diff --git a/seed/api/v3/urls.py b/seed/api/v3/urls.py index baa01d0750..8ee0901b3d 100644 --- a/seed/api/v3/urls.py +++ b/seed/api/v3/urls.py @@ -67,6 +67,7 @@ from seed.views.v3.salesforce_mappings import SalesforceMappingViewSet from seed.views.v3.sensors import SensorViewSet from seed.views.v3.services import ServiceViewSet +from seed.views.v3.statistics import StastisticsSetupViewSet from seed.views.v3.systems import SystemViewSet from seed.views.v3.tax_lot_properties import TaxLotPropertyViewSet from seed.views.v3.taxlot_views import TaxlotViewViewSet @@ -120,6 +121,7 @@ api_v3_router.register(r"report_configurations", ReportConfigurationViewSet, basename="report_configurations") api_v3_router.register(r"salesforce_configs", SalesforceConfigViewSet, basename="salesforce_configs") api_v3_router.register(r"salesforce_mappings", SalesforceMappingViewSet, basename="salesforce_mappings") +api_v3_router.register(r"statistics", StastisticsSetupViewSet, basename="statistics") api_v3_router.register(r"tax_lot_properties", TaxLotPropertyViewSet, basename="tax_lot_properties") api_v3_router.register(r"taxlot_views", TaxlotViewViewSet, basename="taxlot_views") api_v3_router.register(r"taxlots", TaxlotViewSet, basename="taxlots") diff --git a/seed/migrations/0243_statisticssetup.py b/seed/migrations/0243_statisticssetup.py new file mode 100644 index 0000000000..f727a535fb --- /dev/null +++ b/seed/migrations/0243_statisticssetup.py @@ -0,0 +1,104 @@ +# Generated by Django 3.2.25 on 2025-02-27 20:06 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("orgs", "0041_add_at_tracking_fields"), + ("seed", "0242_add_meter_types"), + ] + + operations = [ + migrations.CreateModel( + name="StatisticsSetup", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("gfa_units", models.CharField(choices=[("ft2", "ft2"), ("m2", ",2")], default="ft2", max_length=20)), + ( + "electricity_units", + models.CharField( + choices=[("GJ", "GJ"), ("kBtu", "kBtu"), ("kWh", "kWh"), ("MBtu/MMBtu", "MBtu/MMBtu"), ("MWh", "MWh")], + default=None, + max_length=50, + ), + ), + ( + "natural_gas_units", + models.CharField( + choices=[ + ("GJ", "GJ"), + ("kBtu", "kBtu"), + ("MBtu/MMBtu", "MBtu/MMBtu"), + ("therms", "therms"), + ("kWh", "kWh"), + ("kcf", "kcf"), + ("Mcf", "Mcf"), + ], + default=None, + max_length=50, + ), + ), + ( + "electricity_eui_units", + models.CharField( + choices=[ + ("kBtu/ft2", "kBtu/ft2"), + ("kBtu/m2", "kBtu/m2"), + ("kWh/ft2", "kWh/ft2"), + ("kWh/m2", "kWh/m2"), + ("GJ/m2", "GJ/m2"), + ("GJ/ft2", "GJ/ft2"), + ], + default="kBtu/ft2", + max_length=50, + ), + ), + ( + "natural_gas_eui_units", + models.CharField( + choices=[ + ("kBtu/ft2", "kBtu/ft2"), + ("kBtu/m2", "kBtu/m2"), + ("kWh/ft2", "kWh/ft2"), + ("kWh/m2", "kWh/m2"), + ("GJ/m2", "GJ/m2"), + ("GJ/ft2", "GJ/ft2"), + ], + default="kBtu/ft2", + max_length=50, + ), + ), + ( + "electricity_column", + models.ForeignKey( + null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="electricity_column", to="seed.column" + ), + ), + ( + "electricity_eui_column", + models.ForeignKey( + null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="electricity_eui_column", to="seed.column" + ), + ), + ( + "gfa_column", + models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="gfa_column", to="seed.column"), + ), + ( + "natural_gas_column", + models.ForeignKey( + null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="natural_gas_column", to="seed.column" + ), + ), + ( + "natural_gas_eui_column", + models.ForeignKey( + null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="natural_gas_eui_column", to="seed.column" + ), + ), + ("organization", models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to="orgs.organization")), + ], + ), + ] diff --git a/seed/models/__init__.py b/seed/models/__init__.py index 683f58dfa9..01c734001c 100644 --- a/seed/models/__init__.py +++ b/seed/models/__init__.py @@ -29,6 +29,7 @@ from .salesforce_mappings import * from .sensors import * from .simulations import * +from .statistics_setups import * from .building_file import * from .inventory_document import * from .inventory_groups import * diff --git a/seed/models/statistics_setups.py b/seed/models/statistics_setups.py new file mode 100644 index 0000000000..036ecc18e6 --- /dev/null +++ b/seed/models/statistics_setups.py @@ -0,0 +1,64 @@ +""" +SEED Platform (TM), Copyright (c) Alliance for Sustainable Energy, LLC, and other contributors. +See also https://github.com/SEED-platform/seed/blob/main/LICENSE.md +""" + +import logging + +from django.db import models + +from seed.data_importer.utils import kbtu_thermal_conversion_factors +from seed.lib.superperms.orgs.models import Organization +from seed.models.columns import Column + +logger = logging.getLogger(__name__) + + +class StatisticsSetup(models.Model): + # Stores all the configuration needed to calculate organization statistics + # Retrieve default values + # find the kbtu_thermal_conversion_factors entry under Electric that has "kWh" in it + electric_factors = kbtu_thermal_conversion_factors("US").get("Electric", {}) + electric_default = next((item for item in electric_factors.items() if "kWh" in item), None) + # find the value under "Natural Gas" that has "therms" in it + gas_factors = kbtu_thermal_conversion_factors("US").get("Natural Gas", {}) + gas_default = next((item for item in electric_factors.items() if "therms" in item), None) + # set area default to 'ft2' + area_units_default = "ft2" + AREA_UNITS = ( + ("ft2", "ft2"), + ("m2", ",2"), + ) + + ELECTRIC_UNITS = (("GJ", "GJ"), ("kBtu", "kBtu"), ("kWh", "kWh"), ("MBtu/MMBtu", "MBtu/MMBtu"), ("MWh", "MWh")) + + EUI_UNITS = ( + ("kBtu/ft2", "kBtu/ft2"), + ("kBtu/m2", "kBtu/m2"), + ("kWh/ft2", "kWh/ft2"), + ("kWh/m2", "kWh/m2"), + ("GJ/m2", "GJ/m2"), + ("GJ/ft2", "GJ/ft2"), + ) + + GAS_UNITS = ( + ("GJ", "GJ"), + ("kBtu", "kBtu"), + ("MBtu/MMBtu", "MBtu/MMBtu"), + ("therms", "therms"), + ("kWh", "kWh"), + ("kcf", "kcf"), + ("Mcf", "Mcf"), + ) + + organization = models.OneToOneField(Organization, on_delete=models.CASCADE) + gfa_column = models.ForeignKey(Column, related_name="gfa_column", null=True, on_delete=models.SET_NULL) + gfa_units = models.CharField(max_length=20, choices=AREA_UNITS, default=area_units_default) + electricity_column = models.ForeignKey(Column, related_name="electricity_column", null=True, on_delete=models.SET_NULL) + electricity_units = models.CharField(max_length=50, choices=ELECTRIC_UNITS, default=electric_default) + natural_gas_column = models.ForeignKey(Column, related_name="natural_gas_column", null=True, on_delete=models.SET_NULL) + natural_gas_units = models.CharField(max_length=50, choices=GAS_UNITS, default=gas_default) + electricity_eui_column = models.ForeignKey(Column, related_name="electricity_eui_column", null=True, on_delete=models.SET_NULL) + electricity_eui_units = models.CharField(max_length=50, choices=EUI_UNITS, default="kBtu/ft2") + natural_gas_eui_column = models.ForeignKey(Column, related_name="natural_gas_eui_column", null=True, on_delete=models.SET_NULL) + natural_gas_eui_units = models.CharField(max_length=50, choices=EUI_UNITS, default="kBtu/ft2") diff --git a/seed/serializers/statistics_setups.py b/seed/serializers/statistics_setups.py new file mode 100644 index 0000000000..17755d8444 --- /dev/null +++ b/seed/serializers/statistics_setups.py @@ -0,0 +1,29 @@ +""" +SEED Platform (TM), Copyright (c) Alliance for Sustainable Energy, LLC, and other contributors. +See also https://github.com/SEED-platform/seed/blob/main/LICENSE.md +""" + +from rest_framework import serializers + +from seed.models.statistics_setups import StatisticsSetup + + +class StatisticsSetupSerializer(serializers.ModelSerializer): + organization_id = serializers.IntegerField(required=True) + + class Meta: + model = StatisticsSetup + fields = ( + "id", + "organization_id", + "gfa_column", + "gfa_units", + "electricity_column", + "electricity_units", + "natural_gas_column", + "natural_gas_units", + "electricity_eui_column", + "electricity_eui_units", + "natural_gas_eui_column", + "natural_gas_eui_units", + ) diff --git a/seed/static/seed/js/controllers/organization_stats_controller.js b/seed/static/seed/js/controllers/organization_stats_controller.js new file mode 100644 index 0000000000..4cf5572b64 --- /dev/null +++ b/seed/static/seed/js/controllers/organization_stats_controller.js @@ -0,0 +1,169 @@ +/** + * SEED Platform (TM), Copyright (c) Alliance for Sustainable Energy, LLC, and other contributors. + * See also https://github.com/SEED-platform/seed/blob/main/LICENSE.md + */ +angular.module('SEED.controller.organization_stats', []).controller('organization_stats_controller', [ + '$scope', + 'all_columns', + 'organization_payload', + 'auth_payload', + 'statistics_payload', + 'statistics_service', + 'Notification', + + // eslint-disable-next-line func-names + function ($scope, all_columns, organization_payload, auth_payload, statistics_payload, statistics_service, Notification) { + $scope.fields = all_columns.columns; + $scope.org = organization_payload.organization; + $scope.auth = auth_payload.auth; + $scope.data_fetched = false; + $scope.data = []; + + $scope.conf = {}; + if (statistics_payload.length > 0) { + $scope.conf = statistics_payload[0]; + } + + // hardcoding these for now but we should make a service for them at some point + $scope.electric_units = [ + 'GJ', + 'kBtu', + 'kWh', + 'MBtu/MMBtu', + 'MWh' + ]; + $scope.gas_units = [ + 'GJ', + 'kBtu', + 'MBtu/MMBtu', + 'therms', + 'kWh', + 'kcf', + 'Mcf' + ]; + $scope.area_units = ['ft2', 'm2']; + + $scope.eui_units = [ + 'kBtu/ft2', + 'kBtu/m2', + 'kWh/ft2', + 'kWh/m2', + 'GJ/m2', + 'GJ/ft2' + ]; + + $scope.btnText = 'Expand Configurations'; + $scope.changeText = (btnText) => { + if (btnText === 'Collapse Configurations') { + $scope.btnText = 'Expand Configurations'; + } else { + $scope.btnText = 'Collapse Configurations'; + } + }; + + /* save settings */ + $scope.save_settings = () => { + $scope.settings_updated = false; + if ($scope.conf.id) { + // update + statistics_service + .update_statistic($scope.org.id, $scope.conf.id, $scope.conf) + .then((response) => { + if (response.status === 'error') { + $scope.config_errors = response.errors; + } else { + statistics_service.get_statistics($scope.org.id).then((data) => { + $scope.conf = data.length > 0 ? data[0] : {}; + }); + $scope.settings_updated = true; + } + }) + .catch((response) => { + if (response.data && response.data.status === 'error') { + $scope.config_errors = response.data.message; + } else { + $scope.config_errors = 'An unknown error has occurred'; + } + Notification.error({ message: `Error: ${$scope.config_errors}`, delay: 15000, closeOnClick: true }); + }); + } else { + // create + statistics_service + .new_statistic($scope.org.id, $scope.conf) + .then(() => { + statistics_service.get_statistics($scope.org.id).then((data) => { + $scope.conf = data.length > 0 ? data[0] : {}; + }); + $scope.settings_updated = true; + }) + .catch((response) => { + if (response.data && response.data.status === 'error') { + $scope.config_errors = response.data.message; + } else { + $scope.config_errors = 'An unknown error has occurred'; + } + Notification.error({ message: `Error: ${$scope.config_errors}`, delay: 15000, closeOnClick: true }); + }); + } + }; + + /* calculate stats and update data display */ + $scope.calculate = () => { + // calculate statistics + statistics_service + .calculate_statistics($scope.org.id, $scope.conf.id) + .then((response) => { + if (response.status === 'error') { + $scope.config_errors = response.errors; + } else { + // fetched data + $scope.data = response.data.data; + // $scope.boxplotData = {}; + // $scope.elec_chart = $scope.create_chart('canvas_elec', $scope.data.chart_data_elec, 'Electricity Energy Use Intensity'); + // $scope.gas_chart = $scope.create_chart('canvas_gas', $scope.data.chart_data_gas, 'Natural Gas Energy Use Intensity'); + $scope.data_fetched = true; + } + }) + .catch((response) => { + if (response.data && response.status === 'error') { + $scope.config_errors = response.data.message; + } else { + $scope.config_errors = 'An unknown error has occurred'; + } + Notification.error({ message: `Error: ${$scope.config_errors}`, delay: 15000, closeOnClick: true }); + }); + }; + + // $scope.create_chart = (chartType, data, title_text) => { + // $scope.boxplotData[chartType] = { + // // define label tree + // // labels are the cycle in $scope.data + // labels: Object.keys(data), + // datasets: [] + // }; + // // iterate over keys of $scope.chart_data_elec hash and add data to datasets + // for (const key in data) { + // $scope.boxplotData[chartType].datasets.push({ + // label: key, + // data: data[key] + // }); + // } + // const canvas = document.getElementById(chartType); + // const ctx = canvas.getContext('2d'); + // return new Chart(ctx, { + // type: 'boxplot', + // data: $scope.boxplotData[chartType], + // options: { + // responsive: true, + // legend: { + // position: 'top' + // }, + // title: { + // display: true, + // text: title_text + // } + // } + // }); + // }; + } +]); diff --git a/seed/static/seed/js/seed.js b/seed/static/seed/js/seed.js index 84d8900c42..802e2aab53 100644 --- a/seed/static/seed/js/seed.js +++ b/seed/static/seed/js/seed.js @@ -136,6 +136,7 @@ 'SEED.controller.organization_edit_access_level_instance_modal', 'SEED.controller.organization_settings', 'SEED.controller.organization_sharing', + 'SEED.controller.organization_stats', 'SEED.controller.pairing', 'SEED.controller.pairing_settings', 'SEED.controller.service_detail', @@ -241,6 +242,7 @@ 'SEED.service.search', 'SEED.service.sensor', 'SEED.service.service', + 'SEED.service.statistics', 'SEED.service.system', 'SEED.service.simple_modal', 'SEED.service.two_factor', @@ -1277,6 +1279,55 @@ ] } }) + .state({ + name: 'organization_stats', + url: '/accounts/{organization_id:int}/stats', + templateUrl: `${static_url}seed/partials/organization_stats.html`, + controller: 'organization_stats_controller', + resolve: { + all_columns: [ + '$stateParams', + 'analyses_service', + ($stateParams, analyses_service) => { + const { organization_id } = $stateParams; + return analyses_service.get_used_columns(organization_id); + } + ], + organization_payload: [ + 'organization_service', + '$stateParams', + (organization_service, $stateParams) => { + const { organization_id } = $stateParams; + return organization_service.get_organization(organization_id); + } + ], + auth_payload: [ + 'auth_service', + '$stateParams', + '$q', + (auth_service, $stateParams, $q) => { + const { organization_id } = $stateParams; + return auth_service.is_authorized(organization_id, ['requires_owner']).then( + (data) => { + if (data.auth.requires_owner) { + return data; + } + return $q.reject('not authorized'); + }, + (data) => $q.reject(data.message) + ); + } + ], + statistics_payload: [ + 'statistics_service', + '$stateParams', + (statistics_service, $stateParams) => { + const { organization_id } = $stateParams; + return statistics_service.get_statistics(organization_id); + } + ] + } + }) .state({ name: 'programs', url: '/accounts/{organization_id:int}/program_setup', diff --git a/seed/static/seed/js/services/statistics_service.js b/seed/static/seed/js/services/statistics_service.js new file mode 100644 index 0000000000..71ad11de89 --- /dev/null +++ b/seed/static/seed/js/services/statistics_service.js @@ -0,0 +1,64 @@ +angular.module('SEED.service.statistics', []).factory('statistics_service', [ + '$http', + ($http) => { + // get all statistics for org (should just be 1) + const get_statistics = (organization_id) => $http + .get('/api/v3/statistics/', { + params: { + organization_id + } + }) + .then((response) => response.data.statistics) + .catch((response) => response.data); + + // retrieve statistic by ID + const get_statistic = (organization_id, id) => $http + .get(`/api/v3/statistics/${id}/`, { + params: { + organization_id + } + }) + .then((response) => response.data.statistic) + .catch((response) => response.data); + + // update + const update_statistic = (organization_id, id, data) => $http + .put(`/api/v3/statistics/${id}/`, data, { + params: { + organization_id + } + }) + .then((response) => response.data.statistic) + .catch((response) => response.data); + + // create + const new_statistic = (organization_id, data) => $http + .post('/api/v3/statistics/', data, { + params: { + organization_id + } + }) + .then((response) => response.data.statistic) + .catch((response) => response.data); + + // calculate statistics + const calculate_statistics = (organization_id, id) => $http + .post(`/api/v3/statistics/${id}/calculate/`, null, { + params: { + organization_id + } + }) + .then((response) => response) + .catch((response) => response); + + const statistics_factory = { + get_statistics, + get_statistic, + update_statistic, + new_statistic, + calculate_statistics + }; + + return statistics_factory; + } +]); diff --git a/seed/static/seed/partials/organization_stats.html b/seed/static/seed/partials/organization_stats.html new file mode 100644 index 0000000000..0834c054dc --- /dev/null +++ b/seed/static/seed/partials/organization_stats.html @@ -0,0 +1,200 @@ +
+ +
+
+
+
+
+
+
+
+

{$:: 'Statistics' | translate $}

+
+
+
+
+
+

Here is where you can see stats about your org. You will need to expand the configurations section below to indicate which columns your data is stored in.

+
+ +
+

+ Select the columns where your organization stores the relevant data to generate statistics. Note that quantities will be converted to calculate EUIs in kBtu/ft2 where + appropriate. Enter EITHER Annual Electricity and Natural Gas usage quantities OR Electricity and Gas EUIs. EUIs will be used if given; otherwise they will be calculated from usage + quantities. +

+
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+
+
+
+ +
+
+
+

Statistics

+
+ +
+
+

Overall Statistics - Gross Floor Area

+ + + + + + + + + + + +
CycleNumber of PropertiesTotal Gross Floor Area (ft2)
{$ item.cycle $}{$ item.num_records $}{$ item.GFA | number: 0 $}
+ +

Overall Statistics - Electricity

+ + + + + + + + + + + + + +
CycleNumber of PropertiesGross Floor Area Represented (ft2)Electricity Use (kWh)
{$ item.cycle $}{$ item.num_records $}{$ item.GFA | number: 0 $}{$ item.Electricity_Use | number: 0 $}
+ +

Overall Statistics - Natural Gas

+ + + + + + + + + + + + + +
CycleNumber of PropertiesGross Floor Area Represented (ft2)Natural Gas Use (therms)
{$ item.cycle $}{$ item.num_records $}{$ item.GFA | number: 0 $}{$ item.Natural_Gas_Use | number: 0 $}
+ +

EUI Quantiles - Electricity

+ + + + + + + + + + + + + + + + + +
Cycle5%25%50%75%95%
{$ key $}{$ item['0.05'] | number: 2 $}{$ item['0.25'] | number: 2 $}{$ item['0.5'] | number: 2 $}{$ item['0.75'] | number: 2 $}{$ item['0.95'] | number: 2 $}
+ +

EUI Quantiles - Natural Gas

+ + + + + + + + + + + + + + + + + +
Cycle5%25%50%75%95%
{$ key $}{$ item["0.05"] | number: 2 $}{$ item["0.25"] | number: 2 $}{$ item["0.5"] | number: 2 $}{$ item["0.75"] | number: 2 $}{$ item["0.95"] | number: 2 $}
+ + +
+
+
+
+
diff --git a/seed/static/seed/scss/style.scss b/seed/static/seed/scss/style.scss index b374fdd4df..dd4d36177f 100755 --- a/seed/static/seed/scss/style.scss +++ b/seed/static/seed/scss/style.scss @@ -5893,3 +5893,9 @@ tags-input .tags .tag-item { margin-left: 20px; margin-right: 20px; } + +.stat_instructions_container { + border: 1px solid #ccc; + padding: 20px; + margin-top: 10px; +} diff --git a/seed/templates/seed/_header.html b/seed/templates/seed/_header.html index 4eb5a41d08..b4379f2206 100644 --- a/seed/templates/seed/_header.html +++ b/seed/templates/seed/_header.html @@ -82,6 +82,9 @@
  • Sharing
  • +
  • + Stats +
  • Sub-Organizations + @@ -213,6 +214,7 @@ + diff --git a/seed/templates/seed/_vendor_scripts.html b/seed/templates/seed/_vendor_scripts.html index 25e8cc57ec..1fb6de7985 100644 --- a/seed/templates/seed/_vendor_scripts.html +++ b/seed/templates/seed/_vendor_scripts.html @@ -35,6 +35,7 @@ + diff --git a/seed/views/v3/statistics.py b/seed/views/v3/statistics.py new file mode 100644 index 0000000000..4ad283fc9e --- /dev/null +++ b/seed/views/v3/statistics.py @@ -0,0 +1,487 @@ +""" +SEED Platform (TM), Copyright (c) Alliance for Sustainable Energy, LLC, and other contributors. +See also https://github.com/SEED-platform/seed/blob/main/LICENSE.md +""" + +import re +from copy import deepcopy + +import django.core.exceptions +import numpy as np +import pandas as pd +from django.db import IntegrityError, models +from django.db.models.fields.json import KeyTransform +from django.http import JsonResponse +from drf_yasg.utils import swagger_auto_schema +from rest_framework import status, viewsets +from rest_framework.decorators import action + +from seed.decorators import ajax_request_class, require_organization_id_class +from seed.lib.superperms.orgs.decorators import has_perm_class +from seed.lib.superperms.orgs.models import AccessLevelInstance, Organization +from seed.models.columns import Column +from seed.models.cycles import Cycle +from seed.models.properties import PropertyView +from seed.models.statistics_setups import StatisticsSetup +from seed.serializers.statistics_setups import StatisticsSetupSerializer +from seed.utils.api import OrgMixin, api_endpoint_class +from seed.utils.api_schema import AutoSchemaHelper, swagger_auto_schema_org_query_param + + +def _validate_data(data, org_id): + error = False + msgs = [] + + # Validate Columns + column_names = ["gfa_column", "electricity_column", "natural_gas_column"] + for item in column_names: + c_id = data.get(item) + if c_id: + c_col = Column.objects.get(pk=c_id) + + if c_col.organization_id != org_id: + # error, this column does not belong to this org + error = True + msgs.append("The selected column for " + item + " does not belong to this organization") + + return error, msgs + + +def _remove_nan(the_obj): + """Convert all NaN values in a nested dictionary to None""" + if isinstance(the_obj, dict): + for key, value in the_obj.items(): + if isinstance(value, dict): + _remove_nan(value) + elif pd.isna(value): + the_obj[key] = None + else: + for item in the_obj: + if isinstance(item, dict): + _remove_nan(item) + elif pd.isna(item): + the_obj[item] = None + return the_obj + + +def _convert_to_numeric(df, exclude_cols=[]): + """ + Convert all columns in a DataFrame to numeric types. + For columns with text patterns like '2042.0 foot ** 2', + extract the numeric value before conversion. + + Parameters: + ----------- + df : pandas.DataFrame + DataFrame with mixed type columns + + Returns: + -------- + pandas.DataFrame + DataFrame with all columns converted to numeric types + """ + # Create a copy to avoid modifying the original + df_numeric = df.copy() + + for col in df_numeric.columns: + # Skip columns in exclude_cols + if col in exclude_cols: + continue + # Check if column is already numeric + if pd.api.types.is_numeric_dtype(df_numeric[col]): + continue + + # If it's an object type, try to extract numeric values + if df_numeric[col].dtype == "object": + # Function to extract numeric part from strings + def extract_numeric(val): + if pd.isna(val): + return np.nan + if isinstance(val, (int, float)): + return val + + # Convert to string if not already + val_str = str(val) + + # Extract the first number (including decimals) + # This pattern matches numbers like 123, 123.45, .45 + match = re.search(r"([-+]?\d*\.?\d+)", val_str) + if match: + return match.group(1) + return val + + # Apply extraction and then convert to numeric + df_numeric[col] = df_numeric[col].apply(extract_numeric) + + # Convert to numeric, with errors='coerce' to convert non-numerics to NaN + df_numeric[col] = pd.to_numeric(df_numeric[col], errors="coerce") + + return df_numeric + + +class StastisticsSetupViewSet(viewsets.ViewSet, OrgMixin): + serializer_class = StatisticsSetupSerializer + model = StatisticsSetup + + @swagger_auto_schema_org_query_param + @api_endpoint_class + @ajax_request_class + @has_perm_class("requires_owner") + def list(self, request): + organization_id = self.get_organization(request) + stats = StatisticsSetup.objects.filter(organization=organization_id) + + s_data = StatisticsSetupSerializer(stats, many=True).data + + return JsonResponse({"status": "success", "statistics": s_data}, status=status.HTTP_200_OK) + + @swagger_auto_schema_org_query_param + @require_organization_id_class + @api_endpoint_class + @ajax_request_class + @has_perm_class("requires_owner") + def retrieve(self, request, pk=0): + organization = self.get_organization(request) + if pk == 0: + try: + return JsonResponse( + { + "status": "success", + "statistic": StatisticsSetupSerializer(StatisticsSetup.objects.filter(organization=organization).first()).data, + }, + status=status.HTTP_200_OK, + ) + except Exception: + return JsonResponse( + {"status": "error", "message": "No statistics setup exist with this identifier"}, status=status.HTTP_404_NOT_FOUND + ) + else: + try: + data = StatisticsSetupSerializer(StatisticsSetup.objects.get(id=pk, organization=organization)).data + return JsonResponse({"status": "success", "stastic": data}, status=status.HTTP_200_OK) + except StatisticsSetup.DoesNotExist: + return JsonResponse( + {"status": "error", "message": f"Statistics Setup with id {pk} does not exist"}, status=status.HTTP_404_NOT_FOUND + ) + + @swagger_auto_schema_org_query_param + @require_organization_id_class + @api_endpoint_class + @ajax_request_class + @has_perm_class("requires_owner") + def destroy(self, request, pk): + organization_id = self.get_organization(request) + + try: + StatisticsSetup.objects.get(id=pk, organization=organization_id).delete() + except StatisticsSetup.DoesNotExist: + return JsonResponse({"status": "error", "message": f"Statistics with id {pk} does not exist"}, status=status.HTTP_404_NOT_FOUND) + + return JsonResponse({"status": "success", "message": f"Successfully deleted Statistics ID {pk}"}, status=status.HTTP_200_OK) + + @swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_org_id_field()], + request_body=AutoSchemaHelper.schema_factory( + { + "gfa_column": "integer", + "gfa_units": "string", + "electricity_column": "integer", + "electricity_units": "string", + "natural_gas_column": "integer", + "natural_gas_units": "string", + "electricity_eui_column": "integer", + "electricity_eui_units": "string", + "natural_gas_eui_column": "integer", + "natural_gas_eui_units": "string", + }, + ), + ) + @require_organization_id_class + @api_endpoint_class + @ajax_request_class + @has_perm_class("requires_owner") + def create(self, request): + org_id = int(self.get_organization(request)) + try: + Organization.objects.get(pk=org_id) + except Organization.DoesNotExist: + return JsonResponse({"status": "error", "message": "bad organization_id"}, status=status.HTTP_400_BAD_REQUEST) + + data = deepcopy(request.data) + data.update({"organization_id": org_id}) + + error, msgs = _validate_data(data, org_id) + if error is True: + return JsonResponse({"status": "error", "message": ",".join(msgs)}, status=status.HTTP_400_BAD_REQUEST) + serializer = StatisticsSetupSerializer(data=data) + + if not serializer.is_valid(): + error_response = {"status": "error", "message": "Data Validation Error", "errors": serializer.errors} + return JsonResponse(error_response, status=status.HTTP_400_BAD_REQUEST) + + try: + serializer.save() + + return JsonResponse({"status": "success", "statistic": serializer.data}, status=status.HTTP_200_OK) + except IntegrityError: + return JsonResponse( + {"status": "error", "message": "Only one statistics setup can be created per organization"}, + status=status.HTTP_400_BAD_REQUEST, + ) + except django.core.exceptions.ValidationError as e: + message_dict = e.message_dict + return JsonResponse({"status": "error", "message": "Bad Request", "errors": message_dict}, status=status.HTTP_400_BAD_REQUEST) + + @swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_org_id_field()], + request_body=AutoSchemaHelper.schema_factory( + { + "gfa_column": "integer", + "gfa_units": "string", + "electricity_column": "integer", + "electricity_units": "string", + "natural_gas_column": "integer", + "natural_gas_units": "string", + "electricity_eui_column": "integer", + "electricity_eui_units": "string", + "natural_gas_eui_column": "integer", + "natural_gas_eui_units": "string", + }, + ), + ) + @require_organization_id_class + @api_endpoint_class + @ajax_request_class + @has_perm_class("requires_owner") + def update(self, request, pk): + org_id = self.get_organization(request) + + statistic = None + try: + statistic = StatisticsSetup.objects.get(id=pk, organization=org_id) + except StatisticsSetup.DoesNotExist: + return JsonResponse({"status": "error", "message": f"Statistics with id {pk} does not exist"}, status=status.HTTP_404_NOT_FOUND) + + data = deepcopy(request.data) + error, msgs = _validate_data(data, org_id) + if error is True: + return JsonResponse({"status": "error", "message": ",".join(msgs)}, status=status.HTTP_400_BAD_REQUEST) + + serializer = StatisticsSetupSerializer(statistic, data=data, partial=True) + + if not serializer.is_valid(): + return JsonResponse( + {"status": "error", "message": "Bad Request", "errors": serializer.errors}, status=status.HTTP_400_BAD_REQUEST + ) + + try: + serializer.save() + + return JsonResponse( + { + "status": "success", + "statistic": serializer.data, + }, + status=status.HTTP_200_OK, + ) + except django.core.exceptions.ValidationError as e: + message_dict = e.message_dict + # rename key __all__ to general to make it more user-friendly + if "__all__" in message_dict: + message_dict["general"] = message_dict.pop("__all__") + + return JsonResponse( + { + "status": "error", + "message": "Bad request", + "errors": message_dict, + }, + status=status.HTTP_400_BAD_REQUEST, + ) + except Exception as e: + return JsonResponse( + { + "status": "error", + "message": "Bad request", + "errors": str(e), + }, + status=status.HTTP_400_BAD_REQUEST, + ) + + @require_organization_id_class + @api_endpoint_class + @ajax_request_class + @has_perm_class("requires_owner") + @action(detail=True, methods=["POST"]) + def calculate(self, request, pk): + org_id = self.get_organization(request) + ali = AccessLevelInstance.objects.get(pk=request.access_level_instance_id) + # get statistic setup for this org + try: + statistic = StatisticsSetup.objects.get(id=pk, organization=org_id) + except StatisticsSetup.DoesNotExist: + return JsonResponse({"status": "error", "message": f"Statistics with id {pk} does not exist"}, status=status.HTTP_404_NOT_FOUND) + + # get all cycles in this org + cycles = Cycle.objects.filter(organization=org_id) + + # column ids we are interested in, only those non-null + + cols = {} + extra_data_cols = {} + if statistic.gfa_column: + if statistic.gfa_column.is_extra_data: + extra_data_cols["gfa"] = statistic.gfa_column.column_name + else: + cols["gfa"] = "state__" + statistic.gfa_column.column_name + else: + # if we don't have GFA column, we can't calculate anything + return JsonResponse( + { + "status": "error", + "message": "GFA column is required to calculate statistics", + }, + status=status.HTTP_400_BAD_REQUEST, + ) + # prefer EUI column over usage column + if statistic.electricity_eui_column: + if statistic.electricity_eui_column.is_extra_data: + extra_data_cols["elec_eui"] = statistic.electricity_eui_column.column_name + else: + cols["elec_eui"] = "state__" + statistic.electricity_eui_column.column_name + elif statistic.electricity_column: + if statistic.electricity_column.is_extra_data: + extra_data_cols["electricity"] = statistic.electricity_column.column_name + else: + cols["electricity"] = "state__" + statistic.electricity_column.column_name + if statistic.natural_gas_eui_column: + if statistic.natural_gas_eui_column.is_extra_data: + extra_data_cols["gas_eui"] = statistic.natural_gas_eui_column.column_name + else: + cols["gas_eui"] = "state__" + statistic.natural_gas_eui_column.column_name + elif statistic.natural_gas_column: + if statistic.natural_gas_column.is_extra_data: + extra_data_cols["natural_gas"] = statistic.natural_gas_column.column_name + else: + cols["natural_gas"] = "state__" + statistic.natural_gas_column.column_name + + # make sure we have at least electricity or gas here to calculate? + if len(cols.keys()) + len(extra_data_cols.keys()) < 2: + return JsonResponse( + { + "status": "error", + "message": "At least one of electricity or natural gas column is required to calculate statistics", + }, + status=status.HTTP_400_BAD_REQUEST, + ) + + # reverse the cols dictionary to have the keys be the values + # this is to make it easier to map the results back to the correct columns in df + cols_reverse = {v: k for k, v in cols.items()} + + annotations: dict[str, models.Func] = {} + for key, val in extra_data_cols.items(): + annotations[key] = KeyTransform(val, "state__extra_data") + + results = {} + df_full = pd.DataFrame() + # get all properties that fall in all cycles for this org + for cycle in cycles: + # get Property-Views for this Cycle + property_views = ( + PropertyView.objects.select_related("state") + .filter( + property__organization_id=org_id, + cycle_id=cycle.id, + property__access_level_instance__lft__gte=ali.lft, + property__access_level_instance__rgt__lte=ali.rgt, + ) + .annotate(**annotations) + .values("property_id", "state_id", *(list(cols.values())), *(list(extra_data_cols.keys()))) + ) + + # dataframe column names will be: + # property_id, state_id, electricity, natural_gas, cycle_id, gfa, + # elec_eui, gas_eui + + df_temp = pd.DataFrame.from_records(list(property_views)) + df_temp["cycle"] = cycle.name # or use ID and get name in frontend? + df_full = pd.concat([df_full, df_temp]) + + # remap the column names to the original column names + df_full = df_full.rename(columns=cols_reverse) + df_full = _convert_to_numeric(df_full, ["cycle"]) + + # calculate the statistics + # check if we have EUIs or Usage columns + # electricity + if "electricity" in df_full.columns: + # we have usage, compute EUI + df_full["elec_eui"] = (df_full["electricity"] / df_full["gfa"]).replace([np.inf, -np.inf], np.nan) + # gas + if "natural_gas" in df_full.columns: + df_full["gas_eui"] = (df_full["natural_gas"] / df_full["gfa"]).replace([np.inf, -np.inf], np.nan) + + # reverse calculate electricity and/or natural gas from EUIs + # electricity + if "elec_eui" in df_full.columns: + df_full["electricity"] = df_full["elec_eui"] * df_full["gfa"] + # gas + if "gas_eui" in df_full.columns: + df_full["natural_gas"] = df_full["gas_eui"] * df_full["gfa"] + + # high-level counts: + yearly_gfa = ( + df_full.dropna(subset=["gfa"]) + .groupby("cycle") + .agg(num_records=("property_id", "count"), cycle=("cycle", "first"), GFA=("gfa", "sum")) + ) + yearly_elec = ( + df_full.dropna(subset=["gfa", "elec_eui"]) + .groupby("cycle") + .agg(num_records=("property_id", "count"), cycle=("cycle", "first"), GFA=("gfa", "sum"), Electricity_Use=("electricity", "sum")) + ) + yearly_elec.style.format("{:,.0f}") + # print(yearly_elec) # format numbers with commas + + yearly_gas = ( + df_full.dropna(subset=["gfa", "gas_eui"]) + .groupby("cycle") + .agg(num_records=("property_id", "count"), cycle=("cycle", "first"), GFA=("gfa", "sum"), Natural_Gas_Use=("natural_gas", "sum")) + ) + yearly_gas.style.format("{:,.0f}") + # print(yearly_gas) # format numbers with commas + + # elec quartiles for each cycle (include cycle key) + elec_eui_quantiles = df_full.groupby("cycle")["elec_eui"].quantile([0.05, 0.25, 0.50, 0.75, 0.95]).unstack() # noqa: PD010 + + # print(f"Elec Quantiles") + # print(elec_eui_quantiles) + + # do the same for gas eui + gas_eui_quantiles = df_full.groupby("cycle")["gas_eui"].quantile([0.05, 0.25, 0.50, 0.75, 0.95]).unstack() # noqa: PD010 + # print(f"Gas Quantiles") + # print(gas_eui_quantiles) + + df_return = df_full[["property_id", "cycle", "elec_eui", "gas_eui"]] + + # transform df_return to have a key for each cycle and an array of elec_eui + chart_data_elec = {} + chart_data_gas = {} + for year_value in df_return["cycle"].unique(): + chart_data_elec[year_value] = df_return[df_return["cycle"] == year_value]["elec_eui"].dropna().tolist() + chart_data_gas[year_value] = df_return[df_return["cycle"] == year_value]["gas_eui"].dropna().tolist() + + # return the results and convert NaN to None + results["elec_quantiles"] = elec_eui_quantiles.to_dict(orient="index") + results["gas_quantiles"] = gas_eui_quantiles.to_dict(orient="index") + results["chart_data_elec"] = chart_data_elec + results["chart_data_gas"] = chart_data_gas + results["annual_gfa"] = _remove_nan(yearly_gfa.to_dict(orient="records")) + results["annual_elec"] = _remove_nan(yearly_elec.to_dict(orient="records")) + results["annual_gas"] = _remove_nan(yearly_gas.to_dict(orient="records")) + + return JsonResponse( + {"status": "success", "data": results}, + status=status.HTTP_200_OK, + ) diff --git a/vendors/package-lock.json b/vendors/package-lock.json index 6c6cc48bf6..fbc8236130 100644 --- a/vendors/package-lock.json +++ b/vendors/package-lock.json @@ -12,6 +12,7 @@ "@fontsource/pt-sans": "^5.1.0", "@fontsource/pt-sans-narrow": "^5.1.0", "@fortawesome/fontawesome-free": "^6.6.0", + "@sgratzl/chartjs-chart-boxplot": "^3.10.0", "angular": "^1.8.3", "angular-animate": "^1.8.3", "angular-aria": "^1.8.3", @@ -396,6 +397,22 @@ "license": "MIT", "peer": true }, + "node_modules/@sgratzl/boxplots": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@sgratzl/boxplots/-/boxplots-1.3.4.tgz", + "integrity": "sha512-wRlHp9sZ2aQ/BBq+6Xg8DBsf4zXs/DQhNpLRlfwyO62IqTANmu7d64yE6jSKMnixM+nbtZqbeGa9spiaU/S++g==" + }, + "node_modules/@sgratzl/chartjs-chart-boxplot": { + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/@sgratzl/chartjs-chart-boxplot/-/chartjs-chart-boxplot-3.10.0.tgz", + "integrity": "sha512-+XzmRr9unHzTAMrNVjQ1U6qKid2fEYpV6byC7WIKyU/nT3zB1g98RDuF+z8cFwBjZm/NdVw0uaQYdjHWwHiPbA==", + "dependencies": { + "@sgratzl/boxplots": "^1.3.0" + }, + "peerDependencies": { + "chart.js": "^3.9.1" + } + }, "node_modules/@socket.io/component-emitter": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/@socket.io/component-emitter/-/component-emitter-3.1.2.tgz", diff --git a/vendors/package.json b/vendors/package.json index e41ed01207..21a8ad9f3a 100644 --- a/vendors/package.json +++ b/vendors/package.json @@ -30,6 +30,7 @@ "chartjs-chart-sankey": "^0.14.0", "chartjs-plugin-annotation": "^2.2.1", "chartjs-plugin-zoom": "=1.2.1", + "@sgratzl/chartjs-chart-boxplot": "^3.10.0", "d3": "^5.16.0", "dimple": "PMSI-AlignAlytics/dimple#2.3.0", "file-saver": "^2.0.5",