diff --git a/seed/api/v3/urls.py b/seed/api/v3/urls.py
index baa01d0750..8ee0901b3d 100644
--- a/seed/api/v3/urls.py
+++ b/seed/api/v3/urls.py
@@ -67,6 +67,7 @@
from seed.views.v3.salesforce_mappings import SalesforceMappingViewSet
from seed.views.v3.sensors import SensorViewSet
from seed.views.v3.services import ServiceViewSet
+from seed.views.v3.statistics import StastisticsSetupViewSet
from seed.views.v3.systems import SystemViewSet
from seed.views.v3.tax_lot_properties import TaxLotPropertyViewSet
from seed.views.v3.taxlot_views import TaxlotViewViewSet
@@ -120,6 +121,7 @@
api_v3_router.register(r"report_configurations", ReportConfigurationViewSet, basename="report_configurations")
api_v3_router.register(r"salesforce_configs", SalesforceConfigViewSet, basename="salesforce_configs")
api_v3_router.register(r"salesforce_mappings", SalesforceMappingViewSet, basename="salesforce_mappings")
+api_v3_router.register(r"statistics", StastisticsSetupViewSet, basename="statistics")
api_v3_router.register(r"tax_lot_properties", TaxLotPropertyViewSet, basename="tax_lot_properties")
api_v3_router.register(r"taxlot_views", TaxlotViewViewSet, basename="taxlot_views")
api_v3_router.register(r"taxlots", TaxlotViewSet, basename="taxlots")
diff --git a/seed/migrations/0243_statisticssetup.py b/seed/migrations/0243_statisticssetup.py
new file mode 100644
index 0000000000..f727a535fb
--- /dev/null
+++ b/seed/migrations/0243_statisticssetup.py
@@ -0,0 +1,104 @@
+# Generated by Django 3.2.25 on 2025-02-27 20:06
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("orgs", "0041_add_at_tracking_fields"),
+ ("seed", "0242_add_meter_types"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="StatisticsSetup",
+ fields=[
+ ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+ ("gfa_units", models.CharField(choices=[("ft2", "ft2"), ("m2", ",2")], default="ft2", max_length=20)),
+ (
+ "electricity_units",
+ models.CharField(
+ choices=[("GJ", "GJ"), ("kBtu", "kBtu"), ("kWh", "kWh"), ("MBtu/MMBtu", "MBtu/MMBtu"), ("MWh", "MWh")],
+ default=None,
+ max_length=50,
+ ),
+ ),
+ (
+ "natural_gas_units",
+ models.CharField(
+ choices=[
+ ("GJ", "GJ"),
+ ("kBtu", "kBtu"),
+ ("MBtu/MMBtu", "MBtu/MMBtu"),
+ ("therms", "therms"),
+ ("kWh", "kWh"),
+ ("kcf", "kcf"),
+ ("Mcf", "Mcf"),
+ ],
+ default=None,
+ max_length=50,
+ ),
+ ),
+ (
+ "electricity_eui_units",
+ models.CharField(
+ choices=[
+ ("kBtu/ft2", "kBtu/ft2"),
+ ("kBtu/m2", "kBtu/m2"),
+ ("kWh/ft2", "kWh/ft2"),
+ ("kWh/m2", "kWh/m2"),
+ ("GJ/m2", "GJ/m2"),
+ ("GJ/ft2", "GJ/ft2"),
+ ],
+ default="kBtu/ft2",
+ max_length=50,
+ ),
+ ),
+ (
+ "natural_gas_eui_units",
+ models.CharField(
+ choices=[
+ ("kBtu/ft2", "kBtu/ft2"),
+ ("kBtu/m2", "kBtu/m2"),
+ ("kWh/ft2", "kWh/ft2"),
+ ("kWh/m2", "kWh/m2"),
+ ("GJ/m2", "GJ/m2"),
+ ("GJ/ft2", "GJ/ft2"),
+ ],
+ default="kBtu/ft2",
+ max_length=50,
+ ),
+ ),
+ (
+ "electricity_column",
+ models.ForeignKey(
+ null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="electricity_column", to="seed.column"
+ ),
+ ),
+ (
+ "electricity_eui_column",
+ models.ForeignKey(
+ null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="electricity_eui_column", to="seed.column"
+ ),
+ ),
+ (
+ "gfa_column",
+ models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="gfa_column", to="seed.column"),
+ ),
+ (
+ "natural_gas_column",
+ models.ForeignKey(
+ null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="natural_gas_column", to="seed.column"
+ ),
+ ),
+ (
+ "natural_gas_eui_column",
+ models.ForeignKey(
+ null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="natural_gas_eui_column", to="seed.column"
+ ),
+ ),
+ ("organization", models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to="orgs.organization")),
+ ],
+ ),
+ ]
diff --git a/seed/models/__init__.py b/seed/models/__init__.py
index 683f58dfa9..01c734001c 100644
--- a/seed/models/__init__.py
+++ b/seed/models/__init__.py
@@ -29,6 +29,7 @@
from .salesforce_mappings import *
from .sensors import *
from .simulations import *
+from .statistics_setups import *
from .building_file import *
from .inventory_document import *
from .inventory_groups import *
diff --git a/seed/models/statistics_setups.py b/seed/models/statistics_setups.py
new file mode 100644
index 0000000000..036ecc18e6
--- /dev/null
+++ b/seed/models/statistics_setups.py
@@ -0,0 +1,64 @@
+"""
+SEED Platform (TM), Copyright (c) Alliance for Sustainable Energy, LLC, and other contributors.
+See also https://github.com/SEED-platform/seed/blob/main/LICENSE.md
+"""
+
+import logging
+
+from django.db import models
+
+from seed.data_importer.utils import kbtu_thermal_conversion_factors
+from seed.lib.superperms.orgs.models import Organization
+from seed.models.columns import Column
+
+logger = logging.getLogger(__name__)
+
+
+class StatisticsSetup(models.Model):
+ # Stores all the configuration needed to calculate organization statistics
+ # Retrieve default values
+ # find the kbtu_thermal_conversion_factors entry under Electric that has "kWh" in it
+ electric_factors = kbtu_thermal_conversion_factors("US").get("Electric", {})
+ electric_default = next((item for item in electric_factors.items() if "kWh" in item), None)
+ # find the value under "Natural Gas" that has "therms" in it
+ gas_factors = kbtu_thermal_conversion_factors("US").get("Natural Gas", {})
+ gas_default = next((item for item in electric_factors.items() if "therms" in item), None)
+ # set area default to 'ft2'
+ area_units_default = "ft2"
+ AREA_UNITS = (
+ ("ft2", "ft2"),
+ ("m2", ",2"),
+ )
+
+ ELECTRIC_UNITS = (("GJ", "GJ"), ("kBtu", "kBtu"), ("kWh", "kWh"), ("MBtu/MMBtu", "MBtu/MMBtu"), ("MWh", "MWh"))
+
+ EUI_UNITS = (
+ ("kBtu/ft2", "kBtu/ft2"),
+ ("kBtu/m2", "kBtu/m2"),
+ ("kWh/ft2", "kWh/ft2"),
+ ("kWh/m2", "kWh/m2"),
+ ("GJ/m2", "GJ/m2"),
+ ("GJ/ft2", "GJ/ft2"),
+ )
+
+ GAS_UNITS = (
+ ("GJ", "GJ"),
+ ("kBtu", "kBtu"),
+ ("MBtu/MMBtu", "MBtu/MMBtu"),
+ ("therms", "therms"),
+ ("kWh", "kWh"),
+ ("kcf", "kcf"),
+ ("Mcf", "Mcf"),
+ )
+
+ organization = models.OneToOneField(Organization, on_delete=models.CASCADE)
+ gfa_column = models.ForeignKey(Column, related_name="gfa_column", null=True, on_delete=models.SET_NULL)
+ gfa_units = models.CharField(max_length=20, choices=AREA_UNITS, default=area_units_default)
+ electricity_column = models.ForeignKey(Column, related_name="electricity_column", null=True, on_delete=models.SET_NULL)
+ electricity_units = models.CharField(max_length=50, choices=ELECTRIC_UNITS, default=electric_default)
+ natural_gas_column = models.ForeignKey(Column, related_name="natural_gas_column", null=True, on_delete=models.SET_NULL)
+ natural_gas_units = models.CharField(max_length=50, choices=GAS_UNITS, default=gas_default)
+ electricity_eui_column = models.ForeignKey(Column, related_name="electricity_eui_column", null=True, on_delete=models.SET_NULL)
+ electricity_eui_units = models.CharField(max_length=50, choices=EUI_UNITS, default="kBtu/ft2")
+ natural_gas_eui_column = models.ForeignKey(Column, related_name="natural_gas_eui_column", null=True, on_delete=models.SET_NULL)
+ natural_gas_eui_units = models.CharField(max_length=50, choices=EUI_UNITS, default="kBtu/ft2")
diff --git a/seed/serializers/statistics_setups.py b/seed/serializers/statistics_setups.py
new file mode 100644
index 0000000000..17755d8444
--- /dev/null
+++ b/seed/serializers/statistics_setups.py
@@ -0,0 +1,29 @@
+"""
+SEED Platform (TM), Copyright (c) Alliance for Sustainable Energy, LLC, and other contributors.
+See also https://github.com/SEED-platform/seed/blob/main/LICENSE.md
+"""
+
+from rest_framework import serializers
+
+from seed.models.statistics_setups import StatisticsSetup
+
+
+class StatisticsSetupSerializer(serializers.ModelSerializer):
+ organization_id = serializers.IntegerField(required=True)
+
+ class Meta:
+ model = StatisticsSetup
+ fields = (
+ "id",
+ "organization_id",
+ "gfa_column",
+ "gfa_units",
+ "electricity_column",
+ "electricity_units",
+ "natural_gas_column",
+ "natural_gas_units",
+ "electricity_eui_column",
+ "electricity_eui_units",
+ "natural_gas_eui_column",
+ "natural_gas_eui_units",
+ )
diff --git a/seed/static/seed/js/controllers/organization_stats_controller.js b/seed/static/seed/js/controllers/organization_stats_controller.js
new file mode 100644
index 0000000000..4cf5572b64
--- /dev/null
+++ b/seed/static/seed/js/controllers/organization_stats_controller.js
@@ -0,0 +1,169 @@
+/**
+ * SEED Platform (TM), Copyright (c) Alliance for Sustainable Energy, LLC, and other contributors.
+ * See also https://github.com/SEED-platform/seed/blob/main/LICENSE.md
+ */
+angular.module('SEED.controller.organization_stats', []).controller('organization_stats_controller', [
+ '$scope',
+ 'all_columns',
+ 'organization_payload',
+ 'auth_payload',
+ 'statistics_payload',
+ 'statistics_service',
+ 'Notification',
+
+ // eslint-disable-next-line func-names
+ function ($scope, all_columns, organization_payload, auth_payload, statistics_payload, statistics_service, Notification) {
+ $scope.fields = all_columns.columns;
+ $scope.org = organization_payload.organization;
+ $scope.auth = auth_payload.auth;
+ $scope.data_fetched = false;
+ $scope.data = [];
+
+ $scope.conf = {};
+ if (statistics_payload.length > 0) {
+ $scope.conf = statistics_payload[0];
+ }
+
+ // hardcoding these for now but we should make a service for them at some point
+ $scope.electric_units = [
+ 'GJ',
+ 'kBtu',
+ 'kWh',
+ 'MBtu/MMBtu',
+ 'MWh'
+ ];
+ $scope.gas_units = [
+ 'GJ',
+ 'kBtu',
+ 'MBtu/MMBtu',
+ 'therms',
+ 'kWh',
+ 'kcf',
+ 'Mcf'
+ ];
+ $scope.area_units = ['ft2', 'm2'];
+
+ $scope.eui_units = [
+ 'kBtu/ft2',
+ 'kBtu/m2',
+ 'kWh/ft2',
+ 'kWh/m2',
+ 'GJ/m2',
+ 'GJ/ft2'
+ ];
+
+ $scope.btnText = 'Expand Configurations';
+ $scope.changeText = (btnText) => {
+ if (btnText === 'Collapse Configurations') {
+ $scope.btnText = 'Expand Configurations';
+ } else {
+ $scope.btnText = 'Collapse Configurations';
+ }
+ };
+
+ /* save settings */
+ $scope.save_settings = () => {
+ $scope.settings_updated = false;
+ if ($scope.conf.id) {
+ // update
+ statistics_service
+ .update_statistic($scope.org.id, $scope.conf.id, $scope.conf)
+ .then((response) => {
+ if (response.status === 'error') {
+ $scope.config_errors = response.errors;
+ } else {
+ statistics_service.get_statistics($scope.org.id).then((data) => {
+ $scope.conf = data.length > 0 ? data[0] : {};
+ });
+ $scope.settings_updated = true;
+ }
+ })
+ .catch((response) => {
+ if (response.data && response.data.status === 'error') {
+ $scope.config_errors = response.data.message;
+ } else {
+ $scope.config_errors = 'An unknown error has occurred';
+ }
+ Notification.error({ message: `Error: ${$scope.config_errors}`, delay: 15000, closeOnClick: true });
+ });
+ } else {
+ // create
+ statistics_service
+ .new_statistic($scope.org.id, $scope.conf)
+ .then(() => {
+ statistics_service.get_statistics($scope.org.id).then((data) => {
+ $scope.conf = data.length > 0 ? data[0] : {};
+ });
+ $scope.settings_updated = true;
+ })
+ .catch((response) => {
+ if (response.data && response.data.status === 'error') {
+ $scope.config_errors = response.data.message;
+ } else {
+ $scope.config_errors = 'An unknown error has occurred';
+ }
+ Notification.error({ message: `Error: ${$scope.config_errors}`, delay: 15000, closeOnClick: true });
+ });
+ }
+ };
+
+ /* calculate stats and update data display */
+ $scope.calculate = () => {
+ // calculate statistics
+ statistics_service
+ .calculate_statistics($scope.org.id, $scope.conf.id)
+ .then((response) => {
+ if (response.status === 'error') {
+ $scope.config_errors = response.errors;
+ } else {
+ // fetched data
+ $scope.data = response.data.data;
+ // $scope.boxplotData = {};
+ // $scope.elec_chart = $scope.create_chart('canvas_elec', $scope.data.chart_data_elec, 'Electricity Energy Use Intensity');
+ // $scope.gas_chart = $scope.create_chart('canvas_gas', $scope.data.chart_data_gas, 'Natural Gas Energy Use Intensity');
+ $scope.data_fetched = true;
+ }
+ })
+ .catch((response) => {
+ if (response.data && response.status === 'error') {
+ $scope.config_errors = response.data.message;
+ } else {
+ $scope.config_errors = 'An unknown error has occurred';
+ }
+ Notification.error({ message: `Error: ${$scope.config_errors}`, delay: 15000, closeOnClick: true });
+ });
+ };
+
+ // $scope.create_chart = (chartType, data, title_text) => {
+ // $scope.boxplotData[chartType] = {
+ // // define label tree
+ // // labels are the cycle in $scope.data
+ // labels: Object.keys(data),
+ // datasets: []
+ // };
+ // // iterate over keys of $scope.chart_data_elec hash and add data to datasets
+ // for (const key in data) {
+ // $scope.boxplotData[chartType].datasets.push({
+ // label: key,
+ // data: data[key]
+ // });
+ // }
+ // const canvas = document.getElementById(chartType);
+ // const ctx = canvas.getContext('2d');
+ // return new Chart(ctx, {
+ // type: 'boxplot',
+ // data: $scope.boxplotData[chartType],
+ // options: {
+ // responsive: true,
+ // legend: {
+ // position: 'top'
+ // },
+ // title: {
+ // display: true,
+ // text: title_text
+ // }
+ // }
+ // });
+ // };
+ }
+]);
diff --git a/seed/static/seed/js/seed.js b/seed/static/seed/js/seed.js
index 84d8900c42..802e2aab53 100644
--- a/seed/static/seed/js/seed.js
+++ b/seed/static/seed/js/seed.js
@@ -136,6 +136,7 @@
'SEED.controller.organization_edit_access_level_instance_modal',
'SEED.controller.organization_settings',
'SEED.controller.organization_sharing',
+ 'SEED.controller.organization_stats',
'SEED.controller.pairing',
'SEED.controller.pairing_settings',
'SEED.controller.service_detail',
@@ -241,6 +242,7 @@
'SEED.service.search',
'SEED.service.sensor',
'SEED.service.service',
+ 'SEED.service.statistics',
'SEED.service.system',
'SEED.service.simple_modal',
'SEED.service.two_factor',
@@ -1277,6 +1279,55 @@
]
}
})
+ .state({
+ name: 'organization_stats',
+ url: '/accounts/{organization_id:int}/stats',
+ templateUrl: `${static_url}seed/partials/organization_stats.html`,
+ controller: 'organization_stats_controller',
+ resolve: {
+ all_columns: [
+ '$stateParams',
+ 'analyses_service',
+ ($stateParams, analyses_service) => {
+ const { organization_id } = $stateParams;
+ return analyses_service.get_used_columns(organization_id);
+ }
+ ],
+ organization_payload: [
+ 'organization_service',
+ '$stateParams',
+ (organization_service, $stateParams) => {
+ const { organization_id } = $stateParams;
+ return organization_service.get_organization(organization_id);
+ }
+ ],
+ auth_payload: [
+ 'auth_service',
+ '$stateParams',
+ '$q',
+ (auth_service, $stateParams, $q) => {
+ const { organization_id } = $stateParams;
+ return auth_service.is_authorized(organization_id, ['requires_owner']).then(
+ (data) => {
+ if (data.auth.requires_owner) {
+ return data;
+ }
+ return $q.reject('not authorized');
+ },
+ (data) => $q.reject(data.message)
+ );
+ }
+ ],
+ statistics_payload: [
+ 'statistics_service',
+ '$stateParams',
+ (statistics_service, $stateParams) => {
+ const { organization_id } = $stateParams;
+ return statistics_service.get_statistics(organization_id);
+ }
+ ]
+ }
+ })
.state({
name: 'programs',
url: '/accounts/{organization_id:int}/program_setup',
diff --git a/seed/static/seed/js/services/statistics_service.js b/seed/static/seed/js/services/statistics_service.js
new file mode 100644
index 0000000000..71ad11de89
--- /dev/null
+++ b/seed/static/seed/js/services/statistics_service.js
@@ -0,0 +1,64 @@
+angular.module('SEED.service.statistics', []).factory('statistics_service', [
+ '$http',
+ ($http) => {
+ // get all statistics for org (should just be 1)
+ const get_statistics = (organization_id) => $http
+ .get('/api/v3/statistics/', {
+ params: {
+ organization_id
+ }
+ })
+ .then((response) => response.data.statistics)
+ .catch((response) => response.data);
+
+ // retrieve statistic by ID
+ const get_statistic = (organization_id, id) => $http
+ .get(`/api/v3/statistics/${id}/`, {
+ params: {
+ organization_id
+ }
+ })
+ .then((response) => response.data.statistic)
+ .catch((response) => response.data);
+
+ // update
+ const update_statistic = (organization_id, id, data) => $http
+ .put(`/api/v3/statistics/${id}/`, data, {
+ params: {
+ organization_id
+ }
+ })
+ .then((response) => response.data.statistic)
+ .catch((response) => response.data);
+
+ // create
+ const new_statistic = (organization_id, data) => $http
+ .post('/api/v3/statistics/', data, {
+ params: {
+ organization_id
+ }
+ })
+ .then((response) => response.data.statistic)
+ .catch((response) => response.data);
+
+ // calculate statistics
+ const calculate_statistics = (organization_id, id) => $http
+ .post(`/api/v3/statistics/${id}/calculate/`, null, {
+ params: {
+ organization_id
+ }
+ })
+ .then((response) => response)
+ .catch((response) => response);
+
+ const statistics_factory = {
+ get_statistics,
+ get_statistic,
+ update_statistic,
+ new_statistic,
+ calculate_statistics
+ };
+
+ return statistics_factory;
+ }
+]);
diff --git a/seed/static/seed/partials/organization_stats.html b/seed/static/seed/partials/organization_stats.html
new file mode 100644
index 0000000000..0834c054dc
--- /dev/null
+++ b/seed/static/seed/partials/organization_stats.html
@@ -0,0 +1,200 @@
+
+
+
+
+
+
+
Here is where you can see stats about your org. You will need to expand the configurations section below to indicate which columns your data is stored in.
+
+
{$ btnText $}
+
+
+ Select the columns where your organization stores the relevant data to generate statistics. Note that quantities will be converted to calculate EUIs in kBtu/ft2 where
+ appropriate. Enter EITHER Annual Electricity and Natural Gas usage quantities OR Electricity and Gas EUIs. EUIs will be used if given; otherwise they will be calculated from usage
+ quantities.
+
+
+
+
+
+ {$:: 'Save Changes' | translate $}
+
+
+
+
+
+
Statistics
+
+ {$:: 'Calculate Statistics' | translate $}
+
+
+
Overall Statistics - Gross Floor Area
+
+
+ Cycle
+ Number of Properties
+ Total Gross Floor Area (ft2)
+
+
+ {$ item.cycle $}
+ {$ item.num_records $}
+ {$ item.GFA | number: 0 $}
+
+
+
+
Overall Statistics - Electricity
+
+
+ Cycle
+ Number of Properties
+ Gross Floor Area Represented (ft2)
+ Electricity Use (kWh)
+
+
+ {$ item.cycle $}
+ {$ item.num_records $}
+ {$ item.GFA | number: 0 $}
+ {$ item.Electricity_Use | number: 0 $}
+
+
+
+
Overall Statistics - Natural Gas
+
+
+ Cycle
+ Number of Properties
+ Gross Floor Area Represented (ft2)
+ Natural Gas Use (therms)
+
+
+ {$ item.cycle $}
+ {$ item.num_records $}
+ {$ item.GFA | number: 0 $}
+ {$ item.Natural_Gas_Use | number: 0 $}
+
+
+
+
EUI Quantiles - Electricity
+
+
+ Cycle
+ 5%
+ 25%
+ 50%
+ 75%
+ 95%
+
+
+ {$ key $}
+ {$ item['0.05'] | number: 2 $}
+ {$ item['0.25'] | number: 2 $}
+ {$ item['0.5'] | number: 2 $}
+ {$ item['0.75'] | number: 2 $}
+ {$ item['0.95'] | number: 2 $}
+
+
+
+
EUI Quantiles - Natural Gas
+
+
+ Cycle
+ 5%
+ 25%
+ 50%
+ 75%
+ 95%
+
+
+ {$ key $}
+ {$ item["0.05"] | number: 2 $}
+ {$ item["0.25"] | number: 2 $}
+ {$ item["0.5"] | number: 2 $}
+ {$ item["0.75"] | number: 2 $}
+ {$ item["0.95"] | number: 2 $}
+
+
+
+
+
+
+
+
+
diff --git a/seed/static/seed/scss/style.scss b/seed/static/seed/scss/style.scss
index b374fdd4df..dd4d36177f 100755
--- a/seed/static/seed/scss/style.scss
+++ b/seed/static/seed/scss/style.scss
@@ -5893,3 +5893,9 @@ tags-input .tags .tag-item {
margin-left: 20px;
margin-right: 20px;
}
+
+.stat_instructions_container {
+ border: 1px solid #ccc;
+ padding: 20px;
+ margin-top: 10px;
+}
diff --git a/seed/templates/seed/_header.html b/seed/templates/seed/_header.html
index 4eb5a41d08..b4379f2206 100644
--- a/seed/templates/seed/_header.html
+++ b/seed/templates/seed/_header.html
@@ -82,6 +82,9 @@
Sharing
+
+ Stats
+
Sub-Organizations
+
@@ -213,6 +214,7 @@
+
diff --git a/seed/templates/seed/_vendor_scripts.html b/seed/templates/seed/_vendor_scripts.html
index 25e8cc57ec..1fb6de7985 100644
--- a/seed/templates/seed/_vendor_scripts.html
+++ b/seed/templates/seed/_vendor_scripts.html
@@ -35,6 +35,7 @@
+
diff --git a/seed/views/v3/statistics.py b/seed/views/v3/statistics.py
new file mode 100644
index 0000000000..4ad283fc9e
--- /dev/null
+++ b/seed/views/v3/statistics.py
@@ -0,0 +1,487 @@
+"""
+SEED Platform (TM), Copyright (c) Alliance for Sustainable Energy, LLC, and other contributors.
+See also https://github.com/SEED-platform/seed/blob/main/LICENSE.md
+"""
+
+import re
+from copy import deepcopy
+
+import django.core.exceptions
+import numpy as np
+import pandas as pd
+from django.db import IntegrityError, models
+from django.db.models.fields.json import KeyTransform
+from django.http import JsonResponse
+from drf_yasg.utils import swagger_auto_schema
+from rest_framework import status, viewsets
+from rest_framework.decorators import action
+
+from seed.decorators import ajax_request_class, require_organization_id_class
+from seed.lib.superperms.orgs.decorators import has_perm_class
+from seed.lib.superperms.orgs.models import AccessLevelInstance, Organization
+from seed.models.columns import Column
+from seed.models.cycles import Cycle
+from seed.models.properties import PropertyView
+from seed.models.statistics_setups import StatisticsSetup
+from seed.serializers.statistics_setups import StatisticsSetupSerializer
+from seed.utils.api import OrgMixin, api_endpoint_class
+from seed.utils.api_schema import AutoSchemaHelper, swagger_auto_schema_org_query_param
+
+
+def _validate_data(data, org_id):
+ error = False
+ msgs = []
+
+ # Validate Columns
+ column_names = ["gfa_column", "electricity_column", "natural_gas_column"]
+ for item in column_names:
+ c_id = data.get(item)
+ if c_id:
+ c_col = Column.objects.get(pk=c_id)
+
+ if c_col.organization_id != org_id:
+ # error, this column does not belong to this org
+ error = True
+ msgs.append("The selected column for " + item + " does not belong to this organization")
+
+ return error, msgs
+
+
+def _remove_nan(the_obj):
+ """Convert all NaN values in a nested dictionary to None"""
+ if isinstance(the_obj, dict):
+ for key, value in the_obj.items():
+ if isinstance(value, dict):
+ _remove_nan(value)
+ elif pd.isna(value):
+ the_obj[key] = None
+ else:
+ for item in the_obj:
+ if isinstance(item, dict):
+ _remove_nan(item)
+ elif pd.isna(item):
+ the_obj[item] = None
+ return the_obj
+
+
+def _convert_to_numeric(df, exclude_cols=[]):
+ """
+ Convert all columns in a DataFrame to numeric types.
+ For columns with text patterns like '2042.0 foot ** 2',
+ extract the numeric value before conversion.
+
+ Parameters:
+ -----------
+ df : pandas.DataFrame
+ DataFrame with mixed type columns
+
+ Returns:
+ --------
+ pandas.DataFrame
+ DataFrame with all columns converted to numeric types
+ """
+ # Create a copy to avoid modifying the original
+ df_numeric = df.copy()
+
+ for col in df_numeric.columns:
+ # Skip columns in exclude_cols
+ if col in exclude_cols:
+ continue
+ # Check if column is already numeric
+ if pd.api.types.is_numeric_dtype(df_numeric[col]):
+ continue
+
+ # If it's an object type, try to extract numeric values
+ if df_numeric[col].dtype == "object":
+ # Function to extract numeric part from strings
+ def extract_numeric(val):
+ if pd.isna(val):
+ return np.nan
+ if isinstance(val, (int, float)):
+ return val
+
+ # Convert to string if not already
+ val_str = str(val)
+
+ # Extract the first number (including decimals)
+ # This pattern matches numbers like 123, 123.45, .45
+ match = re.search(r"([-+]?\d*\.?\d+)", val_str)
+ if match:
+ return match.group(1)
+ return val
+
+ # Apply extraction and then convert to numeric
+ df_numeric[col] = df_numeric[col].apply(extract_numeric)
+
+ # Convert to numeric, with errors='coerce' to convert non-numerics to NaN
+ df_numeric[col] = pd.to_numeric(df_numeric[col], errors="coerce")
+
+ return df_numeric
+
+
+class StastisticsSetupViewSet(viewsets.ViewSet, OrgMixin):
+ serializer_class = StatisticsSetupSerializer
+ model = StatisticsSetup
+
+ @swagger_auto_schema_org_query_param
+ @api_endpoint_class
+ @ajax_request_class
+ @has_perm_class("requires_owner")
+ def list(self, request):
+ organization_id = self.get_organization(request)
+ stats = StatisticsSetup.objects.filter(organization=organization_id)
+
+ s_data = StatisticsSetupSerializer(stats, many=True).data
+
+ return JsonResponse({"status": "success", "statistics": s_data}, status=status.HTTP_200_OK)
+
+ @swagger_auto_schema_org_query_param
+ @require_organization_id_class
+ @api_endpoint_class
+ @ajax_request_class
+ @has_perm_class("requires_owner")
+ def retrieve(self, request, pk=0):
+ organization = self.get_organization(request)
+ if pk == 0:
+ try:
+ return JsonResponse(
+ {
+ "status": "success",
+ "statistic": StatisticsSetupSerializer(StatisticsSetup.objects.filter(organization=organization).first()).data,
+ },
+ status=status.HTTP_200_OK,
+ )
+ except Exception:
+ return JsonResponse(
+ {"status": "error", "message": "No statistics setup exist with this identifier"}, status=status.HTTP_404_NOT_FOUND
+ )
+ else:
+ try:
+ data = StatisticsSetupSerializer(StatisticsSetup.objects.get(id=pk, organization=organization)).data
+ return JsonResponse({"status": "success", "stastic": data}, status=status.HTTP_200_OK)
+ except StatisticsSetup.DoesNotExist:
+ return JsonResponse(
+ {"status": "error", "message": f"Statistics Setup with id {pk} does not exist"}, status=status.HTTP_404_NOT_FOUND
+ )
+
+ @swagger_auto_schema_org_query_param
+ @require_organization_id_class
+ @api_endpoint_class
+ @ajax_request_class
+ @has_perm_class("requires_owner")
+ def destroy(self, request, pk):
+ organization_id = self.get_organization(request)
+
+ try:
+ StatisticsSetup.objects.get(id=pk, organization=organization_id).delete()
+ except StatisticsSetup.DoesNotExist:
+ return JsonResponse({"status": "error", "message": f"Statistics with id {pk} does not exist"}, status=status.HTTP_404_NOT_FOUND)
+
+ return JsonResponse({"status": "success", "message": f"Successfully deleted Statistics ID {pk}"}, status=status.HTTP_200_OK)
+
+ @swagger_auto_schema(
+ manual_parameters=[AutoSchemaHelper.query_org_id_field()],
+ request_body=AutoSchemaHelper.schema_factory(
+ {
+ "gfa_column": "integer",
+ "gfa_units": "string",
+ "electricity_column": "integer",
+ "electricity_units": "string",
+ "natural_gas_column": "integer",
+ "natural_gas_units": "string",
+ "electricity_eui_column": "integer",
+ "electricity_eui_units": "string",
+ "natural_gas_eui_column": "integer",
+ "natural_gas_eui_units": "string",
+ },
+ ),
+ )
+ @require_organization_id_class
+ @api_endpoint_class
+ @ajax_request_class
+ @has_perm_class("requires_owner")
+ def create(self, request):
+ org_id = int(self.get_organization(request))
+ try:
+ Organization.objects.get(pk=org_id)
+ except Organization.DoesNotExist:
+ return JsonResponse({"status": "error", "message": "bad organization_id"}, status=status.HTTP_400_BAD_REQUEST)
+
+ data = deepcopy(request.data)
+ data.update({"organization_id": org_id})
+
+ error, msgs = _validate_data(data, org_id)
+ if error is True:
+ return JsonResponse({"status": "error", "message": ",".join(msgs)}, status=status.HTTP_400_BAD_REQUEST)
+ serializer = StatisticsSetupSerializer(data=data)
+
+ if not serializer.is_valid():
+ error_response = {"status": "error", "message": "Data Validation Error", "errors": serializer.errors}
+ return JsonResponse(error_response, status=status.HTTP_400_BAD_REQUEST)
+
+ try:
+ serializer.save()
+
+ return JsonResponse({"status": "success", "statistic": serializer.data}, status=status.HTTP_200_OK)
+ except IntegrityError:
+ return JsonResponse(
+ {"status": "error", "message": "Only one statistics setup can be created per organization"},
+ status=status.HTTP_400_BAD_REQUEST,
+ )
+ except django.core.exceptions.ValidationError as e:
+ message_dict = e.message_dict
+ return JsonResponse({"status": "error", "message": "Bad Request", "errors": message_dict}, status=status.HTTP_400_BAD_REQUEST)
+
+ @swagger_auto_schema(
+ manual_parameters=[AutoSchemaHelper.query_org_id_field()],
+ request_body=AutoSchemaHelper.schema_factory(
+ {
+ "gfa_column": "integer",
+ "gfa_units": "string",
+ "electricity_column": "integer",
+ "electricity_units": "string",
+ "natural_gas_column": "integer",
+ "natural_gas_units": "string",
+ "electricity_eui_column": "integer",
+ "electricity_eui_units": "string",
+ "natural_gas_eui_column": "integer",
+ "natural_gas_eui_units": "string",
+ },
+ ),
+ )
+ @require_organization_id_class
+ @api_endpoint_class
+ @ajax_request_class
+ @has_perm_class("requires_owner")
+ def update(self, request, pk):
+ org_id = self.get_organization(request)
+
+ statistic = None
+ try:
+ statistic = StatisticsSetup.objects.get(id=pk, organization=org_id)
+ except StatisticsSetup.DoesNotExist:
+ return JsonResponse({"status": "error", "message": f"Statistics with id {pk} does not exist"}, status=status.HTTP_404_NOT_FOUND)
+
+ data = deepcopy(request.data)
+ error, msgs = _validate_data(data, org_id)
+ if error is True:
+ return JsonResponse({"status": "error", "message": ",".join(msgs)}, status=status.HTTP_400_BAD_REQUEST)
+
+ serializer = StatisticsSetupSerializer(statistic, data=data, partial=True)
+
+ if not serializer.is_valid():
+ return JsonResponse(
+ {"status": "error", "message": "Bad Request", "errors": serializer.errors}, status=status.HTTP_400_BAD_REQUEST
+ )
+
+ try:
+ serializer.save()
+
+ return JsonResponse(
+ {
+ "status": "success",
+ "statistic": serializer.data,
+ },
+ status=status.HTTP_200_OK,
+ )
+ except django.core.exceptions.ValidationError as e:
+ message_dict = e.message_dict
+ # rename key __all__ to general to make it more user-friendly
+ if "__all__" in message_dict:
+ message_dict["general"] = message_dict.pop("__all__")
+
+ return JsonResponse(
+ {
+ "status": "error",
+ "message": "Bad request",
+ "errors": message_dict,
+ },
+ status=status.HTTP_400_BAD_REQUEST,
+ )
+ except Exception as e:
+ return JsonResponse(
+ {
+ "status": "error",
+ "message": "Bad request",
+ "errors": str(e),
+ },
+ status=status.HTTP_400_BAD_REQUEST,
+ )
+
+ @require_organization_id_class
+ @api_endpoint_class
+ @ajax_request_class
+ @has_perm_class("requires_owner")
+ @action(detail=True, methods=["POST"])
+ def calculate(self, request, pk):
+ org_id = self.get_organization(request)
+ ali = AccessLevelInstance.objects.get(pk=request.access_level_instance_id)
+ # get statistic setup for this org
+ try:
+ statistic = StatisticsSetup.objects.get(id=pk, organization=org_id)
+ except StatisticsSetup.DoesNotExist:
+ return JsonResponse({"status": "error", "message": f"Statistics with id {pk} does not exist"}, status=status.HTTP_404_NOT_FOUND)
+
+ # get all cycles in this org
+ cycles = Cycle.objects.filter(organization=org_id)
+
+ # column ids we are interested in, only those non-null
+
+ cols = {}
+ extra_data_cols = {}
+ if statistic.gfa_column:
+ if statistic.gfa_column.is_extra_data:
+ extra_data_cols["gfa"] = statistic.gfa_column.column_name
+ else:
+ cols["gfa"] = "state__" + statistic.gfa_column.column_name
+ else:
+ # if we don't have GFA column, we can't calculate anything
+ return JsonResponse(
+ {
+ "status": "error",
+ "message": "GFA column is required to calculate statistics",
+ },
+ status=status.HTTP_400_BAD_REQUEST,
+ )
+ # prefer EUI column over usage column
+ if statistic.electricity_eui_column:
+ if statistic.electricity_eui_column.is_extra_data:
+ extra_data_cols["elec_eui"] = statistic.electricity_eui_column.column_name
+ else:
+ cols["elec_eui"] = "state__" + statistic.electricity_eui_column.column_name
+ elif statistic.electricity_column:
+ if statistic.electricity_column.is_extra_data:
+ extra_data_cols["electricity"] = statistic.electricity_column.column_name
+ else:
+ cols["electricity"] = "state__" + statistic.electricity_column.column_name
+ if statistic.natural_gas_eui_column:
+ if statistic.natural_gas_eui_column.is_extra_data:
+ extra_data_cols["gas_eui"] = statistic.natural_gas_eui_column.column_name
+ else:
+ cols["gas_eui"] = "state__" + statistic.natural_gas_eui_column.column_name
+ elif statistic.natural_gas_column:
+ if statistic.natural_gas_column.is_extra_data:
+ extra_data_cols["natural_gas"] = statistic.natural_gas_column.column_name
+ else:
+ cols["natural_gas"] = "state__" + statistic.natural_gas_column.column_name
+
+ # make sure we have at least electricity or gas here to calculate?
+ if len(cols.keys()) + len(extra_data_cols.keys()) < 2:
+ return JsonResponse(
+ {
+ "status": "error",
+ "message": "At least one of electricity or natural gas column is required to calculate statistics",
+ },
+ status=status.HTTP_400_BAD_REQUEST,
+ )
+
+ # reverse the cols dictionary to have the keys be the values
+ # this is to make it easier to map the results back to the correct columns in df
+ cols_reverse = {v: k for k, v in cols.items()}
+
+ annotations: dict[str, models.Func] = {}
+ for key, val in extra_data_cols.items():
+ annotations[key] = KeyTransform(val, "state__extra_data")
+
+ results = {}
+ df_full = pd.DataFrame()
+ # get all properties that fall in all cycles for this org
+ for cycle in cycles:
+ # get Property-Views for this Cycle
+ property_views = (
+ PropertyView.objects.select_related("state")
+ .filter(
+ property__organization_id=org_id,
+ cycle_id=cycle.id,
+ property__access_level_instance__lft__gte=ali.lft,
+ property__access_level_instance__rgt__lte=ali.rgt,
+ )
+ .annotate(**annotations)
+ .values("property_id", "state_id", *(list(cols.values())), *(list(extra_data_cols.keys())))
+ )
+
+ # dataframe column names will be:
+ # property_id, state_id, electricity, natural_gas, cycle_id, gfa,
+ # elec_eui, gas_eui
+
+ df_temp = pd.DataFrame.from_records(list(property_views))
+ df_temp["cycle"] = cycle.name # or use ID and get name in frontend?
+ df_full = pd.concat([df_full, df_temp])
+
+ # remap the column names to the original column names
+ df_full = df_full.rename(columns=cols_reverse)
+ df_full = _convert_to_numeric(df_full, ["cycle"])
+
+ # calculate the statistics
+ # check if we have EUIs or Usage columns
+ # electricity
+ if "electricity" in df_full.columns:
+ # we have usage, compute EUI
+ df_full["elec_eui"] = (df_full["electricity"] / df_full["gfa"]).replace([np.inf, -np.inf], np.nan)
+ # gas
+ if "natural_gas" in df_full.columns:
+ df_full["gas_eui"] = (df_full["natural_gas"] / df_full["gfa"]).replace([np.inf, -np.inf], np.nan)
+
+ # reverse calculate electricity and/or natural gas from EUIs
+ # electricity
+ if "elec_eui" in df_full.columns:
+ df_full["electricity"] = df_full["elec_eui"] * df_full["gfa"]
+ # gas
+ if "gas_eui" in df_full.columns:
+ df_full["natural_gas"] = df_full["gas_eui"] * df_full["gfa"]
+
+ # high-level counts:
+ yearly_gfa = (
+ df_full.dropna(subset=["gfa"])
+ .groupby("cycle")
+ .agg(num_records=("property_id", "count"), cycle=("cycle", "first"), GFA=("gfa", "sum"))
+ )
+ yearly_elec = (
+ df_full.dropna(subset=["gfa", "elec_eui"])
+ .groupby("cycle")
+ .agg(num_records=("property_id", "count"), cycle=("cycle", "first"), GFA=("gfa", "sum"), Electricity_Use=("electricity", "sum"))
+ )
+ yearly_elec.style.format("{:,.0f}")
+ # print(yearly_elec) # format numbers with commas
+
+ yearly_gas = (
+ df_full.dropna(subset=["gfa", "gas_eui"])
+ .groupby("cycle")
+ .agg(num_records=("property_id", "count"), cycle=("cycle", "first"), GFA=("gfa", "sum"), Natural_Gas_Use=("natural_gas", "sum"))
+ )
+ yearly_gas.style.format("{:,.0f}")
+ # print(yearly_gas) # format numbers with commas
+
+ # elec quartiles for each cycle (include cycle key)
+ elec_eui_quantiles = df_full.groupby("cycle")["elec_eui"].quantile([0.05, 0.25, 0.50, 0.75, 0.95]).unstack() # noqa: PD010
+
+ # print(f"Elec Quantiles")
+ # print(elec_eui_quantiles)
+
+ # do the same for gas eui
+ gas_eui_quantiles = df_full.groupby("cycle")["gas_eui"].quantile([0.05, 0.25, 0.50, 0.75, 0.95]).unstack() # noqa: PD010
+ # print(f"Gas Quantiles")
+ # print(gas_eui_quantiles)
+
+ df_return = df_full[["property_id", "cycle", "elec_eui", "gas_eui"]]
+
+ # transform df_return to have a key for each cycle and an array of elec_eui
+ chart_data_elec = {}
+ chart_data_gas = {}
+ for year_value in df_return["cycle"].unique():
+ chart_data_elec[year_value] = df_return[df_return["cycle"] == year_value]["elec_eui"].dropna().tolist()
+ chart_data_gas[year_value] = df_return[df_return["cycle"] == year_value]["gas_eui"].dropna().tolist()
+
+ # return the results and convert NaN to None
+ results["elec_quantiles"] = elec_eui_quantiles.to_dict(orient="index")
+ results["gas_quantiles"] = gas_eui_quantiles.to_dict(orient="index")
+ results["chart_data_elec"] = chart_data_elec
+ results["chart_data_gas"] = chart_data_gas
+ results["annual_gfa"] = _remove_nan(yearly_gfa.to_dict(orient="records"))
+ results["annual_elec"] = _remove_nan(yearly_elec.to_dict(orient="records"))
+ results["annual_gas"] = _remove_nan(yearly_gas.to_dict(orient="records"))
+
+ return JsonResponse(
+ {"status": "success", "data": results},
+ status=status.HTTP_200_OK,
+ )
diff --git a/vendors/package-lock.json b/vendors/package-lock.json
index 6c6cc48bf6..fbc8236130 100644
--- a/vendors/package-lock.json
+++ b/vendors/package-lock.json
@@ -12,6 +12,7 @@
"@fontsource/pt-sans": "^5.1.0",
"@fontsource/pt-sans-narrow": "^5.1.0",
"@fortawesome/fontawesome-free": "^6.6.0",
+ "@sgratzl/chartjs-chart-boxplot": "^3.10.0",
"angular": "^1.8.3",
"angular-animate": "^1.8.3",
"angular-aria": "^1.8.3",
@@ -396,6 +397,22 @@
"license": "MIT",
"peer": true
},
+ "node_modules/@sgratzl/boxplots": {
+ "version": "1.3.4",
+ "resolved": "https://registry.npmjs.org/@sgratzl/boxplots/-/boxplots-1.3.4.tgz",
+ "integrity": "sha512-wRlHp9sZ2aQ/BBq+6Xg8DBsf4zXs/DQhNpLRlfwyO62IqTANmu7d64yE6jSKMnixM+nbtZqbeGa9spiaU/S++g=="
+ },
+ "node_modules/@sgratzl/chartjs-chart-boxplot": {
+ "version": "3.10.0",
+ "resolved": "https://registry.npmjs.org/@sgratzl/chartjs-chart-boxplot/-/chartjs-chart-boxplot-3.10.0.tgz",
+ "integrity": "sha512-+XzmRr9unHzTAMrNVjQ1U6qKid2fEYpV6byC7WIKyU/nT3zB1g98RDuF+z8cFwBjZm/NdVw0uaQYdjHWwHiPbA==",
+ "dependencies": {
+ "@sgratzl/boxplots": "^1.3.0"
+ },
+ "peerDependencies": {
+ "chart.js": "^3.9.1"
+ }
+ },
"node_modules/@socket.io/component-emitter": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/@socket.io/component-emitter/-/component-emitter-3.1.2.tgz",
diff --git a/vendors/package.json b/vendors/package.json
index e41ed01207..21a8ad9f3a 100644
--- a/vendors/package.json
+++ b/vendors/package.json
@@ -30,6 +30,7 @@
"chartjs-chart-sankey": "^0.14.0",
"chartjs-plugin-annotation": "^2.2.1",
"chartjs-plugin-zoom": "=1.2.1",
+ "@sgratzl/chartjs-chart-boxplot": "^3.10.0",
"d3": "^5.16.0",
"dimple": "PMSI-AlignAlytics/dimple#2.3.0",
"file-saver": "^2.0.5",