From 729c1ece0e5dd20fef907555eb44ea51712bc81c Mon Sep 17 00:00:00 2001 From: Hannah Eslinger Date: Mon, 24 Nov 2025 16:18:03 -0700 Subject: [PATCH 1/3] Add pm custom download --- seed/tests/test_portfoliomanager.py | 120 ++++++++++++++++++++++ seed/tests/test_taxlot_views.py | 1 - seed/views/v3/portfolio_manager.py | 154 +++++++++++++++++++++++----- 3 files changed, 249 insertions(+), 26 deletions(-) diff --git a/seed/tests/test_portfoliomanager.py b/seed/tests/test_portfoliomanager.py index 7b3ff5d236..b9e52ce464 100644 --- a/seed/tests/test_portfoliomanager.py +++ b/seed/tests/test_portfoliomanager.py @@ -8,10 +8,12 @@ import json import locale import os +from datetime import datetime from os import path from pathlib import Path from unittest import skip, skipIf +import pandas as pd import pytest import requests import xmltodict @@ -537,3 +539,121 @@ def test_parse_pm_report(self): self.assertEqual(properties[0]["portfolioManagerPropertyId"], "22178843") self.assertIsNone(properties[0]["parentPropertyId"]) self.assertEqual(properties[0]["propertyFloorAreaBuildingsAndParking"], "89250.0") + + +class PortfolioManagerCustomDownloadTest(TestCase): + def setUp(self): + user_details = { + "username": "test_user@demo.com", + "password": "test_pass", + } + self.user = User.objects.create_superuser(email="test_user@demo.com", **user_details) + self.org, _, _ = create_organization(self.user) + self.client.login(**user_details) + + self.pm_un = os.environ.get(PM_UN) + self.pm_pw = os.environ.get(PM_PW) + if not self.pm_un or not self.pm_pw: + self.fail(f"Somehow PM test was initiated without {PM_UN} or {PM_PW} in the environment") + + def test_generate_and_download_meter_data(self): + # SetUp + pm = PortfolioManagerImport(self.pm_un, self.pm_pw) + ids = ["16731961"] + start_date = datetime(2019, 1, 1) + end_date = datetime(2019, 12, 31) + + # Action + excel_b = pm.generate_and_download_meter_data(ids, start_date, end_date) + + # Assertion + xl = pd.ExcelFile(excel_b) + assert xl.sheet_names == ["Meters", "Meter Entries"] + + meters_df = xl.parse("Meters", header=5) + assert meters_df[["Portfolio Manager ID", "Portfolio Manager Meter ID"]].to_dict("records") == [ + {"Portfolio Manager ID": 16731961, "Portfolio Manager Meter ID": 110290980}, + {"Portfolio Manager ID": 16731961, "Portfolio Manager Meter ID": 110290981}, + {"Portfolio Manager ID": 16731961, "Portfolio Manager Meter ID": 110291021}, + {"Portfolio Manager ID": 16731961, "Portfolio Manager Meter ID": 110291022}, + ] + + meter_readings_df = xl.parse("Meter Entries", header=5) + assert meter_readings_df["Portfolio Manager Meter ID"].value_counts().to_dict() == {110291021: 11, 110291022: 11} + + def test_custom_download(self): + # SetUp + ids = ["16731961"] + + # Action + resp = self.client.post( + reverse_lazy("api:v3:portfolio_manager-download"), + json.dumps( + { + "username": self.pm_un, + "password": self.pm_pw, + "property_ids": ids, + "start_date": "01/01/2019", + "end_date": "12/31/2019", + } + ), + content_type="application/json", + ) + + # Assertion + assert resp.status_code == 200 + xl = pd.ExcelFile(resp.content) + assert xl.sheet_names == ["Meters", "Meter Entries"] + + meters_df = xl.parse("Meters", header=5) + assert meters_df[["Portfolio Manager ID", "Portfolio Manager Meter ID"]].to_dict("records") == [ + {"Portfolio Manager ID": 16731961, "Portfolio Manager Meter ID": 110290980}, + {"Portfolio Manager ID": 16731961, "Portfolio Manager Meter ID": 110290981}, + {"Portfolio Manager ID": 16731961, "Portfolio Manager Meter ID": 110291021}, + {"Portfolio Manager ID": 16731961, "Portfolio Manager Meter ID": 110291022}, + ] + + meter_readings_df = xl.parse("Meter Entries", header=5) + assert meter_readings_df["Portfolio Manager Meter ID"].value_counts().to_dict() == {110291021: 11, 110291022: 11} + + def test_custom_download_multiple(self): + # SetUp + ids = ["16731961", "22178849"] + + # Action + resp = self.client.post( + reverse_lazy("api:v3:portfolio_manager-download"), + json.dumps( + { + "username": self.pm_un, + "password": self.pm_pw, + "property_ids": ids, + "start_date": "01/01/2019", + "end_date": "12/31/2019", + } + ), + content_type="application/json", + ) + + # Assertion + assert resp.status_code == 200 + xl = pd.ExcelFile(resp.content) + assert xl.sheet_names == ["Meters", "Meter Entries"] + + meters_df = xl.parse("Meters", header=5) + assert meters_df[["Portfolio Manager ID", "Portfolio Manager Meter ID"]].to_dict("records") == [ + {"Portfolio Manager ID": 22178849, "Portfolio Manager Meter ID": 139067336}, + {"Portfolio Manager ID": 22178849, "Portfolio Manager Meter ID": 139067339}, + {"Portfolio Manager ID": 16731961, "Portfolio Manager Meter ID": 110290980}, + {"Portfolio Manager ID": 16731961, "Portfolio Manager Meter ID": 110290981}, + {"Portfolio Manager ID": 16731961, "Portfolio Manager Meter ID": 110291021}, + {"Portfolio Manager ID": 16731961, "Portfolio Manager Meter ID": 110291022}, + ] + + meter_readings_df = xl.parse("Meter Entries", header=5) + assert meter_readings_df["Portfolio Manager Meter ID"].value_counts().to_dict() == { + 139067336: 12, + 139067339: 12, + 110291021: 11, + 110291022: 11, + } diff --git a/seed/tests/test_taxlot_views.py b/seed/tests/test_taxlot_views.py index cbf6a07856..a8cbdff370 100644 --- a/seed/tests/test_taxlot_views.py +++ b/seed/tests/test_taxlot_views.py @@ -120,7 +120,6 @@ def test_edit_properties_creates_notes_after_initial_edit(self): ] self.assertEqual(note.note_type, Note.LOG) self.assertEqual(note.name, "Automatically Created") - # import pdb; pdb.set_trace() self.assertCountEqual(note.log_data, expected_log_data) def test_first_lat_long_edit(self): diff --git a/seed/views/v3/portfolio_manager.py b/seed/views/v3/portfolio_manager.py index e7d4653f42..d2ada45797 100644 --- a/seed/views/v3/portfolio_manager.py +++ b/seed/views/v3/portfolio_manager.py @@ -5,6 +5,7 @@ import json import logging +import re import time from datetime import datetime @@ -31,7 +32,7 @@ class PortfolioManagerSerializer(serializers.Serializer): class PortfolioManagerViewSet(GenericViewSet): """ - This ViewSet contains two API views: /template_list/ and /report/ that are used to interface SEED with ESPM + This ViewSet contains four API views: /template_list/, /report/, /download/, and /custom_download/ that are used to interface SEED with ESPM """ serializer_class = PortfolioManagerSerializer @@ -219,43 +220,42 @@ def report(self, request): required=["username", "password"], ), ) - @action(detail=True, methods=["POST"]) - def download(self, request, pk): + @action(detail=False, methods=["POST"]) + def download(self, request): """Download a single property report from Portfolio Manager. The PK is the PM property ID that is on ESPM""" - if "username" not in request.data: - _log.debug(f"Invalid call to PM worker: missing username for PM account: {request.data!s}") + required_fields = {"username", "password", "property_ids", "start_date", "end_date"} + missing_fields = required_fields - set(request.data.keys()) + if len(missing_fields) > 0: return JsonResponse( - {"status": "error", "message": "Invalid call to PM worker: missing username for PM account"}, - status=status.HTTP_400_BAD_REQUEST, - ) - if "password" not in request.data: - _log.debug(f"Invalid call to PM worker: missing password for PM account: {request.data!s}") - return JsonResponse( - {"status": "error", "message": "Invalid call to PM worker: missing password for PM account"}, + {"status": "error", "message": f"Invalid call to PM worker: missing fields {missing_fields}"}, status=status.HTTP_400_BAD_REQUEST, ) username = request.data["username"] password = request.data["password"] - if "filename" not in request.data: - filename = f"pm_{pk}_{datetime.strftime(datetime.now(), '%Y%m%d_%H%M%S')}.xlsx" - else: - filename = request.data["filename"] + property_ids = request.data["property_ids"] + start_date = datetime.strptime(request.data.get("start_date"), "%m/%d/%Y") + end_date = datetime.strptime(request.data.get("end_date"), "%m/%d/%Y") + # generate and download meter data pm = PortfolioManagerImport(username, password) try: - content = pm.return_single_property_report(pk) + content = pm.generate_and_download_meter_data(property_ids, start_date, end_date) + except PMError as pme: + _log.debug(f"{pme!s}: {property_ids!s}") + return JsonResponse({"status": "error", "message": str(pme)}, status=status.HTTP_400_BAD_REQUEST) - # return the excel file as the HTTP response + # return the Excel file + try: response = HttpResponse(content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") - response["Content-Disposition"] = f'attachment; filename="{filename}"' + response["Content-Disposition"] = 'attachment; filename="pm_report_export.xlsx"' response.write(content) - return response + except Exception as e: + _log.debug(f"ERROR downloading EXCEL report: {e}") + return JsonResponse({"status": "error", "message": "Malformed XML from template download"}, status=status.HTTP_400_BAD_REQUEST) - except PMError as pme: - _log.debug(f"{pme!s}: PM Property ID {pk}") - return JsonResponse({"status": "error", "message": str(pme)}, status=status.HTTP_400_BAD_REQUEST) + return response # TODO: Move this object to /seed/utils/portfolio_manager.py @@ -472,7 +472,7 @@ def generate_and_download_template_report(self, matched_template, report_format= # Now we need to wait while the report is being generated attempt_count = 0 report_generation_complete = False - while attempt_count < 90: + while attempt_count < 24: attempt_count += 1 # get the report data @@ -496,7 +496,7 @@ def generate_and_download_template_report(self, matched_template, report_format= if not this_matched_template: raise PMError("Could not find a match for this report template id... odd at this point") if this_matched_template["pending"] == 1: - time.sleep(2) + time.sleep(5) continue else: report_generation_complete = True @@ -587,6 +587,110 @@ def return_single_property_report(self, pm_property_id: int): except requests.exceptions.SSLError: raise PMError("SSL Error in Portfolio Manager Query; check VPN/Network/Proxy.") + def generate_and_download_meter_data(self, pm_property_ids: list[int], start_date: datetime, end_date: datetime): + # login if needed + if not self.authenticated_headers: + self.login_and_set_cookie_header() + + # Get the csrf + response = requests.get( + "https://portfoliomanager.energystar.gov/pm/property/createCustomDownload/loadPage/maxPropertiesWithinLimit/true", + headers=self.authenticated_headers, + allow_redirects=True, + timeout=300, + ) + csrf_header = re.findall('(?<=meta name="_csrf_header" content=")(.*)(?="/>)', response.text)[0] + csrf_token = re.findall('(?<=meta name="_csrf" content=")(.*)(?="/>)', response.text)[0] + + # create custom download + response = requests.post( + "https://portfoliomanager.energystar.gov/pm/property/createCustomDownloadSubmit/", + headers={ + **self.authenticated_headers, + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.5", + "Accept-Encoding": "gzip, deflate, br, zstd", + "Content-Type": "application/x-www-form-urlencoded", + csrf_header: csrf_token, + }, + data={ + "_aggregateMeter": "on", + "_energyMeter": "on", + "_energyMeterEntries": "on", + "_meterEntries": "on", + "_meterType": "on", + "_propertyIds": "on", + "energyMeter": "true", + "energyMeterEntries": "true", + "includeInactiveMeters": "true", + "includeUnassociatedMeters": "false", + "meterEntries": "true", + "meterType": "true", + "selectionType": "MULTIPLE", + "selectedPropertyIdsAsString": pm_property_ids, + "customDownloadStartDate": start_date.strftime("%m/%d/%Y"), + "customDownloadEndDate": end_date.strftime("%m/%d/%Y"), + }, + stream=True, + timeout=300, + ) + + # Now we need to wait while the report is being generated + attempt_count = 0 + custom_download_found = False + while attempt_count < 24: + attempt_count += 1 + + # get the report data + try: + response = requests.post( + "https://portfoliomanager.energystar.gov/pm/notifications.json", + headers={ + **self.authenticated_headers, + "Accept": "application/json", + "Content-Type": "application/json", + csrf_header: csrf_token, + }, + timeout=300, + data='{"page":1,"pageSize":100,"sort":{"column":"CREATE_DATE","ascending":true},"type":"Notices"}', + ) + except requests.exceptions.SSLError: + raise PMError("SSL Error in Portfolio Manager Query; check VPN/Network/Proxy.") + if not response.status_code == status.HTTP_200_OK: + raise PMError("Unsuccessful response from report template rows query; aborting.") + + notifications = response.json()["result"]["items"] + if len(notifications) == 0: + time.sleep(5) + continue + + latest_notification = notifications[-1] + if latest_notification["notificationTypeCode"]["code"] == "CUSTOMPORTFOLIODOWNLOAD": + custom_download_found = True + break + + else: + time.sleep(5) + continue + + # Finally we can download the generated report + try: + response = requests.get( + f"https://portfoliomanager.energystar.gov/pm{latest_notification['notificationParameters'][1]['url']}", + headers=self.authenticated_headers, + timeout=300, + ) + except requests.exceptions.SSLError: + raise PMError("SSL Error in Portfolio Manager Query; check VPN/Network/Proxy.") + if response.status_code != status.HTTP_200_OK: + error_message = "Unsuccessful response from GET trying to download generated report;" + error_message += f"Returned with a status code = {response.status_code};" + raise PMError(error_message) + if not custom_download_found: + raise PMError("custom report not generated successfully; aborting.") + + return response.content + def _parse_properties_v1(self, xml): """Parse the XML (in dict format) response from the ESPM API and return a list of properties. This version was implemented prior to 02/13/2023 From 0baa49f7c9052127e5a616f3e5e37dbc587a852e Mon Sep 17 00:00:00 2001 From: Hannah Eslinger Date: Tue, 25 Nov 2025 15:17:32 -0700 Subject: [PATCH 2/3] Fix custom download endpoint --- seed/tests/test_portfoliomanager.py | 4 ++-- seed/views/v3/portfolio_manager.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/seed/tests/test_portfoliomanager.py b/seed/tests/test_portfoliomanager.py index b9e52ce464..0a095b93eb 100644 --- a/seed/tests/test_portfoliomanager.py +++ b/seed/tests/test_portfoliomanager.py @@ -587,7 +587,7 @@ def test_custom_download(self): # Action resp = self.client.post( - reverse_lazy("api:v3:portfolio_manager-download"), + reverse_lazy("api:v3:portfolio_manager-meter-download"), json.dumps( { "username": self.pm_un, @@ -622,7 +622,7 @@ def test_custom_download_multiple(self): # Action resp = self.client.post( - reverse_lazy("api:v3:portfolio_manager-download"), + reverse_lazy("api:v3:portfolio_manager-meter-download"), json.dumps( { "username": self.pm_un, diff --git a/seed/views/v3/portfolio_manager.py b/seed/views/v3/portfolio_manager.py index d2ada45797..f70a29203d 100644 --- a/seed/views/v3/portfolio_manager.py +++ b/seed/views/v3/portfolio_manager.py @@ -221,7 +221,7 @@ def report(self, request): ), ) @action(detail=False, methods=["POST"]) - def download(self, request): + def meter_download(self, request): """Download a single property report from Portfolio Manager. The PK is the PM property ID that is on ESPM""" required_fields = {"username", "password", "property_ids", "start_date", "end_date"} From 7b481fe22769f80ff38a82be8b9d0fdf86fa4539 Mon Sep 17 00:00:00 2001 From: heslinge Date: Fri, 9 Jan 2026 10:34:23 -0700 Subject: [PATCH 3/3] Re-run jobs