From 50895c69cb64b348090a5f113843f8dfe611bc01 Mon Sep 17 00:00:00 2001 From: emunsing Date: Mon, 6 Feb 2023 14:37:07 -0800 Subject: [PATCH 1/3] Update requirements.txt and setup.py to reflect dependencies --- requirements.txt | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index de5314f..fa9918a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ lxml>=3.6.4 mock nose-timer nose -pandas>=0.18,<0.21 +pandas parameterized python-dateutil pytz @@ -16,4 +16,4 @@ requests-cache requests-mock requests Sphinx -xlrd \ No newline at end of file +xlrd diff --git a/setup.py b/setup.py index 65092cf..ee1d6ba 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ def find_version(*file_paths): test_suite='nose.collector', install_requires=[ 'beautifulsoup4>=4.5.0', - 'pandas>=0.18,<0.21', + 'pandas', 'python-dateutil', 'pytz', 'requests', From 9838f469574ea8af763bcac1d4a7d73c1c8f0d3d Mon Sep 17 00:00:00 2001 From: emunsing Date: Mon, 6 Feb 2023 16:11:15 -0800 Subject: [PATCH 2/3] Fix generation forecast processing --- pyiso/eu.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/pyiso/eu.py b/pyiso/eu.py index 464c06e..33a599f 100644 --- a/pyiso/eu.py +++ b/pyiso/eu.py @@ -222,7 +222,7 @@ def get_load(self, control_area=None, latest=False, start_at=None, end_at=None, def get_generation(self, control_area=None, latest=False, yesterday=False, start_at=False, end_at=False, forecast=False, **kwargs): self.handle_options(data='gen', start_at=start_at, end_at=end_at, yesterday=yesterday, - latest=latest, control_area=control_area, forecast=False, **kwargs) + latest=latest, control_area=control_area, forecast=forecast, **kwargs) response = self.fetch_entsoe() return self.parse_response(response) @@ -303,15 +303,19 @@ def parse_response(self, response): 'timestamp': timestamp, 'freq': 'n/a', } - if (self.options['forecast']): - datapoint['market'] = 'DAM' if self.options['data'] == 'gen': - datapoint['market'] = self.CONTROL_AREAS[self.options['control_area']]['gen_market'] - datapoint['freq'] = self.CONTROL_AREAS[self.options['control_area']]['gen_freq'] - datapoint['gen_MW'] = int(point.quantity.text) - datapoint['fuel_name'] = self.fuels[ts.MktPSRType.psrType.text] + datapoint['market'] = self.CONTROL_AREAS[self.options['control_area']]['gen_market'] + datapoint['freq'] = self.CONTROL_AREAS[self.options['control_area']]['gen_freq'] + datapoint['gen_MW'] = int(point.quantity.text) + if hasattr(ts, 'MktPSRType'): + datapoint['fuel_name'] = self.fuels[ts.MktPSRType.psrType.text] + else: + datapoint['fuel_name'] = 'other' + elif self.options['data'] == 'load': - datapoint['load_MW'] = int(point.quantity.text) + if self.options['forecast']: + datapoint['market'] = 'DAM' + datapoint['load_MW'] = int(point.quantity.text) data.append(datapoint) return data From fd501efb39cdfe86e3f3a684e0970d871593c9aa Mon Sep 17 00:00:00 2001 From: emunsing Date: Tue, 7 Feb 2023 17:21:40 -0800 Subject: [PATCH 3/3] Bugfixes in EU XML parsing and handling of renewable forecasts --- pyiso/eu.py | 49 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/pyiso/eu.py b/pyiso/eu.py index 33a599f..825e531 100644 --- a/pyiso/eu.py +++ b/pyiso/eu.py @@ -1,10 +1,5 @@ -import time from pyiso.base import BaseClient -from pyiso import LOGGER -import requests -import pandas as pd -import numpy as np -from io import StringIO +from typing import Optional from datetime import datetime, timedelta import pytz from os import environ @@ -48,7 +43,7 @@ class EUClient(BaseClient): 'ENTSOe_ID': '10YAT-APG------L', 'gen_freq': '15m', 'gen_market': 'RTPD'}, 'BY': {'country': 'Belarus', 'Code': 'CTA|BY', - 'ENSTOe_ID': '10Y1001A1001S51S', + 'ENTSOe_ID': '10Y1001A1001S51S', 'gen_freq': '1hr', 'gen_market': 'RTHR'}, 'BE': {'country': 'Belgium', 'Code': 'CTA|BE', 'ENTSOe_ID': '10YBE----------2', @@ -219,10 +214,25 @@ def get_load(self, control_area=None, latest=False, start_at=None, end_at=None, response = self.fetch_entsoe() return self.parse_response(response) - def get_generation(self, control_area=None, latest=False, yesterday=False, start_at=False, - end_at=False, forecast=False, **kwargs): + def get_generation(self, control_area: Optional[str]=None, + latest: bool=False, yesterday: bool=False, start_at: bool=False, + end_at: bool=False, forecast: bool=False, renewable_forecast=False, **kwargs): + """ + :param control_area: String, matching one of the EU regions defined in the CONTROL AREAS + :param latest: + :param yesterday: + :param start_at: + :param end_at: + :param renewable_forecast: The default forecast only returns the aggregate generation forecast. A separate + report covers solar, wind, and offshore wind as named forecast categories. + :param forecast: + :param kwargs: + :return: + """ + assert control_area in self.CONTROL_AREAS, "Control area must be defined" self.handle_options(data='gen', start_at=start_at, end_at=end_at, yesterday=yesterday, - latest=latest, control_area=control_area, forecast=forecast, **kwargs) + latest=latest, control_area=control_area, forecast=forecast, + renewable_forecast=renewable_forecast, **kwargs) response = self.fetch_entsoe() return self.parse_response(response) @@ -237,8 +247,8 @@ def handle_options(self, **kwargs): end_at=datetime.now(pytz.utc)) # workaround for base.handle_options setting forecast to false if end_at too far in past - if 'forecast' in kwargs and kwargs['forecast']: - self.options['forecast'] = True + self.options['forecast'] = kwargs.get('forecast', False) + self.options['renewable_forecast'] = kwargs.get('renewable_forecast', False) def fetch_entsoe(self): payload = { @@ -257,7 +267,10 @@ def fetch_entsoe(self): elif self.options['data'] == 'gen': domainType = 'in_Domain' if self.options['forecast']: - documentType = 'A71' + if self.options['renewable_forecast']: + documentType = 'A69' + else: + documentType = 'A71' else: documentType = 'A75' @@ -285,6 +298,11 @@ def parse_response(self, response): """ data = [] xmldoc = objectify.fromstring(response) + + if not hasattr(xmldoc, 'TimeSeries'): + print("No matching time series returned") + return data + for ts in xmldoc.TimeSeries: for period in ts.Period: initialOffset = self.utcify(period.timeInterval.start.text) @@ -310,7 +328,7 @@ def parse_response(self, response): if hasattr(ts, 'MktPSRType'): datapoint['fuel_name'] = self.fuels[ts.MktPSRType.psrType.text] else: - datapoint['fuel_name'] = 'other' + datapoint['fuel_name'] = 'undefined' elif self.options['data'] == 'load': if self.options['forecast']: @@ -319,7 +337,6 @@ def parse_response(self, response): data.append(datapoint) return data - def parse_resolution(self, resolution): """ Resolutions are given as ISO8601 durations. @@ -349,5 +366,3 @@ def get_tso_id(self): msg = 'Control area code not found for %s. Options are %s' % (self.options['control_area'], sorted(self.CONTROL_AREAS.keys())) raise ValueError(msg) - -