diff --git a/.github/workflows/tests-coverage.yml b/.github/workflows/tests-coverage.yml index 6bcc81f6d..58ce5870e 100644 --- a/.github/workflows/tests-coverage.yml +++ b/.github/workflows/tests-coverage.yml @@ -66,12 +66,16 @@ jobs: run: | python -m pip install pytest pytest-notebook python -m pytest --runslow --runonlinux --disable-warnings --color=yes -v + env: + TOEP_TOKEN_KH: ${{ secrets.TOEP_TOKEN_KH }} - name: Run tests Windows if: runner.os == 'Windows' run: | python -m pip install pytest pytest-notebook python -m pytest --runslow --disable-warnings --color=yes -v + env: + TOEP_TOKEN_KH: ${{ secrets.TOEP_TOKEN_KH }} - name: Run tests, coverage and send to coveralls if: runner.os == 'Linux' && matrix.python-version == 3.9 && matrix.name-suffix == 'coverage' @@ -80,5 +84,6 @@ jobs: coverage run --source=edisgo -m pytest --runslow --runonlinux --disable-warnings --color=yes -v coveralls env: + TOEP_TOKEN_KH: ${{ secrets.TOEP_TOKEN_KH }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} COVERALLS_SERVICE_NAME: github diff --git a/.gitignore b/.gitignore index e13855310..b02956662 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,5 @@ eDisGo.egg-info/ /edisgo/opf/opf_solutions/*.json /edisgo/opf/eDisGo_OPF.jl/.vscode .vscode/settings.json + +*TOEP_TOKEN.* diff --git a/doc/whatsnew/v0-3-0.rst b/doc/whatsnew/v0-3-0.rst index 1335a26c7..95ab687b0 100644 --- a/doc/whatsnew/v0-3-0.rst +++ b/doc/whatsnew/v0-3-0.rst @@ -28,4 +28,6 @@ Changes * Move function to assign feeder to Topology class and add methods to the Grid class to get information on the feeders `#360 `_ * Added a storage operation strategy where the storage is charged when PV feed-in is higher than electricity demand of the household and discharged when electricity demand exceeds PV generation `#386 `_ * Added an estimation of the voltage deviation over a cable when selecting a suitable cable to connect a new component `#411 `_ -* Added clipping of heat pump electrical power at its maximum value #428 +* Added clipping of heat pump electrical power at its maximum value `#428 `_ +* Made OEP database call optional in get_database_alias_dictionaries, allowing setup without OEP when using an alternative eGon-data database. `#451 `_ +* Fixed database import issues by addressing table naming assumptions and added support for external SSH tunneling in eGon-data configurations. `#451 `_ diff --git a/edisgo/edisgo.py b/edisgo/edisgo.py index 4fda72aa2..77e6ba0a7 100755 --- a/edisgo/edisgo.py +++ b/edisgo/edisgo.py @@ -71,6 +71,11 @@ class EDisGo: ---------- ding0_grid : :obj:`str` Path to directory containing csv files of network to be loaded. + engine : :sqlalchemy:`sqlalchemy.Engine` or None + Database engine for connecting to the `OpenEnergy DataBase OEDB + `_ or other eGon-data + databases. Defaults to the OEDB engine. Can be set to None if no scenario is to + be loaded. generator_scenario : None or :obj:`str`, optional If None, the generator park of the imported grid is kept as is. Otherwise defines which scenario of future generator park to use @@ -158,8 +163,10 @@ class EDisGo: """ def __init__(self, **kwargs): + # Set database engine for future scenarios + self.engine: Engine | None = kwargs.pop("engine", toep_engine()) # load configuration - self._config = Config(**kwargs) + self._config = Config(engine=self.engine, **kwargs) # instantiate topology object and load grid data self.topology = Topology(config=self.config) @@ -418,12 +425,9 @@ def set_time_series_active_power_predefined( Technology- and weather cell-specific hourly feed-in time series are obtained from the `OpenEnergy DataBase - `_. See - :func:`edisgo.io.timeseries_import.feedin_oedb` for more information. - - This option requires that the parameter `engine` is provided in case - new ding0 grids with geo-referenced LV grids are used. For further - settings, the parameter `timeindex` can also be provided. + `_ or other eGon-data + databases. See :func:`edisgo.io.timeseries_import.feedin_oedb` for more + information. * :pandas:`pandas.DataFrame` @@ -536,9 +540,6 @@ def set_time_series_active_power_predefined( Other Parameters ------------------ - engine : :sqlalchemy:`sqlalchemy.Engine` - Database engine. This parameter is only required in case - `conventional_loads_ts` or `fluctuating_generators_ts` is 'oedb'. scenario : str Scenario for which to retrieve demand data. Possible options are 'eGon2035' and 'eGon100RE'. This parameter is only required in case @@ -569,7 +570,7 @@ def set_time_series_active_power_predefined( self, fluctuating_generators_ts, fluctuating_generators_names, - engine=kwargs.get("engine", toep_engine()), + engine=self.engine, timeindex=kwargs.get("timeindex", None), ) if dispatchable_generators_ts is not None: @@ -584,7 +585,7 @@ def set_time_series_active_power_predefined( loads_ts_df = timeseries_import.electricity_demand_oedb( edisgo_obj=self, scenario=kwargs.get("scenario"), - engine=kwargs.get("engine", toep_engine()), + engine=self.engine, timeindex=kwargs.get("timeindex", None), load_names=conventional_loads_names, ) @@ -971,9 +972,7 @@ def import_generators(self, generator_scenario=None, **kwargs): Other Parameters ---------------- kwargs : - In case you are using new ding0 grids, where the LV is geo-referenced, a - database engine needs to be provided through keyword argument `engine`. - In case you are using old ding0 grids, where the LV is not geo-referenced, + If you are using old ding0 grids, where the LV is not geo-referenced, you can check :func:`edisgo.io.generators_import.oedb_legacy` for possible keyword arguments. @@ -985,7 +984,7 @@ def import_generators(self, generator_scenario=None, **kwargs): else: generators_import.oedb( edisgo_object=self, - engine=kwargs.get("engine", toep_engine()), + engine=self.engine, scenario=generator_scenario, ) @@ -1350,7 +1349,7 @@ def reinforce( """ if copy_grid: - edisgo_obj = copy.deepcopy(self) + edisgo_obj = self.copy() else: edisgo_obj = self @@ -1846,9 +1845,11 @@ def _aggregate_time_series(attribute, groups, naming): [ pd.DataFrame( { - naming.format("_".join(k)) - if isinstance(k, tuple) - else naming.format(k): getattr(self.timeseries, attribute) + ( + naming.format("_".join(k)) + if isinstance(k, tuple) + else naming.format(k) + ): getattr(self.timeseries, attribute) .loc[:, v] .sum(axis=1) } @@ -1917,9 +1918,8 @@ def _aggregate_time_series(attribute, groups, naming): def import_electromobility( self, - data_source: str, + data_source: str = "oedb", scenario: str = None, - engine: Engine = None, charging_processes_dir: PurePath | str = None, potential_charging_points_dir: PurePath | str = None, import_electromobility_data_kwds=None, @@ -1961,10 +1961,8 @@ def import_electromobility( * "oedb" Electromobility data is obtained from the `OpenEnergy DataBase - `_. - - This option requires that the parameters `scenario` and `engine` are - provided. + `_ or other eGon-data + databases depending on the provided Engine. * "directory" @@ -1974,9 +1972,6 @@ def import_electromobility( scenario : str Scenario for which to retrieve electromobility data in case `data_source` is set to "oedb". Possible options are "eGon2035" and "eGon100RE". - engine : :sqlalchemy:`sqlalchemy.Engine` - Database engine. Needs to be provided in case `data_source` is set to - "oedb". charging_processes_dir : str or pathlib.PurePath Directory holding data on charging processes (standing times, charging demand, etc. per vehicle), including metadata, from SimBEV. @@ -2038,7 +2033,7 @@ def import_electromobility( import_electromobility_from_oedb( self, scenario=scenario, - engine=engine, + engine=self.engine, **import_electromobility_data_kwds, ) elif data_source == "directory": @@ -2131,10 +2126,11 @@ def apply_charging_strategy(self, strategy="dumb", **kwargs): """ charging_strategy(self, strategy=strategy, **kwargs) - def import_heat_pumps(self, scenario, engine, timeindex=None, import_types=None): + def import_heat_pumps(self, scenario, timeindex=None, import_types=None): """ - Gets heat pump data for specified scenario from oedb and integrates the heat - pumps into the grid. + Gets heat pump data for specified scenario from the OEDB or other eGon-data + databases depending on the provided Engine and integrates the heat pumps into + the grid. Besides heat pump capacity the heat pump's COP and heat demand to be served are as well retrieved. @@ -2189,8 +2185,6 @@ def import_heat_pumps(self, scenario, engine, timeindex=None, import_types=None) scenario : str Scenario for which to retrieve heat pump data. Possible options are 'eGon2035' and 'eGon100RE'. - engine : :sqlalchemy:`sqlalchemy.Engine` - Database engine. timeindex : :pandas:`pandas.DatetimeIndex` or None Specifies time steps for which to set COP and heat demand data. Leap years can currently not be handled. In case the given @@ -2231,7 +2225,7 @@ def import_heat_pumps(self, scenario, engine, timeindex=None, import_types=None) year = tools.get_year_based_on_scenario(scenario) return self.import_heat_pumps( scenario, - engine, + self.engine, timeindex=pd.date_range(f"1/1/{year}", periods=8760, freq="H"), import_types=import_types, ) @@ -2239,7 +2233,7 @@ def import_heat_pumps(self, scenario, engine, timeindex=None, import_types=None) integrated_heat_pumps = import_heat_pumps_oedb( edisgo_object=self, scenario=scenario, - engine=engine, + engine=self.engine, import_types=import_types, ) if len(integrated_heat_pumps) > 0: @@ -2247,7 +2241,7 @@ def import_heat_pumps(self, scenario, engine, timeindex=None, import_types=None) self, "oedb", heat_pump_names=integrated_heat_pumps, - engine=engine, + engine=self.engine, scenario=scenario, timeindex=timeindex, ) @@ -2255,7 +2249,7 @@ def import_heat_pumps(self, scenario, engine, timeindex=None, import_types=None) self, "oedb", heat_pump_names=integrated_heat_pumps, - engine=engine, + engine=self.engine, timeindex=timeindex, ) @@ -2303,7 +2297,7 @@ def apply_heat_pump_operating_strategy( """ hp_operating_strategy(self, strategy=strategy, heat_pump_names=heat_pump_names) - def import_dsm(self, scenario: str, engine: Engine, timeindex=None): + def import_dsm(self, scenario: str, timeindex=None): """ Gets industrial and CTS DSM profiles from the `OpenEnergy DataBase `_. @@ -2322,8 +2316,6 @@ def import_dsm(self, scenario: str, engine: Engine, timeindex=None): scenario : str Scenario for which to retrieve DSM data. Possible options are 'eGon2035' and 'eGon100RE'. - engine : :sqlalchemy:`sqlalchemy.Engine` - Database engine. timeindex : :pandas:`pandas.DatetimeIndex` or None Specifies time steps for which to get data. Leap years can currently not be handled. In case the given timeindex contains a leap year, the data will be @@ -2336,7 +2328,7 @@ def import_dsm(self, scenario: str, engine: Engine, timeindex=None): """ dsm_profiles = dsm_import.oedb( - edisgo_obj=self, scenario=scenario, engine=engine, timeindex=timeindex + edisgo_obj=self, scenario=scenario, engine=self.engine, timeindex=timeindex ) self.dsm.p_min = dsm_profiles["p_min"] self.dsm.p_max = dsm_profiles["p_max"] @@ -2346,7 +2338,6 @@ def import_dsm(self, scenario: str, engine: Engine, timeindex=None): def import_home_batteries( self, scenario: str, - engine: Engine, ): """ Gets home battery data for specified scenario and integrates the batteries into @@ -2357,7 +2348,8 @@ def import_home_batteries( between two scenarios: 'eGon2035' and 'eGon100RE'. The data is retrieved from the - `open energy platform `_. + `open energy platform `_ or other eGon-data + databases depending on the given Engine. The batteries are integrated into the grid (added to :attr:`~.network.topology.Topology.storage_units_df`) based on their building @@ -2374,14 +2366,12 @@ def import_home_batteries( scenario : str Scenario for which to retrieve home battery data. Possible options are 'eGon2035' and 'eGon100RE'. - engine : :sqlalchemy:`sqlalchemy.Engine` - Database engine. """ home_batteries_oedb( edisgo_obj=self, scenario=scenario, - engine=engine, + engine=self.engine, ) def plot_mv_grid_topology(self, technologies=False, **kwargs): @@ -3131,7 +3121,7 @@ def spatial_complexity_reduction( """ if copy_edisgo is True: - edisgo_obj = copy.deepcopy(self) + edisgo_obj = self.copy() else: edisgo_obj = self busmap_df, linemap_df = spatial_complexity_reduction( @@ -3345,6 +3335,44 @@ def resample_timeseries( self.heat_pump.resample_timeseries(method=method, freq=freq) self.overlying_grid.resample(method=method, freq=freq) + def copy(self, deep=True): + """ + Returns a copy of the object, with an option for a deep copy. + + The SQLAlchemy engine is excluded from the copying process and restored + afterward. + + Parameters + ---------- + deep : bool + If True, performs a deep copy; otherwise, performs a shallow copy. + + Returns + --------- + :class:`~.EDisGo` + Copied EDisGo object. + + """ + tmp_engine = ( + getattr(self, "engine", None) + if isinstance(getattr(self, "engine", None), Engine) + else None + ) + + if tmp_engine: + logging.info("Temporarily removing the SQLAlchemy engine before copying.") + self.engine = self.config._engine = None + + cpy = copy.deepcopy(self) if deep else copy.copy(self) + + if tmp_engine: + logging.info("Restoring the SQLAlchemy engine after copying.") + self.engine = self.config._engine = cpy.engine = cpy.config._engine = ( + tmp_engine + ) + + return cpy + def import_edisgo_from_pickle(filename, path=""): """ diff --git a/edisgo/io/db.py b/edisgo/io/db.py index e920dee47..5b46f2d9f 100644 --- a/edisgo/io/db.py +++ b/edisgo/io/db.py @@ -1,6 +1,9 @@ from __future__ import annotations +import importlib.util import logging +import os +import re from contextlib import contextmanager from pathlib import Path @@ -149,17 +152,24 @@ def ssh_tunnel(cred: dict) -> str: return str(server.local_bind_port) -def engine(path: Path | str = None, ssh: bool = False) -> Engine: +def engine( + path: Path | str = None, ssh: bool = False, token: Path | str = None +) -> Engine: """ Engine for local or remote database. Parameters ---------- - path : str + path : str or pathlib.Path, optional (default=None) Path to configuration YAML file of egon-data database. - ssh : bool + ssh : bool (default=False) If True try to establish ssh tunnel from given information within the configuration YAML. If False try to connect to local database. + token : str or pathlib.Path, optional (default=None) + Token for database connection or path to text file containing token. + If empty the default token file in the config folder TOEP_TOKEN.txt + will be used. If the default token file is not found, no token + will be used and the connection will be established without token. Returns ------- @@ -168,15 +178,57 @@ def engine(path: Path | str = None, ssh: bool = False) -> Engine: """ - if not ssh: + if path is None: + # Github Actions KHs token + if "TOEP_TOKEN_KH" in os.environ: + token = os.environ["TOEP_TOKEN_KH"] + + read = True + else: + read = False + + if token is None: + spec = importlib.util.find_spec("edisgo") + token = Path(spec.origin).resolve().parent / "config" / "TOEP_TOKEN.txt" + + if token.is_file(): + logger.info(f"Getting OEP token from file {token}.") + + with open(token) as file: + token = file.read().strip() + + read = True + database_url = "toep.iks.cs.ovgu.de" + + msg = "" + + if not read: + msg = f"Token file {token} not found" + token = "" + # Check if the token format is valid + elif not re.match(r"^[a-f0-9]{40}$", token): + msg = ( + f"Invalid token format for token {token}. A 40 character " + f"hexadecimal string was expected" + ) + token = "" + + if msg: + logger.warning( + f"{msg}. Connecting to {database_url} without a user token. This may " + f"cause connection errors due to connection limitations. Consider " + f"setting up an OEP account and providing your user token." + ) + return create_engine( - "postgresql+oedialect://:@" f"{database_url}", + f"postgresql+oedialect://:{token}@{database_url}", echo=False, ) cred = credentials(path=path) - local_port = ssh_tunnel(cred) + + local_port = ssh_tunnel(cred) if ssh else int(cred["--database-port"]) return create_engine( f"postgresql+psycopg2://{cred['POSTGRES_USER']}:" diff --git a/edisgo/io/heat_pump_import.py b/edisgo/io/heat_pump_import.py index 4bf862485..da6d26cc7 100644 --- a/edisgo/io/heat_pump_import.py +++ b/edisgo/io/heat_pump_import.py @@ -327,7 +327,7 @@ def _get_individual_heat_pump_capacity(): "boundaries", ) egon_etrago_bus, egon_etrago_link = config.import_tables_from_oep( - engine, ["egon_etrago_bus", "egon_etrago_link"], "supply" + engine, ["egon_etrago_bus", "egon_etrago_link"], "grid" ) building_ids = edisgo_object.topology.loads_df.building_id.unique() diff --git a/edisgo/tools/config.py b/edisgo/tools/config.py index 0341299f7..6cf74c77c 100644 --- a/edisgo/tools/config.py +++ b/edisgo/tools/config.py @@ -130,6 +130,8 @@ class Config: """ def __init__(self, **kwargs): + self._engine = kwargs.get("engine", None) + if not kwargs.get("from_json", False): self._data = self.from_cfg(kwargs.get("config_path", "default")) else: @@ -164,13 +166,17 @@ def _set_db_mappings(self) -> None: """ Sets the database table and schema mappings by retrieving alias dictionaries. """ - name_mapping, schema_mapping = self.get_database_alias_dictionaries() + if self._engine is not None and "toep.iks.cs.ovgu.de" in self._engine.url.host: + name_mapping, schema_mapping = self.get_database_alias_dictionaries() + else: + name_mapping = schema_mapping = {} + self.db_table_mapping = name_mapping self.db_schema_mapping = schema_mapping def get_database_alias_dictionaries(self) -> tuple[dict[str, str], dict[str, str]]: """ - Retrieves the database alias dictionaries for table and schema mappings. + Retrieves the OEP database alias dictionaries for table and schema mappings. Returns ------- @@ -181,20 +187,16 @@ def get_database_alias_dictionaries(self) -> tuple[dict[str, str], dict[str, str - schema_mapping: A dictionary mapping source schema names to target schema names. """ - OEP_CONNECTION = "postgresql+oedialect://:@{platform}" - platform = "toep.iks.cs.ovgu.de" - conn_str = OEP_CONNECTION.format(platform=platform) - engine = sa.create_engine(conn_str) dictionary_schema_name = ( "model_draft" # Replace with the actual schema name if needed ) dictionary_module_name = f"saio.{dictionary_schema_name}" - register_schema(dictionary_schema_name, engine) + register_schema(dictionary_schema_name, self._engine) dictionary_table_name = "edut_00" dictionary_table = importlib.import_module(dictionary_module_name).__getattr__( dictionary_table_name ) - with session_scope_egon_data(engine) as session: + with session_scope_egon_data(self._engine) as session: query = session.query(dictionary_table) dictionary_entries = query.all() name_mapping = { @@ -228,13 +230,21 @@ def import_tables_from_oep( list of sqlalchemy.Table A list of SQLAlchemy Table objects corresponding to the imported tables. """ - schema = self.db_schema_mapping.get(schema_name) - saio.register_schema(schema, engine) tables = [] - for table in table_names: - table = self.db_table_mapping.get(table) - module_name = f"saio.{schema}" - tables.append(importlib.import_module(module_name).__getattr__(table)) + + if "toep" in engine.url.host: + schema = self.db_schema_mapping.get(schema_name) + saio.register_schema(schema, engine) + for table in table_names: + table = self.db_table_mapping.get(table) + module_name = f"saio.{schema}" + tables.append(importlib.import_module(module_name).__getattr__(table)) + else: + saio.register_schema(schema_name, engine) + for table in table_names: + module_name = f"saio.{schema_name}" + tables.append(importlib.import_module(module_name).__getattr__(table)) + return tables def from_cfg(self, config_path=None): @@ -295,21 +305,28 @@ def from_cfg(self, config_path=None): config_dict["demandlib"]["day_start"] = datetime.datetime.strptime( config_dict["demandlib"]["day_start"], "%H:%M" ) + config_dict["demandlib"]["day_start"] = datetime.time( config_dict["demandlib"]["day_start"].hour, config_dict["demandlib"]["day_start"].minute, ) + config_dict["demandlib"]["day_end"] = datetime.datetime.strptime( config_dict["demandlib"]["day_end"], "%H:%M" ) + config_dict["demandlib"]["day_end"] = datetime.time( config_dict["demandlib"]["day_end"].hour, config_dict["demandlib"]["day_end"].minute, ) - ( - config_dict["db_tables_dict"], - config_dict["db_schema_dict"], - ) = self.get_database_alias_dictionaries() + + if self._engine is not None and "toep.iks.cs.ovgu.de" in self._engine.url.host: + config_dict["db_tables_dict"], config_dict["db_schema_dict"] = ( + self.get_database_alias_dictionaries() + ) + else: + config_dict["db_tables_dict"] = config_dict["db_schema_dict"] = {} + return config_dict def to_json(self, directory, filename=None): diff --git a/edisgo/tools/spatial_complexity_reduction.py b/edisgo/tools/spatial_complexity_reduction.py index 6e64d0133..1d2ebc37a 100644 --- a/edisgo/tools/spatial_complexity_reduction.py +++ b/edisgo/tools/spatial_complexity_reduction.py @@ -79,7 +79,7 @@ def find_buses_of_interest(edisgo_root: EDisGo) -> set: """ logger.debug("Find buses of interest.") - edisgo_obj = copy.deepcopy(edisgo_root) + edisgo_obj = edisgo_root.copy() edisgo_obj.timeseries = timeseries.TimeSeries() edisgo_obj.timeseries.set_worst_case(edisgo_obj, ["feed-in_case", "load_case"]) edisgo_obj.analyze() @@ -501,9 +501,9 @@ def rename_new_buses(series): for index, new_bus in zip( partial_busmap_df.index, partial_busmap_df.new_bus ): - partial_busmap_df.loc[ - index, ["new_x", "new_y"] - ] = kmeans.cluster_centers_[new_bus] + partial_busmap_df.loc[index, ["new_x", "new_y"]] = ( + kmeans.cluster_centers_[new_bus] + ) elif mode == "kmeansdijkstra": # Use dijkstra to select clusters @@ -760,9 +760,9 @@ def transform_coordinates_back(ser): if n_clusters == 0: for index in feeder_buses_df.index.tolist(): partial_busmap_df.loc[index, "new_bus"] = transformer_node - partial_busmap_df.loc[ - index, ["new_x", "new_y"] - ] = transformer_coordinates + partial_busmap_df.loc[index, ["new_x", "new_y"]] = ( + transformer_coordinates + ) else: kmeans = KMeans(n_clusters=n_clusters, n_init=10, random_state=42) kmeans.fit( @@ -776,9 +776,9 @@ def transform_coordinates_back(ser): partial_busmap_df.loc[index, "new_bus"] = make_name( kmeans.labels_[n] + 1 ) - partial_busmap_df.loc[ - index, ["new_x", "new_y"] - ] = kmeans.cluster_centers_[kmeans.labels_[n]] + partial_busmap_df.loc[index, ["new_x", "new_y"]] = ( + kmeans.cluster_centers_[kmeans.labels_[n]] + ) n = n + 1 elif mode == "kmeansdijkstra": dist_to_cluster_center = pd.DataFrame( @@ -815,11 +815,11 @@ def transform_coordinates_back(ser): partial_busmap_df.loc[index, "new_bus"] = make_name( medoid_bus_name[feeder_buses_df.loc[index, "medoid"]] + 1 ) - partial_busmap_df.loc[ - index, ["new_x", "new_y"] - ] = feeder_buses_df.loc[ - feeder_buses_df.loc[index, "medoid"], ["x", "y"] - ].values + partial_busmap_df.loc[index, ["new_x", "new_y"]] = ( + feeder_buses_df.loc[ + feeder_buses_df.loc[index, "medoid"], ["x", "y"] + ].values + ) number_of_feeder = number_of_feeder + 1 if str(grid).split("_")[0] == "MVGrid": @@ -1167,9 +1167,9 @@ def short_coordinates(root_node, end_node, branch_length, node_number): if n_clusters == 0: for index in feeder_buses_df.index.tolist(): partial_busmap_df.loc[index, "new_bus"] = transformer_node - partial_busmap_df.loc[ - index, ["new_x", "new_y"] - ] = transformer_coordinates + partial_busmap_df.loc[index, ["new_x", "new_y"]] = ( + transformer_coordinates + ) else: kmeans = KMeans(n_clusters=n_clusters, n_init=10, random_state=42) kmeans.fit( @@ -1183,9 +1183,9 @@ def short_coordinates(root_node, end_node, branch_length, node_number): partial_busmap_df.loc[index, "new_bus"] = make_name( kmeans.labels_[n] + 1 ) - partial_busmap_df.loc[ - index, ["new_x", "new_y"] - ] = kmeans.cluster_centers_[kmeans.labels_[n]] + partial_busmap_df.loc[index, ["new_x", "new_y"]] = ( + kmeans.cluster_centers_[kmeans.labels_[n]] + ) n = n + 1 elif mode == "kmeansdijkstra": dist_to_cluster_center = pd.DataFrame( @@ -1223,11 +1223,11 @@ def short_coordinates(root_node, end_node, branch_length, node_number): medoid_bus_name[feeder_buses_df.loc[index, "medoid"]] + 1 ) - partial_busmap_df.loc[ - index, ["new_x", "new_y"] - ] = feeder_buses_df.loc[ - feeder_buses_df.loc[index, "medoid"], ["x", "y"] - ].values + partial_busmap_df.loc[index, ["new_x", "new_y"]] = ( + feeder_buses_df.loc[ + feeder_buses_df.loc[index, "medoid"], ["x", "y"] + ].values + ) if mode != "aggregate_to_main_feeder": # Backmap diff --git a/tests/flex_opt/test_reinforce_grid.py b/tests/flex_opt/test_reinforce_grid.py index e073b2326..24a10eb50 100644 --- a/tests/flex_opt/test_reinforce_grid.py +++ b/tests/flex_opt/test_reinforce_grid.py @@ -27,8 +27,7 @@ def test_reinforce_grid(self): modes = [None, "mv", "mvlv", "lv"] results_dict = { - mode: reinforce_grid(edisgo=copy.deepcopy(self.edisgo), mode=mode) - for mode in modes + mode: reinforce_grid(edisgo=self.edisgo.copy(), mode=mode) for mode in modes } for mode, result in results_dict.items(): @@ -57,14 +56,14 @@ def test_reinforce_grid(self): ) # test reduced analysis res_reduced = reinforce_grid( - edisgo=copy.deepcopy(self.edisgo), + edisgo=self.edisgo.copy(), reduced_analysis=True, num_steps_loading=2, ) assert len(res_reduced.i_res) == 2 def test_run_separate_lv_grids(self): - edisgo = copy.deepcopy(self.edisgo) + edisgo = self.edisgo.copy() edisgo.timeseries.scale_timeseries(p_scaling_factor=5, q_scaling_factor=5) diff --git a/tests/flex_opt/test_reinforce_measures.py b/tests/flex_opt/test_reinforce_measures.py index 5ea720064..128b8dec9 100644 --- a/tests/flex_opt/test_reinforce_measures.py +++ b/tests/flex_opt/test_reinforce_measures.py @@ -15,7 +15,7 @@ def setup_class(cls): cls.edisgo.set_time_series_worst_case_analysis() cls.edisgo.analyze() - cls.edisgo_root = copy.deepcopy(cls.edisgo) + cls.edisgo_root = cls.edisgo.copy() cls.timesteps = pd.date_range("1/1/1970", periods=2, freq="H") def test_reinforce_mv_lv_station_overloading(self): @@ -24,7 +24,7 @@ def test_reinforce_mv_lv_station_overloading(self): # create problems such that in LVGrid_1 existing transformer is # exchanged with standard transformer and in LVGrid_4 a third # transformer is added - self.edisgo = copy.deepcopy(self.edisgo_root) + self.edisgo = self.edisgo_root.copy() lv_grid_1 = self.edisgo.topology.get_lv_grid(1) lv_grid_4 = self.edisgo.topology.get_lv_grid(4) @@ -94,7 +94,7 @@ def test_reinforce_hv_mv_station_overloading(self): # implicitly checks function _station_overloading # check adding transformer of same MVA - self.edisgo = copy.deepcopy(self.edisgo_root) + self.edisgo = self.edisgo_root.copy() crit_mv_station = pd.DataFrame( { @@ -158,7 +158,7 @@ def test_reinforce_hv_mv_station_overloading(self): ) def test_reinforce_mv_lv_station_voltage_issues(self): - self.edisgo = copy.deepcopy(self.edisgo_root) + self.edisgo = self.edisgo_root.copy() crit_stations = pd.DataFrame( { @@ -211,7 +211,7 @@ def test_reinforce_lines_voltage_issues(self): # * check problem in same feeder => Bus_BranchTee_MVGrid_1_10 (node # has higher voltage issue than Bus_BranchTee_MVGrid_1_11, but # Bus_BranchTee_MVGrid_1_10 is farther away from station) - self.edisgo = copy.deepcopy(self.edisgo_root) + self.edisgo = self.edisgo_root.copy() crit_nodes = pd.DataFrame( { @@ -378,7 +378,7 @@ def test_reinforce_lines_overloading(self): # and Line_50000002 # * check for replacement by parallel standard lines (MV and LV) => # problems at Line_10003 and Line_60000001 - self.edisgo = copy.deepcopy(self.edisgo_root) + self.edisgo = self.edisgo_root.copy() # create crit_lines dataframe crit_lines = pd.DataFrame( @@ -455,7 +455,7 @@ def test_reinforce_lines_overloading(self): assert line.num_parallel == 1 def test_separate_lv_grid(self): - self.edisgo = copy.deepcopy(self.edisgo_root) + self.edisgo = self.edisgo_root.copy() crit_lines_lv = check_tech_constraints.lv_line_max_relative_overload( self.edisgo diff --git a/tests/network/test_timeseries.py b/tests/network/test_timeseries.py index 76807a7f6..1e37c845b 100644 --- a/tests/network/test_timeseries.py +++ b/tests/network/test_timeseries.py @@ -1,4 +1,3 @@ -import copy import logging import os import shutil @@ -739,9 +738,9 @@ def test_worst_case_generators(self): def test_worst_case_conventional_load(self): # connect one load to MV - self.edisgo.topology._loads_df.at[ - "Load_agricultural_LVGrid_1_1", "bus" - ] = "Bus_BranchTee_MVGrid_1_2" + self.edisgo.topology._loads_df.at["Load_agricultural_LVGrid_1_1", "bus"] = ( + "Bus_BranchTee_MVGrid_1_2" + ) # ######### check both feed-in and load case df = assign_voltage_level_to_component( @@ -1491,9 +1490,9 @@ def test_predefined_dispatchable_generators_by_technology(self): # ############# all generators (default), with "gas" and "other" # overwrite type of generator GeneratorFluctuating_2 - self.edisgo.topology._generators_df.at[ - "GeneratorFluctuating_2", "type" - ] = "coal" + self.edisgo.topology._generators_df.at["GeneratorFluctuating_2", "type"] = ( + "coal" + ) gens_p = pd.DataFrame( data={ "other": [5, 6], @@ -1808,9 +1807,9 @@ def test_predefined_conventional_loads_by_sector(self, caplog): == "The annual consumption of some loads is missing. Please provide" ) # Restore the original 'annual_consumption' values - self.edisgo.topology.loads_df[ - "annual_consumption" - ] = original_annual_consumption + self.edisgo.topology.loads_df["annual_consumption"] = ( + original_annual_consumption + ) def test_predefined_charging_points_by_use_case(self, caplog): index = pd.date_range("1/1/2018", periods=3, freq="H") @@ -2477,7 +2476,7 @@ def test_resample(self): def test_scale_timeseries(self): self.edisgo.set_time_series_worst_case_analysis() - edisgo_scaled = copy.deepcopy(self.edisgo) + edisgo_scaled = self.edisgo.copy() edisgo_scaled.timeseries.scale_timeseries( p_scaling_factor=0.5, q_scaling_factor=0.4 ) diff --git a/tests/test_edisgo.py b/tests/test_edisgo.py index a4d20407a..4ebddc331 100755 --- a/tests/test_edisgo.py +++ b/tests/test_edisgo.py @@ -1,4 +1,3 @@ -import copy import logging import os import shutil @@ -39,7 +38,7 @@ def test_config_setter(self): save_dir = os.path.join(os.getcwd(), "config_dir") # test default - config_orig = copy.deepcopy(self.edisgo.config) + config_orig = self.edisgo.copy().config self.edisgo.config = {} assert config_orig._data == self.edisgo.config._data @@ -51,7 +50,7 @@ def test_config_setter(self): # test json and config_path=None # save changed config to json self.edisgo.config["geo"]["srid"] = 2 - config_json = copy.deepcopy(self.edisgo.config) + config_json = self.edisgo.copy().config self.edisgo.save( save_dir, save_topology=False, @@ -533,7 +532,7 @@ def test_enhanced_reinforce_grid(self): p_scaling_factor=50, q_scaling_factor=50 ) - edisgo_obj = copy.deepcopy(self.edisgo) + edisgo_obj = self.edisgo.copy() edisgo_obj = enhanced_reinforce_grid( edisgo_obj, activate_cost_results_disturbing_mode=True, @@ -546,7 +545,7 @@ def test_enhanced_reinforce_grid(self): assert len(results.equipment_changes) == 892 assert results.v_res.shape == (4, 148) - edisgo_obj = copy.deepcopy(self.edisgo) + edisgo_obj = self.edisgo.copy() edisgo_obj = enhanced_reinforce_grid( edisgo_obj, reduced_analysis=True, @@ -934,9 +933,9 @@ def test_aggregate_components(self): # ##### test without any aggregation - self.edisgo.topology._loads_df.at[ - "Load_residential_LVGrid_1_4", "bus" - ] = "Bus_BranchTee_LVGrid_1_10" + self.edisgo.topology._loads_df.at["Load_residential_LVGrid_1_4", "bus"] = ( + "Bus_BranchTee_LVGrid_1_10" + ) # save original values number_gens_before = len(self.edisgo.topology.generators_df) @@ -1054,9 +1053,9 @@ def test_aggregate_components(self): ) # manipulate grid so that more than one load of the same sector is # connected at the same bus - self.edisgo.topology._loads_df.at[ - "Load_residential_LVGrid_1_4", "bus" - ] = "Bus_BranchTee_LVGrid_1_10" + self.edisgo.topology._loads_df.at["Load_residential_LVGrid_1_4", "bus"] = ( + "Bus_BranchTee_LVGrid_1_10" + ) # save original values (only loads, as generators did not change) loads_p_set_before = self.edisgo.topology.loads_df.p_set.sum() @@ -1136,9 +1135,9 @@ def test_aggregate_components(self): # manipulate grid so that two generators of different types are # connected at the same bus - self.edisgo.topology._generators_df.at[ - "GeneratorFluctuating_13", "type" - ] = "misc" + self.edisgo.topology._generators_df.at["GeneratorFluctuating_13", "type"] = ( + "misc" + ) # save original values (values of loads were changed in previous aggregation) loads_p_set_before = self.edisgo.topology.loads_df.p_set.sum() @@ -1672,7 +1671,7 @@ def test_spatial_complexity_reduction(self): assert len(edisgo_obj.topology.lines_df) == 23 # test without copying edisgo object - edisgo_orig = copy.deepcopy(self.edisgo) + edisgo_orig = self.edisgo.copy() ( _, busmap_df, diff --git a/tests/tools/test_plots.py b/tests/tools/test_plots.py index 25f8f8e8c..5feb5a96f 100644 --- a/tests/tools/test_plots.py +++ b/tests/tools/test_plots.py @@ -1,5 +1,3 @@ -import copy - import pytest from edisgo import EDisGo @@ -11,13 +9,13 @@ class TestPlots: def setup_class(cls): cls.edisgo_root = EDisGo(ding0_grid=pytest.ding0_test_network_path) cls.edisgo_root.set_time_series_worst_case_analysis() - cls.edisgo_analyzed = copy.deepcopy(cls.edisgo_root) - cls.edisgo_reinforced = copy.deepcopy(cls.edisgo_root) + cls.edisgo_analyzed = cls.edisgo_root.copy() + cls.edisgo_reinforced = cls.edisgo_root.copy() cls.edisgo_analyzed.analyze() cls.edisgo_reinforced.reinforce() - cls.edisgo_reinforced.results.equipment_changes.loc[ - "Line_10006", "change" - ] = "added" + cls.edisgo_reinforced.results.equipment_changes.loc["Line_10006", "change"] = ( + "added" + ) @pytest.mark.parametrize( "line_color," diff --git a/tests/tools/test_spatial_complexity_reduction.py b/tests/tools/test_spatial_complexity_reduction.py index 83ae18dcd..e10795829 100644 --- a/tests/tools/test_spatial_complexity_reduction.py +++ b/tests/tools/test_spatial_complexity_reduction.py @@ -1,5 +1,3 @@ -import copy - from contextlib import nullcontext as does_not_raise import numpy as np @@ -245,9 +243,9 @@ def test_apply_busmap( busmap_df = self.setup_busmap_df(test_edisgo_obj) # Add second line to test line reduction - test_edisgo_obj.topology.lines_df.loc[ - "Line_10003_2" - ] = test_edisgo_obj.topology.lines_df.loc["Line_10003"] + test_edisgo_obj.topology.lines_df.loc["Line_10003_2"] = ( + test_edisgo_obj.topology.lines_df.loc["Line_10003"] + ) assert test_edisgo_obj.topology.buses_df.shape[0] == 142 assert test_edisgo_obj.topology.lines_df.shape[0] == 132 @@ -315,7 +313,7 @@ def test_spatial_complexity_reduction(self, test_edisgo_obj): test_edisgo_obj.reinforce() def test_compare_voltage(self, test_edisgo_obj): - edisgo_reduced = copy.deepcopy(test_edisgo_obj) + edisgo_reduced = test_edisgo_obj.copy() ( busmap_df, linemap_df, @@ -334,7 +332,7 @@ def test_compare_voltage(self, test_edisgo_obj): assert np.isclose(rms, 0.00766, atol=1e-5) def test_compare_apparent_power(self, test_edisgo_obj): - edisgo_reduced = copy.deepcopy(test_edisgo_obj) + edisgo_reduced = test_edisgo_obj.copy() ( busmap_df, @@ -354,7 +352,7 @@ def test_compare_apparent_power(self, test_edisgo_obj): assert np.isclose(rms, 2.873394, atol=1e-5) def test_remove_short_end_lines(self, test_edisgo_obj): - edisgo_root = copy.deepcopy(test_edisgo_obj) + edisgo_root = test_edisgo_obj.copy() # change line length of line to switch to under 1 meter to check that it # is not deleted @@ -382,7 +380,7 @@ def test_remove_short_end_lines(self, test_edisgo_obj): ) # def test_remove_lines_under_one_meter(self, test_edisgo_obj, caplog): - # edisgo_root = copy.deepcopy(test_edisgo_obj) + # edisgo_root = test_edisgo_obj.copy() # edisgo_root.topology.lines_df.at["Line_50000002", "length"] = 0.0006 # edisgo_root.topology.lines_df.at["Line_90000009", "length"] = 0.0007 # edisgo_root.topology.lines_df.at["Line_90000013", "length"] = 0.0008 diff --git a/tests/tools/test_tools.py b/tests/tools/test_tools.py index f6d08f729..48e20c03b 100644 --- a/tests/tools/test_tools.py +++ b/tests/tools/test_tools.py @@ -1,5 +1,3 @@ -import copy - import numpy as np import pandas as pd import pytest @@ -460,7 +458,7 @@ def test_add_line_susceptance(self): assert self.edisgo.topology.lines_df.loc["Line_50000002", "b"] == 0 # test mode no_b - edisgo_root = copy.deepcopy(self.edisgo) + edisgo_root = self.edisgo.copy() edisgo_root.topology.lines_df.loc["Line_10006", "b"] = 1 edisgo_root.topology.lines_df.loc["Line_50000002", "b"] = 1 edisgo_root = tools.add_line_susceptance(edisgo_root, mode="no_b") @@ -468,7 +466,7 @@ def test_add_line_susceptance(self): assert edisgo_root.topology.lines_df.loc["Line_50000002", "b"] == 0 # test mode mv_b - edisgo_root = copy.deepcopy(self.edisgo) + edisgo_root = self.edisgo.copy() edisgo_root.topology.lines_df.loc["Line_10006", "b"] = 1 edisgo_root.topology.lines_df.loc["Line_50000002", "b"] = 1 edisgo_root = tools.add_line_susceptance(edisgo_root, mode="mv_b") @@ -478,7 +476,7 @@ def test_add_line_susceptance(self): assert edisgo_root.topology.lines_df.loc["Line_50000002", "b"] == 0 # test mode all_b - edisgo_root = copy.deepcopy(self.edisgo) + edisgo_root = self.edisgo.copy() edisgo_root = tools.add_line_susceptance(edisgo_root, mode="all_b") assert edisgo_root.topology.lines_df.loc[ "Line_10006", "b"