From 25445e039565836a6911dcd84a856df4df9ba94f Mon Sep 17 00:00:00 2001 From: SabineHaas Date: Fri, 6 Dec 2024 12:50:21 +0100 Subject: [PATCH 1/6] Add units --- data_adapter/unit_conversion.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/data_adapter/unit_conversion.py b/data_adapter/unit_conversion.py index f2c24ee..adacb51 100644 --- a/data_adapter/unit_conversion.py +++ b/data_adapter/unit_conversion.py @@ -68,6 +68,7 @@ def define_energy_model_units(): scaled_unit("PJ", "J", 1e15) scaled_unit("kWh", "PJ", 3.6e-9) + scaled_unit("MWh", "PJ", 3.6e-6) NamedComposedUnit("Wh", unit("W") * unit("h")) NamedComposedUnit("kWh", unit("kW") * unit("h")) @@ -100,6 +101,7 @@ def define_energy_model_units(): NamedComposedUnit("MWh/MW", unit("MWh") / unit("MW")) NamedComposedUnit("MWh/t", unit("MWh") / unit("t")) NamedComposedUnit("MWh/kt", unit("MWh") / unit("kt")) + NamedComposedUnit("MWh/Mt", unit("MWh") / unit("Mt")) NamedComposedUnit("MWh/k_units", unit("MWh") / unit("k_units")) NamedComposedUnit("MWh/M_units", unit("MWh") / unit("M_units")) NamedComposedUnit("MWh/G_units", unit("MWh") / unit("G_units")) @@ -205,6 +207,7 @@ def define_energy_model_units(): NamedComposedUnit("Kt/PJ", unit("kt") / unit("PJ")) NamedComposedUnit("kt/PJ", unit("kt") / unit("PJ")) NamedComposedUnit("Mt/PJ", unit("Mt") / unit("PJ")) + NamedComposedUnit("Mt/MWh", unit("Mt") / unit("MWh")) NamedComposedUnit("PJ/Million units", unit("PJ") / unit("M_units")) NamedComposedUnit("PJ/M_units", unit("PJ") / unit("M_units")) From 6f47a81628694a0a114310fed3feec26e797d118 Mon Sep 17 00:00:00 2001 From: SabineHaas Date: Fri, 6 Dec 2024 14:06:23 +0100 Subject: [PATCH 2/6] Strings cannot be converted Unit is turned back to input unit if conversion does not take place. Foreign keys are added and converted later. --- data_adapter/preprocessing.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/data_adapter/preprocessing.py b/data_adapter/preprocessing.py index 208c00f..7475866 100644 --- a/data_adapter/preprocessing.py +++ b/data_adapter/preprocessing.py @@ -265,7 +265,11 @@ def convert_series(series: list[float], factor: float) -> list[float]: if "array" in field["type"]: df[field["name"]] = df[field["name"]].apply(convert_series, factor=conversion_factor) else: - df[field["name"]] = df[field["name"]] * conversion_factor + try: + df[field["name"]] = df[field["name"]] * conversion_factor + except TypeError: # occurs for foreign keys + # rewrite unit as conversion did not take place + df_units[field["name"]] = field["unit"] return df, df_units def __unpack_bandwidths(self, df: pd.DataFrame) -> pd.DataFrame: From 1557d61a1f8ef83afb02e22e0d1c03cef5bb9bd9 Mon Sep 17 00:00:00 2001 From: SabineHaas Date: Fri, 6 Dec 2024 14:06:48 +0100 Subject: [PATCH 3/6] Add units of foreign keys to units dict --- data_adapter/preprocessing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data_adapter/preprocessing.py b/data_adapter/preprocessing.py index 7475866..4465cad 100644 --- a/data_adapter/preprocessing.py +++ b/data_adapter/preprocessing.py @@ -117,7 +117,8 @@ def get_process(self, process: str) -> Process: f"Foreign key for process '{process}' points to subject '{foreign_key.process}' " "which is not unique.", ) - foreign_df = self.__get_df_from_artifact(artifacts[0], foreign_key.process, foreign_key.parameter)[0] + foreign_df, foreign_units = self.__get_df_from_artifact(artifacts[0], foreign_key.process, foreign_key.parameter) + units[fk_column] = foreign_units[getattr(foreign_key, "parameter")] foreign_df = foreign_df.rename({foreign_key.parameter: fk_column}, axis=1) if artifacts[0].datatype == collection.DataType.Scalar: scalar_dfs.append(foreign_df) From c50b589949968d104f43b338f062426bcd61e0fc Mon Sep 17 00:00:00 2001 From: SabineHaas Date: Fri, 6 Dec 2024 14:06:55 +0100 Subject: [PATCH 4/6] Add units --- data_adapter/unit_conversion.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/data_adapter/unit_conversion.py b/data_adapter/unit_conversion.py index adacb51..df57518 100644 --- a/data_adapter/unit_conversion.py +++ b/data_adapter/unit_conversion.py @@ -46,6 +46,8 @@ def define_energy_model_units(): scaled_unit("ktCO2eq", "tCO2eq", 1e3) scaled_unit("MtCO2eq", "ktCO2eq", 1e3) scaled_unit("GtCO2eq", "MtCO2eq", 1e3) + scaled_unit("Mt CO2-eq.", "ktCO2eq", 1e3) + scaled_unit("GtCO2eq", "Mt CO2-eq.", 1e3) scaled_unit("kEUR", "EUR", 1e3) scaled_unit("MEUR", "kEUR", 1e3) From 8ab9d72ffa5470b27b116acd313dfb9c41847a49 Mon Sep 17 00:00:00 2001 From: SabineHaas Date: Mon, 9 Dec 2024 15:53:26 +0100 Subject: [PATCH 5/6] Add units --- data_adapter/unit_conversion.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/data_adapter/unit_conversion.py b/data_adapter/unit_conversion.py index df57518..fdd9ac6 100644 --- a/data_adapter/unit_conversion.py +++ b/data_adapter/unit_conversion.py @@ -170,6 +170,8 @@ def define_energy_model_units(): NamedComposedUnit("M€/Million units", unit("MEUR") / unit("M_units")) NamedComposedUnit("MEUR/PJ", unit("MEUR") / unit("PJ")) NamedComposedUnit("M€/PJ", unit("MEUR") / unit("PJ")) + NamedComposedUnit("MEUR/Mt", unit("MEUR") / unit("Mt")) + NamedComposedUnit("M€/Mt", unit("MEUR") / unit("Mt")) NamedComposedUnit("kWh/100km", unit("kWh") / unit("100km")) NamedComposedUnit("MWh/100km", unit("MWh") / unit("100km")) From 332a1405f86db04beeba395801d7f7ca68616282 Mon Sep 17 00:00:00 2001 From: SabineHaas Date: Mon, 9 Dec 2024 17:00:00 +0100 Subject: [PATCH 6/6] Move check above --- data_adapter/preprocessing.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/data_adapter/preprocessing.py b/data_adapter/preprocessing.py index 4465cad..67e27a3 100644 --- a/data_adapter/preprocessing.py +++ b/data_adapter/preprocessing.py @@ -253,6 +253,8 @@ def convert_series(series: list[float], factor: float) -> list[float]: continue if field["name"] not in df.columns: continue + if isinstance(df[field["name"]].iloc[-1], str): + continue df_units[field["name"]] = field["unit"] conversion_factor = None for unit in self.units: @@ -266,11 +268,7 @@ def convert_series(series: list[float], factor: float) -> list[float]: if "array" in field["type"]: df[field["name"]] = df[field["name"]].apply(convert_series, factor=conversion_factor) else: - try: - df[field["name"]] = df[field["name"]] * conversion_factor - except TypeError: # occurs for foreign keys - # rewrite unit as conversion did not take place - df_units[field["name"]] = field["unit"] + df[field["name"]] = df[field["name"]] * conversion_factor return df, df_units def __unpack_bandwidths(self, df: pd.DataFrame) -> pd.DataFrame: