From 9d491d1269200b98e096167312edc9a728ab6ea2 Mon Sep 17 00:00:00 2001 From: jucordero Date: Wed, 13 Aug 2025 18:54:15 +0100 Subject: [PATCH 1/4] First commit of pipeline manager code to agrifoodpy --- README.md | 79 +++----- agrifoodpy/pipeline/__init__.py | 6 + agrifoodpy/pipeline/pipeline.py | 189 ++++++++++++++++++ agrifoodpy/pipeline/tests/__init__.py | 0 agrifoodpy/pipeline/tests/test_pipeline.py | 103 ++++++++++ agrifoodpy/pipeline/tests/test_utils.py | 46 +++++ agrifoodpy/pipeline/utils.py | 86 ++++++++ agrifoodpy/utils/add_items.py | 63 ++++++ agrifoodpy/utils/copy_datablock.py | 23 +++ agrifoodpy/utils/extend_years.py | 29 +++ agrifoodpy/utils/load_dataset.py | 65 ++++++ agrifoodpy/utils/print_datablock.py | 21 ++ docs/conf.py | 2 +- .../modules/plot_emissions_animal_scaling.py | 2 +- setup.py | 2 +- 15 files changed, 664 insertions(+), 52 deletions(-) create mode 100644 agrifoodpy/pipeline/__init__.py create mode 100644 agrifoodpy/pipeline/pipeline.py create mode 100644 agrifoodpy/pipeline/tests/__init__.py create mode 100644 agrifoodpy/pipeline/tests/test_pipeline.py create mode 100644 agrifoodpy/pipeline/tests/test_utils.py create mode 100644 agrifoodpy/pipeline/utils.py create mode 100644 agrifoodpy/utils/add_items.py create mode 100644 agrifoodpy/utils/copy_datablock.py create mode 100644 agrifoodpy/utils/extend_years.py create mode 100644 agrifoodpy/utils/load_dataset.py create mode 100644 agrifoodpy/utils/print_datablock.py diff --git a/README.md b/README.md index 151cb31..3bb0522 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,10 @@ including food consumption paterns, environmental impact and emissions data, population and land use. It also provides an interface to run external models by using xarray as the data container. +AgriFoodPy also provides a pipeline manager to build end-to-end simulations and +analysis toolchains. Modules can also be executed in standalone mode, which +does not require a pipeline to be defined. + In addition to this package, we have also pre-packaged some datasets for use with agrifood. These can be found on the agrifoodpy_data repository https://github.com/FixOurFood/agrifoodpy-data @@ -36,69 +40,46 @@ pip install git+https://github.com/FixOurFood/agrifoodpy-data.git@importable ## Usage: -Each of the four basic modules on AgriFoodPy (Food, Land, Impact, Population) -has its own set of basic array manipulation functionality, a set of -modelling methods to extract basic metrics from datasets, and interfaces with -external modelling packages and code. - -Agrifoodpy employs _xarray_ accesors to provide additional functionality on top -of the array manipulation provided by xarray. +AgriFoodPy modules can be used to manipulate food system data in standalone mode +or by constructing a pipeline of modules which can be executed partially or +completely. -Basic usage of the accesors depend on the type of array being manipulated. -The following examples uses the **food** module with the importable UK data -mentioned above: +To build a pipeline ```python -# import the FoodBalanceSheet accessor and FAOSTAT from agrifoodpy_data -from agrifoodpy.food.food import FoodBalanceSheet -from agrifoodpy_data.food import FAOSTAT +from agrifoodpy.pipeline import Pipeline +from agrifoodpy.utils.load_dataset import load_dataset +from agrifoodpy.food.model import import matplotlib.pyplot as plt -# Extract data for the UK (Region=229) -food_uk = FAOSTAT.sel(Region=229) +# Create pipeline object +fs = Pipeline() -# Compute the Self-sufficiency ratio using the fbs accessor SSR function -SSR = food_uk.fbs.SSR(per_item=True) +# Add node to load food balance sheet data from external module. +fs.add_node(load_dataset, + { + "datablock_path": "food", + "module": "agrifoodpy_data.food", + "data_attr": "FAOSTAT", + "coords": {"Year":np.arange(1990, 2010), "Region":229} + }) -# Plot the results using the fbs accessor plot_years function -SSR.fbs.plot_years() -plt.show() -``` -To use the specific models and interfaces to external code, these need to be -imported +# Add node convert scale Food Balance Sheet by a constant +fs.add_node(fbs_convert, + { + "fbs":"food", + "convertion_arr":1e-6 # From 1000 Tonnes to kg + }) -```python -# import the FoodBalanceSheet accessor and FAOSTAT from agrifoodpy_data -from agrifoodpy.food.food import FoodBalanceSheet -from agrifoodpy_data.food import FAOSTAT -import agrifoodpy.food.model as food_model -import matplotlib.pyplot as plt +fs.run() -# Extract data for the UK in 2020 (Region=229, Year=2020) -food_uk = FAOSTAT.sel(Region=229, Year=2020) - -# Scale consumption of meat to 50%, -food_uk_scaled = food_model.balanced_scaling(food_uk, - items=2731, - element="food", - origin="production", - scale=0.5, - constant=True) - -# Plot bar summary of resultant food quantities -food_uk_scaled.fbs.plot_bars(elements=["production","imports"], - inverted_elements=["exports","food"]) -plt.show() +results = fs.datablock ``` -In he future, we plan to implement a pipeline manager to automatize certain -aspects of the agrifood execution, and to simulate a comprehensive model where -all aspects of the food system are considered simultaneously. - ## Examples and documentation -[Examples](https://agrifoodpy.readthedocs.io/en/latest/examples/index.html#modules) +[Examples](https://agrifoodpy.readthedocs.io/en/latest/examples/index.html) demonstrating the functionality of AgriFoodPy can be the found in the [package documentation](https://agrifoodpy.readthedocs.io/en/latest/). These include the use of accessors to manipulate data and access to basic diff --git a/agrifoodpy/pipeline/__init__.py b/agrifoodpy/pipeline/__init__.py new file mode 100644 index 0000000..82e5e72 --- /dev/null +++ b/agrifoodpy/pipeline/__init__.py @@ -0,0 +1,6 @@ +""" +This module provides methods to build a pipeline for the AgriFoodPy package. +""" + +from .pipeline import * +from .utils import * \ No newline at end of file diff --git a/agrifoodpy/pipeline/pipeline.py b/agrifoodpy/pipeline/pipeline.py new file mode 100644 index 0000000..344dc3b --- /dev/null +++ b/agrifoodpy/pipeline/pipeline.py @@ -0,0 +1,189 @@ +"""Pipeline implementation + +This class provides methods to build and manage a pipeline for end to end +simulations using the agrifoodpy package. +""" + +import copy +from functools import wraps +from inspect import signature +import time + +class Pipeline(): + '''Class for constructing and running pipelines of functions with + individual sets of parameters.''' + def __init__(self, datablock=None): + self.nodes = [] + self.params = [] + self.names = [] + if datablock is not None: + self.datablock = datablock + else: + self.datablock = {} + + @classmethod + def read(cls, filename): + """Read a pipeline from a configuration file + + Parameters + ---------- + filename : str + The name of the configuration file. + + Returns + ------- + pipeline : Pipeline + The pipeline object. + """ + raise NotImplementedError("This method is not yet implemented.") + + def datablock_write(self, path, value): + """Writes a single value to the datablock at the specified path. + + Parameters + ---------- + path : list + The datablock path to the value to be written. + value : any + The value to be written. + """ + current = self.datablock + + for key in path[:-1]: + current = current.setdefault(key, {}) + current[path[-1]] = value + + def add_node(self, node, params={}, name=None): + """Adds a node to the pipeline, including its function and execution + parameters. + + Parameters + ---------- + node : function + The function to be executed on this node. + params : dict, optional + The parameters to be passed to the node function. + name : str, optional + The name of the node. If not provided, a generic name will be + assigned. + """ + + # Copy the parameters to avoid modifying the original dictionaries + params = copy.deepcopy(params) + + if name is None: + name = "Node {}".format(len(self.nodes) + 1) + + self.names.append(name) + self.nodes.append(node) + self.params.append(params) + + def run(self, from_node=0, to_node=None, timing=False): + """Runs the pipeline + + Parameters + ---------- + from_node : int, optional + The index of the first node to be executed. Defaults to 0. + + to_node : int, optional + The index of the last node to be executed. If not provided, all + nodes will be executed + + timing : bool, optional + If True, the execution time of each node will be printed. Defaults + to False. + """ + + if to_node is None: + to_node = len(self.nodes) + + pipeline_start_time = time.time() + + # Execute the node functions within the specified range + for i in range(from_node, to_node): + node = self.nodes[i] + params = self.params[i] + + node_start_time = time.time() + + # Run node + self.datablock = node(datablock=self.datablock, **params) + + node_end_time = time.time() + node_time = node_end_time - node_start_time + + if timing: + print(f"Node {i + 1} ({self.names[i]}) executed in {node_time:.4f} seconds.") + + pipeline_end_time = time.time() + pipeline_time = pipeline_end_time - pipeline_start_time + + if timing: + print(f"Pipeline executed in {pipeline_time:.4f} seconds.") + +def standalone(input_keys, return_keys): + """ Decorator to make a pipeline node available as a standalone function + + If datablock is not passed as a kwarg, and datasets are passed directly + instead of datablock keys, a temporary datablock is created and the datasets + associated with the arguments in input_keys are added to it. The function + then returns the specified datasets in return_keys. + + Parameters + ---------- + key_list: list of strings + List of dataset keys to be added to the temporary datablock + return_list: list of strings + List of keys to datablock datasets to be returned by the decorated + function. + + Returns + ------- + + wrapper: function + The decorated function + + """ + def pipeline_decorator(test_func): + @wraps(test_func) + def wrapper(*args, **kwargs): + + # Identify positional arguments + func_sig = signature(test_func) + func_params = func_sig.parameters + + kwargs.update({key: arg for key, arg in zip(func_params.keys(), args)}) + + # Make sure that the datablock is passed as a kwarg, if not, create it + datablock = kwargs.get("datablock", None) + + # Fill in missing arguments with their default values + for key, param in func_params.items(): + if key not in kwargs: + if param.default is not param.empty: # Check if there's a default value + kwargs[key] = param.default + + standalone = datablock is None + if standalone: + # Create datablock + datablock = {key: kwargs[key] for key in kwargs if key in input_keys} + kwargs["datablock"] = datablock + + # Create list of keys for passed arguments only + for key in input_keys: + if kwargs.get(key, None) is not None: + kwargs[key] = key + + result = test_func(**kwargs) + + # return tuple of results + if standalone: + if len(return_keys) == 1: + return result[kwargs[return_keys[0]]] + else: + return tuple(kwargs[result[key]] for key in return_keys) + + return result + return wrapper + return pipeline_decorator \ No newline at end of file diff --git a/agrifoodpy/pipeline/tests/__init__.py b/agrifoodpy/pipeline/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/agrifoodpy/pipeline/tests/test_pipeline.py b/agrifoodpy/pipeline/tests/test_pipeline.py new file mode 100644 index 0000000..9692862 --- /dev/null +++ b/agrifoodpy/pipeline/tests/test_pipeline.py @@ -0,0 +1,103 @@ +from agrifoodpy.pipeline.pipeline import Pipeline, standalone + +def test_init(): + pipeline = Pipeline() + +def test_add_node(): + pipeline = Pipeline() + def dummy_node(datablock, param1): + datablock['result'] = param1 + return datablock + + pipeline.add_node(dummy_node, params={'param1': 10}, name='Test Node') + assert(len(pipeline.nodes) == 1) + assert(pipeline.names[0] == 'Test Node') + assert(pipeline.params[0] == {'param1': 10}) + +def test_run_pipeline(): + pipeline = Pipeline() + def node1(datablock, param1): + datablock['result1'] = param1 + return datablock + + def node2(datablock, param2): + datablock['result2'] = param2 + return datablock + + pipeline.add_node(node1, params={'param1': 10}) + pipeline.add_node(node2, params={'param2': 20}) + + pipeline.run() + assert(pipeline.datablock['result1'] == 10) + assert(pipeline.datablock['result2'] == 20) + +def test_run_first_node_only(): + pipeline = Pipeline() + def node1(datablock, param1): + datablock['result1'] = param1 + return datablock + + def node2(datablock, param2): + datablock['result2'] = param2 + return datablock + + pipeline.add_node(node1, params={'param1': 10}) + pipeline.add_node(node2, params={'param2': 20}) + + pipeline.run(to_node=1) + assert(pipeline.datablock['result1'] == 10) + assert('result2' not in pipeline.datablock) + +def test_run_nodes_separately(): + pipeline = Pipeline() + def node1(datablock, param1): + datablock['result1'] = param1 + return datablock + + def node2(datablock, param2): + datablock['result1'] *= param2 + return datablock + + pipeline.add_node(node1, params={'param1': 10}) + pipeline.add_node(node2, params={'param2': 2}) + + pipeline.run(to_node=1) + assert(pipeline.datablock['result1'] == 10) + assert('result2' not in pipeline.datablock) + + pipeline.run(from_node=1) + assert(pipeline.datablock['result1'] == 20) + +def test_standalone_decorator(): + pipeline = Pipeline() + @standalone([], ['output1']) + def test_func(input1, output1="output1", datablock=None): + datablock[output1] = input1 * 2 + return datablock + + result = test_func(5) + assert(result == 10) + + pipeline.add_node(test_func, params={'input1': 5, 'output1': 'output1'}) + pipeline.run() + assert(pipeline.datablock['output1'] == 10) + +def test_datablock_write(): + pipeline = Pipeline() + pipeline.datablock_write(['a', 'b', 'c'], 10) + assert(pipeline.datablock['a']['b']['c'] == 10) + + pipeline.datablock_write(['a', 'b', 'd'], 20) + assert(pipeline.datablock['a']['b']['c'] == 10) + assert(pipeline.datablock['a']['b']['d'] == 20) + + pipeline.datablock_write(['a', 'e'], 30) + assert(pipeline.datablock['a']['b']['c'] == 10) + assert(pipeline.datablock['a']['b']['d'] == 20) + assert(pipeline.datablock['a']['e'] == 30) + + pipeline.datablock_write(['f'], 40) + assert(pipeline.datablock['a']['b']['c'] == 10) + assert(pipeline.datablock['a']['b']['d'] == 20) + assert(pipeline.datablock['a']['e'] == 30) + assert(pipeline.datablock['f'] == 40) diff --git a/agrifoodpy/pipeline/tests/test_utils.py b/agrifoodpy/pipeline/tests/test_utils.py new file mode 100644 index 0000000..603cfba --- /dev/null +++ b/agrifoodpy/pipeline/tests/test_utils.py @@ -0,0 +1,46 @@ +import numpy as np +import xarray as xr + +from agrifoodpy.pipeline.utils import item_parser + +def test_item_parser_none(): + + items = ["Beef", "Apples", "Poultry"] + item_origin = ["Animal", "Vegetal", "Animal"] + + data = np.random.rand(3, 2, 2) + + fbs = xr.Dataset({"data": (("Item", "X", "Y"), data)}, + coords={"Item": items, "X": [0, 1], "Y": [0, 1]}) + fbs = fbs.assign_coords({"Item_origin":("Item", item_origin)}) + + items = item_parser(fbs, None) + assert items is None + +def test_item_parser_tuple(): + items = ["Beef", "Apples", "Poultry"] + item_origin = ["Animal", "Vegetal", "Animal"] + + data = np.random.rand(3, 2, 2) + + fbs = xr.Dataset({"data": (("Item", "X", "Y"), data)}, + coords={"Item": items, "X": [0, 1], "Y": [0, 1]}) + fbs = fbs.assign_coords({"Item_origin":("Item", item_origin)}) + + items = item_parser(fbs, ("Item_origin", ["Animal"])) + assert np.array_equal(items, ["Beef", "Poultry"]) + +def test_item_parser_scalar(): + items = ["Beef", "Apples", "Poultry"] + item_origin = ["Animal", "Vegetal", "Animal"] + + data = np.random.rand(3, 2, 2) + + fbs = xr.Dataset({"data": (("Item", "X", "Y"), data)}, + coords={"Item": items, "X": [0, 1], "Y": [0, 1]}) + fbs = fbs.assign_coords({"Item_origin":("Item", item_origin)}) + + items = item_parser(fbs, "Beef") + + assert np.array_equal(items, ["Beef"]) + diff --git a/agrifoodpy/pipeline/utils.py b/agrifoodpy/pipeline/utils.py new file mode 100644 index 0000000..b6c63e6 --- /dev/null +++ b/agrifoodpy/pipeline/utils.py @@ -0,0 +1,86 @@ +"""Pipeline utilities""" + +import numpy as np + +def item_parser(fbs, items): + """Extracts a list of items from a dataset using a coordinate-key tuple, + or converts a scalar item to a list + + Parameters + ---------- + + fbs : xarray.Dataset + The FBS dataset + + items : tuple, scalar + If a tuple, the first element is the name of the coordinate and the + second element is a list of items to extract. If a scalar, the item + is converted to a list. + + Returns + ------- + list + A list of items matching the coordinate-key description, or containing + the scalar item. + """ + + if items is None: + return None + + if isinstance(items, tuple): + items = fbs.sel(Item = fbs[items[0]].isin(items[1])).Item.values + elif np.isscalar(items): + items = [items] + + return items + +def get_dict_path(datablock, keys): + """Returns an element from a dictionary using a key or tuple of keys used to + describe a path of keys + + Parameters + ---------- + + datablock : dict + The input dictionary + + keys : str or tuple + Dictionary key, or tuple of keys + """ + + if isinstance(keys, tuple): + out = datablock + for key in keys: + out = out[key] + else: + out = datablock[keys] + + return out + +def set_dict_path(datablock, keys, object): + """Sets an element in a dictionary using a key or tuple of keys used to + describe a path of keys + + Parameters + ---------- + + datablock : dict + The input dictionary + + keys : str or tuple + Dictionary key, or tuple of keys + + object : any + The object to set in the dictionary + """ + + if isinstance(keys, tuple): + out = datablock + for key in keys[:-1]: + out = out[key] + out[keys[-1]] = object + else: + datablock[keys] = object + + return datablock + diff --git a/agrifoodpy/utils/add_items.py b/agrifoodpy/utils/add_items.py new file mode 100644 index 0000000..296887d --- /dev/null +++ b/agrifoodpy/utils/add_items.py @@ -0,0 +1,63 @@ +import xarray as xr +import numpy as np +import copy + +from agrifoodpy.pipeline import standalone +from agrifoodpy.pipeline.utils import get_dict_path, set_dict_path +from agrifoodpy.food import food + +@standalone(["dataset"], ["dataset"]) +def add_items(dataset, items, values=None, copy_from=None, datablock=None): + """Adds a list of items to a selected dataset in the datablock and + initializes their values. + + Parameters + ---------- + datablock : Datablock + Datablock object. + dataset : dict + Datablock path to the datasets to modify. + items : list, dict + List of items or dictionary of items attributes to add. If a dictionary, + keys are the item names, and non-dimension coordinates. Must contain a + key named "Item". + values : list, optional + List of values to initialize the items. If not set, values are set to 0, + unless the copy from parameter is set. + copy_from : list, optional + Items to copy the values from. + + Returns + ------- + dict or xarray.Dataset + - If no datablock is provided, returns a xarray.Dataset with the new + items. + - If a datablock is provided, returns the datablock with the modified + datasets on the corresponding keys. + + """ + + # Check if items is a dictionary + if isinstance(items, dict): + items_src = copy.deepcopy(items) + new_items = items_src.pop("Item") + else: + new_items = items + items_src = {} + + # Add new items to the datasets + data = get_dict_path(datablock, dataset) + + data = data.fbs.add_items(new_items, copy_from=copy_from) + for key, val in items_src.items(): + data[key].loc[{"Item":new_items}] = val + + if values is not None: + data.loc[{"Item":new_items}] = values + + elif copy_from is None: + data.loc[{"Item":new_items}] = 0 + + datablock = set_dict_path(datablock, dataset, data) + + return datablock diff --git a/agrifoodpy/utils/copy_datablock.py b/agrifoodpy/utils/copy_datablock.py new file mode 100644 index 0000000..c0895ec --- /dev/null +++ b/agrifoodpy/utils/copy_datablock.py @@ -0,0 +1,23 @@ +import copy + +def copy_datablock(datablock, key, out_key): + """Copy a datablock element into a new key in the datablock + + Parameters + ---------- + datablock : xarray.Dataset + The datablock to print + key : str + The key of the datablock to print + out_key : str + The key of the datablock to copy to + + Returns + ------- + datablock : dict + Datablock to with added key + """ + + datablock[out_key] = copy.deepcopy(datablock[key]) + + return datablock diff --git a/agrifoodpy/utils/extend_years.py b/agrifoodpy/utils/extend_years.py new file mode 100644 index 0000000..1f2251d --- /dev/null +++ b/agrifoodpy/utils/extend_years.py @@ -0,0 +1,29 @@ +from agrifoodpy.pipeline import standalone +from agrifoodpy.impact.impact import Impact + +@standalone(["dataset"], ["dataset"]) +def extend_years(dataset, years, projection='empty', datablock=None): + """ + Extends the dimensions of a dataset. + + Parameters + ---------- + datablock : dict + The datablock dictionary where the dataset is stored. + dataset : str + Datablock key of the dataset to extend. + years : list + List of years to extend the dataset to. + projection : str + Projection mode. If "constant", the last year of the input array + is copied to every new year. If "empty", values are initialized and + set to zero. If a float array is given, these are used to populate + the new year using a scaling of the last year of the array + """ + + data = datablock[dataset].copy(deep=True) + + data = data.fbs.add_years(years, projection) + + datablock[dataset] = data + return dataset diff --git a/agrifoodpy/utils/load_dataset.py b/agrifoodpy/utils/load_dataset.py new file mode 100644 index 0000000..2e0d0ad --- /dev/null +++ b/agrifoodpy/utils/load_dataset.py @@ -0,0 +1,65 @@ +import numpy as np +import xarray as xr +import importlib + +from agrifoodpy.pipeline import standalone + +def import_dataset(module_name, dataset_name): + module = importlib.import_module(module_name) + dataset = getattr(module, dataset_name) + return dataset + +@standalone([], ['datablock_path']) +def load_dataset(datablock_path="data", path=None, module=None, + data_attr=None, da=None, coords=None, scale=1., + datablock=None): + """Loads a dataset to the specified datablock dictionary. Can only be used + in pipeline mode. + + Parameters + ---------- + datablock : dict + The datablock dictionary where the dataset is stored, containing all + the model parameters + datablock_path : str + The path to the datablock where the dataset is stored. + path : str + The path to the dataset stored in a netCDF file. + module : str + The module name where the dataset will be imported from. + data_attr : str + The attribute name of the dataset in the module. + da : str + The dataarray to be loaded. + coords : dict + Dictionary containing the coordinates of the dataset to be loaded. + scale : float + Optional scale factor to be applied to the dataset on load. + + """ + + # Load dataset from Netcdf file + if path is not None: + try: + with xr.open_dataset(path) as data: + dataset = data.load() + + except ValueError: + with xr.open_dataarray(path) as data: + dataset = data.load() + + # Load dataset from module + elif module is not None and data_attr is not None: + dataset = import_dataset(module, data_attr) + + # Select dataarray and coords from dataset + if da is not None: + dataset = dataset[da] + + if coords is not None: + dataset = dataset.sel(coords) + + # Add dataset to datablock + datablock[datablock_path] = dataset * scale + + return datablock diff --git a/agrifoodpy/utils/print_datablock.py b/agrifoodpy/utils/print_datablock.py new file mode 100644 index 0000000..01018b8 --- /dev/null +++ b/agrifoodpy/utils/print_datablock.py @@ -0,0 +1,21 @@ +def print_datablock(datablock, key, sel={}): + """Prints a datablock element at any point in the pipeline execution + + Parameters + ---------- + datablock : xarray.Dataset + The datablock to print + key : str + The key of the datablock to print + sel : dict, optional + The selection to apply to the datablock + + Returns + ------- + datablock : xarray.Dataset + Unmodified datablock to continue execution + """ + + print(datablock[key].sel(sel)) + + return datablock diff --git a/docs/conf.py b/docs/conf.py index 1e3bcd6..9b60950 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -9,7 +9,7 @@ project = 'agrifoodpy' copyright = '2023, AgriFoodPy developers' author = 'Juan P. Cordero' -release = '0.1.0' +release = '0.2.0' # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration diff --git a/examples/modules/plot_emissions_animal_scaling.py b/examples/modules/plot_emissions_animal_scaling.py index 13dea0f..37654af 100644 --- a/examples/modules/plot_emissions_animal_scaling.py +++ b/examples/modules/plot_emissions_animal_scaling.py @@ -33,7 +33,7 @@ food = FAOSTAT.sel(Region=country_codes)["production"] # Convert emissions from [g CO2e] to [Gt CO2e] -ghg_emissions = PN18["GHG Emissions"] / 1e6 +ghg_emissions = PN18["GHG Emissions (IPCC 2013)"] / 1e6 food_emissions = fbs_impacts(food, ghg_emissions) ax = food_emissions.fbs.plot_years(show="Region", labels=["UK", "USA"]) diff --git a/setup.py b/setup.py index 649a794..046c7f3 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ HERE = pathlib.Path(__file__).parent -VERSION = '0.1.1' +VERSION = '0.2.0' PACKAGE_NAME = 'AgriFoodPy' AUTHOR = 'FixOurFood developers' AUTHOR_EMAIL = 'juanpablo.cordero@york.ac.uk' From 0c1f5c9a921503d78e03be1d1a46523276933304 Mon Sep 17 00:00:00 2001 From: jucordero Date: Fri, 29 Aug 2025 16:49:06 +0100 Subject: [PATCH 2/4] Tests and examples for all modules --- agrifoodpy/food/food.py | 48 ++- agrifoodpy/food/model.py | 404 +++++++++++++----- agrifoodpy/food/tests/test_model.py | 364 +++++++++++++++- agrifoodpy/impact/impact.py | 2 +- agrifoodpy/land/model.py | 2 +- agrifoodpy/pipeline/__init__.py | 2 +- agrifoodpy/pipeline/pipeline.py | 15 +- agrifoodpy/pipeline/tests/test_pipeline.py | 55 +++ agrifoodpy/pipeline/tests/test_utils.py | 46 -- agrifoodpy/population/population.py | 2 +- agrifoodpy/tests/test_base_class.py | 2 +- agrifoodpy/utils/add_items.py | 18 +- .../utils/{extend_years.py => add_years.py} | 22 +- .../utils.py => utils/dict_utils.py} | 27 +- agrifoodpy/utils/load_dataset.py | 31 +- agrifoodpy/utils/print_datablock.py | 66 ++- .../utils/tests/data/generate_dataset.py | 13 + agrifoodpy/utils/tests/data/test_dataset.nc | Bin 0 -> 8368 bytes agrifoodpy/utils/tests/test_add_items.py | 55 +++ agrifoodpy/utils/tests/test_add_years.py | 39 ++ agrifoodpy/utils/tests/test_copy_datablock.py | 36 ++ agrifoodpy/utils/tests/test_dict_utils.py | 81 ++++ agrifoodpy/utils/tests/test_load_dataset.py | 39 ++ .../utils/tests/test_print_datablock.py | 53 +++ .../utils/tests/test_write_to_datablock.py | 34 ++ agrifoodpy/utils/write_to_datablock.py | 29 ++ .../plot_animal_consumption_scaling.py | 42 +- .../modules/plot_emissions_animal_scaling.py | 13 +- examples/modules/plot_reforest_ALC_4_5.py | 15 +- examples/pipeline/README.rst | 6 + ...ced_scaling_food_balance_sheet_pipeline.py | 212 +++++++++ 31 files changed, 1513 insertions(+), 260 deletions(-) delete mode 100644 agrifoodpy/pipeline/tests/test_utils.py rename agrifoodpy/utils/{extend_years.py => add_years.py} (63%) rename agrifoodpy/{pipeline/utils.py => utils/dict_utils.py} (73%) create mode 100644 agrifoodpy/utils/tests/data/generate_dataset.py create mode 100644 agrifoodpy/utils/tests/data/test_dataset.nc create mode 100644 agrifoodpy/utils/tests/test_add_items.py create mode 100644 agrifoodpy/utils/tests/test_add_years.py create mode 100644 agrifoodpy/utils/tests/test_copy_datablock.py create mode 100644 agrifoodpy/utils/tests/test_dict_utils.py create mode 100644 agrifoodpy/utils/tests/test_load_dataset.py create mode 100644 agrifoodpy/utils/tests/test_print_datablock.py create mode 100644 agrifoodpy/utils/tests/test_write_to_datablock.py create mode 100644 agrifoodpy/utils/write_to_datablock.py create mode 100644 examples/pipeline/README.rst create mode 100644 examples/pipeline/plot_balanced_scaling_food_balance_sheet_pipeline.py diff --git a/agrifoodpy/food/food.py b/agrifoodpy/food/food.py index 933f856..ee103a6 100644 --- a/agrifoodpy/food/food.py +++ b/agrifoodpy/food/food.py @@ -13,7 +13,7 @@ import copy import warnings -from agrifoodpy.array_accessor import XarrayAccessorBase +from ..array_accessor import XarrayAccessorBase import matplotlib.pyplot as plt @@ -127,9 +127,14 @@ def FoodSupply(items, years, quantities, regions=None, elements=None, @xr.register_dataset_accessor("fbs") class FoodBalanceSheet(XarrayAccessorBase): - def scale_element(self, element, scale, items=None): - """Scales list of items from an element in a food balance sheet like - DataSet. + def scale_element( + self, + element, + scale, + items=None + ): + """Scales list of items from an element in a Food Balance Sheet like + Dataset. Parameters ---------- @@ -172,8 +177,16 @@ def scale_element(self, element, scale, items=None): return out - def scale_add(self, element_in, element_out, scale, items=None, add=True, - elasticity=None): + def scale_add( + self, + element_in, + element_out, + scale, + items=None, + add=True, + elasticity=None + ): + """Scales item quantities of an element and adds the difference to another element DataArray @@ -327,8 +340,16 @@ def IDR(self, items=None, per_item=False, imports="imports", domestic=None, return fbs["imports"].sum(dim="Item") / domestic_use.sum(dim="Item") - def plot_bars(self, show="Item", elements=None, inverted_elements=None, - ax=None, colors=None, labels=None, **kwargs): + def plot_bars( + self, + show="Item", + elements=None, + inverted_elements=None, + ax=None, + colors=None, + labels=None, + **kwargs + ): """Plot total quantities per element on a horizontal bar plot Produces a horizontal bar plot with a bar per element on the vertical @@ -479,8 +500,15 @@ def plot_bars(self, show="Item", elements=None, inverted_elements=None, @xr.register_dataarray_accessor("fbs") class FoodElementSheet(XarrayAccessorBase): - def plot_years(self, show=None, stack=True,ax=None, colors=None, - labels=None, **kwargs): + def plot_years( + self, + show=None, + stack=True, + ax=None, + colors=None, + labels=None, + **kwargs + ): """ Fill plot with quantities at each year value Produces a vertical fill plot with quantities for each year on the diff --git a/agrifoodpy/food/model.py b/agrifoodpy/food/model.py index 3f106b0..c6854bf 100644 --- a/agrifoodpy/food/model.py +++ b/agrifoodpy/food/model.py @@ -3,146 +3,324 @@ import xarray as xr import numpy as np -# from .food_supply import FoodBalanceSheet import warnings +import copy +from ..pipeline import standalone +from ..utils.dict_utils import get_dict, set_dict, item_parser -def balanced_scaling(fbs, items, scale, element, year=None, adoption=None, - timescale=10, origin=None, constant=False, - fallback=None): - """Scale items quantities across multiple elements in a FoodBalanceSheet - Dataset +@standalone(input_keys=["fbs"], return_keys=["out_key"]) +def balanced_scaling( + fbs, + scale, + element, + items=None, + constant=False, + origin=None, + add_to_origin=True, + elasticity=None, + fallback=None, + add_to_fallback=True, + datablock=None, + out_key=None +): + """ Scales items in a Food Balance Sheet, while optionally maintaining total + quantities - Scales selected item quantities on a food balance sheet and with the - posibility to keep the sum of selected elements constant. - Optionally, produce an Dataset with a sequence of quantities over the years - following a smooth scaling according to the selected functional form. - - The elements used to supply the modified quantities can be selected to keep - a balanced food balance sheet. + Scales selected item quantities on a Food Balance Sheet, with the option + to keep the sum over an element DataArray constant. + Changes can be propagated to a set of origin FBS elements according to an + elasticity parameter. Parameters ---------- fbs : xarray.Dataset - Input food balance sheet Dataset. - items : list - List of items to scale in the food balance sheet. + Input food balance sheet Dataset element : string - Name of the DataArray to scale. + Name of the DataArray to scale scale : float - Scaling parameter after full adoption. - year : int, optional - Year of the Food Balance Sheet to use. If not set, the last year of the - array is used - adoption : string, optional - Shape of the scaling adoption curve. "logistic" uses a logistic model - for a slow-fast-slow adoption. "linear" uses a constant slope adoption - during the the "timescale period" - timescale : int, optional - Timescale for the scaling to be applied completely. If "year" + - "timescale" is greater than the last year in the array, it is extended - to accomodate the extra years. - origin : string, optional - Name of the DataArray which will be used to balance the food balance - sheets. Any change to the "element" DataArray will be reflected in this - DataArray. + Scaling parameter after full adoption + items : list, optional + List of items to scaled in the food balance sheet. If None, all items + are scaled and 'constant' is ignored constant : bool, optional If set to True, the sum of element remains constant by scaling the non - selected items accordingly. - fallback : string, optional - Name of the DataArray used to provide the excess required to balance the - food balance sheet in case the "origin" falls below zero. - + selected items accordingly + origin : string, list, optional + Names of the DataArrays which will be used as source for the quantity + changes. Any change to the "element" DataArray will be reflected in this + DataArray + add_to_origin : bool, array, optional + Whether to add or subtract the difference from the respective origins + elasticity : float, array, optional + Relative fraction of the total difference to be assigned to each origin + element. Values are not normalized. + fallback : string + Name of the DataArray to use as fallback in case the origin quantities + fall below zero + add_to_fallback : bool, optional + Whether to add or subtract the difference below zero in the origin + DataArray to the fallback array. + out_key : string, tuple + Output datablock path to write results to. If not given, input path is + overwritten + datablock : dict, optional + Dictionary containing data + Returns ------- data : xarray.Dataarray - Food balance sheet Dataset with scaled "food" values. + Food balance sheet Dataset with scaled values. """ - # Check for single item inputs - if np.isscalar(items): - items = [items] - - # Check for single item list fbs - input_item_list = fbs.Item.values - if np.isscalar(input_item_list): - input_item_list = [input_item_list] - if constant: - warnings.warn("Constant set to true but input only has a single item.") - constant = False - - # If no items are provided, we scale all of them. - if items is None or np.sort(items) is np.sort(input_item_list): - items = fbs.Item.values - if constant: - warnings.warn("Cannot keep food constant when scaling all items.") - constant = False - - # Define Dataarray to use as pivot - if "Year" in fbs.dims: - if year is None: - if np.isscalar(fbs.Year.values): - year = fbs.Year.values - fbs_toscale = fbs - else: - year = fbs.Year.values[-1] - fbs_toscale = fbs.isel(Year=-1) - else: - fbs_toscale = fbs.sel(Year=year) + # Pepare inputs + data = copy.deepcopy(get_dict(datablock, fbs)) + out = copy.deepcopy(data) + + if out_key is None: + out_key = fbs + if items is None: + scaled_items = data.Item.values + constant = False else: - fbs_toscale = fbs - try: - year = fbs.Year.values - except AttributeError: - year=0 - - # Define scale array based on year range - if adoption is not None: - if adoption == "linear": - from agrifoodpy.utils.scaling import linear_scale as scale_func - elif adoption == "logistic": - from agrifoodpy.utils.scaling import logistic_scale as scale_func - else: - raise ValueError("Adoption must be one of 'linear' or 'logistic'") + scaled_items = item_parser(data, items) + + if origin is not None and np.isscalar(origin): + origin = [origin] + + if np.isscalar(add_to_origin): + add_to_origin = [add_to_origin]*len(origin) + + if elasticity is None: + elasticity = [1/len(origin)]*len(origin) + + # Scale input + if origin is None: + out = out.fbs.scale_element( + element=element, + scale=scale, + items=scaled_items + ) + + else: + out = out.fbs.scale_add( + element_in=element, + element_out=origin, + scale=scale, + items=scaled_items, + add=add_to_origin, + elasticity=elasticity + ) + + # If quantities are set to be constant + if constant: + + delta = out[element] - data[element] - scale_arr = scale_func(year, year, year+timescale-1, year+timescale-1, - c_init=1, c_end = scale) + # Identify non selected items and scaling + non_sel_items = np.setdiff1d(data.Item.values, scaled_items) + non_sel_scale = (data.sel(Item=non_sel_items)[element].sum(dim="Item") + - delta.sum(dim="Item")) \ + / data.sel(Item=non_sel_items)[element].sum(dim="Item") - fbs_toscale = fbs_toscale * xr.ones_like(scale_arr) + # Make sure no scaling occurs on inf and nan + non_sel_scale = non_sel_scale.where(np.isfinite(non_sel_scale)).fillna(1.0) + + if origin is None: + out = out.fbs.scale_element( + element=element, + scale=non_sel_scale, + items=non_sel_items + ) + + else: + out = out.fbs.scale_add( + element_in=element, + element_out=origin, + scale=non_sel_scale, + items=non_sel_items, + add=add_to_origin, + elasticity=elasticity + ) + + # If a fallback DataArray is defined, transfer the excess negative + # quantities to it + if fallback is not None: + for orig in origin: + dif = out[orig].where(out[orig]<0).fillna(0) + out[fallback] -= np.where(add_to_fallback, 1, -1)*dif + out[orig] = out[orig].where(out[orig] > 0, 0) + + set_dict(datablock, out_key, out) + + return datablock + + +@standalone(input_keys=["fbs", "convertion_arr"], return_keys=["out_key"]) +def fbs_convert( + fbs, + convertion_arr, + out_key=None, + datablock=None +): + """Scales quantities in a food balance sheet using a conversion + dataarray, dataset, or scaling factor. + Parameters + ---------- + datablock : Dict + Dictionary containing data. + dataset : str, xarray.Dataset + Datablock paths to the food balance sheet datasets or the datasets + themselves. + convertion_arr : str, xarray.DataArray, tuple or float + Datablock path to the conversion array, datablock-key tuple, or the + array or float itself. + keys : str, list + Datablock key of the resulting dataset to be stored in the datablock. + + Returns + ------- + dict or xarray.Dataset + - Updated datablock if a datablock is provided. + - xarray.Dataset with converted quantities if no datablock is provided. + """ + + # Retrieve target array + data = get_dict(datablock, fbs) + + # retrieve convertion array + if isinstance(convertion_arr, xr.DataArray): + convertion_arr = convertion_arr.where(np.isfinite(convertion_arr), other=0) else: - scale_arr = scale + convertion_arr = get_dict(datablock, convertion_arr) - # Create a deep copy to modify and return - out = fbs_toscale.copy(deep=True) - osplit = origin.split("-")[-1] - - out = out.fbs.scale_add(element, osplit, scale_arr, items, - add = origin.startswith("-")) - + # If no output key is provided, overwrite original dataset + if out_key is None: + out_key = fbs - if constant: + out = data*convertion_arr + set_dict(datablock, out_key, out) - delta = out[element] - fbs_toscale[element] + return datablock - # Scale non selected items - non_sel_items = np.setdiff1d(fbs_toscale.Item.values, items) - non_sel_scale = (fbs_toscale.sel(Item=non_sel_items)[element].sum(dim="Item") - delta.sum(dim="Item")) / fbs_toscale.sel(Item=non_sel_items)[element].sum(dim="Item") - - # Make sure inf and nan values are not scaled - non_sel_scale = non_sel_scale.where(np.isfinite(non_sel_scale)).fillna(1.0) +@standalone(["fbs"], ["out_key"]) +def SSR( + fbs, + items=None, + per_item=False, + production="production", + imports="imports", + exports="exports", + out_key=None, + datablock=None, +): + """Self-sufficiency ratio + + Self-sufficiency ratio (SSR) or ratios for a list of item imports, + exports and production quantities. + + Parameters + ---------- + fbs : xarray.Dataset + Input Dataset containing an "Item" coordinate and, optionally, a + "Year" coordinate. + items : list, optional + list of items to compute the SSR for from the food Dataset. If no + list is provided, the SSR is computed for all items. + per_item : bool, optional + Whether to return an SSR for each item separately. Default is false + production : string, optional + Name of the DataArray containing the production data + imports : string, optional + Name of the DataArray containing the imports data + exports : string, optional + Name of the DataArray containing the exports data + datablock : dict, optional + Dictionary containing the food balance sheet Dataset. + + Returns + ------- + data : xarray.Dataarray + Self-sufficiency ratio or ratios for the list of items, one for each + year of the input food Dataset "Year" coordinate. + + """ + + fbs = get_dict(datablock, fbs) + + if items is not None: + if np.isscalar(items): + items = [items] + fbs = fbs.sel(Item=items) + + domestic_use = fbs[production] + fbs[imports] - fbs[exports] - if np.any(non_sel_scale < 0): - warnings.warn("Additional consumption cannot be compensated by \ - reduction of non-selected items") + if per_item: + ssr = fbs[production] / domestic_use + else: + ssr = fbs[production].sum(dim="Item") / domestic_use.sum(dim="Item") + + set_dict(datablock, out_key, ssr) + + return datablock + +@standalone(["fbs"], ["out_key"]) +def IDR( + fbs, + items=None, + per_item=False, + imports="imports", + production="production", + exports="exports", + out_key=None, + datablock=None, +): + """Import-dependency ratio + + Import-ependency ratio (IDR) or ratios for a list of item imports, + exports and production quantities. + + Parameters + ---------- + fbs : xarray.Dataset + Input Dataset containing an "Item" coordinate and, optionally, a + "Year" coordinate. + items : list, optional + list of items to compute the IDR for from the food Dataset. If no + list is provided, the IDR is computed for all items. + per_item : bool, optional + Whether to return an IDR for each item separately. Default is false. + imports : string, optional + Name of the DataArray containing the imports data + exports : string, optional + Name of the DataArray containing the exports data + production : string, optional + Name of the DataArray containing the production data + datablock : dict, optional + Dictionary containing the food balance sheet Dataset. - out = out.fbs.scale_add(element, osplit, non_sel_scale, - non_sel_items, add = origin.startswith("-")) + Returns + ------- + data : xarray.Datarray + Import-dependency ratio or ratios for the list of items, one for + each year of the input food Dataset "Year" coordinate. + """ + + fbs = get_dict(datablock, fbs) + + if items is not None: + if np.isscalar(items): + items = [items] + fbs = fbs.sel(Item=items) + + domestic_use = fbs[production] + fbs[imports] - fbs[exports] + + if per_item: + idr = fbs["imports"] / domestic_use + else: + idr = fbs["imports"].sum(dim="Item") / domestic_use.sum(dim="Item") - # If fallback is defined, adjust to prevent negative values - if fallback is not None: - df = out[osplit].where(out[osplit] < 0).fillna(0) - out[fallback.split("-")[-1]] -= np.where(fallback.startswith("-"), 1, -1)*df - out[osplit] = out[osplit].where(out[osplit] > 0, 0) + set_dict(datablock, out_key, idr) - return out \ No newline at end of file + return datablock \ No newline at end of file diff --git a/agrifoodpy/food/tests/test_model.py b/agrifoodpy/food/tests/test_model.py index b18aa65..1da1db8 100644 --- a/agrifoodpy/food/tests/test_model.py +++ b/agrifoodpy/food/tests/test_model.py @@ -4,5 +4,367 @@ import warnings def test_balanced_scaling(): - + from agrifoodpy.food.model import balanced_scaling + + items = ["Beef", "Apples"] + years = [2020, 2021] + + fbs = xr.Dataset( + data_vars=dict( + imports=(["Year", "Item"], [[10., 20.], [30., 40.]]), + production=(["Year", "Item"], [[50., 60.], [70., 80.]]), + exports=(["Year", "Item"], [[5., 10.], [15., 20.]]), + food=(["Year", "Item"], [[55., 70.], [85., 100.]]) + ), + + coords=dict(Item=("Item", items), Year=("Year", years)) + ) + + # Test basic result + result_basic = balanced_scaling( + fbs, + scale=1.0, + element="food" + ) + + xr.testing.assert_equal(result_basic, fbs) + + # Test result with scalar scaling factor + result_scalar = balanced_scaling( + fbs, + scale=2.0, + element="food" + ) + + ex_result_scalar = xr.Dataset( + data_vars=dict( + imports=(["Year", "Item"], [[10., 20.], [30., 40.]]), + production=(["Year", "Item"], [[50., 60.], [70., 80.]]), + exports=(["Year", "Item"], [[5., 10.], [15., 20.]]), + food=(["Year", "Item"], [[110., 140.], [170., 200.]]) + ), + + coords=dict(Item=("Item", items), Year=("Year", years)) + ) + + xr.testing.assert_equal(result_scalar, ex_result_scalar) + + # Test results with selected items + result_items = balanced_scaling( + fbs, + scale=2.0, + element="food", + items="Beef" + ) + + ex_result_items = xr.Dataset( + data_vars=dict( + imports=(["Year", "Item"], [[10., 20.], [30., 40.]]), + production=(["Year", "Item"], [[50., 60.], [70., 80.]]), + exports=(["Year", "Item"], [[5., 10.], [15., 20.]]), + food=(["Year", "Item"], [[110., 70.], [170., 100.]]) + ), + + coords=dict(Item=("Item", items), Year=("Year", years)) + ) + + xr.testing.assert_equal(result_items, ex_result_items) + + # Test results with selected items and setting constant to True + result_constant = balanced_scaling( + fbs, + scale=2.0, + element="food", + items="Beef", + constant=True + ) + + ex_result_constant = xr.Dataset( + data_vars=dict( + imports=(["Year", "Item"], [[10., 20.], [30., 40.]]), + production=(["Year", "Item"], [[50., 60.], [70., 80.]]), + exports=(["Year", "Item"], [[5., 10.], [15., 20.]]), + food=(["Year", "Item"], [[110., 15.], [170., 15.]]) + ), + + coords=dict(Item=("Item", items), Year=("Year", years)) + ) + + xr.testing.assert_equal(result_constant, ex_result_constant) + + # Test selected items, constant to True, origin from "production" + result_origin = balanced_scaling( + fbs, + scale=2.0, + element="food", + items="Beef", + constant=True, + origin="production" + ) + + ex_result_origin = xr.Dataset( + data_vars=dict( + imports=(["Year", "Item"], [[10., 20.], [30., 40.]]), + # production=(["Year", "Item"], [[50., 60.], [70., 80.]]), + production=(["Year", "Item"], [[105., 5.], [155., -5.]]), + exports=(["Year", "Item"], [[5., 10.], [15., 20.]]), + # food=(["Year", "Item"], [[55., 70.], [85., 100.]]) + food=(["Year", "Item"], [[110., 15.], [170., 15.]]) + ), + + coords=dict(Item=("Item", items), Year=("Year", years)) + ) + + xr.testing.assert_equal(result_origin, ex_result_origin) + + # Test with fallback + result_fallback = balanced_scaling( + fbs, + scale=2.0, + element="food", + items="Beef", + constant=True, + origin="production", + fallback="imports" + ) + + ex_result_fallback = xr.Dataset( + data_vars=dict( + # imports=(["Year", "Item"], [[10., 20.], [30., 40.]]), + imports=(["Year", "Item"], [[10., 20.], [30., 45.]]), + # production=(["Year", "Item"], [[50., 60.], [70., 80.]]), + production=(["Year", "Item"], [[105., 5.], [155., 0.]]), + exports=(["Year", "Item"], [[5., 10.], [15., 20.]]), + # food=(["Year", "Item"], [[55., 70.], [85., 100.]]) + food=(["Year", "Item"], [[110., 15.], [170., 15.]]) + ), + + coords=dict(Item=("Item", items), Year=("Year", years)) + ) + + xr.testing.assert_equal(result_fallback, ex_result_fallback) + + # Test selected items, constant to True, origin from "exports" + # add_to_origin set to False + result_add_origin = balanced_scaling( + fbs, + scale=2.0, + element="food", + items="Beef", + constant=True, + origin="exports", + add_to_origin=False + ) + + ex_result_add_origin = xr.Dataset( + data_vars=dict( + imports=(["Year", "Item"], [[10., 20.], [30., 40.]]), + production=(["Year", "Item"], [[50., 60.], [70., 80.]]), + # exports=(["Year", "Item"], [[5., 10.], [15., 20.]]), + exports=(["Year", "Item"], [[-50., 65.], [-70., 105.]]), + # food=(["Year", "Item"], [[55., 70.], [85., 100.]]) + food=(["Year", "Item"], [[110., 15.], [170., 15.]]) + ), + + coords=dict(Item=("Item", items), Year=("Year", years)) + ) + + xr.testing.assert_equal(result_add_origin, ex_result_add_origin) + + + # Test with multiple origins and separate elasticity values + result_elasticity = balanced_scaling( + fbs, + scale=2.0, + element="food", + items="Beef", + constant=True, + origin=["production", "imports"], + elasticity=[0.8, 0.2] + ) + + ex_result_elasticity = xr.Dataset( + data_vars=dict( + # imports=(["Year", "Item"], [[10., 20.], [30., 40.]]), + imports=(["Year", "Item"], [[21., 9.], [47., 23.]]), + # production=(["Year", "Item"], [[50., 60.], [70., 80.]]), + production=(["Year", "Item"], [[94., 16.], [138., 12.]]), + exports=(["Year", "Item"], [[5., 10.], [15., 20.]]), + # food=(["Year", "Item"], [[55., 70.], [85., 100.]]) + food=(["Year", "Item"], [[110., 15.], [170., 15.]]) + ), + + coords=dict(Item=("Item", items), Year=("Year", years)) + ) + + xr.testing.assert_equal(result_elasticity, ex_result_elasticity) + + # Test with a scaling array + from agrifoodpy.utils.scaling import linear_scale + + scale_arr = linear_scale( + 2020, + 2020, + 2021, + 2021, + c_init=1.0, + c_end=2.0 + ) + + result_scale_arr = balanced_scaling( + fbs, + scale=scale_arr, + element="food", + items="Beef", + ) + + ex_result_scale_arr = xr.Dataset( + data_vars=dict( + imports=(["Year", "Item"], [[10., 20.], [30., 40.]]), + production=(["Year", "Item"], [[50., 60.], [70., 80.]]), + exports=(["Year", "Item"], [[5., 10.], [15., 20.]]), + # food=(["Year", "Item"], [[55., 70.], [85., 100.]]) + food=(["Year", "Item"], [[55., 70.], [170., 100.]]) + ), + + coords=dict(Item=("Item", items), Year=("Year", years)) + ) + + xr.testing.assert_equal(result_scale_arr, ex_result_scale_arr) + + +def test_SSR(): + + from agrifoodpy.food.model import SSR + + items = ["Beef", "Apples"] + years = [2020, 2021] + + fbs = xr.Dataset( + data_vars=dict( + imports=(["Year", "Item"], [[10, 20], [30, 40]]), + exports=(["Year", "Item"], [[5, 10], [15, 20]]), + production=(["Year", "Item"], [[50, 60], [70, 80]]), + ), + + coords=dict(Item=("Item", items), Year=("Year", years)) + ) + + # Test basic result on all items + result_basic = SSR(fbs) + ex_result_basic = xr.DataArray([0.88, 0.810810], dims=("Year"), + coords={"Year": years}) + + xr.testing.assert_allclose(result_basic, ex_result_basic) + + # Test for an item subset + result_subset = SSR(fbs, items="Beef") + ex_result_subset = xr.DataArray([0.909090, 0.823529], dims=("Year"), + coords={"Year": years}) + + xr.testing.assert_allclose(result_subset, ex_result_subset) + + # Test per item + result_peritem = SSR(fbs, per_item=True) + ex_result_peritem = xr.DataArray([[0.909090, 0.857142], [0.823529, 0.8]], + dims=(["Year", "Item"]), + coords={"Year": years, "Item": items}) + + xr.testing.assert_allclose(result_peritem, ex_result_peritem) + +def test_IDR(): + + from agrifoodpy.food.model import IDR + + items = ["Beef", "Apples"] + years = [2020, 2021] + + fbs = xr.Dataset( + data_vars=dict( + imports=(["Year", "Item"], [[10, 20], [30, 40]]), + exports=(["Year", "Item"], [[5, 10], [15, 20]]), + production=(["Year", "Item"], [[50, 60], [70, 80]]), + ), + + coords=dict(Item=("Item", items), Year=("Year", years)) + ) + + # Test basic result on all items + result_basic = IDR(fbs) + ex_result_basic = xr.DataArray([0.24, 0.37837838], dims=("Year"), + coords={"Year": years}) + + xr.testing.assert_allclose(result_basic, ex_result_basic) + + # Test for an item subset + result_subset = IDR(fbs, items="Beef") + ex_result_subset = xr.DataArray([0.1818181, 0.352941], dims=("Year"), + coords={"Year": years}) + + xr.testing.assert_allclose(result_subset, ex_result_subset) + + # Test per item + result_peritem = IDR(fbs, per_item=True) + ex_result_peritem = xr.DataArray([[0.1818181, 0.285714], [0.352941, 0.4]], + dims=(["Year", "Item"]), + coords={"Year": years, "Item": items}) + + xr.testing.assert_allclose(result_peritem, ex_result_peritem) + +def test_fbs_convert(): + + from agrifoodpy.food.model import fbs_convert + + items = ["Beef", "Apples"] + years = [2020, 2021] + + fbs = xr.Dataset( + data_vars=dict( + imports=(["Year", "Item"], [[10, 20], [30, 40]]), + exports=(["Year", "Item"], [[5, 10], [15, 20]]), + production=(["Year", "Item"], [[50, 60], [70, 80]]), + ), + + coords=dict(Item=("Item", items), Year=("Year", years)) + ) + + # Test basic result + result_basic = fbs_convert(fbs, convertion_arr=1.0) + ex_result_basic = xr.Dataset( + data_vars=dict( + imports=(["Year", "Item"], [[10, 20], [30, 40]]), + exports=(["Year", "Item"], [[5, 10], [15, 20]]), + production=(["Year", "Item"], [[50, 60], [70, 80]]), + ), + coords=dict(Item=("Item", items), Year=("Year", years)) + ) + + xr.testing.assert_allclose(result_basic, ex_result_basic) + + # Test with a conversion array + conversion_arr = xr.DataArray([1.0, 2.0], dims=["Item"], coords={"Item": items}) + result_conversion = fbs_convert(fbs, convertion_arr=conversion_arr) + ex_result_conversion = xr.Dataset( + data_vars=dict( + imports=(["Year", "Item"], [[10, 40], [30, 80]]), + exports=(["Year", "Item"], [[5, 20], [15, 40]]), + production=(["Year", "Item"], [[50, 120], [70, 160]]), + ), + coords=dict(Item=("Item", items), Year=("Year", years)) + ) + + xr.testing.assert_allclose(result_conversion, ex_result_conversion) + + # Test with a conversion factor + result_factor = fbs_convert(fbs, convertion_arr=2.0) + ex_result_factor = xr.Dataset( + data_vars=dict( + imports=(["Year", "Item"], [[20, 40], [60, 80]]), + exports=(["Year", "Item"], [[10, 20], [30, 40]]), + production=(["Year", "Item"], [[100, 120], [140, 160]]), + ), + coords=dict(Item=("Item", items), Year=("Year", years)) + ) + + xr.testing.assert_allclose(result_factor, ex_result_factor) diff --git a/agrifoodpy/impact/impact.py b/agrifoodpy/impact/impact.py index 863e7dd..a409660 100644 --- a/agrifoodpy/impact/impact.py +++ b/agrifoodpy/impact/impact.py @@ -3,7 +3,7 @@ import numpy as np import xarray as xr -from agrifoodpy.array_accessor import XarrayAccessorBase +from ..array_accessor import XarrayAccessorBase def impact(items, regions, quantities, datasets=None, long_format=True): """Impact style dataset constructor diff --git a/agrifoodpy/land/model.py b/agrifoodpy/land/model.py index 0021dc8..5db6a4e 100644 --- a/agrifoodpy/land/model.py +++ b/agrifoodpy/land/model.py @@ -3,7 +3,7 @@ """ import numpy as np -from agrifoodpy.land.land import LandDataArray +from ..land.land import LandDataArray def land_sequestration(land_da, use_id, fraction, max_seq, years=None, growth_timescale=10, growth="linear", ha_per_pixel=1): diff --git a/agrifoodpy/pipeline/__init__.py b/agrifoodpy/pipeline/__init__.py index 82e5e72..26952e7 100644 --- a/agrifoodpy/pipeline/__init__.py +++ b/agrifoodpy/pipeline/__init__.py @@ -3,4 +3,4 @@ """ from .pipeline import * -from .utils import * \ No newline at end of file +from ..utils.dict_utils import * \ No newline at end of file diff --git a/agrifoodpy/pipeline/pipeline.py b/agrifoodpy/pipeline/pipeline.py index 344dc3b..963faf0 100644 --- a/agrifoodpy/pipeline/pipeline.py +++ b/agrifoodpy/pipeline/pipeline.py @@ -114,7 +114,7 @@ def run(self, from_node=0, to_node=None, timing=False): node_time = node_end_time - node_start_time if timing: - print(f"Node {i + 1} ({self.names[i]}) executed in {node_time:.4f} seconds.") + print(f"Node {i + 1}: {self.names[i]}, executed in {node_time:.4f} seconds.") pipeline_end_time = time.time() pipeline_time = pipeline_end_time - pipeline_start_time @@ -132,9 +132,9 @@ def standalone(input_keys, return_keys): Parameters ---------- - key_list: list of strings + input_keys: list of strings List of dataset keys to be added to the temporary datablock - return_list: list of strings + return_keys: list of strings List of keys to datablock datasets to be returned by the decorated function. @@ -174,7 +174,12 @@ def wrapper(*args, **kwargs): for key in input_keys: if kwargs.get(key, None) is not None: kwargs[key] = key - + + # Fill return keys if they are not passed or are None + for key in return_keys: + if kwargs.get(key, None) is None: + kwargs[key] = key + result = test_func(**kwargs) # return tuple of results @@ -182,7 +187,7 @@ def wrapper(*args, **kwargs): if len(return_keys) == 1: return result[kwargs[return_keys[0]]] else: - return tuple(kwargs[result[key]] for key in return_keys) + return tuple(result[key] for key in return_keys) return result return wrapper diff --git a/agrifoodpy/pipeline/tests/test_pipeline.py b/agrifoodpy/pipeline/tests/test_pipeline.py index 9692862..1562c91 100644 --- a/agrifoodpy/pipeline/tests/test_pipeline.py +++ b/agrifoodpy/pipeline/tests/test_pipeline.py @@ -101,3 +101,58 @@ def test_datablock_write(): assert(pipeline.datablock['a']['b']['d'] == 20) assert(pipeline.datablock['a']['e'] == 30) assert(pipeline.datablock['f'] == 40) + +def test_standalone_decorator(): + + from agrifoodpy.pipeline.pipeline import Pipeline, standalone + + test_datablock = {'x': 5, 'y': 10} + + # Test decorated function with single input key + @standalone(['x'], ['out_key']) + def double_numbers_decorated(x, out_key, datablock=None): + datablock[out_key] = datablock[x]*2 + return datablock + + result_double = double_numbers_decorated(5) + assert result_double == 10 + + # Test decorated function with multiple input keys + @standalone(['x', 'y'], ['out_key']) + def sum_numbers_decorated(x, y, out_key, datablock=None): + datablock[out_key] = datablock[x] + datablock[y] + return datablock + + result_sum = sum_numbers_decorated(5, 10) + assert result_sum == 15 + + # Test decorated function with no input keys + @standalone([], ['out_key']) + def return_constant_decorated(out_key, datablock=None): + datablock[out_key] = 42 + return datablock + + result_constant = return_constant_decorated() + assert result_constant == 42 + + # Test decorated function with multiple return keys + @standalone(['x'], ['out_key1', 'out_key2']) + def multiple_returns_decorated(x, out_key1, out_key2, datablock=None): + datablock[out_key1] = datablock[x] * 2 + datablock[out_key2] = datablock[x] + 10 + return datablock + + result_multiple = multiple_returns_decorated(5) + assert result_multiple[0] == 10 + assert result_multiple[1] == 15 + + # Test decorated function inside a pipeline + pipeline = Pipeline(test_datablock) + @standalone(['x'], ['out_key']) + def pipeline_decorated(x, out_key, datablock=None): + datablock[out_key] = datablock[x] * 3 + return datablock + + pipeline.add_node(pipeline_decorated, params={'x': 'x', 'out_key': 'result'}) + pipeline.run() + assert pipeline.datablock['result'] == 15 \ No newline at end of file diff --git a/agrifoodpy/pipeline/tests/test_utils.py b/agrifoodpy/pipeline/tests/test_utils.py deleted file mode 100644 index 603cfba..0000000 --- a/agrifoodpy/pipeline/tests/test_utils.py +++ /dev/null @@ -1,46 +0,0 @@ -import numpy as np -import xarray as xr - -from agrifoodpy.pipeline.utils import item_parser - -def test_item_parser_none(): - - items = ["Beef", "Apples", "Poultry"] - item_origin = ["Animal", "Vegetal", "Animal"] - - data = np.random.rand(3, 2, 2) - - fbs = xr.Dataset({"data": (("Item", "X", "Y"), data)}, - coords={"Item": items, "X": [0, 1], "Y": [0, 1]}) - fbs = fbs.assign_coords({"Item_origin":("Item", item_origin)}) - - items = item_parser(fbs, None) - assert items is None - -def test_item_parser_tuple(): - items = ["Beef", "Apples", "Poultry"] - item_origin = ["Animal", "Vegetal", "Animal"] - - data = np.random.rand(3, 2, 2) - - fbs = xr.Dataset({"data": (("Item", "X", "Y"), data)}, - coords={"Item": items, "X": [0, 1], "Y": [0, 1]}) - fbs = fbs.assign_coords({"Item_origin":("Item", item_origin)}) - - items = item_parser(fbs, ("Item_origin", ["Animal"])) - assert np.array_equal(items, ["Beef", "Poultry"]) - -def test_item_parser_scalar(): - items = ["Beef", "Apples", "Poultry"] - item_origin = ["Animal", "Vegetal", "Animal"] - - data = np.random.rand(3, 2, 2) - - fbs = xr.Dataset({"data": (("Item", "X", "Y"), data)}, - coords={"Item": items, "X": [0, 1], "Y": [0, 1]}) - fbs = fbs.assign_coords({"Item_origin":("Item", item_origin)}) - - items = item_parser(fbs, "Beef") - - assert np.array_equal(items, ["Beef"]) - diff --git a/agrifoodpy/population/population.py b/agrifoodpy/population/population.py index 36be12d..5637135 100644 --- a/agrifoodpy/population/population.py +++ b/agrifoodpy/population/population.py @@ -4,7 +4,7 @@ import numpy as np import xarray as xr -from agrifoodpy.array_accessor import XarrayAccessorBase +from ..array_accessor import XarrayAccessorBase def population(years, regions, quantities, datasets=None, long_format=True): """Population style dataset constructor diff --git a/agrifoodpy/tests/test_base_class.py b/agrifoodpy/tests/test_base_class.py index b385b80..f2a062a 100644 --- a/agrifoodpy/tests/test_base_class.py +++ b/agrifoodpy/tests/test_base_class.py @@ -1,6 +1,6 @@ import numpy as np import xarray as xr -from agrifoodpy.array_accessor import XarrayAccessorBase +from ..array_accessor import XarrayAccessorBase def test_add_years(): diff --git a/agrifoodpy/utils/add_items.py b/agrifoodpy/utils/add_items.py index 296887d..4c26dc4 100644 --- a/agrifoodpy/utils/add_items.py +++ b/agrifoodpy/utils/add_items.py @@ -2,12 +2,18 @@ import numpy as np import copy -from agrifoodpy.pipeline import standalone -from agrifoodpy.pipeline.utils import get_dict_path, set_dict_path -from agrifoodpy.food import food +from ..pipeline import standalone +from ..utils.dict_utils import get_dict, set_dict +from ..food import food @standalone(["dataset"], ["dataset"]) -def add_items(dataset, items, values=None, copy_from=None, datablock=None): +def add_items( + dataset, + items, + values=None, + copy_from=None, + datablock=None +): """Adds a list of items to a selected dataset in the datablock and initializes their values. @@ -46,7 +52,7 @@ def add_items(dataset, items, values=None, copy_from=None, datablock=None): items_src = {} # Add new items to the datasets - data = get_dict_path(datablock, dataset) + data = get_dict(datablock, dataset) data = data.fbs.add_items(new_items, copy_from=copy_from) for key, val in items_src.items(): @@ -58,6 +64,6 @@ def add_items(dataset, items, values=None, copy_from=None, datablock=None): elif copy_from is None: data.loc[{"Item":new_items}] = 0 - datablock = set_dict_path(datablock, dataset, data) + set_dict(datablock, dataset, data) return datablock diff --git a/agrifoodpy/utils/extend_years.py b/agrifoodpy/utils/add_years.py similarity index 63% rename from agrifoodpy/utils/extend_years.py rename to agrifoodpy/utils/add_years.py index 1f2251d..0835a36 100644 --- a/agrifoodpy/utils/extend_years.py +++ b/agrifoodpy/utils/add_years.py @@ -1,10 +1,17 @@ -from agrifoodpy.pipeline import standalone -from agrifoodpy.impact.impact import Impact +from ..pipeline import standalone +from ..impact.impact import Impact +from ..food.food import FoodBalanceSheet +from .dict_utils import get_dict, set_dict @standalone(["dataset"], ["dataset"]) -def extend_years(dataset, years, projection='empty', datablock=None): +def add_years( + dataset, + years, + projection='empty', + datablock=None +): """ - Extends the dimensions of a dataset. + Extends the Year coordinates of a dataset. Parameters ---------- @@ -21,9 +28,10 @@ def extend_years(dataset, years, projection='empty', datablock=None): the new year using a scaling of the last year of the array """ - data = datablock[dataset].copy(deep=True) + data = get_dict(datablock, dataset) data = data.fbs.add_years(years, projection) - datablock[dataset] = data - return dataset + set_dict(datablock, dataset, data) + + return datablock diff --git a/agrifoodpy/pipeline/utils.py b/agrifoodpy/utils/dict_utils.py similarity index 73% rename from agrifoodpy/pipeline/utils.py rename to agrifoodpy/utils/dict_utils.py index b6c63e6..585f379 100644 --- a/agrifoodpy/pipeline/utils.py +++ b/agrifoodpy/utils/dict_utils.py @@ -9,9 +9,8 @@ def item_parser(fbs, items): Parameters ---------- - fbs : xarray.Dataset - The FBS dataset - + fbs : xarray.Dataset or xarray.DataArray + The dataset containing the coordinate-key to extract items from. items : tuple, scalar If a tuple, the first element is the name of the coordinate and the second element is a list of items to extract. If a scalar, the item @@ -34,7 +33,7 @@ def item_parser(fbs, items): return items -def get_dict_path(datablock, keys): +def get_dict(datablock, keys): """Returns an element from a dictionary using a key or tuple of keys used to describe a path of keys @@ -43,7 +42,6 @@ def get_dict_path(datablock, keys): datablock : dict The input dictionary - keys : str or tuple Dictionary key, or tuple of keys """ @@ -57,7 +55,7 @@ def get_dict_path(datablock, keys): return out -def set_dict_path(datablock, keys, object): +def set_dict(datablock, keys, object, create_missing=True): """Sets an element in a dictionary using a key or tuple of keys used to describe a path of keys @@ -66,21 +64,28 @@ def set_dict_path(datablock, keys, object): datablock : dict The input dictionary - keys : str or tuple Dictionary key, or tuple of keys - object : any The object to set in the dictionary + create_missing : bool, optional + If True, creates missing keys in the dictionary. Defaults to True. + + Raises + ------ + KeyError + If a key in the path does not exist and create_missing is False. """ if isinstance(keys, tuple): out = datablock for key in keys[:-1]: + if key not in out: + if create_missing: + out[key] = {} + else: + raise KeyError(f"Key '{key}' not found in datablock.") out = out[key] out[keys[-1]] = object else: datablock[keys] = object - - return datablock - diff --git a/agrifoodpy/utils/load_dataset.py b/agrifoodpy/utils/load_dataset.py index 2e0d0ad..967ea98 100644 --- a/agrifoodpy/utils/load_dataset.py +++ b/agrifoodpy/utils/load_dataset.py @@ -2,27 +2,30 @@ import xarray as xr import importlib -from agrifoodpy.pipeline import standalone +from ..pipeline import standalone -def import_dataset(module_name, dataset_name): +def _import_dataset(module_name, dataset_name): module = importlib.import_module(module_name) dataset = getattr(module, dataset_name) return dataset @standalone([], ['datablock_path']) -def load_dataset(datablock_path="data", path=None, module=None, - data_attr=None, da=None, coords=None, scale=1., - datablock=None): - """Loads a dataset to the specified datablock dictionary. Can only be used - in pipeline mode. +def load_dataset( + datablock_path, + path=None, + module=None, + data_attr=None, + da=None, + coords=None, + scale=1., + datablock=None +): + """Loads a dataset to the specified datablock dictionary. Parameters ---------- datablock : dict - The datablock dictionary where the dataset is stored, containing all - the model parameters - datablock_path : str - The path to the datablock where the dataset is stored. + The datablock path where the dataset is stored path : str The path to the dataset stored in a netCDF file. module : str @@ -34,7 +37,9 @@ def load_dataset(datablock_path="data", path=None, module=None, coords : dict Dictionary containing the coordinates of the dataset to be loaded. scale : float - Optional scale factor to be applied to the dataset on load. + Optional multiplicative factor to be applied to the dataset on load. + datablock_path : str + The path to the datablock where the dataset is stored. """ @@ -50,7 +55,7 @@ def load_dataset(datablock_path="data", path=None, module=None, # Load dataset from module elif module is not None and data_attr is not None: - dataset = import_dataset(module, data_attr) + dataset = _import_dataset(module, data_attr) # Select dataarray and coords from dataset if da is not None: diff --git a/agrifoodpy/utils/print_datablock.py b/agrifoodpy/utils/print_datablock.py index 01018b8..27cecf7 100644 --- a/agrifoodpy/utils/print_datablock.py +++ b/agrifoodpy/utils/print_datablock.py @@ -1,21 +1,65 @@ -def print_datablock(datablock, key, sel={}): - """Prints a datablock element at any point in the pipeline execution +def print_datablock( + datablock, + key, + attr=None, + method=None, + args=None, + kwargs=None, + preffix="", + suffix="" +): + """Prints a datablock element or its attributes/methods at any point in the + pipeline execution. Parameters ---------- - datablock : xarray.Dataset - The datablock to print + datablock : dict + The datablock to print from. key : str - The key of the datablock to print - sel : dict, optional - The selection to apply to the datablock + The key of the datablock to print. + attr : str, optional + Name of an attribute of the object to print. + method : str, optional + Name of a method of the object to call and print. + args : list, optional + Positional arguments for the method call. + kwargs : dict, optional + Keyword arguments for the method call. Returns ------- - datablock : xarray.Dataset - Unmodified datablock to continue execution + datablock : dict + Unmodified datablock to continue execution. """ + obj = datablock[key] - print(datablock[key].sel(sel)) + # Extract attribute + if attr is not None: + if hasattr(obj, attr): + obj = getattr(obj, attr) + else: + print(f"Object has no attribute '{attr}'") + return datablock - return datablock + # Call method + if method is not None: + if hasattr(obj, method): + func = getattr(obj, method) + if callable(func): + args = args or [] + kwargs = kwargs or {} + try: + obj = func(*args, **kwargs) + except Exception as e: + print(f"Error calling {method} on {key}: {e}") + return datablock + else: + print(f"'{method}' is not callable on {key}") + return datablock + else: + print(f"Object has no method '{method}'") + return datablock + + # Final print + print(f"{preffix}{obj}{suffix}") + return datablock \ No newline at end of file diff --git a/agrifoodpy/utils/tests/data/generate_dataset.py b/agrifoodpy/utils/tests/data/generate_dataset.py new file mode 100644 index 0000000..ad7db80 --- /dev/null +++ b/agrifoodpy/utils/tests/data/generate_dataset.py @@ -0,0 +1,13 @@ +import xarray as xr +import numpy as np + +items = ["Beef", "Apples", "Poultry"] + +shape = (3, 2, 2) + +data = np.reshape(np.arange(np.prod(shape)), shape) + +ds = xr.Dataset({"data": (("Item", "X", "Y"), data)}, + coords={"Item": items, "X": [0, 1], "Y": [0, 1]}) + +xr.Dataset.to_netcdf(ds, "test_dataset.nc") \ No newline at end of file diff --git a/agrifoodpy/utils/tests/data/test_dataset.nc b/agrifoodpy/utils/tests/data/test_dataset.nc new file mode 100644 index 0000000000000000000000000000000000000000..ad1cefa9d41d5db60e538562267dff88f3aaa095 GIT binary patch literal 8368 zcmeI1&1(}u6u{qPlQhQdhZ-&Y5SNMywuV;HB0|BKY;CknV&kC&AuY`ogK1Kdw9t!q zQVJfVH*X$1cn}mRcn~~z@*gND9t1sk5IyKYFLmC`yd=g}S}gJ43~b)c%$t`tzj-q| zv%4}H9clJ=`dvOBP})%Gi6u+=RmHuh58D%?(FvEi=IRk;Mw6k_J44}syi~)Lum*@caz;g(1jg!J#w61Otoa{nM{pn?_<>_EI zp!bFc>bX$f45OX_0WkRm>9|v#&CG=&g+eit&Cisb60~vql7R*2uylvq!sY;GFXj#K zayPkdWGtLY!2w=@E+Uv{Y}}5gVu^S<6$y{oU>?D813a=pJUng-edLMy$eN4Q*>5)_ zGP?$uA}tiT5*Q|eK|f^-odZvcM1^^!eN!TOpb)KML!KIf2$Ag^QTOJEus?WB3X=wh z*un>oQ>Vd;X6Lh+Ux$sW%VB%=eSJ{^4eS!oAvJ9p8gtnHT?abFjjo)U3jdj-45t_E z?6Bl`hp`ZE2ri>Km!(c6^!O0T^jrD10@t@6TN?~4X=Pf9O6VF}&+Zti?r3(Ct0d6? zhK?Yfle4^)PiwwdROj?qEOiAsxE4-#TxrF;+hlPQL8z83)UVBV*5$i+9>*A$iETwV z*N0^C;(8)7dPz?9ak(AM#s zdGUlA4h7Ey2l}pM<_3ocf&-!8;HU0ax!Qr6ws#K&d~3>%AY2)p~T_k~J#+ z8|V=Y%-Ye7+&si`Bs&u7X!({ zVy;|VuI$(7SRP(j$niY}>{I%32L^E7aptfHcc+-hA3FHsE)^4%)3G0StC*;q4*jwe zz#_+}Kub+P6VL=S0Zl*?&;&FAO+XXS1T+Cn;6EflUpMGgHGO}gb2)uzr}I0V@9Dg~ kM=|p&xmQWiUcGV0v%YAr@+gEIm9IUblRwIg`g{uT6R638SO5S3 literal 0 HcmV?d00001 diff --git a/agrifoodpy/utils/tests/test_add_items.py b/agrifoodpy/utils/tests/test_add_items.py new file mode 100644 index 0000000..6cf57e4 --- /dev/null +++ b/agrifoodpy/utils/tests/test_add_items.py @@ -0,0 +1,55 @@ +import numpy as np +import xarray as xr + +def test_add_items(): + + from agrifoodpy.utils.add_items import add_items + + items = ["Beef", "Apples", "Poultry"] + item_origin = ["Animal", "Vegetal", "Animal"] + new_items = ["Tomatoes", "Potatoes", "Eggs"] + + data = np.random.rand(3, 2, 2) + expected_items = np.concatenate([items, new_items]) + + ds = xr.Dataset({"data": (("Item", "X", "Y"), data)}, + coords={"Item": items, "X": [0, 1], "Y": [0, 1]}) + ds = ds.assign_coords({"Item_origin":("Item", item_origin)}) + + # Test basic functionality + result_add = add_items(ds, new_items) + + assert np.array_equal(result_add["Item"].values, expected_items) + for item in new_items: + assert np.all(result_add["data"].sel(Item=item).values == 0) + + # Test copying from a single existing item + result_copy = add_items(ds, new_items, copy_from="Beef") + + assert np.array_equal(result_copy["Item"], expected_items) + for item_i in new_items: + assert np.array_equal(result_copy["data"].sel(Item=item_i), + ds.data.sel(Item="Beef")) + + # Test copying from multiple existing items + result_copy_multiple = add_items(ds, new_items, copy_from=["Beef", + "Apples", + "Poultry"]) + + assert np.array_equal(result_copy_multiple["Item"], expected_items) + assert np.array_equal(result_copy_multiple["data"].sel(Item=new_items), + ds.data.sel(Item=["Beef", "Apples", "Poultry"])) + + # Test providing values as dictionary + new_items_dict = { + "Item": new_items, + "Item_origin": ["Vegetal", "Vegetal", "Animal"], + } + + result_dict = add_items(ds, new_items_dict) + + assert np.array_equal(result_dict["Item"].values, expected_items) + assert np.array_equal(result_dict["Item_origin"].values, + ["Animal", "Vegetal", "Animal", "Vegetal", "Vegetal", "Animal"]) + for item in new_items: + assert np.all(result_dict["data"].sel(Item=item).values == 0) diff --git a/agrifoodpy/utils/tests/test_add_years.py b/agrifoodpy/utils/tests/test_add_years.py new file mode 100644 index 0000000..964cf0e --- /dev/null +++ b/agrifoodpy/utils/tests/test_add_years.py @@ -0,0 +1,39 @@ +import numpy as np +import xarray as xr + +def test_add_items(): + + from agrifoodpy.utils.add_years import add_years + + items = ["Beef", "Apples", "Poultry"] + years = [2010, 2011, 2012] + + shape = (3, 3) + data = np.reshape(np.arange(np.prod(shape)), shape) + + ds = xr.Dataset({"data": (("Item", "Year"), data)}, + coords={"Item": items, "Year": years}) + + # Test basic functionality + new_years = [2013, 2014] + result_add = add_years(ds, new_years) + expected_years = years + new_years + assert np.array_equal(result_add["Year"].values, expected_years) + for year in new_years: + assert np.all(np.isnan(result_add["data"].sel(Year=year).values)) + + # Test projection mode 'constant' + result_constant = add_years(ds, new_years, projection='constant') + assert np.array_equal(result_constant["Year"].values, expected_years) + for year in new_years: + assert np.array_equal(result_constant["data"].sel(Year=year).values, + ds.data.isel(Year=-1).values) + + # Test projection mode with float array + scaling_factors = np.array([1.0, 2.0]) + result_scaled = add_years(ds, new_years, projection=scaling_factors) + assert np.array_equal(result_scaled["Year"].values, expected_years) + for i, year in enumerate(new_years): + expected_values = ds.data.isel(Year=-1).values * scaling_factors[i] + assert np.array_equal(result_scaled["data"].sel(Year=year).values, + expected_values) \ No newline at end of file diff --git a/agrifoodpy/utils/tests/test_copy_datablock.py b/agrifoodpy/utils/tests/test_copy_datablock.py new file mode 100644 index 0000000..9c5743b --- /dev/null +++ b/agrifoodpy/utils/tests/test_copy_datablock.py @@ -0,0 +1,36 @@ +import numpy as np + +def test_copy_datablock(): + + from agrifoodpy.pipeline import Pipeline + from agrifoodpy.utils.copy_datablock import copy_datablock + + datablock = { + 'test_dataset': { + 'fbs': { + 'data': np.array([[1, 2], [3, 4]]), + 'years': [2020, 2021] + } + } + } + + # Test copying the datablock + pipeline = Pipeline(datablock=datablock) + pipeline.add_node( + copy_datablock, + params={ + 'key': 'test_dataset', + 'out_key': 'copied_dataset' + }, + ) + + # Execute the pipeline + pipeline.run() + + # Check if the copied dataset exists in the datablock + assert 'copied_dataset' in pipeline.datablock + assert np.array_equal( + pipeline.datablock['copied_dataset']['fbs']['data'], + datablock['test_dataset']['fbs']['data'] + ) + diff --git a/agrifoodpy/utils/tests/test_dict_utils.py b/agrifoodpy/utils/tests/test_dict_utils.py new file mode 100644 index 0000000..662649e --- /dev/null +++ b/agrifoodpy/utils/tests/test_dict_utils.py @@ -0,0 +1,81 @@ +import numpy as np +import xarray as xr + + +def test_item_parser(): + + from agrifoodpy.utils.dict_utils import item_parser + + items = ["Beef", "Apples", "Poultry"] + item_origin = ["Animal", "Vegetal", "Animal"] + + data = np.random.rand(3, 2, 2) + + fbs = xr.Dataset({"data": (("Item", "X", "Y"), data)}, + coords={"Item": items, "X": [0, 1], "Y": [0, 1]}) + fbs = fbs.assign_coords({"Item_origin":("Item", item_origin)}) + + # Test case for item_parser with None input + items = item_parser(fbs, None) + assert items is None + + # Test case for item_parser with tuple input + items = item_parser(fbs, ("Item_origin", ["Animal"])) + assert np.array_equal(items, ["Beef", "Poultry"]) + + # Tesct case for item_parser with scalar input + items = item_parser(fbs, "Beef") + assert np.array_equal(items, ["Beef"]) + +def test_get_dict(): + + from agrifoodpy.utils.dict_utils import get_dict + + datablock = { + "key1": { + "key2": { + "key3": "value" + } + }, + "key4": "another_value" + } + + # Test case for get_dict with a single key + value = get_dict(datablock, "key4") + assert value == "another_value" + + # Test case for get_dict with a tuple of keys + value = get_dict(datablock, ("key1", "key2", "key3")) + assert value == "value" + +def test_set_dict(): + + from agrifoodpy.utils.dict_utils import set_dict + + datablock = { + "key1": { + "key2": { + "key3": "value" + } + }, + "key4": "another_value" + } + + # Test case for set_dict with a single key + set_dict(datablock, "key4", "new_value") + assert datablock["key4"] == "new_value" + + # Test case for set_dict with a tuple of keys + set_dict(datablock, ("key1", "key2", "key3"), "new_nested_value") + assert datablock["key1"]["key2"]["key3"] == "new_nested_value" + + # Test case for set_dict with create_missing=True + set_dict(datablock, ("key5", "key6"), "missing_value", create_missing=True) + assert datablock["key5"]["key6"] == "missing_value" + + # Test case for set_dict with create_missing=False + try: + set_dict(datablock, ("key7", "key8"), "should_fail", create_missing=False) + except KeyError: + pass + \ No newline at end of file diff --git a/agrifoodpy/utils/tests/test_load_dataset.py b/agrifoodpy/utils/tests/test_load_dataset.py new file mode 100644 index 0000000..7ee1ca9 --- /dev/null +++ b/agrifoodpy/utils/tests/test_load_dataset.py @@ -0,0 +1,39 @@ +import numpy as np +import xarray as xr + +from agrifoodpy.utils.load_dataset import load_dataset +import os + +def test_load_dataset(): + + items = ["Beef", "Apples", "Poultry"] + shape = (3, 2, 2) + data = np.reshape(np.arange(np.prod(shape)), shape) + + expected_ds = xr.Dataset({"data": (("Item", "X", "Y"), data)}, + coords={"Item": items, "X": [0, 1], "Y": [0, 1]}) + + script_dir = os.path.dirname(__file__) + test_data_path = os.path.join(script_dir, "data/test_dataset.nc") + + # Test loading a dataset from a file path + ds = load_dataset(path=test_data_path) + assert isinstance(ds, xr.Dataset) + assert ds.equals(expected_ds) + + # Test loading a dataset and selecting a specific dataarray + ds_dataarray = load_dataset(path=test_data_path, da="data") + assert isinstance(ds_dataarray, xr.DataArray) + assert ds_dataarray.equals(expected_ds["data"]) + + # Test loading a dataset and selecting specific items + sel = {"Item": ["Beef", "Apples"]} + ds_selected = load_dataset(path=test_data_path, coords=sel) + expected_selected_ds = expected_ds.sel(sel) + assert ds_selected.equals(expected_selected_ds) + + # Test loading a dataset and applying a scale factor + scaled_ds = load_dataset(path=test_data_path, scale=2.0) + expected_scaled_data = expected_ds * 2.0 + assert scaled_ds.equals(expected_scaled_data) + diff --git a/agrifoodpy/utils/tests/test_print_datablock.py b/agrifoodpy/utils/tests/test_print_datablock.py new file mode 100644 index 0000000..b43d80e --- /dev/null +++ b/agrifoodpy/utils/tests/test_print_datablock.py @@ -0,0 +1,53 @@ +import numpy as np +import xarray as xr + +def test_print_datablock(): + + from agrifoodpy.utils.print_datablock import print_datablock + + items = ["Beef", "Apples", "Poultry"] + + data = np.random.rand(3, 2, 2) + + ds = xr.Dataset({"data": (("Item", "X", "Y"), data)}, + coords={"Item": items, "X": [0, 1], "Y": [0, 1]}) + + datablock = { + 'test_dict': { + 'data': np.array([[1, 2], [3, 4]]), + 'years': [2020, 2021] + }, + + 'test_xarray': ds, + 'test_string': "Hello, World!", + 'test_list': items, + 'test_array': data + + } + + # Test printing a dictionary element + datablock = print_datablock(datablock, 'test_dict') + + # Test printing an xarray Dataset + datablock = print_datablock(datablock, 'test_xarray') + + # Test printing a string + datablock = print_datablock(datablock, 'test_string') + + # Test printing a list + datablock = print_datablock(datablock, 'test_list') + + # Test printing an array + datablock = print_datablock(datablock, 'test_array') + + # Test printing an attribute of the xarray Dataset + datablock = print_datablock(datablock, 'test_xarray', attr='data_vars') + + # Test calling a method on the xarray Dataset + datablock = print_datablock(datablock, 'test_xarray', method='mean', args=[('X', 'Y')]) + + # Test calling a method with keyword arguments + datablock = print_datablock(datablock, 'test_xarray', method='sel', kwargs={'Item': 'Beef'}) + + # Test error handling for non-existent attribute + datablock = print_datablock(datablock, 'test_xarray', attr='non_existent_attr') diff --git a/agrifoodpy/utils/tests/test_write_to_datablock.py b/agrifoodpy/utils/tests/test_write_to_datablock.py new file mode 100644 index 0000000..237590e --- /dev/null +++ b/agrifoodpy/utils/tests/test_write_to_datablock.py @@ -0,0 +1,34 @@ +def test_write_to_datablock(): + from agrifoodpy.utils.write_to_datablock import write_to_datablock + + datablock_basic = {} + + # Basic write to the datablock + write_to_datablock(datablock_basic, "test_key", "test_value") + assert datablock_basic["test_key"] == "test_value" + + # Write to the datablock with a tuple value + datablock_tuple = {} + write_to_datablock( + datablock=datablock_tuple, + key=("test_key_1", "test_key_2"), + value="test_tuple_value" + ) + assert datablock_tuple["test_key_1"]["test_key_2"] == "test_tuple_value" + + # Overwrite existing key + datablock_overwrite = {"test_key": "old_value"} + write_to_datablock(datablock_overwrite, "test_key", "new_value") + assert datablock_overwrite["test_key"] == "new_value" + + # Attempt to write without overwriting an existing key + datablock_no_overwrite = {"test_key": "existing_value"} + try: + write_to_datablock( + datablock=datablock_no_overwrite, + key="test_key", + value="new_value", + overwrite=False) + + except KeyError as e: + assert str(e) == "'Key already exists in datablock and overwrite is set to False.'" \ No newline at end of file diff --git a/agrifoodpy/utils/write_to_datablock.py b/agrifoodpy/utils/write_to_datablock.py new file mode 100644 index 0000000..3ff5506 --- /dev/null +++ b/agrifoodpy/utils/write_to_datablock.py @@ -0,0 +1,29 @@ +from .dict_utils import set_dict + +def write_to_datablock(datablock, key, value, overwrite=True): + """Writes a value to a specified key in the datablock. + + Parameters + ---------- + datablock : dict + The datablock to write to. + key : str + The key in the datablock where the value will be written. + value : any + The value to write to the datablock. + overwrite : bool, optional + If True, overwrite the existing value at the key. + If False, do not overwrite. + + Returns + ------- + datablock : dict + The updated datablock with the new key-value pair. + """ + + if not overwrite and key in datablock: + raise KeyError(f"Key already exists in datablock and overwrite is set to False.") + + set_dict(datablock, key, value) + + return datablock \ No newline at end of file diff --git a/examples/modules/plot_animal_consumption_scaling.py b/examples/modules/plot_animal_consumption_scaling.py index 7caede3..c2e2829 100644 --- a/examples/modules/plot_animal_consumption_scaling.py +++ b/examples/modules/plot_animal_consumption_scaling.py @@ -2,10 +2,10 @@ ======================================== Reducing UK's animal product consumption ======================================== - -This example demonstrates how to combine a food supply array with a scaling -model to reduce animal product consumption. -It also employs a few functions from the ``fbs`` accessor to group and plot +This example demonstrates how to combine a Food Balance Sheet array with a +scaling model to reduce animal product consumption, while keeping total +consumption constant across all items. +It also employs a few ``fbs`` accessor class functions to group and plot food balance sheet arrays. Consumption of animal based products is halved, while keeping total comsumed @@ -13,31 +13,33 @@ """ import numpy as np +from matplotlib import pyplot as plt from agrifoodpy_data.food import FAOSTAT - -import agrifoodpy.food from agrifoodpy.food.model import balanced_scaling - -from matplotlib import pyplot as plt - # Select food items and production values for the last year of data in the UK # Values are in 1000 Tonnes country_code = 229 food_uk = FAOSTAT.isel(Year=-1).sel(Region=country_code) +# Select all Animal Products according to its Item_origin, which is a +# non-dimension coordinate animal_items = food_uk.sel(Item=food_uk.Item_origin=="Animal Products").Item.values - # Scale domestic use of animal items by a factor of 0.5, while keeping # the sum of domestic use constant. Reduce imports to account for the new -# consumption values -food_uk_scaled = balanced_scaling(food_uk, - element="domestic", - items=animal_items, - scale=0.5, - origin="-imports", - fallback="-exports", - constant=True) +# consumption values and use production as fallback, subtracting any negative +# excess if required + +food_uk_scaled = balanced_scaling( + food_uk, + element="domestic", + scale=0.5, + items=animal_items, + constant=True, + origin="imports", + fallback="production", + add_to_fallback=False +) # We group the original and scaled quantities by origin and plot to compare food_uk_origin = food_uk.fbs.group_sum("Item_origin") @@ -45,8 +47,8 @@ #%% # From the plot we can see that domestic use of animal products is reduced by -# half, while keeping total weight constant. We used ``-exports`` as the -# fallback for any extra origin required. If any item domestic use reduction +# half, while keeping total weight constant. We used ``production`` as the +# fallback for any extra origin required. If any domestic use reduction # requires more origin reduction than available, the remaining is taken from # the ``fallback`` DataArray element. diff --git a/examples/modules/plot_emissions_animal_scaling.py b/examples/modules/plot_emissions_animal_scaling.py index 37654af..0c4680c 100644 --- a/examples/modules/plot_emissions_animal_scaling.py +++ b/examples/modules/plot_emissions_animal_scaling.py @@ -2,10 +2,9 @@ =================================================== Plot emissions from different item groups and years =================================================== - -This example demonstrates how manipulate a Food Balance Sheet array, add +This example demonstrates how to manipulate a Food Balance Sheet array, add items and years to it and combine it with impact data to plot total GHG -emissions dissagregated by selected coordinates. +emissions dissagregated by a selected coordinate. Two datasets are imported from the agrifoodpy_data package: @@ -24,12 +23,12 @@ from matplotlib import pyplot as plt -# Load FAOSTAT array to memory. -FAOSTAT.load(); - # Select food items and production values for the UK and the US # Values are in [1000 Tonnes] -country_codes = [229, 231] +UK_FAO_CODE = 229 +US_FAO_CODE = 231 + +country_codes = [UK_FAO_CODE, US_FAO_CODE] food = FAOSTAT.sel(Region=country_codes)["production"] # Convert emissions from [g CO2e] to [Gt CO2e] diff --git a/examples/modules/plot_reforest_ALC_4_5.py b/examples/modules/plot_reforest_ALC_4_5.py index ea9ff97..d995d06 100644 --- a/examples/modules/plot_reforest_ALC_4_5.py +++ b/examples/modules/plot_reforest_ALC_4_5.py @@ -56,13 +56,18 @@ coniferous_max_seq = 14 broadleaf_fraction = 0.5 -seq_forest= broadleaf_max_seq * (broadleaf_fraction) + \ +seq_forest = broadleaf_max_seq * (broadleaf_fraction) + \ coniferous_max_seq * (1-broadleaf_fraction) -co2e_seq = land_sequestration(land_use, [1,2], max_seq=seq_forest, - fraction=[0.0, pasture_4_5/total_area_england*0.5], - years = np.arange(2020,2070), - growth_timescale=25) +# We can now compute the additional carbon sequestration from reforesting +co2e_seq = land_sequestration( + land_da=land_use, + use_id=[1,2], + max_seq=seq_forest, + fraction=[0.0, pasture_4_5/total_area_england*0.5], + years = np.arange(2020,2070), + growth_timescale=25 + ) ax = co2e_seq.fbs.plot_years() ax.set_ylabel("[t CO2 / yr]") diff --git a/examples/pipeline/README.rst b/examples/pipeline/README.rst new file mode 100644 index 0000000..38b3bea --- /dev/null +++ b/examples/pipeline/README.rst @@ -0,0 +1,6 @@ +.. _pipeline_examples: + +Pipeline +-------- + +Basic pipeline examples \ No newline at end of file diff --git a/examples/pipeline/plot_balanced_scaling_food_balance_sheet_pipeline.py b/examples/pipeline/plot_balanced_scaling_food_balance_sheet_pipeline.py new file mode 100644 index 0000000..a4b9a64 --- /dev/null +++ b/examples/pipeline/plot_balanced_scaling_food_balance_sheet_pipeline.py @@ -0,0 +1,212 @@ +""" +====================================== +Building a Food Balance Sheet Pipeline +====================================== + +This example demonstrates the use of the pipeline manager to create a simple +pipeline of modules. + +In this particular example, we will load a food balance sheet dataset, compute +prelimimary Self-Sufficiency and Import Dependency Ratios (SSR, IDR) and add new +items and years to the dataset. +We will also print the SSR and IDR values to the console. +Finally, we will scale the food balance sheet to reduce animal products and plot +the results. +""" + +#%% +# We start by creating a pipeline object, which will manage the flow of data +# through the different modules. + +import numpy as np +from matplotlib import pyplot as plt + +from agrifoodpy.pipeline.pipeline import Pipeline +from agrifoodpy.food.food import FoodBalanceSheet + +import agrifoodpy.food.model as afpm +from agrifoodpy.utils.load_dataset import load_dataset +from agrifoodpy.utils.write_to_datablock import write_to_datablock +from agrifoodpy.utils.add_items import add_items +from agrifoodpy.utils.print_datablock import print_datablock +from agrifoodpy.utils.add_items import add_items +from agrifoodpy.utils.add_years import add_years +from agrifoodpy.utils.scaling import linear_scale + + +# Create a pipeline object +pipeline = Pipeline() + +#%% +# We add a node to the pipeline to load a food balance sheet dataset. + +# Load a dataset +pipeline.add_node( + load_dataset, + name="Load Dataset", + params={ + "datablock_path": "food", + "module": "agrifoodpy_data.food", + "data_attr":"FAOSTAT", + "coords":{ + "Item": [2731, 2511], + "Year": [2019, 2020], + "Region": 229,}, + } +) + +#%% +# We add a node to the pipeline to store a conversion factor in the +# datablock. This conversion factor will be used to convert the food balance +# sheet data from 1000 tonnes to kgs. + +# Add convertion factors to the datablock +pipeline.add_node( + write_to_datablock, + name="Write to datablock", + params={ + "key": "tonnes_to_kgs", + "value": 1e6, + } +) + +# Convert food data from 1000 tonnes to kgs +pipeline.add_node( + afpm.fbs_convert, + name="Convert from 1000 tonnes to kgs", + params={ + "fbs": "food", + "convertion_arr": "tonnes_to_kgs", + } +) + +#%% +# Compute preliminary Self-Sufficiency Ratio (SSR) and Import Dependency Ratio (IDR) + + +# Compute IDR and SSR for food +pipeline.add_node( + afpm.SSR, + name="Compute SSR for food", + params={ + "fbs":"food", + "out_key":"SSR" + } +) + +# Compute IDR and SSR for food +pipeline.add_node( + afpm.IDR, + name="Compute IDR for food", + params={ + "fbs":"food", + "out_key":"IDR" + } +) + +#%% +# Print the SSR and IDR values to the console + +# Add a print node to display the SSR +pipeline.add_node( + print_datablock, + name="Print SSR", + params={ + "key": "SSR", + "method": "to_numpy", + "preffix": "SSR values: ", + } +) + +#%% +# Now we can add new items to the food balance sheet dataset. + +# Add an item to the food dataset +pipeline.add_node( + add_items, + name="Add item to food", + params={ + "dataset": "food", + "items": { + "Item":5000, + "Item_name":"Cultured meat", + "Item_group":"Cultured products", + "Item_origin":"Synthetic origin", + }, + "copy_from":2731 + } +) + +#%% +# We can also add new years to the food balance sheet dataset. + +projection = np.linspace(1.1, 2.0, 10) +new_years = np.arange(2021, 2031) + +# Extend the year range of the food dataset +pipeline.add_node( + add_years, + name="Add years to food", + params={ + "dataset": "food", + "years": new_years, + "projection": projection, + } +) + +#%% +# We execute the pipeline to run all the nodes in order. + +pipeline.run(timing=True) + +#%%# Finally, we plot the results + +# Get the food results from the pipeline and plot using the fbs accessor +food_results = pipeline.datablock["food"]["food"] + +f, ax = plt.subplots(figsize=(10, 6)) +food_results.fbs.plot_years(show="Item_name", labels="show", ax=ax) +plt.show() + +#%% +# We can continue adding nodes to the pipeline, even after being executed +# once. To pick up where we left, we indicate which node to start execution from + +# Define a year dependent linear scale starting decreasing at 2021 from 1 to 0.5 +scaling = linear_scale( + 2019, + 2021, + 2030, + 2030, + 1, + 0.5 +) + +# We will add a node to scale consumption +pipeline.add_node( + afpm.balanced_scaling, + name="Balanced scaling of items", + params={ + "fbs":"food", + "scale":scaling, + "element":"food", + "items":("Item_name", "Bovine Meat"), + "constant":True, + "out_key":"food_scaled" + } +) + +# Execute the recently added node +pipeline.run(from_node=8, timing=True) + +# Get the food results from the pipeline and plot using the fbs accessor +scaled_food_results = pipeline.datablock["food_scaled"]["food"] + +f, ax = plt.subplots(figsize=(10, 6)) +scaled_food_results.fbs.plot_years(show="Item_name", labels="show", ax=ax) +plt.show() + +#%% +# We can see in the scaled Food Balance Sheet that Bovine Meat consumption is +# reduced by half by 2030, while the total sum across all items remains +# constant. \ No newline at end of file From bf9a9c3d66e110baeacc51663445fbe1ee00d837 Mon Sep 17 00:00:00 2001 From: jucordero Date: Mon, 1 Sep 2025 13:09:16 +0100 Subject: [PATCH 3/4] Some PEP8 formatting and reworked utils --- agrifoodpy/food/food.py | 196 +++++++----- agrifoodpy/food/model.py | 58 ++-- agrifoodpy/impact/data/PN18.nc | Bin 22288 -> 0 bytes agrifoodpy/impact/data/PN18_FAOSTAT.nc | Bin 13227 -> 0 bytes agrifoodpy/impact/impact.py | 36 ++- agrifoodpy/impact/model.py | 62 ++-- agrifoodpy/land/land.py | 124 +++++--- agrifoodpy/land/model.py | 33 +- agrifoodpy/pipeline/pipeline.py | 34 ++- agrifoodpy/population/population.py | 16 +- agrifoodpy/utils/add_items.py | 69 ----- agrifoodpy/utils/add_years.py | 37 --- agrifoodpy/utils/copy_datablock.py | 23 -- agrifoodpy/utils/dict_utils.py | 17 +- agrifoodpy/utils/load_dataset.py | 70 ----- agrifoodpy/utils/nodes.py | 289 ++++++++++++++++++ agrifoodpy/utils/print_datablock.py | 65 ---- agrifoodpy/utils/scaling.py | 24 +- agrifoodpy/utils/tests/test_add_items.py | 16 +- agrifoodpy/utils/tests/test_add_years.py | 11 +- agrifoodpy/utils/tests/test_copy_datablock.py | 4 +- agrifoodpy/utils/tests/test_dict_utils.py | 14 +- agrifoodpy/utils/tests/test_load_dataset.py | 6 +- .../utils/tests/test_print_datablock.py | 15 +- agrifoodpy/utils/tests/test_scaling.py | 17 +- .../utils/tests/test_write_to_datablock.py | 6 +- agrifoodpy/utils/write_to_datablock.py | 29 -- ...ced_scaling_food_balance_sheet_pipeline.py | 110 ++++--- 28 files changed, 743 insertions(+), 638 deletions(-) delete mode 100644 agrifoodpy/impact/data/PN18.nc delete mode 100644 agrifoodpy/impact/data/PN18_FAOSTAT.nc delete mode 100644 agrifoodpy/utils/add_items.py delete mode 100644 agrifoodpy/utils/add_years.py delete mode 100644 agrifoodpy/utils/copy_datablock.py delete mode 100644 agrifoodpy/utils/load_dataset.py create mode 100644 agrifoodpy/utils/nodes.py delete mode 100644 agrifoodpy/utils/print_datablock.py delete mode 100644 agrifoodpy/utils/write_to_datablock.py diff --git a/agrifoodpy/food/food.py b/agrifoodpy/food/food.py index ee103a6..ee327f2 100644 --- a/agrifoodpy/food/food.py +++ b/agrifoodpy/food/food.py @@ -1,7 +1,7 @@ """ Food supply module. The Food module provides the FoodBalanceSheet and FoodElementSheet accessor -classes to manipulate and analyse Food data stored in xarray.Dataset and +classes to manipulate and analyse Food data stored in xarray.Dataset and xarray.DataArray formats, respectively. It also provides a constructor style function which allows the creation of a @@ -11,20 +11,25 @@ import numpy as np import xarray as xr import copy -import warnings - from ..array_accessor import XarrayAccessorBase import matplotlib.pyplot as plt -def FoodSupply(items, years, quantities, regions=None, elements=None, - long_format=True): + +def FoodSupply( + items, + years, + quantities, + regions=None, + elements=None, + long_format=True +): """ Food Supply style dataset constructor - Constructs a food balance sheet style xarray.Dataset or xarray.DataArray for - a given list of items, years and regions, and an array data shaped - accordingly. - + Constructs a food balance sheet style xarray.Dataset or xarray.DataArray + for a given list of items, years and regions, and an array data shaped + accordingly. + Parameters ---------- items : (ni,) array_like @@ -42,14 +47,15 @@ def FoodSupply(items, years, quantities, regions=None, elements=None, Boolean flag to interpret data in long or wide format elements : (ne,) array_like, optional Array with element name strings. If `elements` is provided, a dataset - is created for each element in `elements` with the quantities being each - of the sub-arrays indexed by the first coordinate of the input array. + is created for each element in `elements` with the quantities being + each of the sub-arrays indexed by the first coordinate of the input + array. Returns ------- fbs : xarray.Dataset - Food Supply dataset containing the food quantity for each `Item`, `Year` - and `Region` with one dataarray per element in `elements`. + Food Supply dataset containing the food quantity for each `Item`, + `Year` and `Region` with one dataarray per element in `elements`. """ # if the input has a single element, proceed with long format @@ -61,14 +67,14 @@ def FoodSupply(items, years, quantities, regions=None, elements=None, # Identify unique values in coordinates _items = np.unique(items) _years = np.unique(years) - coords = {"Item" : _items, - "Year" : _years,} + coords = {"Item": _items, + "Year": _years} # find positions in output array to organize data ii = [np.searchsorted(_items, items), np.searchsorted(_years, years)] size = (len(_items), len(_years)) - # If regions and are provided, add the coordinate information + # If regions and are provided, add the coordinate information if regions is not None: _regions = np.unique(regions) ii.append(np.searchsorted(_regions, regions)) @@ -76,7 +82,7 @@ def FoodSupply(items, years, quantities, regions=None, elements=None, coords["Region"] = _regions # Create empty dataset - fbs = xr.Dataset(coords = coords) + fbs = xr.Dataset(coords=coords) if long_format: # dataset, quantities @@ -84,7 +90,7 @@ def FoodSupply(items, years, quantities, regions=None, elements=None, else: # dataset, coords ndims = len(coords)+1 - + # make sure the long format has the right number of dimensions while len(quantities.shape) < ndims: quantities = np.expand_dims(quantities, axis=0) @@ -124,6 +130,7 @@ def FoodSupply(items, years, quantities, regions=None, elements=None, return fbs + @xr.register_dataset_accessor("fbs") class FoodBalanceSheet(XarrayAccessorBase): @@ -148,12 +155,12 @@ def scale_element( Destination element DataArray to which the difference is added to items : list of int or list of str, optional List of items to be scaled. If not provided, all items are scaled. - + Returns ------- out : xarray.Dataset FAOSTAT formatted Food Supply dataset with scaled quantities. - + """ fbs = self._obj @@ -171,7 +178,7 @@ def scale_element( out = copy.deepcopy(fbs) # Scale items - sel = {"Item":items} + sel = {"Item": items} out[element].loc[sel] = out[element].loc[sel] * scale @@ -185,11 +192,11 @@ def scale_add( items=None, add=True, elasticity=None - ): - + ): + """Scales item quantities of an element and adds the difference to another element DataArray - + Parameters ---------- fbs : xarray.Dataset @@ -207,7 +214,7 @@ def scale_add( elasticity : float, float array_like optional Fractional percentage of the difference that is added to each element in element_out. - + Returns ------- out : xarray.Dataset @@ -216,7 +223,7 @@ def scale_add( """ fbs = self._obj - + if np.isscalar(element_out): element_out = [element_out] @@ -234,11 +241,18 @@ def scale_add( for elmnt, add_el, elast in zip(element_out, add, elasticity): out[elmnt] = out[elmnt] + np.where(add_el, -1, 1)*dif*elast - + return out - def SSR(self, items=None, per_item=False, domestic=None, - production="production", imports="imports", exports="exports"): + def SSR( + self, + items=None, + per_item=False, + domestic=None, + production="production", + imports="imports", + exports="exports" + ): """Self-sufficiency ratio Self-sufficiency ratio (SSR) or ratios for a list of item imports, @@ -266,8 +280,8 @@ def SSR(self, items=None, per_item=False, domestic=None, Returns ------- data : xarray.Dataarray - Self-sufficiency ratio or ratios for the list of items, one for each - year of the input food Dataset "Year" coordinate. + Self-sufficiency ratio or ratios for the list of items, one for + each year of the input food Dataset "Year" coordinate. """ @@ -288,8 +302,15 @@ def SSR(self, items=None, per_item=False, domestic=None, return fbs[production].sum(dim="Item") / domestic_use.sum(dim="Item") - def IDR(self, items=None, per_item=False, imports="imports", domestic=None, - production="production", exports="exports"): + def IDR( + self, + items=None, + per_item=False, + imports="imports", + domestic=None, + production="production", + exports="exports" + ): """Import-dependency ratio Import-ependency ratio (IDR) or ratios for a list of item imports, @@ -304,7 +325,8 @@ def IDR(self, items=None, per_item=False, imports="imports", domestic=None, list of items to compute the IDR for from the food Dataset. If no list is provided, the IDR is computed for all items. per_item : bool, optional - Whether to return an IDR for each item separately. Default is false. + Whether to return an IDR for each item separately. Default is + false. domestic : string, optional Name of the DataArray containing the domestic use data imports : string, optional @@ -313,7 +335,7 @@ def IDR(self, items=None, per_item=False, imports="imports", domestic=None, Name of the DataArray containing the exports data production : string, optional Name of the DataArray containing the production data - + Returns ------- @@ -341,25 +363,25 @@ def IDR(self, items=None, per_item=False, imports="imports", domestic=None, return fbs["imports"].sum(dim="Item") / domestic_use.sum(dim="Item") def plot_bars( - self, - show="Item", - elements=None, - inverted_elements=None, - ax=None, - colors=None, - labels=None, - **kwargs - ): + self, + show="Item", + elements=None, + inverted_elements=None, + ax=None, + colors=None, + labels=None, + **kwargs + ): """Plot total quantities per element on a horizontal bar plot Produces a horizontal bar plot with a bar per element on the vertical axis plotted on a cumulative form. Each bar is the sum of quantities on each element, broken down by the selected coordinate "show". The - starting x-axis position of each bar will depend on the cumulative value - up to that element. The order of elements can be defined by the + starting x-axis position of each bar will depend on the cumulative + value up to that element. The order of elements can be defined by the "element" parameter. A second set of "inverted_elements" can be given, and these will be plotted from right to left starting from the previous - cumulative sum, minus the corresponding sum of the inverted elements. + cumulative sum, minus the corresponding sum of the inverted elements. Parameters ---------- @@ -380,11 +402,12 @@ def plot_bars( colors : list of str, optional String list containing the colors for each of the elements in the "show" coordinate. - If not defined, a color list is generated from the standard cycling. + If not defined, a color list is generated from the standard + cycling. labels : str, list of str, optional String list containing the labels for the legend of the elements in - the "show" coordinate. If not set, no labels are printed. If "show", - the values of the "show" dimension are used. + the "show" coordinate. If not set, no labels are printed. + If "show", the values of the "show" dimension are used. **kwargs : dict Style options to be passed on to the actual plot function, such as linewidth, alpha, etc. @@ -416,10 +439,14 @@ def plot_bars( new_fbs = FoodBalanceSheet(fbs) new_fbs = FoodBalanceSheet(new_fbs.group_sum(coordinate=show)) - return new_fbs.plot_bars(show=show, elements=elements, - inverted_elements=inverted_elements, - ax=ax, colors=colors, labels=labels, - **kwargs) + return new_fbs.plot_bars( + show=show, + elements=elements, + inverted_elements=inverted_elements, + ax=ax, + colors=colors, + labels=labels, + **kwargs) else: raise ValueError(f"The coordinate {show} is not a valid " "dimension or coordinate of the Dataset.") @@ -444,25 +471,24 @@ def plot_bars( elif np.all(labels == "show"): labels = [str(val) for val in fbs[show].values] - # Plot non inverted elements first cumul = 0 for ie, element in enumerate(elements): ax.hlines(ie, 0, cumul, color="k", alpha=0.2, linestyle="dashed", - linewidth=0.5) + linewidth=0.5) if size_show == 1: - ax.barh(ie, left = cumul, width=food_sum[element], + ax.barh(ie, left=cumul, width=food_sum[element], color=colors[0]) - cumul +=food_sum[element] + cumul += food_sum[element] else: for ii, val in enumerate(food_sum[element]): - ax.barh(ie, left = cumul, width=val, color=colors[ii], + ax.barh(ie, left=cumul, width=val, color=colors[ii], label=labels[ii]) cumul += val # Then the inverted elements if inverted_elements is not None: - + if np.isscalar(inverted_elements): inverted_elements = [inverted_elements] @@ -472,22 +498,22 @@ def plot_bars( cumul = 0 for ie, element in enumerate(reversed(inverted_elements)): ax.hlines(len_elements-1 - ie, 0, cumul, color="k", alpha=0.2, - linestyle="dashed", linewidth=0.5) + linestyle="dashed", linewidth=0.5) if size_show == 1: - ax.barh(len_elements-1 - ie, left = cumul, + ax.barh(len_elements-1 - ie, left=cumul, width=food_sum[element], color=colors[0]) - cumul +=food_sum[element] + cumul += food_sum[element] else: for ii, val in enumerate(food_sum[element]): - ax.barh(len_elements-1 - ie, left = cumul, width=val, + ax.barh(len_elements-1 - ie, left=cumul, width=val, color=colors[ii], label=labels[ii]) cumul += val # Plot decorations ax.set_yticks(np.arange(len_elements), labels=elements) - ax.tick_params(axis="x",direction="in", pad=-12) + ax.tick_params(axis="x", direction="in", pad=-12) ax.invert_yaxis() # labels read top-to-bottom - ax.set_ylim(len_elements,-1) + ax.set_ylim(len_elements, -1) # Unique labels if print_labels: @@ -497,9 +523,9 @@ def plot_bars( return ax + @xr.register_dataarray_accessor("fbs") class FoodElementSheet(XarrayAccessorBase): - def plot_years( self, show=None, @@ -508,12 +534,12 @@ def plot_years( colors=None, labels=None, **kwargs - ): + ): """ Fill plot with quantities at each year value Produces a vertical fill plot with quantities for each year on the "Year" coordinate of the input dataset in the horizontal axis. If the - "show" coordinate exists, then the vertical fill plot is a stack of the + "show" coordinate exists, then the vertical fill plot is a stack of the sums of the other coordinates at that year for each item in the "show" coordinate. @@ -530,21 +556,22 @@ def plot_years( returned. stack : boolean, optional Whether to stack fill plots or not. If 'True', the fill curves are - stacked on top of each other and the upper fill curve represents the - sum of all elements for a given year. + stacked on top of each other and the upper fill curve represents + the sum of all elements for a given year. If 'false', each element along the 'show' dimension is plotted - starting from the origin. + starting from the origin. ax : matplotlib.pyplot.artist, optional Axes on which to draw the plot. If not provided, a new artist is created. colors : list of str, optional String list containing the colors for each of the elements in the "show" coordinate. - If not defined, a color list is generated from the standard cycling. + If not defined, a color list is generated from the standard + cycling. labels : str, list of str, optional String list containing the labels for the legend of the elements in - the "show" coordinate. If not set, no labels are printed. If "show", - the values of the "show" dimension are used. + the "show" coordinate. If not set, no labels are printed. + If "show", the values of the "show" dimension are used. **kwargs : dict Style options to be passed on to the actual plot function, such as linewidth, alpha, etc. @@ -577,7 +604,7 @@ def plot_years( elif show in fbs.coords: new_fbs = FoodElementSheet(fbs) new_fbs = FoodElementSheet(new_fbs.group_sum(coordinate=show)) - + return new_fbs.plot_years(show=show, stack=stack, ax=ax, colors=colors, labels=labels, **kwargs) elif show is None: @@ -611,18 +638,19 @@ def plot_years( # Plot if size_cumsum == 1: ax.fill_between(years, cumsum, color=colors[0], alpha=0.5) - ax.plot(years, cumsum, color=colors[0], linewidth=0.5, label=labels) + ax.plot(years, cumsum, color=colors[0], linewidth=0.5, + label=labels) else: - ax.fill_between(years, cumsum.isel({show:0}), color=colors[0], + ax.fill_between(years, cumsum.isel({show: 0}), color=colors[0], alpha=0.5) - ax.plot(years, cumsum.isel({show:0}), color=colors[0], + ax.plot(years, cumsum.isel({show: 0}), color=colors[0], linewidth=0.5, label=labels[0]) - for id in range(1,size_cumsum): - ax.fill_between(years, cumsum.isel({show:id}), - cumsum.isel({show:id-1}), color=colors[id], + for id in range(1, size_cumsum): + ax.fill_between(years, cumsum.isel({show: id}), + cumsum.isel({show: id-1}), color=colors[id], alpha=0.5) - ax.plot(years, cumsum.isel({show:id}), color=colors[id], - linewidth=0.5,label=labels[id]) + ax.plot(years, cumsum.isel({show: id}), color=colors[id], + linewidth=0.5, label=labels[id]) ax.set_xlim(years.min(), years.max()) ax.set_ylim(bottom=0) diff --git a/agrifoodpy/food/model.py b/agrifoodpy/food/model.py index c6854bf..d1b8021 100644 --- a/agrifoodpy/food/model.py +++ b/agrifoodpy/food/model.py @@ -3,11 +3,11 @@ import xarray as xr import numpy as np -import warnings import copy from ..pipeline import standalone from ..utils.dict_utils import get_dict, set_dict, item_parser + @standalone(input_keys=["fbs"], return_keys=["out_key"]) def balanced_scaling( fbs, @@ -20,12 +20,12 @@ def balanced_scaling( elasticity=None, fallback=None, add_to_fallback=True, - datablock=None, - out_key=None + out_key=None, + datablock=None ): - """ Scales items in a Food Balance Sheet, while optionally maintaining total - quantities - + """ Scales items in a Food Balance Sheet, while optionally maintaining + total quantities + Scales selected item quantities on a Food Balance Sheet, with the option to keep the sum over an element DataArray constant. Changes can be propagated to a set of origin FBS elements according to an @@ -35,10 +35,10 @@ def balanced_scaling( ---------- fbs : xarray.Dataset Input food balance sheet Dataset - element : string - Name of the DataArray to scale scale : float Scaling parameter after full adoption + element : string + Name of the DataArray to scale items : list, optional List of items to scaled in the food balance sheet. If None, all items are scaled and 'constant' is ignored @@ -47,8 +47,8 @@ def balanced_scaling( selected items accordingly origin : string, list, optional Names of the DataArrays which will be used as source for the quantity - changes. Any change to the "element" DataArray will be reflected in this - DataArray + changes. Any change to the "element" DataArray will be reflected in + this DataArray add_to_origin : bool, array, optional Whether to add or subtract the difference from the respective origins elasticity : float, array, optional @@ -59,13 +59,13 @@ def balanced_scaling( fall below zero add_to_fallback : bool, optional Whether to add or subtract the difference below zero in the origin - DataArray to the fallback array. + DataArray to the fallback array. out_key : string, tuple Output datablock path to write results to. If not given, input path is overwritten datablock : dict, optional Dictionary containing data - + Returns ------- data : xarray.Dataarray @@ -116,15 +116,16 @@ def balanced_scaling( if constant: delta = out[element] - data[element] - + # Identify non selected items and scaling non_sel_items = np.setdiff1d(data.Item.values, scaled_items) non_sel_scale = (data.sel(Item=non_sel_items)[element].sum(dim="Item") - delta.sum(dim="Item")) \ - / data.sel(Item=non_sel_items)[element].sum(dim="Item") - + / data.sel(Item=non_sel_items)[element].sum(dim="Item") + # Make sure no scaling occurs on inf and nan - non_sel_scale = non_sel_scale.where(np.isfinite(non_sel_scale)).fillna(1.0) + non_sel_scale = non_sel_scale.where( + np.isfinite(non_sel_scale)).fillna(1.0) if origin is None: out = out.fbs.scale_element( @@ -147,13 +148,13 @@ def balanced_scaling( # quantities to it if fallback is not None: for orig in origin: - dif = out[orig].where(out[orig]<0).fillna(0) + dif = out[orig].where(out[orig] < 0).fillna(0) out[fallback] -= np.where(add_to_fallback, 1, -1)*dif out[orig] = out[orig].where(out[orig] > 0, 0) set_dict(datablock, out_key, out) - return datablock + return datablock @standalone(input_keys=["fbs", "convertion_arr"], return_keys=["out_key"]) @@ -165,19 +166,19 @@ def fbs_convert( ): """Scales quantities in a food balance sheet using a conversion dataarray, dataset, or scaling factor. - + Parameters ---------- - datablock : Dict - Dictionary containing data. - dataset : str, xarray.Dataset + fbs : str, xarray.Dataset Datablock paths to the food balance sheet datasets or the datasets themselves. convertion_arr : str, xarray.DataArray, tuple or float Datablock path to the conversion array, datablock-key tuple, or the array or float itself. - keys : str, list + out_key : str, list Datablock key of the resulting dataset to be stored in the datablock. + datablock : Dict + Dictionary containing data. Returns ------- @@ -191,7 +192,8 @@ def fbs_convert( # retrieve convertion array if isinstance(convertion_arr, xr.DataArray): - convertion_arr = convertion_arr.where(np.isfinite(convertion_arr), other=0) + convertion_arr = convertion_arr.where( + np.isfinite(convertion_arr), other=0) else: convertion_arr = get_dict(datablock, convertion_arr) @@ -204,6 +206,7 @@ def fbs_convert( return datablock + @standalone(["fbs"], ["out_key"]) def SSR( fbs, @@ -265,6 +268,7 @@ def SSR( return datablock + @standalone(["fbs"], ["out_key"]) def IDR( fbs, @@ -298,8 +302,8 @@ def IDR( production : string, optional Name of the DataArray containing the production data datablock : dict, optional - Dictionary containing the food balance sheet Dataset. - + Dictionary containing the food balance sheet Dataset. + Returns ------- data : xarray.Datarray @@ -323,4 +327,4 @@ def IDR( set_dict(datablock, out_key, idr) - return datablock \ No newline at end of file + return datablock diff --git a/agrifoodpy/impact/data/PN18.nc b/agrifoodpy/impact/data/PN18.nc deleted file mode 100644 index c3a369ed418214ecbd7bc58a784f82a695956948..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 22288 zcmeHP4OCQB9=|gn5+YEVn2Jtn!Y;`|mtK0`QpFQY+a6Eui*vJ!aFd36#mR2 za{Pspix(Ugdv$!&;%Z*3Fq*8Ea>kkwF*71+ai!68w~QGT5fvFRliu3sBUNIMGRm!> z_CG|qh#H8-Zs|K@(4}YkUq)y+wlENhXe`Q2NlnR3$=0Qm>NE245gMb$p5dZKPUIZC z$SU!e2V;pZ=v!=jHTXg`f%kJM2rGCK1r7ODsR*cO7%jwd2ewC^7BB&2PP|<%Md+C+ z1L+yl2GWU>?rGIvuyx6)8W$&6xN<8Wp`mF;G*%X8+-4>rF%qd%W1%#&Eu2=xa<9=4 z2{&sdxjhN?MZrt4)-ZpNsJ+xG$;UlQy?uehs1dV)Qc>U37(o^Ob5uG_CM!?1Ac`cu zkvCa6k#}R`3t@!3)N0&*B!NLt^{o>~=H-nB2Q8zepglxmVyje~M$SITI#fcrIBR;j zH5y`uRjm4jitA|=tT(t##(}lCa|;PSJyfqkBhYAUerc-3YAxWbPF}j{Pk~AFX?S4N zNMbocdzSjoz)<5F1CT%Ia-=8CNVeE4My&gzK%+H~Y>b>3c>Qcav=`XxGXeN2Zx(kq z72>84H1i$2JG91w#?L4$Sv&6-aXkmO8a6Z#_pq@jIU_42J1@hKO?%#LxDx1jtVgk^ zW3CR!Y4?J@Uc5}O;!J}-*iH{P4emxU(e3p&NRWRTMm$5mHpsWr$K${t-zYuh1`vY3XU26r05Xk2ME-@c59{R(Z4L@YQqGMhFK6@yh4l*4>bS9xA!d z#*5L36&9n#RA;F+UqgXCZhc6jsJv-^N4KUpMLf_|4%B83V;_iV+=pn3Any#3w@teP zm{MACDv&)?8S7e}rZeaX+R!_2aw9L1D3dZ+yV1Vk<}-%|^O@~~`K+f8<}>T*ZSf!~ zC8oz5Vl>Mc|0WOm(!@p@MnQ$?kMN)m69`CwQigKvc2QHnn|ToiGY6VY`Rsm40*6Dm zf`bVNq!=k94;#2Nh?V$(cCCU~D)*5JFwpwgY;9(X42&ID4*GvYH?I*!c3fCc1$G{NqyHda$9LfKT#;w8^sX9HK*yp`3LZ#OwXHHYf6cE|eV z*cgq$0^=M)_fWYTPtIxsVJ_t}Kl7Z`Cae7c9NSXKJSt~+=R)xDypdI$PvxxN7_1f- z&xBYupUT;N^X)cHwDXPv%4hLrsD|+ZM}VH^t1v=^RL=a@;qJ(}IJAKB8E!fbM`51R z%n6cU;UIYu&x>;Ty;RQpk@`oD(~8F;5Zy=R4Bs4HdIS_vKC{Qbi8h|IqT)gNxt0pj z57~T>`JH1IiQnWu9bAUtox}-NzD^T{?Gzxc%4mJezAVmC&uhYPcw0e?I4U1S^TJB# zfzvd(byzD&;5ibVR0tUZO2$$y6$BIn6a*9m6a*9m6a*9mh8_a>xhW~|dX&AxWcfA^ zu11Cp6ogZf&4UL@6-q%sK|n!3K|n!3L0~8#0E;iaw}2;JsxtagZ!29Kfb~yr_sW0| zTT1XWmH3Ur@$eg8eIl*m)&16&s=J~jEMYPR6EU`;?0Ko$Uk@dI8wVx{ny6+d-IP3RTc8Vdp3NLPM??u>oc$xaLp(ohmPfLRo}tQigCP?sV|evxc1T8zdx`WJ-45$RT+rT_+tFl4X0Q1d->~9ni=s zyb%bRIXf|ntcF4}55pfe{=$9$^#AYUxx3kSuR8!xZAs?$1vcxe*;6;wq_voSIOVj* z9iKlpE?l^5Ju<7;`NyK7ru!yMEIE6-y{IvMQ*4&*jrZSgEbi*~uz8)X^dqlHyH@45 zFT8Y7-{teySI%ARD=p1zY`nVlSnZ)tm#n#~?X{!(R~~($Wd3Uxg5OJjujPwlyG}b6 z4?FtIy8cIUA8qXsBaWYV`?DimSH0WXAAB%A=!EutZS3sW546kD(;Bqe25m#jhTR)m zx3@;;9K0#a3$0tx~O0tx~{5rNtg zZQS9RVOdwtFI*Ec@skyCej`S_dwlwT%$LVZnf&#+@jcBu{<*Meb$q}}-P?+D8!z|i zHzf4+ME9MZc>kBRl~q-3XG&`Rl(=!}N3kPU_X(C4qr1kLDk&dkdU*Y|#9T6v}Ib@Kz?l$v*TpP!y;d~;UsI_Kp5+PIHvia&Y# z5B4h^#~xpmdAk1I^|tjzWraKqkX{jYT|h=@A>UHQwpj{W1d z7>b{cD7<2QqG$Vaw>In!k86(0n!aG~o|>kWjrJoKb%DM8_jcCrH{M>gtA5*oB@GP? z!-s1dhHGqRk#WWH-61W*Cx02z{^W%A2_YfN-ntoO7o{0IO>ee(Xp(NW9iPgT>5T`V z6t;qZf`Ed6f`Ed+5JKSkEat;cqn1xE)Lft1N8s9Pw@&*~Y6e)igY;{+3gGua1Csmq z^OPG>usMBzfgG!OL`a8HXYoxp>MsmvvgL(=o8#{vH0U}SDP-j zIoC&R-|6?6t!w__OP8L`3g7(k%~LPUuWtXod0&*UDtLTvXY0Q|cz);F@W|AuZ^ds~ zddAsX5V3lCW_v(;MsN6J^H)0(-&+^jy`j-|py`Q>b7>tfylQ%6$@#f2CA7?~JbNnR zod(dw;o^PJn?2gKtOYLz_-l-37=O6zLA~K`Cxp`SpDpn7_Bx&wQoXuyGnf| zHPzGm4NqHUJ7F7_EGh3dWMC+l3IYlO3IYlO3IanEf!dOa$T?L{KEA)j6xV0{VDpZJ z=A+$r&1rKTzkH`|oL9^1=rOIfYRu~kHQIA~&qVLHS^TYS_U}G-Zu-|ZF&7*c+Rf%a zA4O~DJr(nA+2+SzoVx8V%hRIGhiC3P-~4D?jQ=}pW<7fMoUTyes@@sV*ZkOl{_cpz z`-MfFr|Z7$SRXj)?}s{r(^t6`xkQ(@Z&B%CfGD#f%Cg;^qx(w6gd>cx`nK$Iz97@AkI%L-^rQr-6a!F~}6fM_> zvw+AkHdGF00Lf`M^GAMvIO9iNKb+|!r{Om;^72`jmfsJ~UXts>nM?3ChxwbsE~!0m zwvyc6TujUDfyo^?ABpP2&#TBKrQwVuat;!u;cO&wP8p@)>@sKXCRK+3Zj2gEPnE<>9O`xxaHTE%z7B6qA=vz_i@nL`;)&!D#*Pa|?3*7o}mZ syBwcnOcN26!|ryu{i&Fi+mnWA`Tf%e$ulr5uYVq<$+=v#{!C2&4;P>YfdBvi diff --git a/agrifoodpy/impact/data/PN18_FAOSTAT.nc b/agrifoodpy/impact/data/PN18_FAOSTAT.nc deleted file mode 100644 index 828d4743aafcfc477a5e36c97a82c18ab47dd215..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 13227 zcmeI23s98T6@c%uh=2d{i#z#cK_~<$Jp0n)!1Gb%DCYkJ+ zk?%j}o^${EpZnPRd;gcR(kF+7J{lSj94uJwlWa}UmUt={n6YkZURL__0L$)x7-cgD zQ4xn285$r==+e74{@oOYe}$N$A#|DOJ3s`2#cUGBaMpErT7Zr#pzxr_@WK?R?f_T8 zImK0_)|pGo8bZmVVa39RP)2KOr~h`2r!vU#pKE~Td-BVAyNlR(IXr!=HaMQ z3+k{zJ;mYCI`|0H!4DGB<%3vK(bX)Xg$4eJqTI6DlG5^VX-k&WlvY#~*Oo0+x5jh` zVO0%;dn;k0Vi|$CGm9W-B>1M~q!tv2K2U&Z#x|yBPtC|J$j-|xDo9Js$q<&u(QXz! zbU<$E)C}d{gZ#5=%PLi^W^f9J#zC#zxb%w3ic;TZL6nx6oBkA&mzFifl7Tvgss2#c$xSy@e1)e@e^Vz@fPt@ zVjJ-;@gDI$QP99N5d(;3Vkof}(LxL(h7%)*y@`>;C}JOCG_fzSAMsIQ9C0}D31R|q z95InNp7hr5{rm-VlnYs#1i5P;#T4|;+w?n z#2v()#J7mMiF=6qhzE&(CjN!!Cca1fEAbHVZ^XmIX5uO0Y2q#7nqGR`uO;4j$ekor_j3cVb@K5uQ(Gnm-n+GCUx30Yu=dHKoPl%DT%FgU-`r}}^HSp!+ zrvsxrgfQjL_T??>B47SN3*MK-UO#E51f?z|)Q7ItEo0Emb>D(DN%ozAvHP6`h%9?_oI;@_o)bUY=d4@^9UqsoKZ7eee22Y>%@#B_FAT z^T9gMCr_Y5yy{1L{r+osA12$qLe-^peo3u_D_84)?;s&RDt|4F)v?|>7IlX}dwNQ! z$Vg+|kngkDexumS%Y!5s?d9hOBU@HR_ec8WE0h7fO)GoQHZ2-9KqAs=^K8?`^%CRm zn}yGIO{c7FRd5tunN90)zXK8sekiIS#1vgGKUTms=VBOV9PYP(+S|Il-3UdI*Sa9|8@y69xI)hk#D+ReA+_UAICA^$x>I zC!E*#_}q@(>1oj}b?iyKV@2H*h&ybQu*|H?j&5;K(Cw@lN7ZKi3pyCF^$QAX2Vxs} z#%@+asiV8PclhG~jrk<}Hs*|T=XrTVBM4_tYw+6V4r_a8>(fwum(P80TE3t|0+*Z3 zO%1s?B!X~JgF&uSBt#&-v|b@CXn-!H1zv>AhL`CY{LcXUDD844^m_x!YCD$3(u&Hm zDqOZiFfMtzTB~~H=0%SA--ZOQd(;+D`0Zb~^$_XPZR?j_K7JCe?@>dy`V#(tK^Jc6 zAIGWcoSA8PIcjC2A0xfW>7O)=;9loBi$8TJZ0C+M+Ij34?L2CXc3rCwE9i%fbwB+X z@MplE0e=Sk8SrPop8Zrkgs` zL_!=BF10i_s&h9uKf8kQf^@ErbvPFdk!iE`Y(I4Ej_vKMk!LOS?H)py@@M{BsW-_556RCvr!8*_7GpeI)iu z?EG6cUGGrY6`=;WpE_Umq0t}w;5^;uoSSbB`HGaFz&Kaq5Be=iezbc1<++#JZBNgh zm;YF}ng{UQw>>)BjpRy!Q13~~{|z6~Jb>~WwzL74eqAb#+1@$p-tdhWH4k81sdg|u zRZ>5|xxCLjfc7?H{oEfg|3c)Jf%m>U@zfpLyYE$h@YeI~9)g?lXZtUYkGnE`g&HS( zA2|0_^8n`IBNB{E7_Vv`a5)q1I{^KR^pwMCAC`9&4}P7`19~ogJ|m6u0Q%=2mB#B2vG3fro%-gH69-E#pdG%yQ0oa3 zOiCbLe@C%0dXNMQ>-O@mCwS{He@4oAM}LUrRK)%=HX6@dPr zn;t~y&rmF9!a8;f^T+Y99+x-!pj4pSM>Q5M_iKzSKE0h86lkkQB{os5yY*~k&Ae%*mh{q2>re{pfn4XcFmQkcW z1Q`$Bcz9M?TwA>Ke?0c&(=q1%ee6jebZ`f~GUfi_hjmik81I{nFX2IG_kPOI1M*R6 z|L64w4w-FugxZ}$2Jec({H#NU59@WULhL&E{HnHs4>$eXwMH(eJdpe9@Z}Q9?BzF~ z@#t(flIsAv+&`R(aYlj7g?brL_C+y^C$znhD;GxWzJ24F6bVL`+rHYH$`Ag?gyiSY z?sObnfn?p~9=g+8mkHO_$=81sQuSujDtR}4(#~5?6nY48%Af5Ad#21QJELA!p-73n z{5O>nuAGjG&w5iPtg1(y%bBoARxgc4c9j3tiE&-UyWILCQHURjesJePq|3c+#)Alc z^Rb*M6IPd#uHuD=FG1buNLs1wT8EE3lTdfL&(&!=D4Y+tWbw%p7@dxN^yA%MZNi)m zrnYk&6WZS(L6Hd~)@ep4LQQLYb#Uqiak-aol~9k7#;To;zb`;Pqqy8Z-YCK7bZk`P K&*=zzLH-9m$rt

= 1: if labels is None: @@ -89,32 +89,40 @@ def plot(self, ax=None, class_coord=None, colors=None, labels=None, ymin, ymax = map.y.values[[0, -1]] ax.imshow(map, interpolation="none", origin="lower", - extent=[xmin-dx_low, xmax+dx_high, ymin-dy_low, ymax+dy_high], + extent=[xmin-dx_low, + xmax+dx_high, + ymin-dy_low, + ymax+dy_high], cmap=cmap, norm=norm) - + patches = [mpatches.Patch(color=colors[i], - label=labels[i]) for i in np.arange(len(labels))] - + label=labels[i]) + for i in np.arange(len(labels))] + ax.legend(handles=patches, loc="best") - + return ax - - def area_by_type(self, values = None, dim = None): + + def area_by_type( + self, + values=None, + dim=None + ): """Area per map category in a LandDataArray - - Returns a DataArray with the total number of pixels for each category or - category subset of the LandDataArray. + + Returns a DataArray with the total number of pixels for each category + or category subset of the LandDataArray. Parameters ---------- values : int, array - List of category types to return the total area for. If not set, the - function returns areas for all values found on the map, excluding - nan values. + List of category types to return the total area for. If not set, + the function returns areas for all values found on the map, + excluding nan values. dim : string Name to assign to the categories coordinate. If not set, the input DataArray name is used instead. - + Returns ------- xarray.DataArray @@ -136,17 +144,25 @@ def area_by_type(self, values = None, dim = None): # Prevent nan values from being counted nan_indices = np.isnan(values) values = values[~nan_indices] - area = [ones.where(map==value).sum() for value in values] - - area_arr = xr.DataArray(area, dims=dim, coords={dim:values}) + area = [ones.where(map == value).sum() for value in values] + + area_arr = xr.DataArray(area, dims=dim, coords={dim: values}) return area_arr - def area_overlap(self, map_right, values_left = None, values_right = None, dim_left=None, dim_right=None): + def area_overlap( + self, + map_right, + values_left=None, + values_right=None, + dim_left=None, + dim_right=None + ): """Area overlap of selected categories between two maps - - Returns a DataArray with the total number of pixels for each combination - of categories from the left and right map selected categories. Casa + + Returns a DataArray with the total number of pixels for each + combination of categories from the left and right map selected + categories. Parameters ---------- @@ -161,12 +177,14 @@ def area_overlap(self, map_right, values_left = None, values_right = None, dim_l overlaps for. If not set, all category types are used, except nan values. dim_left : string - Names to assign to the category coordinates on the output DataArray. + Names to assign to the category coordinates on the output + DataArray. If not set, the input DataArray name is used instead. dim_right : string - Names to assign to the category coordinates on the output DataArray. + Names to assign to the category coordinates on the output + DataArray. If not set, the input DataArray name is used instead. - + Returns ------- area_arr : xarray.DataArray @@ -175,8 +193,8 @@ def area_overlap(self, map_right, values_left = None, values_right = None, dim_l """ map_left = self._obj - # Check that both maps have the same dimensions and coordinates. if not, - # this raises a ValueError (alternatively, align the maps and use ) + # Check that both maps have the same dimensions and coordinates. + # Otherwise, this raises a ValueError xr.align(map_left, map_right, join='exact') if dim_left is None: @@ -203,14 +221,23 @@ def area_overlap(self, map_right, values_left = None, values_right = None, dim_l values_right = values_right[~nan_indices_right] ones = xr.ones_like(map_left) - area = [[ones.where(map_left==vl).where(map_right==vr).sum().values for vr in values_right] for vl in values_left] + area = [[ones.where(map_left == vl).where(map_right == vr).sum().values + for vr in values_right] for vl in values_left] - area_arr = xr.DataArray(area, dims=[dim_left, dim_right], coords={dim_left:values_left, dim_right:values_right}) + area_arr = xr.DataArray(area, + dims=[dim_left, dim_right], + coords={dim_left: values_left, + dim_right: values_right}) return area_arr - - def category_match(self, map_right, values_left=None, values_right=None, - join="left"): + + def category_match( + self, + map_right, + values_left=None, + values_right=None, + join="left" + ): """Returns a land Dataarray with values where a selected overlap occurs between categories from two maps. This returns the values from the left map where coincidence occurs between the left and right map. @@ -225,7 +252,7 @@ def category_match(self, map_right, values_left=None, values_right=None, values_right : int, array List of category types from the right map to match. If not set, all category types are used, except nan values. - + Returns ------- category_match : xarray.DataArray @@ -257,8 +284,13 @@ def category_match(self, map_right, values_left=None, values_right=None, return category_match - def dominant_class(self, class_coord=None, return_index=False): - """Returns a land DataArray with the dominant land class for each pixel. + def dominant_class( + self, + class_coord=None, + return_index=False + ): + """Returns a land DataArray with the dominant land class for each + pixel. Parameters ---------- @@ -282,8 +314,8 @@ class value. if return_index: len_class = len(map[class_coord].values) - map = map.assign_coords({class_coord:np.arange(len_class)}) + map = map.assign_coords({class_coord: np.arange(len_class)}) map = map.idxmax(dim=class_coord, skipna=True) - return map \ No newline at end of file + return map diff --git a/agrifoodpy/land/model.py b/agrifoodpy/land/model.py index 5db6a4e..0134455 100644 --- a/agrifoodpy/land/model.py +++ b/agrifoodpy/land/model.py @@ -5,14 +5,23 @@ import numpy as np from ..land.land import LandDataArray -def land_sequestration(land_da, use_id, fraction, max_seq, years=None, - growth_timescale=10, growth="linear", ha_per_pixel=1): - + +def land_sequestration( + land_da, + use_id, + fraction, + max_seq, + years=None, + growth_timescale=10, + growth="linear", + ha_per_pixel=1 +): + """Additional land use sequestration model. - + Computes the anual additional sequestration from land use change as a function of the different land category converted fractional areas. - + Given a Land Data Array map with pixel id values for the different land use types, the model computes additional sequestration from land given the new value in [t CO2e / yr]. @@ -25,7 +34,7 @@ def land_sequestration(land_da, use_id, fraction, max_seq, years=None, use_id : int, array Land category identifiers for the land uses to be converted. fraction : float, array - Fraction of each repurposed land category + Fraction of each repurposed land category max_seq : float Maximum sequestration achieved at the end of the growth period in [t CO2e / yr] @@ -44,7 +53,7 @@ def land_sequestration(land_da, use_id, fraction, max_seq, years=None, seq : xarray.DataArray DataArray with the per year sequestration """ - + if np.isscalar(use_id): use_id = np.array(use_id) @@ -56,9 +65,9 @@ def land_sequestration(land_da, use_id, fraction, max_seq, years=None, if not (fraction >= 0).all() and (fraction <= 1).all(): raise ValueError("Input fraction values must be between 0 and 1") - + pixel_count_category = land_da.land.area_by_type(values=use_id) - + # area in hectares area_category = pixel_count_category * ha_per_pixel @@ -68,7 +77,7 @@ def land_sequestration(land_da, use_id, fraction, max_seq, years=None, from agrifoodpy.utils.scaling import logistic_scale as growth_shape else: raise ValueError("Growth must be one of 'linear' or 'logistic'") - + if years is not None: # single scalar value if np.isscalar(years): @@ -83,8 +92,8 @@ def land_sequestration(land_da, use_id, fraction, max_seq, years=None, else: scale = 1 - # agroforestry + # agroforestry area = area_category * fraction total_seq = area.sum() * scale * max_seq - return total_seq \ No newline at end of file + return total_seq diff --git a/agrifoodpy/pipeline/pipeline.py b/agrifoodpy/pipeline/pipeline.py index 963faf0..ecc3f60 100644 --- a/agrifoodpy/pipeline/pipeline.py +++ b/agrifoodpy/pipeline/pipeline.py @@ -9,6 +9,7 @@ from inspect import signature import time + class Pipeline(): '''Class for constructing and running pipelines of functions with individual sets of parameters.''' @@ -36,7 +37,7 @@ def read(cls, filename): The pipeline object. """ raise NotImplementedError("This method is not yet implemented.") - + def datablock_write(self, path, value): """Writes a single value to the datablock at the specified path. @@ -52,7 +53,7 @@ def datablock_write(self, path, value): for key in path[:-1]: current = current.setdefault(key, {}) current[path[-1]] = value - + def add_node(self, node, params={}, name=None): """Adds a node to the pipeline, including its function and execution parameters. @@ -65,7 +66,7 @@ def add_node(self, node, params={}, name=None): The parameters to be passed to the node function. name : str, optional The name of the node. If not provided, a generic name will be - assigned. + assigned. """ # Copy the parameters to avoid modifying the original dictionaries @@ -114,7 +115,8 @@ def run(self, from_node=0, to_node=None, timing=False): node_time = node_end_time - node_start_time if timing: - print(f"Node {i + 1}: {self.names[i]}, executed in {node_time:.4f} seconds.") + print(f"Node {i + 1}: {self.names[i]}, \ + executed in {node_time:.4f} seconds.") pipeline_end_time = time.time() pipeline_time = pipeline_end_time - pipeline_start_time @@ -122,13 +124,14 @@ def run(self, from_node=0, to_node=None, timing=False): if timing: print(f"Pipeline executed in {pipeline_time:.4f} seconds.") + def standalone(input_keys, return_keys): """ Decorator to make a pipeline node available as a standalone function If datablock is not passed as a kwarg, and datasets are passed directly - instead of datablock keys, a temporary datablock is created and the datasets - associated with the arguments in input_keys are added to it. The function - then returns the specified datasets in return_keys. + instead of datablock keys, a temporary datablock is created and the + datasets associated with the arguments in input_keys are added to it. + The function then returns the specified datasets in return_keys. Parameters ---------- @@ -143,7 +146,7 @@ def standalone(input_keys, return_keys): wrapper: function The decorated function - + """ def pipeline_decorator(test_func): @wraps(test_func) @@ -153,23 +156,26 @@ def wrapper(*args, **kwargs): func_sig = signature(test_func) func_params = func_sig.parameters - kwargs.update({key: arg for key, arg in zip(func_params.keys(), args)}) + kwargs.update({key: arg for key, arg in zip(func_params.keys(), + args)}) - # Make sure that the datablock is passed as a kwarg, if not, create it + # Make sure the datablock is passed as a kwarg, if not, create it datablock = kwargs.get("datablock", None) # Fill in missing arguments with their default values for key, param in func_params.items(): if key not in kwargs: - if param.default is not param.empty: # Check if there's a default value + # Check if there is a default value + if param.default is not param.empty: kwargs[key] = param.default standalone = datablock is None if standalone: # Create datablock - datablock = {key: kwargs[key] for key in kwargs if key in input_keys} + datablock = {key: kwargs[key] + for key in kwargs if key in input_keys} kwargs["datablock"] = datablock - + # Create list of keys for passed arguments only for key in input_keys: if kwargs.get(key, None) is not None: @@ -191,4 +197,4 @@ def wrapper(*args, **kwargs): return result return wrapper - return pipeline_decorator \ No newline at end of file + return pipeline_decorator diff --git a/agrifoodpy/population/population.py b/agrifoodpy/population/population.py index 5637135..40e8c34 100644 --- a/agrifoodpy/population/population.py +++ b/agrifoodpy/population/population.py @@ -6,7 +6,14 @@ from ..array_accessor import XarrayAccessorBase -def population(years, regions, quantities, datasets=None, long_format=True): + +def population( + years, + regions, + quantities, + datasets=None, + long_format=True +): """Population style dataset constructor Parameters @@ -42,15 +49,15 @@ def population(years, regions, quantities, datasets=None, long_format=True): # Identify unique values in coordinates _years = np.unique(years) _regions = np.unique(regions) - coords = {"Year" : _years, - "Region" : _regions} + coords = {"Year": _years, + "Region": _regions} # find positions in output array to organize data ii = [np.searchsorted(_years, years), np.searchsorted(_regions, regions)] size = (len(_years), len(_regions)) # Create empty dataset - data = xr.Dataset(coords = coords) + data = xr.Dataset(coords=coords) if long_format: # dataset, quantities @@ -95,6 +102,7 @@ def population(years, regions, quantities, datasets=None, long_format=True): return data + @xr.register_dataarray_accessor("pop") class PopulationDataArray(XarrayAccessorBase): pass diff --git a/agrifoodpy/utils/add_items.py b/agrifoodpy/utils/add_items.py deleted file mode 100644 index 4c26dc4..0000000 --- a/agrifoodpy/utils/add_items.py +++ /dev/null @@ -1,69 +0,0 @@ -import xarray as xr -import numpy as np -import copy - -from ..pipeline import standalone -from ..utils.dict_utils import get_dict, set_dict -from ..food import food - -@standalone(["dataset"], ["dataset"]) -def add_items( - dataset, - items, - values=None, - copy_from=None, - datablock=None -): - """Adds a list of items to a selected dataset in the datablock and - initializes their values. - - Parameters - ---------- - datablock : Datablock - Datablock object. - dataset : dict - Datablock path to the datasets to modify. - items : list, dict - List of items or dictionary of items attributes to add. If a dictionary, - keys are the item names, and non-dimension coordinates. Must contain a - key named "Item". - values : list, optional - List of values to initialize the items. If not set, values are set to 0, - unless the copy from parameter is set. - copy_from : list, optional - Items to copy the values from. - - Returns - ------- - dict or xarray.Dataset - - If no datablock is provided, returns a xarray.Dataset with the new - items. - - If a datablock is provided, returns the datablock with the modified - datasets on the corresponding keys. - - """ - - # Check if items is a dictionary - if isinstance(items, dict): - items_src = copy.deepcopy(items) - new_items = items_src.pop("Item") - else: - new_items = items - items_src = {} - - # Add new items to the datasets - data = get_dict(datablock, dataset) - - data = data.fbs.add_items(new_items, copy_from=copy_from) - for key, val in items_src.items(): - data[key].loc[{"Item":new_items}] = val - - if values is not None: - data.loc[{"Item":new_items}] = values - - elif copy_from is None: - data.loc[{"Item":new_items}] = 0 - - set_dict(datablock, dataset, data) - - return datablock diff --git a/agrifoodpy/utils/add_years.py b/agrifoodpy/utils/add_years.py deleted file mode 100644 index 0835a36..0000000 --- a/agrifoodpy/utils/add_years.py +++ /dev/null @@ -1,37 +0,0 @@ -from ..pipeline import standalone -from ..impact.impact import Impact -from ..food.food import FoodBalanceSheet -from .dict_utils import get_dict, set_dict - -@standalone(["dataset"], ["dataset"]) -def add_years( - dataset, - years, - projection='empty', - datablock=None -): - """ - Extends the Year coordinates of a dataset. - - Parameters - ---------- - datablock : dict - The datablock dictionary where the dataset is stored. - dataset : str - Datablock key of the dataset to extend. - years : list - List of years to extend the dataset to. - projection : str - Projection mode. If "constant", the last year of the input array - is copied to every new year. If "empty", values are initialized and - set to zero. If a float array is given, these are used to populate - the new year using a scaling of the last year of the array - """ - - data = get_dict(datablock, dataset) - - data = data.fbs.add_years(years, projection) - - set_dict(datablock, dataset, data) - - return datablock diff --git a/agrifoodpy/utils/copy_datablock.py b/agrifoodpy/utils/copy_datablock.py deleted file mode 100644 index c0895ec..0000000 --- a/agrifoodpy/utils/copy_datablock.py +++ /dev/null @@ -1,23 +0,0 @@ -import copy - -def copy_datablock(datablock, key, out_key): - """Copy a datablock element into a new key in the datablock - - Parameters - ---------- - datablock : xarray.Dataset - The datablock to print - key : str - The key of the datablock to print - out_key : str - The key of the datablock to copy to - - Returns - ------- - datablock : dict - Datablock to with added key - """ - - datablock[out_key] = copy.deepcopy(datablock[key]) - - return datablock diff --git a/agrifoodpy/utils/dict_utils.py b/agrifoodpy/utils/dict_utils.py index 585f379..c662fda 100644 --- a/agrifoodpy/utils/dict_utils.py +++ b/agrifoodpy/utils/dict_utils.py @@ -2,6 +2,7 @@ import numpy as np + def item_parser(fbs, items): """Extracts a list of items from a dataset using a coordinate-key tuple, or converts a scalar item to a list @@ -27,16 +28,17 @@ def item_parser(fbs, items): return None if isinstance(items, tuple): - items = fbs.sel(Item = fbs[items[0]].isin(items[1])).Item.values + items = fbs.sel(Item=fbs[items[0]].isin(items[1])).Item.values elif np.isscalar(items): items = [items] return items + def get_dict(datablock, keys): - """Returns an element from a dictionary using a key or tuple of keys used to - describe a path of keys - + """Returns an element from a dictionary using a key or tuple of keys used + to describe a path of keys + Parameters ---------- @@ -45,7 +47,7 @@ def get_dict(datablock, keys): keys : str or tuple Dictionary key, or tuple of keys """ - + if isinstance(keys, tuple): out = datablock for key in keys: @@ -55,10 +57,11 @@ def get_dict(datablock, keys): return out + def set_dict(datablock, keys, object, create_missing=True): """Sets an element in a dictionary using a key or tuple of keys used to describe a path of keys - + Parameters ---------- @@ -76,7 +79,7 @@ def set_dict(datablock, keys, object, create_missing=True): KeyError If a key in the path does not exist and create_missing is False. """ - + if isinstance(keys, tuple): out = datablock for key in keys[:-1]: diff --git a/agrifoodpy/utils/load_dataset.py b/agrifoodpy/utils/load_dataset.py deleted file mode 100644 index 967ea98..0000000 --- a/agrifoodpy/utils/load_dataset.py +++ /dev/null @@ -1,70 +0,0 @@ -import numpy as np -import xarray as xr -import importlib - -from ..pipeline import standalone - -def _import_dataset(module_name, dataset_name): - module = importlib.import_module(module_name) - dataset = getattr(module, dataset_name) - return dataset - -@standalone([], ['datablock_path']) -def load_dataset( - datablock_path, - path=None, - module=None, - data_attr=None, - da=None, - coords=None, - scale=1., - datablock=None -): - """Loads a dataset to the specified datablock dictionary. - - Parameters - ---------- - datablock : dict - The datablock path where the dataset is stored - path : str - The path to the dataset stored in a netCDF file. - module : str - The module name where the dataset will be imported from. - data_attr : str - The attribute name of the dataset in the module. - da : str - The dataarray to be loaded. - coords : dict - Dictionary containing the coordinates of the dataset to be loaded. - scale : float - Optional multiplicative factor to be applied to the dataset on load. - datablock_path : str - The path to the datablock where the dataset is stored. - - """ - - # Load dataset from Netcdf file - if path is not None: - try: - with xr.open_dataset(path) as data: - dataset = data.load() - - except ValueError: - with xr.open_dataarray(path) as data: - dataset = data.load() - - # Load dataset from module - elif module is not None and data_attr is not None: - dataset = _import_dataset(module, data_attr) - - # Select dataarray and coords from dataset - if da is not None: - dataset = dataset[da] - - if coords is not None: - dataset = dataset.sel(coords) - - # Add dataset to datablock - datablock[datablock_path] = dataset * scale - - return datablock diff --git a/agrifoodpy/utils/nodes.py b/agrifoodpy/utils/nodes.py new file mode 100644 index 0000000..77a7dfe --- /dev/null +++ b/agrifoodpy/utils/nodes.py @@ -0,0 +1,289 @@ +import copy +import xarray as xr +import importlib + +from ..pipeline import standalone +from ..utils.dict_utils import get_dict, set_dict + + +@standalone(["dataset"], ["dataset"]) +def add_items( + dataset, + items, + values=None, + copy_from=None, + datablock=None +): + """Adds a list of items to a selected dataset in the datablock and + initializes their values. + + Parameters + ---------- + datablock : Datablock + Datablock object. + dataset : dict + Datablock path to the datasets to modify. + items : list, dict + List of items or dictionary of items attributes to add. If a + dictionary, keys are the item names, and non-dimension coordinates. + values : list, optional + List of values to initialize the items. If not set, values are set to + 0, unless the copy from parameter is set. + copy_from : list, optional + Items to copy the values from. + + Returns + ------- + dict or xarray.Dataset + - If no datablock is provided, returns a xarray.Dataset with the new + items. + - If a datablock is provided, returns the datablock with the modified + datasets on the corresponding keys. + + """ + + # Check if items is a dictionary + if isinstance(items, dict): + items_src = copy.deepcopy(items) + new_items = items_src.pop("Item") + else: + new_items = items + items_src = {} + + # Add new items to the datasets + data = get_dict(datablock, dataset) + + data = data.fbs.add_items(new_items, copy_from=copy_from) + for key, val in items_src.items(): + data[key].loc[{"Item": new_items}] = val + + if values is not None: + data.loc[{"Item": new_items}] = values + + elif copy_from is None: + data.loc[{"Item": new_items}] = 0 + + set_dict(datablock, dataset, data) + + return datablock + + +@standalone(["dataset"], ["dataset"]) +def add_years( + dataset, + years, + projection='empty', + datablock=None +): + """ + Extends the Year coordinates of a dataset. + + Parameters + ---------- + datablock : dict + The datablock dictionary where the dataset is stored. + dataset : str + Datablock key of the dataset to extend. + years : list + List of years to extend the dataset to. + projection : str + Projection mode. If "constant", the last year of the input array + is copied to every new year. If "empty", values are initialized and + set to zero. If a float array is given, these are used to populate + the new year using a scaling of the last year of the array + """ + + data = get_dict(datablock, dataset) + + data = data.fbs.add_years(years, projection) + + set_dict(datablock, dataset, data) + + return datablock + + +def copy_datablock(datablock, key, out_key): + """Copy a datablock element into a new key in the datablock + + Parameters + ---------- + datablock : xarray.Dataset + The datablock to print + key : str + The key of the datablock to print + out_key : str + The key of the datablock to copy to + + Returns + ------- + datablock : dict + Datablock to with added key + """ + + datablock[out_key] = copy.deepcopy(datablock[key]) + + return datablock + + +def print_datablock( + datablock, + key, + attr=None, + method=None, + args=None, + kwargs=None, + preffix="", + suffix="" +): + """Prints a datablock element or its attributes/methods at any point in the + pipeline execution. + + Parameters + ---------- + datablock : dict + The datablock to print from. + key : str + The key of the datablock to print. + attr : str, optional + Name of an attribute of the object to print. + method : str, optional + Name of a method of the object to call and print. + args : list, optional + Positional arguments for the method call. + kwargs : dict, optional + Keyword arguments for the method call. + + Returns + ------- + datablock : dict + Unmodified datablock to continue execution. + """ + obj = datablock[key] + + # Extract attribute + if attr is not None: + if hasattr(obj, attr): + obj = getattr(obj, attr) + else: + print(f"Object has no attribute '{attr}'") + return datablock + + # Call method + if method is not None: + if hasattr(obj, method): + func = getattr(obj, method) + if callable(func): + args = args or [] + kwargs = kwargs or {} + try: + obj = func(*args, **kwargs) + except Exception as e: + print(f"Error calling {method} on {key}: {e}") + return datablock + else: + print(f"'{method}' is not callable on {key}") + return datablock + else: + print(f"Object has no method '{method}'") + return datablock + + # Final print + print(f"{preffix}{obj}{suffix}") + return datablock + + +def write_to_datablock(datablock, key, value, overwrite=True): + """Writes a value to a specified key in the datablock. + + Parameters + ---------- + datablock : dict + The datablock to write to. + key : str + The key in the datablock where the value will be written. + value : any + The value to write to the datablock. + overwrite : bool, optional + If True, overwrite the existing value at the key. + If False, do not overwrite. + + Returns + ------- + datablock : dict + The updated datablock with the new key-value pair. + """ + + if not overwrite and key in datablock: + raise KeyError( + "Key already exists in datablock and overwrite is set to False.") + + set_dict(datablock, key, value) + + return datablock + + +def _import_dataset(module_name, dataset_name): + module = importlib.import_module(module_name) + dataset = getattr(module, dataset_name) + return dataset + + +@standalone([], ['datablock_path']) +def load_dataset( + datablock_path, + path=None, + module=None, + data_attr=None, + da=None, + coords=None, + scale=1., + datablock=None +): + """Loads a dataset to the specified datablock dictionary. + + Parameters + ---------- + datablock : dict + The datablock path where the dataset is stored + path : str + The path to the dataset stored in a netCDF file. + module : str + The module name where the dataset will be imported from. + data_attr : str + The attribute name of the dataset in the module. + da : str + The dataarray to be loaded. + coords : dict + Dictionary containing the coordinates of the dataset to be loaded. + scale : float + Optional multiplicative factor to be applied to the dataset on load. + datablock_path : str + The path to the datablock where the dataset is stored. + + """ + + # Load dataset from Netcdf file + if path is not None: + try: + with xr.open_dataset(path) as data: + dataset = data.load() + + except ValueError: + with xr.open_dataarray(path) as data: + dataset = data.load() + + # Load dataset from module + elif module is not None and data_attr is not None: + dataset = _import_dataset(module, data_attr) + + # Select dataarray and coords from dataset + if da is not None: + dataset = dataset[da] + + if coords is not None: + dataset = dataset.sel(coords) + + # Add dataset to datablock + datablock[datablock_path] = dataset * scale + + return datablock diff --git a/agrifoodpy/utils/print_datablock.py b/agrifoodpy/utils/print_datablock.py deleted file mode 100644 index 27cecf7..0000000 --- a/agrifoodpy/utils/print_datablock.py +++ /dev/null @@ -1,65 +0,0 @@ -def print_datablock( - datablock, - key, - attr=None, - method=None, - args=None, - kwargs=None, - preffix="", - suffix="" -): - """Prints a datablock element or its attributes/methods at any point in the - pipeline execution. - - Parameters - ---------- - datablock : dict - The datablock to print from. - key : str - The key of the datablock to print. - attr : str, optional - Name of an attribute of the object to print. - method : str, optional - Name of a method of the object to call and print. - args : list, optional - Positional arguments for the method call. - kwargs : dict, optional - Keyword arguments for the method call. - - Returns - ------- - datablock : dict - Unmodified datablock to continue execution. - """ - obj = datablock[key] - - # Extract attribute - if attr is not None: - if hasattr(obj, attr): - obj = getattr(obj, attr) - else: - print(f"Object has no attribute '{attr}'") - return datablock - - # Call method - if method is not None: - if hasattr(obj, method): - func = getattr(obj, method) - if callable(func): - args = args or [] - kwargs = kwargs or {} - try: - obj = func(*args, **kwargs) - except Exception as e: - print(f"Error calling {method} on {key}: {e}") - return datablock - else: - print(f"'{method}' is not callable on {key}") - return datablock - else: - print(f"Object has no method '{method}'") - return datablock - - # Final print - print(f"{preffix}{obj}{suffix}") - return datablock \ No newline at end of file diff --git a/agrifoodpy/utils/scaling.py b/agrifoodpy/utils/scaling.py index 945893d..a98ff9d 100644 --- a/agrifoodpy/utils/scaling.py +++ b/agrifoodpy/utils/scaling.py @@ -3,6 +3,7 @@ import numpy as np import xarray as xr + def logistic_scale(y0, y1, y2, y3, c_init, c_end): """ Create an xarray DataArray with a logistic growth interval @@ -20,7 +21,7 @@ def logistic_scale(y0, y1, y2, y3, c_init, c_end): The initial constant value. c_end : (float) The final constant value. - + Returns: xarray DataArray An xarray DataArray object with 'year' as the coordinate and values set by a logistic growth between the user defined intervals. @@ -33,8 +34,8 @@ def logistic_scale(y0, y1, y2, y3, c_init, c_end): # Set values between y1 and y2 using a logistic curve var_segment = np.logical_and(years >= y1, years < y2) t = (years[var_segment] - y1) / (y2 - y1) - values[var_segment] = c_init + \ - (c_end - c_init) *(1 / (1 + np.exp(-10 * (t - 0.5)))) + values[var_segment] = c_init \ + + (c_end - c_init)*(1 / (1 + np.exp(-10 * (t - 0.5)))) # Set values between y2 and y3 to c_end values[years >= y2] = c_end @@ -42,15 +43,16 @@ def logistic_scale(y0, y1, y2, y3, c_init, c_end): data_array = xr.DataArray(values, dims='Year', coords={'Year': years}) return data_array + def linear_scale(y0, y1, y2, y3, c_init, c_end): """ Create an xarray DataArray with a single coordinate called 'year'. - + The values from the first year 'y0' up to a given year 'y1' will be constant, then they will vary linearly between 'y1' and another given year 'y2', and from 'y2' until the end of the array 'y3', the values will continue with a constant value. - + Parameters: y0 : (int) Starting year. @@ -63,12 +65,12 @@ def linear_scale(y0, y1, y2, y3, c_init, c_end): c_init : (float) Value to use for initial constant scale segment. c_end : (float) - + Returns: xr.DataArray An xarray DataArray object with 'year' as the coordinate and values set by a linear growth between the user defined intervals. """ - + # Create arrays and set values between y0 and y1 to c_init years = np.arange(y0, y3 + 1) values = np.ones_like(years, dtype=float) * c_init @@ -80,10 +82,10 @@ def linear_scale(y0, y1, y2, y3, c_init, c_end): else: slope = float((c_end - c_init) / (y2 - y1)) values[var_segment] = slope * (years[var_segment] - y1) + c_init - + # Set values between y2 and y3 to c_end values[years >= y2] = c_end - + data_array = xr.DataArray(values, coords={'Year': years}, dims=['Year']) - - return data_array \ No newline at end of file + + return data_array diff --git a/agrifoodpy/utils/tests/test_add_items.py b/agrifoodpy/utils/tests/test_add_items.py index 6cf57e4..2e8dc1b 100644 --- a/agrifoodpy/utils/tests/test_add_items.py +++ b/agrifoodpy/utils/tests/test_add_items.py @@ -1,10 +1,11 @@ import numpy as np import xarray as xr + def test_add_items(): - - from agrifoodpy.utils.add_items import add_items - + + from agrifoodpy.utils.nodes import add_items + items = ["Beef", "Apples", "Poultry"] item_origin = ["Animal", "Vegetal", "Animal"] new_items = ["Tomatoes", "Potatoes", "Eggs"] @@ -14,7 +15,7 @@ def test_add_items(): ds = xr.Dataset({"data": (("Item", "X", "Y"), data)}, coords={"Item": items, "X": [0, 1], "Y": [0, 1]}) - ds = ds.assign_coords({"Item_origin":("Item", item_origin)}) + ds = ds.assign_coords({"Item_origin": ("Item", item_origin)}) # Test basic functionality result_add = add_items(ds, new_items) @@ -30,12 +31,12 @@ def test_add_items(): for item_i in new_items: assert np.array_equal(result_copy["data"].sel(Item=item_i), ds.data.sel(Item="Beef")) - + # Test copying from multiple existing items result_copy_multiple = add_items(ds, new_items, copy_from=["Beef", "Apples", "Poultry"]) - + assert np.array_equal(result_copy_multiple["Item"], expected_items) assert np.array_equal(result_copy_multiple["data"].sel(Item=new_items), ds.data.sel(Item=["Beef", "Apples", "Poultry"])) @@ -50,6 +51,7 @@ def test_add_items(): assert np.array_equal(result_dict["Item"].values, expected_items) assert np.array_equal(result_dict["Item_origin"].values, - ["Animal", "Vegetal", "Animal", "Vegetal", "Vegetal", "Animal"]) + ["Animal", "Vegetal", "Animal", + "Vegetal", "Vegetal", "Animal"]) for item in new_items: assert np.all(result_dict["data"].sel(Item=item).values == 0) diff --git a/agrifoodpy/utils/tests/test_add_years.py b/agrifoodpy/utils/tests/test_add_years.py index 964cf0e..ed57e54 100644 --- a/agrifoodpy/utils/tests/test_add_years.py +++ b/agrifoodpy/utils/tests/test_add_years.py @@ -1,9 +1,10 @@ import numpy as np import xarray as xr + def test_add_items(): - - from agrifoodpy.utils.add_years import add_years + + from agrifoodpy.utils.nodes import add_years items = ["Beef", "Apples", "Poultry"] years = [2010, 2011, 2012] @@ -13,7 +14,7 @@ def test_add_items(): ds = xr.Dataset({"data": (("Item", "Year"), data)}, coords={"Item": items, "Year": years}) - + # Test basic functionality new_years = [2013, 2014] result_add = add_years(ds, new_years) @@ -28,7 +29,7 @@ def test_add_items(): for year in new_years: assert np.array_equal(result_constant["data"].sel(Year=year).values, ds.data.isel(Year=-1).values) - + # Test projection mode with float array scaling_factors = np.array([1.0, 2.0]) result_scaled = add_years(ds, new_years, projection=scaling_factors) @@ -36,4 +37,4 @@ def test_add_items(): for i, year in enumerate(new_years): expected_values = ds.data.isel(Year=-1).values * scaling_factors[i] assert np.array_equal(result_scaled["data"].sel(Year=year).values, - expected_values) \ No newline at end of file + expected_values) diff --git a/agrifoodpy/utils/tests/test_copy_datablock.py b/agrifoodpy/utils/tests/test_copy_datablock.py index 9c5743b..f176eea 100644 --- a/agrifoodpy/utils/tests/test_copy_datablock.py +++ b/agrifoodpy/utils/tests/test_copy_datablock.py @@ -1,9 +1,10 @@ import numpy as np + def test_copy_datablock(): from agrifoodpy.pipeline import Pipeline - from agrifoodpy.utils.copy_datablock import copy_datablock + from agrifoodpy.utils.nodes import copy_datablock datablock = { 'test_dataset': { @@ -33,4 +34,3 @@ def test_copy_datablock(): pipeline.datablock['copied_dataset']['fbs']['data'], datablock['test_dataset']['fbs']['data'] ) - diff --git a/agrifoodpy/utils/tests/test_dict_utils.py b/agrifoodpy/utils/tests/test_dict_utils.py index 662649e..4eca715 100644 --- a/agrifoodpy/utils/tests/test_dict_utils.py +++ b/agrifoodpy/utils/tests/test_dict_utils.py @@ -5,17 +5,17 @@ def test_item_parser(): from agrifoodpy.utils.dict_utils import item_parser - items = ["Beef", "Apples", "Poultry"] + item_origin = ["Animal", "Vegetal", "Animal"] data = np.random.rand(3, 2, 2) fbs = xr.Dataset({"data": (("Item", "X", "Y"), data)}, - coords={"Item": items, "X": [0, 1], "Y": [0, 1]}) - fbs = fbs.assign_coords({"Item_origin":("Item", item_origin)}) - + coords={"Item": items, "X": [0, 1], "Y": [0, 1]}) + fbs = fbs.assign_coords({"Item_origin": ("Item", item_origin)}) # Test case for item_parser with None input + items = item_parser(fbs, None) assert items is None @@ -27,6 +27,7 @@ def test_item_parser(): items = item_parser(fbs, "Beef") assert np.array_equal(items, ["Beef"]) + def test_get_dict(): from agrifoodpy.utils.dict_utils import get_dict @@ -48,6 +49,7 @@ def test_get_dict(): value = get_dict(datablock, ("key1", "key2", "key3")) assert value == "value" + def test_set_dict(): from agrifoodpy.utils.dict_utils import set_dict @@ -75,7 +77,7 @@ def test_set_dict(): # Test case for set_dict with create_missing=False try: - set_dict(datablock, ("key7", "key8"), "should_fail", create_missing=False) + set_dict(datablock, ("key7", "key8"), "should_fail", + create_missing=False) except KeyError: pass - \ No newline at end of file diff --git a/agrifoodpy/utils/tests/test_load_dataset.py b/agrifoodpy/utils/tests/test_load_dataset.py index 7ee1ca9..55b44ff 100644 --- a/agrifoodpy/utils/tests/test_load_dataset.py +++ b/agrifoodpy/utils/tests/test_load_dataset.py @@ -1,9 +1,10 @@ import numpy as np import xarray as xr -from agrifoodpy.utils.load_dataset import load_dataset +from agrifoodpy.utils.nodes import load_dataset import os + def test_load_dataset(): items = ["Beef", "Apples", "Poultry"] @@ -11,7 +12,7 @@ def test_load_dataset(): data = np.reshape(np.arange(np.prod(shape)), shape) expected_ds = xr.Dataset({"data": (("Item", "X", "Y"), data)}, - coords={"Item": items, "X": [0, 1], "Y": [0, 1]}) + coords={"Item": items, "X": [0, 1], "Y": [0, 1]}) script_dir = os.path.dirname(__file__) test_data_path = os.path.join(script_dir, "data/test_dataset.nc") @@ -36,4 +37,3 @@ def test_load_dataset(): scaled_ds = load_dataset(path=test_data_path, scale=2.0) expected_scaled_data = expected_ds * 2.0 assert scaled_ds.equals(expected_scaled_data) - diff --git a/agrifoodpy/utils/tests/test_print_datablock.py b/agrifoodpy/utils/tests/test_print_datablock.py index b43d80e..9d3622b 100644 --- a/agrifoodpy/utils/tests/test_print_datablock.py +++ b/agrifoodpy/utils/tests/test_print_datablock.py @@ -1,9 +1,10 @@ import numpy as np import xarray as xr + def test_print_datablock(): - from agrifoodpy.utils.print_datablock import print_datablock + from agrifoodpy.utils.nodes import print_datablock items = ["Beef", "Apples", "Poultry"] @@ -41,13 +42,17 @@ def test_print_datablock(): datablock = print_datablock(datablock, 'test_array') # Test printing an attribute of the xarray Dataset - datablock = print_datablock(datablock, 'test_xarray', attr='data_vars') + datablock = print_datablock(datablock, 'test_xarray', + attr='data_vars') # Test calling a method on the xarray Dataset - datablock = print_datablock(datablock, 'test_xarray', method='mean', args=[('X', 'Y')]) + datablock = print_datablock(datablock, 'test_xarray', + method='mean', args=[('X', 'Y')]) # Test calling a method with keyword arguments - datablock = print_datablock(datablock, 'test_xarray', method='sel', kwargs={'Item': 'Beef'}) + datablock = print_datablock(datablock, 'test_xarray', + method='sel', kwargs={'Item': 'Beef'}) # Test error handling for non-existent attribute - datablock = print_datablock(datablock, 'test_xarray', attr='non_existent_attr') + datablock = print_datablock(datablock, 'test_xarray', + attr='non_existent_attr') diff --git a/agrifoodpy/utils/tests/test_scaling.py b/agrifoodpy/utils/tests/test_scaling.py index a71c90d..5b6e86e 100644 --- a/agrifoodpy/utils/tests/test_scaling.py +++ b/agrifoodpy/utils/tests/test_scaling.py @@ -1,7 +1,7 @@ import numpy as np -import xarray as xr from agrifoodpy.utils.scaling import linear_scale, logistic_scale + def test_logistic_scale(): # Basic functionality test @@ -12,7 +12,7 @@ def test_logistic_scale(): truth = [0, 0, 0, 0, 0, 0.06692851, 0.47425873, 2.68941421, 7.31058579, 9.52574127, 10, 10, 10, 10, 10, 10] - + assert np.allclose(basic_result, truth) assert np.array_equal(basic_result["Year"].values, np.arange(2000, 2016)) @@ -23,7 +23,7 @@ def test_logistic_scale(): truth = [-1, -1, -1, -1, -1, -1.06023566, -1.42683286, -3.42047279, -7.57952721, -9.57316714, -10, -10, -10, -10, -10, -10] - + assert np.allclose(negative_result, truth) # Instant change test @@ -46,11 +46,12 @@ def test_logistic_scale(): c_init, c_end = 5.5, 5.5 constant_value = logistic_scale(y0, y1, y2, y3, c_init, c_end) - assert np.array_equal(constant_value, c_init * np.ones(y3+1-y0)) + assert np.array_equal(constant_value, c_init * np.ones(y3+1-y0)) assert np.array_equal(constant_value, c_end * np.ones(y3+1-y0)) + def test_linear_scale(): - + # Basic functionality test basic_result = linear_scale(2000, 2005, 2010, 2015, 0, 10) truth = [0, 0, 0, 0, 0, 0, 2, 4, 6, 8, 10, 10, 10, 10, 10, 10] @@ -62,8 +63,8 @@ def test_linear_scale(): y0, y1, y2, y3 = 2000, 2005, 2010, 2015 c_init, c_end = -1, -10 negative_result = linear_scale(y0, y1, y2, y3, c_init, c_end) - truth = [ -1, -1, -1, -1, -1, -1, -2.8, -4.6, -6.4, -8.2, - -10. , -10. , -10. , -10. , -10. , -10. ] + truth = [-1, -1, -1, -1, -1, -1, -2.8, -4.6, -6.4, -8.2, + -10., -10., -10., -10., -10., -10.] assert np.allclose(negative_result, truth) @@ -89,5 +90,3 @@ def test_linear_scale(): assert np.array_equal(constant_value, c_init * np.ones(y3+1-y0)) assert np.array_equal(constant_value, c_end * np.ones(y3+1-y0)) - - diff --git a/agrifoodpy/utils/tests/test_write_to_datablock.py b/agrifoodpy/utils/tests/test_write_to_datablock.py index 237590e..8986e31 100644 --- a/agrifoodpy/utils/tests/test_write_to_datablock.py +++ b/agrifoodpy/utils/tests/test_write_to_datablock.py @@ -1,8 +1,8 @@ def test_write_to_datablock(): - from agrifoodpy.utils.write_to_datablock import write_to_datablock + from agrifoodpy.utils.nodes import write_to_datablock datablock_basic = {} - + # Basic write to the datablock write_to_datablock(datablock_basic, "test_key", "test_value") assert datablock_basic["test_key"] == "test_value" @@ -29,6 +29,6 @@ def test_write_to_datablock(): key="test_key", value="new_value", overwrite=False) - + except KeyError as e: assert str(e) == "'Key already exists in datablock and overwrite is set to False.'" \ No newline at end of file diff --git a/agrifoodpy/utils/write_to_datablock.py b/agrifoodpy/utils/write_to_datablock.py deleted file mode 100644 index 3ff5506..0000000 --- a/agrifoodpy/utils/write_to_datablock.py +++ /dev/null @@ -1,29 +0,0 @@ -from .dict_utils import set_dict - -def write_to_datablock(datablock, key, value, overwrite=True): - """Writes a value to a specified key in the datablock. - - Parameters - ---------- - datablock : dict - The datablock to write to. - key : str - The key in the datablock where the value will be written. - value : any - The value to write to the datablock. - overwrite : bool, optional - If True, overwrite the existing value at the key. - If False, do not overwrite. - - Returns - ------- - datablock : dict - The updated datablock with the new key-value pair. - """ - - if not overwrite and key in datablock: - raise KeyError(f"Key already exists in datablock and overwrite is set to False.") - - set_dict(datablock, key, value) - - return datablock \ No newline at end of file diff --git a/examples/pipeline/plot_balanced_scaling_food_balance_sheet_pipeline.py b/examples/pipeline/plot_balanced_scaling_food_balance_sheet_pipeline.py index a4b9a64..59e04e4 100644 --- a/examples/pipeline/plot_balanced_scaling_food_balance_sheet_pipeline.py +++ b/examples/pipeline/plot_balanced_scaling_food_balance_sheet_pipeline.py @@ -7,14 +7,14 @@ pipeline of modules. In this particular example, we will load a food balance sheet dataset, compute -prelimimary Self-Sufficiency and Import Dependency Ratios (SSR, IDR) and add new -items and years to the dataset. +prelimimary Self-Sufficiency and Import Dependency Ratios (SSR, IDR) and add +new items and years to the dataset. We will also print the SSR and IDR values to the console. -Finally, we will scale the food balance sheet to reduce animal products and plot -the results. +Finally, we will scale the food balance sheet to reduce animal products and +plot the results. """ -#%% +# %% # We start by creating a pipeline object, which will manage the flow of data # through the different modules. @@ -22,47 +22,40 @@ from matplotlib import pyplot as plt from agrifoodpy.pipeline.pipeline import Pipeline -from agrifoodpy.food.food import FoodBalanceSheet - -import agrifoodpy.food.model as afpm -from agrifoodpy.utils.load_dataset import load_dataset -from agrifoodpy.utils.write_to_datablock import write_to_datablock -from agrifoodpy.utils.add_items import add_items -from agrifoodpy.utils.print_datablock import print_datablock -from agrifoodpy.utils.add_items import add_items -from agrifoodpy.utils.add_years import add_years -from agrifoodpy.utils.scaling import linear_scale +from agrifoodpy.food import model +from agrifoodpy.utils import nodes +from agrifoodpy.utils.scaling import linear_scale # Create a pipeline object pipeline = Pipeline() -#%% +# %% # We add a node to the pipeline to load a food balance sheet dataset. # Load a dataset pipeline.add_node( - load_dataset, + nodes.load_dataset, name="Load Dataset", params={ "datablock_path": "food", "module": "agrifoodpy_data.food", - "data_attr":"FAOSTAT", - "coords":{ + "data_attr": "FAOSTAT", + "coords": { "Item": [2731, 2511], "Year": [2019, 2020], - "Region": 229,}, + "Region": 229}, } ) -#%% +# %% # We add a node to the pipeline to store a conversion factor in the # datablock. This conversion factor will be used to convert the food balance # sheet data from 1000 tonnes to kgs. # Add convertion factors to the datablock pipeline.add_node( - write_to_datablock, + nodes.write_to_datablock, name="Write to datablock", params={ "key": "tonnes_to_kgs", @@ -72,7 +65,7 @@ # Convert food data from 1000 tonnes to kgs pipeline.add_node( - afpm.fbs_convert, + model.fbs_convert, name="Convert from 1000 tonnes to kgs", params={ "fbs": "food", @@ -80,36 +73,37 @@ } ) -#%% -# Compute preliminary Self-Sufficiency Ratio (SSR) and Import Dependency Ratio (IDR) +# %% +# Compute preliminary Self-Sufficiency Ratio (SSR) and Import Dependency Ratio +# (IDR) # Compute IDR and SSR for food pipeline.add_node( - afpm.SSR, + model.SSR, name="Compute SSR for food", params={ - "fbs":"food", - "out_key":"SSR" + "fbs": "food", + "out_key": "SSR" } ) # Compute IDR and SSR for food pipeline.add_node( - afpm.IDR, + model.IDR, name="Compute IDR for food", params={ - "fbs":"food", - "out_key":"IDR" + "fbs": "food", + "out_key": "IDR" } ) -#%% +# %% # Print the SSR and IDR values to the console # Add a print node to display the SSR pipeline.add_node( - print_datablock, + nodes.print_datablock, name="Print SSR", params={ "key": "SSR", @@ -118,26 +112,26 @@ } ) -#%% +# %% # Now we can add new items to the food balance sheet dataset. # Add an item to the food dataset pipeline.add_node( - add_items, + nodes.add_items, name="Add item to food", params={ "dataset": "food", "items": { - "Item":5000, - "Item_name":"Cultured meat", - "Item_group":"Cultured products", - "Item_origin":"Synthetic origin", + "Item": 5000, + "Item_name": "Cultured meat", + "Item_group": "Cultured products", + "Item_origin": "Synthetic origin", }, - "copy_from":2731 + "copy_from": 2731 } ) -#%% +# %% # We can also add new years to the food balance sheet dataset. projection = np.linspace(1.1, 2.0, 10) @@ -145,7 +139,7 @@ # Extend the year range of the food dataset pipeline.add_node( - add_years, + nodes.add_years, name="Add years to food", params={ "dataset": "food", @@ -154,12 +148,12 @@ } ) -#%% +# %% # We execute the pipeline to run all the nodes in order. pipeline.run(timing=True) -#%%# Finally, we plot the results +# %% Finally, we plot the results # Get the food results from the pipeline and plot using the fbs accessor food_results = pipeline.datablock["food"]["food"] @@ -168,11 +162,13 @@ food_results.fbs.plot_years(show="Item_name", labels="show", ax=ax) plt.show() -#%% +# %% # We can continue adding nodes to the pipeline, even after being executed -# once. To pick up where we left, we indicate which node to start execution from +# once. To pick up where we left, we indicate which node to start execution +# from -# Define a year dependent linear scale starting decreasing at 2021 from 1 to 0.5 +# Define a year dependent linear scale starting decreasing at 2021 from 1 to +# 0.5 scaling = linear_scale( 2019, 2021, @@ -184,15 +180,15 @@ # We will add a node to scale consumption pipeline.add_node( - afpm.balanced_scaling, + model.balanced_scaling, name="Balanced scaling of items", params={ - "fbs":"food", - "scale":scaling, - "element":"food", - "items":("Item_name", "Bovine Meat"), - "constant":True, - "out_key":"food_scaled" + "fbs": "food", + "scale": scaling, + "element": "food", + "items": ("Item_name", "Bovine Meat"), + "constant": True, + "out_key": "food_scaled" } ) @@ -206,7 +202,7 @@ scaled_food_results.fbs.plot_years(show="Item_name", labels="show", ax=ax) plt.show() -#%% -# We can see in the scaled Food Balance Sheet that Bovine Meat consumption is +# %% +# We can see in the scaled Food Balance Sheet that Bovine Meat consumption is # reduced by half by 2030, while the total sum across all items remains -# constant. \ No newline at end of file +# constant. From dee8aab4bc7fa01ee88f08087c9f3e26b803bda8 Mon Sep 17 00:00:00 2001 From: jucordero Date: Mon, 1 Sep 2025 14:38:27 +0100 Subject: [PATCH 4/4] missing init file --- agrifoodpy/utils/tests/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 agrifoodpy/utils/tests/__init__.py diff --git a/agrifoodpy/utils/tests/__init__.py b/agrifoodpy/utils/tests/__init__.py new file mode 100644 index 0000000..e69de29