Use esgvoc in dr2xml

.github/workflows/test_python.yml

@@ -15,7 +15,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
 
     steps:
     - uses: actions/checkout@v4
@@ -29,6 +29,7 @@ jobs:
         python -m pip install flake8 pytest
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
         python -m pip install git+https://github.com/CMIP-Data-Request/CMIP7_DReq_Software.git
+        python -m pip install git+https://github.com/ESGF/esgf-vocab.git
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
@@ -39,5 +40,4 @@ jobs:
       run: |
         export CMIP7_DR_API_CONFIGFILE=$PWD/dr2xml/dr_interface/CMIP7_config
         python3 -m data_request_api.command_line.config offline true
-        for f in $(ls tests/test_*/__init__.py); do echo $f; python3 -m unittest $f; done
-        for f in $(ls tests/test_*.py); do echo $f; python3 -m unittest $f; done
+        bash launch_tests.sh

.gitignore

@@ -26,7 +26,7 @@ pycallgraph_tree/
 dr2xml_*.xml
 pycallgraph_use.py
 .coverage
-create_file_defs.sh
+create_file_defs*.sh
 tests/htmlcov
 tests/test*/output_*
 /sphinx/build/

dr2xml/Xwrite.py

@@ -430,10 +430,13 @@ def write_xios_file_def(filename, svars_per_table, year, dummies, skipped_vars_p
     required_components = internal_dict["required_model_components"]
     if not isinstance(required_components, list):
         required_components = [required_components, ]
+    required_components = [elt.lower() for elt in required_components]
     allowed_components = internal_dict["additional_allowed_model_components"]
     if not isinstance(allowed_components, list):
         allowed_components = [allowed_components, ]
+    allowed_components = [elt.lower() for elt in allowed_components]
     actual_components = source_type.split(" ")
+    actual_components = [elt.lower() for elt in actual_components]
     ok = True
     for c in required_components:
         if c not in actual_components:
@@ -547,30 +550,33 @@ def write_xios_file_def_for_svars_list(vars_list, hgrid, xml_file_definition, fr
     internal_dict = get_settings_values("internal")
     context = internal_dict["context"]
     prefix = internal_dict["ping_variables_prefix"]
-    # Initialize xml file
-    xml_file = DR2XMLElement(tag="file", default_tag="file_output",
-                             output_freq=freq, split_freq=split_freq,
-                             split_freq_format=split_freq_format, split_start_offset=split_start_offset,
-                             split_end_offset=split_end_offset, split_last_date=split_last_date, grid=grid_description,
-                             grid_label=grid_label, nominal_resolution=grid_resolution, variable=vars_list,
-                             context=context, table_id=table)
-    # Add several attributes
-    for name, value in sorted(list(attributes)):
-        xml_file.append(wrv(name, value))
-    non_stand_att = internal_dict["non_standard_attributes"]
-    for name in sorted(list(non_stand_att)):
-        xml_file.append(wrv(name, non_stand_att[name]))
-    # For each variable, add the elements about the variable
-    found = False
-    found_A = False
-    found_AH = False
-    found_begin_A = False
-    freq_ps = vars_list[0].frequency
+    # Check if this file should be written
+    real_vars_list = list()
     for svar in sorted(vars_list):
         rep = find_alias(svar, skipped_vars_per_table, debug)
         if rep is not None:
+            real_vars_list.append((svar, rep))
+    if len(real_vars_list) > 0:
+        # Initialize xml file
+        xml_file = DR2XMLElement(tag="file", default_tag="file_output",
+                                 output_freq=freq, split_freq=split_freq,
+                                 split_freq_format=split_freq_format, split_start_offset=split_start_offset,
+                                 split_end_offset=split_end_offset, split_last_date=split_last_date, grid=grid_description,
+                                 grid_label=grid_label, nominal_resolution=grid_resolution, variable=[real_vars_list[0][0], ],
+                                 context=context, table_id=table)
+        # Add several attributes
+        for name, value in sorted(list(attributes)):
+            xml_file.append(wrv(name, value))
+        non_stand_att = internal_dict["non_standard_attributes"]
+        for name in sorted(list(non_stand_att)):
+            xml_file.append(wrv(name, non_stand_att[name]))
+        # For each variable, add the elements about the variable
+        found_A = False
+        found_AH = False
+        found_begin_A = False
+        freq_ps = real_vars_list[0][0].frequency
+        for (svar, rep) in sorted(real_vars_list):
             alias, alias_ping = rep
-            found = True
             if svar.spatial_shp.startswith("XY-A") or svar.spatial_shp.startswith("S-A"):
                 found_begin_A = True
                 if svar.spatial_shp in ["XY-A", "S-A"]:
@@ -584,12 +590,11 @@ def write_xios_file_def_for_svars_list(vars_list, hgrid, xml_file_definition, fr
             xml_file.append(end_field)
             actually_written_vars.append((svar.label, svar.long_name, svar.stdname, svar.mipTable, svar.frequency,
                                           svar.Priority, svar.spatial_shp))
-    # Add content to xml_file to out
-    if found:
+        # Add content to xml_file to out
         if found_begin_A:
             # create a field_def entry for surface pressure
             # print "Searching for ps for var %s, freq %s="%(alias,freq)
-            sv_psol = get_simplevar("ps", table, freq_ps)
+            sv_psol = get_simplevar("ps", real_vars_list[0][0])
 
             if sv_psol:
                 # if not sv_psol.cell_measures : sv_psol.cell_measures = "cell measure is not specified in DR "+

dr2xml/__init__.py

@@ -652,6 +652,7 @@ def generate_file_defs(year, enddate, context, pingfiles=None, dummies='include'
 
     from .settings_interface import get_settings_values
 
+    init_settings = get_settings_values("init")
     internal_settings = get_settings_values("internal")
 
     print("\n {}\n*".format(50 * "*"))
@@ -663,7 +664,7 @@ def generate_file_defs(year, enddate, context, pingfiles=None, dummies='include'
     # TBS# from os import path as os_path
     # TBS# prog_path=os_path.abspath(os_path.split(__file__)[0])
 
-    print("* %29s" % "{} Data Request version: ".format(internal_settings["data_request_used"]), get_dr_object("get_data_request").get_version())
+    print("* %29s" % "{} Data Request version: ".format(init_settings["data_request_used"]), get_dr_object("get_data_request").get_version())
     print("\n*\n {}".format(50 * "*"))
 
     logger = get_logger()

dr2xml/analyzer.py

@@ -256,6 +256,15 @@ def analyze_cell_time_method(cm, label, table):
                     " for %15s in table %s is well handled by 'detect_missing'" % (label, table))
         operation = "average"
         detect_missing = True
+        # ----------------------------------------------------------------------------------------------------------------
+    elif "time: mean where land" in cm:
+        # Weighted Time Mean on Land Tiles
+        add_value_in_list_config_variable("cell_method_warnings",
+                                          ('time: mean where land', label, table))
+        logger.info("Note: assuming that 'time: mean where land' "
+                    " for %15s in table %s is well handled by 'detect_missing'" % (label, table))
+        operation = "average"
+        detect_missing = True
     # ----------------------------------------------------------------------------------------------------------------
     elif "time: mean where crops" in cm:
         # [amc-twm]: Weighted Time Mean on Crops (uniquement des

dr2xml/dr_interface/C3S.py

@@ -18,7 +18,7 @@
 from .definition import SimpleDim as SimpleDimBasic
 from dr2xml.settings_interface import get_settings_values
 
-data_request_path = get_settings_values("internal", "data_request_path")
+data_request_path = get_settings_values("init", "data_request_path")
 if data_request_path is not None:
     data_request_filename = os.path.basename(data_request_path)
     data_request_module = SourceFileLoader(data_request_filename, data_request_path).load_module(data_request_filename)
@@ -88,6 +88,9 @@ def get_cmorvars_list(self, **kwargs):
                 rep[id].add(grid)
         return rep
 
+    def get_ps_data(self, reference_var):
+        return None
+
 
 def initialize_data_request():
     global data_request

dr2xml/dr_interface/CMIP6.py

@@ -21,15 +21,15 @@
 from .definition import SimpleObject
 from .definition import SimpleDim as SimpleDimBasic
 from .definition import SimpleCMORVar as SimpleCMORVarBasic
-from ..projects.dr2xml import format_sizes
-from ..utils import Dr2xmlError, print_struct, is_elt_applicable, convert_string_to_year
+from dr2xml.projects.dr2xml_func import format_sizes
+from dr2xml.utils import Dr2xmlError, print_struct, is_elt_applicable, convert_string_to_year
 from dr2xml.settings_interface import get_settings_values, get_values_from_internal_settings
 
-data_request_path = get_settings_values("internal", "data_request_path")
+data_request_path = get_settings_values("init", "data_request_path")
 if data_request_path is not None:
     sys.path.insert(0, data_request_path)
 
-data_request_content_version = get_settings_values("internal", "data_request_content_version")
+data_request_content_version = get_settings_values("init", "data_request_content_version")
 if data_request_content_version not in ["latest_stable", "stable", "latest"]:
     reset_manifest = True
     os.environ["DRQ_CONFIG_DIR"] = data_request_content_version
@@ -177,7 +177,7 @@ def _check_requestitem_for_exp_and_year(self, ri, experiment, year, filter_on_re
 
         if ri_applies_to_experiment:
             logger.debug("Year considered: %s %s" % (year, type(year)))
-            if year is None:
+            if year is False:
                 rep = True
                 endyear = None
                 logger.debug(" ..applies because arg year is None")
@@ -514,6 +514,25 @@ def _get_vars_by_request_link(self, request_link, pmax):
             request_link = [request_link, ]
         return self.scope.varsByRql(request_link, pmax)
 
+    def get_ps_data(self, reference_var):
+        rep = None
+        table = reference_var.mipTable
+        freq = reference_var.frequency
+        if freq in ["3h", "3hr", "3hrPt"]:
+            rep = dict(label='ps', mipTable='E3hrPt')
+        elif freq in ["6h", "6hr"]:
+             rep = dict(label='ps', mipTable='6hrLev')
+        elif freq in ["day", ]:
+            rep = dict(label='ps', mipTable='CFday')
+        elif freq in ["mon", "1mo"]:
+            rep = dict(label='ps', mipTable='Emon')
+        elif freq in ["subhr", ]:
+            if table in ["CFsubhr", ]:
+                rep = dict(label='ps', mipTable='CFsubhr')
+            else:
+                rep = dict(label='ps', mipTable='Esubhr')
+        return rep
+
 
 data_request = None
 

dr2xml/dr_interface/CMIP7.py

@@ -18,11 +18,11 @@
 from .definition import SimpleCMORVar as SimpleCMORVarBasic
 from .definition import SimpleDim as SimpleDimBasic
 from dr2xml.settings_interface import get_settings_values
-from ..utils import Dr2xmlError, is_elt_applicable
+from dr2xml.utils import Dr2xmlError, is_elt_applicable
 
-data_request_path = get_settings_values("internal", "data_request_path")
+data_request_path = get_settings_values("init", "data_request_path")
 sys.path.append(data_request_path)
-os.environ["CMIP7_DR_API_CONFIGFILE"] = get_settings_values("internal", "data_request_config")
+os.environ["CMIP7_DR_API_CONFIGFILE"] = get_settings_values("init", "data_request_config")
 from data_request_api.query.vocabulary_server import ConstantValueObj
 from data_request_api.query.data_request import DataRequest as CMIP7DataRequest
 from data_request_api.content.dump_transformation import get_transformed_content
@@ -100,7 +100,12 @@ def get_element_uid(self, id=None, elt_type=None, error_msg=None, raise_on_error
         if elt_type is None:
             raise ValueError("Unable to find out uid with elt_type None")
         if id is None:
-            return self.data_request.get_elements_per_kind(elt_type)
+            rep = self.data_request.get_elements_per_kind(elt_type)
+            if elt_type in ["variable", ]:
+                rep = [SimpleCMORVar.get_from_dr(elt, id=elt.id, **kwargs) for elt in rep]
+            elif elt_type in ["dimension", ]:
+                rep = [SimpleDim.get_from_dr(elt, id=elt.id) for elt in rep]
+            return rep
         else:
             if elt_type in ["dim", ]:
                 elt_type = "dimension"
@@ -137,26 +142,29 @@ def get_single_levels_list(self):
     def get_grids_dict(self):
         rep = OrderedDict()
         dims = self.get_list_by_id("coordinates_and_dimensions")
-        for dim in dims.items:
+        for dim in dims:
             rep[dim.name] = dim.id
         return rep
 
     def get_dimensions_dict(self):
         rep = OrderedDict()
-        for spshp in self.get_list_by_id("spatial_shape").items:
-            dims = [elt.name for elt in spshp.dimensions]
-            new_dims = list()
-            for key in ["longitude", "latitude"]:
-                if key in dims:
-                    dims.remove(key)
-                    new_dims.append(key)
-            new_dims.extend(sorted(dims))
-            new_dims = "|".join([str(dim) for dim in new_dims])
-            rep[new_dims] = str(spshp.name)
+        excluded_spshapes = get_settings_values("internal", "excluded_spshapes_lset")
+        for spshp in self.get_list_by_id("spatial_shape"):
+            shape_name = str(spshp.name)
+            if shape_name not in excluded_spshapes:
+                dims = [elt.name for elt in spshp.dimensions]
+                new_dims = list()
+                for key in ["longitude", "latitude"]:
+                    if key in dims:
+                        dims.remove(key)
+                        new_dims.append(key)
+                new_dims.extend(sorted(dims))
+                new_dims = "|".join([str(dim) for dim in new_dims])
+                rep[new_dims] = str(shape_name)
         return rep
 
     def _is_timesubset_applicable(self, year, select_on_year, time_subset):
-        if year is None or select_on_year is None:
+        if year is False or select_on_year is False:
             return None, None
         else:
             return ((time_subset.start is None or (time_subset.start <= int(year))) and
@@ -166,9 +174,11 @@ def _get_filtering_elements(self, experiment=None, variable=None):
         internal_dict = get_settings_values("internal")
         request_dict_all_of_any = dict(opportunities=internal_dict["select_included_opportunities"],
                                        variable_groups=internal_dict["select_included_vargroups"],
+                                       experiment_groups=internal_dict["select_included_expgroups"],
                                        max_priority_level=internal_dict["select_max_priority"])
         not_request_dict_any = dict(opportunity=internal_dict["select_excluded_opportunities"],
-                                    variable_groups=internal_dict["select_excluded_vargroups"])
+                                    variable_groups=internal_dict["select_excluded_vargroups"],
+                                    experiment_groups=internal_dict["select_excluded_expgroups"])
         select_mips = internal_dict["select_mips"]
         if len(select_mips) > 0:
             request_dict_all_of_any["mip"] = select_mips
@@ -218,12 +228,17 @@ def get_endyear_for_cmorvar(self, cmorvar, experiment, year, internal_dict):
         else:
             return max(time_subsets)
 
+    def get_ps_data(self, reference_var):
+        rep = dict(label="ps", frequency=reference_var.frequency, spatial_shp=reference_var.spatial_shp,
+                   temporal_shp=reference_var.temporal_shp)
+        return rep
+
 
 def initialize_data_request():
     global data_request
     if data_request is None:
-        internal_dict = get_settings_values("internal")
-        data_request_content_version = internal_dict["data_request_content_version"]
+        init_dict = get_settings_values("init")
+        data_request_content_version = init_dict["data_request_content_version"]
         content = get_transformed_content(version=data_request_content_version,
                                           force_retrieve=False)
         data_request = DataRequest(CMIP7DataRequest.from_separated_inputs(**content), print_DR_errors=True,
@@ -266,14 +281,17 @@ def get_from_dr(cls, input_var, **kwargs):
             cell_measures = [cell_measures, ]
         else:
             cell_measures = [cell_measures.name, ]
+        cell_measures = " ".join([str(elt) for elt in cell_measures])
         cell_methods = input_var.cell_methods.cell_methods
+        official_label = str(input_var.branded_variable_name)
         logger = get_logger()
-        logger.debug(f"Variable considered: {input_var.name}")
+        logger.debug(f"Variable considered: {input_var.name} (branded name {input_var.branded_variable_name}, official label {official_label})")
         return cls(from_dr=True,
-                   type=input_var.type,
+                   type="cmor",
                    modeling_realm=[realm.id for realm in input_var.modelling_realm],
                    label=input_var.physical_parameter.name,
                    mipVarLabel=input_var.physical_parameter.name,
+                   official_label=official_label,
                    label_without_psuffix=input_var.physical_parameter.name,
                    label_non_ambiguous=input_var.name,
                    frequency=input_var.cmip7_frequency.name,
@@ -291,7 +309,8 @@ def get_from_dr(cls, input_var, **kwargs):
                    temporal_shp=input_var.temporal_shape.name,
                    id=input_var.id,
                    cmvar=input_var,
-                   Priority=data_request.data_request.find_priority_per_variable(input_var)
+                   Priority=data_request.data_request.find_priority_per_variable(input_var),
+                   region=input_var.region
                    )
 
 

dr2xml/dr_interface/__init__.py

@@ -46,7 +46,7 @@ def load_correct_dr():
     global data_request, DataRequest, initialize_data_request, get_data_request, \
         normalize_grid, SimpleDim, SimpleObject, SimpleCMORVar
 
-    data_request_version = get_settings_values("internal", "data_request_used")
+    data_request_version = get_settings_values("init", "data_request_used")
 
     if data_request_version in ["CMIP6", ]:
         from .CMIP6 import data_request, DataRequest, initialize_data_request, get_data_request, \