Prepare V3.2 version

.coveragerc

@@ -0,0 +1,15 @@
+[run]
+parallel = True
+branch = True
+command_line = -m unittest
+source =
+    cfsites
+    dr2xml
+    scripts
+    xml_writer
+    logger.py
+    utilities
+
+[report]
+exclude_lines =
+    if __name__ == "__main__":

.github/workflows/test_python.yml

@@ -0,0 +1,43 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Test python 3.8
+
+on:
+  - push
+  - workflow_dispatch
+  - pull_request
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install flake8 pytest
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        python -m pip install git+https://github.com/CMIP-Data-Request/CMIP7_DReq_Software.git
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+    - name: Test with unittest
+      run: |
+        export CMIP7_DR_API_CONFIGFILE=$PWD/dr2xml/dr_interface/CMIP7_config
+        python3 -m data_request_api.command_line.config offline true
+        for f in $(ls tests/test_*/__init__.py); do echo $f; python3 -m unittest $f; done
+        for f in $(ls tests/test_*.py); do echo $f; python3 -m unittest $f; done

dr2xml/Xparse.py

@@ -18,7 +18,7 @@
 from .xml_interface import get_root_of_xml_file, is_xml_element_to_parse, find_rank_xml_subelement
 
 # Logger
-from logger import get_logger
+from utilities.logger import get_logger
 
 
 # Define for each object kind those attributes useful for grid inheritance
@@ -169,7 +169,7 @@ def attrib_by_ref(elt, attrib, index, level):
             if rep:
                 return rep
         except:
-            if not refid.startswith("dummy_"):
+            if not "dummy" in refid:
                 raise XparseError("Error : reference '%s' is invalid" % refid)
 
 
@@ -285,25 +285,6 @@ def id_has_expr_with_at(field_id, index):
         # raise Xparse_error("field %s is not known"%field_id)
 
 
-if False:
-
-    nemo = init_context('nemo', "./", False)
-    # print list(nemo)
-    grid = id2grid("CMIP6_O18sw", nemo, True)
-    print(grid.attrib['id'])
-    print()
-
-    arpsfx = init_context('arpsfx', "./", False)
-    grid = id2grid("CMIP6_cdnc", arpsfx, True)
-    # grid=None
-    if grid is not None:
-        # print "Grid id is :"+grid.attrib['id']
-        print(create_string_from_xml_element(grid))
-        grid_string = create_string_from_xml_element(grid)
-        new_grid_string = re.sub(r'axis_ref= *.([\w_])*.', 'axis_ref="axis_autre"', grid_string)
-        print(new_grid_string)
-
-
 class XparseError(Exception):
     """
     Xparse exceptions class.

dr2xml/Xwrite.py

@@ -18,7 +18,7 @@
 from .utils import Dr2xmlError
 
 # Logger
-from logger import get_logger
+from utilities.logger import get_logger, change_log_level
 
 # Global variables and configuration tools
 from .config import get_config_variable, set_config_variable, add_value_in_dict_config_variable
@@ -156,7 +156,7 @@ def create_xios_aux_elmts_defs(sv, alias, table, context, target_hgrid_id, zgrid
     # Build a construct for computing a climatology (if applicable)
     # --------------------------------------------------------------------
     if clim:
-        if sv.frequency in ["1hrCM", ]:
+        if sv.frequency in ["1hrCM", "1hr"]:
             last_field_id, last_grid_id = process_diurnal_cycle(last_field_id)
         else:
             raise Dr2xmlError("Cannot handle climatology cell_method for frequency %s and variable %s"
@@ -462,7 +462,7 @@ def write_xios_file_def(filename, svars_per_table, year, dummies, skipped_vars_p
     set_config_variable("domain_defs", OrderedDict())
     # Add xml_file_definition
     xml_file_definition = DR2XMLElement(tag="file_definition")
-    _, hgrid, _, _, _ = internal_dict['grids'][get_settings_values("internal_values", "grid_choice")][context]
+    _, hgrid, _, _, _ = internal_dict['grids'][internal_dict["select_grid_choice"]][context]
     files_list = determine_files_list(svars_per_table, enddate, year, debug)
     for file_dict in files_list:
         write_xios_file_def_for_svars_list(hgrid=hgrid, xml_file_definition=xml_file_definition, dummies=dummies,
@@ -767,8 +767,7 @@ def get_split_info(sv, table, enddate, year, debug):
             endmonth = "01"
             endday = "01"
             split_last_date = "{}-{}-{} 00:00:00".format(endyear, endmonth, endday)
-        sc = get_dr_object("get_scope")
-        split_freq = determine_split_freq(sv, grid_choice, sc.mcfg, context)
+        split_freq = determine_split_freq(sv, grid_choice, context)
     return split_freq_format, split_last_date, split_start_offset, split_end_offset, split_freq
 
 

dr2xml/__init__.py

@@ -61,7 +61,7 @@
 from .utils import print_struct
 
 # Logger
-from logger import initialize_logger, get_logger, change_log_level
+from utilities.logger import initialize_logger, get_logger, change_log_level
 
 # Global variables and configuration tools
 from .config import get_config_variable, set_config_variable, initialize_config_variables
@@ -575,7 +575,7 @@ def configuration_init(func):
     :return: The initial function with initialized environment to use dr2xml.
     """
     def make_configuration(lset, sset, cvs_path=None, printout=False, prefix="", debug=False, force_reset=False,
-                           **kwargs):
+                           select="on_expt_and_year", **kwargs):
         year = kwargs.get("year", 0)
         context = kwargs.get("context")
         dirname = kwargs.get("dirname")
@@ -588,14 +588,13 @@ def make_configuration(lset, sset, cvs_path=None, printout=False, prefix="", deb
         initialize_logger(default=True, level=default_level)
         initialize_config_variables()
         initialize_settings(lset=lset, sset=sset, cvspath=cvs_path, context=context, prefix=prefix,
-                            year=year, dirname=dirname, force_reset=force_reset)
+                            year=year, dirname=dirname, force_reset=force_reset, select=select)
         return func(**kwargs)
     return make_configuration
 
 
 @configuration_init
-def generate_file_defs(year, enddate, context, pingfiles=None, dummies='include', dirname="./", attributes=list(),
-                       select="on_expt_and_year"):
+def generate_file_defs(year, enddate, context, pingfiles=None, dummies='include', dirname="./", attributes=list()):
     """
     Using the DR module, a dict of lab settings ``lset``, and a dict
     of simulation settings ``sset``, generate an XIOS file_defs 'file' for a
@@ -664,7 +663,7 @@ def generate_file_defs(year, enddate, context, pingfiles=None, dummies='include'
     # TBS# from os import path as os_path
     # TBS# prog_path=os_path.abspath(os_path.split(__file__)[0])
 
-    print("* %29s" % "CMIP6 Data Request version: ", get_dr_object("get_data_request").get_version())
+    print("* %29s" % "{} Data Request version: ".format(internal_settings["data_request_used"]), get_dr_object("get_data_request").get_version())
     print("\n*\n {}".format(50 * "*"))
 
     logger = get_logger()
@@ -699,7 +698,7 @@ def generate_file_defs(year, enddate, context, pingfiles=None, dummies='include'
     # --------------------------------------------------------------------
     skipped_vars_per_table = OrderedDict()
     actually_written_vars = list()
-    svars_per_table = select_variables_to_be_processed(year, context, select)
+    svars_per_table = select_variables_to_be_processed()
     #
     # --------------------------------------------------------------------
     # Read ping_file defined variables
@@ -753,14 +752,15 @@ def generate_file_defs(year, enddate, context, pingfiles=None, dummies='include'
 
 
 @configuration_init
-def create_ping_files(context, path_special, dummy="field_atm", dummy_with_shape=False, exact=False, comments=False,
-                      filename=None, debug=list(), by_realm=False):
+def create_ping_files(context, dummy="field_atm", dummy_with_shape=False, exact=False, comments=False,
+                      filename=None, debug=list(), by_realm=False, **kwargs):
     from .settings_interface import get_settings_values
     from .vars_interface.selection import select_variables_to_be_processed
     from .pingfiles_interface import ping_file_for_realms_list
 
     considered_realms = get_settings_values("internal", "realms_per_context")
-    svars = select_variables_to_be_processed(None, context, "no")
+    path_special = get_settings_values("internal", "path_special_defs")
+    svars = select_variables_to_be_processed()
     if by_realm:
         for realm in considered_realms:
             ping_file_for_realms_list(context=context, svars=svars, lrealms=[realm, ], path_special=path_special,

dr2xml/analyzer.py

@@ -24,7 +24,7 @@
 from .utils import Dr2xmlError
 
 # Logger
-from logger import get_logger
+from utilities.logger import get_logger
 
 # Global variables and configuration tools
 from .config import add_value_in_list_config_variable
@@ -210,8 +210,12 @@ def analyze_cell_time_method(cm, label, table):
         operation = "average"
         detect_missing = True
     elif "time: mean where sea" in cm:  # [amnesi-tmn]:
-        # Area Mean of Ext. Prop. on Sea Ice : pas utilisee
-        logger.warning("time: mean where sea is not supposed to be used (%s,%s)" % (label, table))
+        add_value_in_list_config_variable("cell_method_warnings",
+                                          ('time: mean where sea', label, table))
+        logger.info("Note: assuming that 'time: mean where sea' "
+                    " for %15s in table %s is well handled by 'detect_missing'" % (label, table))
+        operation = "average"
+        detect_missing = True
     # -------------------------------------------------------------------------------------
     elif "time: mean where floating_ice_shelf" in cm:
         # [amnfi-twmn]: Weighted Time Mean on Floating Ice Shelf (presque que des

dr2xml/dr_interface/C3S.py

@@ -12,7 +12,6 @@
 from importlib.machinery import SourceFileLoader
 
 from .definition import ListWithItems
-from .definition import Scope as ScopeBasic
 from .definition import DataRequest as DataRequestBasic
 from .definition import SimpleObject
 from .definition import SimpleCMORVar as SimpleCMORVarBasic
@@ -31,13 +30,12 @@
     from .C3S_DR import c3s_nc_dims, c3s_nc_coords, c3s_nc_comvars, c3s_nc_vars
 
 
-scope = None
 data_request = None
 
 
 class DataRequest(DataRequestBasic):
     def get_version(self):
-        return "No Data Request"
+        return "undef"
 
     def get_list_by_id(self, collection, **kwargs):
         return ListWithItems()
@@ -83,27 +81,12 @@ def get_grids_dict(self):
     def get_dimensions_dict(self):
         return OrderedDict()
 
-    def get_cmorvars_list(self, sizes=None, **kwargs):
-        if sizes is not None:
-            sc = get_scope()
-            sc.update_mcfg(sizes)
+    def get_cmorvars_list(self, **kwargs):
         rep = defaultdict(set)
         for id in self.get_element_uid(elt_type="variable"):
             for grid in self.get_element_uid(id=id, elt_type="variable").grids:
                 rep[id].add(grid)
-        return rep, list()
-
-
-class Scope(ScopeBasic):
-
-    def __init__(self, scope=None):
-        super().__init__(scope=scope)
-
-    def get_request_link_by_mip(self, mips_list):
-        return list()
-
-    def get_vars_by_request_link(self, request_link, pmax):
-        return list()
+        return rep
 
 
 def initialize_data_request():
@@ -120,27 +103,6 @@ def get_data_request():
         return data_request
 
 
-def initialize_scope(tier_max):
-    global scope
-    dq = get_data_request()
-    if scope is None:
-        scope = Scope()
-    return scope
-
-
-def get_scope(tier_max=None):
-    if scope is None:
-        return initialize_scope(tier_max)
-    else:
-        return scope
-
-
-def set_scope(sc):
-    if sc is not None:
-        global scope
-        scope = sc
-
-
 def normalize_grid(grid):
     return grid