Update documentation

dr2xml/dr_interface/CMIP7_config

@@ -1,3 +1,4 @@
+cache_dir: /home/rigoudyg/dev/data_request/CMIP7_DReq_Software/data_request_api/data_request_api/content/dreq_api/dreq_res
 check_api_version: false
 consolidate: false
 export: release

dr2xml/projects/dr2xml.py

@@ -84,7 +84,7 @@ def sort_mips(*mips):
 	),
 	synchronisation_frequency=ParameterSettings(
 		key="synchronisation_frequency",
-		help="Frequency at which the synchornisation between buffer and filesystem is done.",
+		help="Frequency at which the synchronisation between buffer and filesystem is done.",
 		default_value=[
 			ValueSettings(key_type="simulation", keys="synchronisation_frequency"),
 			ValueSettings(key_type="laboratory", keys="synchronisation_frequency"),

dr2xml/projects/projects_interface_definitions.py

@@ -346,7 +346,7 @@ def init_dict_default(self):
         return dict(skip_values=list(), forbidden_patterns=list(), conditions=list(), default_values=list(),
                     cases=list(), authorized_values=list(), authorized_types=list(), corrections=dict(),
                     output_key=None, num_type="string", is_default=False, fatal=False, key=None, help="TODO",
-                    target_type=None)
+                    target_type=None, section=None)
 
     def dump_doc(self, force_void=False):
         rep = list()

dr2xml/settings_interface/py_project_interface.py

@@ -7,6 +7,7 @@
 from __future__ import print_function, division, absolute_import, unicode_literals
 
 import os
+from collections import defaultdict
 from importlib.machinery import SourceFileLoader
 
 from .py_settings_interface import get_variable_from_lset_with_default_in_lset, get_variable_from_lset_with_default, \
@@ -32,26 +33,52 @@ def initialize_project_settings(dirname, doc_writer=False):
     return internal_values, common_values, project_settings
 
 
-def write_project_documentation(internal_values, common_values, project_settings, dirname, project):
-    target_filename = os.sep.join([dirname, project + ".rst"])
+def write_project_dict_content(list_values, title, title_level, level=0):
     content = list()
-    content.append("Parameters available for project %s" % project)
-    content.append("=" * len(content[0]))
-    content.append("")
-    content.append("Internal values")
-    content.append("---------------")
+    content.append(title)
+    content.append(title_level * len(title))
     content.append(".. glossary::")
     content.append("   :sorted:")
     content.append("   ")
+    content.extend(list_values)
+    content = ["   " * level + elt for elt in content]
+    return content
+
+
+def write_project_documentation(internal_values, common_values, project_settings, dirname, project):
+    param_content = defaultdict(lambda: defaultdict(list))
     for value in sorted(list(internal_values)):
-        content.extend(internal_values[value].dump_doc())
-    content.append("Common values")
-    content.append("-------------")
-    content.append(".. glossary::")
-    content.append("   :sorted:")
-    content.append("   ")
+        param_content[internal_values[value].section]["internal"].extend(internal_values[value].dump_doc())
     for value in sorted(list(common_values)):
-        content.extend(common_values[value].dump_doc())
+        param_content[common_values[value].section]["common"].extend(common_values[value].dump_doc())
+    list_sections = list(param_content)
+    if None in list_sections:
+        list_sections.remove(None)
+        list_sections = sorted(list_sections)
+        list_sections.insert(0, None)
+    else:
+        list_sections = sorted(list_sections)
+    target_filename = os.sep.join([dirname, project + ".rst"])
+    content = list()
+    content.append("Parameters available for project %s" % project)
+    content.append("=" * len(content[0]))
+    content.append("")
+
+    for section in list_sections:
+        if section is None:
+            section_title = "Unsorted parameters"
+        else:
+            section_title = section
+        level = 1
+        title_level = "^"
+        content.append(section_title)
+        content.append("-" * len(section_title))
+        if "internal" in param_content[section]:
+            content.extend(write_project_dict_content(param_content[section]["internal"], "Internal values", title_level, level=level))
+        if "common" in param_content[section]:
+            content.extend(write_project_dict_content(param_content[section]["common"], "Common values", title_level, level=level))
+        if section is not None:
+            content.append("")
     content.append("Project settings")
     content.append("----------------")
     content.append(".. glossary::")

sphinx/source/userguide/configuration.rst

@@ -26,7 +26,7 @@ default values can be found.
 
 Currently, three types of Data Request can be used in dr2xml:
    - the CMIP6 Data Request (default one)
-   - a C3S seasonal forecast kind of Data Request (python-like file with defined entries)
+   - a C3S seasonal forecast kind of Data Request (json-like file with defined entries)
    - no Data Request (in this case, the definitions of the variables are given through json-like tables and everything
      must be defined)
 
@@ -45,10 +45,11 @@ Beside those projects, the ones available currently for "real" usage are:
    - :module:`dr2xml.projects.CORDEX-CMIP6` for CORDEX-CMIP6 kind' usages
    - :module:`dr2xml.projects.C3S-SF` for C3S seasonal forecast kind' usages
 
-The settings defined in the projects are splitted in three categories (determined in this order):
-   - the internal settings which are required to run dr2xml
-   - the common settings which are used after having be read once at the launch of dr2xml
-   - for each XIOS-related object, the settings associated with this object (attributes and meta-data)
+The settings defined in the projects are splitted in four categories (determined in this order):
+   - the init settings which are required to initialize dr2xml
+   - the internal settings which are required to run dr2xml (need init settings to get the right values)
+   - the common settings which are only used in the XIOS-related objects settings (not properly in dr2xml source code), can use init and internal settings values
+   - for each XIOS-related object, the settings associated with this object (attributes and meta-data), some common values are regrouped at this level too
 That means that a setting in the common list can be defined from an other one in the internal list but not the reverse.
 
 The lab and model settings

sphinx/source/userguide/how_to.rst

@@ -0,0 +1,77 @@
+How to start with a new model/laboratory?
+=========================================
+
+This section aims at giving a few tips to start with the use of dr2xml with a new model/laboratory.
+
+First, from where to start?
+---------------------------
+
+The first step is to have XIOS working in the targetted model.
+One it is the case, that's means that you have many of the initialized files needed (fields, grids...).
+
+Then, the simplest way to start with dr2xml is to use an existing configuration and to adapt it.
+The notebook `DR2xml.ipynb` or the script `examples/DR2xml.py` are good starting points as long as unitary tests.
+
+Deal with a new laboratory
+--------------------------
+
+The short name of the new laboratory is specified in the lab_and_model_settings.py using the `institution_id` key.
+It is used to get information from outer sources (including CVs for CMIP6) to determine metadada and also to determine
+which variables should be output or not following lab grid policy. Lab grid policy can be seen as a first set of
+filtering on grids on which variables are output.
+
+For CNRM-CERFACS, the file is already included in the repository under `dr2xml.laboratories.CNRM-CERFACS.py`.
+For other groups, this file must be created (can be a copy of the CNRM-CERFACS one's with the new lab's name) and adapted if needed.
+The location of this file should be added in lab_and_model_settings.py using key `laboratory_used`).
+It can also be integrated in the repository.
+
+Other keys such as `contact` will have to be updated (but it is not mandatory at this stage).
+
+Creating ping files
+-------------------
+
+Ping files are used to make the link between the names defined in the fields definitions (i.e. the 'model' names) and the names defined in the data request.
+Example of ping files are in tests/common/xml_files directory.
+
+To create ping files, you can use the notebook `create_ping_files.ipynb` or adapt the tests under tests/test_pingfiles_*.
+
+Deal with a new model
+---------------------
+
+Then, the next step is to define the model within the lab_and_model_settings.py file.
+
+First, the different grids of the model (if several, list all), must defined in `grids` and `sizes` for each context as
+long as the sampling timestep in `sampling_timestep`. You must also defined the list of mips associated with each grid
+in `mips` (can be a void set if no filtering should be applied).
+
+Then, for each model id, you must provide:
+   - the grid associated with this model `grid_choice` (if a model is associated with several grids, you must provide two different ids)
+   - the types of sources associated with each model using `source_types`
+   - for child simulations, the start year in parent simulation must be provided in `branching`
+   - the `configuration` should also be updated
+
+Deal with a new project
+-----------------------
+
+If you need to configure your output for an other project, it's the most tricky part of dr2xml.
+
+The different projects usable with dr2xml can be found in `dr2xml.projects`.
+
+The base one is `dr2xml.projects.dr2xml.json` which come with some functions in `dr2xml.projects.dr2xml_funcs.py`.
+All others must inherit from it as it defines all elements that are needed to run dr2xml.
+
+The second base is `dr2xml.projects.basics.json` which come with some functions in `dr2xml.projects.basics_funcs.py`.
+All others (except dr2xml ones) should inherit from it too, as it give default values to all elements needed to run
+dr2xml (where the dr2xml files defines them). This class can be used as it but do not have all attributes you may need.
+
+For your new project, if possible, start from an existing one, as close as possible as what is needed for you.
+Then, copy the settings (json and py) and rename them with the id of your project. You need then to modify it to fit
+with the requirements of your project. See the configuration section to have more elements.
+
+And then?
+---------
+
+You should now be able to start playing with dr2xml for your model.
+
+Keep in mind that dr2xml will try to take into account all the requirements you give but may fail and just set a
+warning if it does not manage to find everything.

sphinx/source/userguide/index.rst

@@ -14,4 +14,5 @@ DR2XML User Guide
    Ping file <ping>
    Home data request <home_dr>
    Supplementary <supplementary>
+   How to start with a new model/laboratory? <how_to>
    Acknowledgments <acknowledgement>