Add a RuntimeError in some cases when using the UDA backend (#99)

Author: maarten-ic

docs/source/multi-dd.rst

@@ -276,27 +276,110 @@ You need to explicitly convert the data, which you can do as follows:
       entry.put(imas.convert_ids(equilibrium, entry.dd_version))
 
 
+.. _`UDA backend and DD versions`:
 
-.. _`DD background`:
+UDA backend caching and Data Dictionary versions
+------------------------------------------------
 
-Background information
-----------------------
+If you try to load data from a different Data Dictionary version with the UDA backend,
+you may see the following error:
+
+.. code-block:: text
+
+  The Data Dictionary version of the data (3.38.1) is different from the Data
+  Dictionary version of the DBEntry (3.42.0). This is not supported when using the
+  UDA backend.
+
+There are three possible workarounds. The first two require passing an additional option
+in the IMAS UDA URI: please see the `imas-core documentation
+<https://imas-core.readthedocs.io/en/stable/user_guide/backends_guide.html#query-keys-specific-for-the-uda-backend>`__
+for more details on these URI options.
+
+1. Use UDA fetch to bypass the cache problem. You can do this by appending ``&fetch=1``
+   to the URI when you create the :py:class:`~imas.db_entry.DBEntry`.
+
+   Note that this will download the entire IDS files from the remote server, this may
+   not be desired if you only want to read a single time slice.
+2. Disable the UDA cache. You can do this by appending ``&cache_mode=none`` to the URI
+   when you create the :py:class:`~imas.db_entry.DBEntry`.
+
+   Note that this may make the ``get()`` (a lot) slower, since a separate request needs
+   to be sent to the remote UDA server for every data variable. However, this may still
+   be the best performing option if you are only interested in a subset of all the data
+   in an IDS (and use :ref:`lazy loading`).
+3. Explicitly provide the data dictionary version when you create the
+   :py:class:`~imas.db_entry.DBEntry`, setting it to match the Data Dictionary version
+   of the data you want to load. To obtain the version of the data on the remote server
+   from the field `ids_properties.put_version.data_dictionary` via a _lazy_ ``get()``
+   with ``autoconvert=False`` option and using the ``&cache_mode=none`` query in the URI.
+   
+   Note that you may need to call ``imas.convert_ids`` to convert the IDS to your
+   desired Data Dictionary version.
+
+All three possible workarounds are shown in the examples below:
+
+.. md-tab-set::
+
+    .. md-tab-item:: Original code
+
+      .. code-block:: python
+
+        import imas
+
+        URI = (
+            "imas://uda.iter.org:56565/uda?backend=hdf5"
+            "&path=/work/imas/shared/imasdb/ITER/3/121013/50"
+        )
+        with imas.DBEntry(URI, "r") as entry:
+            cp = entry.get("core_profiles")
 
-Since IMAS-Python needs to have access to multiple DD versions it was chosen to
-bundle these with the code at build-time, in setup.py. If a git clone of the
-Data Dictionary succeeds, the setup tools automatically download saxon and
-generate ``IDSDef.xml`` for each of the tagged versions in the DD git
-repository. These are then gathered into ``IDSDef.zip``, which is
-distributed inside the IMAS-Python package.
+    .. md-tab-item:: 1. Use UDA fetch
 
-To update the set of data dictionaries new versions can be added to the zipfile.
-A reinstall of the package will ensure that all available versions are included
-in IMAS-Python. Additionally an explicit path to an XML file can be specified, which
-is useful for development.
+      .. code-block:: python
 
-Automated tests have been provided that check the loading of all of the DD
-versions tagged in the data-dictionary git repository.
+        import imas
 
+        URI = (
+            "imas://uda.iter.org:56565/uda?backend=hdf5"
+            "&path=/work/imas/shared/imasdb/ITER/3/121013/50&fetch=1"
+        )
+        with imas.DBEntry(URI, "r") as entry:
+            cp = entry.get("core_profiles")
+
+    .. md-tab-item:: 2. Disable the UDA cache
+
+      .. code-block:: python
+
+        import imas
+
+        URI = (
+            "imas://uda.iter.org:56565/uda?backend=hdf5"
+            "&path=/work/imas/shared/imasdb/ITER/3/121013/50&cache_mode=none"
+        )
+        with imas.DBEntry(URI, "r") as entry:
+            cp = entry.get("core_profiles")
+
+    .. md-tab-item:: 3. Explicitly provide the DD version
+
+      .. code-block:: python
+
+        import imas
+
+        URI = (
+            "imas://uda.iter.org:56565/uda?backend=hdf5"
+            "&path=/work/imas/shared/imasdb/ITER/3/121013/50"
+        )
+        with imas.DBEntry(URI, "r", dd_version="3.38.1") as entry:
+            cp = entry.get("core_profiles")
+
+        # Optional: convert the IDS to your desired DD version
+        cp = imas.convert_ids(cp, "3.42.0")
+
+
+.. _`DD background`:
+
+Background information
+----------------------
 
 Data Dictionary definitions
 '''''''''''''''''''''''''''

imas/backends/imas_core/db_entry_al.py

@@ -256,6 +256,22 @@ def read_dd_version(self, ids_name: str, occurrence: int) -> str:
             raise DataEntryException(
                 f"IDS {ids_name!r}, occurrence {occurrence} is empty."
             )
+
+        # UDA caching doesn't play well when the DD version of the on-disk IDS doesn't
+        # match the DD version of this DBEntry. See GH#97
+        if self.backend == "uda" and dd_version != self._ids_factory.dd_version:
+            cache_mode = self._querydict.get("cache_mode")
+            fetch = self._querydict.get("fetch")
+            if cache_mode != "none" and fetch not in ("1", "true"):
+                raise RuntimeError(
+                    f"The Data Dictionary version of the data ({dd_version}) is "
+                    "different from the Data Dictionary version of the DBEntry "
+                    f"({self._ids_factory.dd_version}). This is not supported when "
+                    f"using the UDA backend. See {imas.PUBLISHED_DOCUMENTATION_ROOT}"
+                    "multi-dd.html#uda-backend-caching-and-data-dictionary-versions "
+                    "for more details and workarounds."
+                )
+
         return dd_version
 
     def put(self, ids: IDSToplevel, occurrence: int, is_slice: bool) -> None:

imas/test/test_dbentry_uda.py

@@ -0,0 +1,80 @@
+from pathlib import Path
+import os
+from unittest.mock import patch
+
+import pytest
+from packaging.version import Version
+
+from imas import DBEntry
+from imas.ids_defs import READ_OP
+
+
+@pytest.fixture
+def mock_read_data():
+    return {
+        "ids_properties/homogeneous_time": 1,
+        "ids_properties/version_put/data_dictionary": "4.0.0",
+    }
+
+
+@pytest.fixture
+def mock_ll_interface(mock_read_data):
+    """Mock the IMAS lowlevel interface so we can still test the our UDA-specific logic.
+
+    Since we don't have a public UDA server available to test against, this is the
+    next-best thing.
+    """
+    with patch("imas.backends.imas_core.db_entry_al.ll_interface") as mock_ll_interface:
+        mock_ll_interface.begin_dataentry_action.return_value = (0, 0)
+        mock_ll_interface.begin_global_action.return_value = (0, 0)
+        mock_ll_interface.begin_arraystruct_action.return_value = (0, 0, 0)
+        mock_ll_interface.close_pulse.return_value = 0
+        mock_ll_interface._al_version = Version("5.6.0")
+
+        def read_data(ctx, fieldPath, pyTimebasePath, ualDataType, dim):
+            return 0, mock_read_data.get(fieldPath)
+
+        mock_ll_interface.read_data.side_effect = read_data
+
+        # Also patch in al_context.py:
+        with patch(
+            "imas.backends.imas_core.al_context.ll_interface", mock_ll_interface
+        ):
+            yield mock_ll_interface
+
+
+def test_uda_idsdef_path(mock_ll_interface):
+    # Check that IDSDEF_PATH env variable is set for the UDA backend
+    with DBEntry("imas:uda?mock", "r", dd_version="4.0.0"):
+        assert "IDSDEF_PATH" in os.environ
+        path1 = Path(os.environ["IDSDEF_PATH"])
+        assert path1.exists()
+    with DBEntry("imas:uda?mock", "r", dd_version="3.42.0"):
+        assert "IDSDEF_PATH" in os.environ
+        path2 = Path(os.environ["IDSDEF_PATH"])
+        assert path2.exists()
+    assert path1 != path2
+
+
+def test_uda_datapath(mock_ll_interface):
+    # Check that datapath is set when requesting the dd version
+    with DBEntry("imas:uda?mock", "r", dd_version="4.0.0") as entry:
+        mock_ll_interface.begin_global_action.assert_not_called()
+        entry.get("mhd", lazy=True)
+        # pulseCtx=0, dataobjectname="mhd", rwmode=READ_OP, datapath="ids_properties"
+        mock_ll_interface.begin_global_action.assert_called_with(
+            0, "mhd", READ_OP, "ids_properties"
+        )
+
+
+def test_uda_version_mismatch_exception(mock_ll_interface):
+    # Check that we get an exception when versions mismatch
+    with pytest.raises(RuntimeError, match="Data Dictionary version"):
+        DBEntry("imas:uda?path=mock", "r", dd_version="4.1.0").get("mhd")
+    # No exceptions when using cache_mode=none
+    DBEntry("imas:uda?path=mock&cache_mode=none", "r", dd_version="4.1.0").get("mhd")
+    # Or when using fetch
+    DBEntry("imas:uda?path=mock&fetch=true", "r", dd_version="4.1.0").get("mhd")
+    DBEntry("imas:uda?path=mock&fetch=1", "r", dd_version="4.1.0").get("mhd")
+    # Or when using the exact same DD version
+    DBEntry("imas:uda?path=mock", "r", dd_version="4.0.0").get("mhd")