Rework reader

.pre-commit-config.yaml

@@ -79,6 +79,7 @@ repos:
     hooks:
       - id: codespell
         additional_dependencies: [ 'tomli' ]
+        exclude: tests/.*\.py
   - repo: https://github.com/python-jsonschema/check-jsonschema
     rev: 0.31.1
     hooks:

CHANGES.rst

@@ -22,7 +22,8 @@ New features and enhancements
 
 Breaking changes
 ^^^^^^^^^^^^^^^^
-* ``cdm_reader_mapper.cdm_mapper``: rename `map_and_covnert` to helper function `_map_and_convert` (:pull:`343`)
+* ``cdm_reader_mapper.cdm_mapper``: rename `map_and_convert` to helepr function `_map_and_convert` (:pull:`343`)
+* replace `logging.error` with `raise` error statements (:pull:`345`)
 
 Internal changes
 ^^^^^^^^^^^^^^^^
@@ -34,6 +35,22 @@ Internal changes
 * ``cdm_reader_mapper.cdm_mapper``: introduce some helper functions (:pull:`324`)
 * add more unit tests (:issue:`311`, :pull:`324`)
 * ``cdm_reader_mapper.cdm_mapper``: split `map_and_convert` into multiple helper functions (:issue:`333`, :pull:`343`)
+* exclude tests/*.py from `pre-commit` codespell hook (:pull:`345`)
+* replace many `os` functions with `pathlib.Path` (:pull:`345`)
+* re-work `mdf_reader` (:issue:`334`, :pull:`345`)
+
+  * remove `reader.MDFFileReader` class
+  * remove `utils.configurator` module
+  * remove both `utils.decoder` and `mdf_reader.utils.converter` modules
+  * introduce `utils.parser` module: bunch of functions to parse input data into MDF data
+  * introduce `utils.convert_and_decode`: make converter and decoder functions more modular
+  * make `utils.validator` module more modular
+  * `utils.filereader.FileReader` uses `utils.parser` function for parsing
+  * move many helper function to `utils.utilities`
+  * serialize `schemas.schemas` module
+
+* add type hints and docstrings to `mdf_reader` (:pull:`345`)
+* add unit tests for `mdf_reader` module to testing suite (:pull:`345`)
 
 Bug fixes
 ^^^^^^^^^

cdm_reader_mapper/common/inspect.py

@@ -12,7 +12,7 @@
 
 import pandas as pd
 
-from cdm_reader_mapper.common import pandas_TextParser_hdlr
+from . import pandas_TextParser_hdlr
 
 
 def _count_by_cat(series) -> dict:

cdm_reader_mapper/common/replace.py

@@ -22,7 +22,7 @@
 
 import pandas as pd
 
-from cdm_reader_mapper.common import logging_hdlr
+from . import logging_hdlr
 
 
 def replace_columns(

cdm_reader_mapper/mdf_reader/codes/codes.py

@@ -9,8 +9,6 @@
 
 from __future__ import annotations
 
-import logging
-import os
 from pathlib import Path
 
 from cdm_reader_mapper.common.json_dict import (
@@ -23,53 +21,60 @@
 
 
 def read_table(
-    code_table_name,
-    imodel=None,
-    ext_table_path=None,
+    code_table_name: str,
+    imodel: str | None = None,
+    ext_table_path: str | None = None,
 ) -> dict:
     """
-    Read a data model code table file to a dictionary.
+    Load a data model code table into a Python dictionary.
 
-    It completes the code table to the full complexity
-    the data reader expects, by appending information
-    on secondary keys and expanding range keys.
+    The code table may define secondary keys, range expansions, or other
+    structures required by the data reader. This function resolves the
+    file location either from an external path or an internal data model.
 
-    Parameter
-    ---------
-    code_table_name: str
-        The external code table file.
-    imodel: str, optional
-        Name of internally available input data model.
-        e.g. icoads_r300_d704
-    ext_table_path: str, optional
-        The path to the external code table file.
-        One of ``imodel`` and ``ext_table_path`` must be set.
+    Parameters
+    ----------
+    code_table_name : str
+        The name of the code table (without file extension).
+        e.g., `"ICOADS.C0.IM"`
+    imodel : str, optional
+        Internal data model name, e.g., `"icoads_r300_d704"`. Required if
+        `ext_table_path` is not provided.
+    ext_table_path : str, optional
+        External path containing the code table file. If set, this path
+        takes precedence over `imodel`.
 
     Returns
     -------
-    dict
-        Code table
+    Dict
+        The fully combined code table dictionary.
+
+    Raises
+    ------
+    FileNotFoundError
+        If the specified table file cannot be found.
+    ValueError
+        If neither `imodel` nor `ext_table_path` is provided.
     """
-    # 1. Validate input
     if ext_table_path:
-        table_path = os.path.abspath(ext_table_path)
-        table_files = os.path.join(table_path, code_table_name + ".json")
-        if not os.path.isfile(table_files):
-            logging.error(f"Can't find input code table file {table_files}")
-            return
-        table_files = Path(table_files)
-    else:
-        imodel = imodel.split("_")
+        table_path = Path(ext_table_path).resolve()
+        table_file = table_path / f"{code_table_name}.json"
+        if not table_file.is_file():
+            raise FileNotFoundError(f"Can't find input code table file {table_file}")
+        table_files = [table_file]
+    elif imodel:
+        parts = imodel.split("_")
         table_files = collect_json_files(
-            *imodel,
+            *parts,
             base=f"{properties._base}.codes",
             name=code_table_name,
         )
 
-    if isinstance(table_files, Path):
-        table_files = [table_files]
-    # 2. Get tables
+        if isinstance(table_files, Path):
+            table_files = [table_files]
+    else:
+        raise ValueError("One of 'imodel' or 'ext_table_path' must be set")
+
     tables = [open_json_file(ifile) for ifile in table_files]
 
-    # 3. Combine tables
     return combine_dicts(tables)