Update to work with Pydantic upgrade of mt-metadata

.github/workflows/tests.yaml

@@ -18,7 +18,7 @@ jobs:
       fail-fast: false
       matrix:
         os: ["ubuntu-latest"]
-        python-version: [3.9, "3.10", "3.11", "3.12"]
+        python-version: ["3.10", "3.11", "3.12"]
 
     steps:
     - uses: actions/checkout@v4
@@ -35,31 +35,31 @@ jobs:
       run: |
         uv venv --python ${{ matrix.python-version }}
         uv pip install -e ".[dev,test]"
-        uv pip install "mt_metadata[obspy] @ git+https://github.com/kujaku11/mt_metadata.git"
-        uv pip install git+https://github.com/kujaku11/mth5.git
+        uv pip install "mt_metadata[obspy] @ git+https://github.com/kujaku11/mt_metadata.git@pydantic"
+        uv pip install git+https://github.com/kujaku11/mth5.git@old_pydantic
         uv pip install jupyter ipykernel pytest pytest-cov codecov
 
     - name: Install system dependencies
       run: |
         sudo apt-get update
         sudo apt-get install -y pandoc
 
-    - name: Execute Jupyter Notebooks
-      run: |
-        source .venv/bin/activate
-        python -m ipykernel install --user --name aurora-test
-        jupyter nbconvert --to notebook --execute docs/examples/dataset_definition.ipynb
-        jupyter nbconvert --to notebook --execute docs/examples/operate_aurora.ipynb
-        jupyter nbconvert --to notebook --execute docs/tutorials/pkd_units_check.ipynb
-        jupyter nbconvert --to notebook --execute docs/tutorials/pole_zero_fitting/lemi_pole_zero_fitting_example.ipynb
-        jupyter nbconvert --to notebook --execute docs/tutorials/processing_configuration.ipynb
-        jupyter nbconvert --to notebook --execute docs/tutorials/process_cas04_multiple_station.ipynb
-        jupyter nbconvert --to notebook --execute docs/tutorials/synthetic_data_processing.ipynb
+    # - name: Execute Jupyter Notebooks
+    #   run: |
+    #     source .venv/bin/activate
+    #     python -m ipykernel install --user --name aurora-test
+    #     jupyter nbconvert --to notebook --execute docs/examples/dataset_definition.ipynb
+    #     jupyter nbconvert --to notebook --execute docs/examples/operate_aurora.ipynb
+    #     jupyter nbconvert --to notebook --execute docs/tutorials/pkd_units_check.ipynb
+    #     jupyter nbconvert --to notebook --execute docs/tutorials/pole_zero_fitting/lemi_pole_zero_fitting_example.ipynb
+    #     jupyter nbconvert --to notebook --execute docs/tutorials/processing_configuration.ipynb
+    #     jupyter nbconvert --to notebook --execute docs/tutorials/process_cas04_multiple_station.ipynb
+    #     jupyter nbconvert --to notebook --execute docs/tutorials/synthetic_data_processing.ipynb
 
     - name: Run Tests
       run: |
         source .venv/bin/activate
-        pytest -s -v --cov=./ --cov-report=xml --cov=aurora
+        pytest -s -v --cov=./ --cov-report=xml --cov=aurora -n auto tests
         # pytest -s -v tests/synthetic/test_fourier_coefficients.py
         # pytest -s -v tests/config/test_config_creator.py 
 

.pre-commit-config.yaml

@@ -1,10 +1,45 @@
+# .pre-commit-config.yaml
 repos:
--   repo: https://github.com/ambv/black
-    rev: 22.6.0
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
     hooks:
-    - id: black
-      language_version: python3.10
--   repo: https://github.com/pycqa/flake8
-    rev: 3.9.2
+    -   id: trailing-whitespace
+        types: [python]
+    -   id: end-of-file-fixer
+        types: [python]
+    -   id: check-yaml
+        exclude: '^(?!.*\.py$).*$'
+
+-   repo: https://github.com/pycqa/isort
+    rev: 5.12.0
     hooks:
-    - id: flake8
+    -   id: isort
+        types: [python]
+        exclude: (__init__.py)$
+        files: \.py$
+        args: ["--profile", "black",
+               "--skip-glob","*/__init__.py",
+               "--force-alphabetical-sort-within-sections",
+               "--order-by-type",
+               "--lines-after-imports=2"]
+
+-   repo: https://github.com/psf/black
+    rev: 23.3.0
+    hooks:
+    -   id: black
+        types: [python]
+        files: \.py$
+        language_version: python3
+
+-   repo: https://github.com/pycqa/autoflake
+    rev: v2.1.1
+    hooks:
+    -   id: autoflake
+        types: [python]
+        files: \.py$
+        args: [
+            "--remove-all-unused-imports",
+            "--expand-star-imports",
+            "--ignore-init-module-imports",
+            "--in-place"
+        ]

aurora/__init__.py

@@ -13,7 +13,7 @@
             "sink": sys.stdout,
             "level": "INFO",
             "colorize": True,
-            "format": "<level>{time} | {level: <3} | {name} | {function} | {message}</level>",
+            "format": "<level>{time} | {level: <3} | {name} | {function} | line: {line} | {message}</level>",
         },
     ],
     "extra": {"user": "someone"},

aurora/config/config_creator.py

@@ -16,7 +16,7 @@
 from aurora.config.metadata import Processing
 from aurora.sandbox.io_helpers.emtf_band_setup import EMTFBandSetupFile
 from mth5.processing.kernel_dataset import KernelDataset
-from mt_metadata.transfer_functions.processing.window import Window
+from mt_metadata.processing.window import Window
 
 import pathlib
 
@@ -127,6 +127,7 @@ def create_from_kernel_dataset(
         kernel_dataset: KernelDataset,
         input_channels: Optional[list] = None,
         output_channels: Optional[list] = None,
+        remote_channels: Optional[list] = None,
         estimator: Optional[str] = None,
         emtf_band_file: Optional[Union[str, pathlib.Path]] = None,
         band_edges: Optional[dict] = None,
@@ -166,6 +167,8 @@ def create_from_kernel_dataset(
             List of the input channels that will be used in TF estimation (usually "hx", "hy")
         output_channels: list
             List of the output channels that will be estimated by TF (usually "ex", "ey", "hz")
+        remote_channels: list
+            List of the remote reference channels (usually "hx", "hy" at remote site)
         estimator:  Optional[Union[str, None]]
             The name of the regression estimator to use for TF estimation.
         emtf_band_file: Optional[Union[str, pathlib.Path, None]]
@@ -241,6 +244,10 @@ def create_from_kernel_dataset(
             else:
                 decimation_obj.output_channels = output_channels
 
+            if remote_channels is None:
+                if kernel_dataset.remote_channels is not None:
+                    decimation_obj.reference_channels = kernel_dataset.remote_channels
+
             if num_samples_window is not None:
                 decimation_obj.stft.window.num_samples = num_samples_window[key]
             # set estimator if provided as kwarg

aurora/config/metadata/processing.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 """
-Extend the mt_metadata.transfer_functions.processing.aurora.processing.Processing class
+Extend the mt_metadata.processing.aurora.processing.Processing class
 with some aurora-specific methods.
 """
 
@@ -10,10 +10,10 @@
 
 from aurora.time_series.windowing_scheme import window_scheme_from_decimation
 from loguru import logger
-from mt_metadata.transfer_functions.processing.aurora.processing import (
+from mt_metadata.processing.aurora.processing import (
     Processing as AuroraProcessing,
 )
-from mt_metadata.utils.list_dict import ListDict
+from mt_metadata.common.list_dict import ListDict
 from typing import Optional, Union
 
 import json
@@ -192,7 +192,7 @@ class EMTFTFHeader(ListDict):
     def __init__(self, **kwargs):
         """
         Parameters
-        _local_station : mt_metadata.transfer_functions.tf.station.Station()
+        _local_station : mt_metadata.processing.tf.station.Station()
             Station metadata object for the station to be estimated (
             location, channel_azimuths, etc.)
         _remote_station: same object type as local station