From 65314c45150c2d213864f3aab007d6596c321e4a Mon Sep 17 00:00:00 2001 From: "Karl N. Kappler" Date: Sun, 3 Aug 2025 16:47:47 -0700 Subject: [PATCH 1/8] do not exit executing notebooks if one fails --- docs/execute_notebooks.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/execute_notebooks.py b/docs/execute_notebooks.py index fcfce1d6..a3ddcc48 100644 --- a/docs/execute_notebooks.py +++ b/docs/execute_notebooks.py @@ -17,7 +17,7 @@ notebook_dir = DOCS_PATH.joinpath("tutorials") notebooks += sorted( nb for nb in notebook_dir.rglob("*.ipynb") if ".ipynb_checkpoints" not in str(nb) -) +) # Execute each notebook in-place for nb_path in notebooks: @@ -29,19 +29,19 @@ "%matplotlib inline", ) print(f"Executing: {nb_path} (in cwd={working_dir})") - + try: pm.execute_notebook( input_path=str(nb_path), output_path=str(nb_path), - kernel_name="aurora-test", # Adjust if using a different kernel ("dipole-st") + kernel_name="aurora-test", # Adjust if using a different kernel ("dipole-st") request_save_on_cell_execute=True, - cwd=str(working_dir) # <- this sets the working directory! + cwd=str(working_dir), # <- this sets the working directory! ) print(f"✓ Executed successfully: {nb_path}") except Exception as e: print(f"✗ Failed to execute {nb_path}: {e}") - exit(1) + # exit(1) # Replace the matplotlib inline magic back to widget for interactive plots replace_in_file( @@ -50,4 +50,3 @@ "%matplotlib widget", ) print("All notebooks executed and updated successfully.") - From 3742687ae51951d2eeb2d7dc777e5f9ec9ed0f50 Mon Sep 17 00:00:00 2001 From: "Karl N. Kappler" Date: Sun, 3 Aug 2025 16:51:33 -0700 Subject: [PATCH 2/8] use uv for pytest --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c72b31da..0efd91b1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -79,7 +79,7 @@ jobs: run: | # pytest -s -v tests/synthetic/test_fourier_coefficients.py # pytest -s -v tests/config/test_config_creator.py - pytest -s -v --cov=./ --cov-report=xml --cov=aurora + uv pytest -s -v --cov=./ --cov-report=xml --cov=aurora - name: "Upload coverage reports to Codecov" uses: codecov/codecov-action@v4 From c13634fc0220281be21f739857ad0d7ebf2cda96 Mon Sep 17 00:00:00 2001 From: "Karl N. Kappler" Date: Sun, 3 Aug 2025 18:13:45 -0700 Subject: [PATCH 3/8] rm incorrect uv prefix --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0efd91b1..c72b31da 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -79,7 +79,7 @@ jobs: run: | # pytest -s -v tests/synthetic/test_fourier_coefficients.py # pytest -s -v tests/config/test_config_creator.py - uv pytest -s -v --cov=./ --cov-report=xml --cov=aurora + pytest -s -v --cov=./ --cov-report=xml --cov=aurora - name: "Upload coverage reports to Codecov" uses: codecov/codecov-action@v4 From 531faa1aa001f6ed08ff120a454c25395fd05e44 Mon Sep 17 00:00:00 2001 From: "Karl N. Kappler" Date: Sat, 18 Oct 2025 16:03:03 -0700 Subject: [PATCH 4/8] try better utilizing uv, (test can move away from conda) --- .github/workflows/tests.yml | 45 ++++++++++++++++--------------------- 1 file changed, 19 insertions(+), 26 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c72b31da..7ae25ef0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,7 +13,7 @@ jobs: runs-on: ${{ matrix.os }} defaults: run: - shell: bash -l {0} + shell: bash strategy: fail-fast: false matrix: @@ -23,40 +23,31 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v2.1.1 + - name: Install uv + uses: astral-sh/setup-uv@v3 with: - activate-environment: aurora-test - python-version: ${{ matrix.python-version }} + version: "latest" + - name: Set up Python ${{ matrix.python-version }} + run: uv python install ${{ matrix.python-version }} - - name: Install uv and project dependencies + - name: Create virtual environment and install dependencies run: | - python --version - echo $CONDA_PREFIX - pip install uv - uv pip install -e ".[dev]" + uv venv --python ${{ matrix.python-version }} + uv pip install -e ".[dev,test]" uv pip install "mt_metadata[obspy] @ git+https://github.com/kujaku11/mt_metadata.git" uv pip install git+https://github.com/kujaku11/mth5.git - conda install -c conda-forge certifi">=2017.4.17" pandoc + uv pip install jupyter ipykernel pytest pytest-cov codecov - - name: Install Our Package + - name: Install system dependencies run: | - echo $CONDA_PREFIX - uv pip install -e . - echo "Install complete" - conda list - pip freeze - - - name: Install Jupyter and dependencies - run: | - pip install jupyter - pip install ipykernel - python -m ipykernel install --user --name aurora-test - # Install any other dependencies you need + sudo apt-get update + sudo apt-get install -y pandoc - name: Execute Jupyter Notebooks run: | + source .venv/bin/activate + python -m ipykernel install --user --name aurora-test jupyter nbconvert --to notebook --execute docs/examples/dataset_definition.ipynb jupyter nbconvert --to notebook --execute docs/examples/operate_aurora.ipynb jupyter nbconvert --to notebook --execute docs/tutorials/pkd_units_check.ipynb @@ -64,7 +55,6 @@ jobs: jupyter nbconvert --to notebook --execute docs/tutorials/processing_configuration.ipynb jupyter nbconvert --to notebook --execute docs/tutorials/process_cas04_multiple_station.ipynb jupyter nbconvert --to notebook --execute docs/tutorials/synthetic_data_processing.ipynb - # Replace "notebook.ipynb" with your notebook's filename # - name: Commit changes (if any) # run: | @@ -77,9 +67,11 @@ jobs: - name: Run Tests run: | + source .venv/bin/activate + pytest -s -v --cov=./ --cov-report=xml --cov=aurora # pytest -s -v tests/synthetic/test_fourier_coefficients.py # pytest -s -v tests/config/test_config_creator.py - pytest -s -v --cov=./ --cov-report=xml --cov=aurora + - name: "Upload coverage reports to Codecov" uses: codecov/codecov-action@v4 @@ -92,6 +84,7 @@ jobs: - name: Build Doc if: ${{ (github.ref == 'refs/heads/main') && (matrix.python-version == '3.8')}} run: | + source .venv/bin/activate cd docs make html cd .. From a5c110fc66bee79eb8c20f177f2bc87ffeb17f94 Mon Sep 17 00:00:00 2001 From: "Karl N. Kappler" Date: Sat, 18 Oct 2025 16:13:09 -0700 Subject: [PATCH 5/8] add some doc --- tests/synthetic/test_feature_weighting.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/synthetic/test_feature_weighting.py b/tests/synthetic/test_feature_weighting.py index 4c6e580f..b811975e 100644 --- a/tests/synthetic/test_feature_weighting.py +++ b/tests/synthetic/test_feature_weighting.py @@ -1,13 +1,21 @@ """ - Integrated test of the functionality of feature weights. -1. This test uses degraded sythetic data to test the feature weighting. +1. This test uses degraded synthetic data to test the feature weighting. Noise is added to some fraction (50-75%) of the data. Then regular (single station) processing is called on the data and feature weighting processing is called on the data. +--- +Feature weights are specified using the mt_metadata.features.weights module. +This test demonstrates how feature-based channel weighting (e.g., striding_window_coherence) +can be injected into Aurora's processing pipeline. In the future, these features will be +used to enable more robust, data-driven weighting strategies for transfer function estimation, +including integration of new features from mt_metadata and more flexible weighting schemes. + +See also: mt_metadata.features.weights.channel_weight_spec and test_feature_weighting.py for +examples of how to define, load, and use feature weights in Aurora workflows. """ from aurora.config.metadata import Processing From b36862e6ce497f59e7fa746b92598baf8a6c4126 Mon Sep 17 00:00:00 2001 From: "Karl N. Kappler" Date: Sat, 18 Oct 2025 16:25:20 -0700 Subject: [PATCH 6/8] Update python versions for tests - next mth5 and mt_metdata have dropped support for 3.8 and added 3.12 --- .github/workflows/tests.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7ae25ef0..1345bf48 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -18,7 +18,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest"] - python-version: [3.8, 3.9, "3.10", "3.11"] + python-version: [3.9, "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 @@ -56,15 +56,6 @@ jobs: jupyter nbconvert --to notebook --execute docs/tutorials/process_cas04_multiple_station.ipynb jupyter nbconvert --to notebook --execute docs/tutorials/synthetic_data_processing.ipynb -# - name: Commit changes (if any) -# run: | -# git config --local user.email "action@github.com" -# git config --local user.name "GitHub Action" -# git commit -a -m "Execute Jupyter notebook" -# git push -# if: ${{ success() }} - - - name: Run Tests run: | source .venv/bin/activate @@ -79,7 +70,6 @@ jobs: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: false flags: tests - # token: ${{ secrets.CODECOV_TOKEN }} - name: Build Doc if: ${{ (github.ref == 'refs/heads/main') && (matrix.python-version == '3.8')}} From a6e106eca07ca4a27eb194f1596b197deefd7041 Mon Sep 17 00:00:00 2001 From: "Karl N. Kappler" Date: Sat, 18 Oct 2025 16:30:27 -0700 Subject: [PATCH 7/8] rename yml to yaml --- .github/workflows/{tests.yml => tests.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{tests.yml => tests.yaml} (100%) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yaml similarity index 100% rename from .github/workflows/tests.yml rename to .github/workflows/tests.yaml From c3af257cdd41db238e55907b17383aa1b22db0c3 Mon Sep 17 00:00:00 2001 From: JP Date: Mon, 8 Dec 2025 20:15:02 -0800 Subject: [PATCH 8/8] Fix station metadata by converting timeseries.Station to dict --- .pre-commit-config.yaml | 2 +- aurora/pipelines/transfer_function_kernel.py | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c2bcdcad..43204dcd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ repos: rev: 22.6.0 hooks: - id: black - language_version: python3.10 + language_version: python3.11 - repo: https://github.com/pycqa/flake8 rev: 3.9.2 hooks: diff --git a/aurora/pipelines/transfer_function_kernel.py b/aurora/pipelines/transfer_function_kernel.py index b9826150..bd19873c 100644 --- a/aurora/pipelines/transfer_function_kernel.py +++ b/aurora/pipelines/transfer_function_kernel.py @@ -1,6 +1,6 @@ """ - This module contains the TrasnferFunctionKernel class which is the main object that - links the KernelDataset to Processing configuration. +This module contains the TrasnferFunctionKernel class which is the main object that +links the KernelDataset to Processing configuration. """ @@ -602,6 +602,17 @@ def make_decimation_dict_for_tf( # Set key as first el't of dict, nor currently supporting mixed surveys in TF tf_cls.survey_metadata = self.dataset.local_survey_metadata + # Explicitly set station_metadata to ensure station ID is correct + # (TF.__init__ creates a default station with ID '0', we need to replace it) + # Convert timeseries.Station to dict, which TF._validate_station_metadata will convert to tf.Station + if ( + hasattr(self.dataset.local_survey_metadata, "stations") + and len(self.dataset.local_survey_metadata.stations) > 0 + ): + tf_cls.station_metadata = self.dataset.local_survey_metadata.stations[ + 0 + ].to_dict() + # pack the station metadata into the TF object # station_id = self.processing_config.stations.local.id # station_sub_df = self.dataset_df[self.dataset_df["station"] == station_id]