From 74d72fcf0804bfdeafe0ce68906e4c4a6b042eaf Mon Sep 17 00:00:00 2001 From: Caden Myers Date: Thu, 30 Oct 2025 15:02:25 -0400 Subject: [PATCH 01/17] add workflow to run tests on release --- .github/workflows/tests-on-release.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/workflows/tests-on-release.yml diff --git a/.github/workflows/tests-on-release.yml b/.github/workflows/tests-on-release.yml new file mode 100644 index 0000000..f15088a --- /dev/null +++ b/.github/workflows/tests-on-release.yml @@ -0,0 +1,20 @@ +name: Full Tests on Release + +on: + push: + tags: + - "v*" + +jobs: + tests-on-release: + uses: scikit-package/release-scripts/.github/workflows/_tests-on-pr.yml@v0 + with: + project: diffpy.cmi + c_extension: false + headless: false + run: | + echo "Running examples" + export RUN_EXAMPLES=1 + pytest --cov + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} From 5604c491e3b3046fa72384b95194939901e01e17 Mon Sep 17 00:00:00 2001 From: Caden Myers Date: Thu, 30 Oct 2025 15:13:27 -0400 Subject: [PATCH 02/17] mark test_examples so it only runs on a cmi release --- tests/test_examples.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_examples.py b/tests/test_examples.py index fafbfab..da5e6cb 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -1,5 +1,12 @@ +import os import runpy +import pytest + +# Run example scripts only on release +if not os.getenv("RUN_EXAMPLES"): + pytest.skip("Skipping test_examples.py", allow_module_level=True) + def test_all_examples(tmp_examples): """Run all example scripts to ensure they execute without error.""" From a35b635934547d203cb006aad560f11c073c917a Mon Sep 17 00:00:00 2001 From: Caden Myers Date: Thu, 30 Oct 2025 15:13:34 -0400 Subject: [PATCH 03/17] news --- news/run-ex-on-releases.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 news/run-ex-on-releases.rst diff --git a/news/run-ex-on-releases.rst b/news/run-ex-on-releases.rst new file mode 100644 index 0000000..909f6cf --- /dev/null +++ b/news/run-ex-on-releases.rst @@ -0,0 +1,23 @@ +**Added:** + +* + +**Changed:** + +* Changed workflow so that ``test_examples.py`` runs on release. + +**Deprecated:** + +* + +**Removed:** + +* + +**Fixed:** + +* + +**Security:** + +* From 380757f0b2d6f18919c59fd2d5a0158f8e4870c1 Mon Sep 17 00:00:00 2001 From: Caden Myers Date: Thu, 30 Oct 2025 15:19:25 -0400 Subject: [PATCH 04/17] add instructive comment for running locally --- tests/test_examples.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_examples.py b/tests/test_examples.py index da5e6cb..4af5766 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -3,11 +3,12 @@ import pytest -# Run example scripts only on release +# Run example scripts only on release. if not os.getenv("RUN_EXAMPLES"): pytest.skip("Skipping test_examples.py", allow_module_level=True) +# To run this test locally, run `RUN_EXAMPLES=1 pytest` def test_all_examples(tmp_examples): """Run all example scripts to ensure they execute without error.""" scripts = list(tmp_examples.rglob("**/solutions/diffpy-cmi/*.py")) From eaad3f91943f938eb1dd64a47164150e9ece1ef1 Mon Sep 17 00:00:00 2001 From: Caden Myers Date: Thu, 30 Oct 2025 21:04:07 -0400 Subject: [PATCH 05/17] rm env variable --- tests/test_examples.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/test_examples.py b/tests/test_examples.py index 4af5766..fafbfab 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -1,14 +1,6 @@ -import os import runpy -import pytest -# Run example scripts only on release. -if not os.getenv("RUN_EXAMPLES"): - pytest.skip("Skipping test_examples.py", allow_module_level=True) - - -# To run this test locally, run `RUN_EXAMPLES=1 pytest` def test_all_examples(tmp_examples): """Run all example scripts to ensure they execute without error.""" scripts = list(tmp_examples.rglob("**/solutions/diffpy-cmi/*.py")) From 8b03054fa7022003cd21347655abc54c2090eedd Mon Sep 17 00:00:00 2001 From: Caden Myers Date: Thu, 30 Oct 2025 21:04:23 -0400 Subject: [PATCH 06/17] add PYTEST_ADDOPT --- .github/workflows/tests-on-pr.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests-on-pr.yml b/.github/workflows/tests-on-pr.yml index e8ef0fc..d817c38 100644 --- a/.github/workflows/tests-on-pr.yml +++ b/.github/workflows/tests-on-pr.yml @@ -12,6 +12,7 @@ jobs: c_extension: false headless: false run: | + export PYTEST_ADDOPTS="--ignore=tests/test_examples.py" set -Eeuo pipefail echo "Test cmds" cmi -h From 84ed9c02fe1e6db1d92726b47c93aa343567f783 Mon Sep 17 00:00:00 2001 From: Caden Myers Date: Thu, 30 Oct 2025 21:09:59 -0400 Subject: [PATCH 07/17] workflow to run examples script manually --- .../{tests-on-release.yml => run-examples.yml} | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) rename .github/workflows/{tests-on-release.yml => run-examples.yml} (58%) diff --git a/.github/workflows/tests-on-release.yml b/.github/workflows/run-examples.yml similarity index 58% rename from .github/workflows/tests-on-release.yml rename to .github/workflows/run-examples.yml index f15088a..3e44ff0 100644 --- a/.github/workflows/tests-on-release.yml +++ b/.github/workflows/run-examples.yml @@ -1,20 +1,18 @@ -name: Full Tests on Release +name: Run Example Scripts on: - push: - tags: - - "v*" + workflow_dispatch: jobs: - tests-on-release: + run-examples: uses: scikit-package/release-scripts/.github/workflows/_tests-on-pr.yml@v0 with: project: diffpy.cmi c_extension: false headless: false run: | - echo "Running examples" - export RUN_EXAMPLES=1 - pytest --cov + set -Eeuo pipefail + echo "Running example scripts" + pytest --cov tests/test_examples.py -v secrets: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} From adac539d3e7a277569a4d29e86d5481d17b44706 Mon Sep 17 00:00:00 2001 From: Caden Myers Date: Thu, 30 Oct 2025 21:12:17 -0400 Subject: [PATCH 08/17] add step to the release checklist --- .github/ISSUE_TEMPLATE/release_checklist.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/release_checklist.md b/.github/ISSUE_TEMPLATE/release_checklist.md index 6107962..73f5184 100644 --- a/.github/ISSUE_TEMPLATE/release_checklist.md +++ b/.github/ISSUE_TEMPLATE/release_checklist.md @@ -6,6 +6,10 @@ labels: "release" assignees: "" --- +### Run diffpy.cmi example scripts + +- [ ] Manually trigger the `run-examples.yml` workflow to run all example scripts. + ### PyPI/GitHub rc-release preparation checklist: - [ ] All PRs/issues attached to the release are merged. From fff1bbe0975cff75d42f17b4ce6e951658762792 Mon Sep 17 00:00:00 2001 From: Caden Myers Date: Fri, 31 Oct 2025 15:12:59 -0400 Subject: [PATCH 09/17] change test name to validate_examples.py --- tests/validate_examples.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 tests/validate_examples.py diff --git a/tests/validate_examples.py b/tests/validate_examples.py new file mode 100644 index 0000000..fafbfab --- /dev/null +++ b/tests/validate_examples.py @@ -0,0 +1,12 @@ +import runpy + + +def test_all_examples(tmp_examples): + """Run all example scripts to ensure they execute without error.""" + scripts = list(tmp_examples.rglob("**/solutions/diffpy-cmi/*.py")) + # sort list so that fitBulkNi.py runs first + scripts.sort(key=lambda s: 0 if s.name == "fitBulkNi.py" else 1) + for script in scripts: + script_relative_path = script.relative_to(tmp_examples) + print(f"Testing {script_relative_path}") + runpy.run_path(str(script), run_name="__main__") From 980422975e6725459634af926f575480680731b4 Mon Sep 17 00:00:00 2001 From: Caden Myers Date: Fri, 31 Oct 2025 15:14:11 -0400 Subject: [PATCH 10/17] add workflow to run examples scripts manually --- .github/workflows/validate-examples.yml | 52 +++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 .github/workflows/validate-examples.yml diff --git a/.github/workflows/validate-examples.yml b/.github/workflows/validate-examples.yml new file mode 100644 index 0000000..7c4d1e2 --- /dev/null +++ b/.github/workflows/validate-examples.yml @@ -0,0 +1,52 @@ +name: Validate Examples + +on: + workflow_dispatch: + +jobs: + validate-examples: + defaults: + run: + shell: bash -l {0} + + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Initialize miniconda + uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: test + channels: conda-forge + auto-update-conda: true + auto-activate-base: false + python-version: 3.13 + + - name: Conda config + run: >- + conda config --set always_yes yes + --set changeps1 no + + - name: Install dependencies + run: | + conda install --file requirements/conda.txt + conda install --file requirements/tests.txt + + if [ -d requirements/packs ]; then + for f in requirements/packs/*.txt; do + if [ -f "$f" ]; then + echo "Installing dependencies from $f" + conda install --file "$f" + fi + done + fi + + python -m pip install . --no-deps + + - name: Run example scripts + run: | + set -Eeuo pipefail + echo "Running example scripts" + pytest tests/validate_examples.py -v --cov From 8d7a3b6dca8f780b68aa6d76b5dcfb4262edbaef Mon Sep 17 00:00:00 2001 From: Caden Myers Date: Fri, 31 Oct 2025 15:14:52 -0400 Subject: [PATCH 11/17] fix name in release checklist --- .github/ISSUE_TEMPLATE/release_checklist.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/release_checklist.md b/.github/ISSUE_TEMPLATE/release_checklist.md index 73f5184..5be9636 100644 --- a/.github/ISSUE_TEMPLATE/release_checklist.md +++ b/.github/ISSUE_TEMPLATE/release_checklist.md @@ -8,7 +8,7 @@ assignees: "" ### Run diffpy.cmi example scripts -- [ ] Manually trigger the `run-examples.yml` workflow to run all example scripts. +- [ ] Manually trigger the `validate-examples.yml` workflow to run all example scripts. ### PyPI/GitHub rc-release preparation checklist: From 220ed1ae4f30571b686086dd61f9697c7a5cc018 Mon Sep 17 00:00:00 2001 From: Caden Myers Date: Fri, 31 Oct 2025 15:15:14 -0400 Subject: [PATCH 12/17] test file name change --- tests/test_examples.py | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 tests/test_examples.py diff --git a/tests/test_examples.py b/tests/test_examples.py deleted file mode 100644 index fafbfab..0000000 --- a/tests/test_examples.py +++ /dev/null @@ -1,12 +0,0 @@ -import runpy - - -def test_all_examples(tmp_examples): - """Run all example scripts to ensure they execute without error.""" - scripts = list(tmp_examples.rglob("**/solutions/diffpy-cmi/*.py")) - # sort list so that fitBulkNi.py runs first - scripts.sort(key=lambda s: 0 if s.name == "fitBulkNi.py" else 1) - for script in scripts: - script_relative_path = script.relative_to(tmp_examples) - print(f"Testing {script_relative_path}") - runpy.run_path(str(script), run_name="__main__") From 5a5d5a9e51ad86f8a7a2422662542343a2cbd91e Mon Sep 17 00:00:00 2001 From: Caden Myers Date: Fri, 31 Oct 2025 15:15:52 -0400 Subject: [PATCH 13/17] remove PYTEST_ADDOPTS --- .github/workflows/run-examples.yml | 18 ------------------ .github/workflows/tests-on-pr.yml | 1 - 2 files changed, 19 deletions(-) delete mode 100644 .github/workflows/run-examples.yml diff --git a/.github/workflows/run-examples.yml b/.github/workflows/run-examples.yml deleted file mode 100644 index 3e44ff0..0000000 --- a/.github/workflows/run-examples.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Run Example Scripts - -on: - workflow_dispatch: - -jobs: - run-examples: - uses: scikit-package/release-scripts/.github/workflows/_tests-on-pr.yml@v0 - with: - project: diffpy.cmi - c_extension: false - headless: false - run: | - set -Eeuo pipefail - echo "Running example scripts" - pytest --cov tests/test_examples.py -v - secrets: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/tests-on-pr.yml b/.github/workflows/tests-on-pr.yml index d817c38..e8ef0fc 100644 --- a/.github/workflows/tests-on-pr.yml +++ b/.github/workflows/tests-on-pr.yml @@ -12,7 +12,6 @@ jobs: c_extension: false headless: false run: | - export PYTEST_ADDOPTS="--ignore=tests/test_examples.py" set -Eeuo pipefail echo "Test cmds" cmi -h From e813014c3e3abf5ad2b59e84e3f40a7b4258ae62 Mon Sep 17 00:00:00 2001 From: Caden Myers Date: Fri, 31 Oct 2025 15:16:38 -0400 Subject: [PATCH 14/17] remove -v pytest flag --- .github/workflows/validate-examples.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-examples.yml b/.github/workflows/validate-examples.yml index 7c4d1e2..7e57ae1 100644 --- a/.github/workflows/validate-examples.yml +++ b/.github/workflows/validate-examples.yml @@ -49,4 +49,4 @@ jobs: run: | set -Eeuo pipefail echo "Running example scripts" - pytest tests/validate_examples.py -v --cov + pytest tests/validate_examples.py --cov From d73618b2b03aba9a17762516cce163afce2f042a Mon Sep 17 00:00:00 2001 From: Caden Myers Date: Fri, 31 Oct 2025 15:18:08 -0400 Subject: [PATCH 15/17] news update --- news/run-ex-on-releases.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/news/run-ex-on-releases.rst b/news/run-ex-on-releases.rst index 909f6cf..20c2924 100644 --- a/news/run-ex-on-releases.rst +++ b/news/run-ex-on-releases.rst @@ -1,10 +1,10 @@ **Added:** -* +* Add workflow to run ``examples/validate_examples.py`` manually. **Changed:** -* Changed workflow so that ``test_examples.py`` runs on release. +* Changed workflow so that ``validate_examples.py`` is ran only on manual triggers. **Deprecated:** From 8dc85e5cd07836016139e61d2c61ca435d8aacd5 Mon Sep 17 00:00:00 2001 From: Caden Myers Date: Fri, 31 Oct 2025 20:10:22 -0400 Subject: [PATCH 16/17] mv temp dir to validators, create validators dir --- tests/conftest.py | 17 ----------------- tests/validate_examples.py | 12 ------------ validators/validate_examples.py | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 29 deletions(-) delete mode 100644 tests/validate_examples.py create mode 100644 validators/validate_examples.py diff --git a/tests/conftest.py b/tests/conftest.py index 5a4edca..658294d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,26 +1,9 @@ import json -import shutil from pathlib import Path import matplotlib import pytest -PROJECT_ROOT = Path(__file__).resolve().parents[1] -EXAMPLES_ROOT = PROJECT_ROOT / "docs" / "examples" - - -@pytest.fixture(scope="session") -def tmp_examples(tmp_path_factory): - """Copy the entire examples/ tree into a temp directory once per - test session. - - Returns the path to that copy. - """ - tmpdir = tmp_path_factory.mktemp("examples") - tmp_examples = tmpdir / "examples" - shutil.copytree(EXAMPLES_ROOT, tmp_examples) - yield tmp_examples - @pytest.fixture(scope="session") def example_cases(tmp_path_factory): diff --git a/tests/validate_examples.py b/tests/validate_examples.py deleted file mode 100644 index fafbfab..0000000 --- a/tests/validate_examples.py +++ /dev/null @@ -1,12 +0,0 @@ -import runpy - - -def test_all_examples(tmp_examples): - """Run all example scripts to ensure they execute without error.""" - scripts = list(tmp_examples.rglob("**/solutions/diffpy-cmi/*.py")) - # sort list so that fitBulkNi.py runs first - scripts.sort(key=lambda s: 0 if s.name == "fitBulkNi.py" else 1) - for script in scripts: - script_relative_path = script.relative_to(tmp_examples) - print(f"Testing {script_relative_path}") - runpy.run_path(str(script), run_name="__main__") diff --git a/validators/validate_examples.py b/validators/validate_examples.py new file mode 100644 index 0000000..3fd2cde --- /dev/null +++ b/validators/validate_examples.py @@ -0,0 +1,32 @@ +import runpy +import shutil +from pathlib import Path + +import pytest + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +EXAMPLES_ROOT = PROJECT_ROOT / "docs" / "examples" + + +@pytest.fixture(scope="session") +def tmp_examples(tmp_path_factory): + """Copy the entire examples/ tree into a temp directory once per + test session. + + Returns the path to that copy. + """ + tmpdir = tmp_path_factory.mktemp("examples") + tmp_examples = tmpdir / "examples" + shutil.copytree(EXAMPLES_ROOT, tmp_examples) + yield tmp_examples + + +def test_all_examples(tmp_examples): + """Run all example scripts to ensure they execute without error.""" + scripts = list(tmp_examples.rglob("**/solutions/diffpy-cmi/*.py")) + # sort list so that fitBulkNi.py runs first + scripts.sort(key=lambda s: 0 if s.name == "fitBulkNi.py" else 1) + for script in scripts: + script_relative_path = script.relative_to(tmp_examples) + print(f"Testing {script_relative_path}") + runpy.run_path(str(script), run_name="__main__") From ca07824ce80bcc9a73c03c35531b37b6d580a21a Mon Sep 17 00:00:00 2001 From: Caden Myers Date: Fri, 31 Oct 2025 20:12:20 -0400 Subject: [PATCH 17/17] change dir in CI --- .github/workflows/validate-examples.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-examples.yml b/.github/workflows/validate-examples.yml index 7e57ae1..2ccf311 100644 --- a/.github/workflows/validate-examples.yml +++ b/.github/workflows/validate-examples.yml @@ -49,4 +49,4 @@ jobs: run: | set -Eeuo pipefail echo "Running example scripts" - pytest tests/validate_examples.py --cov + pytest validators/validate_examples.py --cov