From f57b8a41d2fb77ab1011ddb06ba0277aa97955c9 Mon Sep 17 00:00:00 2001 From: Matt Henderson Date: Fri, 23 Dec 2022 17:53:11 -0800 Subject: [PATCH 1/5] testing --- .github/workflows/python-app.yml | 226 ++++++++++++++++++++++++++++++- 1 file changed, 222 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index fc8f014cb..c5a2aee9f 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -12,17 +12,25 @@ on: permissions: contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: build: runs-on: ubuntu-latest - + continue-on-error: true + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v3 - - name: Set up Python 3.10 - uses: actions/setup-python@v3 + - name: Set up Python 3 + uses: actions/setup-python@v4 with: - python-version: "3.8" + python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip @@ -35,6 +43,216 @@ jobs: # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide # flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Test that the module imports + id: install + run: | + pip install . + python -c "import py4DSTEM; print(py4DSTEM.__version__)" + cache: + + needs: build + runs-on: ubuntu-latest + continue-on-error: true + strategy: + fail-fast: false + matrix: + python-version: ["3.8"] + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3 + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Test that the module imports + id: install run: | pip install . python -c "import py4DSTEM; print(py4DSTEM.__version__)" + - name: Checkout tutorials + uses: actions/checkout@v3 + with: + repository: mlhenderson/py4DSTEM_tutorials + path: './py4DSTEM_tutorials' + - name: Install download dependencies + run: | + python -m pip install gdown jupyter + - name: Cache data files + id: cache-data + uses: actions/cache@v3 + env: + cache-name: cache-data + with: + path: data + key: ${{ env.cache-name }}-${{ hashFiles('data/download_files.json') }} + - if: ${{ steps.cache-data.outputs.cache-hit != 'true' }} + name: Download tutorial data + uses: jannekem/run-python-script-action@v1 + continue-on-error: true + with: + script: | + import json + import os + import shutil + + import gdown + from nbconvert.preprocessors import ExecutePreprocessor, CellExecutionError + import nbformat + + data_dir = os.path.abspath("data") + if not os.path.exists(data_dir): + os.makedirs(data_dir) + notebooks_dir = os.path.abspath("./py4DSTEM_tutorials/notebooks/") + google_url = 'https://drive.google.com' + downloads = {} + notebooks = sorted([p.name for p in os.scandir(notebooks_dir) if '.ipynb' in p.name and p.is_file()]) + failures = 0 + + for nb_file in notebooks: + downloads[nb_file] = [] + + print("\tChecking for downloads and hardcoded paths in {}".format(nb_file)) + with open(os.path.join(notebooks_dir, nb_file), 'r') as f: + nb = nbformat.read(f, as_version=4) + for i in range(len(nb['cells'])): + text = nb['cells'][i]['source'] + if google_url in text: + lines = text.split() + urls = [x.strip() for x in lines if google_url in x] + # download file + print("\tDownloading input files for {}".format(nb_file)) + for u in urls: + download_url = u.split('(')[1][:-1] + name = gdown.download(url=download_url, fuzzy=True) + if name is not None: + destination = os.path.join(data_dir, name) + shutil.move(name, destination) + downloads[nb_file].append(destination) + else: + print("Unable to download {}".format(download_url)) + failures += 1 + + with open(os.path.join(data_dir, 'download_files.json'), 'w') as f: + json.dump(downloads, f) + print(json.dumps(downloads)) + print("{}: {}".format(data_dir, os.listdir(data_dir))) + + if failures > 0: + import sys + sys.exit(1) + + test: + + needs: [build, cache] + runs-on: ubuntu-latest + continue-on-error: true + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.9", "3.10"] + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3 + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Test that the module imports + id: install + run: | + pip install . + python -c "import py4DSTEM; print(py4DSTEM.__version__)" + - name: Checkout tutorials + uses: actions/checkout@v3 + with: + repository: mlhenderson/py4DSTEM_tutorials + path: './py4DSTEM_tutorials' + - name: Install tutorial dependencies + run: | + python -m pip install . + python -m pip install "jedi==0.17.2" "dask[complete]" pymatgen gdown jupyter + - name: Get cached data files + uses: actions/cache@v3 + env: + cache-name: cache-data + with: + path: data + key: ${{ env.cache-name }}-${{ hashFiles('data/download_files.json') }} + - name: Run tutorial notebooks + uses: jannekem/run-python-script-action@v1 + with: + script: | + import json + import os + import re + import traceback + + from nbconvert.preprocessors import ExecutePreprocessor, CellExecutionError + import nbformat + + data_dir = os.path.abspath("data") + + print("{}: {}".format(data_dir, os.listdir(data_dir))) + + notebooks_dir = os.path.abspath("./py4DSTEM_tutorials/notebooks/") + os.chdir(notebooks_dir) + google_url = 'https://drive.google.com' + with open(os.path.join(data_dir, 'download_files.json'), 'r') as f: + downloads = json.load(f) + print(downloads) + notebooks = sorted([p.name for p in os.scandir(notebooks_dir) if '.ipynb' in p.name and p.is_file()]) + + failed = [] + succeeded = [] + + for nb_file in notebooks: + print("Testing {}".format(nb_file)) + print("{} in downloads: {}, {}".format(nb_file, nb_file in downloads, downloads[nb_file])) + changed = False + + print("\tChecking for downloads and hardcoded paths in {}".format(nb_file)) + with open(nb_file, 'r') as f: + nb = nbformat.read(f, as_version=4) + for i in range(len(nb['cells'])): + text = nb['cells'][i]['source'] + replaced_text = "" + if 'file_' in text: + #print("Found file_ in {}".format(text)) + results = [m for m in re.finditer(r'file_\w+\s*=\s*[r]?[\'\"](.*)[\'\"]', text)] + for r in results: + #print("matched regex: {}".format(r.group())) + name_matches = 0 + #print("possible filenames to insert: {}".format(downloads[nb_file])) + for fpath in downloads[nb_file]: + fname = os.path.split(fpath)[-1] + #print("Filename without path: {}".format(fname)) + if fname in r.group(): + parts = r.group().split('=') + replacement = "{} = '{}'".format(parts[0], fpath) + #print("Inserting {} into {}".format( + # fpath, + # replacement)) + name_matches += 1 + text = text.replace(r.group(), replacement) + changed = True + #print('After replacement: {}'.format(text)) + break + nb['cells'][i]['source'] = text + + if changed: + print("\tWriting path updates back to {}".format(nb_file)) + with open(nb_file, 'w', encoding='utf-8') as f: + nbformat.write(nb, f) + + print("\tExecuting {}".format(nb_file)) + try: + ep = ExecutePreprocessor(timeout=600, kernel_name='python3') + ep.preprocess(nb, {'metadata': {'path': notebooks_dir}}) + succeeded.append(nb_file) + except Exception as e: + failed.append(nb_file) + tb = "\n".join(["\t{}".format(line) for line in traceback.format_exc().splitlines()]) + print("\n\t{0} {1} {0}\n\n{2}\n\t{0} {1} {0}\n".format("*"*20, nb_file, tb)) + + print("{} Notebooks ran without errors.".format(len(succeeded))) + print("{} Notebooks failed with errors".format(len(failed))) + if len(failed) > 0: + import sys + sys.exit(1) From a7af4631f8827088f6b89f791fb2cb9f1534929d Mon Sep 17 00:00:00 2001 From: Matt Henderson Date: Fri, 23 Dec 2022 17:59:36 -0800 Subject: [PATCH 2/5] switch to py4DSTEM main of tutorials repo --- .github/workflows/python-app.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index c5a2aee9f..1c092011c 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -70,7 +70,7 @@ jobs: - name: Checkout tutorials uses: actions/checkout@v3 with: - repository: mlhenderson/py4DSTEM_tutorials + repository: py4DSTEM/py4DSTEM_tutorials path: './py4DSTEM_tutorials' - name: Install download dependencies run: | @@ -162,7 +162,7 @@ jobs: - name: Checkout tutorials uses: actions/checkout@v3 with: - repository: mlhenderson/py4DSTEM_tutorials + repository: py4DSTEM/py4DSTEM_tutorials path: './py4DSTEM_tutorials' - name: Install tutorial dependencies run: | From 28853cef4826a55450aad476ce514969d0aa6256 Mon Sep 17 00:00:00 2001 From: Matt Henderson Date: Fri, 23 Dec 2022 18:29:20 -0800 Subject: [PATCH 3/5] hash all notebook files for cache key --- .github/workflows/python-app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 1c092011c..80328875d 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -82,7 +82,7 @@ jobs: cache-name: cache-data with: path: data - key: ${{ env.cache-name }}-${{ hashFiles('data/download_files.json') }} + key: ${{ env.cache-name }}-${{ hashFiles('./py4DSTEM_tutorials/**/.ipynb') }} - if: ${{ steps.cache-data.outputs.cache-hit != 'true' }} name: Download tutorial data uses: jannekem/run-python-script-action@v1 From dcf9921e2871dd555ccef83f5491a288f1f4573b Mon Sep 17 00:00:00 2001 From: Matt Henderson Date: Fri, 23 Dec 2022 18:30:13 -0800 Subject: [PATCH 4/5] hash all notebook files for cache key --- .github/workflows/python-app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 80328875d..a22115f40 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -174,7 +174,7 @@ jobs: cache-name: cache-data with: path: data - key: ${{ env.cache-name }}-${{ hashFiles('data/download_files.json') }} + key: ${{ env.cache-name }}-${{ hashFiles('./py4DSTEM_tutorials/**/.ipynb') }} - name: Run tutorial notebooks uses: jannekem/run-python-script-action@v1 with: From 2a7a9070221e4c12280c4ac89802f3518e56deef Mon Sep 17 00:00:00 2001 From: Matt Henderson Date: Fri, 23 Dec 2022 19:04:53 -0800 Subject: [PATCH 5/5] removing debug statements --- .github/workflows/python-app.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index a22115f40..88489862c 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -214,25 +214,17 @@ jobs: text = nb['cells'][i]['source'] replaced_text = "" if 'file_' in text: - #print("Found file_ in {}".format(text)) results = [m for m in re.finditer(r'file_\w+\s*=\s*[r]?[\'\"](.*)[\'\"]', text)] for r in results: - #print("matched regex: {}".format(r.group())) name_matches = 0 - #print("possible filenames to insert: {}".format(downloads[nb_file])) for fpath in downloads[nb_file]: fname = os.path.split(fpath)[-1] - #print("Filename without path: {}".format(fname)) if fname in r.group(): parts = r.group().split('=') replacement = "{} = '{}'".format(parts[0], fpath) - #print("Inserting {} into {}".format( - # fpath, - # replacement)) name_matches += 1 text = text.replace(r.group(), replacement) changed = True - #print('After replacement: {}'.format(text)) break nb['cells'][i]['source'] = text