diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..fc8df89 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,32 @@ +name: Build + +on: + push: + branches: + - develop + tags: + - 'v*' + pull_request: + branches: + - main + - develop + +jobs: + call-version-info-workflow: + uses: ASFHyP3/actions/.github/workflows/reusable-version-info.yml@v0.20.0 + permissions: + contents: read + with: + python_version: '3.10' + + call-docker-ghcr-workflow: + needs: call-version-info-workflow + uses: ASFHyP3/actions/.github/workflows/reusable-docker-ghcr.yml@v0.20.0 + permissions: + contents: read + packages: write + with: + version_tag: ${{ needs.call-version-info-workflow.outputs.version_tag }} + user: ${{ github.actor }} + secrets: + USER_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 3e4a03f..5360e08 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -14,4 +14,4 @@ on: jobs: call-changelog-check-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.18.1 + uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.20.0 diff --git a/.github/workflows/create-jira-issue.yml b/.github/workflows/create-jira-issue.yml index 5765047..c77e1e8 100644 --- a/.github/workflows/create-jira-issue.yml +++ b/.github/workflows/create-jira-issue.yml @@ -6,7 +6,7 @@ on: jobs: call-create-jira-issue-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.18.1 + uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.20.0 secrets: JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} diff --git a/.github/workflows/labeled-pr.yml b/.github/workflows/labeled-pr.yml index 4f8dcd9..d729153 100644 --- a/.github/workflows/labeled-pr.yml +++ b/.github/workflows/labeled-pr.yml @@ -13,4 +13,4 @@ on: jobs: call-labeled-pr-check-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.18.1 + uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.20.0 diff --git a/.github/workflows/release-checklist-comment.yml b/.github/workflows/release-checklist-comment.yml index 5170ee3..a2bb05d 100644 --- a/.github/workflows/release-checklist-comment.yml +++ b/.github/workflows/release-checklist-comment.yml @@ -10,7 +10,7 @@ on: jobs: call-release-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.18.1 + uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.20.0 permissions: pull-requests: write secrets: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bd1d47c..2791b93 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,7 +8,7 @@ on: jobs: call-release-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.18.1 + uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.20.0 with: release_prefix: HyP3 OPERA-RTC release_branch: main diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index 6b1b024..15f80f6 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -4,10 +4,10 @@ on: push jobs: call-secrets-analysis-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.18.1 + uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.20.0 call-ruff-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-ruff.yml@v0.18.1 + uses: ASFHyP3/actions/.github/workflows/reusable-ruff.yml@v0.20.0 call-mypy-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-mypy.yml@v0.18.1 + uses: ASFHyP3/actions/.github/workflows/reusable-mypy.yml@v0.20.0 diff --git a/.github/workflows/tag-version.yml b/.github/workflows/tag-version.yml index 44ad9eb..cd5ee66 100644 --- a/.github/workflows/tag-version.yml +++ b/.github/workflows/tag-version.yml @@ -8,7 +8,7 @@ on: jobs: call-bump-version-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.18.1 + uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.20.0 with: user: tools-bot email: UAF-asf-apd@alaska.edu diff --git a/.github/workflows/test-and-build.yml b/.github/workflows/test-and-build.yml deleted file mode 100644 index 9e40789..0000000 --- a/.github/workflows/test-and-build.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Test and build - -on: - push: - branches: - - main - - develop - pull_request: - branches: - - main - - develop - -jobs: - call-pytest-workflow: - # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-pytest.yml@v0.18.1 - with: - local_package_name: hyp3_opera_rtc - python_versions: >- - ["3.10", "3.11", "3.12", "3.13"] - - call-version-info-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-version-info.yml@v0.18.1 - with: - python_version: '3.10' - - call-docker-ecr-workflow: - needs: call-version-info-workflow - uses: ASFHyP3/actions/.github/workflows/reusable-docker-ecr.yml@v0.18.1 - with: - version_tag: ${{ needs.call-version-info-workflow.outputs.version_tag }} - ecr_registry: 845172464411.dkr.ecr.us-west-2.amazonaws.com - aws_region: us-west-2 - release_branch: main - develop_branch: develop - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.V2_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.V2_AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..685768b --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,21 @@ +name: Test + +on: + push: + branches: + - main + - develop + pull_request: + branches: + - main + - develop + +jobs: + call-pytest-workflow: + uses: ASFHyP3/actions/.github/workflows/reusable-pytest.yml@v0.20.0 + permissions: + contents: read + with: + local_package_name: hyp3_opera_rtc + python_versions: >- + ["3.10", "3.11", "3.12", "3.13"] diff --git a/CHANGELOG.md b/CHANGELOG.md index d0a7ee5..9ad38bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.1.3] + +### Changed +- Download opera burst db during container build instead of at runtime. +- Publish docker container to public GHCR repo instead of private Amazon ECR repo. + ## [0.1.2] ### Changed diff --git a/Dockerfile b/Dockerfile index 0dcbc40..a235dbd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM 845172464411.dkr.ecr.us-west-2.amazonaws.com/opera_pge/rtc_s1:2.1.1 +FROM ghcr.io/nasa/opera-sds-pge/opera_pge/rtc_s1:2.1.1 # For opencontainers label definitions, see: # https://github.com/opencontainers/image-spec/blob/master/annotations.md @@ -15,6 +15,8 @@ USER root RUN chown rtc_user:rtc_user /home/rtc_user/scratch USER rtc_user +RUN curl https://asf-dem-west.s3.amazonaws.com/AUX/opera-burst-bbox-only.sqlite3 -o /home/rtc_user/opera-burst-bbox-only.sqlite3 + COPY --chown=rtc_user:rtc_user . /home/rtc_user/hyp3-opera-rtc/ RUN conda env create -f /home/rtc_user/hyp3-opera-rtc/environment.yml && \ conda clean -afy && \ diff --git a/README.md b/README.md index e843a17..713c386 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,8 @@ A HyP3 plugin for OPERA-RTC processing **ALL CREDIT FOR THIS PLUGIN'S RTC PRODUCTS GOES TO THE [JPL OPERA TEAM](https://www.jpl.nasa.gov/go/opera). THIS PLUGIN MERELY ALLOWS US TO RUN THEIR WORKFLOW IN A HYP3 ENVIRONMENT.** +**VALIDATION OF THIS PLUGIN IS STILL ONGOING. CONFIDENCE THAT THIS PLUGIN PRODUCES EQUIVALENT RTC DATA PRODUCTS AS THE OPERA MISSION IS NOT (YET) HIGH ENOUGH FOR PRODUCTION USE.** + ## Earthdata Login Credentials To use this plugin, the user must provide their Earthdata Login credentials in order to download input data. @@ -14,20 +16,11 @@ Your credentials can be passed to the workflows via environmental variables that ## Usage -> [!WARNING] -> This plugin is designed to run within the HyP3 processing system, and directly relies on the JPL OPERA OPERA-RTC-S1 Product Generation Executable (PGE) docker container (see architecture section below). Currently this container is not publicly available, but the OPERA team is working to make it available in the near future. - +This plugin is designed to run within the HyP3 processing system, +and directly relies on the JPL OPERA OPERA-RTC-S1 Product Generation Executable (PGE) docker container (see architecture section below). For this reason, the plugin is only runnable via the docker container. -Because our docker image is based on `845172464411.dkr.ecr.us-west-2.amazonaws.com/opera_pge/rtc_s1` (see the [Dockerfile](./Dockerfile)), -you'll need to authenticate to Amazon ECR in the HyP3 AWS account before building our docker image -(you must be an ASF developer with access to the HyP3 AWS account): - -``` -aws --profile hyp3 ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 845172464411.dkr.ecr.us-west-2.amazonaws.com -``` - -Then build our container: +Build the container: ```bash git clone https://github.com/ASFHyP3/hyp3-OPERA-RTC.git diff --git a/requirements-static.txt b/requirements-static.txt index ab2d803..989a2ca 100644 --- a/requirements-static.txt +++ b/requirements-static.txt @@ -1,4 +1,4 @@ -ruff==0.11.8 +ruff==0.11.11 mypy==1.15.0 lxml-stubs types-shapely diff --git a/src/hyp3_opera_rtc/prep_rtc.py b/src/hyp3_opera_rtc/prep_rtc.py index 593543e..eb72f05 100644 --- a/src/hyp3_opera_rtc/prep_rtc.py +++ b/src/hyp3_opera_rtc/prep_rtc.py @@ -1,5 +1,6 @@ import argparse import os +import shutil import warnings from pathlib import Path from zipfile import ZipFile @@ -18,16 +19,13 @@ CMR_URL = 'https://cmr.earthdata.nasa.gov/search/granules.umm_json' -def download_burst_db(save_dir: Path) -> Path: - db_path = save_dir / 'opera-burst-bbox-only.sqlite3' +def prep_burst_db(save_dir: Path) -> Path: + db_filename = 'opera-burst-bbox-only.sqlite3' + db_path = save_dir / db_filename - if db_path.exists(): - return db_path + shutil.copy(Path.home() / db_filename, db_path) - # Currently using a version created using opera-adt/burst_db v0.4.0, but hope to switch to ASF-provide source. - url = 'https://ffwilliams2-shenanigans.s3.us-west-2.amazonaws.com/opera/opera-burst-bbox-only.sqlite3' - db_path = hyp3lib.fetch.download_file(url, str(save_dir)) - return Path(db_path) + return db_path def get_s1_granule_bbox(granule_path: Path, buffer: float = 0.025) -> Polygon: @@ -135,8 +133,8 @@ def prep_rtc( orbit_path = orbit.get_orbit(safe_path.with_suffix('').name, save_dir=input_dir) print(f'Downloaded orbit file: {orbit_path}') - db_path = download_burst_db(input_dir) - print(f'Downloaded burst database: {db_path}') + db_path = prep_burst_db(input_dir) + print(f'Burst database: {db_path}') dem_path = input_dir / 'dem.tif' granule_bbox = get_s1_granule_bbox(safe_path)