From 452206b3b6ff3d1f118aa5a63a2cf93193d5a997 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Jul 2025 11:55:26 +0000 Subject: [PATCH 01/10] Bump the pip-deps group with 2 updates Bumps the pip-deps group with 2 updates: [ruff](https://github.com/astral-sh/ruff) and [mypy](https://github.com/python/mypy). Updates `ruff` from 0.12.2 to 0.12.5 - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/ruff/compare/0.12.2...0.12.5) Updates `mypy` from 1.16.1 to 1.17.0 - [Changelog](https://github.com/python/mypy/blob/master/CHANGELOG.md) - [Commits](https://github.com/python/mypy/compare/v1.16.1...v1.17.0) --- updated-dependencies: - dependency-name: ruff dependency-version: 0.12.5 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: pip-deps - dependency-name: mypy dependency-version: 1.17.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: pip-deps ... Signed-off-by: dependabot[bot] --- requirements-static.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-static.txt b/requirements-static.txt index 32b7581..edeac2c 100644 --- a/requirements-static.txt +++ b/requirements-static.txt @@ -1,5 +1,5 @@ -ruff==0.12.2 -mypy==1.16.1 +ruff==0.12.5 +mypy==1.17.0 lxml-stubs types-shapely types-requests From 1d2f88719f406b8ee87e65df9d7c7a664c314bbb Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Tue, 29 Jul 2025 14:30:19 -0500 Subject: [PATCH 02/10] update sas version --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 287560b..89fca67 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/nasa/opera-sds-pge/opera_pge/rtc_s1:2.1.1 +FROM ghcr.io/nasa/opera-sds-pge/opera_pge/rtc_s1:2.1.3 # For opencontainers label definitions, see: # https://github.com/opencontainers/image-spec/blob/master/annotations.md From bba59da37115d02e4c50ffa2243c086cee17b514 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Tue, 29 Jul 2025 14:31:21 -0500 Subject: [PATCH 03/10] update changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62b535d..70dc500 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,11 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [0.1.4] +### Changed +- Updated OPERA PGE container version to 2.1.3. + +## [0.1.4] + ### Added - Added functionality to `prep_rtc.py` and `upload_rtc.py` to accept SLCs or co-pol bursts. - Add `--num-workers` as `prep_rtc` cli param From 593f3a824fe7dc80aed92d0f364c7bc892d67e7c Mon Sep 17 00:00:00 2001 From: Andrew Johnston Date: Thu, 31 Jul 2025 11:27:19 -0800 Subject: [PATCH 04/10] Revert "Update PGE version" --- CHANGELOG.md | 5 ----- Dockerfile | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70dc500..62b535d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,11 +8,6 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [0.1.4] -### Changed -- Updated OPERA PGE container version to 2.1.3. - -## [0.1.4] - ### Added - Added functionality to `prep_rtc.py` and `upload_rtc.py` to accept SLCs or co-pol bursts. - Add `--num-workers` as `prep_rtc` cli param diff --git a/Dockerfile b/Dockerfile index 89fca67..287560b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/nasa/opera-sds-pge/opera_pge/rtc_s1:2.1.3 +FROM ghcr.io/nasa/opera-sds-pge/opera_pge/rtc_s1:2.1.1 # For opencontainers label definitions, see: # https://github.com/opencontainers/image-spec/blob/master/annotations.md From b8e7080b7cb64c42220142d6e1240a3bae8e97ad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Aug 2025 10:21:06 +0000 Subject: [PATCH 05/10] Bump the pip-deps group with 2 updates Bumps the pip-deps group with 2 updates: [ruff](https://github.com/astral-sh/ruff) and [mypy](https://github.com/python/mypy). Updates `ruff` from 0.12.5 to 0.12.7 - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/ruff/compare/0.12.5...0.12.7) Updates `mypy` from 1.17.0 to 1.17.1 - [Changelog](https://github.com/python/mypy/blob/master/CHANGELOG.md) - [Commits](https://github.com/python/mypy/compare/v1.17.0...v1.17.1) --- updated-dependencies: - dependency-name: ruff dependency-version: 0.12.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: pip-deps - dependency-name: mypy dependency-version: 1.17.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: pip-deps ... Signed-off-by: dependabot[bot] --- requirements-static.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-static.txt b/requirements-static.txt index edeac2c..01b0c99 100644 --- a/requirements-static.txt +++ b/requirements-static.txt @@ -1,5 +1,5 @@ -ruff==0.12.5 -mypy==1.17.0 +ruff==0.12.7 +mypy==1.17.1 lxml-stubs types-shapely types-requests From a2feab6b13c28ffa29bf48fda9f4eabbfc11792a Mon Sep 17 00:00:00 2001 From: William Horn Date: Mon, 4 Aug 2025 07:52:25 -0800 Subject: [PATCH 06/10] Update pge template yml to add new field --- Dockerfile | 2 +- src/hyp3_opera_rtc/templates/pge.yml.j2 | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 287560b..89fca67 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/nasa/opera-sds-pge/opera_pge/rtc_s1:2.1.1 +FROM ghcr.io/nasa/opera-sds-pge/opera_pge/rtc_s1:2.1.3 # For opencontainers label definitions, see: # https://github.com/opencontainers/image-spec/blob/master/annotations.md diff --git a/src/hyp3_opera_rtc/templates/pge.yml.j2 b/src/hyp3_opera_rtc/templates/pge.yml.j2 index b26ea48..4d00272 100644 --- a/src/hyp3_opera_rtc/templates/pge.yml.j2 +++ b/src/hyp3_opera_rtc/templates/pge.yml.j2 @@ -32,6 +32,7 @@ RunConfig: ErrorCodeBase: 300000 SchemaPath: /home/rtc_user/opera/pge/rtc_s1/schema/rtc_s1_sas_schema.yaml IsoTemplatePath: /home/rtc_user/opera/pge/rtc_s1/templates/OPERA_ISO_metadata_L2_RTC_S1_template.xml.jinja2 + IsoMeasuredParameterDescriptions: /home/rtc_user/opera/pge/rtc_s1/templates/rtc_s1_measured_parameters.yaml # Date field which designates the point after which the # RTC static layer product(s) should be considered valid. # This field must be provided for RTC-S1 jobs when static layer From b3de56d1f28245c74978f10893b66fd8093e48dc Mon Sep 17 00:00:00 2001 From: William Horn Date: Mon, 4 Aug 2025 07:58:08 -0800 Subject: [PATCH 07/10] Update changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62b535d..41b0ef4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.1.5] + +### Changed +- Updated OPERA PGE container version to 2.1.3. + ## [0.1.4] ### Added From 3795e00d87ff8c1ea0db22d66717dfe94eb9bd1d Mon Sep 17 00:00:00 2001 From: Andrew Johnston Date: Tue, 12 Aug 2025 11:41:24 -0800 Subject: [PATCH 08/10] Revert "Update pge template yml to add new field" This reverts commit a2feab6b13c28ffa29bf48fda9f4eabbfc11792a. --- Dockerfile | 2 +- src/hyp3_opera_rtc/templates/pge.yml.j2 | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 89fca67..287560b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/nasa/opera-sds-pge/opera_pge/rtc_s1:2.1.3 +FROM ghcr.io/nasa/opera-sds-pge/opera_pge/rtc_s1:2.1.1 # For opencontainers label definitions, see: # https://github.com/opencontainers/image-spec/blob/master/annotations.md diff --git a/src/hyp3_opera_rtc/templates/pge.yml.j2 b/src/hyp3_opera_rtc/templates/pge.yml.j2 index 4d00272..b26ea48 100644 --- a/src/hyp3_opera_rtc/templates/pge.yml.j2 +++ b/src/hyp3_opera_rtc/templates/pge.yml.j2 @@ -32,7 +32,6 @@ RunConfig: ErrorCodeBase: 300000 SchemaPath: /home/rtc_user/opera/pge/rtc_s1/schema/rtc_s1_sas_schema.yaml IsoTemplatePath: /home/rtc_user/opera/pge/rtc_s1/templates/OPERA_ISO_metadata_L2_RTC_S1_template.xml.jinja2 - IsoMeasuredParameterDescriptions: /home/rtc_user/opera/pge/rtc_s1/templates/rtc_s1_measured_parameters.yaml # Date field which designates the point after which the # RTC static layer product(s) should be considered valid. # This field must be provided for RTC-S1 jobs when static layer From bd8383387ec55e16640ca1b8552bab640b829832 Mon Sep 17 00:00:00 2001 From: Andrew Johnston Date: Wed, 13 Aug 2025 11:43:19 -0800 Subject: [PATCH 09/10] Revert "Update pge template yml to add new field" --- CHANGELOG.md | 6 ------ Dockerfile | 2 +- src/hyp3_opera_rtc/templates/pge.yml.j2 | 1 - 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41b0ef4..62b535d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,12 +6,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - -## [0.1.5] - -### Changed -- Updated OPERA PGE container version to 2.1.3. - ## [0.1.4] ### Added diff --git a/Dockerfile b/Dockerfile index 89fca67..287560b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/nasa/opera-sds-pge/opera_pge/rtc_s1:2.1.3 +FROM ghcr.io/nasa/opera-sds-pge/opera_pge/rtc_s1:2.1.1 # For opencontainers label definitions, see: # https://github.com/opencontainers/image-spec/blob/master/annotations.md diff --git a/src/hyp3_opera_rtc/templates/pge.yml.j2 b/src/hyp3_opera_rtc/templates/pge.yml.j2 index 4d00272..b26ea48 100644 --- a/src/hyp3_opera_rtc/templates/pge.yml.j2 +++ b/src/hyp3_opera_rtc/templates/pge.yml.j2 @@ -32,7 +32,6 @@ RunConfig: ErrorCodeBase: 300000 SchemaPath: /home/rtc_user/opera/pge/rtc_s1/schema/rtc_s1_sas_schema.yaml IsoTemplatePath: /home/rtc_user/opera/pge/rtc_s1/templates/OPERA_ISO_metadata_L2_RTC_S1_template.xml.jinja2 - IsoMeasuredParameterDescriptions: /home/rtc_user/opera/pge/rtc_s1/templates/rtc_s1_measured_parameters.yaml # Date field which designates the point after which the # RTC static layer product(s) should be considered valid. # This field must be provided for RTC-S1 jobs when static layer From fa53cb23c64ed6c6c4ce36ec25f6fb1708cd7d2f Mon Sep 17 00:00:00 2001 From: Andrew Johnston Date: Wed, 13 Aug 2025 16:48:54 -0800 Subject: [PATCH 10/10] set chunk size when uploading output files to s3 --- CHANGELOG.md | 5 +++++ environment.yml | 2 +- src/hyp3_opera_rtc/upload_rtc.py | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62b535d..6eaf082 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.1.5] + +### Changed +- Avoid multipart uploads by using a very large chunk_size so the S3 ETag will always be an md5 checksum. + ## [0.1.4] ### Added diff --git a/environment.yml b/environment.yml index 9c7dc94..633e390 100644 --- a/environment.yml +++ b/environment.yml @@ -9,7 +9,7 @@ dependencies: - pip - gdal>=3.0 - shapely - - hyp3lib>=4,<5 + - hyp3lib>=4.0.1,<5 - boto3 - jinja2 # For packaging, and testing diff --git a/src/hyp3_opera_rtc/upload_rtc.py b/src/hyp3_opera_rtc/upload_rtc.py index dca6e2b..16a297e 100644 --- a/src/hyp3_opera_rtc/upload_rtc.py +++ b/src/hyp3_opera_rtc/upload_rtc.py @@ -14,7 +14,7 @@ def upload_rtc(bucket: str, bucket_prefix: str, output_dir: Path) -> None: output_files.append(output_zip) for output_file in output_files: - upload_file_to_s3(output_file, bucket, bucket_prefix) + upload_file_to_s3(output_file, bucket, bucket_prefix, chunk_size=100_000_000) def make_zip(output_files: list[Path], output_dir: Path) -> Path: