From 1efaf5cd403a81e148b2c8c7e13d2077e8e5b674 Mon Sep 17 00:00:00 2001 From: William Horn Date: Fri, 18 Jul 2025 09:51:58 -0800 Subject: [PATCH 1/8] Add opera SLC job spec --- job_spec/OPERA_RTC_S1_SLC.yml | 43 +++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 job_spec/OPERA_RTC_S1_SLC.yml diff --git a/job_spec/OPERA_RTC_S1_SLC.yml b/job_spec/OPERA_RTC_S1_SLC.yml new file mode 100644 index 000000000..e8d2109c2 --- /dev/null +++ b/job_spec/OPERA_RTC_S1_SLC.yml @@ -0,0 +1,43 @@ +OPERA_RTC_S1_SLC: + required_parameters: + - granules + parameters: + granules: + api_schema: + type: array + minItems: 1 + maxItems: 1 + example: + - S1A_IW_SLC__1SSV_20150621T120220_20150621T120232_006471_008934_72D8 + items: + anyOf: + - description: The name of the Sentinel-1 SLC granule to process + type: string + pattern: "^S1[ABC]_IW_SLC__1S[SD][VH]" + minLength: 67 + maxLength: 67 + example: S1A_IW_SLC__1SSV_20150621T120220_20150621T120232_006471_008934_72D8 + cost_profiles: + EDC: + cost: 1.0 + DEFAULT: + cost: 1.0 + validators: + - check_opera_rtc_s1_date + - check_opera_rtc_s1_bounds + steps: + - name: '' + image: ghcr.io/asfhyp3/hyp3-opera-rtc + command: + - Ref::granules + - --bucket + - '!Ref Bucket' + - --bucket-prefix + - Ref::job_id + timeout: 3600 # 1 hr + compute_environment: Default + vcpu: 1 + memory: 7600 + secrets: + - EARTHDATA_USERNAME + - EARTHDATA_PASSWORD From 25b8f2cc3bf297b3aa3981d362efd4a3e764d6da Mon Sep 17 00:00:00 2001 From: William Horn Date: Fri, 18 Jul 2025 10:07:23 -0800 Subject: [PATCH 2/8] Update changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 40bf66c76..ac083db6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [10.9.2] + +### Added +- Add `OPERA_RTC_S1_SLC` job spec that processes all bursts in a SLC. + ## [10.10.0] ### Added From 30bbc405505a63060cb24b465deb532bdf23a696 Mon Sep 17 00:00:00 2001 From: William Horn Date: Tue, 22 Jul 2025 10:31:53 -0800 Subject: [PATCH 3/8] Add new opera test deployment to gh action --- .github/workflows/deploy-daac-test.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/deploy-daac-test.yml b/.github/workflows/deploy-daac-test.yml index dc9792a0b..9000dd5b7 100644 --- a/.github/workflows/deploy-daac-test.yml +++ b/.github/workflows/deploy-daac-test.yml @@ -40,6 +40,24 @@ jobs: ami_id: /ngap/amis/image_id_ecs_al2023_x86 distribution_url: 'https://d1riv60tezqha9.cloudfront.net' + - environment: hyp3-opera-uat + api_name: hyp3-opera-test + template_bucket: cf-templates-118ylv0o6jp2n-us-west-2 + image_tag: test + product_lifetime_in_days: 14 + default_credits_per_user: 0 + default_application_status: APPROVED + cost_profile: EDC + opera_rtc_s1_end_date: 2026-01-01 + job_files: >- + job_spec/OPERA_RTC_S1_SLC.yml + instance_types: c6id.xlarge,c6id.2xlarge,c6id.4xlarge,c6id.8xlarge,c6idn.xlarge + default_max_vcpus: 3000 + expanded_max_vcpus: 3000 + required_surplus: 0 + security_environment: EDC + ami_id: /ngap/amis/image_id_ecs_al2023_x86 + distribution_url: 'https://d1riv60tezqha9.cloudfront.net' environment: name: ${{ matrix.environment }} From 3fafab8929908e2e0e3312d3d74470c6663496cf Mon Sep 17 00:00:00 2001 From: William Horn Date: Tue, 22 Jul 2025 10:32:32 -0800 Subject: [PATCH 4/8] Remove date check from opera-slc job type --- job_spec/OPERA_RTC_S1_SLC.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/job_spec/OPERA_RTC_S1_SLC.yml b/job_spec/OPERA_RTC_S1_SLC.yml index e8d2109c2..cf7086c32 100644 --- a/job_spec/OPERA_RTC_S1_SLC.yml +++ b/job_spec/OPERA_RTC_S1_SLC.yml @@ -23,7 +23,6 @@ OPERA_RTC_S1_SLC: DEFAULT: cost: 1.0 validators: - - check_opera_rtc_s1_date - check_opera_rtc_s1_bounds steps: - name: '' From a4e5b771284e3c717377f80b402f3f279972ef1a Mon Sep 17 00:00:00 2001 From: William Horn Date: Tue, 22 Jul 2025 13:15:59 -0800 Subject: [PATCH 5/8] Add ++omp-num-theads to opera slc job type --- job_spec/OPERA_RTC_S1_SLC.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/job_spec/OPERA_RTC_S1_SLC.yml b/job_spec/OPERA_RTC_S1_SLC.yml index cf7086c32..4633659cd 100644 --- a/job_spec/OPERA_RTC_S1_SLC.yml +++ b/job_spec/OPERA_RTC_S1_SLC.yml @@ -33,6 +33,8 @@ OPERA_RTC_S1_SLC: - '!Ref Bucket' - --bucket-prefix - Ref::job_id + - ++omp-num-threads + - '2' timeout: 3600 # 1 hr compute_environment: Default vcpu: 1 From f8492205ee113ba6a2d7bc77867306bb9c3e6e7f Mon Sep 17 00:00:00 2001 From: Andrew Johnston Date: Tue, 22 Jul 2025 13:55:48 -0800 Subject: [PATCH 6/8] update resources for opera_rtc_s1_slc jobs --- .github/workflows/deploy-daac-test.yml | 2 +- job_spec/OPERA_RTC_S1_SLC.yml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy-daac-test.yml b/.github/workflows/deploy-daac-test.yml index 9000dd5b7..4e807c704 100644 --- a/.github/workflows/deploy-daac-test.yml +++ b/.github/workflows/deploy-daac-test.yml @@ -51,7 +51,7 @@ jobs: opera_rtc_s1_end_date: 2026-01-01 job_files: >- job_spec/OPERA_RTC_S1_SLC.yml - instance_types: c6id.xlarge,c6id.2xlarge,c6id.4xlarge,c6id.8xlarge,c6idn.xlarge + instance_types: c6i.2xlarge,c7i.2xlarge,c6id.2xlarge,c6id.4xlarge,c6id.8xlarge default_max_vcpus: 3000 expanded_max_vcpus: 3000 required_surplus: 0 diff --git a/job_spec/OPERA_RTC_S1_SLC.yml b/job_spec/OPERA_RTC_S1_SLC.yml index 4633659cd..f25d6a5b5 100644 --- a/job_spec/OPERA_RTC_S1_SLC.yml +++ b/job_spec/OPERA_RTC_S1_SLC.yml @@ -33,12 +33,12 @@ OPERA_RTC_S1_SLC: - '!Ref Bucket' - --bucket-prefix - Ref::job_id - - ++omp-num-threads - - '2' + - --num-workers + - '8' timeout: 3600 # 1 hr compute_environment: Default - vcpu: 1 - memory: 7600 + vcpu: 8 + memory: 15200 secrets: - EARTHDATA_USERNAME - EARTHDATA_PASSWORD From c426dd92ba2e7d9c203d5bd409a7b94e2a453128 Mon Sep 17 00:00:00 2001 From: Andrew Johnston Date: Tue, 22 Jul 2025 16:11:06 -0800 Subject: [PATCH 7/8] update opera_rtc_s1_slc timeout to 2 hours --- job_spec/OPERA_RTC_S1_SLC.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/job_spec/OPERA_RTC_S1_SLC.yml b/job_spec/OPERA_RTC_S1_SLC.yml index f25d6a5b5..3e4aef636 100644 --- a/job_spec/OPERA_RTC_S1_SLC.yml +++ b/job_spec/OPERA_RTC_S1_SLC.yml @@ -35,7 +35,7 @@ OPERA_RTC_S1_SLC: - Ref::job_id - --num-workers - '8' - timeout: 3600 # 1 hr + timeout: 7200 # 2 hr compute_environment: Default vcpu: 8 memory: 15200 From 1fe091f8e76b942e2132cdada0d38cfb03573517 Mon Sep 17 00:00:00 2001 From: Andrew Johnston Date: Wed, 23 Jul 2025 09:40:42 -0800 Subject: [PATCH 8/8] add hyp3-opera-prod deployment --- .github/workflows/deploy-daac-prod.yml | 19 +++++++++++++++++++ .github/workflows/deploy-daac-test.yml | 4 ++-- CHANGELOG.md | 3 ++- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy-daac-prod.yml b/.github/workflows/deploy-daac-prod.yml index ff01d2e78..bf38d7d05 100644 --- a/.github/workflows/deploy-daac-prod.yml +++ b/.github/workflows/deploy-daac-prod.yml @@ -38,6 +38,25 @@ jobs: ami_id: /ngap/amis/image_id_ecs_al2023_x86 distribution_url: 'https://d3gm2hf49xd6jj.cloudfront.net' + - environment: hyp3-opera-prod + api_name: hyp3-opera-prod + template_bucket: cf-templates-118mtzosmrltk-us-west-2 + image_tag: latest + product_lifetime_in_days: 14 + default_credits_per_user: 0 + default_application_status: APPROVED + cost_profile: EDC + opera_rtc_s1_end_date: 2026-01-01 + job_files: >- + job_spec/OPERA_RTC_S1_SLC.yml + instance_types: c6i.2xlarge,c7i.2xlarge,c6id.2xlarge,c6id.4xlarge,c6id.8xlarge + default_max_vcpus: 10000 + expanded_max_vcpus: 10000 + required_surplus: 0 + security_environment: EDC + ami_id: /ngap/amis/image_id_ecs_al2023_x86 + distribution_url: 'https://d3gm2hf49xd6jj.cloudfront.net' + environment: name: ${{ matrix.environment }} diff --git a/.github/workflows/deploy-daac-test.yml b/.github/workflows/deploy-daac-test.yml index 4e807c704..a679b5ccb 100644 --- a/.github/workflows/deploy-daac-test.yml +++ b/.github/workflows/deploy-daac-test.yml @@ -52,8 +52,8 @@ jobs: job_files: >- job_spec/OPERA_RTC_S1_SLC.yml instance_types: c6i.2xlarge,c7i.2xlarge,c6id.2xlarge,c6id.4xlarge,c6id.8xlarge - default_max_vcpus: 3000 - expanded_max_vcpus: 3000 + default_max_vcpus: 800 + expanded_max_vcpus: 800 required_surplus: 0 security_environment: EDC ami_id: /ngap/amis/image_id_ecs_al2023_x86 diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c80cd956..95283a80d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added a new `ARIA_S1_COSEIS` job spec to ARIA JPL deployments. -- Added `OPERA_RTC_S1_SLC` job spec that processes all bursts in a SLC as well as `hyp3-opera-uat` deployment. +- Added `OPERA_RTC_S1_SLC` job spec that processes all bursts in an SLC. +- Created new hyp3-opera-uat and hyp3-opera-prod deployments. ## [10.10.0]