From cdbeb8a6ce6b444f3143a0c21dd28c5cc85ff711 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Fri, 25 Apr 2025 14:20:12 -0800 Subject: [PATCH 01/16] add jira integration --- .github/workflows/create-jira-issue.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .github/workflows/create-jira-issue.yml diff --git a/.github/workflows/create-jira-issue.yml b/.github/workflows/create-jira-issue.yml new file mode 100644 index 0000000..5765047 --- /dev/null +++ b/.github/workflows/create-jira-issue.yml @@ -0,0 +1,15 @@ +name: Create Jira issue + +on: + issues: + types: [labeled] + +jobs: + call-create-jira-issue-workflow: + uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.18.1 + secrets: + JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} + JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} + JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }} + JIRA_PROJECT: ${{ secrets.JIRA_PROJECT }} + JIRA_FIELDS: ${{ secrets.JIRA_FIELDS }} From 530347d64ce941a324a2c42267717f688d4e68e4 Mon Sep 17 00:00:00 2001 From: William Horn Date: Tue, 29 Apr 2025 10:01:11 -0800 Subject: [PATCH 02/16] Add inner folder to zip and exclude some files --- src/hyp3_opera_rtc/upload_rtc.py | 16 +++++++++++----- tests/test_upload_rtc.py | 13 ++++++++++++- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/hyp3_opera_rtc/upload_rtc.py b/src/hyp3_opera_rtc/upload_rtc.py index 4cc4eb1..b5cfb5e 100644 --- a/src/hyp3_opera_rtc/upload_rtc.py +++ b/src/hyp3_opera_rtc/upload_rtc.py @@ -15,20 +15,26 @@ def upload_rtc(bucket: str, bucket_prefix: str, output_dir: Path) -> None: def make_zip(output_files: list[Path], output_dir: Path) -> Path: + zip_name = make_zip_name(output_files) + zip_path = output_dir / zip_name + zip_archive_path = output_dir / 'zip' - zip_archive_path.mkdir(exist_ok=True) + (zip_archive_path / zip_name).mkdir(exist_ok=True, parents=True) + file_extensions_to_include = set(['.png', '.xml', '.tif', '.h5']) for output_file in output_files: - copyfile(output_file, zip_archive_path / output_file.name) + if output_file.suffix not in file_extensions_to_include: + continue - zip_path = output_dir / make_zip_name(output_files) - output_zip = make_archive(base_name=str(zip_path), format='zip', root_dir=zip_archive_path) + zip_dest_path = zip_archive_path / zip_name / output_file.name + copyfile(output_file, zip_dest_path) + output_zip = make_archive(base_name=str(zip_path), format='zip', root_dir=zip_archive_path) return Path(output_zip) def make_zip_name(product_files: list[Path]) -> str: - h5_file = [f for f in product_files if f.name.endswith('h5')].pop() + h5_file = next(f for f in product_files if f.name.endswith('h5')) return h5_file.name.split('.h5')[0] diff --git a/tests/test_upload_rtc.py b/tests/test_upload_rtc.py index 602eff4..6bf2340 100644 --- a/tests/test_upload_rtc.py +++ b/tests/test_upload_rtc.py @@ -18,14 +18,25 @@ def test_upload_rtc(rtc_results_dir, rtc_output_files, s3_bucket): assert len(resp['Contents']) == 9 + product_name = 'OPERA_L2_RTC-S1_T115-245714-IW1_20240809T141633Z_20250411T185446Z_S1A_30_v1.0' zip_s3_key = [c['Key'] for c in resp['Contents'] if c['Key'].endswith('.zip')].pop() + assert Path(zip_s3_key).name == f'{product_name}.zip' + zip_download_path = rtc_results_dir / 'output.zip' aws.S3_CLIENT.download_file(s3_bucket, zip_s3_key, zip_download_path) with ZipFile(zip_download_path) as zf: files_in_zip = set([f.filename for f in zf.infolist()]) - assert files_in_zip == set(rtc_output_files) + assert files_in_zip == set([ + f'{product_name}/', + f'{product_name}/OPERA_L2_RTC-S1_T115-245714-IW1_20240809T141633Z_20250411T185446Z_S1A_30_v1.0_BROWSE.png', + f'{product_name}/OPERA_L2_RTC-S1_T115-245714-IW1_20240809T141633Z_20250411T185446Z_S1A_30_v1.0.iso.xml', + f'{product_name}/OPERA_L2_RTC-S1_T115-245714-IW1_20240809T141633Z_20250411T185446Z_S1A_30_v1.0.h5', + f'{product_name}/OPERA_L2_RTC-S1_T115-245714-IW1_20240809T141633Z_20250411T185446Z_S1A_30_v1.0_mask.tif', + f'{product_name}/OPERA_L2_RTC-S1_T115-245714-IW1_20240809T141633Z_20250411T185446Z_S1A_30_v1.0_VH.tif', + f'{product_name}/OPERA_L2_RTC-S1_T115-245714-IW1_20240809T141633Z_20250411T185446Z_S1A_30_v1.0_VV.tif', + ]) def test_make_zip_name(rtc_output_files): From 998b2820b44c7937959240fbb31451e6595f38c0 Mon Sep 17 00:00:00 2001 From: William Horn Date: Tue, 29 Apr 2025 10:19:42 -0800 Subject: [PATCH 03/16] Improve test to check zip filename --- tests/test_upload_rtc.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_upload_rtc.py b/tests/test_upload_rtc.py index 6bf2340..9bae20d 100644 --- a/tests/test_upload_rtc.py +++ b/tests/test_upload_rtc.py @@ -20,7 +20,9 @@ def test_upload_rtc(rtc_results_dir, rtc_output_files, s3_bucket): product_name = 'OPERA_L2_RTC-S1_T115-245714-IW1_20240809T141633Z_20250411T185446Z_S1A_30_v1.0' zip_s3_key = [c['Key'] for c in resp['Contents'] if c['Key'].endswith('.zip')].pop() - assert Path(zip_s3_key).name == f'{product_name}.zip' + zip_filename = zip_s3_key.split(f'{prefix}/').pop() + + assert zip_filename == f'{product_name}.zip' zip_download_path = rtc_results_dir / 'output.zip' aws.S3_CLIENT.download_file(s3_bucket, zip_s3_key, zip_download_path) From 563258cc67a466c67ec950b047db4003c9372a49 Mon Sep 17 00:00:00 2001 From: William Horn Date: Tue, 29 Apr 2025 10:23:07 -0800 Subject: [PATCH 04/16] Changelog and ruff --- CHANGELOG.md | 6 ++++++ tests/test_upload_rtc.py | 20 +++++++++++--------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b035187..e15f6d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.1.1] + +### Changed +- All files in zip now contained in folder named after the product +- Remove `log` and `catalog.json` from the zip + ## [0.1.0] ### Added diff --git a/tests/test_upload_rtc.py b/tests/test_upload_rtc.py index 9bae20d..0a3ec30 100644 --- a/tests/test_upload_rtc.py +++ b/tests/test_upload_rtc.py @@ -30,15 +30,17 @@ def test_upload_rtc(rtc_results_dir, rtc_output_files, s3_bucket): with ZipFile(zip_download_path) as zf: files_in_zip = set([f.filename for f in zf.infolist()]) - assert files_in_zip == set([ - f'{product_name}/', - f'{product_name}/OPERA_L2_RTC-S1_T115-245714-IW1_20240809T141633Z_20250411T185446Z_S1A_30_v1.0_BROWSE.png', - f'{product_name}/OPERA_L2_RTC-S1_T115-245714-IW1_20240809T141633Z_20250411T185446Z_S1A_30_v1.0.iso.xml', - f'{product_name}/OPERA_L2_RTC-S1_T115-245714-IW1_20240809T141633Z_20250411T185446Z_S1A_30_v1.0.h5', - f'{product_name}/OPERA_L2_RTC-S1_T115-245714-IW1_20240809T141633Z_20250411T185446Z_S1A_30_v1.0_mask.tif', - f'{product_name}/OPERA_L2_RTC-S1_T115-245714-IW1_20240809T141633Z_20250411T185446Z_S1A_30_v1.0_VH.tif', - f'{product_name}/OPERA_L2_RTC-S1_T115-245714-IW1_20240809T141633Z_20250411T185446Z_S1A_30_v1.0_VV.tif', - ]) + assert files_in_zip == set( + [ + f'{product_name}/', + f'{product_name}/OPERA_L2_RTC-S1_T115-245714-IW1_20240809T141633Z_20250411T185446Z_S1A_30_v1.0_BROWSE.png', + f'{product_name}/OPERA_L2_RTC-S1_T115-245714-IW1_20240809T141633Z_20250411T185446Z_S1A_30_v1.0.iso.xml', + f'{product_name}/OPERA_L2_RTC-S1_T115-245714-IW1_20240809T141633Z_20250411T185446Z_S1A_30_v1.0.h5', + f'{product_name}/OPERA_L2_RTC-S1_T115-245714-IW1_20240809T141633Z_20250411T185446Z_S1A_30_v1.0_mask.tif', + f'{product_name}/OPERA_L2_RTC-S1_T115-245714-IW1_20240809T141633Z_20250411T185446Z_S1A_30_v1.0_VH.tif', + f'{product_name}/OPERA_L2_RTC-S1_T115-245714-IW1_20240809T141633Z_20250411T185446Z_S1A_30_v1.0_VV.tif', + ] + ) def test_make_zip_name(rtc_output_files): From 6cc47a17c301b5987ec112f15284cb6ac690dc04 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Thu, 1 May 2025 07:36:04 -0500 Subject: [PATCH 05/16] switch slc-based download --- src/hyp3_opera_rtc/prep_rtc.py | 36 +++++++++++++++++++------ src/hyp3_opera_rtc/templates/pge.yml.j2 | 3 +++ 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/src/hyp3_opera_rtc/prep_rtc.py b/src/hyp3_opera_rtc/prep_rtc.py index a3147f1..e2532ab 100644 --- a/src/hyp3_opera_rtc/prep_rtc.py +++ b/src/hyp3_opera_rtc/prep_rtc.py @@ -2,13 +2,13 @@ import os import warnings from pathlib import Path -from shutil import make_archive from zipfile import ZipFile import hyp3lib.fetch import lxml.etree as ET import requests -from burst2safe.burst2safe import burst2safe +from hyp3lib.fetch import download_file +from hyp3lib.scene import get_download_url from jinja2 import Template from shapely.geometry import Polygon, box @@ -49,13 +49,33 @@ def get_s1_granule_bbox(granule_path: Path, buffer: float = 0.025) -> Polygon: return box(*footprint.bounds) -def granule_exists(granule: str) -> bool: +def get_granule_cmr(granule: str) -> bool: params = (('short_name', 'SENTINEL-1_BURSTS'), ('granule_ur', granule)) response = requests.get(CMR_URL, params=params) response.raise_for_status() + return response + + +def granule_exists(granule: str) -> bool: + response = get_granule_cmr(granule) return bool(response.json()['items']) +def get_granule_slc_params(granule: str) -> tuple[str, str]: + response = get_granule_cmr(granule) + items = response.json()['items'] + assert len(items) == 1 + item = items[0] + + source_slc = item['umm']['InputGranules'][0] + + opera_burst_id = [attr for attr in item['umm']['AdditionalAttributes'] if attr['Name'] == 'BURST_ID_FULL'] + assert len(opera_burst_id) == 1 + opera_burst_id = opera_burst_id[0]['Values'][0] + + return source_slc, opera_burst_id + + def validate_co_pol_granule(granule: str) -> None: pol = granule.split('_')[4] if pol not in {'VV', 'HH'}: @@ -105,16 +125,15 @@ def prep_rtc( dual_pol = granule_exists(cross_pol_granule) if dual_pol: print(f'Found cross-pol granule: {cross_pol_granule}') - granules = [co_pol_granule, cross_pol_granule] else: print('No cross-pol granule found') - granules = [co_pol_granule] - safe_path = burst2safe(granules=granules, all_anns=True, work_dir=input_dir) - granule_path = Path(make_archive(base_name=str(safe_path.with_suffix('')), format='zip', base_dir=str(safe_path))) + source_slc, opera_burst_id = get_granule_slc_params(co_pol_granule) + download_url = get_download_url(source_slc) + granule_path = download_file(download_url, chunk_size=10485760) print(f'Created archive: {granule_path}') - orbit_path = orbit.get_orbit(safe_path.with_suffix('').name, save_dir=input_dir) + orbit_path = orbit.get_orbit(granule_path.with_suffix('').name, save_dir=input_dir) print(f'Downloaded orbit file: {orbit_path}') db_path = download_burst_db(input_dir) @@ -130,6 +149,7 @@ def prep_rtc( 'orbit_path': str(orbit_path), 'db_path': str(db_path), 'dem_path': str(dem_path), + 'opera_burst_id': opera_burst_id, 'scratch_dir': str(scratch_dir), 'output_dir': str(output_dir), 'dual_pol': dual_pol, diff --git a/src/hyp3_opera_rtc/templates/pge.yml.j2 b/src/hyp3_opera_rtc/templates/pge.yml.j2 index 171d153..13bc048 100644 --- a/src/hyp3_opera_rtc/templates/pge.yml.j2 +++ b/src/hyp3_opera_rtc/templates/pge.yml.j2 @@ -135,6 +135,9 @@ RunConfig: # Required. List of SAFE files (min=1) safe_file_path: - {{ granule_path }} + # Optional. Burst ID to process (empty for all bursts) + burst_id: + - {{ opera_burst_id }} # Required. List of orbit (EOF) files (min=1) orbit_file_path: - {{ orbit_path }} From 45d49b6ec318a4261ef14e02d9a1aec2f26c17a8 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Thu, 1 May 2025 08:03:14 -0500 Subject: [PATCH 06/16] add tests --- src/hyp3_opera_rtc/prep_rtc.py | 15 +++++++++------ tests/test_prep_rtc.py | 9 +++++++++ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/hyp3_opera_rtc/prep_rtc.py b/src/hyp3_opera_rtc/prep_rtc.py index e2532ab..5b0ba74 100644 --- a/src/hyp3_opera_rtc/prep_rtc.py +++ b/src/hyp3_opera_rtc/prep_rtc.py @@ -61,13 +61,11 @@ def granule_exists(granule: str) -> bool: return bool(response.json()['items']) -def get_granule_slc_params(granule: str) -> tuple[str, str]: - response = get_granule_cmr(granule) - items = response.json()['items'] - assert len(items) == 1 - item = items[0] +def parse_response_for_slc_params(response_dict: dict) -> tuple[str, str]: + assert len(response_dict['items']) == 1 + item = response_dict['items'][0] - source_slc = item['umm']['InputGranules'][0] + source_slc = item['umm']['InputGranules'][0][:-4] opera_burst_id = [attr for attr in item['umm']['AdditionalAttributes'] if attr['Name'] == 'BURST_ID_FULL'] assert len(opera_burst_id) == 1 @@ -76,6 +74,11 @@ def get_granule_slc_params(granule: str) -> tuple[str, str]: return source_slc, opera_burst_id +def get_granule_slc_params(granule: str) -> tuple[str, str]: + response = get_granule_cmr(granule) + return parse_response_for_slc_params(response.json()) + + def validate_co_pol_granule(granule: str) -> None: pol = granule.split('_')[4] if pol not in {'VV', 'HH'}: diff --git a/tests/test_prep_rtc.py b/tests/test_prep_rtc.py index 00b89de..3d63829 100644 --- a/tests/test_prep_rtc.py +++ b/tests/test_prep_rtc.py @@ -1,4 +1,6 @@ +import json import unittest.mock +from pathlib import Path import pytest import requests @@ -7,6 +9,13 @@ from hyp3_opera_rtc import prep_rtc +def test_parse_response_for_slc_params(): + test_response = json.loads(Path('tests/data/burst_response.json').read_text()) + slc_name, burst_id = prep_rtc.parse_response_for_slc_params(test_response) + assert slc_name == 'S1A_IW_SLC__1SDV_20250413T020809_20250413T020836_058732_07464F_EF1E' + assert burst_id == '035_073251_IW2' + + @responses.activate def test_granule_exists(): responses.get( From 4b3ea7b43c1105c1601d984c66af46b401213749 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Thu, 1 May 2025 08:05:48 -0500 Subject: [PATCH 07/16] add tests --- src/hyp3_opera_rtc/prep_rtc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hyp3_opera_rtc/prep_rtc.py b/src/hyp3_opera_rtc/prep_rtc.py index 5b0ba74..7b63937 100644 --- a/src/hyp3_opera_rtc/prep_rtc.py +++ b/src/hyp3_opera_rtc/prep_rtc.py @@ -49,7 +49,7 @@ def get_s1_granule_bbox(granule_path: Path, buffer: float = 0.025) -> Polygon: return box(*footprint.bounds) -def get_granule_cmr(granule: str) -> bool: +def get_granule_cmr(granule: str) -> requests.Response: params = (('short_name', 'SENTINEL-1_BURSTS'), ('granule_ur', granule)) response = requests.get(CMR_URL, params=params) response.raise_for_status() @@ -61,7 +61,7 @@ def granule_exists(granule: str) -> bool: return bool(response.json()['items']) -def parse_response_for_slc_params(response_dict: dict) -> tuple[str, str]: +def parse_response_for_slc_params(response_dict: dict) -> tuple: assert len(response_dict['items']) == 1 item = response_dict['items'][0] From 7952788441b451e96d6c8d2e06a8030b204ba4b1 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Thu, 1 May 2025 08:06:03 -0500 Subject: [PATCH 08/16] add test data --- tests/data/burst_response.json | 305 +++++++++++++++++++++++++++++++++ 1 file changed, 305 insertions(+) create mode 100644 tests/data/burst_response.json diff --git a/tests/data/burst_response.json b/tests/data/burst_response.json new file mode 100644 index 0000000..f9adb8c --- /dev/null +++ b/tests/data/burst_response.json @@ -0,0 +1,305 @@ +{ + "hits": 1, + "took": 28, + "items": [ + { + "meta": { + "concept-type": "granule", + "concept-id": "G3493873323-ASF", + "revision-id": 1, + "native-id": "S1_073251_IW2_20250413T020809_VV_EF1E-BURST", + "collection-concept-id": "C2709161906-ASF", + "provider-id": "ASF", + "format": "application/vnd.nasa.cmr.umm+json", + "revision-date": "2025-04-13T05:21:49.972Z" + }, + "umm": { + "TemporalExtent": { + "RangeDateTime": { + "BeginningDateTime": "2025-04-13T02:08:10.453534Z", + "EndingDateTime": "2025-04-13T02:08:13.555368Z" + } + }, + "OrbitCalculatedSpatialDomains": [ + { + "OrbitNumber": 58732 + } + ], + "GranuleUR": "S1_073251_IW2_20250413T020809_VV_EF1E-BURST", + "AdditionalAttributes": [ + { + "Name": "ASC_NODE_TIME", + "Values": [ + "2025-04-13T01:57:59.280569Z" + ] + }, + { + "Name": "ASCENDING_DESCENDING", + "Values": [ + "ASCENDING" + ] + }, + { + "Name": "AZIMUTH_ANX_TIME", + "Values": [ + "610.1584306672" + ] + }, + { + "Name": "AZIMUTH_TIME", + "Values": [ + "2025-04-13T02:08:09.435967" + ] + }, + { + "Name": "AZIMUTH_TIME_INTERVAL", + "Values": [ + "0.002055556299999998" + ] + }, + { + "Name": "BEAM_MODE", + "Values": [ + "IW" + ] + }, + { + "Name": "BEAM_MODE_DESC", + "Values": [ + "Interferometric Wide. 250 km swath, 5 m x 20 m spatial resolution and burst synchronization for interferometry. IW is considered to be the standard mode over land masses." + ] + }, + { + "Name": "BURST_ID_ABSOLUTE", + "Values": [ + "126150173" + ] + }, + { + "Name": "BURST_ID_FULL", + "Values": [ + "035_073251_IW2" + ] + }, + { + "Name": "BURST_ID_RELATIVE", + "Values": [ + "73251" + ] + }, + { + "Name": "BURST_INDEX", + "Values": [ + "0" + ] + }, + { + "Name": "BYTE_LENGTH", + "Values": [ + "154781148" + ] + }, + { + "Name": "BYTE_OFFSET", + "Values": [ + "108963" + ] + }, + { + "Name": "CENTER_LAT", + "Values": [ + "37.84383923820902" + ] + }, + { + "Name": "CENTER_LON", + "Values": [ + "-122.37700996731327" + ] + }, + { + "Name": "GROUP_ID", + "Values": [ + "S1A_IWDV_0121_0127_058732_035" + ] + }, + { + "Name": "LINES_PER_BURST", + "Values": [ + "1509" + ] + }, + { + "Name": "PATH_NUMBER", + "Values": [ + "35" + ] + }, + { + "Name": "POLARIZATION", + "Values": [ + "VV" + ] + }, + { + "Name": "PROCESSING_LEVEL", + "Values": [ + "L1" + ] + }, + { + "Name": "PROCESSING_TYPE", + "Values": [ + "BURST" + ] + }, + { + "Name": "SAMPLES_PER_BURST", + "Values": [ + "25643" + ] + }, + { + "Name": "SUBSWATH_NAME", + "Values": [ + "IW2" + ] + }, + { + "Name": "SV_POSITION_POST", + "Values": [ + "-3466943.640335,-4432648.526834,4280636.897919,2025-04-13T02:08:19.000000" + ] + }, + { + "Name": "SV_POSITION_PRE", + "Values": [ + "-3480520.469779,-4478505.865553,4221649.253332,2025-04-13T02:08:09.000000" + ] + }, + { + "Name": "SV_VELOCITY_POST", + "Values": [ + "1380.492592,4609.671815,5874.702457,2025-04-13T02:08:19.000000" + ] + }, + { + "Name": "SV_VELOCITY_PRE", + "Values": [ + "1334.859471,4561.698961,5922.715631,2025-04-13T02:08:09.000000" + ] + } + ], + "SpatialExtent": { + "HorizontalSpatialDomain": { + "Geometry": { + "GPolygons": [ + { + "Boundary": { + "Points": [ + { + "Latitude": 37.679959, + "Longitude": -122.867058 + }, + { + "Latitude": 37.751761, + "Longitude": -122.345075 + }, + { + "Latitude": 37.817265, + "Longitude": -121.851681 + }, + { + "Latitude": 38.009685, + "Longitude": -121.893464 + }, + { + "Latitude": 37.940016, + "Longitude": -122.387251 + }, + { + "Latitude": 37.86404, + "Longitude": -122.909645 + }, + { + "Latitude": 37.679959, + "Longitude": -122.867058 + } + ] + } + } + ] + } + } + }, + "ProviderDates": [ + { + "Type": "Insert", + "Date": "2025-04-13T05:21:49Z" + }, + { + "Type": "Update", + "Date": "2025-04-13T05:21:49Z" + } + ], + "CollectionReference": { + "ShortName": "SENTINEL-1_BURSTS", + "Version": "1" + }, + "PGEVersionClass": { + "PGEName": "Sentinel-1 IPF", + "PGEVersion": "003.91" + }, + "RelatedUrls": [ + { + "URL": "https://sentinel1-burst.asf.alaska.edu/S1A_IW_SLC__1SDV_20250413T020809_20250413T020836_058732_07464F_EF1E/IW2/VV/0.tiff", + "Type": "USE SERVICE API", + "Description": "Use the link to extract the burst as a GeoTIFF from the input granule.", + "Format": "GeoTIFF" + }, + { + "URL": "https://sentinel1-burst.asf.alaska.edu/S1A_IW_SLC__1SDV_20250413T020809_20250413T020836_058732_07464F_EF1E/IW2/VV/0.xml", + "Type": "USE SERVICE API", + "Description": "Use the link to extract an extended metadata file of the burst from the input granule.", + "Format": "XML" + }, + { + "URL": "https://sentinel1-burst.asf.alaska.edu/S1A_IW_SLC__1SDV_20250413T020809_20250413T020836_058732_07464F_EF1E/IW2/VV/0.zip", + "Type": "USE SERVICE API", + "Description": "Use the link to extract the burst in ESA SAFE format from the input granule.", + "Format": "SAFE" + }, + { + "URL": "https://sentinel1-burst-docs.asf.alaska.edu/", + "Type": "VIEW RELATED INFORMATION", + "Description": "Usage information for the API used to extract a burst product from a Sentinel-1 SLC granule." + } + ], + "InputGranules": [ + "S1A_IW_SLC__1SDV_20250413T020809_20250413T020836_058732_07464F_EF1E-SLC" + ], + "Platforms": [ + { + "Instruments": [ + { + "ShortName": "C-SAR", + "Characteristics": [ + { + "Name": "LookDirection", + "Value": "RIGHT" + } + ] + } + ], + "ShortName": "SENTINEL-1A" + } + ], + "MetadataSpecification": { + "URL": "https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6", + "Name": "UMM-G", + "Version": "1.6.6" + } + } + } + ] +} \ No newline at end of file From b6f6286f85d00469b0311d92c556eb213bff189a Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Thu, 1 May 2025 08:10:09 -0500 Subject: [PATCH 09/16] update dependencies --- CHANGELOG.md | 1 + environment.yml | 1 - pyproject.toml | 1 - 3 files changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e15f6d9..0f21ea1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Changed - All files in zip now contained in folder named after the product - Remove `log` and `catalog.json` from the zip +- Switch a full-SLC based data prep strategy ## [0.1.0] diff --git a/environment.yml b/environment.yml index 2ed7937..0046512 100644 --- a/environment.yml +++ b/environment.yml @@ -12,7 +12,6 @@ dependencies: - hyp3lib - boto3 - jinja2 - - burst2safe # For packaging, and testing - setuptools - setuptools_scm diff --git a/pyproject.toml b/pyproject.toml index 85a2fcc..a27bc1a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,6 @@ dependencies = [ "urllib3", "lxml", "jinja2", - "burst2safe" ] dynamic = ["version", "readme"] From 3472555a6060244c4112241d257b88df7c8cffae Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Thu, 1 May 2025 08:14:14 -0500 Subject: [PATCH 10/16] add path --- src/hyp3_opera_rtc/prep_rtc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hyp3_opera_rtc/prep_rtc.py b/src/hyp3_opera_rtc/prep_rtc.py index 7b63937..c77edf6 100644 --- a/src/hyp3_opera_rtc/prep_rtc.py +++ b/src/hyp3_opera_rtc/prep_rtc.py @@ -132,8 +132,8 @@ def prep_rtc( print('No cross-pol granule found') source_slc, opera_burst_id = get_granule_slc_params(co_pol_granule) - download_url = get_download_url(source_slc) - granule_path = download_file(download_url, chunk_size=10485760) + granule_path = download_file(get_download_url(source_slc), chunk_size=10485760) + granule_path = Path(granule_path) print(f'Created archive: {granule_path}') orbit_path = orbit.get_orbit(granule_path.with_suffix('').name, save_dir=input_dir) From a83991c259838ee8b2c8a1fc12263339b2c2b2b9 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Thu, 1 May 2025 08:28:22 -0500 Subject: [PATCH 11/16] fix path --- src/hyp3_opera_rtc/prep_rtc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hyp3_opera_rtc/prep_rtc.py b/src/hyp3_opera_rtc/prep_rtc.py index c77edf6..ecb75c9 100644 --- a/src/hyp3_opera_rtc/prep_rtc.py +++ b/src/hyp3_opera_rtc/prep_rtc.py @@ -132,7 +132,7 @@ def prep_rtc( print('No cross-pol granule found') source_slc, opera_burst_id = get_granule_slc_params(co_pol_granule) - granule_path = download_file(get_download_url(source_slc), chunk_size=10485760) + granule_path = download_file(get_download_url(source_slc), directory=str(input_dir), chunk_size=10485760) granule_path = Path(granule_path) print(f'Created archive: {granule_path}') From 03640fb017334213deca996cd8a62f2f4d7a5226 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Thu, 1 May 2025 08:46:02 -0500 Subject: [PATCH 12/16] update burst id --- src/hyp3_opera_rtc/prep_rtc.py | 12 +++++++----- tests/test_prep_rtc.py | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/hyp3_opera_rtc/prep_rtc.py b/src/hyp3_opera_rtc/prep_rtc.py index ecb75c9..c1c724c 100644 --- a/src/hyp3_opera_rtc/prep_rtc.py +++ b/src/hyp3_opera_rtc/prep_rtc.py @@ -61,17 +61,19 @@ def granule_exists(granule: str) -> bool: return bool(response.json()['items']) -def parse_response_for_slc_params(response_dict: dict) -> tuple: +def parse_response_for_slc_params(response_dict: dict) -> tuple[str,str]: assert len(response_dict['items']) == 1 item = response_dict['items'][0] source_slc = item['umm']['InputGranules'][0][:-4] + assert isinstance(source_slc, str) - opera_burst_id = [attr for attr in item['umm']['AdditionalAttributes'] if attr['Name'] == 'BURST_ID_FULL'] - assert len(opera_burst_id) == 1 - opera_burst_id = opera_burst_id[0]['Values'][0] + opera_burst_ids = [attr for attr in item['umm']['AdditionalAttributes'] if attr['Name'] == 'BURST_ID_FULL'] + assert len(opera_burst_ids) == 1 + opera_burst_id = opera_burst_ids[0]['Values'][0] + assert isinstance(opera_burst_id, str) - return source_slc, opera_burst_id + return source_slc, f't{opera_burst_id.lower()}' def get_granule_slc_params(granule: str) -> tuple[str, str]: diff --git a/tests/test_prep_rtc.py b/tests/test_prep_rtc.py index 3d63829..96f7c48 100644 --- a/tests/test_prep_rtc.py +++ b/tests/test_prep_rtc.py @@ -13,7 +13,7 @@ def test_parse_response_for_slc_params(): test_response = json.loads(Path('tests/data/burst_response.json').read_text()) slc_name, burst_id = prep_rtc.parse_response_for_slc_params(test_response) assert slc_name == 'S1A_IW_SLC__1SDV_20250413T020809_20250413T020836_058732_07464F_EF1E' - assert burst_id == '035_073251_IW2' + assert burst_id == 't035_073251_iw2' @responses.activate From fc0a44e07d22300af12889145fe8ad4a275e614e Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Thu, 1 May 2025 09:10:44 -0500 Subject: [PATCH 13/16] fix ruff --- src/hyp3_opera_rtc/prep_rtc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hyp3_opera_rtc/prep_rtc.py b/src/hyp3_opera_rtc/prep_rtc.py index c1c724c..3917fc0 100644 --- a/src/hyp3_opera_rtc/prep_rtc.py +++ b/src/hyp3_opera_rtc/prep_rtc.py @@ -61,7 +61,7 @@ def granule_exists(granule: str) -> bool: return bool(response.json()['items']) -def parse_response_for_slc_params(response_dict: dict) -> tuple[str,str]: +def parse_response_for_slc_params(response_dict: dict) -> tuple[str, str]: assert len(response_dict['items']) == 1 item = response_dict['items'][0] From 580021d97ecfeccd8a1063a5c124d0870d3c787e Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Thu, 1 May 2025 13:44:01 -0500 Subject: [PATCH 14/16] fix grammar --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f21ea1..af7b2e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Changed - All files in zip now contained in folder named after the product - Remove `log` and `catalog.json` from the zip -- Switch a full-SLC based data prep strategy +- Switch to a full-SLC based data prep strategy ## [0.1.0] From 4070631eb98e9534660ba8ec742224df50fe9551 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Thu, 1 May 2025 13:54:45 -0500 Subject: [PATCH 15/16] address andrew feedback --- src/hyp3_opera_rtc/prep_rtc.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/hyp3_opera_rtc/prep_rtc.py b/src/hyp3_opera_rtc/prep_rtc.py index 3917fc0..a837b3e 100644 --- a/src/hyp3_opera_rtc/prep_rtc.py +++ b/src/hyp3_opera_rtc/prep_rtc.py @@ -126,31 +126,25 @@ def prep_rtc( validate_co_pol_granule(co_pol_granule) - cross_pol_granule = get_cross_pol_name(co_pol_granule) - dual_pol = granule_exists(cross_pol_granule) - if dual_pol: - print(f'Found cross-pol granule: {cross_pol_granule}') - else: - print('No cross-pol granule found') - source_slc, opera_burst_id = get_granule_slc_params(co_pol_granule) - granule_path = download_file(get_download_url(source_slc), directory=str(input_dir), chunk_size=10485760) - granule_path = Path(granule_path) - print(f'Created archive: {granule_path}') + safe_path = download_file(get_download_url(source_slc), directory=str(input_dir), chunk_size=10485760) + safe_path = Path(safe_path) + dual_pol = safe_path.name[14] == 'D' + print(f'Created archive: {safe_path}') - orbit_path = orbit.get_orbit(granule_path.with_suffix('').name, save_dir=input_dir) + orbit_path = orbit.get_orbit(safe_path.with_suffix('').name, save_dir=input_dir) print(f'Downloaded orbit file: {orbit_path}') db_path = download_burst_db(input_dir) print(f'Downloaded burst database: {db_path}') dem_path = input_dir / 'dem.tif' - granule_bbox = get_s1_granule_bbox(granule_path) + granule_bbox = get_s1_granule_bbox(safe_path) dem.download_opera_dem_for_footprint(dem_path, granule_bbox) print(f'Downloaded DEM: {dem_path}') runconfig_dict = { - 'granule_path': str(granule_path), + 'granule_path': str(safe_path), 'orbit_path': str(orbit_path), 'db_path': str(db_path), 'dem_path': str(dem_path), From 864e38828a9f7d32137abfd86c8242e9f9fd9315 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Thu, 1 May 2025 13:55:48 -0500 Subject: [PATCH 16/16] address andrew feedback --- src/hyp3_opera_rtc/prep_rtc.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/hyp3_opera_rtc/prep_rtc.py b/src/hyp3_opera_rtc/prep_rtc.py index a837b3e..fc6fb28 100644 --- a/src/hyp3_opera_rtc/prep_rtc.py +++ b/src/hyp3_opera_rtc/prep_rtc.py @@ -49,21 +49,21 @@ def get_s1_granule_bbox(granule_path: Path, buffer: float = 0.025) -> Polygon: return box(*footprint.bounds) -def get_granule_cmr(granule: str) -> requests.Response: +def get_granule_cmr(granule: str) -> dict: params = (('short_name', 'SENTINEL-1_BURSTS'), ('granule_ur', granule)) response = requests.get(CMR_URL, params=params) response.raise_for_status() - return response + return response.json() def granule_exists(granule: str) -> bool: response = get_granule_cmr(granule) - return bool(response.json()['items']) + return bool(response['items']) -def parse_response_for_slc_params(response_dict: dict) -> tuple[str, str]: - assert len(response_dict['items']) == 1 - item = response_dict['items'][0] +def parse_response_for_slc_params(response: dict) -> tuple[str, str]: + assert len(response['items']) == 1 + item = response['items'][0] source_slc = item['umm']['InputGranules'][0][:-4] assert isinstance(source_slc, str) @@ -78,7 +78,7 @@ def parse_response_for_slc_params(response_dict: dict) -> tuple[str, str]: def get_granule_slc_params(granule: str) -> tuple[str, str]: response = get_granule_cmr(granule) - return parse_response_for_slc_params(response.json()) + return parse_response_for_slc_params(response) def validate_co_pol_granule(granule: str) -> None: