From 5b087154f7327462cbf9e844780aed8ac1e6efaa Mon Sep 17 00:00:00 2001 From: tcchase Date: Thu, 20 Feb 2025 11:27:02 -0500 Subject: [PATCH 1/4] feat: add initial alos-2 support --- SearchAPI/CMR/SubQuery.py | 2 +- SearchAPI/CMR/Translate/datasets.py | 3 +++ SearchAPI/CMR/Translate/parse_cmr_response.py | 10 +++++----- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/SearchAPI/CMR/SubQuery.py b/SearchAPI/CMR/SubQuery.py index 00087a00..5f77553f 100644 --- a/SearchAPI/CMR/SubQuery.py +++ b/SearchAPI/CMR/SubQuery.py @@ -61,7 +61,7 @@ def combine_params(self, params, extra_params): def should_use_asf_frame(self): asf_frame_platforms = ['SENTINEL-1A', 'SENTINEL-1B', 'ALOS'] - asf_frame_datasets = ['SENTINEL-1', 'OPERA-S1', 'SLC-BURST', 'ALOS PALSAR', 'ALOS AVNIR-2'] + asf_frame_datasets = ['SENTINEL-1', 'OPERA-S1', 'SLC-BURST', 'ALOS PALSAR', 'ALOS AVNIR-2', 'ALOS-2'] asf_frame_collections = [] for dataset in asf_frame_datasets: diff --git a/SearchAPI/CMR/Translate/datasets.py b/SearchAPI/CMR/Translate/datasets.py index 7b9da157..00c76509 100644 --- a/SearchAPI/CMR/Translate/datasets.py +++ b/SearchAPI/CMR/Translate/datasets.py @@ -132,6 +132,9 @@ "C2803501097-ASF", ], "SLC-BURST": ["C2709161906-ASF", "C1257024016-ASF"], + "ALOS-2": [ + "C3315903479-ASF" + ], "ALOS PALSAR": [ "C1206487504-ASF", "C1206485940-ASF", diff --git a/SearchAPI/CMR/Translate/parse_cmr_response.py b/SearchAPI/CMR/Translate/parse_cmr_response.py index 194fbb59..1525995e 100644 --- a/SearchAPI/CMR/Translate/parse_cmr_response.py +++ b/SearchAPI/CMR/Translate/parse_cmr_response.py @@ -70,7 +70,7 @@ def remove_field(f): if 'frameNumber' in req_fields: asf_frame_platforms = [ 'Sentinel-1A', 'Sentinel-1B', 'ALOS', 'SENTINEL-1A', 'SENTINEL-1B', - 'ERS-1', 'ERS-2', 'JERS-1', 'RADARSAT-1' + 'ERS-1', 'ERS-2', 'JERS-1', 'RADARSAT-1', 'ALOS-2' ] if result['platform'] in asf_frame_platforms: @@ -90,7 +90,7 @@ def remove_field(f): result['fileName'] = file_name.split('/')[-1] if file_name else None remove_field('fileName') - if 'stateVectors' in req_fields or ('canInsar' in req_fields and result['platform'] not in ['ALOS', 'RADARSAT-1', 'JERS-1', 'ERS-1', 'ERS-2']): + if 'stateVectors' in req_fields or ('canInsar' in req_fields and result['platform'] not in ['ALOS', 'ALOS-2', 'RADARSAT-1', 'JERS-1', 'ERS-1', 'ERS-2']): def parse_sv(sv): def float_or_none(a): try: @@ -131,7 +131,7 @@ def float_or_none(a): remove_field('stateVectors') if 'canInsar' in req_fields: - if result['platform'] in ['ALOS', 'RADARSAT-1', 'JERS-1', 'ERS-1', 'ERS-2']: + if result['platform'] in ['ALOS', 'ALOS-2', 'RADARSAT-1', 'JERS-1', 'ERS-1', 'ERS-2']: result['insarGrouping'] = get_val(field_paths['insarGrouping']) insarBaseline = get_val(field_paths['insarBaseline']) @@ -204,9 +204,9 @@ def float_or_none(a): if len(urls): result['downloadUrl'] = urls[0] result['fileName'] = result['granuleName'] + '.' + urls[0].split('.')[-1] + if result['platform'] in ['ALOS-2']: + result['beamMode'] = get_val(attr_path('BEAM_MODE')) - - def get_all_urls(): accessPath = './OnlineAccessURLs/OnlineAccessURL/URL' resourcesPath = './OnlineResources/OnlineResource/URL' From a4358d7c95b73d3df28a4b22915c776de5d5a9b2 Mon Sep 17 00:00:00 2001 From: tcchase Date: Mon, 24 Feb 2025 14:03:19 -0500 Subject: [PATCH 2/4] feat: add opera-disp dataset --- SearchAPI/CMR/Output/jsonlite.py | 3 +-- SearchAPI/CMR/Translate/datasets.py | 3 +++ SearchAPI/CMR/Translate/parse_cmr_response.py | 6 ++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/SearchAPI/CMR/Output/jsonlite.py b/SearchAPI/CMR/Output/jsonlite.py index 9183f8ac..3e893053 100644 --- a/SearchAPI/CMR/Output/jsonlite.py +++ b/SearchAPI/CMR/Output/jsonlite.py @@ -179,8 +179,7 @@ def getItem(self, p): burst['subswath'] = p['subswath'] result['burst'] = burst - - if p.get('operaBurstID') is not None or result['productID'].startswith('OPERA'): + if (p.get('operaBurstID') is not None or result['productID'].startswith('OPERA')) and not result['productID'].startswith('OPERA_L3_DISP'): result['opera'] = { 'operaBurstID': p.get('operaBurstID'), 'additionalUrls': p.get('additionalUrls'), diff --git a/SearchAPI/CMR/Translate/datasets.py b/SearchAPI/CMR/Translate/datasets.py index 00c76509..32cb6f8a 100644 --- a/SearchAPI/CMR/Translate/datasets.py +++ b/SearchAPI/CMR/Translate/datasets.py @@ -135,6 +135,9 @@ "ALOS-2": [ "C3315903479-ASF" ], + "OPERA-DISP" : [ + "C1271830354-ASF" + ], "ALOS PALSAR": [ "C1206487504-ASF", "C1206485940-ASF", diff --git a/SearchAPI/CMR/Translate/parse_cmr_response.py b/SearchAPI/CMR/Translate/parse_cmr_response.py index 1525995e..a321597a 100644 --- a/SearchAPI/CMR/Translate/parse_cmr_response.py +++ b/SearchAPI/CMR/Translate/parse_cmr_response.py @@ -227,7 +227,7 @@ def get_http_urls(): def get_s3_urls(): return [url for url in get_all_urls() if not url.endswith('.md5') and (url.startswith('s3://') or 's3credentials' in url)] - if result.get('product_file_id', '').startswith('OPERA'): + if result.get('product_file_id', '').startswith('OPERA') and not result.get('product_file_id', '').startswith('OPERA_L3_DISP'): result['beamMode'] = get_val(attr_path('BEAM_MODE')) result['additionalUrls'] = get_http_urls() result['configurationName'] = "Interferometric Wide. 250 km swath, 5 m x 20 m spatial resolution and burst synchronization for interferometry. IW is considered to be the standard mode over land masses." @@ -240,11 +240,13 @@ def get_s3_urls(): elif result.get('product_file_id', '').startswith('S1-GUNW') and result.get('ariaVersion') is None: version_unformatted = result.get('granuleName').split('v')[-1] result['ariaVersion'] = re.sub(r'[^0-9\.]', '', version_unformatted.replace("_", '.')) + if result.get('product_file_id', '').startswith('OPERA_L3_DISP'): + if (providerbrowseUrls := get_all_vals('./AssociatedBrowseImageUrls/ProviderBrowseUrl/URL')): + result['browse'] = [url for url in providerbrowseUrls if not url.startswith('s3://')] if result.get('platform', '') == 'NISAR': result['additionalUrls'] = get_http_urls() result['s3Urls'] = get_s3_urls() - return result From 31d936983954836f3e33d0eefe789a68457b7eac Mon Sep 17 00:00:00 2001 From: Tyler Chase Date: Thu, 27 Feb 2025 11:39:57 -0500 Subject: [PATCH 3/4] Update reusable-DeployStack-SearchAPI.yml --- .github/workflows/reusable-DeployStack-SearchAPI.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/reusable-DeployStack-SearchAPI.yml b/.github/workflows/reusable-DeployStack-SearchAPI.yml index 9f2bd9a8..a5c914b5 100644 --- a/.github/workflows/reusable-DeployStack-SearchAPI.yml +++ b/.github/workflows/reusable-DeployStack-SearchAPI.yml @@ -96,6 +96,9 @@ jobs: # If it's writing to a branch, the pull request to that branch already # passed, OR it's a developing branch that doesn't need PR's. No need # to run suite either way. + - uses: actions/setup-python@v5 + with: + python-version: '3.9' - name: Install pytest requirements if: github.event_name == 'pull_request' run: | From e4777ebddb46246c1fbd5cd855f0081f1ebe851c Mon Sep 17 00:00:00 2001 From: tcchase Date: Thu, 27 Feb 2025 16:40:05 -0500 Subject: [PATCH 4/4] fix: results platform access --- SearchAPI/CMR/Translate/parse_cmr_response.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/SearchAPI/CMR/Translate/parse_cmr_response.py b/SearchAPI/CMR/Translate/parse_cmr_response.py index a321597a..57f7ccb4 100644 --- a/SearchAPI/CMR/Translate/parse_cmr_response.py +++ b/SearchAPI/CMR/Translate/parse_cmr_response.py @@ -66,6 +66,8 @@ def remove_field(f): platform = get_val('./Platforms/Platform/ShortName') result['platform'] = platform remove_field('platform') + if result['platform'] in ['ALOS-2']: + result['beamMode'] = get_val(attr_path('BEAM_MODE')) if 'frameNumber' in req_fields: asf_frame_platforms = [ @@ -204,8 +206,7 @@ def float_or_none(a): if len(urls): result['downloadUrl'] = urls[0] result['fileName'] = result['granuleName'] + '.' + urls[0].split('.')[-1] - if result['platform'] in ['ALOS-2']: - result['beamMode'] = get_val(attr_path('BEAM_MODE')) + def get_all_urls(): accessPath = './OnlineAccessURLs/OnlineAccessURL/URL'