From fc91f20bbd06a9e32292846d0609aaf5419c9a39 Mon Sep 17 00:00:00 2001 From: tcchase Date: Wed, 23 Apr 2025 09:50:26 -0400 Subject: [PATCH 1/4] feat: add sentinel-1c dataset --- SearchAPI/CMR/SubQuery.py | 4 ++-- .../CMR/Translate/collections_by_platform.py | 23 +++++++++++++++++++ SearchAPI/CMR/Translate/datasets.py | 17 ++++++++++++++ SearchAPI/CMR/Translate/input_fixer.py | 14 ++++++----- SearchAPI/CMR/Translate/parse_cmr_response.py | 2 +- 5 files changed, 51 insertions(+), 9 deletions(-) diff --git a/SearchAPI/CMR/SubQuery.py b/SearchAPI/CMR/SubQuery.py index 5f77553f..10766cd1 100644 --- a/SearchAPI/CMR/SubQuery.py +++ b/SearchAPI/CMR/SubQuery.py @@ -60,8 +60,8 @@ def combine_params(self, params, extra_params): return final def should_use_asf_frame(self): - asf_frame_platforms = ['SENTINEL-1A', 'SENTINEL-1B', 'ALOS'] - asf_frame_datasets = ['SENTINEL-1', 'OPERA-S1', 'SLC-BURST', 'ALOS PALSAR', 'ALOS AVNIR-2', 'ALOS-2'] + asf_frame_platforms = ['SENTINEL-1A', 'SENTINEL-1B', 'SENTINEL-1C' 'ALOS'] + asf_frame_datasets = ['SENTINEL-1', 'OPERA-S1', 'SLC-BURST', 'ALOS PALSAR', 'ALOS AVNIR-2', 'ALOS-2', 'NISAR'] asf_frame_collections = [] for dataset in asf_frame_datasets: diff --git a/SearchAPI/CMR/Translate/collections_by_platform.py b/SearchAPI/CMR/Translate/collections_by_platform.py index e9f1ec8f..6aae55ec 100644 --- a/SearchAPI/CMR/Translate/collections_by_platform.py +++ b/SearchAPI/CMR/Translate/collections_by_platform.py @@ -333,6 +333,25 @@ "concept-id": "C1327985661-ASF" } ], + "SENTINEL-1C": [ + { "collection": "SENTINEL-1C_RAW", "concept-id": "C3486496642-ASF"}, + { "collection": "SENTINEL-1C_SLC", "concept-id": "C3470873558-ASF"}, + { "collection": "SENTINEL-1C_META_SLC", "concept-id": "C3484561061-ASF"}, + { "collection": "SENTINEL-1C_OCN", "concept-id": "C3484661603-ASF"}, + { "collection": "SENTINEL-1C_META_OCN", "concept-id": "C3484670556-ASF"}, + { "collection": "SENTINEL-1C_DP_GRD_HIGH", "concept-id": "C3486566209-ASF"}, + { "collection": "SENTINEL-1C_DP_META_GRD_HIGH", "concept-id": "C3486580531-ASF"}, + { "collection": "SENTINEL-1C_DP_GRD_MEDIUM", "concept-id": "C3486605959-ASF"}, + { "collection": "SENTINEL-1C_DP_META_GRD_MEDIUM", "concept-id": "C3486616327-ASF"}, + { "collection": "SENTINEL-1C_DP_GRD_FULL", "concept-id": "C3486646217-ASF"}, + { "collection": "SENTINEL-1C_DP_META_GRD_FULL", "concept-id": "C3486655250-ASF"}, + { "collection": "SENTINEL-1C_SP_GRD_HIGH", "concept-id": "C3488402208-ASF"}, + { "collection": "SENTINEL-1C_SP_META_GRD_HIGH", "concept-id": "C3488406939-ASF"}, + { "collection": "SENTINEL-1C_SP_GRD_MEDIUM", "concept-id": "C3488414315-ASF"}, + { "collection": "SENTINEL-1C_SP_META_GRD_MEDIUM", "concept-id": "C3488417229-ASF"}, + { "collection": "SENTINEL-1C_SP_GRD_FULL", "concept-id": "C3488389367-ASF"}, + { "collection": "SENTINEL-1C_SP_META_GRD_FULL", "concept-id": "C3488394269-ASF"}, + ], "SMAP": [ { "collection": "SMAP_L1A_RADAR_METADATA_V001", @@ -942,6 +961,8 @@ "concept-id": "C1216244348-ASF" } ], + "SENTINEL-1C": [ + ], "SMAP": [ { "collection": "SMAP_L1A_RADAR_METADATA_V001", @@ -1477,6 +1498,8 @@ "concept-id": "C1234413263-ASFDEV" } ], + "SENTINEL-1C": [ + ], "STS-59": [ { "collection": "STS-59_BROWSE_GRD", diff --git a/SearchAPI/CMR/Translate/datasets.py b/SearchAPI/CMR/Translate/datasets.py index 874811e0..de22de0f 100644 --- a/SearchAPI/CMR/Translate/datasets.py +++ b/SearchAPI/CMR/Translate/datasets.py @@ -110,6 +110,23 @@ "C1257175154-ASFDEV", "C1244598379-ASFDEV", "C1240784657-ASFDEV", + "C3470873558-ASF", + "C3486496642-ASF", + "C3484561061-ASF", + "C3484661603-ASF", + "C3484670556-ASF", + "C3486566209-ASF", + "C3486580531-ASF", + "C3486605959-ASF", + "C3486616327-ASF", + "C3486646217-ASF", + "C3486655250-ASF", + "C3488402208-ASF", + "C3488406939-ASF", + "C3488414315-ASF", + "C3488417229-ASF", + "C3488389367-ASF", + "C3488394269-ASF" ], "OPERA-S1": [ "C2777443834-ASF", diff --git a/SearchAPI/CMR/Translate/input_fixer.py b/SearchAPI/CMR/Translate/input_fixer.py index fb581bba..9828cd22 100644 --- a/SearchAPI/CMR/Translate/input_fixer.py +++ b/SearchAPI/CMR/Translate/input_fixer.py @@ -61,9 +61,9 @@ def input_fixer(params, is_prod: bool = False, provider: str = "ASF"): # conversion plat_aliases = { - 'S1': ['SENTINEL-1A', 'SENTINEL-1B'], - 'SENTINEL-1': ['SENTINEL-1A', 'SENTINEL-1B'], - 'SENTINEL': ['SENTINEL-1A', 'SENTINEL-1B'], + 'S1': ['SENTINEL-1A', 'SENTINEL-1B', 'SENTINEL-1C'], + 'SENTINEL-1': ['SENTINEL-1A', 'SENTINEL-1B', 'SENTINEL-1C'], + 'SENTINEL': ['SENTINEL-1A', 'SENTINEL-1B', 'SENTINEL-1C'], 'ERS': ['ERS-1', 'ERS-2'], 'SIR-C': ['STS-59', 'STS-68'] } @@ -80,6 +80,7 @@ def input_fixer(params, is_prod: bool = False, provider: str = "ASF"): 'SEASAT': 'SEASAT 1', 'SA': 'SENTINEL-1A', 'SB': 'SENTINEL-1B', + 'SC': 'SENTINEL-1C', 'SP': 'SMAP', 'UA': 'G-III', 'UAVSAR': 'G-III' @@ -105,12 +106,13 @@ def input_fixer(params, is_prod: bool = False, provider: str = "ASF"): for p in v: if p.upper() in plat_aliases: for x in plat_aliases[p.upper()]: - if x in ['SENTINEL-1A', 'SENTINEL-1B'] and any_processing_level: + if x in ['SENTINEL-1A', 'SENTINEL-1B', 'SENTINEL-1C'] and any_processing_level: collection_list.extend([id_by_platform['concept-id'] for id_by_platform in to_collections[x]]) platform_list.append(x) else: - if ((p.upper() in plat_names and p.upper() in ['SA', 'SB']) or p.upper() in ['SENTINEL-1A', 'SENTINEL-1B']) and any_processing_level: - if p.upper() in plat_names and p.upper() in ['SA', 'SB']: + if ((p.upper() in plat_names and p.upper() in ['SA', 'SB', 'SC']) or p.upper() in ['SENTINEL-1A', 'SENTINEL-1B', 'SENTINEL-1C']) and any_processing_level: + if p.upper() in plat_names and p.upper() in ['SA', 'SB', 'SC']: + to_collections collection_list.extend([id_by_platform['concept-id'] for id_by_platform in to_collections[plat_names[p.upper()]]]) else: collection_list.extend([id_by_platform['concept-id'] for id_by_platform in to_collections[p.upper()]]) diff --git a/SearchAPI/CMR/Translate/parse_cmr_response.py b/SearchAPI/CMR/Translate/parse_cmr_response.py index bffc6605..a3476a3c 100644 --- a/SearchAPI/CMR/Translate/parse_cmr_response.py +++ b/SearchAPI/CMR/Translate/parse_cmr_response.py @@ -71,7 +71,7 @@ def remove_field(f): if 'frameNumber' in req_fields: asf_frame_platforms = [ - 'Sentinel-1A', 'Sentinel-1B', 'ALOS', 'SENTINEL-1A', 'SENTINEL-1B', + 'Sentinel-1A', 'Sentinel-1B', 'Sentinel-1C', 'ALOS', 'SENTINEL-1A', 'SENTINEL-1B', 'SENTINEL-1C' 'ERS-1', 'ERS-2', 'JERS-1', 'RADARSAT-1', 'ALOS-2', 'NISAR' ] From efff6cbdbf1a0980c1ce1364aad9dcee7126f1f8 Mon Sep 17 00:00:00 2001 From: tcchase Date: Wed, 23 Apr 2025 10:11:42 -0400 Subject: [PATCH 2/4] chore: format collections --- .../CMR/Translate/collections_by_platform.py | 85 +++++++++++++++---- SearchAPI/CMR/Translate/datasets.py | 2 +- 2 files changed, 69 insertions(+), 18 deletions(-) diff --git a/SearchAPI/CMR/Translate/collections_by_platform.py b/SearchAPI/CMR/Translate/collections_by_platform.py index 6aae55ec..5b646647 100644 --- a/SearchAPI/CMR/Translate/collections_by_platform.py +++ b/SearchAPI/CMR/Translate/collections_by_platform.py @@ -334,23 +334,74 @@ } ], "SENTINEL-1C": [ - { "collection": "SENTINEL-1C_RAW", "concept-id": "C3486496642-ASF"}, - { "collection": "SENTINEL-1C_SLC", "concept-id": "C3470873558-ASF"}, - { "collection": "SENTINEL-1C_META_SLC", "concept-id": "C3484561061-ASF"}, - { "collection": "SENTINEL-1C_OCN", "concept-id": "C3484661603-ASF"}, - { "collection": "SENTINEL-1C_META_OCN", "concept-id": "C3484670556-ASF"}, - { "collection": "SENTINEL-1C_DP_GRD_HIGH", "concept-id": "C3486566209-ASF"}, - { "collection": "SENTINEL-1C_DP_META_GRD_HIGH", "concept-id": "C3486580531-ASF"}, - { "collection": "SENTINEL-1C_DP_GRD_MEDIUM", "concept-id": "C3486605959-ASF"}, - { "collection": "SENTINEL-1C_DP_META_GRD_MEDIUM", "concept-id": "C3486616327-ASF"}, - { "collection": "SENTINEL-1C_DP_GRD_FULL", "concept-id": "C3486646217-ASF"}, - { "collection": "SENTINEL-1C_DP_META_GRD_FULL", "concept-id": "C3486655250-ASF"}, - { "collection": "SENTINEL-1C_SP_GRD_HIGH", "concept-id": "C3488402208-ASF"}, - { "collection": "SENTINEL-1C_SP_META_GRD_HIGH", "concept-id": "C3488406939-ASF"}, - { "collection": "SENTINEL-1C_SP_GRD_MEDIUM", "concept-id": "C3488414315-ASF"}, - { "collection": "SENTINEL-1C_SP_META_GRD_MEDIUM", "concept-id": "C3488417229-ASF"}, - { "collection": "SENTINEL-1C_SP_GRD_FULL", "concept-id": "C3488389367-ASF"}, - { "collection": "SENTINEL-1C_SP_META_GRD_FULL", "concept-id": "C3488394269-ASF"}, + { + "collection": "SENTINEL-1C_RAW", + "concept-id": "C3486496642-ASF" + }, + { + "collection": "SENTINEL-1C_SLC", + "concept-id": "C3470873558-ASF" + }, + { + "collection": "SENTINEL-1C_META_SLC", + "concept-id": "C3484561061-ASF" + }, + { + "collection": "SENTINEL-1C_OCN", + "concept-id": "C3484661603-ASF" + }, + { + "collection": "SENTINEL-1C_META_OCN", + "concept-id": "C3484670556-ASF" + }, + { + "collection": "SENTINEL-1C_DP_GRD_HIGH", + "concept-id": "C3486566209-ASF" + }, + { + "collection": "SENTINEL-1C_DP_META_GRD_HIGH", + "concept-id": "C3486580531-ASF" + }, + { + "collection": "SENTINEL-1C_DP_GRD_MEDIUM", + "concept-id": "C3486605959-ASF" + }, + { + "collection": "SENTINEL-1C_DP_META_GRD_MEDIUM", + "concept-id": "C3486616327-ASF" + }, + { + "collection": "SENTINEL-1C_DP_GRD_FULL", + "concept-id": "C3486646217-ASF" + }, + { + "collection": "SENTINEL-1C_DP_META_GRD_FULL", + "concept-id": "C3486655250-ASF" + }, + { + "collection": "SENTINEL-1C_SP_GRD_HIGH", + "concept-id": "C3488402208-ASF" + }, + { + "collection": "SENTINEL-1C_SP_META_GRD_HIGH", + "concept-id": "C3488406939-ASF" + }, + { + "collection": "SENTINEL-1C_SP_GRD_MEDIUM", + "concept-id": "C3488414315-ASF" + }, + { + "collection": "SENTINEL-1C_SP_META_GRD_MEDIUM", + "concept-id": "C3488417229-ASF" + }, + { + "collection": "SENTINEL-1C_SP_GRD_FULL", + "concept-id": "C3488389367-ASF" + }, + { + "collection": "SENTINEL-1C_SP_META_GRD_FULL", + "concept-id": "C3488394269-ASF" + }, ], "SMAP": [ { diff --git a/SearchAPI/CMR/Translate/datasets.py b/SearchAPI/CMR/Translate/datasets.py index de22de0f..aeb1b975 100644 --- a/SearchAPI/CMR/Translate/datasets.py +++ b/SearchAPI/CMR/Translate/datasets.py @@ -126,7 +126,7 @@ "C3488414315-ASF", "C3488417229-ASF", "C3488389367-ASF", - "C3488394269-ASF" + "C3488394269-ASF", ], "OPERA-S1": [ "C2777443834-ASF", From 1ad54dd6367d30b702f73b4fa67415d2f817d68f Mon Sep 17 00:00:00 2001 From: tcchase Date: Wed, 23 Apr 2025 10:17:54 -0400 Subject: [PATCH 3/4] chore: codefactor fixes --- SearchAPI/CMR/SubQuery.py | 2 +- SearchAPI/CMR/Translate/input_fixer.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/SearchAPI/CMR/SubQuery.py b/SearchAPI/CMR/SubQuery.py index 10766cd1..9000c22e 100644 --- a/SearchAPI/CMR/SubQuery.py +++ b/SearchAPI/CMR/SubQuery.py @@ -60,7 +60,7 @@ def combine_params(self, params, extra_params): return final def should_use_asf_frame(self): - asf_frame_platforms = ['SENTINEL-1A', 'SENTINEL-1B', 'SENTINEL-1C' 'ALOS'] + asf_frame_platforms = ['SENTINEL-1A', 'SENTINEL-1B', 'SENTINEL-1C', 'ALOS'] asf_frame_datasets = ['SENTINEL-1', 'OPERA-S1', 'SLC-BURST', 'ALOS PALSAR', 'ALOS AVNIR-2', 'ALOS-2', 'NISAR'] asf_frame_collections = [] diff --git a/SearchAPI/CMR/Translate/input_fixer.py b/SearchAPI/CMR/Translate/input_fixer.py index 9828cd22..3dd72fc0 100644 --- a/SearchAPI/CMR/Translate/input_fixer.py +++ b/SearchAPI/CMR/Translate/input_fixer.py @@ -112,7 +112,6 @@ def input_fixer(params, is_prod: bool = False, provider: str = "ASF"): else: if ((p.upper() in plat_names and p.upper() in ['SA', 'SB', 'SC']) or p.upper() in ['SENTINEL-1A', 'SENTINEL-1B', 'SENTINEL-1C']) and any_processing_level: if p.upper() in plat_names and p.upper() in ['SA', 'SB', 'SC']: - to_collections collection_list.extend([id_by_platform['concept-id'] for id_by_platform in to_collections[plat_names[p.upper()]]]) else: collection_list.extend([id_by_platform['concept-id'] for id_by_platform in to_collections[p.upper()]]) From f8b22f9e30c9d1734a5e9c9dc2809a05d21eda08 Mon Sep 17 00:00:00 2001 From: tcchase Date: Wed, 23 Apr 2025 11:20:03 -0400 Subject: [PATCH 4/4] fix: platform values for testing --- yml_tests/test_url_manager.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/yml_tests/test_url_manager.py b/yml_tests/test_url_manager.py index 3f865e53..48e28723 100644 --- a/yml_tests/test_url_manager.py +++ b/yml_tests/test_url_manager.py @@ -621,6 +621,7 @@ def renameValsToStandard(self, json_dict): del json_dict["Platform"][i] json_dict["Platform"].append("Sentinel-1A") json_dict["Platform"].append("Sentinel-1B") + json_dict["Platform"].append("Sentinel-1C") json_dict["Platform"].append("Sentinel-1 Interferogram (BETA)") # Sentinel-1A elif platform in ["SENTINEL-1A", "SA"]: @@ -630,6 +631,9 @@ def renameValsToStandard(self, json_dict): elif platform in ["SENTINEL-1B", "SB"]: json_dict["Platform"][i] = "Sentinel-1B" json_dict["Platform"].append("Sentinel-1 Interferogram (BETA)") + elif platform in ["SENTINEL-1C", "SC"]: + json_dict["Platform"][i] = "Sentinel-1C" + json_dict["Platform"].append("Sentinel-1 Interferogram (BETA)") # Sir-C elif platform in ["SIR-C"]: del json_dict["Platform"][i]