From 0faddc796082930615086621a3d5a4fbbf6acca3 Mon Sep 17 00:00:00 2001 From: William Horn Date: Wed, 13 Aug 2025 16:56:20 -0800 Subject: [PATCH 1/7] Use function to update iso xml with asf lineage --- src/hyp3_opera_rtc/upload_rtc.py | 12 + tests/data/opera_v1.0.iso.xml | 1900 ++++++++++++++++++++++++++++++ tests/test_upload_rtc.py | 13 + 3 files changed, 1925 insertions(+) create mode 100644 tests/data/opera_v1.0.iso.xml diff --git a/src/hyp3_opera_rtc/upload_rtc.py b/src/hyp3_opera_rtc/upload_rtc.py index dca6e2b..d058164 100644 --- a/src/hyp3_opera_rtc/upload_rtc.py +++ b/src/hyp3_opera_rtc/upload_rtc.py @@ -1,6 +1,7 @@ import argparse from pathlib import Path from shutil import copyfile, make_archive +from xml.etree import ElementTree as et from hyp3lib.aws import upload_file_to_s3 @@ -42,6 +43,17 @@ def make_zip_name(product_files: list[Path]) -> str: return h5_file.name.split('.h5')[0] +def get_xml_with_asf_lineage(xml_text: str) -> str: + iso_tree = et.fromstring(xml_text) + gmd = '{http://www.isotc211.org/2005/gmd}' + gco = '{http://www.isotc211.org/2005/gco}' + + lineage_el = iso_tree.findall(f'.//{gmd}LI_Lineage/{gmd}statement/{gco}CharacterString')[0] + lineage_el.text = 'OPERA L2 RTC product generated by ASF using Sentinel-1 input data and SAS version 1.0.2 via HyP3 OPERA-RTC v0.1.4' + + return et.tostring(iso_tree, encoding='unicode') + + def main() -> None: """Upload results of OPERA RTC. diff --git a/tests/data/opera_v1.0.iso.xml b/tests/data/opera_v1.0.iso.xml new file mode 100644 index 0000000..b65b1cc --- /dev/null +++ b/tests/data/opera_v1.0.iso.xml @@ -0,0 +1,1900 @@ + + + + + OPERA_L2_RTC-S1_20231204T090220Z_S1A_30_v1.0 + + + + eng + + + + utf8 + + + + dataset + + + + + + Jet Propulsion Laboratory + + + + + + + 4800 Oak Grove Drive + + + Pasadena + + + CA + + + 91109 + + + USA + + + ops@jpl.nasa.gov + + + + + + + pointOfContact + + + + + + 2023-12-04T09:02:20.0710170000Z + + + + ISO 19115-2 Geographic Information - Metadata Part 2 Extensions for imagery and gridded data + + + ISO 19115-2:2019(E) + + + + + 2 + + + + + row + + + 3305 + + + 30.0 + + + + + + + column + + + 1506 + + + -30.0 + + + + + area + + + true + + + + + + + + + + + European Petroleum Survey Group (EPSG) Geodetic Parameter Dataset + + + + + 32626 + + + + + + + + + + + + + OPERA_L2_RTC-S1_20231204T090220Z_S1A_30_v1.0 + + + Radiometric Terrain Corrected (RTC) from Sentinel-1 + + + + + + 2023-12-04T09:02:20.0710170000Z + + + creation + + + + + 1.0 + + + + + + OPERA_L2_RTC-S1_20231204T090220Z_S1A_30_v1.0 + + + gov.nasa.esdis.umm.producergranuleid + + + ProducerGranuleId + + + + + + + + 2.1.0 + + + gov.nasa.esdis.umm.otherid + + + OtherId: PGEVersionId + + + + + + + + 1.0.1 + + + gov.nasa.esdis.umm.otherid + + + OtherId: SASVersionId + + + + + + + Jet Propulsion Repository + + + + + + originator + + + + + documentDigital + + + + + The OPERA Level 2 Radiometric Terrain Corrected (L2_RTC) product is derived from the original Copernicus Sentinel-1 (S1) A/B SLC data, with a temporal product sampling coincident with the availability of Sentinel-1 A/B SLC data. Each OPERA RTC product corresponds to a single Copernicus S1 A/B burst. + + + completed + + + + + + asNeeded + + + + + + + + OPERA + + + JPL + + + RTC + + + Radiometric + + + Terrain + + + Corrected + + + project + + + + + NASA Project Keywords + + + + + + NASA + + + User Support Office + + + + + + + https://support.earthdata.nasa.gov/ + + + Earthdata Support + + + File an issue or provide feedback + + + information + + + + + + + custodian + + + + + + + + + + + + S1 + + + platform + + + + + NASA Platform Keywords + + + + + + NASA + + + User Support Office + + + + + + + https://support.earthdata.nasa.gov/ + + + Earthdata Support + + + File an issue or provide feedback + + + information + + + + + + + custodian + + + + + + + + + + + + Sentinel 1 A/B + + instrument + + + + + NASA Instrument Keywords + + + + + + NASA + + + User Support Office + + + + + + + https://support.earthdata.nasa.gov/ + + + Earthdata Support + + + File an issue or provide feedback + + + information + + + + + + + custodian + + + + + + + + + + + + + + RTC_S1_PGE + + + gov.nasa.esdis.umm.collectionshortname + + + CollectionShortName + + + + + LargerWorkCitation + + + + + + + + + + 2.1.0 + + + gov.nasa.esdis.umm.collectionversion + + + CollectionVersion + + + + + LargerWorkCitation + + + + + + + + + + OPERA Project Homepage + + + + + + + + + + https://www.jpl.nasa.gov/go/opera + + + OPERA Project Homepage + + + information + + + + + + + custodian + + + + + + + LargerWorkCitation + + + + + grid + + + + eng + + + + utf8 + + + geoscientificInformation + + + Data product generated in HDF5 format with ISO 19115 conformant metadata. + + + + + + + + -30.94886170511729 + -29.866406758810424 + 35.9176028763907 + 36.355539817138066 + + + + + + + + + 2017-05-27T19:56:54.267407Z + 2017-05-27T19:56:57.336353Z + + + + + + + + + + + + + + MeasuredParameters + + + physicalMeasurement + + + + + + + + + + + instrumentInformation + + + AbsoluteOrbitNumber + + + Absolute orbit number + + + Unitless + + + int + + + + + 16772 + + + + + + + instrumentInformation + + + AcquisitionMode + + + Acquisition Mode + + + Unitless + + + string + + + + + IW + + + + + + + instrumentInformation + + + BoundingBox + + + Bounding box of the product, in order of xmin, ymin, xmax, ymax + + + string + + + + + [143640.0, 3982020.0, 242790.0, 4027200.0] + + + + + + + instrumentInformation + + + BurstID + + + Burst identification (burst ID) + + + Unitless + + + string + + + + + t075_159159_iw1 + + + + + + + processingInformation + + + CEOSAnalysisReadyDataDocumentIdentifier + + + CEOS Analysis Ready Data Document Identifier + + + boolean + + + + + https://ceos.org/ard/files/PFS/NRB/v5.5/CARD4L-PFS_NRB_v5.5.pdf + + + + + + + processingInformation + + + CEOSAnalysisReadyDataProductType + + + CEOS Analysis Ready Data Product Type + + + string + + + + + Normalised Radar Backscatter + + + + + + + processingInformation + + + ContactInformation + + + Contact email address + + + string + + + + + operasds@jpl.nasa.gov + + + + + + + processingInformation + + + DataAccess + + + Data Access URL + + + string + + + + + https://search.asf.alaska.edu/#/?dataset=OPERA-S1&productTypes=RTC + + + + + + + instrumentInformation + + + TrackNumber + + + Track number + + + Unitless + + + int + + + + + 75 + + + + + + + contentInformation + + + ProductType + + + Product type + + + string + + + + + RTC-S1-STATIC + + + + + + + contentInformation + + + Project + + + Project name + + + string + + + + + OPERA + + + + + + + instrumentInformation + + + RadarBand + + + Acquired frequency band + + + string + + + + + C + + + + + + + processingInformation + + + SubSwathID + + + Sub Swath ID + + + string + + + + + IW1 + + + + + + + contentInformation + + + ProductVersion + + + Product version + + + string + + + + + 1.0 + + + + + + + instrumentInformation + + + LookDirection + + + Look direction can be left or right + + + string + + + + + right + + + + + + + instrumentInformation + + + OrbitPassDirection + + + Orbit direction can be ascending or descending + + + string + + + + + ascending + + + + + + + instrumentInformation + + + Platform + + + Platform name + + + string + + + + + Sentinel-1A + + + + + + + instrumentInformation + + + ZeroDopplerStartTime + + + Azimuth start time of the product + + + dateTimeString + + + + + 2017-05-27T19:56:54.267407Z + + + + + + + instrumentInformation + + + ZeroDopplerEndTime + + + Azimuth stop time of the product + + + dateTimeString + + + + + 2017-05-27T19:56:57.336353Z + + + + + + + contentInformation + + + Institution + + + Institution name + + + string + + + + + NASA JPL + + + + + + + instrumentInformation + + + InstrumentName + + + Name of the instrument used to collect the remote sensing data provided in this product + + + string + + + + + Sentinel-1A CSAR + + + + + + + contentInformation + + + IsGeocoded + + + Flag to indicate radar geometry or geocoded product + + + boolean + + + + + True + + + + + + + processingInformation + + + ProcessingDatetime + + + Processing date and time + + + string + + + + + 2023-12-04T09:13:35.170570Z + + + + + + + processingInformation + + + ProcessingType + + + NOMINAL (or) URGENT (or) CUSTOM (or) UNDEFINED + + + string + + + + + NOMINAL + + + + + + + processingInformation + + + ProductLevel + + + Product level. L0A: Unprocessed instrument data; L0B: Reformatted, unprocessed instrument data; L1: Processed instrument data in radar coordinates system; and L2: Processed instrument data in geocoded coordinates system + + + string + + + + + L2 + + + + + + + processingInformation + + + ProductSpecificationVersion + + + Product specification version which represents the schema of this product + + + string + + + + + 1.0 + + + + + + + contentInformation + + + ListOfPolarizations + + + List of processed polarization layers + + + string + + + + + ["VV"] + + + + + + + contentInformation + + + YCoordinateSpacing + + + Nominal spacing in meters between consecutive lines + + + Meters + + + float + + + + + -30.0 + + + + + + + contentInformation + + + XCoordinateSpacing + + + Nominal spacing in meters between consecutive pixels + + + Meters + + + float + + + + + 30.0 + + + + + + + processingParameter + + + Projection + + + EPSG Projection Value + + + int + + + + + 32626 + + + + + + + processingParameter + + + BistaticDelayCorrectedApplied + + + Flag to indicate if Bi-static Delay correction was applied + + + boolean + + + + + False + + + + + + + processingParameter + + + StaticTroposphericGeolocationCorrectedApplied + + + Flag to indicate if the Static Tropospheric Geolocation correction was applied + + + boolean + + + + + False + + + + + + + processingParameter + + + FilteringApplied + + + Flag to indicate if filtering was applied + + + boolean + + + + + False + + + + + + + processingParameter + + + BurstGeogridSnapX + + + Burst Geogrid Snap X value + + + int + + + + + 30.0 + + + + + + + processingParameter + + + BurstGeogridSnapY + + + Burst Geogrid Snap Y value + + + int + + + + + 30.0 + + + + + + + processingParameter + + + InputBackscatterNormalizationConvention + + + Normalization convention for input backscatter + + + string + + + + + beta0 + + + + + + + processingParameter + + + OutputBackscatterDecibelConversionEquation + + + Output Backscatter Decibel Conversion Equation + + + string + + + + + 10*log10(backscatter_linear) + + + + + + + processingParameter + + + OutputBackscatterExpressionConvention + + + Output Backscatter Expression Convention + + + string + + + + + linear backscatter intensity + + + + + + + processingParameter + + + OutputBackscatterNormalizationConvention + + + Output Backscatter Normalization Convention + + + string + + + + + gamma0 + + + + + + + processingParameter + + + PreprocessingMultilookingApplied + + + Flag to indicate if Pre Processing Multilooking was applied + + + boolean + + + + + False + + + + + + + processingParameter + + + WetTroposphericGeolocationCorrectionApplied + + + Flag to indicate if the Wet Tropospheric Geolocation Correction was applied + + + boolean + + + + + False + + + + + + + processingInformation + + + DemEgmModel + + + DEM Earth Gravitational Model + + + string + + + + + Earth Gravitational Model 2008 (EGM2008) + + + + + + + processingInformation + + + DemInterpolation + + + DEM interpolation method + + + string + + + + + biquintic + + + + + + + processingInformation + + + Geocoding + + + Geocoding algorithm + + + string + + + + + Area-Based SAR Geocoding with Adaptive Multilooking (GEO-AP) + + + + + + + processingInformation + + + GeocodingAlgorithmReference + + + Geocoding algorithm reference document + + + string + + + + + Gustavo H. X. Shiroma, Marco Lavalle, and Sean M. Buckley, "An Area-Based Projection Algorithm for SAR Radiometric Terrain Correction and Geocoding," in IEEE Transactions on Geoscience and Remote Sensing, vol. 60, pp. 1-23, 2022, Art no. 5222723, doi: 10.1109/TGRS.2022.3147472. + + + + + + + processingInformation + + + RadiometricTerrainCorrection + + + Radiometric terrain correction (RTC) algorithm + + + string + + + + + Area-Based SAR Radiometric Terrain Correction (RTC-AP) + + + + + + + processingInformation + + + RadiometricTerrainCorrectionAlgorithmReference + + + Radiometric terrain correction algorithm reference document + + + string + + + + + Gustavo H. X. Shiroma, Marco Lavalle, and Sean M. Buckley, "An Area-Based Projection Algorithm for SAR Radiometric Terrain Correction and Geocoding," in IEEE Transactions on Geoscience and Remote Sensing, vol. 60, pp. 1-23, 2022, Art no. 5222723, doi: 10.1109/TGRS.2022.3147472. + + + + + + + processingInformation + + + S1ReaderVersion + + + S1-Reader version used for processing + + + string + + + + + 0.2.2 + + + + + + + processingInformation + + + ISCEVersion + + + ISCE version used for processing + + + string + + + + + 0.15.0 + + + + + + + processingInformation + + + SoftwareVersion + + + Software version used for processing + + + string + + + + + 1.0.1 + + + + + + + processingInformation + + + L1SlcGranules + + + List of input L1 SLC products used + + + string + + + + + ["S1A_IW_SLC__1SDV_20170527T195629_20170527T195657_016772_01BDE2_30A5.zip"] + + + + + + + processingInformation + + + orbitFiles + + + List of input orbit files used + + + string + + + + + ["S1A_OPER_AUX_POEORB_OPOD_20210302T202525_V20170526T225942_20170528T005942.EOF"] + + + + + + + processingInformation + + + AnnotationFiles + + + List of input annotation files used + + + string + + + + + ["calibration-s1a-iw1-slc-vv-20170527t195629-20170527t195657-016772-01bde2-004.xml", "noise-s1a-iw1-slc-vv-20170527t195629-20170527t195657-016772-01bde2-004.xml"] + + + + + + + processingInformation + + + DemSource + + + Source description for the utilized DEM file + + + string + + + + + Digital Elevation Model (DEM) for the NASA OPERA project version 1.1 (v1.1) based on the Copernicus DEM 30-m and Copernicus 90-m referenced to the WGS84 ellipsoid + + + + + + + instrumentInformation + + + OrbitInterpolationMethod + + + Orbit Interpolation Method + + + string + + + + + Hermite + + + + + + + instrumentInformation + + + OrbitReferenceEpoch + + + Reference epoch in the format YYYY-MM-DDTHH:MM:SS.SSS + + + string + + + + + 2017-05-25T19:56:54.267407000 + + + + + + + instrumentInformation + + + OrbitType + + + PrOE (or) NOE (or) MOE (or) POE (or) Custom + + + string + + + + + POE precise orbit + + + + + + + + + + + + + + + + + + GeoTIFF + + + + + + + + + free + + + + + + + + + + + + + + + dataset + + + + + + + OPERA L2 RTC product generated by JPL using Sentinel-1 input data and SAS version 1.0.1 + + + + + + PGEVersionClass + + + + + + + PGEName: RTC_S1_PGE PGEVersion: 2.1.0 + + + gov.nasa.esdis.umm.pgeversionclass + + + PGEVersionClass + + + + + + + + + Radiometric Terrain Corrected (RTC) Product - OPERA_L2_RTC-S1_20231204T090220Z_S1A_30_v1.0 + + + + + + + + + + ProductionDateTime + + + 2023-12-04T09:02:20.0710170000Z + + + + + + + + GranuleInput + + + + + + /home/rtc_user/input_dir/S1A_IW_SLC__1SDV_20170527T195629_20170527T195657_016772_01BDE2_30A5.zip + + + + + + + + + + + Reference DEM - /home/rtc_user/input_dir/dem.vrt + + + + + + + Orbit Ephemerides - /home/rtc_user/input_dir/S1A_OPER_AUX_POEORB_OPOD_20210302T202525_V20170526T225942_20170528T005942.EOF + + + + + + + Burst Database - /home/rtc_user/input_dir/opera_burst_database.sqlite3 + + + + + + + + + + + asNeeded + + + + \ No newline at end of file diff --git a/tests/test_upload_rtc.py b/tests/test_upload_rtc.py index ba303cc..a790bee 100644 --- a/tests/test_upload_rtc.py +++ b/tests/test_upload_rtc.py @@ -64,6 +64,19 @@ def test_make_zip_name(rtc_burst_output_files): assert zip_filename == 'OPERA_L2_RTC-S1_T115-245714-IW1_20240809T141633Z_20250411T185446Z_S1A_30_v1.0' +def test_get_xml_with_asf_lineage(iso_xml): + updated_iso_xml = upload_rtc.get_xml_with_asf_lineage(iso_xml.read()) + + assert 'OPERA L2 RTC product generated by ASF using Sentinel-1 input data and SAS version 1.0.2 via HyP3 OPERA-RTC v0.1.4' in updated_iso_xml + print(updated_iso_xml) + + +@pytest.fixture +def iso_xml(): + with (Path(__file__).parent / 'data' / 'opera_v1.0.iso.xml').open() as f: + yield f + + @pytest.fixture def rtc_burst_results_dir(tmp_path, rtc_burst_output_files): (tmp_path / 'burst-dir').mkdir(parents=True) From f3563907f4ff7470bea237892f6936d6a90e78dd Mon Sep 17 00:00:00 2001 From: William Horn Date: Fri, 15 Aug 2025 09:01:56 -0800 Subject: [PATCH 2/7] Add updated error checking and better testing --- src/hyp3_opera_rtc/upload_rtc.py | 32 +++++++++++++++++---- tests/test_upload_rtc.py | 48 ++++++++++++++++++++++++++------ 2 files changed, 67 insertions(+), 13 deletions(-) diff --git a/src/hyp3_opera_rtc/upload_rtc.py b/src/hyp3_opera_rtc/upload_rtc.py index ae9118c..85d8db0 100644 --- a/src/hyp3_opera_rtc/upload_rtc.py +++ b/src/hyp3_opera_rtc/upload_rtc.py @@ -6,6 +6,10 @@ from hyp3lib.aws import upload_file_to_s3 +class FailedToFindLineageStatementError(Exception): + pass + + def upload_rtc(bucket: str, bucket_prefix: str, output_dir: Path) -> None: output_files = [f for f in output_dir.iterdir() if not f.is_dir()] @@ -43,15 +47,28 @@ def make_zip_name(product_files: list[Path]) -> str: return h5_file.name.split('.h5')[0] -def get_xml_with_asf_lineage(xml_text: str) -> str: - iso_tree = et.fromstring(xml_text) +def update_xmls_with_asf_lineage(output_dir: Path) -> None: + xml_paths = [f for f in output_dir.iterdir() if f.suffix == '.xml'] + + for xml_path in xml_paths: + update_xml_with_asf_lineage(xml_path) + + +def update_xml_with_asf_lineage(xml_path: Path) -> None: + iso_tree = et.parse(str(xml_path)) + gmd = '{http://www.isotc211.org/2005/gmd}' gco = '{http://www.isotc211.org/2005/gco}' + lineage_tag_path = f'.//{gmd}LI_Lineage/{gmd}statement/{gco}CharacterString' - lineage_el = iso_tree.findall(f'.//{gmd}LI_Lineage/{gmd}statement/{gco}CharacterString')[0] - lineage_el.text = 'OPERA L2 RTC product generated by ASF using Sentinel-1 input data and SAS version 1.0.2 via HyP3 OPERA-RTC v0.1.4' + lineage_seach = iso_tree.findall(lineage_tag_path) + if len(lineage_seach) == 0: + raise FailedToFindLineageStatementError() - return et.tostring(iso_tree, encoding='unicode') + lineage = lineage_seach[0] + lineage.text = f'{lineage.text.replace("JPL", "ASF")} via HyP3 OPERA-RTC v0.1.4' + + iso_tree.write(str(xml_path)) def main() -> None: @@ -69,6 +86,11 @@ def main() -> None: args, _ = parser.parse_known_args() + try: + update_xmls_with_asf_lineage(args.output_dir) + except FailedToFindLineageStatementError: + print('failed to update lineage statement in .iso.xml') + if not args.bucket: print('No bucket provided, skipping upload') else: diff --git a/tests/test_upload_rtc.py b/tests/test_upload_rtc.py index a790bee..a670eab 100644 --- a/tests/test_upload_rtc.py +++ b/tests/test_upload_rtc.py @@ -1,7 +1,9 @@ import json +import shutil from collections import Counter from pathlib import Path from zipfile import ZipFile +from unittest.mock import patch, call import pytest from hyp3lib import aws @@ -64,17 +66,47 @@ def test_make_zip_name(rtc_burst_output_files): assert zip_filename == 'OPERA_L2_RTC-S1_T115-245714-IW1_20240809T141633Z_20250411T185446Z_S1A_30_v1.0' -def test_get_xml_with_asf_lineage(iso_xml): - updated_iso_xml = upload_rtc.get_xml_with_asf_lineage(iso_xml.read()) +@patch('hyp3_opera_rtc.upload_rtc.update_xml_with_asf_lineage') +def test_update_xmls_with_slc_results(update_mock, rtc_slc_results_dir): + upload_rtc.update_xmls(rtc_slc_results_dir) - assert 'OPERA L2 RTC product generated by ASF using Sentinel-1 input data and SAS version 1.0.2 via HyP3 OPERA-RTC v0.1.4' in updated_iso_xml - print(updated_iso_xml) + assert len(update_mock.call_args_list) == 27 + assert all(call[0][0].suffix == '.xml' for call in update_mock.call_args_list) + + +@patch('hyp3_opera_rtc.upload_rtc.update_xml_with_asf_lineage') +def test_update_xmls(update_mock, tmp_path): + upload_rtc.update_xmls(tmp_path) + update_mock.assert_not_called() + + for file in ['f.txt', 'f.xml', 'f.json', 'f2.xml']: + (tmp_path / file).touch() + + upload_rtc.update_xmls(tmp_path) + calls = [call(tmp_path / 'f.xml'), call(tmp_path / 'f2.xml')] + update_mock.assert_has_calls(calls, any_order=True) + + +def test_get_xml_with_asf_lineage(iso_xml_path): + with iso_xml_path.open() as f: + xml_text = f.read() + assert 'ASF' not in xml_text + assert 'via HyP3 OPERA-RTC v0.1.4' not in xml_text + + upload_rtc.update_xml_with_asf_lineage(iso_xml_path) + with iso_xml_path.open() as f: + xml_text = f.read() + + assert 'ASF' in xml_text + assert 'via HyP3 OPERA-RTC v0.1.4' in xml_text @pytest.fixture -def iso_xml(): - with (Path(__file__).parent / 'data' / 'opera_v1.0.iso.xml').open() as f: - yield f +def iso_xml_path(tmp_path): + xml_output_path = tmp_path / 'opera_v1.0.iso.xml' + shutil.copy(Path(__file__).parent / 'data' / 'opera_v1.0.iso.xml', xml_output_path) + + return xml_output_path @pytest.fixture @@ -89,7 +121,7 @@ def rtc_burst_results_dir(tmp_path, rtc_burst_output_files): @pytest.fixture def rtc_slc_results_dir(tmp_path, rtc_slc_output_files): - (tmp_path / 'burst-dir').mkdir(parents=True) + (tmp_path / 'slc-dir').mkdir(parents=True) for file in rtc_slc_output_files: (tmp_path / file).touch() From 8798a0ff8a56451006d53f9e714da2b48bd8cb40 Mon Sep 17 00:00:00 2001 From: William Horn Date: Fri, 15 Aug 2025 09:34:43 -0800 Subject: [PATCH 3/7] Use hyp3_opera_rtc.__version__ for version in statement --- src/hyp3_opera_rtc/upload_rtc.py | 10 ++++------ tests/test_upload_rtc.py | 13 ++++++++----- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/hyp3_opera_rtc/upload_rtc.py b/src/hyp3_opera_rtc/upload_rtc.py index 85d8db0..4acf8d5 100644 --- a/src/hyp3_opera_rtc/upload_rtc.py +++ b/src/hyp3_opera_rtc/upload_rtc.py @@ -4,6 +4,7 @@ from xml.etree import ElementTree as et from hyp3lib.aws import upload_file_to_s3 +import hyp3_opera_rtc class FailedToFindLineageStatementError(Exception): @@ -66,7 +67,8 @@ def update_xml_with_asf_lineage(xml_path: Path) -> None: raise FailedToFindLineageStatementError() lineage = lineage_seach[0] - lineage.text = f'{lineage.text.replace("JPL", "ASF")} via HyP3 OPERA-RTC v0.1.4' + version = hyp3_opera_rtc.__version__ + lineage.text = f'{lineage.text.replace("JPL", "ASF")} via HyP3 OPERA-RTC v{version}' iso_tree.write(str(xml_path)) @@ -85,11 +87,7 @@ def main() -> None: parser.add_argument('--bucket-prefix', default='', help='Add a bucket prefix to products') args, _ = parser.parse_known_args() - - try: - update_xmls_with_asf_lineage(args.output_dir) - except FailedToFindLineageStatementError: - print('failed to update lineage statement in .iso.xml') + update_xmls_with_asf_lineage(args.output_dir) if not args.bucket: print('No bucket provided, skipping upload') diff --git a/tests/test_upload_rtc.py b/tests/test_upload_rtc.py index a670eab..574b39c 100644 --- a/tests/test_upload_rtc.py +++ b/tests/test_upload_rtc.py @@ -10,6 +10,7 @@ from moto import mock_aws from moto.core import patch_client +import hyp3_opera_rtc from hyp3_opera_rtc import upload_rtc @@ -68,7 +69,7 @@ def test_make_zip_name(rtc_burst_output_files): @patch('hyp3_opera_rtc.upload_rtc.update_xml_with_asf_lineage') def test_update_xmls_with_slc_results(update_mock, rtc_slc_results_dir): - upload_rtc.update_xmls(rtc_slc_results_dir) + upload_rtc.update_xmls_with_asf_lineage(rtc_slc_results_dir) assert len(update_mock.call_args_list) == 27 assert all(call[0][0].suffix == '.xml' for call in update_mock.call_args_list) @@ -76,29 +77,31 @@ def test_update_xmls_with_slc_results(update_mock, rtc_slc_results_dir): @patch('hyp3_opera_rtc.upload_rtc.update_xml_with_asf_lineage') def test_update_xmls(update_mock, tmp_path): - upload_rtc.update_xmls(tmp_path) + upload_rtc.update_xmls_with_asf_lineage(tmp_path) update_mock.assert_not_called() for file in ['f.txt', 'f.xml', 'f.json', 'f2.xml']: (tmp_path / file).touch() - upload_rtc.update_xmls(tmp_path) + upload_rtc.update_xmls_with_asf_lineage(tmp_path) calls = [call(tmp_path / 'f.xml'), call(tmp_path / 'f2.xml')] update_mock.assert_has_calls(calls, any_order=True) def test_get_xml_with_asf_lineage(iso_xml_path): + version = hyp3_opera_rtc.__version__ + with iso_xml_path.open() as f: xml_text = f.read() assert 'ASF' not in xml_text - assert 'via HyP3 OPERA-RTC v0.1.4' not in xml_text + assert f'via HyP3 OPERA-RTC {version}' not in xml_text upload_rtc.update_xml_with_asf_lineage(iso_xml_path) with iso_xml_path.open() as f: xml_text = f.read() assert 'ASF' in xml_text - assert 'via HyP3 OPERA-RTC v0.1.4' in xml_text + assert f'via HyP3 OPERA-RTC v{version}' in xml_text @pytest.fixture From 63c2939c577bb3f07a4d414100b9249eadff6352 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Fri, 15 Aug 2025 09:47:38 -0800 Subject: [PATCH 4/7] ruff --- src/hyp3_opera_rtc/upload_rtc.py | 1 + tests/test_upload_rtc.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/hyp3_opera_rtc/upload_rtc.py b/src/hyp3_opera_rtc/upload_rtc.py index 4acf8d5..4bcf836 100644 --- a/src/hyp3_opera_rtc/upload_rtc.py +++ b/src/hyp3_opera_rtc/upload_rtc.py @@ -4,6 +4,7 @@ from xml.etree import ElementTree as et from hyp3lib.aws import upload_file_to_s3 + import hyp3_opera_rtc diff --git a/tests/test_upload_rtc.py b/tests/test_upload_rtc.py index 574b39c..ee125dd 100644 --- a/tests/test_upload_rtc.py +++ b/tests/test_upload_rtc.py @@ -2,8 +2,8 @@ import shutil from collections import Counter from pathlib import Path +from unittest.mock import call, patch from zipfile import ZipFile -from unittest.mock import patch, call import pytest from hyp3lib import aws From fc637d1c8f91642d40d6cca260af59f72b3d3e1b Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Fri, 15 Aug 2025 09:50:11 -0800 Subject: [PATCH 5/7] fix typo and mypy error --- src/hyp3_opera_rtc/upload_rtc.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/hyp3_opera_rtc/upload_rtc.py b/src/hyp3_opera_rtc/upload_rtc.py index 4bcf836..4887591 100644 --- a/src/hyp3_opera_rtc/upload_rtc.py +++ b/src/hyp3_opera_rtc/upload_rtc.py @@ -63,12 +63,13 @@ def update_xml_with_asf_lineage(xml_path: Path) -> None: gco = '{http://www.isotc211.org/2005/gco}' lineage_tag_path = f'.//{gmd}LI_Lineage/{gmd}statement/{gco}CharacterString' - lineage_seach = iso_tree.findall(lineage_tag_path) - if len(lineage_seach) == 0: + lineage_search = iso_tree.findall(lineage_tag_path) + if len(lineage_search) == 0: raise FailedToFindLineageStatementError() - lineage = lineage_seach[0] + lineage = lineage_search[0] version = hyp3_opera_rtc.__version__ + assert lineage.text is not None lineage.text = f'{lineage.text.replace("JPL", "ASF")} via HyP3 OPERA-RTC v{version}' iso_tree.write(str(xml_path)) From d90c3ba8d81abaf33c893da31a8219e1c0326692 Mon Sep 17 00:00:00 2001 From: William Horn Date: Fri, 15 Aug 2025 09:57:10 -0800 Subject: [PATCH 6/7] Changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6eaf082..e32421e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.1.6] + +### Added +- Update lineage statement in ISO XML files output to include mention of ASF and hyp3-OPERA-RTC version + ## [0.1.5] ### Changed From 568f2db230cee3f22f1e094d980d95cbfdbc6147 Mon Sep 17 00:00:00 2001 From: William Horn Date: Fri, 15 Aug 2025 10:08:49 -0800 Subject: [PATCH 7/7] Add test for FailedToFindLineageStatementError --- src/hyp3_opera_rtc/upload_rtc.py | 2 +- tests/test_upload_rtc.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/hyp3_opera_rtc/upload_rtc.py b/src/hyp3_opera_rtc/upload_rtc.py index 4887591..fc23ee5 100644 --- a/src/hyp3_opera_rtc/upload_rtc.py +++ b/src/hyp3_opera_rtc/upload_rtc.py @@ -65,7 +65,7 @@ def update_xml_with_asf_lineage(xml_path: Path) -> None: lineage_search = iso_tree.findall(lineage_tag_path) if len(lineage_search) == 0: - raise FailedToFindLineageStatementError() + raise FailedToFindLineageStatementError('Failed to find lineage statement in iso xml') lineage = lineage_search[0] version = hyp3_opera_rtc.__version__ diff --git a/tests/test_upload_rtc.py b/tests/test_upload_rtc.py index ee125dd..8cddd23 100644 --- a/tests/test_upload_rtc.py +++ b/tests/test_upload_rtc.py @@ -104,6 +104,15 @@ def test_get_xml_with_asf_lineage(iso_xml_path): assert f'via HyP3 OPERA-RTC v{version}' in xml_text +def test_cant_find_lineage_in_xml(tmp_path): + xml_path = tmp_path / 'f.xml' + with xml_path.open(mode='w') as f: + f.write('') + + with pytest.raises(upload_rtc.FailedToFindLineageStatementError): + upload_rtc.update_xml_with_asf_lineage(xml_path) + + @pytest.fixture def iso_xml_path(tmp_path): xml_output_path = tmp_path / 'opera_v1.0.iso.xml'