diff --git a/.github/workflows/create-jira-issue.yml b/.github/workflows/create-jira-issue.yml new file mode 100644 index 0000000..5765047 --- /dev/null +++ b/.github/workflows/create-jira-issue.yml @@ -0,0 +1,15 @@ +name: Create Jira issue + +on: + issues: + types: [labeled] + +jobs: + call-create-jira-issue-workflow: + uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.18.1 + secrets: + JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} + JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} + JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }} + JIRA_PROJECT: ${{ secrets.JIRA_PROJECT }} + JIRA_FIELDS: ${{ secrets.JIRA_FIELDS }} diff --git a/CHANGELOG.md b/CHANGELOG.md index b035187..af7b2e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.1.1] + +### Changed +- All files in zip now contained in folder named after the product +- Remove `log` and `catalog.json` from the zip +- Switch to a full-SLC based data prep strategy + ## [0.1.0] ### Added diff --git a/environment.yml b/environment.yml index 2ed7937..0046512 100644 --- a/environment.yml +++ b/environment.yml @@ -12,7 +12,6 @@ dependencies: - hyp3lib - boto3 - jinja2 - - burst2safe # For packaging, and testing - setuptools - setuptools_scm diff --git a/pyproject.toml b/pyproject.toml index 85a2fcc..a27bc1a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,6 @@ dependencies = [ "urllib3", "lxml", "jinja2", - "burst2safe" ] dynamic = ["version", "readme"] diff --git a/src/hyp3_opera_rtc/prep_rtc.py b/src/hyp3_opera_rtc/prep_rtc.py index a3147f1..fc6fb28 100644 --- a/src/hyp3_opera_rtc/prep_rtc.py +++ b/src/hyp3_opera_rtc/prep_rtc.py @@ -2,13 +2,13 @@ import os import warnings from pathlib import Path -from shutil import make_archive from zipfile import ZipFile import hyp3lib.fetch import lxml.etree as ET import requests -from burst2safe.burst2safe import burst2safe +from hyp3lib.fetch import download_file +from hyp3lib.scene import get_download_url from jinja2 import Template from shapely.geometry import Polygon, box @@ -49,11 +49,36 @@ def get_s1_granule_bbox(granule_path: Path, buffer: float = 0.025) -> Polygon: return box(*footprint.bounds) -def granule_exists(granule: str) -> bool: +def get_granule_cmr(granule: str) -> dict: params = (('short_name', 'SENTINEL-1_BURSTS'), ('granule_ur', granule)) response = requests.get(CMR_URL, params=params) response.raise_for_status() - return bool(response.json()['items']) + return response.json() + + +def granule_exists(granule: str) -> bool: + response = get_granule_cmr(granule) + return bool(response['items']) + + +def parse_response_for_slc_params(response: dict) -> tuple[str, str]: + assert len(response['items']) == 1 + item = response['items'][0] + + source_slc = item['umm']['InputGranules'][0][:-4] + assert isinstance(source_slc, str) + + opera_burst_ids = [attr for attr in item['umm']['AdditionalAttributes'] if attr['Name'] == 'BURST_ID_FULL'] + assert len(opera_burst_ids) == 1 + opera_burst_id = opera_burst_ids[0]['Values'][0] + assert isinstance(opera_burst_id, str) + + return source_slc, f't{opera_burst_id.lower()}' + + +def get_granule_slc_params(granule: str) -> tuple[str, str]: + response = get_granule_cmr(granule) + return parse_response_for_slc_params(response) def validate_co_pol_granule(granule: str) -> None: @@ -101,18 +126,11 @@ def prep_rtc( validate_co_pol_granule(co_pol_granule) - cross_pol_granule = get_cross_pol_name(co_pol_granule) - dual_pol = granule_exists(cross_pol_granule) - if dual_pol: - print(f'Found cross-pol granule: {cross_pol_granule}') - granules = [co_pol_granule, cross_pol_granule] - else: - print('No cross-pol granule found') - granules = [co_pol_granule] - - safe_path = burst2safe(granules=granules, all_anns=True, work_dir=input_dir) - granule_path = Path(make_archive(base_name=str(safe_path.with_suffix('')), format='zip', base_dir=str(safe_path))) - print(f'Created archive: {granule_path}') + source_slc, opera_burst_id = get_granule_slc_params(co_pol_granule) + safe_path = download_file(get_download_url(source_slc), directory=str(input_dir), chunk_size=10485760) + safe_path = Path(safe_path) + dual_pol = safe_path.name[14] == 'D' + print(f'Created archive: {safe_path}') orbit_path = orbit.get_orbit(safe_path.with_suffix('').name, save_dir=input_dir) print(f'Downloaded orbit file: {orbit_path}') @@ -121,15 +139,16 @@ def prep_rtc( print(f'Downloaded burst database: {db_path}') dem_path = input_dir / 'dem.tif' - granule_bbox = get_s1_granule_bbox(granule_path) + granule_bbox = get_s1_granule_bbox(safe_path) dem.download_opera_dem_for_footprint(dem_path, granule_bbox) print(f'Downloaded DEM: {dem_path}') runconfig_dict = { - 'granule_path': str(granule_path), + 'granule_path': str(safe_path), 'orbit_path': str(orbit_path), 'db_path': str(db_path), 'dem_path': str(dem_path), + 'opera_burst_id': opera_burst_id, 'scratch_dir': str(scratch_dir), 'output_dir': str(output_dir), 'dual_pol': dual_pol, diff --git a/src/hyp3_opera_rtc/templates/pge.yml.j2 b/src/hyp3_opera_rtc/templates/pge.yml.j2 index 171d153..13bc048 100644 --- a/src/hyp3_opera_rtc/templates/pge.yml.j2 +++ b/src/hyp3_opera_rtc/templates/pge.yml.j2 @@ -135,6 +135,9 @@ RunConfig: # Required. List of SAFE files (min=1) safe_file_path: - {{ granule_path }} + # Optional. Burst ID to process (empty for all bursts) + burst_id: + - {{ opera_burst_id }} # Required. List of orbit (EOF) files (min=1) orbit_file_path: - {{ orbit_path }} diff --git a/src/hyp3_opera_rtc/upload_rtc.py b/src/hyp3_opera_rtc/upload_rtc.py index 4cc4eb1..b5cfb5e 100644 --- a/src/hyp3_opera_rtc/upload_rtc.py +++ b/src/hyp3_opera_rtc/upload_rtc.py @@ -15,20 +15,26 @@ def upload_rtc(bucket: str, bucket_prefix: str, output_dir: Path) -> None: def make_zip(output_files: list[Path], output_dir: Path) -> Path: + zip_name = make_zip_name(output_files) + zip_path = output_dir / zip_name + zip_archive_path = output_dir / 'zip' - zip_archive_path.mkdir(exist_ok=True) + (zip_archive_path / zip_name).mkdir(exist_ok=True, parents=True) + file_extensions_to_include = set(['.png', '.xml', '.tif', '.h5']) for output_file in output_files: - copyfile(output_file, zip_archive_path / output_file.name) + if output_file.suffix not in file_extensions_to_include: + continue - zip_path = output_dir / make_zip_name(output_files) - output_zip = make_archive(base_name=str(zip_path), format='zip', root_dir=zip_archive_path) + zip_dest_path = zip_archive_path / zip_name / output_file.name + copyfile(output_file, zip_dest_path) + output_zip = make_archive(base_name=str(zip_path), format='zip', root_dir=zip_archive_path) return Path(output_zip) def make_zip_name(product_files: list[Path]) -> str: - h5_file = [f for f in product_files if f.name.endswith('h5')].pop() + h5_file = next(f for f in product_files if f.name.endswith('h5')) return h5_file.name.split('.h5')[0] diff --git a/tests/data/burst_response.json b/tests/data/burst_response.json new file mode 100644 index 0000000..f9adb8c --- /dev/null +++ b/tests/data/burst_response.json @@ -0,0 +1,305 @@ +{ + "hits": 1, + "took": 28, + "items": [ + { + "meta": { + "concept-type": "granule", + "concept-id": "G3493873323-ASF", + "revision-id": 1, + "native-id": "S1_073251_IW2_20250413T020809_VV_EF1E-BURST", + "collection-concept-id": "C2709161906-ASF", + "provider-id": "ASF", + "format": "application/vnd.nasa.cmr.umm+json", + "revision-date": "2025-04-13T05:21:49.972Z" + }, + "umm": { + "TemporalExtent": { + "RangeDateTime": { + "BeginningDateTime": "2025-04-13T02:08:10.453534Z", + "EndingDateTime": "2025-04-13T02:08:13.555368Z" + } + }, + "OrbitCalculatedSpatialDomains": [ + { + "OrbitNumber": 58732 + } + ], + "GranuleUR": "S1_073251_IW2_20250413T020809_VV_EF1E-BURST", + "AdditionalAttributes": [ + { + "Name": "ASC_NODE_TIME", + "Values": [ + "2025-04-13T01:57:59.280569Z" + ] + }, + { + "Name": "ASCENDING_DESCENDING", + "Values": [ + "ASCENDING" + ] + }, + { + "Name": "AZIMUTH_ANX_TIME", + "Values": [ + "610.1584306672" + ] + }, + { + "Name": "AZIMUTH_TIME", + "Values": [ + "2025-04-13T02:08:09.435967" + ] + }, + { + "Name": "AZIMUTH_TIME_INTERVAL", + "Values": [ + "0.002055556299999998" + ] + }, + { + "Name": "BEAM_MODE", + "Values": [ + "IW" + ] + }, + { + "Name": "BEAM_MODE_DESC", + "Values": [ + "Interferometric Wide. 250 km swath, 5 m x 20 m spatial resolution and burst synchronization for interferometry. IW is considered to be the standard mode over land masses." + ] + }, + { + "Name": "BURST_ID_ABSOLUTE", + "Values": [ + "126150173" + ] + }, + { + "Name": "BURST_ID_FULL", + "Values": [ + "035_073251_IW2" + ] + }, + { + "Name": "BURST_ID_RELATIVE", + "Values": [ + "73251" + ] + }, + { + "Name": "BURST_INDEX", + "Values": [ + "0" + ] + }, + { + "Name": "BYTE_LENGTH", + "Values": [ + "154781148" + ] + }, + { + "Name": "BYTE_OFFSET", + "Values": [ + "108963" + ] + }, + { + "Name": "CENTER_LAT", + "Values": [ + "37.84383923820902" + ] + }, + { + "Name": "CENTER_LON", + "Values": [ + "-122.37700996731327" + ] + }, + { + "Name": "GROUP_ID", + "Values": [ + "S1A_IWDV_0121_0127_058732_035" + ] + }, + { + "Name": "LINES_PER_BURST", + "Values": [ + "1509" + ] + }, + { + "Name": "PATH_NUMBER", + "Values": [ + "35" + ] + }, + { + "Name": "POLARIZATION", + "Values": [ + "VV" + ] + }, + { + "Name": "PROCESSING_LEVEL", + "Values": [ + "L1" + ] + }, + { + "Name": "PROCESSING_TYPE", + "Values": [ + "BURST" + ] + }, + { + "Name": "SAMPLES_PER_BURST", + "Values": [ + "25643" + ] + }, + { + "Name": "SUBSWATH_NAME", + "Values": [ + "IW2" + ] + }, + { + "Name": "SV_POSITION_POST", + "Values": [ + "-3466943.640335,-4432648.526834,4280636.897919,2025-04-13T02:08:19.000000" + ] + }, + { + "Name": "SV_POSITION_PRE", + "Values": [ + "-3480520.469779,-4478505.865553,4221649.253332,2025-04-13T02:08:09.000000" + ] + }, + { + "Name": "SV_VELOCITY_POST", + "Values": [ + "1380.492592,4609.671815,5874.702457,2025-04-13T02:08:19.000000" + ] + }, + { + "Name": "SV_VELOCITY_PRE", + "Values": [ + "1334.859471,4561.698961,5922.715631,2025-04-13T02:08:09.000000" + ] + } + ], + "SpatialExtent": { + "HorizontalSpatialDomain": { + "Geometry": { + "GPolygons": [ + { + "Boundary": { + "Points": [ + { + "Latitude": 37.679959, + "Longitude": -122.867058 + }, + { + "Latitude": 37.751761, + "Longitude": -122.345075 + }, + { + "Latitude": 37.817265, + "Longitude": -121.851681 + }, + { + "Latitude": 38.009685, + "Longitude": -121.893464 + }, + { + "Latitude": 37.940016, + "Longitude": -122.387251 + }, + { + "Latitude": 37.86404, + "Longitude": -122.909645 + }, + { + "Latitude": 37.679959, + "Longitude": -122.867058 + } + ] + } + } + ] + } + } + }, + "ProviderDates": [ + { + "Type": "Insert", + "Date": "2025-04-13T05:21:49Z" + }, + { + "Type": "Update", + "Date": "2025-04-13T05:21:49Z" + } + ], + "CollectionReference": { + "ShortName": "SENTINEL-1_BURSTS", + "Version": "1" + }, + "PGEVersionClass": { + "PGEName": "Sentinel-1 IPF", + "PGEVersion": "003.91" + }, + "RelatedUrls": [ + { + "URL": "https://sentinel1-burst.asf.alaska.edu/S1A_IW_SLC__1SDV_20250413T020809_20250413T020836_058732_07464F_EF1E/IW2/VV/0.tiff", + "Type": "USE SERVICE API", + "Description": "Use the link to extract the burst as a GeoTIFF from the input granule.", + "Format": "GeoTIFF" + }, + { + "URL": "https://sentinel1-burst.asf.alaska.edu/S1A_IW_SLC__1SDV_20250413T020809_20250413T020836_058732_07464F_EF1E/IW2/VV/0.xml", + "Type": "USE SERVICE API", + "Description": "Use the link to extract an extended metadata file of the burst from the input granule.", + "Format": "XML" + }, + { + "URL": "https://sentinel1-burst.asf.alaska.edu/S1A_IW_SLC__1SDV_20250413T020809_20250413T020836_058732_07464F_EF1E/IW2/VV/0.zip", + "Type": "USE SERVICE API", + "Description": "Use the link to extract the burst in ESA SAFE format from the input granule.", + "Format": "SAFE" + }, + { + "URL": "https://sentinel1-burst-docs.asf.alaska.edu/", + "Type": "VIEW RELATED INFORMATION", + "Description": "Usage information for the API used to extract a burst product from a Sentinel-1 SLC granule." + } + ], + "InputGranules": [ + "S1A_IW_SLC__1SDV_20250413T020809_20250413T020836_058732_07464F_EF1E-SLC" + ], + "Platforms": [ + { + "Instruments": [ + { + "ShortName": "C-SAR", + "Characteristics": [ + { + "Name": "LookDirection", + "Value": "RIGHT" + } + ] + } + ], + "ShortName": "SENTINEL-1A" + } + ], + "MetadataSpecification": { + "URL": "https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6", + "Name": "UMM-G", + "Version": "1.6.6" + } + } + } + ] +} \ No newline at end of file diff --git a/tests/test_prep_rtc.py b/tests/test_prep_rtc.py index 00b89de..96f7c48 100644 --- a/tests/test_prep_rtc.py +++ b/tests/test_prep_rtc.py @@ -1,4 +1,6 @@ +import json import unittest.mock +from pathlib import Path import pytest import requests @@ -7,6 +9,13 @@ from hyp3_opera_rtc import prep_rtc +def test_parse_response_for_slc_params(): + test_response = json.loads(Path('tests/data/burst_response.json').read_text()) + slc_name, burst_id = prep_rtc.parse_response_for_slc_params(test_response) + assert slc_name == 'S1A_IW_SLC__1SDV_20250413T020809_20250413T020836_058732_07464F_EF1E' + assert burst_id == 't035_073251_iw2' + + @responses.activate def test_granule_exists(): responses.get( diff --git a/tests/test_upload_rtc.py b/tests/test_upload_rtc.py index 602eff4..0a3ec30 100644 --- a/tests/test_upload_rtc.py +++ b/tests/test_upload_rtc.py @@ -18,14 +18,29 @@ def test_upload_rtc(rtc_results_dir, rtc_output_files, s3_bucket): assert len(resp['Contents']) == 9 + product_name = 'OPERA_L2_RTC-S1_T115-245714-IW1_20240809T141633Z_20250411T185446Z_S1A_30_v1.0' zip_s3_key = [c['Key'] for c in resp['Contents'] if c['Key'].endswith('.zip')].pop() + zip_filename = zip_s3_key.split(f'{prefix}/').pop() + + assert zip_filename == f'{product_name}.zip' + zip_download_path = rtc_results_dir / 'output.zip' aws.S3_CLIENT.download_file(s3_bucket, zip_s3_key, zip_download_path) with ZipFile(zip_download_path) as zf: files_in_zip = set([f.filename for f in zf.infolist()]) - assert files_in_zip == set(rtc_output_files) + assert files_in_zip == set( + [ + f'{product_name}/', + f'{product_name}/OPERA_L2_RTC-S1_T115-245714-IW1_20240809T141633Z_20250411T185446Z_S1A_30_v1.0_BROWSE.png', + f'{product_name}/OPERA_L2_RTC-S1_T115-245714-IW1_20240809T141633Z_20250411T185446Z_S1A_30_v1.0.iso.xml', + f'{product_name}/OPERA_L2_RTC-S1_T115-245714-IW1_20240809T141633Z_20250411T185446Z_S1A_30_v1.0.h5', + f'{product_name}/OPERA_L2_RTC-S1_T115-245714-IW1_20240809T141633Z_20250411T185446Z_S1A_30_v1.0_mask.tif', + f'{product_name}/OPERA_L2_RTC-S1_T115-245714-IW1_20240809T141633Z_20250411T185446Z_S1A_30_v1.0_VH.tif', + f'{product_name}/OPERA_L2_RTC-S1_T115-245714-IW1_20240809T141633Z_20250411T185446Z_S1A_30_v1.0_VV.tif', + ] + ) def test_make_zip_name(rtc_output_files):