Release v0.1.1 #48

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged

forrestfwilliams merged 19 commits into main from develop

May 2, 2025

.github/workflows/create-jira-issue.yml

-Original file line number
+Diff line change
@@ -0,0 +1,15 @@
+    name: Create Jira issue
+    on:
+      issues:
+        types: [labeled]
+    jobs:
+      call-create-jira-issue-workflow:
+        uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.18.1
+        secrets:
+          JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }}
+          JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }}
+          JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }}
+          JIRA_PROJECT: ${{ secrets.JIRA_PROJECT }}
+          JIRA_FIELDS: ${{ secrets.JIRA_FIELDS }}

CHANGELOG.md

-Original file line number
+Diff line change
@@ Expand Up @@
     and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
     and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+    ## [0.1.1]
+    ### Changed
+    - All files in zip now contained in folder named after the product
+    - Remove `log` and `catalog.json` from the zip
+    - Switch to a full-SLC based data prep strategy
     ## [0.1.0]
     ### Added
@@ Expand Down @@

environment.yml

-Original file line number
+Diff line change
@@ Expand Up / @@ -12,7 +12,6 @@ dependencies: @@
       - hyp3lib
       - boto3
       - jinja2
-      - burst2safe
       # For packaging, and testing
       - setuptools
       - setuptools_scm
@@ Expand Down @@

pyproject.toml

-Original file line number
+Diff line change
@@ Expand Up / @@ -30,7 +30,6 @@ dependencies = [ @@
         "urllib3",
         "lxml",
         "jinja2",
-        "burst2safe"
     ]
     dynamic = ["version", "readme"]
@@ Expand Down @@

src/hyp3_opera_rtc/prep_rtc.py

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -2,13 +2,13 @@
  
    import os

    import warnings

    from pathlib import Path

    from shutil import make_archive

    from zipfile import ZipFile

    import hyp3lib.fetch

    import lxml.etree as ET

    import requests

    from burst2safe.burst2safe import burst2safe

    from hyp3lib.fetch import download_file

    from hyp3lib.scene import get_download_url

    from jinja2 import Template

    from shapely.geometry import Polygon, box

    @@ -49,11 +49,36 @@ def get_s1_granule_bbox(granule_path: Path, buffer: float = 0.025) -> Polygon:
  
        return box(*footprint.bounds)

    def granule_exists(granule: str) -> bool:

    def get_granule_cmr(granule: str) -> dict:

        params = (('short_name', 'SENTINEL-1_BURSTS'), ('granule_ur', granule))

        response = requests.get(CMR_URL, params=params)

        response.raise_for_status()

        return bool(response.json()['items'])

        return response.json()

    def granule_exists(granule: str) -> bool:

        response = get_granule_cmr(granule)

        return bool(response['items'])

    def parse_response_for_slc_params(response: dict) -> tuple[str, str]:

        assert len(response['items']) == 1

        item = response['items'][0]

        source_slc = item['umm']['InputGranules'][0][:-4]

        assert isinstance(source_slc, str)

        opera_burst_ids = [attr for attr in item['umm']['AdditionalAttributes'] if attr['Name'] == 'BURST_ID_FULL']

        assert len(opera_burst_ids) == 1

        opera_burst_id = opera_burst_ids[0]['Values'][0]

        assert isinstance(opera_burst_id, str)

        return source_slc, f't{opera_burst_id.lower()}'

    def get_granule_slc_params(granule: str) -> tuple[str, str]:

        response = get_granule_cmr(granule)

        return parse_response_for_slc_params(response)

    def validate_co_pol_granule(granule: str) -> None:

    @@ -101,18 +126,11 @@ def prep_rtc(
  
        validate_co_pol_granule(co_pol_granule)

        cross_pol_granule = get_cross_pol_name(co_pol_granule)

        dual_pol = granule_exists(cross_pol_granule)

        if dual_pol:

            print(f'Found cross-pol granule: {cross_pol_granule}')

            granules = [co_pol_granule, cross_pol_granule]

        else:

            print('No cross-pol granule found')

            granules = [co_pol_granule]

        safe_path = burst2safe(granules=granules, all_anns=True, work_dir=input_dir)

        granule_path = Path(make_archive(base_name=str(safe_path.with_suffix('')), format='zip', base_dir=str(safe_path)))

        print(f'Created archive: {granule_path}')

        source_slc, opera_burst_id = get_granule_slc_params(co_pol_granule)

        safe_path = download_file(get_download_url(source_slc), directory=str(input_dir), chunk_size=10485760)

        safe_path = Path(safe_path)

        dual_pol = safe_path.name[14] == 'D'

        print(f'Created archive: {safe_path}')

        orbit_path = orbit.get_orbit(safe_path.with_suffix('').name, save_dir=input_dir)

        print(f'Downloaded orbit file: {orbit_path}')

    @@ -121,15 +139,16 @@ def prep_rtc(
  
        print(f'Downloaded burst database: {db_path}')

        dem_path = input_dir / 'dem.tif'

        granule_bbox = get_s1_granule_bbox(granule_path)

        granule_bbox = get_s1_granule_bbox(safe_path)

        dem.download_opera_dem_for_footprint(dem_path, granule_bbox)

        print(f'Downloaded DEM: {dem_path}')

        runconfig_dict = {

            'granule_path': str(granule_path),

            'granule_path': str(safe_path),

            'orbit_path': str(orbit_path),

            'db_path': str(db_path),

            'dem_path': str(dem_path),

            'opera_burst_id': opera_burst_id,

            'scratch_dir': str(scratch_dir),

            'output_dir': str(output_dir),

            'dual_pol': dual_pol,

src/hyp3_opera_rtc/templates/pge.yml.j2

-Original file line number
+Diff line change
@@ Expand Up / @@ -135,6 +135,9 @@ RunConfig: @@
                             # Required. List of SAFE files (min=1)
                             safe_file_path:
                                 - {{  granule_path  }}
+                            # Optional. Burst ID to process (empty for all bursts)
+                            burst_id:
+                                - {{  opera_burst_id  }}
                             # Required. List of orbit (EOF) files (min=1)
                             orbit_file_path:
                                 - {{  orbit_path  }}
@@ Expand Down @@

src/hyp3_opera_rtc/upload_rtc.py

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -15,20 +15,26 @@ def upload_rtc(bucket: str, bucket_prefix: str, output_dir: Path) -> None:
  
    def make_zip(output_files: list[Path], output_dir: Path) -> Path:

        zip_name = make_zip_name(output_files)

        zip_path = output_dir / zip_name

        zip_archive_path = output_dir / 'zip'

        zip_archive_path.mkdir(exist_ok=True)

        (zip_archive_path / zip_name).mkdir(exist_ok=True, parents=True)

        file_extensions_to_include = set(['.png', '.xml', '.tif', '.h5'])

        for output_file in output_files:

            copyfile(output_file, zip_archive_path / output_file.name)

            if output_file.suffix not in file_extensions_to_include:

                continue

        zip_path = output_dir / make_zip_name(output_files)

        output_zip = make_archive(base_name=str(zip_path), format='zip', root_dir=zip_archive_path)

            zip_dest_path = zip_archive_path / zip_name / output_file.name

            copyfile(output_file, zip_dest_path)

        output_zip = make_archive(base_name=str(zip_path), format='zip', root_dir=zip_archive_path)

        return Path(output_zip)

    def make_zip_name(product_files: list[Path]) -> str:

        h5_file = [f for f in product_files if f.name.endswith('h5')].pop()

        h5_file = next(f for f in product_files if f.name.endswith('h5'))

        return h5_file.name.split('.h5')[0]

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Release v0.1.1 #48

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!