From b0801de6578a6c3a643b8b4b233a0c8bc2905a60 Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Mon, 14 Oct 2024 10:23:36 -0700 Subject: [PATCH 01/11] feat: add AUDIT log level for upload --- .../upload_arbitrary_files_as_granules.py | 1 + .../stage_in_out/upload_granules_abstract.py | 15 + .../upload_granules_by_complete_catalog_s3.py | 2 +- tests/integration_tests/.env.tpl | 6 + tests/integration_tests/__init__.py | 0 tests/integration_tests/test_docker_entry.py | 375 ++++ .../integration_tests/test_docker_stage_in.py | 1928 +++++++++++++++++ .../test_docker_stage_out.py | 1334 ++++++++++++ 8 files changed, 3660 insertions(+), 1 deletion(-) create mode 100644 tests/integration_tests/.env.tpl create mode 100644 tests/integration_tests/__init__.py create mode 100644 tests/integration_tests/test_docker_entry.py create mode 100644 tests/integration_tests/test_docker_stage_in.py create mode 100644 tests/integration_tests/test_docker_stage_out.py diff --git a/mdps_ds_lib/stage_in_out/upload_arbitrary_files_as_granules.py b/mdps_ds_lib/stage_in_out/upload_arbitrary_files_as_granules.py index 9a53a75..a545ce4 100644 --- a/mdps_ds_lib/stage_in_out/upload_arbitrary_files_as_granules.py +++ b/mdps_ds_lib/stage_in_out/upload_arbitrary_files_as_granules.py @@ -77,6 +77,7 @@ def execute_job(self, job_obj, lock) -> bool: sample_stac_item = self.generate_sample_stac(job_obj) updating_assets = {} try: + LOGGER.audit(f'uploading auxiliary file: {job_obj}') s3_url = self.__s3.upload(job_obj, self.__staging_bucket, f'{self.__collection_id}/{self.__collection_id}:{sample_stac_item.id}', self.__delete_files) updating_assets[os.path.basename(s3_url)] = s3_url uploading_current_granule_stac = f'{s3_url}.stac.json' diff --git a/mdps_ds_lib/stage_in_out/upload_granules_abstract.py b/mdps_ds_lib/stage_in_out/upload_granules_abstract.py index 66543e5..5d803d0 100644 --- a/mdps_ds_lib/stage_in_out/upload_granules_abstract.py +++ b/mdps_ds_lib/stage_in_out/upload_granules_abstract.py @@ -1,3 +1,18 @@ +import logging + +# Define the new log level AUDIT +AUDIT_LEVEL = 60 +logging.addLevelName(AUDIT_LEVEL, "AUDIT") + + +# Define a custom method for logging at AUDIT level +def audit(self, message, *args, **kwargs): + if self.isEnabledFor(AUDIT_LEVEL): + self._log(AUDIT_LEVEL, message, args, **kwargs) + + +logging.Logger.audit = audit + import os from abc import ABC, abstractmethod diff --git a/mdps_ds_lib/stage_in_out/upload_granules_by_complete_catalog_s3.py b/mdps_ds_lib/stage_in_out/upload_granules_by_complete_catalog_s3.py index 409a41b..2f7d8e7 100644 --- a/mdps_ds_lib/stage_in_out/upload_granules_by_complete_catalog_s3.py +++ b/mdps_ds_lib/stage_in_out/upload_granules_by_complete_catalog_s3.py @@ -64,7 +64,7 @@ def execute_job(self, each_child, lock) -> bool: uploading_current_granule_stac = None for asset_type, asset_hrefs in current_assets.items(): for each_asset_href in asset_hrefs: - LOGGER.debug(f'uploading {asset_type}, {each_asset_href}') + LOGGER.audit(f'uploading {asset_type}: {each_asset_href}') s3_url = self.__s3.upload(each_asset_href, self.__staging_bucket, f'{self.__collection_id}/{self.__collection_id}:{current_granule_id}', self.__delete_files) diff --git a/tests/integration_tests/.env.tpl b/tests/integration_tests/.env.tpl new file mode 100644 index 0000000..5bb2ddf --- /dev/null +++ b/tests/integration_tests/.env.tpl @@ -0,0 +1,6 @@ +USERNAME=base64-encoded-str +PASSWORD=base64-encoded-str +CLIENT_ID=7a1fglm2d54eoggj13lccivp25 +COGNITO_URL=https://cognito-idp.us-west-2.amazonaws.com +UNITY_URL=https://k3a3qmarxh.execute-api.us-west-2.amazonaws.com/dev +UNITY_STAGE=sbx-uds-dapa diff --git a/tests/integration_tests/__init__.py b/tests/integration_tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration_tests/test_docker_entry.py b/tests/integration_tests/test_docker_entry.py new file mode 100644 index 0000000..df328b3 --- /dev/null +++ b/tests/integration_tests/test_docker_entry.py @@ -0,0 +1,375 @@ +import logging + +logging.basicConfig(level=10, format="%(asctime)s [%(levelname)s] [%(name)s::%(lineno)d] %(message)s") + +import math +from unittest.mock import patch, MagicMock +import json +import os +import tempfile +from glob import glob +from sys import argv +from unittest import TestCase + +from pystac import Item, Asset, Catalog, Link, ItemCollection + +from mdps_ds_lib.lib.constants import Constants + +from cumulus_lambda_functions.docker_entrypoint.__main__ import choose_process +from mdps_ds_lib.lib.utils.time_utils import TimeUtils +from mdps_ds_lib.lib.utils.file_utils import FileUtils + + +class TestDockerEntry(TestCase): + def test_01_search_part_01(self): + """ + :return: + """ + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '6ir9qveln397i0inh9pmsabq1' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test' + os.environ['COLLECTION_ID'] = 'URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030' + os.environ['LIMITS'] = '4000' + os.environ['DATE_FROM'] = '1990-01-14T08:00:00Z' + os.environ['DATE_TO'] = '2022-01-14T11:59:59Z' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' + if len(argv) > 1: + argv.pop(-1) + argv.append('SEARCH') + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + search_result_str = choose_process() + search_result = json.loads(search_result_str) + self.assertTrue('type' in search_result, f'missing type in search_result') + item_collections = ItemCollection.from_dict(search_result) + # self.assertTrue(isinstance(search_result, list), f'search_result is not list: {search_result}') + self.assertEqual(len(item_collections.items), 4000, f'wrong length') + search_result = set([k.id for k in item_collections.items]) + self.assertEqual(len(search_result), 4000, f'wrong length. not unique') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + self.assertEqual(sorted(json.dumps(FileUtils.read_json(os.environ['OUTPUT_FILE']))), sorted(search_result_str), f'not identical result') + return + + def test_01_search_part_02(self): + """ + :return: + """ + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '6ir9qveln397i0inh9pmsabq1' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test' + os.environ['COLLECTION_ID'] = 'SNDR_SNPP_ATMS_L1A___1' + os.environ['LIMITS'] = '100' + os.environ['DATE_FROM'] = '2016-01-14T08:00:00Z' + os.environ['DATE_TO'] = '2016-01-14T11:59:59Z' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' + if len(argv) > 1: + argv.pop(-1) + argv.append('SEARCH') + search_result = choose_process() + print(search_result) + search_result = json.loads(search_result) + self.assertTrue('type' in search_result, f'missing type in search_result') + item_collections = ItemCollection.from_dict(search_result) + # self.assertTrue(isinstance(search_result, list), f'search_result is not list: {search_result}') + self.assertEqual(len(item_collections.items), 20, f'wrong length') + search_result = set([k.id for k in item_collections.items]) + self.assertEqual(len(search_result),20, f'wrong length. not unique') + return + + def test_01_search_part_03(self): + """ + :return: + """ + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '6ir9qveln397i0inh9pmsabq1' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test' + os.environ['COLLECTION_ID'] = 'L0_SNPP_ATMS_SCIENCE___1' + os.environ['LIMITS'] = '-1' + os.environ['DATE_FROM'] = '1990-01-14T08:00:00Z' + os.environ['DATE_TO'] = '2022-01-14T11:59:59Z' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' + if len(argv) > 1: + argv.pop(-1) + argv.append('SEARCH') + search_result = choose_process() + search_result = json.loads(search_result) + self.assertTrue('type' in search_result, f'missing type in search_result') + item_collections = ItemCollection.from_dict(search_result) + # self.assertTrue(isinstance(search_result, list), f'search_result is not list: {search_result}') + self.assertEqual(len(item_collections.items), 4381, f'wrong length') + search_result = set([k.id for k in item_collections.items]) + self.assertEqual(len(search_result), 4381, f'wrong length. not unique') + return + + def test_01_search_part_04(self): + """ + :return: + """ + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '6ir9qveln397i0inh9pmsabq1' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test' + os.environ['COLLECTION_ID'] = 'L0_SNPP_ATMS_SCIENCE___1' + os.environ['LIMITS'] = '347' + os.environ['DATE_FROM'] = '1990-01-14T08:00:00Z' + os.environ['DATE_TO'] = '2022-01-14T11:59:59Z' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' + if len(argv) > 1: + argv.pop(-1) + argv.append('SEARCH') + search_result = choose_process() + search_result = json.loads(search_result) + self.assertTrue('type' in search_result, f'missing type in search_result') + item_collections = ItemCollection.from_dict(search_result) + self.assertEqual(len(item_collections.items), 347, f'wrong length') + search_result = set([k.id for k in item_collections.items]) + self.assertEqual(len(search_result), 347, f'wrong length. not unique') + return + + def test_01_search_part_05(self): + """ + :return: + """ + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['API_PREFIX'] = 'sbx-uds-2-dapa' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://1gp9st60gd.execute-api.us-west-2.amazonaws.com/dev' + os.environ['COLLECTION_ID'] = 'URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030' + os.environ['LIMITS'] = '37' + os.environ['DATE_FROM'] = '1990-01-14T08:00:00Z' + os.environ['DATE_TO'] = '2024-01-14T11:59:59Z' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' + if len(argv) > 1: + argv.pop(-1) + argv.append('SEARCH') + search_result = choose_process() + search_result = json.loads(search_result) + self.assertTrue('type' in search_result, f'missing type in search_result') + item_collections = ItemCollection.from_dict(search_result) + self.assertEqual(len(item_collections.items), 37, f'wrong length') + return + + def test_01_1_search_cmr_part_01(self): + """ + :return: + """ + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '6ir9qveln397i0inh9pmsabq1' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test' + os.environ['COLLECTION_ID'] = 'C1996881146-POCLOUD' # 'C1666605425-PODAAC' # C1996881146-POCLOUD + os.environ['LIMITS'] = '2120' + os.environ['DATE_FROM'] = '2002-06-01T12:06:00.000Z' + os.environ['DATE_TO'] = '2011-10-04T06:51:45.000Z' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['GRANULES_SEARCH_DOMAIN'] = 'CMR' + os.environ['CMR_BASE_URL'] = 'https://cmr.earthdata.nasa.gov' + if len(argv) > 1: + argv.pop(-1) + argv.append('SEARCH') + search_result = choose_process() + search_result = json.loads(search_result) + self.assertTrue('type' in search_result, f'missing type in search_result') + item_collections = ItemCollection.from_dict(search_result) + self.assertEqual(len(item_collections.items), 2120, f'wrong length') + return + + def test_01_1_search_cmr_part_02(self): + """ + :return: + """ + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '6ir9qveln397i0inh9pmsabq1' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test' + os.environ['COLLECTION_ID'] = 'C1996881146-POCLOUD' # 'C1666605425-PODAAC' # C1996881146-POCLOUD + os.environ['LIMITS'] = '23' + os.environ['DATE_FROM'] = '2002-06-01T12:06:00.000Z' + os.environ['DATE_TO'] = '2011-10-04T06:51:45.000Z' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['GRANULES_SEARCH_DOMAIN'] = 'CMR' + os.environ['CMR_BASE_URL'] = 'https://cmr.earthdata.nasa.gov' + if len(argv) > 1: + argv.pop(-1) + argv.append('SEARCH') + search_result = choose_process() + search_result = json.loads(search_result) + self.assertTrue('type' in search_result, f'missing type in search_result') + item_collections = ItemCollection.from_dict(search_result) + self.assertEqual(len(item_collections.items), 23, f'wrong length') + return + + def test_04_catalog(self): + upload_result = [{'id': 'NEW_COLLECTION_EXAMPLE_L1B___9:test_file01', 'collection': 'NEW_COLLECTION_EXAMPLE_L1B___9', 'assets': {'metadata': {'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.cas'}, 'data': {'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc'}}}] + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '6ir9qveln397i0inh9pmsabq1' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['PROVIDER_ID'] = 'SNPP' + os.environ['GRANULES_CATALOG_TYPE'] = 'UNITY' + os.environ['UPLOADED_FILES_JSON'] = json.dumps(upload_result) + if len(argv) > 1: + argv.pop(-1) + argv.append('CATALOG') + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + catalog_result_str = choose_process() + catalog_result = json.loads(catalog_result_str) + self.assertEqual('registered', catalog_result, 'wrong status') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_04_catalog_from_file(self): + upload_result = [{'id': 'NEW_COLLECTION_EXAMPLE_L1B___9:test_file01', 'collection': 'NEW_COLLECTION_EXAMPLE_L1B___9', 'assets': {'metadata': {'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.cas'}, 'data': {'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc'}}}] + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '6ir9qveln397i0inh9pmsabq1' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['PROVIDER_ID'] = 'SNPP' + os.environ['GRANULES_CATALOG_TYPE'] = 'UNITY' + os.environ['DELAY_SECOND'] = '35' + os.environ['REPEAT_TIMES'] = '3' + if len(argv) > 1: + argv.pop(-1) + argv.append('CATALOG') + with tempfile.TemporaryDirectory() as tmp_dir_name: + input_file_path = os.path.join(tmp_dir_name, 'uploaded_files.json') + FileUtils.write_json(input_file_path, upload_result) + os.environ['UPLOADED_FILES_JSON'] = input_file_path + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + catalog_result_str = choose_process() + catalog_result = json.loads(catalog_result_str) + self.assertTrue('cataloging_request_status' in catalog_result, f'missing cataloging_request_status') + self.assertTrue('status_result' in catalog_result, f'missing status_result') + self.assertEqual(catalog_result['cataloging_request_status'], 'registered', f'mismatched cataloging_request_status value') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + + status_result = catalog_result['status_result'] + + self.assertTrue('cataloged' in status_result, f'missing cataloged') + self.assertTrue('missing_granules' in status_result, f'missing missing_granules') + self.assertTrue('registered_granules' in status_result, f'missing registered_granules') + self.assertTrue(isinstance(status_result['cataloged'], bool), f'cataloged is not boolean: {status_result["cataloged"]}') + # Example result: {'cataloging_request_status': 'registered', 'status_result': {'cataloged': False, 'missing_granules': ['NEW_COLLECTION_EXAMPLE_L1B___9:test_file01'], 'registered_granules': []}} + return + + def test_04_catalog_from_file_item_collection(self): + upload_result = {'type': 'FeatureCollection', 'features': [ + {'type': 'Feature', 'stac_version': '1.0.0', 'id': 'NEW_COLLECTION_EXAMPLE_L1B___9:test_file01', + 'properties': {'start_datetime': '2016-01-31T18:00:00.009057Z', 'end_datetime': '2016-01-31T19:59:59.991043Z', + 'created': '2016-02-01T02:45:59.639000Z', 'updated': '2022-03-23T15:48:21.578000Z', + 'datetime': '1970-01-01T00:00:00Z'}, 'geometry': {'type': 'Point', 'coordinates': [0.0, 0.0]}, + 'links': [], 'assets': { + 'data': {'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc', + 'title': 'main data'}, 'metadata__cas': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.cas', + 'title': 'metadata cas'}, 'metadata__stac': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.stac.json', + 'title': 'metadata stac'}}, 'bbox': [0.0, 0.0, 0.0, 0.0], 'stac_extensions': [], + 'collection': 'NEW_COLLECTION_EXAMPLE_L1B___9'}]} + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '6ir9qveln397i0inh9pmsabq1' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['PROVIDER_ID'] = 'SNPP' + os.environ['GRANULES_CATALOG_TYPE'] = 'UNITY' + # os.environ['DELAY_SECOND'] = '5' + # os.environ['REPEAT_TIMES'] = '3' + os.environ['DELAY_SECOND'] = '35' + os.environ['REPEAT_TIMES'] = '3' + if len(argv) > 1: + argv.pop(-1) + argv.append('CATALOG') + with tempfile.TemporaryDirectory() as tmp_dir_name: + input_file_path = os.path.join(tmp_dir_name, 'uploaded_files.json') + FileUtils.write_json(input_file_path, upload_result) + os.environ['UPLOADED_FILES_JSON'] = input_file_path + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + catalog_result_str = choose_process() + catalog_result = json.loads(catalog_result_str) + self.assertTrue('cataloging_request_status' in catalog_result, f'missing cataloging_request_status') + self.assertTrue('status_result' in catalog_result, f'missing status_result') + self.assertEqual(catalog_result['cataloging_request_status'], 'registered', f'mismatched cataloging_request_status value') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + # TODO update this after it is deployed to MCP Test + status_result = catalog_result['status_result'] + + self.assertTrue('cataloged' in status_result, f'missing cataloged') + self.assertTrue('missing_granules' in status_result, f'missing missing_granules') + self.assertTrue('registered_granules' in status_result, f'missing registered_granules') + self.assertTrue(isinstance(status_result['cataloged'], bool), f'cataloged is not boolean: {status_result["cataloged"]}') + return + + def test_04_catalog_from_file_item_collection_large(self): + upload_result = FileUtils.read_json('./stage-out-results.json') + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ[Constants.CLIENT_ID] = '71g0c73jl77gsqhtlfg2ht388c' # MCP Dev + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://1gp9st60gd.execute-api.us-west-2.amazonaws.com/dev' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['PROVIDER_ID'] = 'SNPP' + os.environ['GRANULES_CATALOG_TYPE'] = 'UNITY' + os.environ['CHUNK_SIZE'] = '250' + # os.environ['DELAY_SECOND'] = '5' + # os.environ['REPEAT_TIMES'] = '3' + + if len(argv) > 1: + argv.pop(-1) + argv.append('CATALOG') + with tempfile.TemporaryDirectory() as tmp_dir_name: + input_file_path = os.path.join(tmp_dir_name, 'uploaded_files.json') + FileUtils.write_json(input_file_path, upload_result) + os.environ['UPLOADED_FILES_JSON'] = input_file_path + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + catalog_result_str = choose_process() + catalog_result = json.loads(catalog_result_str) + self.assertTrue(isinstance(catalog_result, list), f'catalog_result is not list. {catalog_result}') + self.assertEqual(len(catalog_result), math.ceil(len(upload_result['features']) / 250), f'mismatched catalog_result count') + + catalog_result = catalog_result[0] + self.assertTrue('cataloging_request_status' in catalog_result, f'missing cataloging_request_status') + self.assertTrue('status_result' in catalog_result, f'missing status_result') + self.assertEqual(catalog_result['cataloging_request_status'], {'message': 'processing'}, f'mismatched cataloging_request_status value') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + + status_result = catalog_result['status_result'] + # TODO disabling this as we are not waiting for them to be registered. + # self.assertTrue('cataloged' in status_result, f'missing cataloged') + # self.assertTrue('missing_granules' in status_result, f'missing missing_granules') + # self.assertTrue('registered_granules' in status_result, f'missing registered_granules') + # self.assertTrue(isinstance(status_result['cataloged'], bool), f'cataloged is not boolean: {status_result["cataloged"]}') + return diff --git a/tests/integration_tests/test_docker_stage_in.py b/tests/integration_tests/test_docker_stage_in.py new file mode 100644 index 0000000..6336640 --- /dev/null +++ b/tests/integration_tests/test_docker_stage_in.py @@ -0,0 +1,1928 @@ +import os + +from mdps_ds_lib.stage_in_out.download_granules_factory import DownloadGranulesFactory + +os.environ['OBJC_DISABLE_INITIALIZE_FORK_SAFETY'] = 'YES' +os.environ['no_proxy'] = '*' +import logging + +import requests + +logging.basicConfig(level=10, format="%(asctime)s [%(levelname)s] [%(name)s::%(lineno)d] %(message)s") + +import math +from unittest.mock import patch, MagicMock +import json +import tempfile +from glob import glob +from sys import argv +from unittest import TestCase + +from pystac import Item, Asset, Catalog, Link, ItemCollection + +from mdps_ds_lib.lib.constants import Constants + +from mdps_ds_lib.lib.utils.time_utils import TimeUtils +from mdps_ds_lib.lib.utils.file_utils import FileUtils + + +class TestDockerStageIn(TestCase): + def test_02_download(self): + granule_json = '{"numberMatched": 20, "numberReturned": 20, "stac_version": "1.0.0", "type": "FeatureCollection", "links": [{"rel": "self", "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=0"}, {"rel": "root", "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com"}, {"rel": "next", "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=100"}, {"rel": "prev", "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=0"}], "features": [{"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.01", "properties": {"start_datetime": "2016-01-14T09:54:00Z", "end_datetime": "2016-01-14T10:00:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:39.830000Z", "datetime": "2022-08-15T06:26:37.029000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.01.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.01.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.01.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.01.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.08", "properties": {"start_datetime": "2016-01-14T10:36:00Z", "end_datetime": "2016-01-14T10:42:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:26.078000Z", "datetime": "2022-08-15T06:26:19.333000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.08.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.08.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.08.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.08.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.08.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.06", "properties": {"start_datetime": "2016-01-14T10:24:00Z", "end_datetime": "2016-01-14T10:30:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:26.068000Z", "datetime": "2022-08-15T06:26:18.641000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.06.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.06.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.06.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.06.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.06.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.18", "properties": {"start_datetime": "2016-01-14T11:36:00Z", "end_datetime": "2016-01-14T11:42:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:26.060000Z", "datetime": "2022-08-15T06:26:19.698000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.18.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.18.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.18.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.18.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.18.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.04", "properties": {"start_datetime": "2016-01-14T10:12:00Z", "end_datetime": "2016-01-14T10:18:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:26.050000Z", "datetime": "2022-08-15T06:26:19.491000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.04.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.04.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.04.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.04.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.04.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.16", "properties": {"start_datetime": "2016-01-14T11:24:00Z", "end_datetime": "2016-01-14T11:30:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:25.917000Z", "datetime": "2022-08-15T06:26:19.027000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.16.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.16.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.16.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.16.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.16.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.16.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.16.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.16.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.16.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.17", "properties": {"start_datetime": "2016-01-14T11:30:00Z", "end_datetime": "2016-01-14T11:36:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:25.907000Z", "datetime": "2022-08-15T06:26:19.042000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.17.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.17.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.17.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.17.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.17.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.17.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.17.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.17.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.17.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.10", "properties": {"start_datetime": "2016-01-14T10:48:00Z", "end_datetime": "2016-01-14T10:54:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:25.446000Z", "datetime": "2022-08-15T06:26:18.730000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.10.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.10.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.10.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.10.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.10.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.10.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.10.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.10.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.10.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.14", "properties": {"start_datetime": "2016-01-14T11:12:00Z", "end_datetime": "2016-01-14T11:18:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:25.354000Z", "datetime": "2022-08-15T06:26:17.758000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.14.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.14.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.14.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.14.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.14.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.14.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.14.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.14.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.14.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.12", "properties": {"start_datetime": "2016-01-14T11:00:00Z", "end_datetime": "2016-01-14T11:06:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:25.344000Z", "datetime": "2022-08-15T06:26:17.938000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.12.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.12.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.09", "properties": {"start_datetime": "2016-01-14T10:42:00Z", "end_datetime": "2016-01-14T10:48:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:24.910000Z", "datetime": "2022-08-15T06:26:20.688000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.09.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.09.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.09.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.09.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.09.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.09.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.09.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.09.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.09.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.20", "properties": {"start_datetime": "2016-01-14T11:48:00Z", "end_datetime": "2016-01-14T11:54:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:23.929000Z", "datetime": "2022-08-15T06:26:19.091000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.20.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.20.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.20.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.20.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.20.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.20.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.20.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.20.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.20.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.15", "properties": {"start_datetime": "2016-01-14T11:18:00Z", "end_datetime": "2016-01-14T11:24:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:23.732000Z", "datetime": "2022-08-15T06:26:19.282000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.15.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.15.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.15.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.15.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.15.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.15.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.15.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.15.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.15.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.07", "properties": {"start_datetime": "2016-01-14T10:30:00Z", "end_datetime": "2016-01-14T10:36:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:23.371000Z", "datetime": "2022-08-15T06:26:19.047000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.07.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.07.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.07.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.07.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.07.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.07.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.07.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.07.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.07.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.19", "properties": {"start_datetime": "2016-01-14T11:42:00Z", "end_datetime": "2016-01-14T11:48:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:23.268000Z", "datetime": "2022-08-15T06:26:18.576000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.19.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.19.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.19.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.19.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.19.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.19.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.19.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.19.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.19.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.03", "properties": {"start_datetime": "2016-01-14T10:06:00Z", "end_datetime": "2016-01-14T10:12:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:22.930000Z", "datetime": "2022-08-15T06:26:17.714000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.03.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.03.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.03.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.03.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.03.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.03.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.03.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.03.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.03.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.11", "properties": {"start_datetime": "2016-01-14T10:54:00Z", "end_datetime": "2016-01-14T11:00:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:22.863000Z", "datetime": "2022-08-15T06:26:17.648000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.11.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.11.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.11.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.11.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.11.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.11.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.11.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.11.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.11.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.05", "properties": {"start_datetime": "2016-01-14T10:18:00Z", "end_datetime": "2016-01-14T10:24:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:22.649000Z", "datetime": "2022-08-15T06:26:18.060000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.05.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.05.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.05.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.05.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.05.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.05.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.05.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.05.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.05.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.13", "properties": {"start_datetime": "2016-01-14T11:06:00Z", "end_datetime": "2016-01-14T11:12:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:22.277000Z", "datetime": "2022-08-15T06:26:18.090000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.13.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.13.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.13.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.13.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.13.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.13.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.13.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.13.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.13.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.02", "properties": {"start_datetime": "2016-01-14T10:00:00Z", "end_datetime": "2016-01-14T10:06:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:22.169000Z", "datetime": "2022-08-15T06:26:17.466000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.02.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.02.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.02.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.02.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.02.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.02.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.02.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.02.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.02.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}]}' + granule_json = json.loads(granule_json) + os.environ['STAC_JSON'] = json.dumps(granule_json) + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'S3' + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + os.environ['DOWNLOAD_DIR'] = tmp_dir_name + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) + 2, len(glob(os.path.join(tmp_dir_name, '*'))), + f'downloaded file does not match') + error_file = os.path.join(tmp_dir_name, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([k['assets']['data']['href'] for k in download_result]) + for each_granule in zip(granule_json['features'], download_result): + remote_filename = os.path.basename(each_granule[0]['assets']['data']['href']) + self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, + f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_02_download__daac(self): + granule_json = '{"type": "FeatureCollection", "stac_version": "1.0.0", "numberMatched": 3413, "numberReturned": 23, "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/granules.stac?collection_concept_id=C1996881146-POCLOUD&page_size=23&temporal%5B%5D=2002-06-01T12%3A06%3A00.000Z%2C2011-10-04T06%3A51%3A45.000Z&page_num=1"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "next", "body": {"collection_concept_id": "C1996881146-POCLOUD", "page_num": "2", "page_size": "23", "temporal": ["2002-06-01T12:06:00.000Z,2011-10-04T06:51:45.000Z"], "temporal[]": "2002-06-01T12:06:00.000Z,2011-10-04T06:51:45.000Z"}, "method": "POST", "merge": true, "href": "https://cmr.earthdata.nasa.gov:443/search/granules.stac"}], "context": {"returned": 23, "limit": 1000000, "matched": 3413}, "features": [{"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"metadata": {"href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.xml", "type": "application/xml"}, "opendap": {"title": "OPeNDAP request URL", "href": "https://opendap.earthdata.nasa.gov/providers/POCLOUD/collections/GHRSST%20Level%204%20MUR%20Global%20Foundation%20Sea%20Surface%20Temperature%20Analysis%20(v4.1)/granules/20020601090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1"}, "data": {"title": "Download 20020601090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc", "href": "https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/MUR-JPL-L4-GLOB-v4.1/20020601090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc"}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-06-01T21:00:00.000Z", "start_datetime": "2002-06-01T21:00:00.000Z", "end_datetime": "2002-06-02T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"metadata": {"href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106835-POCLOUD.xml", "type": "application/xml"}, "opendap": {"title": "OPeNDAP request URL", "href": "https://opendap.earthdata.nasa.gov/providers/POCLOUD/collections/GHRSST%20Level%204%20MUR%20Global%20Foundation%20Sea%20Surface%20Temperature%20Analysis%20(v4.1)/granules/20020602090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1"}, "data": {"title": "Download 20020602090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc", "href": "https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/MUR-JPL-L4-GLOB-v4.1/20020602090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc"}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2028106835-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106835-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106835-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106835-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-06-02T21:00:00.000Z", "start_datetime": "2002-06-02T21:00:00.000Z", "end_datetime": "2002-06-03T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"metadata": {"href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106890-POCLOUD.xml", "type": "application/xml"}, "opendap": {"title": "OPeNDAP request URL", "href": "https://opendap.earthdata.nasa.gov/providers/POCLOUD/collections/GHRSST%20Level%204%20MUR%20Global%20Foundation%20Sea%20Surface%20Temperature%20Analysis%20(v4.1)/granules/20020603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1"}, "data": {"title": "Download 20020603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc", "href": "https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/MUR-JPL-L4-GLOB-v4.1/20020603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc"}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2028106890-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106890-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106890-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106890-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-06-03T21:00:00.000Z", "start_datetime": "2002-06-03T21:00:00.000Z", "end_datetime": "2002-06-04T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"metadata": {"href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106962-POCLOUD.xml", "type": "application/xml"}, "opendap": {"title": "OPeNDAP request URL", "href": "https://opendap.earthdata.nasa.gov/providers/POCLOUD/collections/GHRSST%20Level%204%20MUR%20Global%20Foundation%20Sea%20Surface%20Temperature%20Analysis%20(v4.1)/granules/20020604090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1"}, "data": {"title": "Download 20020604090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc", "href": "https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/MUR-JPL-L4-GLOB-v4.1/20020604090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc"}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2028106962-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106962-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106962-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106962-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-06-04T21:00:00.000Z", "start_datetime": "2002-06-04T21:00:00.000Z", "end_datetime": "2002-06-05T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"metadata": {"href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106862-POCLOUD.xml", "type": "application/xml"}, "opendap": {"title": "OPeNDAP request URL", "href": "https://opendap.earthdata.nasa.gov/providers/POCLOUD/collections/GHRSST%20Level%204%20MUR%20Global%20Foundation%20Sea%20Surface%20Temperature%20Analysis%20(v4.1)/granules/20020605090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1"}, "data": {"title": "Download 20020605090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc", "href": "https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/MUR-JPL-L4-GLOB-v4.1/20020605090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc"}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2028106862-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106862-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106862-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106862-POCLOUD.umm_json"}]}]}' + granule_json = json.loads(granule_json) + os.environ[Constants.EDL_USERNAME] = '/unity/uds/user/wphyo/edl_username' + os.environ[Constants.EDL_PASSWORD] = '/unity/uds/user/wphyo/edl_dwssap' + os.environ[Constants.EDL_PASSWORD_TYPE] = Constants.PARAM_STORE + os.environ[Constants.EDL_BASE_URL] = 'urs.earthdata.nasa.gov' + os.environ['STAC_JSON'] = json.dumps(granule_json) + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'DAAC' + os.environ['DOWNLOADING_KEYS'] = 'metadata' + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + os.environ['DOWNLOAD_DIR'] = tmp_dir_name + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) + 2, len(glob(os.path.join(tmp_dir_name, '*'))), + f'downloaded file does not match') + error_file = os.path.join(tmp_dir_name, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([k['assets']['metadata']['href'] for k in download_result]) + for each_granule in zip(granule_json['features'], download_result): + remote_filename = os.path.basename(each_granule[0]['assets']['metadata']['href']) + self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, + f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_02_download__daac_from_url(self): + os.environ['STAC_JSON'] = 'https://cmr.earthdata.nasa.gov/search/granules.stac?collection_concept_id=C2011289787-GES_DISC&page_num=1&page_size=1' + granule_json = requests.get(os.environ['STAC_JSON']) + granule_json = json.loads(granule_json.text) + os.environ[Constants.EDL_USERNAME] = '/unity/uds/user/wphyo/edl_username' + os.environ[Constants.EDL_PASSWORD] = '/unity/uds/user/wphyo/edl_dwssap' + os.environ[Constants.EDL_PASSWORD_TYPE] = Constants.PARAM_STORE + os.environ[Constants.EDL_BASE_URL] = 'urs.earthdata.nasa.gov' + # os.environ['STAC_JSON'] = json.dumps(granule_json) + + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'DAAC' + os.environ['DOWNLOADING_KEYS'] = 'data,metadata' + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + os.environ['DOWNLOAD_DIR'] = tmp_dir_name + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + print(download_result) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) * len(os.environ['DOWNLOADING_KEYS'].split(',')) + 2, len(glob(os.path.join(tmp_dir_name, '*'))), + f'downloaded file does not match: {glob(os.path.join(tmp_dir_name, "*"))} v. {download_result["features"]}') + error_file = os.path.join(tmp_dir_name, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + print(download_result) + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([k['assets']['data']['href'] for k in download_result]) + for each_granule in zip(granule_json['features'], download_result): + remote_filename = os.path.basename(each_granule[0]['assets']['data']['href']) + self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, + f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_02_download__daac__from_file(self): + granule_json = '{"type": "FeatureCollection", "stac_version": "1.0.0", "numberMatched": 3413, "numberReturned": 23, "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/granules.stac?collection_concept_id=C1996881146-POCLOUD&page_size=23&temporal%5B%5D=2002-06-01T12%3A06%3A00.000Z%2C2011-10-04T06%3A51%3A45.000Z&page_num=1"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "next", "body": {"collection_concept_id": "C1996881146-POCLOUD", "page_num": "2", "page_size": "23", "temporal": ["2002-06-01T12:06:00.000Z,2011-10-04T06:51:45.000Z"], "temporal[]": "2002-06-01T12:06:00.000Z,2011-10-04T06:51:45.000Z"}, "method": "POST", "merge": true, "href": "https://cmr.earthdata.nasa.gov:443/search/granules.stac"}], "context": {"returned": 23, "limit": 1000000, "matched": 3413}, "features": [{"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"metadata": {"href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.xml", "type": "application/xml"}, "opendap": {"title": "OPeNDAP request URL", "href": "https://opendap.earthdata.nasa.gov/providers/POCLOUD/collections/GHRSST%20Level%204%20MUR%20Global%20Foundation%20Sea%20Surface%20Temperature%20Analysis%20(v4.1)/granules/20020601090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1"}, "data": {"title": "Download 20020601090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc", "href": "https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/MUR-JPL-L4-GLOB-v4.1/20020601090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc"}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-06-01T21:00:00.000Z", "start_datetime": "2002-06-01T21:00:00.000Z", "end_datetime": "2002-06-02T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"metadata": {"href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106835-POCLOUD.xml", "type": "application/xml"}, "opendap": {"title": "OPeNDAP request URL", "href": "https://opendap.earthdata.nasa.gov/providers/POCLOUD/collections/GHRSST%20Level%204%20MUR%20Global%20Foundation%20Sea%20Surface%20Temperature%20Analysis%20(v4.1)/granules/20020602090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1"}, "data": {"title": "Download 20020602090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc", "href": "https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/MUR-JPL-L4-GLOB-v4.1/20020602090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc"}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2028106835-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106835-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106835-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106835-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-06-02T21:00:00.000Z", "start_datetime": "2002-06-02T21:00:00.000Z", "end_datetime": "2002-06-03T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"metadata": {"href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106890-POCLOUD.xml", "type": "application/xml"}, "opendap": {"title": "OPeNDAP request URL", "href": "https://opendap.earthdata.nasa.gov/providers/POCLOUD/collections/GHRSST%20Level%204%20MUR%20Global%20Foundation%20Sea%20Surface%20Temperature%20Analysis%20(v4.1)/granules/20020603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1"}, "data": {"title": "Download 20020603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc", "href": "https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/MUR-JPL-L4-GLOB-v4.1/20020603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc"}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2028106890-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106890-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106890-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106890-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-06-03T21:00:00.000Z", "start_datetime": "2002-06-03T21:00:00.000Z", "end_datetime": "2002-06-04T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"metadata": {"href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106962-POCLOUD.xml", "type": "application/xml"}, "opendap": {"title": "OPeNDAP request URL", "href": "https://opendap.earthdata.nasa.gov/providers/POCLOUD/collections/GHRSST%20Level%204%20MUR%20Global%20Foundation%20Sea%20Surface%20Temperature%20Analysis%20(v4.1)/granules/20020604090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1"}, "data": {"title": "Download 20020604090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc", "href": "https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/MUR-JPL-L4-GLOB-v4.1/20020604090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc"}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2028106962-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106962-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106962-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106962-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-06-04T21:00:00.000Z", "start_datetime": "2002-06-04T21:00:00.000Z", "end_datetime": "2002-06-05T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"metadata": {"href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106862-POCLOUD.xml", "type": "application/xml"}, "opendap": {"title": "OPeNDAP request URL", "href": "https://opendap.earthdata.nasa.gov/providers/POCLOUD/collections/GHRSST%20Level%204%20MUR%20Global%20Foundation%20Sea%20Surface%20Temperature%20Analysis%20(v4.1)/granules/20020605090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1"}, "data": {"title": "Download 20020605090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc", "href": "https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/MUR-JPL-L4-GLOB-v4.1/20020605090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc"}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2028106862-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106862-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106862-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106862-POCLOUD.umm_json"}]}]}' + granule_json = json.loads(granule_json) + os.environ[Constants.EDL_USERNAME] = '/unity/uds/user/wphyo/edl_username' + os.environ[Constants.EDL_PASSWORD] = '/unity/uds/user/wphyo/edl_dwssap' + os.environ[Constants.EDL_PASSWORD_TYPE] = Constants.PARAM_STORE + os.environ[Constants.EDL_BASE_URL] = 'urs.earthdata.nasa.gov' + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'DAAC' + os.environ['DOWNLOADING_KEYS'] = 'metadata' + + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + granule_json_file = os.path.join(tmp_dir_name, 'input_file.json') + downloading_dir = os.path.join(tmp_dir_name, 'downloading_dir') + FileUtils.mk_dir_p(downloading_dir) + FileUtils.write_json(granule_json_file, granule_json) + os.environ['STAC_JSON'] = granule_json_file + os.environ['DOWNLOAD_DIR'] = downloading_dir + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) + 1, len(glob(os.path.join(downloading_dir, '*'))), + f'downloaded file does not match') + error_file = os.path.join(downloading_dir, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([k['assets']['metadata']['href'] for k in download_result]) + for each_granule in zip(granule_json['features'], download_result): + remote_filename = os.path.basename(each_granule[0]['assets']['metadata']['href']) + self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, + f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_02_download__daac_502__from_file(self): + granule_json = '''{ + "type": "FeatureCollection", + "stac_version": "1.0.0", + "numberMatched": 242, + "numberReturned": 10, + "links": [ + { + "rel": "self", + "href": "https://cmr.earthdata.nasa.gov:443/search/granules.stac?collection_concept_id=C2011289787-GES_DISC&page_size=10&temporal%5B%5D=2021-02-01T00%3A00%3A00Z%2C2021-02-02T00%3A00%3A00Z&page_num=1" + }, + { + "rel": "root", + "href": "https://cmr.earthdata.nasa.gov:443/search/" + }, + { + "rel": "next", + "body": { + "collection_concept_id": "C2011289787-GES_DISC", + "page_num": "2", + "page_size": "10", + "temporal": [ + "2021-02-01T00:00:00Z,2021-02-02T00:00:00Z" + ], + "temporal[]": "2021-02-01T00:00:00Z,2021-02-02T00:00:00Z" + }, + "method": "POST", + "merge": true, + "href": "https://cmr.earthdata.nasa.gov:443/search/granules.stac" + } + ], + "context": { + "returned": 10, + "limit": 1000000, + "matched": 242 + }, + "features": [ + { + "properties": { + "datetime": "2021-01-31T23:54:00.000Z", + "start_datetime": "2021-01-31T23:54:00.000Z", + "end_datetime": "2021-02-01T00:00:00.000Z" + }, + "bbox": [ + -164.22, + -59.19, + -127.42, + -34.4 + ], + "assets": { + "metadata": { + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327589-GES_DISC.xml", + "type": "application/xml" + }, + "opendap": { + "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)", + "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2021/031/SNDR.SS1330.CHIRP.20210131T2354.m06.g240.L1_J1.std.v02_48.G.200408101645.nc", + "type": "application/x-netcdf" + }, + "data": { + "title": "Download SNDR.SS1330.CHIRP.20210131T2354.m06.g240.L1_J1.std.v02_48.G.200408101645.nc", + "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2021/031/SNDR.SS1330.CHIRP.20210131T2354.m06.g240.L1_J1.std.v02_48.G.200408101645.nc" + } + }, + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -127.42, + -53.49 + ], + [ + -140.59, + -34.4 + ], + [ + -164.22, + -38.4 + ], + [ + -161.84, + -59.19 + ], + [ + -127.42, + -53.49 + ] + ] + ] + }, + "stac_extensions": [], + "id": "G2031327589-GES_DISC", + "stac_version": "1.0.0", + "collection": "C2011289787-GES_DISC", + "links": [ + { + "rel": "self", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327589-GES_DISC.stac" + }, + { + "rel": "parent", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "collection", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "root", + "href": "https://cmr.earthdata.nasa.gov:443/search/" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327589-GES_DISC.json" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327589-GES_DISC.umm_json" + } + ] + }, + { + "properties": { + "datetime": "2021-02-01T00:00:00.000Z", + "start_datetime": "2021-02-01T00:00:00.000Z", + "end_datetime": "2021-02-01T00:06:00.000Z" + }, + "bbox": [ + -167.61, + -38.28, + -140.62, + -14.06 + ], + "assets": { + "metadata": { + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327600-GES_DISC.xml", + "type": "application/xml" + }, + "opendap": { + "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)", + "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0000.m06.g001.L1_J1.std.v02_48.G.200408101657.nc", + "type": "application/x-netcdf" + }, + "data": { + "title": "Download SNDR.SS1330.CHIRP.20210201T0000.m06.g001.L1_J1.std.v02_48.G.200408101657.nc", + "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0000.m06.g001.L1_J1.std.v02_48.G.200408101657.nc" + } + }, + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -140.62, + -34.28 + ], + [ + -147.98, + -14.06 + ], + [ + -167.61, + -17.44 + ], + [ + -164.31, + -38.28 + ], + [ + -140.62, + -34.28 + ] + ] + ] + }, + "stac_extensions": [], + "id": "G2031327600-GES_DISC", + "stac_version": "1.0.0", + "collection": "C2011289787-GES_DISC", + "links": [ + { + "rel": "self", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327600-GES_DISC.stac" + }, + { + "rel": "parent", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "collection", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "root", + "href": "https://cmr.earthdata.nasa.gov:443/search/" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327600-GES_DISC.json" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327600-GES_DISC.umm_json" + } + ] + }, + { + "properties": { + "datetime": "2021-02-01T00:06:00.000Z", + "start_datetime": "2021-02-01T00:06:00.000Z", + "end_datetime": "2021-02-01T00:12:00.000Z" + }, + "bbox": [ + -171.93, + -17.31, + -148.0, + 6.74 + ], + "assets": { + "metadata": { + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327624-GES_DISC.xml", + "type": "application/xml" + }, + "opendap": { + "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)", + "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0006.m06.g002.L1_J1.std.v02_48.G.200408101655.nc", + "type": "application/x-netcdf" + }, + "data": { + "title": "Download SNDR.SS1330.CHIRP.20210201T0006.m06.g002.L1_J1.std.v02_48.G.200408101655.nc", + "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0006.m06.g002.L1_J1.std.v02_48.G.200408101655.nc" + } + }, + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -148.0, + -13.93 + ], + [ + -153.06, + 6.74 + ], + [ + -171.93, + 3.48 + ], + [ + -167.7, + -17.31 + ], + [ + -148.0, + -13.93 + ] + ] + ] + }, + "stac_extensions": [], + "id": "G2031327624-GES_DISC", + "stac_version": "1.0.0", + "collection": "C2011289787-GES_DISC", + "links": [ + { + "rel": "self", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327624-GES_DISC.stac" + }, + { + "rel": "parent", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "collection", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "root", + "href": "https://cmr.earthdata.nasa.gov:443/search/" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327624-GES_DISC.json" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327624-GES_DISC.umm_json" + } + ] + }, + { + "properties": { + "datetime": "2021-02-01T00:12:00.000Z", + "start_datetime": "2021-02-01T00:12:00.000Z", + "end_datetime": "2021-02-01T00:18:00.000Z" + }, + "bbox": [ + -177.83, + 3.6, + -153.07, + 27.71 + ], + "assets": { + "metadata": { + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327611-GES_DISC.xml", + "type": "application/xml" + }, + "opendap": { + "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)", + "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0012.m06.g003.L1_J1.std.v02_48.G.200408101705.nc", + "type": "application/x-netcdf" + }, + "data": { + "title": "Download SNDR.SS1330.CHIRP.20210201T0012.m06.g003.L1_J1.std.v02_48.G.200408101705.nc", + "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0012.m06.g003.L1_J1.std.v02_48.G.200408101705.nc" + } + }, + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -153.07, + 6.87 + ], + [ + -156.97, + 27.71 + ], + [ + -177.83, + 24.15 + ], + [ + -172.02, + 3.6 + ], + [ + -153.07, + 6.87 + ] + ] + ] + }, + "stac_extensions": [], + "id": "G2031327611-GES_DISC", + "stac_version": "1.0.0", + "collection": "C2011289787-GES_DISC", + "links": [ + { + "rel": "self", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327611-GES_DISC.stac" + }, + { + "rel": "parent", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "collection", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "root", + "href": "https://cmr.earthdata.nasa.gov:443/search/" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327611-GES_DISC.json" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327611-GES_DISC.umm_json" + } + ] + }, + { + "properties": { + "datetime": "2021-02-01T00:18:00.000Z", + "start_datetime": "2021-02-01T00:18:00.000Z", + "end_datetime": "2021-02-01T00:24:00.000Z" + }, + "bbox": [ + 172.75, + 24.27, + -156.97, + 48.71 + ], + "assets": { + "metadata": { + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327811-GES_DISC.xml", + "type": "application/xml" + }, + "opendap": { + "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)", + "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0018.m06.g004.L1_J1.std.v02_48.G.200408101716.nc", + "type": "application/x-netcdf" + }, + "data": { + "title": "Download SNDR.SS1330.CHIRP.20210201T0018.m06.g004.L1_J1.std.v02_48.G.200408101716.nc", + "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0018.m06.g004.L1_J1.std.v02_48.G.200408101716.nc" + } + }, + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -156.97, + 27.84 + ], + [ + -160.07, + 48.71 + ], + [ + 172.75, + 44.19 + ], + [ + -177.94, + 24.27 + ], + [ + -156.97, + 27.84 + ] + ] + ] + }, + "stac_extensions": [], + "id": "G2031327811-GES_DISC", + "stac_version": "1.0.0", + "collection": "C2011289787-GES_DISC", + "links": [ + { + "rel": "self", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327811-GES_DISC.stac" + }, + { + "rel": "parent", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "collection", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "root", + "href": "https://cmr.earthdata.nasa.gov:443/search/" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327811-GES_DISC.json" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327811-GES_DISC.umm_json" + } + ] + }, + { + "properties": { + "datetime": "2021-02-01T00:24:00.000Z", + "start_datetime": "2021-02-01T00:24:00.000Z", + "end_datetime": "2021-02-01T00:30:00.000Z" + }, + "bbox": [ + 152.97, + 44.29, + -160.05, + 69.67 + ], + "assets": { + "metadata": { + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327763-GES_DISC.xml", + "type": "application/xml" + }, + "opendap": { + "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)", + "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0024.m06.g005.L1_J1.std.v02_48.G.200408101749.nc", + "type": "application/x-netcdf" + }, + "data": { + "title": "Download SNDR.SS1330.CHIRP.20210201T0024.m06.g005.L1_J1.std.v02_48.G.200408101749.nc", + "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0024.m06.g005.L1_J1.std.v02_48.G.200408101749.nc" + } + }, + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -160.05, + 48.84 + ], + [ + -161.92, + 69.67 + ], + [ + 152.97, + 62.28 + ], + [ + 172.6, + 44.29 + ], + [ + -160.05, + 48.84 + ] + ] + ] + }, + "stac_extensions": [], + "id": "G2031327763-GES_DISC", + "stac_version": "1.0.0", + "collection": "C2011289787-GES_DISC", + "links": [ + { + "rel": "self", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327763-GES_DISC.stac" + }, + { + "rel": "parent", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "collection", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "root", + "href": "https://cmr.earthdata.nasa.gov:443/search/" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327763-GES_DISC.json" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327763-GES_DISC.umm_json" + } + ] + }, + { + "properties": { + "datetime": "2021-02-01T00:30:00.000Z", + "start_datetime": "2021-02-01T00:30:00.000Z", + "end_datetime": "2021-02-01T00:36:00.000Z" + }, + "bbox": [ + -180.0, + 62.35, + 180.0, + 90.0 + ], + "assets": { + "metadata": { + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327782-GES_DISC.xml", + "type": "application/xml" + }, + "opendap": { + "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)", + "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0030.m06.g006.L1_J1.std.v02_48.G.200408101747.nc", + "type": "application/x-netcdf" + }, + "data": { + "title": "Download SNDR.SS1330.CHIRP.20210201T0030.m06.g006.L1_J1.std.v02_48.G.200408101747.nc", + "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0030.m06.g006.L1_J1.std.v02_48.G.200408101747.nc" + } + }, + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -161.83, + 69.79 + ], + [ + -45.22, + 88.83 + ], + [ + 101.33, + 71.69 + ], + [ + 152.69, + 62.35 + ], + [ + -161.83, + 69.79 + ] + ] + ] + }, + "stac_extensions": [], + "id": "G2031327782-GES_DISC", + "stac_version": "1.0.0", + "collection": "C2011289787-GES_DISC", + "links": [ + { + "rel": "self", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327782-GES_DISC.stac" + }, + { + "rel": "parent", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "collection", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "root", + "href": "https://cmr.earthdata.nasa.gov:443/search/" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327782-GES_DISC.json" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327782-GES_DISC.umm_json" + } + ] + }, + { + "properties": { + "datetime": "2021-02-01T00:36:00.000Z", + "start_datetime": "2021-02-01T00:36:00.000Z", + "end_datetime": "2021-02-01T00:42:00.000Z" + }, + "bbox": [ + -41.63, + 61.83, + 100.91, + 89.26695841076085 + ], + "assets": { + "metadata": { + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327806-GES_DISC.xml", + "type": "application/xml" + }, + "opendap": { + "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)", + "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0036.m06.g007.L1_J1.std.v02_48.G.200408101747.nc", + "type": "application/x-netcdf" + }, + "data": { + "title": "Download SNDR.SS1330.CHIRP.20210201T0036.m06.g007.L1_J1.std.v02_48.G.200408101747.nc", + "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0036.m06.g007.L1_J1.std.v02_48.G.200408101747.nc" + } + }, + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -41.63, + 88.73 + ], + [ + 6.42, + 68.41 + ], + [ + 50.76, + 61.83 + ], + [ + 100.91, + 71.65 + ], + [ + -41.63, + 88.73 + ] + ] + ] + }, + "stac_extensions": [], + "id": "G2031327806-GES_DISC", + "stac_version": "1.0.0", + "collection": "C2011289787-GES_DISC", + "links": [ + { + "rel": "self", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327806-GES_DISC.stac" + }, + { + "rel": "parent", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "collection", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "root", + "href": "https://cmr.earthdata.nasa.gov:443/search/" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327806-GES_DISC.json" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327806-GES_DISC.umm_json" + } + ] + }, + { + "properties": { + "datetime": "2021-02-01T00:42:00.000Z", + "start_datetime": "2021-02-01T00:42:00.000Z", + "end_datetime": "2021-02-01T00:48:00.000Z" + }, + "bbox": [ + 4.45, + 43.65, + 50.62, + 68.30202151196218 + ], + "assets": { + "metadata": { + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327773-GES_DISC.xml", + "type": "application/xml" + }, + "opendap": { + "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)", + "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0042.m06.g008.L1_J1.std.v02_48.G.200408101807.nc", + "type": "application/x-netcdf" + }, + "data": { + "title": "Download SNDR.SS1330.CHIRP.20210201T0042.m06.g008.L1_J1.std.v02_48.G.200408101807.nc", + "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0042.m06.g008.L1_J1.std.v02_48.G.200408101807.nc" + } + }, + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 6.29, + 68.29 + ], + [ + 4.45, + 47.46 + ], + [ + 31.46, + 43.65 + ], + [ + 50.62, + 61.71 + ], + [ + 6.29, + 68.29 + ] + ] + ] + }, + "stac_extensions": [], + "id": "G2031327773-GES_DISC", + "stac_version": "1.0.0", + "collection": "C2011289787-GES_DISC", + "links": [ + { + "rel": "self", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327773-GES_DISC.stac" + }, + { + "rel": "parent", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "collection", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "root", + "href": "https://cmr.earthdata.nasa.gov:443/search/" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327773-GES_DISC.json" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327773-GES_DISC.umm_json" + } + ] + }, + { + "properties": { + "datetime": "2021-02-01T00:48:00.000Z", + "start_datetime": "2021-02-01T00:48:00.000Z", + "end_datetime": "2021-02-01T00:54:00.000Z" + }, + "bbox": [ + 1.29, + 23.6, + 31.41, + 47.33 + ], + "assets": { + "metadata": { + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327751-GES_DISC.xml", + "type": "application/xml" + }, + "opendap": { + "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)", + "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0048.m06.g009.L1_J1.std.v02_48.G.200408101736.nc", + "type": "application/x-netcdf" + }, + "data": { + "title": "Download SNDR.SS1330.CHIRP.20210201T0048.m06.g009.L1_J1.std.v02_48.G.200408101736.nc", + "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0048.m06.g009.L1_J1.std.v02_48.G.200408101736.nc" + } + }, + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.35, + 47.33 + ], + [ + 1.29, + 26.46 + ], + [ + 22.18, + 23.6 + ], + [ + 31.41, + 43.52 + ], + [ + 4.35, + 47.33 + ] + ] + ] + }, + "stac_extensions": [], + "id": "G2031327751-GES_DISC", + "stac_version": "1.0.0", + "collection": "C2011289787-GES_DISC", + "links": [ + { + "rel": "self", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327751-GES_DISC.stac" + }, + { + "rel": "parent", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "collection", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "root", + "href": "https://cmr.earthdata.nasa.gov:443/search/" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327751-GES_DISC.json" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327751-GES_DISC.umm_json" + } + ] + } + ] + }''' + granule_json = json.loads(granule_json) + os.environ[Constants.EDL_USERNAME] = '/unity/uds/user/wphyo/edl_username' + os.environ[Constants.EDL_PASSWORD] = '/unity/uds/user/wphyo/edl_dwssap' + os.environ[Constants.EDL_PASSWORD_TYPE] = Constants.PARAM_STORE + os.environ[Constants.EDL_BASE_URL] = 'urs.earthdata.nasa.gov' + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'DAAC' + os.environ['DOWNLOADING_KEYS'] = 'data' + + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + granule_json_file = os.path.join(tmp_dir_name, 'input_file.json') + downloading_dir = os.path.join(tmp_dir_name, 'downloading_dir') + FileUtils.mk_dir_p(downloading_dir) + FileUtils.write_json(granule_json_file, granule_json) + os.environ['STAC_JSON'] = granule_json_file + os.environ['DOWNLOAD_DIR'] = downloading_dir + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + error_file = os.path.join(downloading_dir, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + self.assertEqual(len(download_result['features']) + 1, len(glob(os.path.join(downloading_dir, '*'))), + f'downloaded file does not match') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_02_download__daac_error(self): # TODO update this later + granule_json = '{"type": "FeatureCollection", "stac_version": "1.0.0", "numberMatched": 3413, "numberReturned": 23, "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/granules.stac?collection_concept_id=C1996881146-POCLOUD&page_size=23&temporal%5B%5D=2002-06-01T12%3A06%3A00.000Z%2C2011-10-04T06%3A51%3A45.000Z&page_num=1"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "next", "body": {"collection_concept_id": "C1996881146-POCLOUD", "page_num": "2", "page_size": "23", "temporal": ["2002-06-01T12:06:00.000Z,2011-10-04T06:51:45.000Z"], "temporal[]": "2002-06-01T12:06:00.000Z,2011-10-04T06:51:45.000Z"}, "method": "POST", "merge": true, "href": "https://cmr.earthdata.nasa.gov:443/search/granules.stac"}], "context": {"returned": 23, "limit": 1000000, "matched": 3413}, "features": [{"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/152/20020601161248-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00414.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/152/20020601172624-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00415.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/152/20020601190536-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00416.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/152/20020601204344-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00417.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/152/20020601222152-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00418.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602000000-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00419.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602013912-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00420.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602031720-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00421.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602045528-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00422.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602063440-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00423.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602081248-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00424.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602095056-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00425.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602112904-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00426.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602130816-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00427.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602144624-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00428.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602162432-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00429.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602180240-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00430.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602194152-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00431.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602212000-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00432.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602225808-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00433.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/154/20020603003616-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00434.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/154/20020603021528-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00435.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/154/20020603035336-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00436.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}]}' + # granule_json = [{"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/152/20020601161248-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00414.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/152/20020601172624-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00415.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/152/20020601190536-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00416.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/152/20020601204344-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00417.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/152/20020601222152-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00418.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602000000-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00419.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602013912-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00420.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602031720-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00421.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602045528-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00422.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602063440-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00423.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602081248-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00424.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602095056-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00425.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602112904-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00426.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602130816-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00427.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602144624-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00428.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602162432-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00429.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602180240-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00430.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602194152-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00431.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602212000-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00432.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602225808-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00433.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/154/20020603003616-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00434.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/154/20020603021528-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00435.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/154/20020603035336-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00436.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}] + granule_json = json.loads(granule_json) + os.environ[Constants.EDL_USERNAME] = '/unity/uds/user/wphyo/edl_username' + os.environ[Constants.EDL_PASSWORD] = '/unity/uds/user/wphyo/edl_dwssap' + os.environ[Constants.EDL_PASSWORD_TYPE] = Constants.PARAM_STORE + os.environ[Constants.EDL_BASE_URL] = 'urs.earthdata.nasa.gov' + os.environ['STAC_JSON'] = json.dumps(granule_json) + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'DAAC' + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + # TODO this is downloading a login page HTML + os.environ['DOWNLOAD_DIR'] = tmp_dir_name + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + print(glob(os.path.join(tmp_dir_name, '*'))) + self.assertEqual(len(download_result['features']) + 2, len(glob(os.path.join(tmp_dir_name, '*'))), + f'downloaded file does not match') + error_file = os.path.join(tmp_dir_name, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_02_download__from_dapa_url(self): + dapa_url = 'https://1gp9st60gd.execute-api.us-west-2.amazonaws.com/dev/sbx-uds-2-dapa/collections/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2401310000/items?limit=37&offset=0&datetime=1990-01-14T08:00:00Z/2024-01-14T11:59:59Z' + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['API_PREFIX'] = 'sbx-uds-2-dapa' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://1gp9st60gd.execute-api.us-west-2.amazonaws.com/dev' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['STAC_AUTH_TYPE'] = 'UNITY' + + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'S3' + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + downloading_dir = os.path.join(tmp_dir_name, 'downloading_dir') + FileUtils.mk_dir_p(downloading_dir) + os.environ['STAC_JSON'] = dapa_url + os.environ['DOWNLOAD_DIR'] = downloading_dir + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) * 4 + 1, len(glob(os.path.join(downloading_dir, '*'))), + f'downloaded file does not match: {download_result["features"]}') + error_file = os.path.join(downloading_dir, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + print(download_result) + expected_downloaded_feature = [{'type': 'Feature', 'stac_version': '1.0.0', + 'id': 'URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2401310000:test_file05', + 'properties': {'tag': '#sample', 'c_data1': [1, 10, 100, 1000], + 'c_data2': [False, True, True, False, True], + 'c_data3': ['Bellman Ford'], + 'datetime': '2024-01-31T22:34:06.556000Z', + 'start_datetime': '2016-01-31T18:00:00.009000Z', + 'end_datetime': '2016-01-31T19:59:59.991000Z', + 'created': '1970-01-01T00:00:00Z', + 'updated': '2024-01-31T22:34:49.583000Z', + 'status': 'completed', 'provider': 'unity'}, + 'geometry': {'type': 'Point', 'coordinates': [0.0, 0.0]}, + 'links': [{'rel': 'collection', 'href': '.'}], 'assets': { + 'test_file05.cmr.xml': {'href': './test_file05.cmr.xml', 'title': 'test_file05.cmr.xml', + 'description': 'size=1768;checksumType=md5;checksum=29d1b69df5587d7ee0a945250adfd16f;', + 'roles': ['metadata']}, + 'test_file05.nc.stac.json': {'href': './test_file05.nc.stac.json', + 'title': 'test_file05.nc.stac.json', + 'description': 'size=-1;checksumType=md5;checksum=unknown;', + 'roles': ['metadata']}, + 'test_file05.nc.cas': {'href': './test_file05.nc.cas', 'title': 'test_file05.nc.cas', + 'description': 'size=-1;checksumType=md5;checksum=unknown;', + 'roles': ['metadata']}, + 'test_file05.data.stac.json': {'href': './test_file05.data.stac.json', + 'title': 'test_file05.data.stac.json', + 'description': 'size=-1;checksumType=md5;checksum=unknown;', + 'roles': ['data']}}, 'bbox': [-180.0, -90.0, 180.0, 90.0], + 'stac_extensions': [], + 'collection': 'URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2401310000'}] + + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([k['assets']['test_file05.data.stac.json']['href'] for k in download_result]) + for each_granule in zip(expected_downloaded_feature, download_result): + remote_filename = os.path.basename(each_granule[0]['assets']['test_file05.data.stac.json']['href']) + self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, + f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_02_download__from_file(self): + granule_json = '{"numberMatched": {"total_size": 5}, "numberReturned": 6, "stac_version": "1.0.0", "type": "FeatureCollection", ' \ + '"links": [{"rel": "self", "href": "https://1gp9st60gd.execute-api.us-west-2.amazonaws.com/dev/sbx-uds-2-dapa/collections/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/items/?limit=10"}, {"rel": "root", "href": "https://1gp9st60gd.execute-api.us-west-2.amazonaws.com/dev"}], ' \ + '"features": [' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:21:04.234000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:21:47.477000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.cmr.xml", "title": "test_file01.cmr.xml", "description": "size=1768;checksumType=md5;checksum=4d1935f25f3b508ca1e1a0368eeda10c;"}, "metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.nc.stac.json", "title": "test_file01.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.nc.cas", "title": "test_file01.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.nc", "title": "test_file01.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:44:56.784000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:45:40.118000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.nc.stac.json", "title": "test_file02.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.cmr.xml", "title": "test_file02.cmr.xml", "description": "size=1768;checksumType=md5;checksum=88b82e1824d51713d0bc897d970f3b0a;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.nc.cas", "title": "test_file02.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.nc", "title": "test_file02.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:54:01.078000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:54:42.272000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.cmr.xml", "title": "test_file03.cmr.xml", "description": "size=1768;checksumType=md5;checksum=cd84e6a6138b3aad77d013ca4fb3ded4;"}, "metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.nc.stac.json", "title": "test_file03.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.nc.cas", "title": "test_file03.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.nc", "title": "test_file03.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:54:33.221000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:55:12.198000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.cmr.xml", "title": "test_file04.cmr.xml", "description": "size=1768;checksumType=md5;checksum=47574084df6d14bbe9df60a2d40617ef;"}, "metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.nc.stac.json", "title": "test_file04.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.nc.cas", "title": "test_file04.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.nc", "title": "test_file04.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:58:31.381000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:58:42.027000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.nc.stac.json", "title": "test_file05.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.cmr.xml", "title": "test_file05.cmr.xml", "description": "size=1768;checksumType=md5;checksum=03e639becc6c74ad5128ccd438fc35ae;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.nc.cas", "title": "test_file05.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.nc", "title": "test_file05.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"} ' \ + ']}' + granule_json = json.loads(granule_json) + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'S3' + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + granule_json_file = os.path.join(tmp_dir_name, 'input_file.json') + downloading_dir = os.path.join(tmp_dir_name, 'downloading_dir') + FileUtils.mk_dir_p(downloading_dir) + FileUtils.write_json(granule_json_file, granule_json) + os.environ['STAC_JSON'] = granule_json_file + os.environ['DOWNLOAD_DIR'] = downloading_dir + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) * 4 + 1, len(glob(os.path.join(downloading_dir, '*'))), + f'downloaded file does not match: {download_result["features"]}') + error_file = os.path.join(downloading_dir, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([k['assets']['data']['href'] for k in download_result]) + print(downloaded_file_hrefs) + for each_granule in zip(granule_json['features'], download_result): + remote_filename = os.path.basename(each_granule[0]['assets']['data']['href']) + self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, + f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.get_size(os.path.join(downloading_dir, remote_filename)) > 0, f'empty file: {remote_filename}') + print() + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_02_download__from_file_large_github_data_file(self): + granule_json = ''' + { + "numberMatched": 20, + "numberReturned": 20, + "stac_version": "1.0.0", + "type": "FeatureCollection", + "links": [ + { + "rel": "self", + "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=0" + }, + { + "rel": "root", + "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com" + }, + { + "rel": "next", + "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=100" + }, + { + "rel": "prev", + "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=0" + } + ], + "features": [ + { + "type": "Feature", + "stac_version": "1.0.0", + "id": "SNDR.SNPP.ATMS.L1A.nominal2.01", + "properties": { + "start_datetime": "2016-01-14T09:54:00Z", + "end_datetime": "2016-01-14T10:00:00Z", + "created": "2020-12-14T13:50:00Z", + "updated": "2022-08-15T06:26:39.830000Z", + "datetime": "2022-08-15T06:26:37.029000Z" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 0.0, + 0.0 + ] + }, + "links": [ + { + "rel": "collection", + "href": "." + } + ], + "assets": { + "data": { + "href": "https://raw.githubusercontent.com/unity-sds/unity-tutorial-application/main/test/stage_in/SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc", + "title": "SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc", + "description": "SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc" + } + }, + "bbox": [ + 0.0, + 0.0, + 0.0, + 0.0 + ], + "stac_extensions": [], + "collection": "SNDR_SNPP_ATMS_L1A___1" + }, + { + "type": "Feature", + "stac_version": "1.0.0", + "id": "SNDR.SNPP.ATMS.L1A.nominal2.08", + "properties": { + "start_datetime": "2016-01-14T10:36:00Z", + "end_datetime": "2016-01-14T10:42:00Z", + "created": "2020-12-14T13:50:00Z", + "updated": "2022-08-15T06:26:26.078000Z", + "datetime": "2022-08-15T06:26:19.333000Z" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 0.0, + 0.0 + ] + }, + "links": [ + { + "rel": "collection", + "href": "." + } + ], + "assets": { + "data": { + "href": "https://raw.githubusercontent.com/unity-sds/unity-tutorial-application/main/test/stage_in/SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc", + "title": "SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc", + "description": "SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc" + } + }, + "bbox": [ + 0.0, + 0.0, + 0.0, + 0.0 + ], + "stac_extensions": [], + "collection": "SNDR_SNPP_ATMS_L1A___1" + } + ] + } + ''' + granule_json = json.loads(granule_json) + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'HTTP' + with tempfile.TemporaryDirectory() as tmp_dir_name: + # tmp_dir_name = '/tmp/unity-ds' + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + granule_json_file = os.path.join(tmp_dir_name, 'input_file.json') + downloading_dir = os.path.join(tmp_dir_name, 'downloading_dir') + FileUtils.mk_dir_p(downloading_dir) + FileUtils.write_json(granule_json_file, granule_json) + os.environ['STAC_JSON'] = granule_json_file + os.environ['DOWNLOAD_DIR'] = downloading_dir + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) + 1, len(glob(os.path.join(downloading_dir, '*'))), + f'downloaded file does not match: {download_result["features"]}') + error_file = os.path.join(downloading_dir, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([k['assets']['data']['href'] for k in download_result]) + for each_granule in zip(granule_json['features'], download_result): + remote_filename = os.path.basename(each_granule[0]['assets']['data']['href']) + self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, + f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.get_size(os.path.join(downloading_dir, remote_filename)) > 40 * 2**20, f'empty file: {remote_filename}') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + @patch('requests.get') + def test_02_download__from_file_with_http(self, mock_requests): + granule_json = '{"numberMatched": {"total_size": 5}, "numberReturned": 6, "stac_version": "1.0.0", "type": "FeatureCollection", ' \ + '"links": [{"rel": "self", "href": "https://1gp9st60gd.execute-api.us-west-2.amazonaws.com/dev/sbx-uds-2-dapa/collections/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/items/?limit=10"}, {"rel": "root", "href": "https://1gp9st60gd.execute-api.us-west-2.amazonaws.com/dev"}], ' \ + '"features": [' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:21:04.234000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:21:47.477000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.cmr.xml", "title": "test_file01.cmr.xml", "description": "size=1768;checksumType=md5;checksum=4d1935f25f3b508ca1e1a0368eeda10c;"}, "metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.nc.stac.json", "title": "test_file01.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.nc.cas", "title": "test_file01.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.nc", "title": "test_file01.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:44:56.784000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:45:40.118000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.nc.stac.json", "title": "test_file02.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.cmr.xml", "title": "test_file02.cmr.xml", "description": "size=1768;checksumType=md5;checksum=88b82e1824d51713d0bc897d970f3b0a;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.nc.cas", "title": "test_file02.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.nc", "title": "test_file02.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:54:01.078000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:54:42.272000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.cmr.xml", "title": "test_file03.cmr.xml", "description": "size=1768;checksumType=md5;checksum=cd84e6a6138b3aad77d013ca4fb3ded4;"}, "metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.nc.stac.json", "title": "test_file03.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.nc.cas", "title": "test_file03.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.nc", "title": "test_file03.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:54:33.221000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:55:12.198000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.cmr.xml", "title": "test_file04.cmr.xml", "description": "size=1768;checksumType=md5;checksum=47574084df6d14bbe9df60a2d40617ef;"}, "metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.nc.stac.json", "title": "test_file04.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.nc.cas", "title": "test_file04.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.nc", "title": "test_file04.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:58:31.381000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:58:42.027000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.nc.stac.json", "title": "test_file05.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.cmr.xml", "title": "test_file05.cmr.xml", "description": "size=1768;checksumType=md5;checksum=03e639becc6c74ad5128ccd438fc35ae;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.nc.cas", "title": "test_file05.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.nc", "title": "test_file05.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"} ' \ + ']}' + # granule_json = json.loads(granule_json) + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.return_value.content.decode.return_value = granule_json + + # mock_response.content = granule_json.encode() + # mock_response.content.return_value = granule_json.encode() + # mock_response.json.return_value = json.loads(granule_json) + # specify the return value of the get() method + mock_requests.return_value.get.return_value = mock_response + mock_requests.return_value.content.decode.return_value = granule_json + + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'S3' + os.environ['STAC_JSON'] = 'https://example.com/get_feature_collection' + + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + downloading_dir = os.path.join(tmp_dir_name, 'downloading_dir') + FileUtils.mk_dir_p(downloading_dir) + os.environ['DOWNLOAD_DIR'] = downloading_dir + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) + 1, len(glob(os.path.join(downloading_dir, '*'))), + f'downloaded file does not match') + error_file = os.path.join(downloading_dir, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([k['assets']['data']['href'] for k in download_result]) + granule_json = json.loads(granule_json) + for each_granule in zip(granule_json['features'], download_result): + remote_filename = os.path.basename(each_granule[0]['assets']['data']['href']) + self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, + f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + @patch('requests.get') + def test_02_download__from_file_with_http(self, mock_requests): + granule_json = '{"numberMatched": {"total_size": 5}, "numberReturned": 6, "stac_version": "1.0.0", "type": "FeatureCollection", ' \ + '"links": [{"rel": "self", "href": "https://1gp9st60gd.execute-api.us-west-2.amazonaws.com/dev/sbx-uds-2-dapa/collections/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/items/?limit=10"}, {"rel": "root", "href": "https://1gp9st60gd.execute-api.us-west-2.amazonaws.com/dev"}], ' \ + '"features": [' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:21:04.234000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:21:47.477000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.cmr.xml", "title": "test_file01.cmr.xml", "description": "size=1768;checksumType=md5;checksum=4d1935f25f3b508ca1e1a0368eeda10c;"}, "metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.nc.stac.json", "title": "test_file01.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.nc.cas", "title": "test_file01.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.nc", "title": "test_file01.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:44:56.784000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:45:40.118000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.nc.stac.json", "title": "test_file02.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.cmr.xml", "title": "test_file02.cmr.xml", "description": "size=1768;checksumType=md5;checksum=88b82e1824d51713d0bc897d970f3b0a;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.nc.cas", "title": "test_file02.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.nc", "title": "test_file02.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:54:01.078000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:54:42.272000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.cmr.xml", "title": "test_file03.cmr.xml", "description": "size=1768;checksumType=md5;checksum=cd84e6a6138b3aad77d013ca4fb3ded4;"}, "metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.nc.stac.json", "title": "test_file03.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.nc.cas", "title": "test_file03.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.nc", "title": "test_file03.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:54:33.221000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:55:12.198000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.cmr.xml", "title": "test_file04.cmr.xml", "description": "size=1768;checksumType=md5;checksum=47574084df6d14bbe9df60a2d40617ef;"}, "metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.nc.stac.json", "title": "test_file04.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.nc.cas", "title": "test_file04.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.nc", "title": "test_file04.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:58:31.381000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:58:42.027000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.nc.stac.json", "title": "test_file05.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.cmr.xml", "title": "test_file05.cmr.xml", "description": "size=1768;checksumType=md5;checksum=03e639becc6c74ad5128ccd438fc35ae;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.nc.cas", "title": "test_file05.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.nc", "title": "test_file05.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"} ' \ + ']}' + # granule_json = json.loads(granule_json) + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.return_value.content.decode.return_value = granule_json + + # mock_response.content = granule_json.encode() + # mock_response.content.return_value = granule_json.encode() + # mock_response.json.return_value = json.loads(granule_json) + # specify the return value of the get() method + mock_requests.return_value.get.return_value = mock_response + mock_requests.return_value.content.decode.return_value = granule_json + + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'S3' + os.environ['STAC_JSON'] = 'https://example.com/get_feature_collection' + + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + downloading_dir = os.path.join(tmp_dir_name, 'downloading_dir') + FileUtils.mk_dir_p(downloading_dir) + os.environ['DOWNLOAD_DIR'] = downloading_dir + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) * 4 + 1, len(glob(os.path.join(downloading_dir, '*'))), + f'downloaded file does not match: {glob(os.path.join(downloading_dir, "*"))}') + error_file = os.path.join(downloading_dir, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([k['assets']['data']['href'] for k in download_result]) + granule_json = json.loads(granule_json) + for each_granule in zip(granule_json['features'], download_result): + remote_filename = os.path.basename(each_granule[0]['assets']['data']['href']) + self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, + f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_02_download__from_file_large(self): + granule_json = FileUtils.read_json('./stage-in.json') + # granule_json['features'] = granule_json['features'][0:5] + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + os.environ[Constants.EDL_USERNAME] = '/unity/uds/user/wphyo/edl_username' + os.environ[Constants.EDL_PASSWORD] = '/unity/uds/user/wphyo/edl_dwssap' + os.environ[Constants.EDL_PASSWORD_TYPE] = Constants.PARAM_STORE + os.environ[Constants.EDL_BASE_URL] = 'urs.earthdata.nasa.gov' + os.environ['STAC_JSON'] = json.dumps(granule_json) + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'DAAC' + # os.environ['PARALLEL_COUNT'] = '5' + + with tempfile.TemporaryDirectory() as tmp_dir_name: + print(tmp_dir_name) + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + granule_json_file = os.path.join(tmp_dir_name, 'input_file.json') + downloading_dir = os.path.join(tmp_dir_name, 'downloading_dir') + FileUtils.mk_dir_p(downloading_dir) + FileUtils.write_json(granule_json_file, granule_json) + os.environ['STAC_JSON'] = granule_json_file + os.environ['DOWNLOAD_DIR'] = downloading_dir + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + print(len(download_result['features'])) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) + 1, len(glob(os.path.join(downloading_dir, '*'))), + f'downloaded file does not match') + error_file = os.path.join(downloading_dir, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([k['assets']['data']['href'] for k in download_result]) + for each_granule in zip(granule_json['features'], download_result): + remote_filename = os.path.basename(each_granule[0]['assets']['data']['href']) + self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, + f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_02_download__from_http(self): + granule_json = '{"numberMatched": 20, "numberReturned": 20, "stac_version": "1.0.0", "type": "FeatureCollection", "links": [{"rel": "self", "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=0"}, {"rel": "root", "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com"}, {"rel": "next", "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=100"}, {"rel": "prev", "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=0"}], "features": [{"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.01", "properties": {"start_datetime": "2016-01-14T09:54:00Z", "end_datetime": "2016-01-14T10:00:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:39.830000Z", "datetime": "2022-08-15T06:26:37.029000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "https://raw.githubusercontent.com/unity-sds/unity-data-services/develop/README.md", "title": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.01.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.01.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.01.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.08", "properties": {"start_datetime": "2016-01-14T10:36:00Z", "end_datetime": "2016-01-14T10:42:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:26.078000Z", "datetime": "2022-08-15T06:26:19.333000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "https://raw.githubusercontent.com/unity-sds/unity-data-services/develop/CHANGELOG.md", "title": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.08.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.08.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.08.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.08.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.06", "properties": {"start_datetime": "2016-01-14T10:24:00Z", "end_datetime": "2016-01-14T10:30:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:26.068000Z", "datetime": "2022-08-15T06:26:18.641000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "https://raw.githubusercontent.com/unity-sds/unity-data-services/develop/CODE_OF_CONDUCT.md", "title": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.06.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.06.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.06.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.06.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.18", "properties": {"start_datetime": "2016-01-14T11:36:00Z", "end_datetime": "2016-01-14T11:42:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:26.060000Z", "datetime": "2022-08-15T06:26:19.698000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "https://raw.githubusercontent.com/unity-sds/unity-data-services/develop/CONTRIBUTING.md", "title": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.18.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.18.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.18.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.18.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.04", "properties": {"start_datetime": "2016-01-14T10:12:00Z", "end_datetime": "2016-01-14T10:18:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:26.050000Z", "datetime": "2022-08-15T06:26:19.491000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "https://raw.githubusercontent.com/unity-sds/unity-data-services/develop/LICENSE", "title": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.04.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.04.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.04.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.04.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}]}' + granule_json = json.loads(granule_json) + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'HTTP' + os.environ['PARALLEL_COUNT'] = '3' + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + granule_json_file = os.path.join(tmp_dir_name, 'input_file.json') + downloading_dir = os.path.join(tmp_dir_name, 'downloading_dir') + FileUtils.mk_dir_p(downloading_dir) + FileUtils.write_json(granule_json_file, granule_json) + os.environ['STAC_JSON'] = granule_json_file + os.environ['DOWNLOAD_DIR'] = downloading_dir + os.environ['DOWNLOADING_KEYS'] = 'data' + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) + 1, len(glob(os.path.join(downloading_dir, '*'))), + f'downloaded file does not match: {download_result["features"]} v. {glob(os.path.join(downloading_dir, "*"))}') + error_file = os.path.join(tmp_dir_name, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([k['assets']['data']['href'] for k in download_result]) + for each_granule in zip(granule_json['features'], download_result): + remote_filename = os.path.basename(each_granule[0]['assets']['data']['href']) + self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, + f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.get_size(os.path.join(downloading_dir, remote_filename)) > 0, f'empty file: {remote_filename}') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_02_download__from_http_with_role(self): + granule_json = '''{ + "numberMatched": 20, + "numberReturned": 20, + "stac_version": "1.0.0", + "type": "FeatureCollection", + "links": [ + { + "rel": "self", + "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=0" + }, + { + "rel": "root", + "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com" + }, + { + "rel": "next", + "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=100" + }, + { + "rel": "prev", + "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=0" + } + ], + "features": [ + { + "type": "Feature", + "stac_version": "1.0.0", + "id": "SNDR.SNPP.ATMS.L1A.nominal2.01", + "properties": { + "start_datetime": "2016-01-14T09:54:00Z", + "end_datetime": "2016-01-14T10:00:00Z", + "created": "2020-12-14T13:50:00Z", + "updated": "2022-08-15T06:26:39.830000Z", + "datetime": "2022-08-15T06:26:37.029000Z" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 0.0, + 0.0 + ] + }, + "links": [ + { + "rel": "collection", + "href": "." + } + ], + "assets": { + "SNDR.SNPP.ATMS.L1A.nominal2.01.nc": { + "href": "https://raw.githubusercontent.com/unity-sds/unity-data-services/develop/README.md", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc", + "roles": [ + "data" + ] + }, + "SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cas": { + "href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cas", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cas", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cas", + "roles": [ + "metadata__data" + ] + }, + "SNDR.SNPP.ATMS.L1A.nominal2.01.cmr.xml": { + "href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.01.cmr.xml", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.01.cmr.xml", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.01.cmr.xml", + "roles": [ + "metadata__cmr" + ] + } + }, + "bbox": [ + 0.0, + 0.0, + 0.0, + 0.0 + ], + "stac_extensions": [], + "collection": "SNDR_SNPP_ATMS_L1A___1" + }, + { + "type": "Feature", + "stac_version": "1.0.0", + "id": "SNDR.SNPP.ATMS.L1A.nominal2.08", + "properties": { + "start_datetime": "2016-01-14T10:36:00Z", + "end_datetime": "2016-01-14T10:42:00Z", + "created": "2020-12-14T13:50:00Z", + "updated": "2022-08-15T06:26:26.078000Z", + "datetime": "2022-08-15T06:26:19.333000Z" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 0.0, + 0.0 + ] + }, + "links": [ + { + "rel": "collection", + "href": "." + } + ], + "assets": { + "SNDR.SNPP.ATMS.L1A.nominal2.08.nc": { + "href": "https://raw.githubusercontent.com/unity-sds/unity-data-services/develop/CHANGELOG.md", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc", + "roles": [ + "data" + ] + }, + "SNDR.SNPP.ATMS.L1A.nominal2.08.nc.cas": { + "href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.08.nc.cas", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc.cas", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc.cas", + "roles": [ + "metadata__data" + ] + }, + "SNDR.SNPP.ATMS.L1A.nominal2.08.cmr.xml": { + "href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.08.cmr.xml", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.08.cmr.xml", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.08.cmr.xml", + "roles": [ + "metadata__cmr" + ] + } + }, + "bbox": [ + 0.0, + 0.0, + 0.0, + 0.0 + ], + "stac_extensions": [], + "collection": "SNDR_SNPP_ATMS_L1A___1" + }, + { + "type": "Feature", + "stac_version": "1.0.0", + "id": "SNDR.SNPP.ATMS.L1A.nominal2.06", + "properties": { + "start_datetime": "2016-01-14T10:24:00Z", + "end_datetime": "2016-01-14T10:30:00Z", + "created": "2020-12-14T13:50:00Z", + "updated": "2022-08-15T06:26:26.068000Z", + "datetime": "2022-08-15T06:26:18.641000Z" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 0.0, + 0.0 + ] + }, + "links": [ + { + "rel": "collection", + "href": "." + } + ], + "assets": { + "SNDR.SNPP.ATMS.L1A.nominal2.06.nc": { + "href": "https://raw.githubusercontent.com/unity-sds/unity-data-services/develop/CODE_OF_CONDUCT.md", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc", + "roles": [ + "data" + ] + }, + "SNDR.SNPP.ATMS.L1A.nominal2.06.nc.cas": { + "href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.06.nc.cas", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc.cas", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc.cas", + "roles": [ + "metadata__data" + ] + }, + "SNDR.SNPP.ATMS.L1A.nominal2.06.cmr.xml": { + "href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.06.cmr.xml", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.06.cmr.xml", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.06.cmr.xml", + "roles": [ + "metadata__cmr" + ] + } + }, + "bbox": [ + 0.0, + 0.0, + 0.0, + 0.0 + ], + "stac_extensions": [], + "collection": "SNDR_SNPP_ATMS_L1A___1" + }, + { + "type": "Feature", + "stac_version": "1.0.0", + "id": "SNDR.SNPP.ATMS.L1A.nominal2.18", + "properties": { + "start_datetime": "2016-01-14T11:36:00Z", + "end_datetime": "2016-01-14T11:42:00Z", + "created": "2020-12-14T13:50:00Z", + "updated": "2022-08-15T06:26:26.060000Z", + "datetime": "2022-08-15T06:26:19.698000Z" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 0.0, + 0.0 + ] + }, + "links": [ + { + "rel": "collection", + "href": "." + } + ], + "assets": { + "SNDR.SNPP.ATMS.L1A.nominal2.18.nc": { + "href": "https://raw.githubusercontent.com/unity-sds/unity-data-services/develop/CONTRIBUTING.md", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc", + "roles": [ + "data" + ] + }, + "SNDR.SNPP.ATMS.L1A.nominal2.18.nc.cas": { + "href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.18.nc.cas", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc.cas", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc.cas", + "roles": [ + "metadata__data" + ] + }, + "SNDR.SNPP.ATMS.L1A.nominal2.18.cmr.xml": { + "href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.18.cmr.xml", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.18.cmr.xml", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.18.cmr.xml", + "roles": [ + "metadata__cmr" + ] + } + }, + "bbox": [ + 0.0, + 0.0, + 0.0, + 0.0 + ], + "stac_extensions": [], + "collection": "SNDR_SNPP_ATMS_L1A___1" + }, + { + "type": "Feature", + "stac_version": "1.0.0", + "id": "SNDR.SNPP.ATMS.L1A.nominal2.04", + "properties": { + "start_datetime": "2016-01-14T10:12:00Z", + "end_datetime": "2016-01-14T10:18:00Z", + "created": "2020-12-14T13:50:00Z", + "updated": "2022-08-15T06:26:26.050000Z", + "datetime": "2022-08-15T06:26:19.491000Z" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 0.0, + 0.0 + ] + }, + "links": [ + { + "rel": "collection", + "href": "." + } + ], + "assets": { + "SNDR.SNPP.ATMS.L1A.nominal2.04.nc": { + "href": "https://raw.githubusercontent.com/unity-sds/unity-data-services/develop/LICENSE", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc", + "roles": [ + "data" + ] + }, + "SNDR.SNPP.ATMS.L1A.nominal2.04.nc.cas": { + "href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.04.nc.cas", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc.cas", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc.cas", + "roles": [ + "metadata__data" + ] + }, + "SNDR.SNPP.ATMS.L1A.nominal2.04.cmr.xml": { + "href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.04.cmr.xml", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.04.cmr.xml", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.04.cmr.xml", + "roles": [ + "metadata__cmr" + ] + } + }, + "bbox": [ + 0.0, + 0.0, + 0.0, + 0.0 + ], + "stac_extensions": [], + "collection": "SNDR_SNPP_ATMS_L1A___1" + } + ] + }''' + granule_json = json.loads(granule_json) + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'HTTP' + os.environ['PARALLEL_COUNT'] = '3' + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + granule_json_file = os.path.join(tmp_dir_name, 'input_file.json') + downloading_dir = os.path.join(tmp_dir_name, 'downloading_dir') + FileUtils.mk_dir_p(downloading_dir) + FileUtils.write_json(granule_json_file, granule_json) + os.environ['STAC_JSON'] = granule_json_file + os.environ['DOWNLOAD_DIR'] = downloading_dir + os.environ['DOWNLOADING_ROLES'] = 'data' + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) + 1, len(glob(os.path.join(downloading_dir, '*'))), + f'downloaded file does not match: {download_result["features"]} v. {glob(os.path.join(downloading_dir, "*"))}') + error_file = os.path.join(tmp_dir_name, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([list(k['assets'].values())[0]['href'] for k in download_result]) + # print(granule_json['features'], download_result) + print(FileUtils.read_json(f'{downloading_dir}/downloaded_feature_collection.json')) + for each_granule in zip(granule_json['features'], download_result): + remote_filename = [k['href'] for k in each_granule[0]['assets'].values() if 'data' in k['roles']] + remote_filename = os.path.basename(remote_filename[0]) + # self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, f'mismatched: {remote_filename}') + self.assertTrue(remote_filename in downloaded_file_hrefs, f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.get_size(os.path.join(downloading_dir, remote_filename)) > 0, f'empty file: {remote_filename}') + # self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return diff --git a/tests/integration_tests/test_docker_stage_out.py b/tests/integration_tests/test_docker_stage_out.py new file mode 100644 index 0000000..a1b92b5 --- /dev/null +++ b/tests/integration_tests/test_docker_stage_out.py @@ -0,0 +1,1334 @@ +import logging +logging.basicConfig(level=20, format="%(asctime)s [%(levelname)s] [%(name)s::%(lineno)d] %(message)s") + +from datetime import datetime + +from mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3 import \ + UploadGranulesByCompleteCatalogS3 + +from mdps_ds_lib.lib.aws.aws_s3 import AwsS3 +from mdps_ds_lib.stage_in_out.upoad_granules_factory import UploadGranulesFactory + + +import json +import os +import tempfile +from sys import argv +from unittest import TestCase + +from pystac import Item, Asset, Catalog, Link, ItemCollection + +from mdps_ds_lib.lib.constants import Constants + +from mdps_ds_lib.lib.utils.time_utils import TimeUtils +from mdps_ds_lib.lib.utils.file_utils import FileUtils + + +class TestDockerStageOut(TestCase): + def setUp(self) -> None: + super().setUp() + self.tenant = 'UDS_MY_LOCAL_ARCHIVE_TEST' # 'uds_local_test' # 'uds_sandbox' + self.tenant_venue = 'DEV' # 'DEV1' # 'dev' + self.collection_name = 'UDS_UNIT_COLLECTION' # 'uds_collection' # 'sbx_collection' + self.collection_version = '24.08.29.09.00'.replace('.', '') # '2402011200' + + def not_in_used_test_03_upload(self): + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '6ir9qveln397i0inh9pmsabq1' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test' + os.environ['VERIFY_SSL'] = 'FALSE' + + os.environ['COLLECTION_ID'] = 'NEW_COLLECTION_EXAMPLE_L1B___9' + os.environ['STAGING_BUCKET'] = 'uds-test-cumulus-staging' + + os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' + os.environ['GRANULES_UPLOAD_TYPE'] = 'S3' + if len(argv) > 1: + argv.pop(-1) + argv.append('UPLOAD') + + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + os.environ['UPLOAD_DIR'] = tmp_dir_name + with open(os.path.join(tmp_dir_name, 'test_file01.nc'), 'w') as ff: + ff.write('sample_file') + with open(os.path.join(tmp_dir_name, 'test_file01.nc.cas'), 'w') as ff: + ff.write(''' + + + AggregateDir + snppatmsl1a + + + AutomaticQualityFlag + Passed + + + BuildId + v01.43.00 + + + CollectionLabel + L1AMw_nominal2 + + + DataGroup + sndr + + + EndDateTime + 2016-01-14T10:06:00.000Z + + + EndTAI93 + 726919569.000 + + + FileFormat + nc4 + + + FileLocation + /pge/out + + + Filename + SNDR.SNPP.ATMS.L1A.nominal2.02.nc + + + GranuleNumber + 101 + + + JobId + f163835c-9945-472f-bee2-2bc12673569f + + + ModelId + urn:npp:SnppAtmsL1a + + + NominalDate + 2016-01-14 + + + ProductName + SNDR.SNPP.ATMS.20160114T1000.m06.g101.L1A.L1AMw_nominal2.v03_15_00.D.201214135000.nc + + + ProductType + SNDR_SNPP_ATMS_L1A + + + ProductionDateTime + 2020-12-14T13:50:00.000Z + + + ProductionLocation + Sounder SIPS: JPL/Caltech (Dev) + + + ProductionLocationCode + D + + + RequestId + 1215 + + + StartDateTime + 2016-01-14T10:00:00.000Z + + + StartTAI93 + 726919209.000 + + + TaskId + 8c3ae101-8f7c-46c8-b5c6-63e7b6d3c8cd + + ''') + upload_result = UploadGranulesFactory().get_class(os.getenv('GRANULES_UPLOAD_TYPE', UploadGranulesFactory.UPLOAD_S3_BY_STAC_CATALOG)).upload() + print(upload_result) + self.assertEqual(1, len(upload_result), 'wrong length of upload_result features') + upload_result = upload_result[0] + self.assertTrue('assets' in upload_result, 'missing assets') + self.assertTrue('metadata' in upload_result['assets'], 'missing assets#metadata') + self.assertTrue('href' in upload_result['assets']['metadata'], 'missing assets#metadata#href') + self.assertTrue( + upload_result['assets']['metadata']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/')) + self.assertTrue('data' in upload_result['assets'], 'missing assets#data') + self.assertTrue('href' in upload_result['assets']['data'], 'missing assets#data#href') + self.assertTrue(upload_result['assets']['data']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/')) + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def not_in_used_test_03_upload_catalog(self): + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '6ir9qveln397i0inh9pmsabq1' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test' + os.environ['VERIFY_SSL'] = 'FALSE' + + os.environ['COLLECTION_ID'] = 'NEW_COLLECTION_EXAMPLE_L1B___9' + os.environ['STAGING_BUCKET'] = 'uds-test-cumulus-staging' + + os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' + os.environ['GRANULES_UPLOAD_TYPE'] = 'CATALOG_S3' + + if len(argv) > 1: + argv.pop(-1) + argv.append('UPLOAD') + + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + os.environ['UPLOAD_DIR'] = '' # not needed + os.environ['CATALOG_FILE'] = os.path.join(tmp_dir_name, 'catalog.json') + with open(os.path.join(tmp_dir_name, 'test_file01.nc'), 'w') as ff: + ff.write('sample_file') + with open(os.path.join(tmp_dir_name, 'test_file01.nc.cas'), 'w') as ff: + ff.write(''' + + + AggregateDir + snppatmsl1a + + + AutomaticQualityFlag + Passed + + + BuildId + v01.43.00 + + + CollectionLabel + L1AMw_nominal2 + + + DataGroup + sndr + + + EndDateTime + 2016-01-14T10:06:00.000Z + + + EndTAI93 + 726919569.000 + + + FileFormat + nc4 + + + FileLocation + /pge/out + + + Filename + SNDR.SNPP.ATMS.L1A.nominal2.02.nc + + + GranuleNumber + 101 + + + JobId + f163835c-9945-472f-bee2-2bc12673569f + + + ModelId + urn:npp:SnppAtmsL1a + + + NominalDate + 2016-01-14 + + + ProductName + SNDR.SNPP.ATMS.20160114T1000.m06.g101.L1A.L1AMw_nominal2.v03_15_00.D.201214135000.nc + + + ProductType + SNDR_SNPP_ATMS_L1A + + + ProductionDateTime + 2020-12-14T13:50:00.000Z + + + ProductionLocation + Sounder SIPS: JPL/Caltech (Dev) + + + ProductionLocationCode + D + + + RequestId + 1215 + + + StartDateTime + 2016-01-14T10:00:00.000Z + + + StartTAI93 + 726919209.000 + + + TaskId + 8c3ae101-8f7c-46c8-b5c6-63e7b6d3c8cd + + ''') + stac_item = Item(id='NA', + geometry={ + "type": "Point", + "coordinates": [0.0, 0.0] + }, + bbox=[0.0, 0.0, 0.0, 0.0], + datetime=TimeUtils().parse_from_unix(0, True).get_datetime_obj(), + properties={ + "start_datetime": "2016-01-31T18:00:00.009057Z", + "end_datetime": "2016-01-31T19:59:59.991043Z", + "created": "2016-02-01T02:45:59.639000Z", + "updated": "2022-03-23T15:48:21.578000Z", + "datetime": "2022-03-23T15:48:19.079000Z" + }, + collection='NA', + assets={ + 'data': Asset(os.path.join(tmp_dir_name, 'test_file01.nc'), title='main data'), + 'metadata__cas': Asset(os.path.join(tmp_dir_name, 'test_file01.nc.cas'), title='metadata cas'), + 'metadata__stac': Asset(os.path.join(tmp_dir_name, 'test_file01.nc.stac.json'), title='metadata stac'), + }) + with open(os.path.join(tmp_dir_name, 'test_file01.nc.stac.json'), 'w') as ff: + ff.write(json.dumps(stac_item.to_dict(False, False))) + catalog = Catalog( + id='NA', + description='NA') + catalog.set_self_href(os.environ['CATALOG_FILE']) + catalog.add_link(Link('child', os.path.join(tmp_dir_name, 'test_file01.nc.stac.json'), 'application/json')) + with open(os.environ['CATALOG_FILE'], 'w') as ff: + ff.write(json.dumps(catalog.to_dict(False, False))) + + upload_result = UploadGranulesFactory().get_class(os.getenv('GRANULES_UPLOAD_TYPE', UploadGranulesFactory.UPLOAD_S3_BY_STAC_CATALOG)).upload() + print(upload_result) + self.assertEqual(1, len(upload_result), 'wrong length of upload_result features') + upload_result = upload_result[0] + self.assertTrue('assets' in upload_result, 'missing assets') + self.assertTrue('metadata__cas' in upload_result['assets'], 'missing assets#metadata__cas') + self.assertTrue('href' in upload_result['assets']['metadata__cas'], 'missing assets#metadata__cas#href') + self.assertTrue( + upload_result['assets']['metadata__cas']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/')) + self.assertTrue('data' in upload_result['assets'], 'missing assets#data') + self.assertTrue('href' in upload_result['assets']['data'], 'missing assets#data#href') + self.assertTrue(upload_result['assets']['data']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/')) + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_03_upload_complete_catalog(self): + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['RESULT_PATH_PREFIX'] = 'integration_test/stage_out' + os.environ['COLLECTION_ID'] = 'NEW_COLLECTION_EXAMPLE_L1B___9' + os.environ['STAGING_BUCKET'] = 'uds-sbx-cumulus-staging' + + os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' + # os.environ['GRANULES_UPLOAD_TYPE'] = 'UPLOAD_S3_BY_STAC_CATALOG' + # defaulted to this value + + if len(argv) > 1: + argv.pop(-1) + argv.append('UPLOAD') + + starting_time = datetime.utcnow().strftime('%Y-%m-%dT%H:%M') + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + os.environ['UPLOAD_DIR'] = '' # not needed + os.environ['OUTPUT_DIRECTORY'] = os.path.join(tmp_dir_name, 'output_dir') + FileUtils.mk_dir_p(os.environ.get('OUTPUT_DIRECTORY')) + os.environ['CATALOG_FILE'] = os.path.join(tmp_dir_name, 'catalog.json') + total_files = 10 + # os.environ['PARALLEL_COUNT'] = str(total_files) + granules_dir = os.path.join(tmp_dir_name, 'some_granules') + FileUtils.mk_dir_p(granules_dir) + catalog = Catalog( + id='NA', + description='NA') + catalog.set_self_href(os.environ['CATALOG_FILE']) + + for i in range(1, total_files+1): + filename = f'test_file{i:02d}' + with open(os.path.join(granules_dir, f'{filename}.nc'), 'w') as ff: + ff.write('sample_file') + with open(os.path.join(granules_dir, f'{filename}.nc.cas'), 'w') as ff: + ff.write(''' + + + AggregateDir + snppatmsl1a + + + AutomaticQualityFlag + Passed + + + BuildId + v01.43.00 + + + CollectionLabel + L1AMw_nominal2 + + + DataGroup + sndr + + + EndDateTime + 2016-01-14T10:06:00.000Z + + + EndTAI93 + 726919569.000 + + + FileFormat + nc4 + + + FileLocation + /pge/out + + + Filename + SNDR.SNPP.ATMS.L1A.nominal2.02.nc + + + GranuleNumber + 101 + + + JobId + f163835c-9945-472f-bee2-2bc12673569f + + + ModelId + urn:npp:SnppAtmsL1a + + + NominalDate + 2016-01-14 + + + ProductName + SNDR.SNPP.ATMS.20160114T1000.m06.g101.L1A.L1AMw_nominal2.v03_15_00.D.201214135000.nc + + + ProductType + SNDR_SNPP_ATMS_L1A + + + ProductionDateTime + 2020-12-14T13:50:00.000Z + + + ProductionLocation + Sounder SIPS: JPL/Caltech (Dev) + + + ProductionLocationCode + D + + + RequestId + 1215 + + + StartDateTime + 2016-01-14T10:00:00.000Z + + + StartTAI93 + 726919209.000 + + + TaskId + 8c3ae101-8f7c-46c8-b5c6-63e7b6d3c8cd + + ''') + stac_item = Item(id=filename, + geometry={ + "type": "Point", + "coordinates": [0.0, 0.0] + }, + bbox=[0.0, 0.0, 0.0, 0.0], + datetime=TimeUtils().parse_from_unix(0, True).get_datetime_obj(), + properties={ + "start_datetime": "2016-01-31T18:00:00.009057Z", + "end_datetime": "2016-01-31T19:59:59.991043Z", + "created": "2016-02-01T02:45:59.639000Z", + "updated": "2022-03-23T15:48:21.578000Z", + "datetime": "2022-03-23T15:48:19.079000Z" + }, + href=os.path.join('some_granules', f'{filename}.nc.stac.json'), + collection='NA', + assets={ + f'{filename}.nc': Asset(os.path.join('.', f'{filename}.nc'), title='test_file01.nc', roles=['data']), + f'{filename}.nc.cas': Asset(os.path.join('.', f'{filename}.nc.cas'), title='test_file01.nc.cas', roles=['metadata']), + f'{filename}.nc.stac.json': Asset(os.path.join('.', f'{filename}.nc.stac.json'), title='test_file01.nc.stac.json', roles=['metadata']), + }) + with open(os.path.join(granules_dir, f'{filename}.nc.stac.json'), 'w') as ff: + ff.write(json.dumps(stac_item.to_dict(False, False))) + catalog.add_link(Link('item', os.path.join('some_granules', f'{filename}.nc.stac.json'), 'application/json')) + print(json.dumps(catalog.to_dict(False, False))) + with open(os.environ['CATALOG_FILE'], 'w') as ff: + ff.write(json.dumps(catalog.to_dict(False, False))) + + upload_result = UploadGranulesFactory().get_class(os.getenv('GRANULES_UPLOAD_TYPE', UploadGranulesFactory.UPLOAD_S3_BY_STAC_CATALOG)).upload() + upload_result = json.loads(upload_result) + print(upload_result) + """ + {'type': 'Catalog', 'id': 'NA', 'stac_version': '1.0.0', 'description': 'NA', 'links': [{'rel': 'root', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/catalog.json', 'type': 'application/json'}, {'rel': 'item', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/successful_features.json', 'type': 'application/json'}, {'rel': 'item', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/failed_features.json', 'type': 'application/json'}]} + """ + self.assertTrue('type' in upload_result, 'missing type') + self.assertEqual(upload_result['type'], 'Catalog', 'missing type') + upload_result = Catalog.from_dict(upload_result) + child_links = [k.href for k in upload_result.get_links(rel='item')] + self.assertEqual(len(child_links), 2, f'wrong length: {child_links}') + self.assertTrue(FileUtils.file_exist(child_links[0]), f'missing file: {child_links[0]}') + successful_feature_collection = ItemCollection.from_dict(FileUtils.read_json(child_links[0])) + successful_feature_collection = list(successful_feature_collection.items) + self.assertEqual(len(successful_feature_collection), total_files, f'wrong length: {successful_feature_collection}') + + self.assertTrue(FileUtils.file_exist(child_links[1]), f'missing file: {child_links[1]}') + failed_feature_collection = ItemCollection.from_dict(FileUtils.read_json(child_links[1])) + failed_feature_collection = list(failed_feature_collection.items) + self.assertEqual(len(failed_feature_collection), 0, f'wrong length: {failed_feature_collection}') + + upload_result = successful_feature_collection[0].to_dict(False, False) + print(f'example feature: {upload_result}') + self.assertTrue('assets' in upload_result, 'missing assets') + result_key = [k for k in upload_result['assets'].keys()][0] + self.assertTrue(result_key.startswith('test_file'), f'worng asset key: {result_key}') + result_key_prefix = result_key.split('.')[0] + self.assertTrue(f'{result_key_prefix}.nc.cas' in upload_result['assets'], f'missing assets#metadata asset: {result_key_prefix}.nc.cas') + self.assertTrue('href' in upload_result['assets'][f'{result_key_prefix}.nc.cas'], 'missing assets#metadata__cas#href') + self.assertTrue(upload_result['assets'][f'{result_key_prefix}.nc.cas']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/{os.environ["COLLECTION_ID"]}/')) + self.assertTrue(f'{result_key_prefix}.nc' in upload_result['assets'], f'missing assets#data: {result_key_prefix}.nc') + self.assertTrue('href' in upload_result['assets'][f'{result_key_prefix}.nc'], 'missing assets#data#href') + self.assertTrue(upload_result['assets'][f'{result_key_prefix}.nc']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/{os.environ["COLLECTION_ID"]}/')) + """ + Example output: + { + 'type': 'FeatureCollection', + 'features': [{ + 'type': 'Feature', + 'stac_version': '1.0.0', + 'id': 'NEW_COLLECTION_EXAMPLE_L1B___9:test_file01', + 'properties': {'start_datetime': '2016-01-31T18:00:00.009057Z', + 'end_datetime': '2016-01-31T19:59:59.991043Z', 'created': '2016-02-01T02:45:59.639000Z', + 'updated': '2022-03-23T15:48:21.578000Z', 'datetime': '1970-01-01T00:00:00Z'}, + 'geometry': {'type': 'Point', 'coordinates': [0.0, 0.0]}, 'links': [], + 'assets': {'data': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc', + 'title': 'main data'}, 'metadata__cas': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.cas', + 'title': 'metadata cas'}, 'metadata__stac': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.stac.json', + 'title': 'metadata stac'}}, + 'bbox': [0.0, 0.0, 0.0, 0.0], + 'stac_extensions': [], + 'collection': 'NEW_COLLECTION_EXAMPLE_L1B___9'}]} + """ + s3 = AwsS3() + s3_keys = [k for k in s3.get_child_s3_files(os.environ['STAGING_BUCKET'], + f"{os.environ['RESULT_PATH_PREFIX']}/successful_features_{starting_time}", + )] + s3_keys = sorted(s3_keys) + print(f's3_keys: {s3_keys}') + self.assertTrue(len(s3_keys) > 0, f'empty files in S3') + local_file = s3.set_s3_url(f's3://{os.environ["STAGING_BUCKET"]}/{s3_keys[-1][0]}').download(tmp_dir_name) + successful_feature_collection = ItemCollection.from_dict(FileUtils.read_json(local_file)) + successful_feature_collection = list(successful_feature_collection.items) + self.assertEqual(len(successful_feature_collection), total_files, f'wrong length: {successful_feature_collection}') + return + + def test_03_02_upload_complete_catalog(self): + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['COLLECTION_ID'] = 'NEW_COLLECTION_EXAMPLE_L1B___9' + os.environ['STAGING_BUCKET'] = 'uds-sbx-cumulus-staging' + + os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' + # os.environ['GRANULES_UPLOAD_TYPE'] = 'UPLOAD_S3_BY_STAC_CATALOG' + # defaulted to this value + + if len(argv) > 1: + argv.pop(-1) + argv.append('UPLOAD') + + starting_time = datetime.utcnow().strftime('%Y-%m-%dT%H:%M') + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + os.environ['UPLOAD_DIR'] = '' # not needed + os.environ['OUTPUT_DIRECTORY'] = os.path.join(tmp_dir_name, 'output_dir') + FileUtils.mk_dir_p(os.environ.get('OUTPUT_DIRECTORY')) + os.environ['CATALOG_FILE'] = os.path.join(tmp_dir_name, 'catalog.json') + total_files = 10 + # os.environ['PARALLEL_COUNT'] = str(total_files) + granules_dir = os.path.join(tmp_dir_name, 'some_granules') + FileUtils.mk_dir_p(granules_dir) + catalog = Catalog( + id='NA', + description='NA') + catalog.set_self_href(os.environ['CATALOG_FILE']) + + for i in range(1, total_files+1): + filename = f'test_file{i:02d}' + with open(os.path.join(granules_dir, f'{filename}.nc'), 'w') as ff: + ff.write('sample_file') + with open(os.path.join(granules_dir, f'{filename}.nc.cas'), 'w') as ff: + ff.write(''' + + + AggregateDir + snppatmsl1a + + + AutomaticQualityFlag + Passed + + + BuildId + v01.43.00 + + + CollectionLabel + L1AMw_nominal2 + + + DataGroup + sndr + + + EndDateTime + 2016-01-14T10:06:00.000Z + + + EndTAI93 + 726919569.000 + + + FileFormat + nc4 + + + FileLocation + /pge/out + + + Filename + SNDR.SNPP.ATMS.L1A.nominal2.02.nc + + + GranuleNumber + 101 + + + JobId + f163835c-9945-472f-bee2-2bc12673569f + + + ModelId + urn:npp:SnppAtmsL1a + + + NominalDate + 2016-01-14 + + + ProductName + SNDR.SNPP.ATMS.20160114T1000.m06.g101.L1A.L1AMw_nominal2.v03_15_00.D.201214135000.nc + + + ProductType + SNDR_SNPP_ATMS_L1A + + + ProductionDateTime + 2020-12-14T13:50:00.000Z + + + ProductionLocation + Sounder SIPS: JPL/Caltech (Dev) + + + ProductionLocationCode + D + + + RequestId + 1215 + + + StartDateTime + 2016-01-14T10:00:00.000Z + + + StartTAI93 + 726919209.000 + + + TaskId + 8c3ae101-8f7c-46c8-b5c6-63e7b6d3c8cd + + ''') + stac_item = Item(id=filename, + geometry={ + "type": "Point", + "coordinates": [0.0, 0.0] + }, + bbox=[0.0, 0.0, 0.0, 0.0], + datetime=TimeUtils().parse_from_unix(0, True).get_datetime_obj(), + properties={ + "start_datetime": "2016-01-31T18:00:00.009057Z", + "end_datetime": "2016-01-31T19:59:59.991043Z", + "created": "2016-02-01T02:45:59.639000Z", + "updated": "2022-03-23T15:48:21.578000Z", + "datetime": "2022-03-23T15:48:19.079000Z" + }, + href=os.path.join('some_granules', f'{filename}.nc.stac.json'), + collection='NA', + assets={ + f'{filename}.nc': Asset(os.path.join('.', f'{filename}.nc'), title='test_file01.nc', roles=['data']), + f'{filename}.nc.cas': Asset(os.path.join('.', f'{filename}.nc.cas'), title='test_file01.nc.cas', roles=['metadata']), + f'{filename}.nc.stac.json': Asset(os.path.join('.', f'{filename}.nc.stac.json'), title='test_file01.nc.stac.json', roles=['metadata']), + }) + with open(os.path.join(granules_dir, f'{filename}.nc.stac.json'), 'w') as ff: + ff.write(json.dumps(stac_item.to_dict(False, False))) + catalog.add_link(Link('item', os.path.join('some_granules', f'{filename}.nc.stac.json'), 'application/json')) + print(json.dumps(catalog.to_dict(False, False))) + with open(os.environ['CATALOG_FILE'], 'w') as ff: + ff.write(json.dumps(catalog.to_dict(False, False))) + + upload_result_str = UploadGranulesFactory().get_class(os.getenv('GRANULES_UPLOAD_TYPE', UploadGranulesFactory.UPLOAD_S3_BY_STAC_CATALOG)).upload() + upload_result = json.loads(upload_result_str) + print(upload_result) + """ + {'type': 'Catalog', 'id': 'NA', 'stac_version': '1.0.0', 'description': 'NA', 'links': [{'rel': 'root', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/catalog.json', 'type': 'application/json'}, {'rel': 'item', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/successful_features.json', 'type': 'application/json'}, {'rel': 'item', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/failed_features.json', 'type': 'application/json'}]} + """ + self.assertTrue('type' in upload_result, 'missing type') + self.assertEqual(upload_result['type'], 'Catalog', 'missing type') + upload_result = Catalog.from_dict(upload_result) + child_links = [k.href for k in upload_result.get_links(rel='item')] + self.assertEqual(len(child_links), 2, f'wrong length: {child_links}') + self.assertTrue(FileUtils.file_exist(child_links[0]), f'missing file: {child_links[0]}') + successful_feature_collection = ItemCollection.from_dict(FileUtils.read_json(child_links[0])) + successful_feature_collection = list(successful_feature_collection.items) + self.assertEqual(len(successful_feature_collection), total_files, f'wrong length: {successful_feature_collection}') + + self.assertTrue(FileUtils.file_exist(child_links[1]), f'missing file: {child_links[1]}') + failed_feature_collection = ItemCollection.from_dict(FileUtils.read_json(child_links[1])) + failed_feature_collection = list(failed_feature_collection.items) + self.assertEqual(len(failed_feature_collection), 0, f'wrong length: {failed_feature_collection}') + + upload_result = successful_feature_collection[0].to_dict(False, False) + print(f'example feature: {upload_result}') + self.assertTrue('assets' in upload_result, 'missing assets') + result_key = [k for k in upload_result['assets'].keys()][0] + self.assertTrue(result_key.startswith('test_file'), f'worng asset key: {result_key}') + result_key_prefix = result_key.split('.')[0] + self.assertTrue(f'{result_key_prefix}.nc.cas' in upload_result['assets'], f'missing assets#metadata asset: {result_key_prefix}.nc.cas') + self.assertTrue('href' in upload_result['assets'][f'{result_key_prefix}.nc.cas'], 'missing assets#metadata__cas#href') + self.assertTrue(upload_result['assets'][f'{result_key_prefix}.nc.cas']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/{os.environ["COLLECTION_ID"]}/')) + self.assertTrue(f'{result_key_prefix}.nc' in upload_result['assets'], f'missing assets#data: {result_key_prefix}.nc') + self.assertTrue('href' in upload_result['assets'][f'{result_key_prefix}.nc'], 'missing assets#data#href') + self.assertTrue(upload_result['assets'][f'{result_key_prefix}.nc']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/{os.environ["COLLECTION_ID"]}/')) + """ + Example output: + { + 'type': 'FeatureCollection', + 'features': [{ + 'type': 'Feature', + 'stac_version': '1.0.0', + 'id': 'NEW_COLLECTION_EXAMPLE_L1B___9:test_file01', + 'properties': {'start_datetime': '2016-01-31T18:00:00.009057Z', + 'end_datetime': '2016-01-31T19:59:59.991043Z', 'created': '2016-02-01T02:45:59.639000Z', + 'updated': '2022-03-23T15:48:21.578000Z', 'datetime': '1970-01-01T00:00:00Z'}, + 'geometry': {'type': 'Point', 'coordinates': [0.0, 0.0]}, 'links': [], + 'assets': {'data': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc', + 'title': 'main data'}, 'metadata__cas': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.cas', + 'title': 'metadata cas'}, 'metadata__stac': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.stac.json', + 'title': 'metadata stac'}}, + 'bbox': [0.0, 0.0, 0.0, 0.0], + 'stac_extensions': [], + 'collection': 'NEW_COLLECTION_EXAMPLE_L1B___9'}]} + """ + s3 = AwsS3() + s3_keys = [k for k in s3.get_child_s3_files(os.environ['STAGING_BUCKET'], + f"{UploadGranulesByCompleteCatalogS3.DEFAULT_RESULT_PATH_PREFIX}/successful_features_{starting_time}", + )] + s3_keys = sorted(s3_keys) + print(f's3_keys: {s3_keys}') + self.assertTrue(len(s3_keys) > 0, f'empty files in S3') + local_file = s3.set_s3_url(f's3://{os.environ["STAGING_BUCKET"]}/{s3_keys[-1][0]}').download(tmp_dir_name) + successful_feature_collection = ItemCollection.from_dict(FileUtils.read_json(local_file)) + successful_feature_collection = list(successful_feature_collection.items) + self.assertEqual(len(successful_feature_collection), total_files, f'wrong length: {successful_feature_collection}') + return + + def test_03_03_upload_auxiliary_files(self): + temp_collection_id = f'URN:NASA:UNITY:{self.tenant}:{self.tenant_venue}:{self.collection_name}___{self.collection_version}' + os.environ['GRANULES_UPLOAD_TYPE'] = 'UPLOAD_AUXILIARY_FILE_AS_GRANULE' + os.environ['COLLECTION_ID'] = temp_collection_id + os.environ['STAGING_BUCKET'] = 'uds-sbx-cumulus-staging' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['RESULT_PATH_PREFIX'] = 'stage_out' + os.environ['PARALLEL_COUNT'] = '1' + + if len(argv) > 1: + argv.pop(-1) + argv.append('UPLOAD') + + starting_time = datetime.utcnow().strftime('%Y-%m-%dT%H:%M') + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_DIRECTORY'] = os.path.join(tmp_dir_name, 'output_dir') + FileUtils.mk_dir_p(os.environ.get('OUTPUT_DIRECTORY')) + + real_base_dir = os.path.join(tmp_dir_name, 'auxiliary_base') + FileUtils.mk_dir_p(real_base_dir) + os.environ['BASE_DIRECTORY'] = real_base_dir + + with open(os.path.join(tmp_dir_name, 'excluding_file.json'), 'w') as ff: + ff.write('{"message": "excluding file"}') + with open(os.path.join(real_base_dir, 'test_file_0.json'), 'w') as ff: + ff.write('{"message": "some file at root"}') + sub_folders = [ + os.path.join(real_base_dir, 'son'), + os.path.join(real_base_dir, 'daughter'), + os.path.join(real_base_dir, os.path.join('son', 'grandson')), + os.path.join(real_base_dir, os.path.join('son', 'granddaughter')), + os.path.join(real_base_dir, os.path.join('daughter', 'grandson')), + os.path.join(real_base_dir, os.path.join('daughter', 'granddaughter')), + ] + for i, each_sub_folder in enumerate(sub_folders): + FileUtils.mk_dir_p(each_sub_folder) + with open(os.path.join(each_sub_folder, f'test_file_{i}.json'), 'w') as ff: + ff.write(json.dumps({"message": f"some file at {each_sub_folder}"})) + FileUtils.mk_dir_p(os.path.join(real_base_dir, 'nephew')) # should not throw error for empty folders + FileUtils.mk_dir_p(os.path.join(real_base_dir, os.path.join('nephew', 'grandson'))) # should not throw error for empty folders + total_files = len(sub_folders) + 1 + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + + upload_result_str = UploadGranulesFactory().get_class(os.getenv('GRANULES_UPLOAD_TYPE', UploadGranulesFactory.UPLOAD_S3_BY_STAC_CATALOG)).upload() + upload_result = json.loads(upload_result_str) + print(upload_result) + """ + {'type': 'Catalog', 'id': 'NA', 'stac_version': '1.0.0', 'description': 'NA', 'links': [{'rel': 'root', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/catalog.json', 'type': 'application/json'}, {'rel': 'item', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/successful_features.json', 'type': 'application/json'}, {'rel': 'item', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/failed_features.json', 'type': 'application/json'}]} + """ + self.assertTrue('type' in upload_result, 'missing type') + self.assertEqual(upload_result['type'], 'Catalog', 'missing type') + upload_result = Catalog.from_dict(upload_result) + child_links = [k.href for k in upload_result.get_links(rel='item')] + self.assertEqual(len(child_links), 2, f'wrong length: {child_links}') + self.assertTrue(FileUtils.file_exist(child_links[0]), f'missing file: {child_links[0]}') + successful_feature_collection = ItemCollection.from_dict(FileUtils.read_json(child_links[0])) + successful_feature_collection = list(successful_feature_collection.items) + self.assertEqual(len(successful_feature_collection), total_files, f'wrong length: {successful_feature_collection}') + + self.assertTrue(FileUtils.file_exist(child_links[1]), f'missing file: {child_links[1]}') + failed_feature_collection = ItemCollection.from_dict(FileUtils.read_json(child_links[1])) + failed_feature_collection = list(failed_feature_collection.items) + self.assertEqual(len(failed_feature_collection), 0, f'wrong length: {failed_feature_collection}') + + upload_result = successful_feature_collection[0].to_dict(False, False) + print(f'example feature: {upload_result}') + self.assertTrue('assets' in upload_result, 'missing assets') + result_key = [k for k in upload_result['assets'].keys()][0] + self.assertTrue(result_key.startswith('test_file'), f'worng asset key: {result_key}') + self.assertTrue(f'{result_key}.stac.json' in upload_result['assets'], f'missing assets#metadata asset: test_file_0.json') + self.assertTrue('href' in upload_result['assets'][f'{result_key}.stac.json'], 'missing assets#metadata__cas#href') + self.assertTrue(upload_result['assets'][f'{result_key}.stac.json']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/{os.environ["COLLECTION_ID"]}/'), f"wrong HREF (no S3?): upload_result['assets'][f'{result_key}.stac.json']['href']") + """ + Example output: + { + 'type': 'FeatureCollection', + 'features': [{ + 'type': 'Feature', + 'stac_version': '1.0.0', + 'id': 'NEW_COLLECTION_EXAMPLE_L1B___9:test_file01', + 'properties': {'start_datetime': '2016-01-31T18:00:00.009057Z', + 'end_datetime': '2016-01-31T19:59:59.991043Z', 'created': '2016-02-01T02:45:59.639000Z', + 'updated': '2022-03-23T15:48:21.578000Z', 'datetime': '1970-01-01T00:00:00Z'}, + 'geometry': {'type': 'Point', 'coordinates': [0.0, 0.0]}, 'links': [], + 'assets': {'data': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc', + 'title': 'main data'}, 'metadata__cas': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.cas', + 'title': 'metadata cas'}, 'metadata__stac': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.stac.json', + 'title': 'metadata stac'}}, + 'bbox': [0.0, 0.0, 0.0, 0.0], + 'stac_extensions': [], + 'collection': 'NEW_COLLECTION_EXAMPLE_L1B___9'}]} + """ + s3 = AwsS3() + s3_keys = [k for k in s3.get_child_s3_files(os.environ['STAGING_BUCKET'], + f"{os.environ['RESULT_PATH_PREFIX']}/successful_features_{starting_time}", + )] + s3_keys = sorted(s3_keys) + print(f's3_keys: {s3_keys}') + self.assertTrue(len(s3_keys) > 0, f'empty files in S3') + local_file = s3.set_s3_url(f's3://{os.environ["STAGING_BUCKET"]}/{s3_keys[-1][0]}').download(tmp_dir_name) + successful_feature_collection = ItemCollection.from_dict(FileUtils.read_json(local_file)) + successful_feature_collection = list(successful_feature_collection.items) + self.assertEqual(len(successful_feature_collection), total_files, f'wrong length: {successful_feature_collection}') + return + + def test_03_upload_complete_catalog_invalid_bucket(self): + os.environ['VERIFY_SSL'] = 'FALSE' + + os.environ['COLLECTION_ID'] = 'NEW_COLLECTION_EXAMPLE_L1B___9' + os.environ['STAGING_BUCKET'] = 'invalid_bucket' + + os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' + # os.environ['GRANULES_UPLOAD_TYPE'] = 'UPLOAD_S3_BY_STAC_CATALOG' + # defaulted to this value + + if len(argv) > 1: + argv.pop(-1) + argv.append('UPLOAD') + + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + os.environ['UPLOAD_DIR'] = '' # not needed + os.environ['CATALOG_FILE'] = os.path.join(tmp_dir_name, 'catalog.json') + os.environ['OUTPUT_DIRECTORY'] = os.path.join(tmp_dir_name, 'output_dir') + FileUtils.mk_dir_p(os.environ.get('OUTPUT_DIRECTORY')) + + total_files = 10 + # os.environ['PARALLEL_COUNT'] = str(total_files) + granules_dir = os.path.join(tmp_dir_name, 'some_granules') + FileUtils.mk_dir_p(granules_dir) + catalog = Catalog( + id='NA', + description='NA') + catalog.set_self_href(os.environ['CATALOG_FILE']) + + for i in range(1, total_files+1): + filename = f'test_file{i:02d}' + with open(os.path.join(granules_dir, f'{filename}.nc'), 'w') as ff: + ff.write('sample_file') + with open(os.path.join(granules_dir, f'{filename}.nc.cas'), 'w') as ff: + ff.write(''' + + + AggregateDir + snppatmsl1a + + + AutomaticQualityFlag + Passed + + + BuildId + v01.43.00 + + + CollectionLabel + L1AMw_nominal2 + + + DataGroup + sndr + + + EndDateTime + 2016-01-14T10:06:00.000Z + + + EndTAI93 + 726919569.000 + + + FileFormat + nc4 + + + FileLocation + /pge/out + + + Filename + SNDR.SNPP.ATMS.L1A.nominal2.02.nc + + + GranuleNumber + 101 + + + JobId + f163835c-9945-472f-bee2-2bc12673569f + + + ModelId + urn:npp:SnppAtmsL1a + + + NominalDate + 2016-01-14 + + + ProductName + SNDR.SNPP.ATMS.20160114T1000.m06.g101.L1A.L1AMw_nominal2.v03_15_00.D.201214135000.nc + + + ProductType + SNDR_SNPP_ATMS_L1A + + + ProductionDateTime + 2020-12-14T13:50:00.000Z + + + ProductionLocation + Sounder SIPS: JPL/Caltech (Dev) + + + ProductionLocationCode + D + + + RequestId + 1215 + + + StartDateTime + 2016-01-14T10:00:00.000Z + + + StartTAI93 + 726919209.000 + + + TaskId + 8c3ae101-8f7c-46c8-b5c6-63e7b6d3c8cd + + ''') + stac_item = Item(id=filename, + geometry={ + "type": "Point", + "coordinates": [0.0, 0.0] + }, + bbox=[0.0, 0.0, 0.0, 0.0], + datetime=TimeUtils().parse_from_unix(0, True).get_datetime_obj(), + properties={ + "start_datetime": "2016-01-31T18:00:00.009057Z", + "end_datetime": "2016-01-31T19:59:59.991043Z", + "created": "2016-02-01T02:45:59.639000Z", + "updated": "2022-03-23T15:48:21.578000Z", + "datetime": "2022-03-23T15:48:19.079000Z" + }, + href=os.path.join('some_granules', f'{filename}.nc.stac.json'), + collection='NA', + assets={ + f'{filename}.nc': Asset(os.path.join('.', f'{filename}.nc'), title='test_file01.nc', roles=['data']), + f'{filename}.nc.cas': Asset(os.path.join('.', f'{filename}.nc.cas'), title='test_file01.nc.cas', roles=['metadata']), + f'{filename}.nc.stac.json': Asset(os.path.join('.', f'{filename}.nc.stac.json'), title='test_file01.nc.stac.json', roles=['metadata']), + }) + with open(os.path.join(granules_dir, f'{filename}.nc.stac.json'), 'w') as ff: + ff.write(json.dumps(stac_item.to_dict(False, False))) + catalog.add_link(Link('item', os.path.join('some_granules', f'{filename}.nc.stac.json'), 'application/json')) + print(json.dumps(catalog.to_dict(False, False))) + with open(os.environ['CATALOG_FILE'], 'w') as ff: + ff.write(json.dumps(catalog.to_dict(False, False))) + + upload_result_str = UploadGranulesFactory().get_class(os.getenv('GRANULES_UPLOAD_TYPE', UploadGranulesFactory.UPLOAD_S3_BY_STAC_CATALOG)).upload() + upload_result = json.loads(upload_result_str) + print(upload_result) + """ + {'type': 'Catalog', 'id': 'NA', 'stac_version': '1.0.0', 'description': 'NA', 'links': [{'rel': 'root', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/catalog.json', 'type': 'application/json'}, {'rel': 'item', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/successful_features.json', 'type': 'application/json'}, {'rel': 'item', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/failed_features.json', 'type': 'application/json'}]} + """ + self.assertTrue('type' in upload_result, 'missing type') + self.assertEqual(upload_result['type'], 'Catalog', 'missing type') + upload_result = Catalog.from_dict(upload_result) + child_links = [k.href for k in upload_result.get_links(rel='item')] + self.assertEqual(len(child_links), 2, f'wrong length: {child_links}') + self.assertTrue(FileUtils.file_exist(child_links[0]), f'missing file: {child_links[0]}') + successful_feature_collection = ItemCollection.from_dict(FileUtils.read_json(child_links[0])) + successful_feature_collection = list(successful_feature_collection.items) + self.assertEqual(len(successful_feature_collection), 0, f'wrong length: {successful_feature_collection}') + + self.assertTrue(FileUtils.file_exist(child_links[1]), f'missing file: {child_links[1]}') + failed_feature_collection = ItemCollection.from_dict(FileUtils.read_json(child_links[1])) + failed_feature_collection = list(failed_feature_collection.items) + self.assertEqual(len(failed_feature_collection), total_files, f'wrong length: {failed_feature_collection}') + + upload_result = failed_feature_collection[0].to_dict(False, False) + print(f'example feature: {upload_result}') + self.assertTrue('properties' in upload_result, 'missing properties') + self.assertTrue('upload_error' in upload_result['properties'], 'missing upload_error') + self.assertTrue('An error occurred (NoSuchBucket)'in upload_result['properties']['upload_error'], f"wrong upload_error: {upload_result['properties']['upload_error']}") + + self.assertTrue('assets' in upload_result, 'missing assets') + result_key = [k for k in upload_result['assets'].keys()][0] + self.assertTrue(result_key.startswith('test_file'), f'worng asset key: {result_key}') + result_key_prefix = result_key.split('.')[0] + + self.assertTrue(f'{result_key_prefix}.nc.cas' in upload_result['assets'], f'missing assets#metadata asset: {result_key_prefix}.nc.cas') + self.assertTrue('href' in upload_result['assets'][f'{result_key_prefix}.nc.cas'], 'missing assets#metadata__cas#href') + self.assertTrue(f'{result_key_prefix}.nc' in upload_result['assets'], f'missing assets#data: {result_key_prefix}.nc') + self.assertTrue('href' in upload_result['assets'][f'{result_key_prefix}.nc'], 'missing assets#data#href') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + """ + Example output: + { + 'type': 'FeatureCollection', + 'features': [{ + 'type': 'Feature', + 'stac_version': '1.0.0', + 'id': 'NEW_COLLECTION_EXAMPLE_L1B___9:test_file01', + 'properties': {'start_datetime': '2016-01-31T18:00:00.009057Z', + 'end_datetime': '2016-01-31T19:59:59.991043Z', 'created': '2016-02-01T02:45:59.639000Z', + 'updated': '2022-03-23T15:48:21.578000Z', 'datetime': '1970-01-01T00:00:00Z'}, + 'geometry': {'type': 'Point', 'coordinates': [0.0, 0.0]}, 'links': [], + 'assets': {'data': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc', + 'title': 'main data'}, 'metadata__cas': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.cas', + 'title': 'metadata cas'}, 'metadata__stac': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.stac.json', + 'title': 'metadata stac'}}, + 'bbox': [0.0, 0.0, 0.0, 0.0], + 'stac_extensions': [], + 'collection': 'NEW_COLLECTION_EXAMPLE_L1B___9'}]} + """ + return + + def test_03_upload_complete_catalog_missing_data(self): + os.environ['VERIFY_SSL'] = 'FALSE' + + os.environ['COLLECTION_ID'] = 'NEW_COLLECTION_EXAMPLE_L1B___9' + os.environ['STAGING_BUCKET'] = 'invalid_bucket' + + os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' + # os.environ['GRANULES_UPLOAD_TYPE'] = 'UPLOAD_S3_BY_STAC_CATALOG' + # defaulted to this value + + if len(argv) > 1: + argv.pop(-1) + argv.append('UPLOAD') + + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + os.environ['UPLOAD_DIR'] = '' # not needed + os.environ['CATALOG_FILE'] = os.path.join(tmp_dir_name, 'catalog.json') + os.environ['OUTPUT_DIRECTORY'] = os.path.join(tmp_dir_name, 'output_dir') + FileUtils.mk_dir_p(os.environ.get('OUTPUT_DIRECTORY')) + + total_files = 10 + # os.environ['PARALLEL_COUNT'] = str(total_files) + granules_dir = os.path.join(tmp_dir_name, 'some_granules') + FileUtils.mk_dir_p(granules_dir) + catalog = Catalog( + id='NA', + description='NA') + catalog.set_self_href(os.environ['CATALOG_FILE']) + + for i in range(1, total_files+1): + filename = f'test_file{i:02d}' + with open(os.path.join(granules_dir, f'{filename}.nc'), 'w') as ff: + ff.write('sample_file') + with open(os.path.join(granules_dir, f'{filename}.nc.cas'), 'w') as ff: + ff.write(''' + + + AggregateDir + snppatmsl1a + + + AutomaticQualityFlag + Passed + + + BuildId + v01.43.00 + + + CollectionLabel + L1AMw_nominal2 + + + DataGroup + sndr + + + EndDateTime + 2016-01-14T10:06:00.000Z + + + EndTAI93 + 726919569.000 + + + FileFormat + nc4 + + + FileLocation + /pge/out + + + Filename + SNDR.SNPP.ATMS.L1A.nominal2.02.nc + + + GranuleNumber + 101 + + + JobId + f163835c-9945-472f-bee2-2bc12673569f + + + ModelId + urn:npp:SnppAtmsL1a + + + NominalDate + 2016-01-14 + + + ProductName + SNDR.SNPP.ATMS.20160114T1000.m06.g101.L1A.L1AMw_nominal2.v03_15_00.D.201214135000.nc + + + ProductType + SNDR_SNPP_ATMS_L1A + + + ProductionDateTime + 2020-12-14T13:50:00.000Z + + + ProductionLocation + Sounder SIPS: JPL/Caltech (Dev) + + + ProductionLocationCode + D + + + RequestId + 1215 + + + StartDateTime + 2016-01-14T10:00:00.000Z + + + StartTAI93 + 726919209.000 + + + TaskId + 8c3ae101-8f7c-46c8-b5c6-63e7b6d3c8cd + + ''') + stac_item = Item(id=filename, + geometry={ + "type": "Point", + "coordinates": [0.0, 0.0] + }, + bbox=[0.0, 0.0, 0.0, 0.0], + datetime=TimeUtils().parse_from_unix(0, True).get_datetime_obj(), + properties={ + "start_datetime": "2016-01-31T18:00:00.009057Z", + "end_datetime": "2016-01-31T19:59:59.991043Z", + "created": "2016-02-01T02:45:59.639000Z", + "updated": "2022-03-23T15:48:21.578000Z", + "datetime": "2022-03-23T15:48:19.079000Z" + }, + href=os.path.join('some_granules', f'{filename}.nc.stac.json'), + collection='NA', + assets={ + f'{filename}.nc': Asset(os.path.join('.', f'{filename}.nc'), title='test_file01.nc', roles=['data1']), + f'{filename}.nc.cas': Asset(os.path.join('.', f'{filename}.nc.cas'), title='test_file01.nc.cas', roles=['metadata']), + f'{filename}.nc.stac.json': Asset(os.path.join('.', f'{filename}.nc.stac.json'), title='test_file01.nc.stac.json', roles=['metadata']), + }) + with open(os.path.join(granules_dir, f'{filename}.nc.stac.json'), 'w') as ff: + ff.write(json.dumps(stac_item.to_dict(False, False))) + catalog.add_link(Link('item', os.path.join('some_granules', f'{filename}.nc.stac.json'), 'application/json')) + print(json.dumps(catalog.to_dict(False, False))) + with open(os.environ['CATALOG_FILE'], 'w') as ff: + ff.write(json.dumps(catalog.to_dict(False, False))) + + upload_result_str = UploadGranulesFactory().get_class(os.getenv('GRANULES_UPLOAD_TYPE', UploadGranulesFactory.UPLOAD_S3_BY_STAC_CATALOG)).upload() + upload_result = json.loads(upload_result_str) + print(upload_result) + """ + {'type': 'Catalog', 'id': 'NA', 'stac_version': '1.0.0', 'description': 'NA', 'links': [{'rel': 'root', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/catalog.json', 'type': 'application/json'}, {'rel': 'item', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/successful_features.json', 'type': 'application/json'}, {'rel': 'item', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/failed_features.json', 'type': 'application/json'}]} + """ + self.assertTrue('type' in upload_result, 'missing type') + self.assertEqual(upload_result['type'], 'Catalog', 'missing type') + upload_result = Catalog.from_dict(upload_result) + child_links = [k.href for k in upload_result.get_links(rel='item')] + self.assertEqual(len(child_links), 2, f'wrong length: {child_links}') + self.assertTrue(FileUtils.file_exist(child_links[0]), f'missing file: {child_links[0]}') + successful_feature_collection = ItemCollection.from_dict(FileUtils.read_json(child_links[0])) + successful_feature_collection = list(successful_feature_collection.items) + self.assertEqual(len(successful_feature_collection), 0, f'wrong length: {successful_feature_collection}') + + self.assertTrue(FileUtils.file_exist(child_links[1]), f'missing file: {child_links[1]}') + failed_feature_collection = ItemCollection.from_dict(FileUtils.read_json(child_links[1])) + failed_feature_collection = list(failed_feature_collection.items) + self.assertEqual(len(failed_feature_collection), total_files, f'wrong length: {failed_feature_collection}') + + upload_result = failed_feature_collection[0].to_dict(False, False) + print(f'example feature: {upload_result}') + self.assertTrue('properties' in upload_result, 'missing properties') + self.assertTrue('upload_error' in upload_result['properties'], 'missing upload_error') + self.assertTrue('missing "data" in assets'in upload_result['properties']['upload_error'], f"wrong upload_error: {upload_result['properties']['upload_error']}") + + self.assertTrue('assets' in upload_result, 'missing assets') + result_key = [k for k in upload_result['assets'].keys()][0] + self.assertTrue(result_key.startswith('test_file'), f'worng asset key: {result_key}') + result_key_prefix = result_key.split('.')[0] + + self.assertTrue(f'{result_key_prefix}.nc.cas' in upload_result['assets'], f'missing assets#metadata asset: {result_key_prefix}.nc.cas') + self.assertTrue('href' in upload_result['assets'][f'{result_key_prefix}.nc.cas'], 'missing assets#metadata__cas#href') + self.assertTrue(f'{result_key_prefix}.nc' in upload_result['assets'], f'missing assets#data: {result_key_prefix}.nc') + self.assertTrue('href' in upload_result['assets'][f'{result_key_prefix}.nc'], 'missing assets#data#href') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + """ + Example output: + { + 'type': 'FeatureCollection', + 'features': [{ + 'type': 'Feature', + 'stac_version': '1.0.0', + 'id': 'NEW_COLLECTION_EXAMPLE_L1B___9:test_file01', + 'properties': {'start_datetime': '2016-01-31T18:00:00.009057Z', + 'end_datetime': '2016-01-31T19:59:59.991043Z', 'created': '2016-02-01T02:45:59.639000Z', + 'updated': '2022-03-23T15:48:21.578000Z', 'datetime': '1970-01-01T00:00:00Z'}, + 'geometry': {'type': 'Point', 'coordinates': [0.0, 0.0]}, 'links': [], + 'assets': {'data': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc', + 'title': 'main data'}, 'metadata__cas': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.cas', + 'title': 'metadata cas'}, 'metadata__stac': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.stac.json', + 'title': 'metadata stac'}}, + 'bbox': [0.0, 0.0, 0.0, 0.0], + 'stac_extensions': [], + 'collection': 'NEW_COLLECTION_EXAMPLE_L1B___9'}]} + """ + return From 00d6bded6edaeb42a0d554103e00302296781da2 Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Mon, 14 Oct 2024 10:31:23 -0700 Subject: [PATCH 02/11] chore: update outdated tests --- tests/integration_tests/test_docker_entry.py | 26 +++++++++++--------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/tests/integration_tests/test_docker_entry.py b/tests/integration_tests/test_docker_entry.py index df328b3..c5b396d 100644 --- a/tests/integration_tests/test_docker_entry.py +++ b/tests/integration_tests/test_docker_entry.py @@ -1,5 +1,8 @@ import logging +from mdps_ds_lib.stage_in_out.catalog_granules_factory import CatalogGranulesFactory +from mdps_ds_lib.stage_in_out.search_granules_factory import SearchGranulesFactory + logging.basicConfig(level=10, format="%(asctime)s [%(levelname)s] [%(name)s::%(lineno)d] %(message)s") import math @@ -42,7 +45,7 @@ def test_01_search_part_01(self): argv.append('SEARCH') with tempfile.TemporaryDirectory() as tmp_dir_name: os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') - search_result_str = choose_process() + search_result_str = SearchGranulesFactory().get_class(os.getenv('GRANULES_SEARCH_DOMAIN', 'MISSING_GRANULES_SEARCH_DOMAIN')).search() search_result = json.loads(search_result_str) self.assertTrue('type' in search_result, f'missing type in search_result') item_collections = ItemCollection.from_dict(search_result) @@ -73,7 +76,7 @@ def test_01_search_part_02(self): if len(argv) > 1: argv.pop(-1) argv.append('SEARCH') - search_result = choose_process() + search_result = SearchGranulesFactory().get_class(os.getenv('GRANULES_SEARCH_DOMAIN', 'MISSING_GRANULES_SEARCH_DOMAIN')).search() print(search_result) search_result = json.loads(search_result) self.assertTrue('type' in search_result, f'missing type in search_result') @@ -103,7 +106,7 @@ def test_01_search_part_03(self): if len(argv) > 1: argv.pop(-1) argv.append('SEARCH') - search_result = choose_process() + search_result = SearchGranulesFactory().get_class(os.getenv('GRANULES_SEARCH_DOMAIN', 'MISSING_GRANULES_SEARCH_DOMAIN')).search() search_result = json.loads(search_result) self.assertTrue('type' in search_result, f'missing type in search_result') item_collections = ItemCollection.from_dict(search_result) @@ -132,7 +135,7 @@ def test_01_search_part_04(self): if len(argv) > 1: argv.pop(-1) argv.append('SEARCH') - search_result = choose_process() + search_result = SearchGranulesFactory().get_class(os.getenv('GRANULES_SEARCH_DOMAIN', 'MISSING_GRANULES_SEARCH_DOMAIN')).search() search_result = json.loads(search_result) self.assertTrue('type' in search_result, f'missing type in search_result') item_collections = ItemCollection.from_dict(search_result) @@ -161,7 +164,7 @@ def test_01_search_part_05(self): if len(argv) > 1: argv.pop(-1) argv.append('SEARCH') - search_result = choose_process() + search_result = SearchGranulesFactory().get_class(os.getenv('GRANULES_SEARCH_DOMAIN', 'MISSING_GRANULES_SEARCH_DOMAIN')).search() search_result = json.loads(search_result) self.assertTrue('type' in search_result, f'missing type in search_result') item_collections = ItemCollection.from_dict(search_result) @@ -188,7 +191,7 @@ def test_01_1_search_cmr_part_01(self): if len(argv) > 1: argv.pop(-1) argv.append('SEARCH') - search_result = choose_process() + search_result = SearchGranulesFactory().get_class(os.getenv('GRANULES_SEARCH_DOMAIN', 'MISSING_GRANULES_SEARCH_DOMAIN')).search() search_result = json.loads(search_result) self.assertTrue('type' in search_result, f'missing type in search_result') item_collections = ItemCollection.from_dict(search_result) @@ -215,7 +218,8 @@ def test_01_1_search_cmr_part_02(self): if len(argv) > 1: argv.pop(-1) argv.append('SEARCH') - search_result = choose_process() + search_result = SearchGranulesFactory().get_class(os.getenv('GRANULES_SEARCH_DOMAIN', 'MISSING_GRANULES_SEARCH_DOMAIN')).search() + search_result = json.loads(search_result) self.assertTrue('type' in search_result, f'missing type in search_result') item_collections = ItemCollection.from_dict(search_result) @@ -239,7 +243,7 @@ def test_04_catalog(self): argv.append('CATALOG') with tempfile.TemporaryDirectory() as tmp_dir_name: os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') - catalog_result_str = choose_process() + catalog_result_str = CatalogGranulesFactory().get_class(os.getenv('GRANULES_CATALOG_TYPE', 'MISSING_GRANULES_CATALOG_TYPE')).catalog() catalog_result = json.loads(catalog_result_str) self.assertEqual('registered', catalog_result, 'wrong status') self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') @@ -266,7 +270,7 @@ def test_04_catalog_from_file(self): FileUtils.write_json(input_file_path, upload_result) os.environ['UPLOADED_FILES_JSON'] = input_file_path os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') - catalog_result_str = choose_process() + catalog_result_str = CatalogGranulesFactory().get_class(os.getenv('GRANULES_CATALOG_TYPE', 'MISSING_GRANULES_CATALOG_TYPE')).catalog() catalog_result = json.loads(catalog_result_str) self.assertTrue('cataloging_request_status' in catalog_result, f'missing cataloging_request_status') self.assertTrue('status_result' in catalog_result, f'missing status_result') @@ -317,7 +321,7 @@ def test_04_catalog_from_file_item_collection(self): FileUtils.write_json(input_file_path, upload_result) os.environ['UPLOADED_FILES_JSON'] = input_file_path os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') - catalog_result_str = choose_process() + catalog_result_str = CatalogGranulesFactory().get_class(os.getenv('GRANULES_CATALOG_TYPE', 'MISSING_GRANULES_CATALOG_TYPE')).catalog() catalog_result = json.loads(catalog_result_str) self.assertTrue('cataloging_request_status' in catalog_result, f'missing cataloging_request_status') self.assertTrue('status_result' in catalog_result, f'missing status_result') @@ -355,7 +359,7 @@ def test_04_catalog_from_file_item_collection_large(self): FileUtils.write_json(input_file_path, upload_result) os.environ['UPLOADED_FILES_JSON'] = input_file_path os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') - catalog_result_str = choose_process() + catalog_result_str = CatalogGranulesFactory().get_class(os.getenv('GRANULES_CATALOG_TYPE', 'MISSING_GRANULES_CATALOG_TYPE')).catalog() catalog_result = json.loads(catalog_result_str) self.assertTrue(isinstance(catalog_result, list), f'catalog_result is not list. {catalog_result}') self.assertEqual(len(catalog_result), math.ceil(len(upload_result['features']) / 250), f'mismatched catalog_result count') From a51314b0f34e70a66368f45d04bcc582c76fda57 Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Mon, 21 Oct 2024 12:12:14 -0700 Subject: [PATCH 03/11] fix: allow empty str as RESULT_PATH_PREFIX & replace w/ default val --- mdps_ds_lib/stage_in_out/upload_granules_abstract.py | 2 ++ tests/integration_tests/test_docker_stage_out.py | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/mdps_ds_lib/stage_in_out/upload_granules_abstract.py b/mdps_ds_lib/stage_in_out/upload_granules_abstract.py index 5d803d0..ceedbee 100644 --- a/mdps_ds_lib/stage_in_out/upload_granules_abstract.py +++ b/mdps_ds_lib/stage_in_out/upload_granules_abstract.py @@ -47,6 +47,8 @@ def _set_props_from_env(self): self._collection_id = os.environ.get(self.COLLECTION_ID_KEY) self._staging_bucket = os.environ.get(self.STAGING_BUCKET_KEY) self._result_path_prefix = os.environ.get(self.RESULT_PATH_PREFIX, self.DEFAULT_RESULT_PATH_PREFIX) + if self._result_path_prefix is None or self._result_path_prefix.strip() == '': + self._result_path_prefix = self.DEFAULT_RESULT_PATH_PREFIX self._result_path_prefix = self._result_path_prefix[:-1] if self._result_path_prefix.endswith('/') else self._result_path_prefix self._result_path_prefix = self._result_path_prefix[1:] if self._result_path_prefix.startswith('/') else self._result_path_prefix diff --git a/tests/integration_tests/test_docker_stage_out.py b/tests/integration_tests/test_docker_stage_out.py index a1b92b5..7ee0298 100644 --- a/tests/integration_tests/test_docker_stage_out.py +++ b/tests/integration_tests/test_docker_stage_out.py @@ -1,4 +1,5 @@ import logging + logging.basicConfig(level=20, format="%(asctime)s [%(levelname)s] [%(name)s::%(lineno)d] %(message)s") from datetime import datetime @@ -30,7 +31,7 @@ def setUp(self) -> None: self.tenant = 'UDS_MY_LOCAL_ARCHIVE_TEST' # 'uds_local_test' # 'uds_sandbox' self.tenant_venue = 'DEV' # 'DEV1' # 'dev' self.collection_name = 'UDS_UNIT_COLLECTION' # 'uds_collection' # 'sbx_collection' - self.collection_version = '24.08.29.09.00'.replace('.', '') # '2402011200' + self.collection_version = '24.10.21.12.00'.replace('.', '') # '2402011200' def not_in_used_test_03_upload(self): os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' @@ -791,7 +792,7 @@ def test_03_03_upload_auxiliary_files(self): os.environ['COLLECTION_ID'] = temp_collection_id os.environ['STAGING_BUCKET'] = 'uds-sbx-cumulus-staging' os.environ['VERIFY_SSL'] = 'FALSE' - os.environ['RESULT_PATH_PREFIX'] = 'stage_out' + os.environ['RESULT_PATH_PREFIX'] = '' os.environ['PARALLEL_COUNT'] = '1' if len(argv) > 1: @@ -882,7 +883,7 @@ def test_03_03_upload_auxiliary_files(self): """ s3 = AwsS3() s3_keys = [k for k in s3.get_child_s3_files(os.environ['STAGING_BUCKET'], - f"{os.environ['RESULT_PATH_PREFIX']}/successful_features_{starting_time}", + f"stage_out/successful_features_{starting_time}", )] s3_keys = sorted(s3_keys) print(f's3_keys: {s3_keys}') From 3be75c2253389a30379f53413c95f969d75f7a3a Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Mon, 10 Feb 2025 10:10:41 -0800 Subject: [PATCH 04/11] feat: adding stage-in documentation --- examples/stage-in-stage-out-example.ipynb | 718 ++++++++++++++++++ .../download_granules_amalgamation.py | 2 +- 2 files changed, 719 insertions(+), 1 deletion(-) create mode 100644 examples/stage-in-stage-out-example.ipynb diff --git a/examples/stage-in-stage-out-example.ipynb b/examples/stage-in-stage-out-example.ipynb new file mode 100644 index 0000000..9508459 --- /dev/null +++ b/examples/stage-in-stage-out-example.ipynb @@ -0,0 +1,718 @@ +{ + "cells": [ + { + "metadata": {}, + "cell_type": "markdown", + "source": "## Stage In Stage Out Example", + "id": "30b6016bc9435c51" + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "Steps\n", + "- Install Data Service (DS) Client Library\n", + "- Set Log Level\n", + "- Set Environment variables\n", + "- Try Stage-In\n", + "- Try Stage-Out\n", + "- Check the results" + ], + "id": "41307b0225e83dcc" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T17:41:17.675227Z", + "start_time": "2025-02-10T17:41:16.781958Z" + } + }, + "cell_type": "code", + "source": "%pip install mdps-ds-lib", + "id": "fe80dd452ebd717b", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: mdps-ds-lib in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (1.1.1)\r\n", + "Requirement already satisfied: boto3<2.0.0,>=1.26.51 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (1.36.15)\r\n", + "Requirement already satisfied: elasticsearch==7.13.4 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (7.13.4)\r\n", + "Requirement already satisfied: fastjsonschema<3.0.0,>=2.19.1 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (2.21.1)\r\n", + "Requirement already satisfied: jsonschema<5.0.0,>=4.23.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (4.23.0)\r\n", + "Requirement already satisfied: pystac==1.9.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (1.9.0)\r\n", + "Requirement already satisfied: requests==2.31.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (2.31.0)\r\n", + "Requirement already satisfied: requests-aws4auth==1.2.3 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (1.2.3)\r\n", + "Requirement already satisfied: tenacity==8.2.3 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (8.2.3)\r\n", + "Requirement already satisfied: xmltodict==0.13.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (0.13.0)\r\n", + "Requirement already satisfied: urllib3<2,>=1.21.1 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from elasticsearch==7.13.4->mdps-ds-lib) (1.26.20)\r\n", + "Requirement already satisfied: certifi in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from elasticsearch==7.13.4->mdps-ds-lib) (2024.12.14)\r\n", + "Requirement already satisfied: python-dateutil>=2.7.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from pystac==1.9.0->mdps-ds-lib) (2.9.0.post0)\r\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from requests==2.31.0->mdps-ds-lib) (3.4.1)\r\n", + "Requirement already satisfied: idna<4,>=2.5 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from requests==2.31.0->mdps-ds-lib) (3.10)\r\n", + "Requirement already satisfied: six in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from requests-aws4auth==1.2.3->mdps-ds-lib) (1.17.0)\r\n", + "Requirement already satisfied: botocore<1.37.0,>=1.36.15 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from boto3<2.0.0,>=1.26.51->mdps-ds-lib) (1.36.15)\r\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from boto3<2.0.0,>=1.26.51->mdps-ds-lib) (1.0.1)\r\n", + "Requirement already satisfied: s3transfer<0.12.0,>=0.11.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from boto3<2.0.0,>=1.26.51->mdps-ds-lib) (0.11.2)\r\n", + "Requirement already satisfied: attrs>=22.2.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.23.0->mdps-ds-lib) (24.3.0)\r\n", + "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.23.0->mdps-ds-lib) (2024.10.1)\r\n", + "Requirement already satisfied: referencing>=0.28.4 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.23.0->mdps-ds-lib) (0.35.1)\r\n", + "Requirement already satisfied: rpds-py>=0.7.1 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.23.0->mdps-ds-lib) (0.22.3)\r\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "execution_count": 3 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "### Setting the Log Level\n", + "- Log level mappings:\n", + "- 10 = debug\n", + "- 20 = info\n", + "- 30 = warning\n", + "- 40 = error" + ], + "id": "e6bf69a910209c6c" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T17:41:24.968457Z", + "start_time": "2025-02-10T17:41:24.965823Z" + } + }, + "cell_type": "code", + "source": [ + "import logging\n", + "log_level = 30\n", + "logging.basicConfig(level=log_level, format=\"%(asctime)s [%(levelname)s] [%(name)s::%(lineno)d] %(message)s\")" + ], + "id": "d2aaaee1a57cb095", + "outputs": [], + "execution_count": 4 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "### Setting Environment Variables\n", + "id": "b5f018519b92ea5f" + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "", + "id": "b534ba81c15182d1" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T17:41:38.169484Z", + "start_time": "2025-02-10T17:41:38.166938Z" + } + }, + "cell_type": "code", + "source": [ + "import os\n", + "os.environ['PROJECT'] = 'DEMO'\n", + "os.environ['VENUE'] = 'DEV12'\n", + "\n", + "\n", + "\n", + "#\n", + "# os.environ['VERIFY_SSL'] = 'FALSE'\n", + "# os.environ['RESULT_PATH_PREFIX'] = ''\n", + "# os.environ['STAGING_BUCKET'] = 'uds-sbx-cumulus-staging'\n", + "# os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY'\n", + "# os.environ['OUTPUT_FILE'] = 'normal-stage-out/some_output/output.json'\n", + "# os.environ['UPLOAD_DIR'] = '' # not needed\n", + "# os.environ['OUTPUT_DIRECTORY'] = 'normal-stage-out/output_dir'\n", + "# FileUtils.mk_dir_p(os.environ.get('OUTPUT_DIRECTORY'))\n", + "# os.environ['CATALOG_FILE'] = 'normal-stage-out/catalog.json'" + ], + "id": "36805b7fb91362cb", + "outputs": [], + "execution_count": 5 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "#### Stage-in Only Environment Variables\n", + "\n", + "Follow this link for more information: https://app.gitbook.com/o/xZRqGQeQXJ0RP4VMj7Lq/s/UMIRhLdbRQTvMWop8Il9/developer-docs/data/docs/users-guide/stage-in\n", + "\n" + ], + "id": "8a87921572f1578d" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T17:48:11.471415Z", + "start_time": "2025-02-10T17:48:11.466169Z" + } + }, + "cell_type": "code", + "source": [ + "# The following environment variables are needed to \"stage-in\" from AWS S3\n", + "# They will also be needed if \"EDL\" settings and \"STAC_AUTH_TYPE\" settings are coming from parameter store\n", + "# Note that this may not be needed if JupyterNotebook can take care of the access.\n", + "os.environ['AWS_ACCESS_KEY_ID'] = 'xxx'\n", + "os.environ['AWS_SECRET_ACCESS_KEY'] = 'xxx'\n", + "os.environ['AWS_SESSION_TOKEN'] = 'xxx'\n", + "\n", + "# The following environment variables are ONLY needed if \"stage-in\" requires Earth data Login\n", + "os.environ['EDL_USERNAME'] = '/unity/uds/user/wphyo/edl_username' # Parameter Store Key for EarthData Login Username\n", + "os.environ['EDL_PASSWORD'] = '/unity/uds/user/wphyo/edl_dwssap' # Parameter Store Key for EarthData Login Password\n", + "os.environ['EDL_PASSWORD_TYPE'] = 'PARAM_STORE' # Can hardcode it to PARAM_STORE if that is used.\n", + "os.environ['EDL_BASE_URL'] = 'urs.earthdata.nasa.gov' # Earthdata Login URL to get Earthdata token to download files\n", + "\n", + "# The following environment variables are ONLY needed if \"stage-in\" requires STAC_AUTH_TYPE is \"UNITY\".\n", + "os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' # Look above links for all available options\n", + "os.environ['USERNAME'] = '/unity/uds/user/wphyo/username'\n", + "os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap'\n", + "os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c'\n", + "os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com'\n", + "\n", + "\n", + "# Others environment variables\n", + "os.environ['GRANULES_DOWNLOAD_TYPE'] = 'AMALGAMATION' # Download type to choose download class. AMALGAMATION, S3, DAAC, HTTP, and so on.\n", + "os.environ['DOWNLOADING_KEYS'] = 'data,metadata' # Which asset keys to download\n", + "os.environ['STAC_JSON'] = 'https://raw.githubusercontent.com/GodwinShen/emit-ghg/main/test/catalog.json' # URL to direct which granules + assets to download\n", + "# Other options\n", + "# os.environ['STAC_JSON'] = os.path.join(os.getcwd(), 'stage_in.json') # Alternatively, you can store the file locally, and point it as a path\n", + "os.environ['DOWNLOAD_DIR'] = os.path.join(os.getcwd(), 'downloaded_files') # Base directory where files will be downloaded\n", + "os.environ['OUTPUT_FILE'] = os.path.join(os.getcwd(), 'stage_in_result.json') # file path where the result is written locally for review\n" + ], + "id": "abb5f945876a1a63", + "outputs": [], + "execution_count": 10 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "- This is an example STAC_JSON file (or URL contents) used as a guide for stage-in.\n", + "- It is a STAC \"FeatureCollection\", a wrapper of an array of [Stac items](https://github.com/radiantearth/stac-spec/blob/master/item-spec/item-spec.md)" + ], + "id": "6c17a162ab168924" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T17:41:44.530922Z", + "start_time": "2025-02-10T17:41:44.519332Z" + } + }, + "cell_type": "code", + "source": [ + "{\n", + " \"type\": \"FeatureCollection\",\n", + " \"features\": [\n", + " {\n", + " \"type\": \"Feature\",\n", + " \"stac_version\": \"1.0.0\",\n", + " \"id\": \"G2721220118-LPCLOUD\",\n", + " \"properties\": {\n", + " \"datetime\": \"2023-06-20T08:44:26Z\",\n", + " \"start_datetime\": \"2023-06-20T08:44:26.000Z\",\n", + " \"end_datetime\": \"2023-06-20T08:44:38.000Z\",\n", + " \"eo:cloud_cover\": 41\n", + " },\n", + " \"geometry\": {\n", + " \"type\": \"Polygon\",\n", + " \"coordinates\": [\n", + " [\n", + " [\n", + " 54.235199,\n", + " 39.2283897\n", + " ],\n", + " [\n", + " 53.7996902,\n", + " 38.5576591\n", + " ],\n", + " [\n", + " 54.6095352,\n", + " 38.031826\n", + " ],\n", + " [\n", + " 55.0450439,\n", + " 38.7025566\n", + " ],\n", + " [\n", + " 54.235199,\n", + " 39.2283897\n", + " ]\n", + " ]\n", + " ]\n", + " },\n", + " \"links\": [\n", + " {\n", + " \"rel\": \"self\",\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.stac\"\n", + " },\n", + " {\n", + " \"rel\": \"parent\",\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/C2408009906-LPCLOUD.stac\"\n", + " },\n", + " {\n", + " \"rel\": \"collection\",\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/C2408009906-LPCLOUD.stac\"\n", + " },\n", + " {\n", + " \"rel\": \"root\",\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/\"\n", + " },\n", + " {\n", + " \"rel\": \"via\",\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.json\"\n", + " },\n", + " {\n", + " \"rel\": \"via\",\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.umm_json\"\n", + " }\n", + " ],\n", + " \"assets\": {\n", + " \"metadata\": {\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.xml\",\n", + " \"type\": \"application/xml\"\n", + " },\n", + " \"browse\": {\n", + " \"href\": \"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-public/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_RAD_001_20230620T084426_2317106_011.png\",\n", + " \"type\": \"image/png\",\n", + " \"title\": \"Download EMIT_L1B_RAD_001_20230620T084426_2317106_011.png\"\n", + " },\n", + " \"opendap\": {\n", + " \"href\": \"https://opendap.earthdata.nasa.gov/collections/C2408009906-LPCLOUD/granules/EMIT_L1B_RAD_001_20230620T084426_2317106_011\",\n", + " \"title\": \"OPeNDAP request URL\"\n", + " },\n", + " \"data\": {\n", + " \"href\": \"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc\",\n", + " \"title\": \"Download EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc\"\n", + " },\n", + " \"data1\": {\n", + " \"href\": \"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc\",\n", + " \"title\": \"Download EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc\"\n", + " }\n", + " },\n", + " \"bbox\": [\n", + " 53.7996902,\n", + " 38.031826,\n", + " 55.0450439,\n", + " 39.2283897\n", + " ],\n", + " \"stac_extensions\": [\n", + " \"https://stac-extensions.github.io/eo/v1.0.0/schema.json\"\n", + " ],\n", + " \"collection\": \"C2408009906-LPCLOUD\"\n", + " },\n", + " {\n", + " \"type\": \"Feature\",\n", + " \"stac_version\": \"1.0.0\",\n", + " \"id\": \"G2721699381-LPCLOUD\",\n", + " \"properties\": {\n", + " \"datetime\": \"2023-06-20T08:44:26Z\",\n", + " \"start_datetime\": \"2023-06-20T08:44:26.000Z\",\n", + " \"end_datetime\": \"2023-06-20T08:44:38.000Z\",\n", + " \"eo:cloud_cover\": 41\n", + " },\n", + " \"geometry\": {\n", + " \"type\": \"Polygon\",\n", + " \"coordinates\": [\n", + " [\n", + " [\n", + " 54.235199,\n", + " 39.2283897\n", + " ],\n", + " [\n", + " 53.7996902,\n", + " 38.5576591\n", + " ],\n", + " [\n", + " 54.6095352,\n", + " 38.031826\n", + " ],\n", + " [\n", + " 55.0450439,\n", + " 38.7025566\n", + " ],\n", + " [\n", + " 54.235199,\n", + " 39.2283897\n", + " ]\n", + " ]\n", + " ]\n", + " },\n", + " \"links\": [\n", + " {\n", + " \"rel\": \"self\",\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721699381-LPCLOUD.stac\"\n", + " },\n", + " {\n", + " \"rel\": \"parent\",\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/C2408750690-LPCLOUD.stac\"\n", + " },\n", + " {\n", + " \"rel\": \"collection\",\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/C2408750690-LPCLOUD.stac\"\n", + " },\n", + " {\n", + " \"rel\": \"root\",\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/\"\n", + " },\n", + " {\n", + " \"rel\": \"via\",\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721699381-LPCLOUD.json\"\n", + " },\n", + " {\n", + " \"rel\": \"via\",\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721699381-LPCLOUD.umm_json\"\n", + " }\n", + " ],\n", + " \"assets\": {\n", + " \"metadata\": {\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721699381-LPCLOUD.xml\",\n", + " \"type\": \"application/xml\"\n", + " },\n", + " \"browse\": {\n", + " \"href\": \"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-public/EMITL2ARFL.001/EMIT_L2A_RFL_001_20230620T084426_2317106_011/EMIT_L2A_RFL_001_20230620T084426_2317106_011.png\",\n", + " \"type\": \"image/png\",\n", + " \"title\": \"Download EMIT_L2A_RFL_001_20230620T084426_2317106_011.png\"\n", + " },\n", + " \"opendap\": {\n", + " \"href\": \"https://opendap.earthdata.nasa.gov/collections/C2408750690-LPCLOUD/granules/EMIT_L2A_RFL_001_20230620T084426_2317106_011\",\n", + " \"title\": \"OPeNDAP request URL\"\n", + " },\n", + " \"data\": {\n", + " \"href\": \"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL2ARFL.001/EMIT_L2A_RFL_001_20230620T084426_2317106_011/EMIT_L2A_RFL_001_20230620T084426_2317106_011.nc\",\n", + " \"title\": \"Download EMIT_L2A_RFL_001_20230620T084426_2317106_011.nc\"\n", + " },\n", + " \"data1\": {\n", + " \"href\": \"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL2ARFL.001/EMIT_L2A_RFL_001_20230620T084426_2317106_011/EMIT_L2A_RFLUNCERT_001_20230620T084426_2317106_011.nc\",\n", + " \"title\": \"Download EMIT_L2A_RFLUNCERT_001_20230620T084426_2317106_011.nc\"\n", + " },\n", + " \"data2\": {\n", + " \"href\": \"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL2ARFL.001/EMIT_L2A_RFL_001_20230620T084426_2317106_011/EMIT_L2A_MASK_001_20230620T084426_2317106_011.nc\",\n", + " \"title\": \"Download EMIT_L2A_MASK_001_20230620T084426_2317106_011.nc\"\n", + " }\n", + " },\n", + " \"bbox\": [\n", + " 53.7996902,\n", + " 38.031826,\n", + " 55.0450439,\n", + " 39.2283897\n", + " ],\n", + " \"stac_extensions\": [\n", + " \"https://stac-extensions.github.io/eo/v1.0.0/schema.json\"\n", + " ],\n", + " \"collection\": \"C2408750690-LPCLOUD\"\n", + " }\n", + " ]\n", + "}" + ], + "id": "8a3c3ceaf481e2fe", + "outputs": [ + { + "data": { + "text/plain": [ + "{'type': 'FeatureCollection',\n", + " 'features': [{'type': 'Feature',\n", + " 'stac_version': '1.0.0',\n", + " 'id': 'G2721220118-LPCLOUD',\n", + " 'properties': {'datetime': '2023-06-20T08:44:26Z',\n", + " 'start_datetime': '2023-06-20T08:44:26.000Z',\n", + " 'end_datetime': '2023-06-20T08:44:38.000Z',\n", + " 'eo:cloud_cover': 41},\n", + " 'geometry': {'type': 'Polygon',\n", + " 'coordinates': [[[54.235199, 39.2283897],\n", + " [53.7996902, 38.5576591],\n", + " [54.6095352, 38.031826],\n", + " [55.0450439, 38.7025566],\n", + " [54.235199, 39.2283897]]]},\n", + " 'links': [{'rel': 'self',\n", + " 'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.stac'},\n", + " {'rel': 'parent',\n", + " 'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/C2408009906-LPCLOUD.stac'},\n", + " {'rel': 'collection',\n", + " 'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/C2408009906-LPCLOUD.stac'},\n", + " {'rel': 'root', 'href': 'https://cmr.earthdata.nasa.gov:443/search/'},\n", + " {'rel': 'via',\n", + " 'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.json'},\n", + " {'rel': 'via',\n", + " 'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.umm_json'}],\n", + " 'assets': {'metadata': {'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.xml',\n", + " 'type': 'application/xml'},\n", + " 'browse': {'href': 'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-public/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_RAD_001_20230620T084426_2317106_011.png',\n", + " 'type': 'image/png',\n", + " 'title': 'Download EMIT_L1B_RAD_001_20230620T084426_2317106_011.png'},\n", + " 'opendap': {'href': 'https://opendap.earthdata.nasa.gov/collections/C2408009906-LPCLOUD/granules/EMIT_L1B_RAD_001_20230620T084426_2317106_011',\n", + " 'title': 'OPeNDAP request URL'},\n", + " 'data': {'href': 'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc',\n", + " 'title': 'Download EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc'},\n", + " 'data1': {'href': 'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc',\n", + " 'title': 'Download EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc'}},\n", + " 'bbox': [53.7996902, 38.031826, 55.0450439, 39.2283897],\n", + " 'stac_extensions': ['https://stac-extensions.github.io/eo/v1.0.0/schema.json'],\n", + " 'collection': 'C2408009906-LPCLOUD'},\n", + " {'type': 'Feature',\n", + " 'stac_version': '1.0.0',\n", + " 'id': 'G2721699381-LPCLOUD',\n", + " 'properties': {'datetime': '2023-06-20T08:44:26Z',\n", + " 'start_datetime': '2023-06-20T08:44:26.000Z',\n", + " 'end_datetime': '2023-06-20T08:44:38.000Z',\n", + " 'eo:cloud_cover': 41},\n", + " 'geometry': {'type': 'Polygon',\n", + " 'coordinates': [[[54.235199, 39.2283897],\n", + " [53.7996902, 38.5576591],\n", + " [54.6095352, 38.031826],\n", + " [55.0450439, 38.7025566],\n", + " [54.235199, 39.2283897]]]},\n", + " 'links': [{'rel': 'self',\n", + " 'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/G2721699381-LPCLOUD.stac'},\n", + " {'rel': 'parent',\n", + " 'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/C2408750690-LPCLOUD.stac'},\n", + " {'rel': 'collection',\n", + " 'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/C2408750690-LPCLOUD.stac'},\n", + " {'rel': 'root', 'href': 'https://cmr.earthdata.nasa.gov:443/search/'},\n", + " {'rel': 'via',\n", + " 'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/G2721699381-LPCLOUD.json'},\n", + " {'rel': 'via',\n", + " 'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/G2721699381-LPCLOUD.umm_json'}],\n", + " 'assets': {'metadata': {'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/G2721699381-LPCLOUD.xml',\n", + " 'type': 'application/xml'},\n", + " 'browse': {'href': 'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-public/EMITL2ARFL.001/EMIT_L2A_RFL_001_20230620T084426_2317106_011/EMIT_L2A_RFL_001_20230620T084426_2317106_011.png',\n", + " 'type': 'image/png',\n", + " 'title': 'Download EMIT_L2A_RFL_001_20230620T084426_2317106_011.png'},\n", + " 'opendap': {'href': 'https://opendap.earthdata.nasa.gov/collections/C2408750690-LPCLOUD/granules/EMIT_L2A_RFL_001_20230620T084426_2317106_011',\n", + " 'title': 'OPeNDAP request URL'},\n", + " 'data': {'href': 'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL2ARFL.001/EMIT_L2A_RFL_001_20230620T084426_2317106_011/EMIT_L2A_RFL_001_20230620T084426_2317106_011.nc',\n", + " 'title': 'Download EMIT_L2A_RFL_001_20230620T084426_2317106_011.nc'},\n", + " 'data1': {'href': 'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL2ARFL.001/EMIT_L2A_RFL_001_20230620T084426_2317106_011/EMIT_L2A_RFLUNCERT_001_20230620T084426_2317106_011.nc',\n", + " 'title': 'Download EMIT_L2A_RFLUNCERT_001_20230620T084426_2317106_011.nc'},\n", + " 'data2': {'href': 'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL2ARFL.001/EMIT_L2A_RFL_001_20230620T084426_2317106_011/EMIT_L2A_MASK_001_20230620T084426_2317106_011.nc',\n", + " 'title': 'Download EMIT_L2A_MASK_001_20230620T084426_2317106_011.nc'}},\n", + " 'bbox': [53.7996902, 38.031826, 55.0450439, 39.2283897],\n", + " 'stac_extensions': ['https://stac-extensions.github.io/eo/v1.0.0/schema.json'],\n", + " 'collection': 'C2408750690-LPCLOUD'}]}" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 7 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "#### How to create a Stac Item and a Feature Collection", + "id": "e0d932f5366ff630" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T17:41:47.855545Z", + "start_time": "2025-02-10T17:41:47.853160Z" + } + }, + "cell_type": "code", + "source": "# TODO How do I do this?", + "id": "1429c01da795b287", + "outputs": [], + "execution_count": 8 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "#### Performing Stage-in\n", + "- Creating Stage-in Directory\n", + "- Calling Stage-in method to perform stage-in.\n", + "- Checking the result." + ], + "id": "f7756b2cea8a3f" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T17:54:08.217050Z", + "start_time": "2025-02-10T17:48:17.006382Z" + } + }, + "cell_type": "code", + "source": [ + "from mdps_ds_lib.stage_in_out.download_granules_factory import DownloadGranulesFactory\n", + "from mdps_ds_lib.stage_in_out.stage_in_out_utils import StageInOutUtils\n", + "from mdps_ds_lib.lib.utils.file_utils import FileUtils\n", + "from glob import glob\n", + "\n", + "FileUtils.mk_dir_p(os.environ['DOWNLOAD_DIR']) # Creating a base directory if not created. They can be created manually w/o calling this.\n", + "\n", + "# Hardcoded method call. All params are set via environment previously\n", + "result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download()\n", + "StageInOutUtils.write_output_to_file(result_str)\n", + "print('done')\n", + "\n", + "print(list(glob(os.path.join(os.environ['DOWNLOAD_DIR'], '*')))) # Checking if files are downloaded." + ], + "id": "1353836ca421355e", + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-10 09:48:19,170 [ERROR] [mdps_ds_lib.stage_in_out.download_granules_amalgamation::32] downloading: https://cmr.earthdata.nasa.gov:443/search/concepts/G2721699381-LPCLOUD.xml\n", + "2025-02-10 09:48:19,170 [ERROR] [mdps_ds_lib.stage_in_out.download_granules_amalgamation::32] downloading: https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.xml\n", + "2025-02-10 09:48:19,497 [ERROR] [mdps_ds_lib.stage_in_out.download_granules_amalgamation::32] downloading: https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL2ARFL.001/EMIT_L2A_RFL_001_20230620T084426_2317106_011/EMIT_L2A_RFL_001_20230620T084426_2317106_011.nc\n", + "2025-02-10 09:48:19,511 [ERROR] [mdps_ds_lib.stage_in_out.download_granules_amalgamation::32] downloading: https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "done\n", + "['/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/error.log', '/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/G2721699381-LPCLOUD.xml', '/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/G2721220118-LPCLOUD.xml', '/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/G2721220118-LPCLOUD.stac.json', '/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/G2721699381-LPCLOUD.stac.json', '/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/catalog.json', '/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc', '/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/EMIT_L2A_RFL_001_20230620T084426_2317106_011.nc']\n" + ] + } + ], + "execution_count": 11 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "#### Stage-out Only Environment Variables\n", + "\n", + "Follow this link for more information: https://app.gitbook.com/o/xZRqGQeQXJ0RP4VMj7Lq/s/UMIRhLdbRQTvMWop8Il9/developer-docs/data/docs/users-guide/stage-out" + ], + "id": "44c5455db73d82c0" + }, + { + "metadata": {}, + "cell_type": "code", + "outputs": [], + "execution_count": null, + "source": [ + "os.environ['VERIFY_SSL'] = 'FALSE'\n", + "os.environ['RESULT_PATH_PREFIX'] = ''\n", + "os.environ['STAGING_BUCKET'] = 'uds-sbx-cumulus-staging'\n", + "os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY'\n", + "os.environ['OUTPUT_FILE'] = 'normal-stage-out/some_output/output.json'\n", + "os.environ['UPLOAD_DIR'] = '' # not needed\n", + "os.environ['OUTPUT_DIRECTORY'] = 'normal-stage-out/output_dir'\n", + "FileUtils.mk_dir_p(os.environ.get('OUTPUT_DIRECTORY'))\n", + "os.environ['CATALOG_FILE'] = 'normal-stage-out/catalog.json'" + ], + "id": "518885efaf032c92" + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "Sample catalog.json used as a guide for stage-out", + "id": "c4db172f84e3e689" + }, + { + "metadata": {}, + "cell_type": "code", + "outputs": [], + "execution_count": null, + "source": [ + "{\n", + " \"type\": \"Catalog\",\n", + " \"id\": \"NA\",\n", + " \"stac_version\": \"1.0.0\",\n", + " \"description\": \"NA\",\n", + " \"links\": [\n", + " {\n", + " \"rel\": \"root\",\n", + " \"href\": \"/tmp/normal-stage-out/catalog.json\",\n", + " \"type\": \"application/json\"\n", + " },\n", + " {\n", + " \"rel\": \"item\",\n", + " \"href\": \"some_granules/test_file01.nc.stac.json\",\n", + " \"type\": \"application/json\"\n", + " },\n", + " {\n", + " \"rel\": \"item\",\n", + " \"href\": \"some_granules/test_file02.nc.stac.json\",\n", + " \"type\": \"application/json\"\n", + " },\n", + " {\n", + " \"rel\": \"item\",\n", + " \"href\": \"some_granules/test_file03.nc.stac.json\",\n", + " \"type\": \"application/json\"\n", + " },\n", + " {\n", + " \"rel\": \"item\",\n", + " \"href\": \"some_granules/test_file04.nc.stac.json\",\n", + " \"type\": \"application/json\"\n", + " },\n", + " {\n", + " \"rel\": \"item\",\n", + " \"href\": \"some_granules/test_file05.nc.stac.json\",\n", + " \"type\": \"application/json\"\n", + " },\n", + " {\n", + " \"rel\": \"item\",\n", + " \"href\": \"some_granules/test_file06.nc.stac.json\",\n", + " \"type\": \"application/json\"\n", + " },\n", + " {\n", + " \"rel\": \"item\",\n", + " \"href\": \"some_granules/test_file07.nc.stac.json\",\n", + " \"type\": \"application/json\"\n", + " },\n", + " {\n", + " \"rel\": \"item\",\n", + " \"href\": \"some_granules/test_file08.nc.stac.json\",\n", + " \"type\": \"application/json\"\n", + " },\n", + " {\n", + " \"rel\": \"item\",\n", + " \"href\": \"some_granules/test_file09.nc.stac.json\",\n", + " \"type\": \"application/json\"\n", + " },\n", + " {\n", + " \"rel\": \"item\",\n", + " \"href\": \"some_granules/test_file10.nc.stac.json\",\n", + " \"type\": \"application/json\"\n", + " }\n", + " ]\n", + "}" + ], + "id": "1b7097396bfbb1f3" + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "", + "id": "f71cb9161be6325d" + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/mdps_ds_lib/stage_in_out/download_granules_amalgamation.py b/mdps_ds_lib/stage_in_out/download_granules_amalgamation.py index 65a91f9..9156dae 100644 --- a/mdps_ds_lib/stage_in_out/download_granules_amalgamation.py +++ b/mdps_ds_lib/stage_in_out/download_granules_amalgamation.py @@ -29,7 +29,7 @@ def _set_props_from_env(self): return self def download_one_item(self, downloading_url: str): - LOGGER.error(f'downloading: {downloading_url}') + LOGGER.debug(f'downloading: {downloading_url}') upper_download_url = downloading_url.upper() if upper_download_url.startswith('S3://'): download_s3 = DownloadGranulesS3() From a0d349385dce39075ff9e8af2520ed8783dff0a4 Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Mon, 10 Feb 2025 13:29:57 -0800 Subject: [PATCH 05/11] fix: stage-out --- examples/stage-in-stage-out-example.ipynb | 364 +++++++++++++++++++++- 1 file changed, 347 insertions(+), 17 deletions(-) diff --git a/examples/stage-in-stage-out-example.ipynb b/examples/stage-in-stage-out-example.ipynb index 9508459..0adfee4 100644 --- a/examples/stage-in-stage-out-example.ipynb +++ b/examples/stage-in-stage-out-example.ipynb @@ -13,9 +13,12 @@ "Steps\n", "- Install Data Service (DS) Client Library\n", "- Set Log Level\n", - "- Set Environment variables\n", + "- Set Environment variables for Stage-In\n", "- Try Stage-In\n", + "- Try Stage-Out (Dry-Run Only)\n", + "- Set Environment variables for Stage-In\n", "- Try Stage-Out\n", + "- Try Stage-Out Ancillary Files\n", "- Check the results" ], "id": "41307b0225e83dcc" @@ -28,7 +31,10 @@ } }, "cell_type": "code", - "source": "%pip install mdps-ds-lib", + "source": [ + "from anyio.streams import file\n", + "%pip install mdps-ds-lib" + ], "id": "fe80dd452ebd717b", "outputs": [ { @@ -100,12 +106,6 @@ "source": "### Setting Environment Variables\n", "id": "b5f018519b92ea5f" }, - { - "metadata": {}, - "cell_type": "markdown", - "source": "", - "id": "b534ba81c15182d1" - }, { "metadata": { "ExecuteTime": { @@ -585,34 +585,50 @@ "metadata": {}, "cell_type": "markdown", "source": [ - "#### Stage-out Only Environment Variables\n", + "### Stage-out Only Environment Variables\n", "\n", "Follow this link for more information: https://app.gitbook.com/o/xZRqGQeQXJ0RP4VMj7Lq/s/UMIRhLdbRQTvMWop8Il9/developer-docs/data/docs/users-guide/stage-out" ], "id": "44c5455db73d82c0" }, { - "metadata": {}, + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T21:25:22.300759Z", + "start_time": "2025-02-10T21:25:22.297419Z" + } + }, "cell_type": "code", - "outputs": [], - "execution_count": null, "source": [ + "from mdps_ds_lib.lib.utils.file_utils import FileUtils\n", + "\n", + "# The following environment variables are needed to \"stage-in\" from AWS S3\n", + "# They will also be needed if \"EDL\" settings and \"STAC_AUTH_TYPE\" settings are coming from parameter store\n", + "# Note that this may not be needed if JupyterNotebook can take care of the access.\n", + "os.environ['AWS_ACCESS_KEY_ID'] = 'xxx'\n", + "os.environ['AWS_SECRET_ACCESS_KEY'] = 'xxx'\n", + "os.environ['AWS_SESSION_TOKEN'] = 'xxx'\n", + "\n", + "os.environ['PROJECT'] = 'DEMO'\n", + "os.environ['VENUE'] = 'DEV43'\n", + "\n", "os.environ['VERIFY_SSL'] = 'FALSE'\n", - "os.environ['RESULT_PATH_PREFIX'] = ''\n", + "os.environ['RESULT_PATH_PREFIX'] = '' # We can usually ignore this\n", "os.environ['STAGING_BUCKET'] = 'uds-sbx-cumulus-staging'\n", "os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY'\n", "os.environ['OUTPUT_FILE'] = 'normal-stage-out/some_output/output.json'\n", - "os.environ['UPLOAD_DIR'] = '' # not needed\n", "os.environ['OUTPUT_DIRECTORY'] = 'normal-stage-out/output_dir'\n", "FileUtils.mk_dir_p(os.environ.get('OUTPUT_DIRECTORY'))\n", "os.environ['CATALOG_FILE'] = 'normal-stage-out/catalog.json'" ], - "id": "518885efaf032c92" + "id": "518885efaf032c92", + "outputs": [], + "execution_count": 23 }, { "metadata": {}, "cell_type": "markdown", - "source": "Sample catalog.json used as a guide for stage-out", + "source": "#### Sample catalog.json used as a guide for stage-out", "id": "c4db172f84e3e689" }, { @@ -690,8 +706,322 @@ { "metadata": {}, "cell_type": "markdown", - "source": "", + "source": [ + "- Creating Some Mock Files for stage-out including STAC metadata file and a catalog\n", + "- Note that this is usually not done here as the application will take care of it.\n", + "- How each stac metadata file is created can be used as an example though" + ], "id": "f71cb9161be6325d" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T19:51:39.435978Z", + "start_time": "2025-02-10T19:51:39.425621Z" + } + }, + "cell_type": "code", + "source": [ + "import json\n", + "from mdps_ds_lib.lib.utils.time_utils import TimeUtils\n", + "from pystac import Item, Asset, Catalog, Link\n", + "\n", + "granules_dir = os.path.join('normal-stage-out', 'some_granules')\n", + "FileUtils.mk_dir_p(granules_dir) # base directory\n", + "total_files = 3 # Uploading 3 mock granules\n", + "\n", + "catalog = Catalog( # Creating a STAC catalog (an example from previous cell)\n", + " id='NA', # we don't need to know the ID\n", + " description='NA')\n", + "catalog.set_self_href(os.environ['CATALOG_FILE'])\n", + "\n", + "for i in range(1, total_files+1):\n", + " filename = f'test_file{i:02d}'\n", + " with open(os.path.join(granules_dir, f'{filename}.nc'), 'w') as ff: # Creating Data File\n", + " ff.write('sample_file')\n", + " with open(os.path.join(granules_dir, f'{filename}.nc.cas'), 'w') as ff: # Creating native metadata file\n", + " ff.write('''\n", + " \n", + " \n", + " AggregateDir\n", + " snppatmsl1a\n", + " \n", + " \n", + " AutomaticQualityFlag\n", + " Passed\n", + " \n", + " \n", + " BuildId\n", + " v01.43.00\n", + " \n", + " \n", + " CollectionLabel\n", + " L1AMw_nominal2\n", + " \n", + " \n", + " DataGroup\n", + " sndr\n", + " \n", + " \n", + " EndDateTime\n", + " 2016-01-14T10:06:00.000Z\n", + " \n", + " \n", + " EndTAI93\n", + " 726919569.000\n", + " \n", + " \n", + " FileFormat\n", + " nc4\n", + " \n", + " \n", + " FileLocation\n", + " /pge/out\n", + " \n", + " \n", + " Filename\n", + " SNDR.SNPP.ATMS.L1A.nominal2.02.nc\n", + " \n", + " \n", + " GranuleNumber\n", + " 101\n", + " \n", + " \n", + " JobId\n", + " f163835c-9945-472f-bee2-2bc12673569f\n", + " \n", + " \n", + " ModelId\n", + " urn:npp:SnppAtmsL1a\n", + " \n", + " \n", + " NominalDate\n", + " 2016-01-14\n", + " \n", + " \n", + " ProductName\n", + " SNDR.SNPP.ATMS.20160114T1000.m06.g101.L1A.L1AMw_nominal2.v03_15_00.D.201214135000.nc\n", + " \n", + " \n", + " ProductType\n", + " SNDR_SNPP_ATMS_L1A\n", + " \n", + " \n", + " ProductionDateTime\n", + " 2020-12-14T13:50:00.000Z\n", + " \n", + " \n", + " ProductionLocation\n", + " Sounder SIPS: JPL/Caltech (Dev)\n", + " \n", + " \n", + " ProductionLocationCode\n", + " D\n", + " \n", + " \n", + " RequestId\n", + " 1215\n", + " \n", + " \n", + " StartDateTime\n", + " 2016-01-14T10:00:00.000Z\n", + " \n", + " \n", + " StartTAI93\n", + " 726919209.000\n", + " \n", + " \n", + " TaskId\n", + " 8c3ae101-8f7c-46c8-b5c6-63e7b6d3c8cd\n", + " \n", + " ''')\n", + " # Creating STAC metadata file object which is used by DS during stage-out and cataloging.\n", + " # pystac library is used to create a stac item file to have a standardized stac.\n", + " stac_item = Item(id=filename,\n", + " geometry={ # Set them if the algorithm knows what type of geometry is needed\n", + " \"type\": \"Point\",\n", + " \"coordinates\": [0.0, 0.0]\n", + " },\n", + " bbox=[-180, -90, 180, 90], # Set them if the algorithm knows what type of geometry is needed\n", + " datetime=TimeUtils().parse_from_unix(0, True).get_datetime_obj(), # Current metadata file creation datetime\n", + " properties={ # These 4 fields are mandatory\n", + " \"start_datetime\": \"2016-01-31T18:00:00.009057Z\",\n", + " \"end_datetime\": \"2016-01-31T19:59:59.991043Z\",\n", + " \"created\": \"2016-02-01T02:45:59.639000Z\",\n", + " \"updated\": \"2022-03-23T15:48:21.578000Z\",\n", + " },\n", + " href=os.path.join('some_granules', f'{filename}.nc.stac.json'),\n", + " collection='NA', # No need to find out what collection it belongs to DS will take care of that.\n", + " assets={ # Point to all relevant files including itself\n", + " f'{filename}.nc': Asset(os.path.join('.', f'{filename}.nc'), title='test_file01.nc', roles=['data']),\n", + " f'{filename}.nc.cas': Asset(os.path.join('.', f'{filename}.nc.cas'), title='test_file01.nc.cas', roles=['metadata']),\n", + " f'{filename}.nc.stac.json': Asset(os.path.join('.', f'{filename}.nc.stac.json'), title='test_file01.nc.stac.json', roles=['metadata']),\n", + " })\n", + " with open(os.path.join(granules_dir, f'{filename}.nc.stac.json'), 'w') as ff: # Creating STAC metadata file\n", + " ff.write(json.dumps(stac_item.to_dict(False, False)))\n", + " # Adding to the Catalog so that DS can find out the STAC file and finds out other related files from STAC\n", + " catalog.add_link(Link('item', os.path.join('some_granules', f'{filename}.nc.stac.json'), 'application/json'))\n", + "# Writing the main catalog file\n", + "with open(os.environ['CATALOG_FILE'], 'w') as ff:\n", + " ff.write(json.dumps(catalog.to_dict(False, False)))\n" + ], + "id": "cd2793837298e627", + "outputs": [], + "execution_count": 14 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "### Performing Stage-out as a dry-run to verify if things are setup correctly", + "id": "7cd0f6aaaa07ef8e" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T21:26:41.690596Z", + "start_time": "2025-02-10T21:26:38.269917Z" + } + }, + "cell_type": "code", + "source": [ + "os.environ['DRY_RUN'] = 'TRUE'\n", + "from mdps_ds_lib.stage_in_out.upoad_granules_factory import UploadGranulesFactory\n", + "\n", + "upload_result = UploadGranulesFactory().get_class(UploadGranulesFactory.UPLOAD_S3_BY_STAC_CATALOG).upload()\n", + "print(upload_result)" + ], + "id": "c46d384d19572817", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Result of dry-run\n", + "{\n", + " \"granule_file\": \"normal-stage-out/some_granules/test_file02.nc.stac.json\",\n", + " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file02/test_file02.nc\"\n", + "}\n", + "{\n", + " \"granule_file\": \"normal-stage-out/some_granules/test_file02.nc.stac.json\",\n", + " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file02/test_file02.nc.cas\"\n", + "}\n", + "{\n", + " \"granule_file\": \"normal-stage-out/some_granules/test_file02.nc.stac.json\",\n", + " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file02/test_file02.nc.stac.json\"\n", + "}\n", + "{\n", + " \"granule_file\": \"normal-stage-out/some_granules/test_file03.nc.stac.json\",\n", + " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file03/test_file03.nc\"\n", + "}\n", + "{\n", + " \"granule_file\": \"normal-stage-out/some_granules/test_file03.nc.stac.json\",\n", + " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file03/test_file03.nc.cas\"\n", + "}\n", + "{\n", + " \"granule_file\": \"normal-stage-out/some_granules/test_file03.nc.stac.json\",\n", + " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file03/test_file03.nc.stac.json\"\n", + "}\n", + "{\n", + " \"granule_file\": \"normal-stage-out/some_granules/test_file01.nc.stac.json\",\n", + " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file01/test_file01.nc\"\n", + "}\n", + "{\n", + " \"granule_file\": \"normal-stage-out/some_granules/test_file01.nc.stac.json\",\n", + " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file01/test_file01.nc.cas\"\n", + "}\n", + "{\n", + " \"granule_file\": \"normal-stage-out/some_granules/test_file01.nc.stac.json\",\n", + " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file01/test_file01.nc.stac.json\"\n", + "}\n", + "{}\n" + ] + } + ], + "execution_count": 25 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "### Performing Stage-out", + "id": "e87a8f50d52e90dc" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T21:29:21.314515Z", + "start_time": "2025-02-10T21:29:21.311438Z" + } + }, + "cell_type": "code", + "source": [ + "# Cleaning old result files.\n", + "\n", + "old_result_files = glob(os.path.join(os.environ['OUTPUT_DIRECTORY'], '*'))\n", + "for each_file in old_result_files:\n", + " FileUtils.remove_if_exists(each_file)" + ], + "id": "cda0ee7d594d1513", + "outputs": [], + "execution_count": 32 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T21:29:27.704310Z", + "start_time": "2025-02-10T21:29:23.201698Z" + } + }, + "cell_type": "code", + "source": [ + "\n", + "os.environ['DRY_RUN'] = 'FALSE'\n", + "\n", + "FileUtils.remove_if_exists(os.environ['OUTPUT_DIRECTORY'])\n", + "from mdps_ds_lib.stage_in_out.upoad_granules_factory import UploadGranulesFactory\n", + "\n", + "upload_result = UploadGranulesFactory().get_class(UploadGranulesFactory.UPLOAD_S3_BY_STAC_CATALOG).upload()\n", + "print(upload_result)" + ], + "id": "2e98b386bd09a655", + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-10 13:29:23,648 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=data, name=test_file01.nc, href=normal-stage-out/some_granules/./test_file01.nc\n", + "2025-02-10 13:29:23,648 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=data, name=test_file02.nc, href=normal-stage-out/some_granules/./test_file02.nc\n", + "2025-02-10 13:29:23,650 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=data, name=test_file03.nc, href=normal-stage-out/some_granules/./test_file03.nc\n", + "2025-02-10 13:29:24,041 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=metadata, name=test_file02.nc.cas, href=normal-stage-out/some_granules/./test_file02.nc.cas\n", + "2025-02-10 13:29:24,043 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=metadata, name=test_file03.nc.cas, href=normal-stage-out/some_granules/./test_file03.nc.cas\n", + "2025-02-10 13:29:24,053 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=metadata, name=test_file01.nc.cas, href=normal-stage-out/some_granules/./test_file01.nc.cas\n", + "2025-02-10 13:29:24,202 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=metadata, name=test_file02.nc.stac.json, href=normal-stage-out/some_granules/./test_file02.nc.stac.json\n", + "2025-02-10 13:29:24,207 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=metadata, name=test_file03.nc.stac.json, href=normal-stage-out/some_granules/./test_file03.nc.stac.json\n", + "2025-02-10 13:29:24,215 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=metadata, name=test_file01.nc.stac.json, href=normal-stage-out/some_granules/./test_file01.nc.stac.json\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\"type\": \"Catalog\", \"id\": \"NA\", \"stac_version\": \"1.0.0\", \"description\": \"NA\", \"links\": [{\"rel\": \"root\", \"href\": \"/Users/wphyo/Projects/unity/uds_lib/examples/normal-stage-out/catalog.json\", \"type\": \"application/json\"}, {\"rel\": \"item\", \"href\": \"normal-stage-out/output_dir/successful_features.json\", \"type\": \"application/json\"}, {\"rel\": \"item\", \"href\": \"normal-stage-out/output_dir/failed_features.json\", \"type\": \"application/json\"}]}\n" + ] + } + ], + "execution_count": 33 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "- Successful and failed items are written to 2 files into the directory set in `OUTPUT_DIRECTORY`", + "id": "61efcaa319b065b7" + }, + { + "metadata": {}, + "cell_type": "code", + "outputs": [], + "execution_count": null, + "source": "", + "id": "14569f095beacee0" } ], "metadata": { From 2410f8949ce116b0221ba17713a4c23866f0e6d0 Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Tue, 11 Feb 2025 11:40:33 -0800 Subject: [PATCH 06/11] fix: use latest documentation --- examples/stage-in-stage-out-example.ipynb | 390 ++++++++++++++++-- .../ds_client/test_ds_client_unity.py | 21 +- 2 files changed, 360 insertions(+), 51 deletions(-) diff --git a/examples/stage-in-stage-out-example.ipynb b/examples/stage-in-stage-out-example.ipynb index 0adfee4..eb67c17 100644 --- a/examples/stage-in-stage-out-example.ipynb +++ b/examples/stage-in-stage-out-example.ipynb @@ -100,42 +100,6 @@ "outputs": [], "execution_count": 4 }, - { - "metadata": {}, - "cell_type": "markdown", - "source": "### Setting Environment Variables\n", - "id": "b5f018519b92ea5f" - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-02-10T17:41:38.169484Z", - "start_time": "2025-02-10T17:41:38.166938Z" - } - }, - "cell_type": "code", - "source": [ - "import os\n", - "os.environ['PROJECT'] = 'DEMO'\n", - "os.environ['VENUE'] = 'DEV12'\n", - "\n", - "\n", - "\n", - "#\n", - "# os.environ['VERIFY_SSL'] = 'FALSE'\n", - "# os.environ['RESULT_PATH_PREFIX'] = ''\n", - "# os.environ['STAGING_BUCKET'] = 'uds-sbx-cumulus-staging'\n", - "# os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY'\n", - "# os.environ['OUTPUT_FILE'] = 'normal-stage-out/some_output/output.json'\n", - "# os.environ['UPLOAD_DIR'] = '' # not needed\n", - "# os.environ['OUTPUT_DIRECTORY'] = 'normal-stage-out/output_dir'\n", - "# FileUtils.mk_dir_p(os.environ.get('OUTPUT_DIRECTORY'))\n", - "# os.environ['CATALOG_FILE'] = 'normal-stage-out/catalog.json'" - ], - "id": "36805b7fb91362cb", - "outputs": [], - "execution_count": 5 - }, { "metadata": {}, "cell_type": "markdown", @@ -150,12 +114,13 @@ { "metadata": { "ExecuteTime": { - "end_time": "2025-02-10T17:48:11.471415Z", - "start_time": "2025-02-10T17:48:11.466169Z" + "end_time": "2025-02-10T23:09:05.795762Z", + "start_time": "2025-02-10T23:09:05.790161Z" } }, "cell_type": "code", "source": [ + "import os\n", "# The following environment variables are needed to \"stage-in\" from AWS S3\n", "# They will also be needed if \"EDL\" settings and \"STAC_AUTH_TYPE\" settings are coming from parameter store\n", "# Note that this may not be needed if JupyterNotebook can take care of the access.\n", @@ -188,7 +153,7 @@ ], "id": "abb5f945876a1a63", "outputs": [], - "execution_count": 10 + "execution_count": 43 }, { "metadata": {}, @@ -514,15 +479,212 @@ { "metadata": { "ExecuteTime": { - "end_time": "2025-02-10T17:41:47.855545Z", - "start_time": "2025-02-10T17:41:47.853160Z" + "end_time": "2025-02-10T23:09:49.537304Z", + "start_time": "2025-02-10T23:09:49.530650Z" } }, "cell_type": "code", - "source": "# TODO How do I do this?", + "source": [ + "import json\n", + "\n", + "from pystac import ItemCollection, Item, Asset\n", + "\n", + "from mdps_ds_lib.lib.utils.time_utils import TimeUtils\n", + "\n", + "test1 = ItemCollection(items=[\n", + " Item(id='G2721220118-LPCLOUD',\n", + " geometry={ # Set them if the algorithm knows what type of geometry is needed\n", + " \"type\": \"Point\",\n", + " \"coordinates\": [0.0, 0.0]\n", + " },\n", + " bbox=[53.7996902,\n", + " 38.031826,\n", + " 55.0450439,\n", + " 39.2283897], # Set them if the algorithm knows what type of geometry is needed\n", + " datetime=TimeUtils().parse_from_str('2023-06-20T08:44:26Z').get_datetime_obj(),\n", + " # Current metadata file creation datetime\n", + " properties={ # These 4 fields are mandatory\n", + " \"start_datetime\": \"2016-01-31T18:00:00.009057Z\",\n", + " \"end_datetime\": \"2016-01-31T19:59:59.991043Z\",\n", + " \"created\": \"2016-02-01T02:45:59.639000Z\",\n", + " \"updated\": \"2022-03-23T15:48:21.578000Z\",\n", + " },\n", + " href='https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.stac',\n", + " collection='NA', # No need to find out what collection it belongs to DS will take care of that.\n", + " assets={ # Point to all relevant files including itself\n", + " f'EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc': Asset(\n", + " 'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc',\n", + " title='EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc', roles=['data']),\n", + " f'EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc': Asset(\n", + " 'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc',\n", + " title='EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc', roles=['data']),\n", + " f'EMIT_L1B_RAD_001_20230620T084426_2317106_011.xml': Asset(\n", + " 'https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.xml',\n", + " title='EMIT_L1B_RAD_001_20230620T084426_2317106_011.xml', roles=['metadata']),\n", + " }),\n", + " Item(id='G2721220118-LPCLOUD',\n", + " geometry={ # Set them if the algorithm knows what type of geometry is needed\n", + " \"type\": \"Point\",\n", + " \"coordinates\": [0.0, 0.0]\n", + " },\n", + " bbox=[53.7996902,\n", + " 38.031826,\n", + " 55.0450439,\n", + " 39.2283897], # Set them if the algorithm knows what type of geometry is needed\n", + " datetime=TimeUtils().parse_from_str('2023-06-20T08:44:26Z').get_datetime_obj(),\n", + " # Current metadata file creation datetime\n", + " properties={ # These 4 fields are mandatory\n", + " \"start_datetime\": \"2016-01-31T18:00:00.009057Z\",\n", + " \"end_datetime\": \"2016-01-31T19:59:59.991043Z\",\n", + " \"created\": \"2016-02-01T02:45:59.639000Z\",\n", + " \"updated\": \"2022-03-23T15:48:21.578000Z\",\n", + " },\n", + " href='https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.stac',\n", + " collection='NA', # No need to find out what collection it belongs to DS will take care of that.\n", + " assets={ # Point to all relevant files including itself\n", + " f'EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc': Asset(\n", + " 'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc',\n", + " title='EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc', roles=['data']),\n", + " f'EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc': Asset(\n", + " 'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc',\n", + " title='EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc', roles=['data']),\n", + " f'EMIT_L1B_RAD_001_20230620T084426_2317106_011.xml': Asset(\n", + " 'https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.xml',\n", + " title='EMIT_L1B_RAD_001_20230620T084426_2317106_011.xml', roles=['metadata']),\n", + " })\n", + "])\n", + "\n", + "print(json.dumps(test1.to_dict(False), indent=4))\n" + ], "id": "1429c01da795b287", - "outputs": [], - "execution_count": 8 + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"type\": \"FeatureCollection\",\n", + " \"features\": [\n", + " {\n", + " \"type\": \"Feature\",\n", + " \"stac_version\": \"1.0.0\",\n", + " \"id\": \"G2721220118-LPCLOUD\",\n", + " \"properties\": {\n", + " \"start_datetime\": \"2016-01-31T18:00:00.009057Z\",\n", + " \"end_datetime\": \"2016-01-31T19:59:59.991043Z\",\n", + " \"created\": \"2016-02-01T02:45:59.639000Z\",\n", + " \"updated\": \"2022-03-23T15:48:21.578000Z\",\n", + " \"datetime\": \"2023-06-20T08:44:26Z\"\n", + " },\n", + " \"geometry\": {\n", + " \"type\": \"Point\",\n", + " \"coordinates\": [\n", + " 0.0,\n", + " 0.0\n", + " ]\n", + " },\n", + " \"links\": [\n", + " {\n", + " \"rel\": \"self\",\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.stac\",\n", + " \"type\": \"application/json\"\n", + " }\n", + " ],\n", + " \"assets\": {\n", + " \"EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc\": {\n", + " \"href\": \"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc\",\n", + " \"title\": \"EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc\",\n", + " \"roles\": [\n", + " \"data\"\n", + " ]\n", + " },\n", + " \"EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc\": {\n", + " \"href\": \"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc\",\n", + " \"title\": \"EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc\",\n", + " \"roles\": [\n", + " \"data\"\n", + " ]\n", + " },\n", + " \"EMIT_L1B_RAD_001_20230620T084426_2317106_011.xml\": {\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.xml\",\n", + " \"title\": \"EMIT_L1B_RAD_001_20230620T084426_2317106_011.xml\",\n", + " \"roles\": [\n", + " \"metadata\"\n", + " ]\n", + " }\n", + " },\n", + " \"bbox\": [\n", + " 53.7996902,\n", + " 38.031826,\n", + " 55.0450439,\n", + " 39.2283897\n", + " ],\n", + " \"stac_extensions\": [],\n", + " \"collection\": \"NA\"\n", + " },\n", + " {\n", + " \"type\": \"Feature\",\n", + " \"stac_version\": \"1.0.0\",\n", + " \"id\": \"G2721220118-LPCLOUD\",\n", + " \"properties\": {\n", + " \"start_datetime\": \"2016-01-31T18:00:00.009057Z\",\n", + " \"end_datetime\": \"2016-01-31T19:59:59.991043Z\",\n", + " \"created\": \"2016-02-01T02:45:59.639000Z\",\n", + " \"updated\": \"2022-03-23T15:48:21.578000Z\",\n", + " \"datetime\": \"2023-06-20T08:44:26Z\"\n", + " },\n", + " \"geometry\": {\n", + " \"type\": \"Point\",\n", + " \"coordinates\": [\n", + " 0.0,\n", + " 0.0\n", + " ]\n", + " },\n", + " \"links\": [\n", + " {\n", + " \"rel\": \"self\",\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.stac\",\n", + " \"type\": \"application/json\"\n", + " }\n", + " ],\n", + " \"assets\": {\n", + " \"EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc\": {\n", + " \"href\": \"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc\",\n", + " \"title\": \"EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc\",\n", + " \"roles\": [\n", + " \"data\"\n", + " ]\n", + " },\n", + " \"EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc\": {\n", + " \"href\": \"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc\",\n", + " \"title\": \"EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc\",\n", + " \"roles\": [\n", + " \"data\"\n", + " ]\n", + " },\n", + " \"EMIT_L1B_RAD_001_20230620T084426_2317106_011.xml\": {\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.xml\",\n", + " \"title\": \"EMIT_L1B_RAD_001_20230620T084426_2317106_011.xml\",\n", + " \"roles\": [\n", + " \"metadata\"\n", + " ]\n", + " }\n", + " },\n", + " \"bbox\": [\n", + " 53.7996902,\n", + " 38.031826,\n", + " 55.0450439,\n", + " 39.2283897\n", + " ],\n", + " \"stac_extensions\": [],\n", + " \"collection\": \"NA\"\n", + " }\n", + " ]\n", + "}\n" + ] + } + ], + "execution_count": 44 }, { "metadata": {}, @@ -1015,13 +1177,151 @@ "source": "- Successful and failed items are written to 2 files into the directory set in `OUTPUT_DIRECTORY`", "id": "61efcaa319b065b7" }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "### Ancillary Stage-out Environment Variables", + "id": "c111762146892c41" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T23:11:49.193312Z", + "start_time": "2025-02-10T23:11:49.189141Z" + } + }, + "cell_type": "code", + "source": [ + "import os\n", + "\n", + "os.environ['AWS_ACCESS_KEY_ID'] = 'xxx'\n", + "os.environ['AWS_SECRET_ACCESS_KEY'] = 'xxx'\n", + "os.environ['AWS_SESSION_TOKEN'] = 'xxx'\n", + "\n", + "os.environ['GRANULES_UPLOAD_TYPE'] = 'UPLOAD_AUXILIARY_FILE_AS_GRANULE' # Setting uploading as auxiliary\n", + "os.environ['STAGING_BUCKET'] = 'uds-sbx-cumulus-staging' # S3 bucket where they will reside\n", + "os.environ['VERIFY_SSL'] = 'FALSE' # Optional param.\n", + "os.environ['BASE_DIRECTORY'] = 'auxiliary_files' # Base folder to upload\n", + "\n", + "os.environ['OUTPUT_DIRECTORY'] = 'auxiliary_files/output_dir' # the success / failure results to be stored locally for rewview\n", + "os.environ['OUTPUT_FILE'] = 'auxiliary_files/output_dir/stage_out_result.json' # file path where the result overview is written locally for\n", + "\n", + "tenant = 'UDS_DEMO'\n", + "tenant_venue = 'TEST'\n", + "collection_name = 'UDS_UNIT_COLLECTION'\n", + "collection_version = '24.09.10.11.00'.replace('.', '')\n", + "temp_collection_id = f'URN:NASA:UNITY:{tenant}:{tenant_venue}:{collection_name}___{collection_version}'\n", + "os.environ['COLLECTION_ID'] = temp_collection_id # Setting Collection ID.\n", + "# Note that Collection ID needs to follow the standard.\n", + "\n", + "FileUtils.mk_dir_p(os.environ.get('OUTPUT_DIRECTORY'))\n" + ], + "id": "29f786b4ed0929b5", + "outputs": [], + "execution_count": 45 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "#### Mocking some ancillary files", + "id": "f40f81242b3930f7" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T22:09:55.801596Z", + "start_time": "2025-02-10T22:09:55.795144Z" + } + }, + "cell_type": "code", + "source": [ + "granules_dir = os.path.join('auxiliary_files')\n", + "FileUtils.mk_dir_p(granules_dir) # base directory\n", + "total_files = 3 # Uploading 3 mock files\n", + "\n", + "for i in range(1, total_files+1):\n", + " filename = f'mocked_general_file{i:02d}'\n", + " with open(os.path.join(granules_dir, f'{filename}'), 'w') as ff: # Creating Data File\n", + " ff.write('mocked_general_file')" + ], + "id": "9b97566830b4afc", + "outputs": [], + "execution_count": 34 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "#### Performing actual stage-out\n", + "- Note that there is no dry-run on ancillary stage-out since there are no settings that client setup." + ], + "id": "d516f0a9fffbdc2a" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T22:17:28.437985Z", + "start_time": "2025-02-10T22:17:28.433764Z" + } + }, + "cell_type": "code", + "source": [ + "# Cleaning old result files.\n", + "\n", + "old_result_files = glob(os.path.join(os.environ['OUTPUT_DIRECTORY'], '*'))\n", + "for each_file in old_result_files:\n", + " FileUtils.remove_if_exists(each_file)" + ], + "id": "c61692dd202c889c", + "outputs": [], + "execution_count": 40 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T22:17:35.511806Z", + "start_time": "2025-02-10T22:17:31.251498Z" + } + }, + "cell_type": "code", + "source": [ + "\n", + "from mdps_ds_lib.stage_in_out.upoad_granules_factory import UploadGranulesFactory\n", + "from mdps_ds_lib.stage_in_out.stage_in_out_utils import StageInOutUtils\n", + "\n", + "# Hardcoded method call. All params are set via environment previously\n", + "upload_result_str = UploadGranulesFactory().get_class(os.getenv('GRANULES_UPLOAD_TYPE', UploadGranulesFactory.UPLOAD_S3_BY_STAC_CATALOG)).upload()\n", + "StageInOutUtils.write_output_to_file(upload_result_str)\n", + "print('done')" + ], + "id": "3aef5ceb43098a07", + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-10 14:17:31,715 [AUDIT] [mdps_ds_lib.stage_in_out.upload_arbitrary_files_as_granules::11] uploading auxiliary file: auxiliary_files/mocked_general_file01\n", + "2025-02-10 14:17:31,715 [AUDIT] [mdps_ds_lib.stage_in_out.upload_arbitrary_files_as_granules::11] uploading auxiliary file: auxiliary_files/mocked_general_file03\n", + "2025-02-10 14:17:31,714 [AUDIT] [mdps_ds_lib.stage_in_out.upload_arbitrary_files_as_granules::11] uploading auxiliary file: auxiliary_files/mocked_general_file02\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "done\n" + ] + } + ], + "execution_count": 41 + }, { "metadata": {}, "cell_type": "code", "outputs": [], "execution_count": null, "source": "", - "id": "14569f095beacee0" + "id": "2987dc7d3c9d0898" } ], "metadata": { diff --git a/tests/mdps_ds_lib/ds_client/test_ds_client_unity.py b/tests/mdps_ds_lib/ds_client/test_ds_client_unity.py index 1cd5ecd..12bb24d 100644 --- a/tests/mdps_ds_lib/ds_client/test_ds_client_unity.py +++ b/tests/mdps_ds_lib/ds_client/test_ds_client_unity.py @@ -1,3 +1,4 @@ +import json import os from unittest import TestCase @@ -24,9 +25,13 @@ def test_query_granules_across_collections(self): client.project = 'UNITY' client.tenant = 'UDS_LOCAL_TEST_3' client.tenant_venue = 'DEV' - result = client.query_granules_across_collections(sort_keys='+properties.datetime,-id') - print(result) - print(client.query_granules_next()) + result = client.query_granules_across_collections(sort_keys='+properties.datetime,-id', limit=3) + print(json.dumps(result, indent=4)) + print('--------------------------') + print(json.dumps([{'id': k['id'], 'datetime': k['properties']['datetime']} for k in result['features']], indent=4)) + print('--------------------------') + print(json.dumps(client.query_granules_next(), indent=4)) + print('=================================================') return def test_query_granules(self): @@ -49,7 +54,11 @@ def test_query_granules(self): client.tenant_venue = 'DEV' client.collection = 'DDD-01' client.collection_venue = '001' - result = client.query_granules(sort_keys='+properties.datetime,-id') # bbox='-114,32.5,-113,33.5' - print(result) - print(client.query_granules_next()) + result = client.query_granules(sort_keys='+properties.datetime,-id', limit=3) # bbox='-114,32.5,-113,33.5' + print(json.dumps(result, indent=4)) + print(json.dumps(client.query_granules_next(), indent=4)) + print('=================================================') + result = client.query_granules(sort_keys='+id,-properties.datetime', limit=3) + print(json.dumps(result, indent=4)) + print(json.dumps(client.query_granules_next(), indent=4)) return From 1d3785914a97e9b7eef5b2cdcd79b7dd18ffb6bd Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Tue, 11 Feb 2025 11:52:51 -0800 Subject: [PATCH 07/11] feat: splitting notebooks --- examples/stage-in-example.ipynb | 764 +++++++++++++++++++++ examples/stage-out-ancillary-example.ipynb | 274 ++++++++ examples/stage-out-example.ipynb | 567 +++++++++++++++ 3 files changed, 1605 insertions(+) create mode 100644 examples/stage-in-example.ipynb create mode 100644 examples/stage-out-ancillary-example.ipynb create mode 100644 examples/stage-out-example.ipynb diff --git a/examples/stage-in-example.ipynb b/examples/stage-in-example.ipynb new file mode 100644 index 0000000..7af72c4 --- /dev/null +++ b/examples/stage-in-example.ipynb @@ -0,0 +1,764 @@ +{ + "cells": [ + { + "metadata": {}, + "cell_type": "markdown", + "source": "## Stage In Example", + "id": "30b6016bc9435c51" + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "Steps\n", + "- Install Data Service (DS) Client Library\n", + "- Set Log Level\n", + "- Set Environment variables for Stage-In\n", + "- Try Stage-In\n", + "- Check the results" + ], + "id": "41307b0225e83dcc" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T17:41:17.675227Z", + "start_time": "2025-02-10T17:41:16.781958Z" + } + }, + "cell_type": "code", + "source": [ + "from anyio.streams import file\n", + "%pip install mdps-ds-lib" + ], + "id": "fe80dd452ebd717b", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: mdps-ds-lib in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (1.1.1)\r\n", + "Requirement already satisfied: boto3<2.0.0,>=1.26.51 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (1.36.15)\r\n", + "Requirement already satisfied: elasticsearch==7.13.4 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (7.13.4)\r\n", + "Requirement already satisfied: fastjsonschema<3.0.0,>=2.19.1 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (2.21.1)\r\n", + "Requirement already satisfied: jsonschema<5.0.0,>=4.23.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (4.23.0)\r\n", + "Requirement already satisfied: pystac==1.9.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (1.9.0)\r\n", + "Requirement already satisfied: requests==2.31.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (2.31.0)\r\n", + "Requirement already satisfied: requests-aws4auth==1.2.3 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (1.2.3)\r\n", + "Requirement already satisfied: tenacity==8.2.3 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (8.2.3)\r\n", + "Requirement already satisfied: xmltodict==0.13.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (0.13.0)\r\n", + "Requirement already satisfied: urllib3<2,>=1.21.1 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from elasticsearch==7.13.4->mdps-ds-lib) (1.26.20)\r\n", + "Requirement already satisfied: certifi in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from elasticsearch==7.13.4->mdps-ds-lib) (2024.12.14)\r\n", + "Requirement already satisfied: python-dateutil>=2.7.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from pystac==1.9.0->mdps-ds-lib) (2.9.0.post0)\r\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from requests==2.31.0->mdps-ds-lib) (3.4.1)\r\n", + "Requirement already satisfied: idna<4,>=2.5 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from requests==2.31.0->mdps-ds-lib) (3.10)\r\n", + "Requirement already satisfied: six in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from requests-aws4auth==1.2.3->mdps-ds-lib) (1.17.0)\r\n", + "Requirement already satisfied: botocore<1.37.0,>=1.36.15 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from boto3<2.0.0,>=1.26.51->mdps-ds-lib) (1.36.15)\r\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from boto3<2.0.0,>=1.26.51->mdps-ds-lib) (1.0.1)\r\n", + "Requirement already satisfied: s3transfer<0.12.0,>=0.11.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from boto3<2.0.0,>=1.26.51->mdps-ds-lib) (0.11.2)\r\n", + "Requirement already satisfied: attrs>=22.2.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.23.0->mdps-ds-lib) (24.3.0)\r\n", + "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.23.0->mdps-ds-lib) (2024.10.1)\r\n", + "Requirement already satisfied: referencing>=0.28.4 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.23.0->mdps-ds-lib) (0.35.1)\r\n", + "Requirement already satisfied: rpds-py>=0.7.1 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.23.0->mdps-ds-lib) (0.22.3)\r\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "execution_count": 3 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "### Setting the Log Level\n", + "- Log level mappings:\n", + "- 10 = debug\n", + "- 20 = info\n", + "- 30 = warning\n", + "- 40 = error" + ], + "id": "e6bf69a910209c6c" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T17:41:24.968457Z", + "start_time": "2025-02-10T17:41:24.965823Z" + } + }, + "cell_type": "code", + "source": [ + "import logging\n", + "log_level = 30\n", + "logging.basicConfig(level=log_level, format=\"%(asctime)s [%(levelname)s] [%(name)s::%(lineno)d] %(message)s\")" + ], + "id": "d2aaaee1a57cb095", + "outputs": [], + "execution_count": 4 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "#### Stage-in Only Environment Variables\n", + "\n", + "Follow this link for more information: https://app.gitbook.com/o/xZRqGQeQXJ0RP4VMj7Lq/s/UMIRhLdbRQTvMWop8Il9/developer-docs/data/docs/users-guide/stage-in\n", + "\n" + ], + "id": "8a87921572f1578d" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T23:09:05.795762Z", + "start_time": "2025-02-10T23:09:05.790161Z" + } + }, + "cell_type": "code", + "source": [ + "import os\n", + "# The following environment variables are needed to \"stage-in\" from AWS S3\n", + "# They will also be needed if \"EDL\" settings and \"STAC_AUTH_TYPE\" settings are coming from parameter store\n", + "# Note that this may not be needed if JupyterNotebook can take care of the access.\n", + "os.environ['AWS_ACCESS_KEY_ID'] = 'xxx'\n", + "os.environ['AWS_SECRET_ACCESS_KEY'] = 'xxx'\n", + "os.environ['AWS_SESSION_TOKEN'] = 'xxx'\n", + "\n", + "# The following environment variables are ONLY needed if \"stage-in\" requires Earth data Login\n", + "os.environ['EDL_USERNAME'] = '/unity/uds/user/wphyo/edl_username' # Parameter Store Key for EarthData Login Username\n", + "os.environ['EDL_PASSWORD'] = '/unity/uds/user/wphyo/edl_dwssap' # Parameter Store Key for EarthData Login Password\n", + "os.environ['EDL_PASSWORD_TYPE'] = 'PARAM_STORE' # Can hardcode it to PARAM_STORE if that is used.\n", + "os.environ['EDL_BASE_URL'] = 'urs.earthdata.nasa.gov' # Earthdata Login URL to get Earthdata token to download files\n", + "\n", + "# The following environment variables are ONLY needed if \"stage-in\" requires STAC_AUTH_TYPE is \"UNITY\".\n", + "os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' # Look above links for all available options\n", + "os.environ['USERNAME'] = '/unity/uds/user/wphyo/username'\n", + "os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap'\n", + "os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c'\n", + "os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com'\n", + "\n", + "\n", + "# Others environment variables\n", + "os.environ['GRANULES_DOWNLOAD_TYPE'] = 'AMALGAMATION' # Download type to choose download class. AMALGAMATION, S3, DAAC, HTTP, and so on.\n", + "os.environ['DOWNLOADING_KEYS'] = 'data,metadata' # Which asset keys to download\n", + "os.environ['STAC_JSON'] = 'https://raw.githubusercontent.com/GodwinShen/emit-ghg/main/test/catalog.json' # URL to direct which granules + assets to download\n", + "# Other options\n", + "# os.environ['STAC_JSON'] = os.path.join(os.getcwd(), 'stage_in.json') # Alternatively, you can store the file locally, and point it as a path\n", + "os.environ['DOWNLOAD_DIR'] = os.path.join(os.getcwd(), 'downloaded_files') # Base directory where files will be downloaded\n", + "os.environ['OUTPUT_FILE'] = os.path.join(os.getcwd(), 'stage_in_result.json') # file path where the result is written locally for review\n" + ], + "id": "abb5f945876a1a63", + "outputs": [], + "execution_count": 43 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "- This is an example STAC_JSON file (or URL contents) used as a guide for stage-in.\n", + "- It is a STAC \"FeatureCollection\", a wrapper of an array of [Stac items](https://github.com/radiantearth/stac-spec/blob/master/item-spec/item-spec.md)" + ], + "id": "6c17a162ab168924" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T17:41:44.530922Z", + "start_time": "2025-02-10T17:41:44.519332Z" + } + }, + "cell_type": "code", + "source": [ + "{\n", + " \"type\": \"FeatureCollection\",\n", + " \"features\": [\n", + " {\n", + " \"type\": \"Feature\",\n", + " \"stac_version\": \"1.0.0\",\n", + " \"id\": \"G2721220118-LPCLOUD\",\n", + " \"properties\": {\n", + " \"datetime\": \"2023-06-20T08:44:26Z\",\n", + " \"start_datetime\": \"2023-06-20T08:44:26.000Z\",\n", + " \"end_datetime\": \"2023-06-20T08:44:38.000Z\",\n", + " \"eo:cloud_cover\": 41\n", + " },\n", + " \"geometry\": {\n", + " \"type\": \"Polygon\",\n", + " \"coordinates\": [\n", + " [\n", + " [\n", + " 54.235199,\n", + " 39.2283897\n", + " ],\n", + " [\n", + " 53.7996902,\n", + " 38.5576591\n", + " ],\n", + " [\n", + " 54.6095352,\n", + " 38.031826\n", + " ],\n", + " [\n", + " 55.0450439,\n", + " 38.7025566\n", + " ],\n", + " [\n", + " 54.235199,\n", + " 39.2283897\n", + " ]\n", + " ]\n", + " ]\n", + " },\n", + " \"links\": [\n", + " {\n", + " \"rel\": \"self\",\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.stac\"\n", + " },\n", + " {\n", + " \"rel\": \"parent\",\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/C2408009906-LPCLOUD.stac\"\n", + " },\n", + " {\n", + " \"rel\": \"collection\",\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/C2408009906-LPCLOUD.stac\"\n", + " },\n", + " {\n", + " \"rel\": \"root\",\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/\"\n", + " },\n", + " {\n", + " \"rel\": \"via\",\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.json\"\n", + " },\n", + " {\n", + " \"rel\": \"via\",\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.umm_json\"\n", + " }\n", + " ],\n", + " \"assets\": {\n", + " \"metadata\": {\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.xml\",\n", + " \"type\": \"application/xml\"\n", + " },\n", + " \"browse\": {\n", + " \"href\": \"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-public/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_RAD_001_20230620T084426_2317106_011.png\",\n", + " \"type\": \"image/png\",\n", + " \"title\": \"Download EMIT_L1B_RAD_001_20230620T084426_2317106_011.png\"\n", + " },\n", + " \"opendap\": {\n", + " \"href\": \"https://opendap.earthdata.nasa.gov/collections/C2408009906-LPCLOUD/granules/EMIT_L1B_RAD_001_20230620T084426_2317106_011\",\n", + " \"title\": \"OPeNDAP request URL\"\n", + " },\n", + " \"data\": {\n", + " \"href\": \"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc\",\n", + " \"title\": \"Download EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc\"\n", + " },\n", + " \"data1\": {\n", + " \"href\": \"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc\",\n", + " \"title\": \"Download EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc\"\n", + " }\n", + " },\n", + " \"bbox\": [\n", + " 53.7996902,\n", + " 38.031826,\n", + " 55.0450439,\n", + " 39.2283897\n", + " ],\n", + " \"stac_extensions\": [\n", + " \"https://stac-extensions.github.io/eo/v1.0.0/schema.json\"\n", + " ],\n", + " \"collection\": \"C2408009906-LPCLOUD\"\n", + " },\n", + " {\n", + " \"type\": \"Feature\",\n", + " \"stac_version\": \"1.0.0\",\n", + " \"id\": \"G2721699381-LPCLOUD\",\n", + " \"properties\": {\n", + " \"datetime\": \"2023-06-20T08:44:26Z\",\n", + " \"start_datetime\": \"2023-06-20T08:44:26.000Z\",\n", + " \"end_datetime\": \"2023-06-20T08:44:38.000Z\",\n", + " \"eo:cloud_cover\": 41\n", + " },\n", + " \"geometry\": {\n", + " \"type\": \"Polygon\",\n", + " \"coordinates\": [\n", + " [\n", + " [\n", + " 54.235199,\n", + " 39.2283897\n", + " ],\n", + " [\n", + " 53.7996902,\n", + " 38.5576591\n", + " ],\n", + " [\n", + " 54.6095352,\n", + " 38.031826\n", + " ],\n", + " [\n", + " 55.0450439,\n", + " 38.7025566\n", + " ],\n", + " [\n", + " 54.235199,\n", + " 39.2283897\n", + " ]\n", + " ]\n", + " ]\n", + " },\n", + " \"links\": [\n", + " {\n", + " \"rel\": \"self\",\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721699381-LPCLOUD.stac\"\n", + " },\n", + " {\n", + " \"rel\": \"parent\",\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/C2408750690-LPCLOUD.stac\"\n", + " },\n", + " {\n", + " \"rel\": \"collection\",\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/C2408750690-LPCLOUD.stac\"\n", + " },\n", + " {\n", + " \"rel\": \"root\",\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/\"\n", + " },\n", + " {\n", + " \"rel\": \"via\",\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721699381-LPCLOUD.json\"\n", + " },\n", + " {\n", + " \"rel\": \"via\",\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721699381-LPCLOUD.umm_json\"\n", + " }\n", + " ],\n", + " \"assets\": {\n", + " \"metadata\": {\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721699381-LPCLOUD.xml\",\n", + " \"type\": \"application/xml\"\n", + " },\n", + " \"browse\": {\n", + " \"href\": \"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-public/EMITL2ARFL.001/EMIT_L2A_RFL_001_20230620T084426_2317106_011/EMIT_L2A_RFL_001_20230620T084426_2317106_011.png\",\n", + " \"type\": \"image/png\",\n", + " \"title\": \"Download EMIT_L2A_RFL_001_20230620T084426_2317106_011.png\"\n", + " },\n", + " \"opendap\": {\n", + " \"href\": \"https://opendap.earthdata.nasa.gov/collections/C2408750690-LPCLOUD/granules/EMIT_L2A_RFL_001_20230620T084426_2317106_011\",\n", + " \"title\": \"OPeNDAP request URL\"\n", + " },\n", + " \"data\": {\n", + " \"href\": \"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL2ARFL.001/EMIT_L2A_RFL_001_20230620T084426_2317106_011/EMIT_L2A_RFL_001_20230620T084426_2317106_011.nc\",\n", + " \"title\": \"Download EMIT_L2A_RFL_001_20230620T084426_2317106_011.nc\"\n", + " },\n", + " \"data1\": {\n", + " \"href\": \"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL2ARFL.001/EMIT_L2A_RFL_001_20230620T084426_2317106_011/EMIT_L2A_RFLUNCERT_001_20230620T084426_2317106_011.nc\",\n", + " \"title\": \"Download EMIT_L2A_RFLUNCERT_001_20230620T084426_2317106_011.nc\"\n", + " },\n", + " \"data2\": {\n", + " \"href\": \"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL2ARFL.001/EMIT_L2A_RFL_001_20230620T084426_2317106_011/EMIT_L2A_MASK_001_20230620T084426_2317106_011.nc\",\n", + " \"title\": \"Download EMIT_L2A_MASK_001_20230620T084426_2317106_011.nc\"\n", + " }\n", + " },\n", + " \"bbox\": [\n", + " 53.7996902,\n", + " 38.031826,\n", + " 55.0450439,\n", + " 39.2283897\n", + " ],\n", + " \"stac_extensions\": [\n", + " \"https://stac-extensions.github.io/eo/v1.0.0/schema.json\"\n", + " ],\n", + " \"collection\": \"C2408750690-LPCLOUD\"\n", + " }\n", + " ]\n", + "}" + ], + "id": "8a3c3ceaf481e2fe", + "outputs": [ + { + "data": { + "text/plain": [ + "{'type': 'FeatureCollection',\n", + " 'features': [{'type': 'Feature',\n", + " 'stac_version': '1.0.0',\n", + " 'id': 'G2721220118-LPCLOUD',\n", + " 'properties': {'datetime': '2023-06-20T08:44:26Z',\n", + " 'start_datetime': '2023-06-20T08:44:26.000Z',\n", + " 'end_datetime': '2023-06-20T08:44:38.000Z',\n", + " 'eo:cloud_cover': 41},\n", + " 'geometry': {'type': 'Polygon',\n", + " 'coordinates': [[[54.235199, 39.2283897],\n", + " [53.7996902, 38.5576591],\n", + " [54.6095352, 38.031826],\n", + " [55.0450439, 38.7025566],\n", + " [54.235199, 39.2283897]]]},\n", + " 'links': [{'rel': 'self',\n", + " 'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.stac'},\n", + " {'rel': 'parent',\n", + " 'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/C2408009906-LPCLOUD.stac'},\n", + " {'rel': 'collection',\n", + " 'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/C2408009906-LPCLOUD.stac'},\n", + " {'rel': 'root', 'href': 'https://cmr.earthdata.nasa.gov:443/search/'},\n", + " {'rel': 'via',\n", + " 'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.json'},\n", + " {'rel': 'via',\n", + " 'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.umm_json'}],\n", + " 'assets': {'metadata': {'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.xml',\n", + " 'type': 'application/xml'},\n", + " 'browse': {'href': 'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-public/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_RAD_001_20230620T084426_2317106_011.png',\n", + " 'type': 'image/png',\n", + " 'title': 'Download EMIT_L1B_RAD_001_20230620T084426_2317106_011.png'},\n", + " 'opendap': {'href': 'https://opendap.earthdata.nasa.gov/collections/C2408009906-LPCLOUD/granules/EMIT_L1B_RAD_001_20230620T084426_2317106_011',\n", + " 'title': 'OPeNDAP request URL'},\n", + " 'data': {'href': 'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc',\n", + " 'title': 'Download EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc'},\n", + " 'data1': {'href': 'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc',\n", + " 'title': 'Download EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc'}},\n", + " 'bbox': [53.7996902, 38.031826, 55.0450439, 39.2283897],\n", + " 'stac_extensions': ['https://stac-extensions.github.io/eo/v1.0.0/schema.json'],\n", + " 'collection': 'C2408009906-LPCLOUD'},\n", + " {'type': 'Feature',\n", + " 'stac_version': '1.0.0',\n", + " 'id': 'G2721699381-LPCLOUD',\n", + " 'properties': {'datetime': '2023-06-20T08:44:26Z',\n", + " 'start_datetime': '2023-06-20T08:44:26.000Z',\n", + " 'end_datetime': '2023-06-20T08:44:38.000Z',\n", + " 'eo:cloud_cover': 41},\n", + " 'geometry': {'type': 'Polygon',\n", + " 'coordinates': [[[54.235199, 39.2283897],\n", + " [53.7996902, 38.5576591],\n", + " [54.6095352, 38.031826],\n", + " [55.0450439, 38.7025566],\n", + " [54.235199, 39.2283897]]]},\n", + " 'links': [{'rel': 'self',\n", + " 'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/G2721699381-LPCLOUD.stac'},\n", + " {'rel': 'parent',\n", + " 'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/C2408750690-LPCLOUD.stac'},\n", + " {'rel': 'collection',\n", + " 'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/C2408750690-LPCLOUD.stac'},\n", + " {'rel': 'root', 'href': 'https://cmr.earthdata.nasa.gov:443/search/'},\n", + " {'rel': 'via',\n", + " 'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/G2721699381-LPCLOUD.json'},\n", + " {'rel': 'via',\n", + " 'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/G2721699381-LPCLOUD.umm_json'}],\n", + " 'assets': {'metadata': {'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/G2721699381-LPCLOUD.xml',\n", + " 'type': 'application/xml'},\n", + " 'browse': {'href': 'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-public/EMITL2ARFL.001/EMIT_L2A_RFL_001_20230620T084426_2317106_011/EMIT_L2A_RFL_001_20230620T084426_2317106_011.png',\n", + " 'type': 'image/png',\n", + " 'title': 'Download EMIT_L2A_RFL_001_20230620T084426_2317106_011.png'},\n", + " 'opendap': {'href': 'https://opendap.earthdata.nasa.gov/collections/C2408750690-LPCLOUD/granules/EMIT_L2A_RFL_001_20230620T084426_2317106_011',\n", + " 'title': 'OPeNDAP request URL'},\n", + " 'data': {'href': 'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL2ARFL.001/EMIT_L2A_RFL_001_20230620T084426_2317106_011/EMIT_L2A_RFL_001_20230620T084426_2317106_011.nc',\n", + " 'title': 'Download EMIT_L2A_RFL_001_20230620T084426_2317106_011.nc'},\n", + " 'data1': {'href': 'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL2ARFL.001/EMIT_L2A_RFL_001_20230620T084426_2317106_011/EMIT_L2A_RFLUNCERT_001_20230620T084426_2317106_011.nc',\n", + " 'title': 'Download EMIT_L2A_RFLUNCERT_001_20230620T084426_2317106_011.nc'},\n", + " 'data2': {'href': 'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL2ARFL.001/EMIT_L2A_RFL_001_20230620T084426_2317106_011/EMIT_L2A_MASK_001_20230620T084426_2317106_011.nc',\n", + " 'title': 'Download EMIT_L2A_MASK_001_20230620T084426_2317106_011.nc'}},\n", + " 'bbox': [53.7996902, 38.031826, 55.0450439, 39.2283897],\n", + " 'stac_extensions': ['https://stac-extensions.github.io/eo/v1.0.0/schema.json'],\n", + " 'collection': 'C2408750690-LPCLOUD'}]}" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 7 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "#### How to create a Stac Item and a Feature Collection", + "id": "e0d932f5366ff630" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T23:09:49.537304Z", + "start_time": "2025-02-10T23:09:49.530650Z" + } + }, + "cell_type": "code", + "source": [ + "import json\n", + "\n", + "from pystac import ItemCollection, Item, Asset\n", + "\n", + "from mdps_ds_lib.lib.utils.time_utils import TimeUtils\n", + "\n", + "test1 = ItemCollection(items=[\n", + " Item(id='G2721220118-LPCLOUD',\n", + " geometry={ # Set them if the algorithm knows what type of geometry is needed\n", + " \"type\": \"Point\",\n", + " \"coordinates\": [0.0, 0.0]\n", + " },\n", + " bbox=[53.7996902,\n", + " 38.031826,\n", + " 55.0450439,\n", + " 39.2283897], # Set them if the algorithm knows what type of geometry is needed\n", + " datetime=TimeUtils().parse_from_str('2023-06-20T08:44:26Z').get_datetime_obj(),\n", + " # Current metadata file creation datetime\n", + " properties={ # These 4 fields are mandatory\n", + " \"start_datetime\": \"2016-01-31T18:00:00.009057Z\",\n", + " \"end_datetime\": \"2016-01-31T19:59:59.991043Z\",\n", + " \"created\": \"2016-02-01T02:45:59.639000Z\",\n", + " \"updated\": \"2022-03-23T15:48:21.578000Z\",\n", + " },\n", + " href='https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.stac',\n", + " collection='NA', # No need to find out what collection it belongs to DS will take care of that.\n", + " assets={ # Point to all relevant files including itself\n", + " f'EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc': Asset(\n", + " 'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc',\n", + " title='EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc', roles=['data']),\n", + " f'EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc': Asset(\n", + " 'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc',\n", + " title='EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc', roles=['data']),\n", + " f'EMIT_L1B_RAD_001_20230620T084426_2317106_011.xml': Asset(\n", + " 'https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.xml',\n", + " title='EMIT_L1B_RAD_001_20230620T084426_2317106_011.xml', roles=['metadata']),\n", + " }),\n", + " Item(id='G2721220118-LPCLOUD',\n", + " geometry={ # Set them if the algorithm knows what type of geometry is needed\n", + " \"type\": \"Point\",\n", + " \"coordinates\": [0.0, 0.0]\n", + " },\n", + " bbox=[53.7996902,\n", + " 38.031826,\n", + " 55.0450439,\n", + " 39.2283897], # Set them if the algorithm knows what type of geometry is needed\n", + " datetime=TimeUtils().parse_from_str('2023-06-20T08:44:26Z').get_datetime_obj(),\n", + " # Current metadata file creation datetime\n", + " properties={ # These 4 fields are mandatory\n", + " \"start_datetime\": \"2016-01-31T18:00:00.009057Z\",\n", + " \"end_datetime\": \"2016-01-31T19:59:59.991043Z\",\n", + " \"created\": \"2016-02-01T02:45:59.639000Z\",\n", + " \"updated\": \"2022-03-23T15:48:21.578000Z\",\n", + " },\n", + " href='https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.stac',\n", + " collection='NA', # No need to find out what collection it belongs to DS will take care of that.\n", + " assets={ # Point to all relevant files including itself\n", + " f'EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc': Asset(\n", + " 'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc',\n", + " title='EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc', roles=['data']),\n", + " f'EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc': Asset(\n", + " 'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc',\n", + " title='EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc', roles=['data']),\n", + " f'EMIT_L1B_RAD_001_20230620T084426_2317106_011.xml': Asset(\n", + " 'https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.xml',\n", + " title='EMIT_L1B_RAD_001_20230620T084426_2317106_011.xml', roles=['metadata']),\n", + " })\n", + "])\n", + "\n", + "print(json.dumps(test1.to_dict(False), indent=4))\n" + ], + "id": "1429c01da795b287", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"type\": \"FeatureCollection\",\n", + " \"features\": [\n", + " {\n", + " \"type\": \"Feature\",\n", + " \"stac_version\": \"1.0.0\",\n", + " \"id\": \"G2721220118-LPCLOUD\",\n", + " \"properties\": {\n", + " \"start_datetime\": \"2016-01-31T18:00:00.009057Z\",\n", + " \"end_datetime\": \"2016-01-31T19:59:59.991043Z\",\n", + " \"created\": \"2016-02-01T02:45:59.639000Z\",\n", + " \"updated\": \"2022-03-23T15:48:21.578000Z\",\n", + " \"datetime\": \"2023-06-20T08:44:26Z\"\n", + " },\n", + " \"geometry\": {\n", + " \"type\": \"Point\",\n", + " \"coordinates\": [\n", + " 0.0,\n", + " 0.0\n", + " ]\n", + " },\n", + " \"links\": [\n", + " {\n", + " \"rel\": \"self\",\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.stac\",\n", + " \"type\": \"application/json\"\n", + " }\n", + " ],\n", + " \"assets\": {\n", + " \"EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc\": {\n", + " \"href\": \"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc\",\n", + " \"title\": \"EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc\",\n", + " \"roles\": [\n", + " \"data\"\n", + " ]\n", + " },\n", + " \"EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc\": {\n", + " \"href\": \"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc\",\n", + " \"title\": \"EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc\",\n", + " \"roles\": [\n", + " \"data\"\n", + " ]\n", + " },\n", + " \"EMIT_L1B_RAD_001_20230620T084426_2317106_011.xml\": {\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.xml\",\n", + " \"title\": \"EMIT_L1B_RAD_001_20230620T084426_2317106_011.xml\",\n", + " \"roles\": [\n", + " \"metadata\"\n", + " ]\n", + " }\n", + " },\n", + " \"bbox\": [\n", + " 53.7996902,\n", + " 38.031826,\n", + " 55.0450439,\n", + " 39.2283897\n", + " ],\n", + " \"stac_extensions\": [],\n", + " \"collection\": \"NA\"\n", + " },\n", + " {\n", + " \"type\": \"Feature\",\n", + " \"stac_version\": \"1.0.0\",\n", + " \"id\": \"G2721220118-LPCLOUD\",\n", + " \"properties\": {\n", + " \"start_datetime\": \"2016-01-31T18:00:00.009057Z\",\n", + " \"end_datetime\": \"2016-01-31T19:59:59.991043Z\",\n", + " \"created\": \"2016-02-01T02:45:59.639000Z\",\n", + " \"updated\": \"2022-03-23T15:48:21.578000Z\",\n", + " \"datetime\": \"2023-06-20T08:44:26Z\"\n", + " },\n", + " \"geometry\": {\n", + " \"type\": \"Point\",\n", + " \"coordinates\": [\n", + " 0.0,\n", + " 0.0\n", + " ]\n", + " },\n", + " \"links\": [\n", + " {\n", + " \"rel\": \"self\",\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.stac\",\n", + " \"type\": \"application/json\"\n", + " }\n", + " ],\n", + " \"assets\": {\n", + " \"EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc\": {\n", + " \"href\": \"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc\",\n", + " \"title\": \"EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc\",\n", + " \"roles\": [\n", + " \"data\"\n", + " ]\n", + " },\n", + " \"EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc\": {\n", + " \"href\": \"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc\",\n", + " \"title\": \"EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc\",\n", + " \"roles\": [\n", + " \"data\"\n", + " ]\n", + " },\n", + " \"EMIT_L1B_RAD_001_20230620T084426_2317106_011.xml\": {\n", + " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.xml\",\n", + " \"title\": \"EMIT_L1B_RAD_001_20230620T084426_2317106_011.xml\",\n", + " \"roles\": [\n", + " \"metadata\"\n", + " ]\n", + " }\n", + " },\n", + " \"bbox\": [\n", + " 53.7996902,\n", + " 38.031826,\n", + " 55.0450439,\n", + " 39.2283897\n", + " ],\n", + " \"stac_extensions\": [],\n", + " \"collection\": \"NA\"\n", + " }\n", + " ]\n", + "}\n" + ] + } + ], + "execution_count": 44 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "#### Performing Stage-in\n", + "- Creating Stage-in Directory\n", + "- Calling Stage-in method to perform stage-in.\n", + "- Checking the result." + ], + "id": "f7756b2cea8a3f" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T17:54:08.217050Z", + "start_time": "2025-02-10T17:48:17.006382Z" + } + }, + "cell_type": "code", + "source": [ + "from mdps_ds_lib.stage_in_out.download_granules_factory import DownloadGranulesFactory\n", + "from mdps_ds_lib.stage_in_out.stage_in_out_utils import StageInOutUtils\n", + "from mdps_ds_lib.lib.utils.file_utils import FileUtils\n", + "from glob import glob\n", + "\n", + "FileUtils.mk_dir_p(os.environ['DOWNLOAD_DIR']) # Creating a base directory if not created. They can be created manually w/o calling this.\n", + "\n", + "# Hardcoded method call. All params are set via environment previously\n", + "result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download()\n", + "StageInOutUtils.write_output_to_file(result_str)\n", + "print('done')\n", + "\n", + "print(list(glob(os.path.join(os.environ['DOWNLOAD_DIR'], '*')))) # Checking if files are downloaded." + ], + "id": "1353836ca421355e", + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-10 09:48:19,170 [ERROR] [mdps_ds_lib.stage_in_out.download_granules_amalgamation::32] downloading: https://cmr.earthdata.nasa.gov:443/search/concepts/G2721699381-LPCLOUD.xml\n", + "2025-02-10 09:48:19,170 [ERROR] [mdps_ds_lib.stage_in_out.download_granules_amalgamation::32] downloading: https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.xml\n", + "2025-02-10 09:48:19,497 [ERROR] [mdps_ds_lib.stage_in_out.download_granules_amalgamation::32] downloading: https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL2ARFL.001/EMIT_L2A_RFL_001_20230620T084426_2317106_011/EMIT_L2A_RFL_001_20230620T084426_2317106_011.nc\n", + "2025-02-10 09:48:19,511 [ERROR] [mdps_ds_lib.stage_in_out.download_granules_amalgamation::32] downloading: https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "done\n", + "['/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/error.log', '/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/G2721699381-LPCLOUD.xml', '/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/G2721220118-LPCLOUD.xml', '/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/G2721220118-LPCLOUD.stac.json', '/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/G2721699381-LPCLOUD.stac.json', '/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/catalog.json', '/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc', '/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/EMIT_L2A_RFL_001_20230620T084426_2317106_011.nc']\n" + ] + } + ], + "execution_count": 11 + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/stage-out-ancillary-example.ipynb b/examples/stage-out-ancillary-example.ipynb new file mode 100644 index 0000000..d082231 --- /dev/null +++ b/examples/stage-out-ancillary-example.ipynb @@ -0,0 +1,274 @@ +{ + "cells": [ + { + "metadata": {}, + "cell_type": "markdown", + "source": "## Stage Out (Ancillary) Example", + "id": "30b6016bc9435c51" + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "Steps\n", + "- Install Data Service (DS) Client Library\n", + "- Set Log Level\n", + "- Set Environment variables for Stage-Out (Ancillary\n", + "- Try Stage-Out Ancillary Files\n", + "- Check the results" + ], + "id": "41307b0225e83dcc" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T17:41:17.675227Z", + "start_time": "2025-02-10T17:41:16.781958Z" + } + }, + "cell_type": "code", + "source": [ + "from anyio.streams import file\n", + "%pip install mdps-ds-lib" + ], + "id": "fe80dd452ebd717b", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: mdps-ds-lib in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (1.1.1)\r\n", + "Requirement already satisfied: boto3<2.0.0,>=1.26.51 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (1.36.15)\r\n", + "Requirement already satisfied: elasticsearch==7.13.4 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (7.13.4)\r\n", + "Requirement already satisfied: fastjsonschema<3.0.0,>=2.19.1 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (2.21.1)\r\n", + "Requirement already satisfied: jsonschema<5.0.0,>=4.23.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (4.23.0)\r\n", + "Requirement already satisfied: pystac==1.9.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (1.9.0)\r\n", + "Requirement already satisfied: requests==2.31.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (2.31.0)\r\n", + "Requirement already satisfied: requests-aws4auth==1.2.3 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (1.2.3)\r\n", + "Requirement already satisfied: tenacity==8.2.3 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (8.2.3)\r\n", + "Requirement already satisfied: xmltodict==0.13.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (0.13.0)\r\n", + "Requirement already satisfied: urllib3<2,>=1.21.1 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from elasticsearch==7.13.4->mdps-ds-lib) (1.26.20)\r\n", + "Requirement already satisfied: certifi in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from elasticsearch==7.13.4->mdps-ds-lib) (2024.12.14)\r\n", + "Requirement already satisfied: python-dateutil>=2.7.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from pystac==1.9.0->mdps-ds-lib) (2.9.0.post0)\r\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from requests==2.31.0->mdps-ds-lib) (3.4.1)\r\n", + "Requirement already satisfied: idna<4,>=2.5 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from requests==2.31.0->mdps-ds-lib) (3.10)\r\n", + "Requirement already satisfied: six in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from requests-aws4auth==1.2.3->mdps-ds-lib) (1.17.0)\r\n", + "Requirement already satisfied: botocore<1.37.0,>=1.36.15 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from boto3<2.0.0,>=1.26.51->mdps-ds-lib) (1.36.15)\r\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from boto3<2.0.0,>=1.26.51->mdps-ds-lib) (1.0.1)\r\n", + "Requirement already satisfied: s3transfer<0.12.0,>=0.11.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from boto3<2.0.0,>=1.26.51->mdps-ds-lib) (0.11.2)\r\n", + "Requirement already satisfied: attrs>=22.2.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.23.0->mdps-ds-lib) (24.3.0)\r\n", + "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.23.0->mdps-ds-lib) (2024.10.1)\r\n", + "Requirement already satisfied: referencing>=0.28.4 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.23.0->mdps-ds-lib) (0.35.1)\r\n", + "Requirement already satisfied: rpds-py>=0.7.1 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.23.0->mdps-ds-lib) (0.22.3)\r\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "execution_count": 3 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "### Setting the Log Level\n", + "- Log level mappings:\n", + "- 10 = debug\n", + "- 20 = info\n", + "- 30 = warning\n", + "- 40 = error" + ], + "id": "e6bf69a910209c6c" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T17:41:24.968457Z", + "start_time": "2025-02-10T17:41:24.965823Z" + } + }, + "cell_type": "code", + "source": [ + "import logging\n", + "log_level = 30\n", + "logging.basicConfig(level=log_level, format=\"%(asctime)s [%(levelname)s] [%(name)s::%(lineno)d] %(message)s\")" + ], + "id": "d2aaaee1a57cb095", + "outputs": [], + "execution_count": 4 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "### Stage-out (Ancillary) Only Environment Variables\n", + "\n", + "Follow this link for more information: https://app.gitbook.com/o/xZRqGQeQXJ0RP4VMj7Lq/s/UMIRhLdbRQTvMWop8Il9/developer-docs/data/docs/users-guide/stage-out" + ], + "id": "44c5455db73d82c0" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T23:11:49.193312Z", + "start_time": "2025-02-10T23:11:49.189141Z" + } + }, + "cell_type": "code", + "source": [ + "from mdps_ds_lib.lib.utils.file_utils import FileUtils\n", + "import os\n", + "\n", + "os.environ['AWS_ACCESS_KEY_ID'] = 'xxx'\n", + "os.environ['AWS_SECRET_ACCESS_KEY'] = 'xxx'\n", + "os.environ['AWS_SESSION_TOKEN'] = 'xxx'\n", + "\n", + "os.environ['GRANULES_UPLOAD_TYPE'] = 'UPLOAD_AUXILIARY_FILE_AS_GRANULE' # Setting uploading as auxiliary\n", + "os.environ['STAGING_BUCKET'] = 'uds-sbx-cumulus-staging' # S3 bucket where they will reside\n", + "os.environ['VERIFY_SSL'] = 'FALSE' # Optional param.\n", + "os.environ['BASE_DIRECTORY'] = 'auxiliary_files' # Base folder to upload\n", + "\n", + "os.environ['OUTPUT_DIRECTORY'] = 'auxiliary_files/output_dir' # the success / failure results to be stored locally for rewview\n", + "os.environ['OUTPUT_FILE'] = 'auxiliary_files/output_dir/stage_out_result.json' # file path where the result overview is written locally for\n", + "\n", + "tenant = 'UDS_DEMO'\n", + "tenant_venue = 'TEST'\n", + "collection_name = 'UDS_UNIT_COLLECTION'\n", + "collection_version = '24.09.10.11.00'.replace('.', '')\n", + "temp_collection_id = f'URN:NASA:UNITY:{tenant}:{tenant_venue}:{collection_name}___{collection_version}'\n", + "os.environ['COLLECTION_ID'] = temp_collection_id # Setting Collection ID.\n", + "# Note that Collection ID needs to follow the standard.\n", + "\n", + "FileUtils.mk_dir_p(os.environ.get('OUTPUT_DIRECTORY'))\n" + ], + "id": "29f786b4ed0929b5", + "outputs": [], + "execution_count": 45 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "#### Mocking some ancillary files", + "id": "f40f81242b3930f7" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T22:09:55.801596Z", + "start_time": "2025-02-10T22:09:55.795144Z" + } + }, + "cell_type": "code", + "source": [ + "granules_dir = os.path.join('auxiliary_files')\n", + "FileUtils.mk_dir_p(granules_dir) # base directory\n", + "total_files = 3 # Uploading 3 mock files\n", + "\n", + "for i in range(1, total_files+1):\n", + " filename = f'mocked_general_file{i:02d}'\n", + " with open(os.path.join(granules_dir, f'{filename}'), 'w') as ff: # Creating Data File\n", + " ff.write('mocked_general_file')" + ], + "id": "9b97566830b4afc", + "outputs": [], + "execution_count": 34 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "#### Performing actual stage-out\n", + "- Note that there is no dry-run on ancillary stage-out since there are no settings that client setup." + ], + "id": "d516f0a9fffbdc2a" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T22:17:28.437985Z", + "start_time": "2025-02-10T22:17:28.433764Z" + } + }, + "cell_type": "code", + "source": [ + "from glob import glob\n", + "\n", + "# Cleaning old result files.\n", + "\n", + "old_result_files = glob(os.path.join(os.environ['OUTPUT_DIRECTORY'], '*'))\n", + "for each_file in old_result_files:\n", + " FileUtils.remove_if_exists(each_file)" + ], + "id": "c61692dd202c889c", + "outputs": [], + "execution_count": 40 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T22:17:35.511806Z", + "start_time": "2025-02-10T22:17:31.251498Z" + } + }, + "cell_type": "code", + "source": [ + "\n", + "from mdps_ds_lib.stage_in_out.upoad_granules_factory import UploadGranulesFactory\n", + "from mdps_ds_lib.stage_in_out.stage_in_out_utils import StageInOutUtils\n", + "\n", + "# Hardcoded method call. All params are set via environment previously\n", + "upload_result_str = UploadGranulesFactory().get_class(os.getenv('GRANULES_UPLOAD_TYPE', UploadGranulesFactory.UPLOAD_S3_BY_STAC_CATALOG)).upload()\n", + "StageInOutUtils.write_output_to_file(upload_result_str)\n", + "print('done')" + ], + "id": "3aef5ceb43098a07", + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-10 14:17:31,715 [AUDIT] [mdps_ds_lib.stage_in_out.upload_arbitrary_files_as_granules::11] uploading auxiliary file: auxiliary_files/mocked_general_file01\n", + "2025-02-10 14:17:31,715 [AUDIT] [mdps_ds_lib.stage_in_out.upload_arbitrary_files_as_granules::11] uploading auxiliary file: auxiliary_files/mocked_general_file03\n", + "2025-02-10 14:17:31,714 [AUDIT] [mdps_ds_lib.stage_in_out.upload_arbitrary_files_as_granules::11] uploading auxiliary file: auxiliary_files/mocked_general_file02\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "done\n" + ] + } + ], + "execution_count": 41 + }, + { + "metadata": {}, + "cell_type": "code", + "outputs": [], + "execution_count": null, + "source": "", + "id": "2987dc7d3c9d0898" + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/stage-out-example.ipynb b/examples/stage-out-example.ipynb new file mode 100644 index 0000000..204eb81 --- /dev/null +++ b/examples/stage-out-example.ipynb @@ -0,0 +1,567 @@ +{ + "cells": [ + { + "metadata": {}, + "cell_type": "markdown", + "source": "## Stage Out Example", + "id": "30b6016bc9435c51" + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "Steps\n", + "- Install Data Service (DS) Client Library\n", + "- Set Log Level\n", + "- Set Environment variables for Stage-Out\n", + "- Try Stage-Out (Dry-Run Only)\n", + "- Try Stage-Out\n", + "- Check the results" + ], + "id": "41307b0225e83dcc" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T17:41:17.675227Z", + "start_time": "2025-02-10T17:41:16.781958Z" + } + }, + "cell_type": "code", + "source": [ + "from anyio.streams import file\n", + "%pip install mdps-ds-lib" + ], + "id": "fe80dd452ebd717b", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: mdps-ds-lib in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (1.1.1)\r\n", + "Requirement already satisfied: boto3<2.0.0,>=1.26.51 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (1.36.15)\r\n", + "Requirement already satisfied: elasticsearch==7.13.4 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (7.13.4)\r\n", + "Requirement already satisfied: fastjsonschema<3.0.0,>=2.19.1 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (2.21.1)\r\n", + "Requirement already satisfied: jsonschema<5.0.0,>=4.23.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (4.23.0)\r\n", + "Requirement already satisfied: pystac==1.9.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (1.9.0)\r\n", + "Requirement already satisfied: requests==2.31.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (2.31.0)\r\n", + "Requirement already satisfied: requests-aws4auth==1.2.3 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (1.2.3)\r\n", + "Requirement already satisfied: tenacity==8.2.3 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (8.2.3)\r\n", + "Requirement already satisfied: xmltodict==0.13.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (0.13.0)\r\n", + "Requirement already satisfied: urllib3<2,>=1.21.1 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from elasticsearch==7.13.4->mdps-ds-lib) (1.26.20)\r\n", + "Requirement already satisfied: certifi in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from elasticsearch==7.13.4->mdps-ds-lib) (2024.12.14)\r\n", + "Requirement already satisfied: python-dateutil>=2.7.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from pystac==1.9.0->mdps-ds-lib) (2.9.0.post0)\r\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from requests==2.31.0->mdps-ds-lib) (3.4.1)\r\n", + "Requirement already satisfied: idna<4,>=2.5 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from requests==2.31.0->mdps-ds-lib) (3.10)\r\n", + "Requirement already satisfied: six in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from requests-aws4auth==1.2.3->mdps-ds-lib) (1.17.0)\r\n", + "Requirement already satisfied: botocore<1.37.0,>=1.36.15 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from boto3<2.0.0,>=1.26.51->mdps-ds-lib) (1.36.15)\r\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from boto3<2.0.0,>=1.26.51->mdps-ds-lib) (1.0.1)\r\n", + "Requirement already satisfied: s3transfer<0.12.0,>=0.11.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from boto3<2.0.0,>=1.26.51->mdps-ds-lib) (0.11.2)\r\n", + "Requirement already satisfied: attrs>=22.2.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.23.0->mdps-ds-lib) (24.3.0)\r\n", + "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.23.0->mdps-ds-lib) (2024.10.1)\r\n", + "Requirement already satisfied: referencing>=0.28.4 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.23.0->mdps-ds-lib) (0.35.1)\r\n", + "Requirement already satisfied: rpds-py>=0.7.1 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.23.0->mdps-ds-lib) (0.22.3)\r\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "execution_count": 3 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "### Setting the Log Level\n", + "- Log level mappings:\n", + "- 10 = debug\n", + "- 20 = info\n", + "- 30 = warning\n", + "- 40 = error" + ], + "id": "e6bf69a910209c6c" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T17:41:24.968457Z", + "start_time": "2025-02-10T17:41:24.965823Z" + } + }, + "cell_type": "code", + "source": [ + "import logging\n", + "log_level = 30\n", + "logging.basicConfig(level=log_level, format=\"%(asctime)s [%(levelname)s] [%(name)s::%(lineno)d] %(message)s\")" + ], + "id": "d2aaaee1a57cb095", + "outputs": [], + "execution_count": 4 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "### Stage-out Only Environment Variables\n", + "\n", + "Follow this link for more information: https://app.gitbook.com/o/xZRqGQeQXJ0RP4VMj7Lq/s/UMIRhLdbRQTvMWop8Il9/developer-docs/data/docs/users-guide/stage-out" + ], + "id": "44c5455db73d82c0" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T21:25:22.300759Z", + "start_time": "2025-02-10T21:25:22.297419Z" + } + }, + "cell_type": "code", + "source": [ + "import os\n", + "from mdps_ds_lib.lib.utils.file_utils import FileUtils\n", + "\n", + "# The following environment variables are needed to \"stage-in\" from AWS S3\n", + "# They will also be needed if \"EDL\" settings and \"STAC_AUTH_TYPE\" settings are coming from parameter store\n", + "# Note that this may not be needed if JupyterNotebook can take care of the access.\n", + "os.environ['AWS_ACCESS_KEY_ID'] = 'xxx'\n", + "os.environ['AWS_SECRET_ACCESS_KEY'] = 'xxx'\n", + "os.environ['AWS_SESSION_TOKEN'] = 'xxx'\n", + "\n", + "os.environ['PROJECT'] = 'DEMO'\n", + "os.environ['VENUE'] = 'DEV43'\n", + "\n", + "os.environ['VERIFY_SSL'] = 'FALSE'\n", + "os.environ['RESULT_PATH_PREFIX'] = '' # We can usually ignore this\n", + "os.environ['STAGING_BUCKET'] = 'uds-sbx-cumulus-staging'\n", + "os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY'\n", + "os.environ['OUTPUT_FILE'] = 'normal-stage-out/some_output/output.json'\n", + "os.environ['OUTPUT_DIRECTORY'] = 'normal-stage-out/output_dir'\n", + "FileUtils.mk_dir_p(os.environ.get('OUTPUT_DIRECTORY'))\n", + "os.environ['CATALOG_FILE'] = 'normal-stage-out/catalog.json'" + ], + "id": "518885efaf032c92", + "outputs": [], + "execution_count": 23 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "#### Sample catalog.json used as a guide for stage-out", + "id": "c4db172f84e3e689" + }, + { + "metadata": {}, + "cell_type": "code", + "outputs": [], + "execution_count": null, + "source": [ + "{\n", + " \"type\": \"Catalog\",\n", + " \"id\": \"NA\",\n", + " \"stac_version\": \"1.0.0\",\n", + " \"description\": \"NA\",\n", + " \"links\": [\n", + " {\n", + " \"rel\": \"root\",\n", + " \"href\": \"/tmp/normal-stage-out/catalog.json\",\n", + " \"type\": \"application/json\"\n", + " },\n", + " {\n", + " \"rel\": \"item\",\n", + " \"href\": \"some_granules/test_file01.nc.stac.json\",\n", + " \"type\": \"application/json\"\n", + " },\n", + " {\n", + " \"rel\": \"item\",\n", + " \"href\": \"some_granules/test_file02.nc.stac.json\",\n", + " \"type\": \"application/json\"\n", + " },\n", + " {\n", + " \"rel\": \"item\",\n", + " \"href\": \"some_granules/test_file03.nc.stac.json\",\n", + " \"type\": \"application/json\"\n", + " },\n", + " {\n", + " \"rel\": \"item\",\n", + " \"href\": \"some_granules/test_file04.nc.stac.json\",\n", + " \"type\": \"application/json\"\n", + " },\n", + " {\n", + " \"rel\": \"item\",\n", + " \"href\": \"some_granules/test_file05.nc.stac.json\",\n", + " \"type\": \"application/json\"\n", + " },\n", + " {\n", + " \"rel\": \"item\",\n", + " \"href\": \"some_granules/test_file06.nc.stac.json\",\n", + " \"type\": \"application/json\"\n", + " },\n", + " {\n", + " \"rel\": \"item\",\n", + " \"href\": \"some_granules/test_file07.nc.stac.json\",\n", + " \"type\": \"application/json\"\n", + " },\n", + " {\n", + " \"rel\": \"item\",\n", + " \"href\": \"some_granules/test_file08.nc.stac.json\",\n", + " \"type\": \"application/json\"\n", + " },\n", + " {\n", + " \"rel\": \"item\",\n", + " \"href\": \"some_granules/test_file09.nc.stac.json\",\n", + " \"type\": \"application/json\"\n", + " },\n", + " {\n", + " \"rel\": \"item\",\n", + " \"href\": \"some_granules/test_file10.nc.stac.json\",\n", + " \"type\": \"application/json\"\n", + " }\n", + " ]\n", + "}" + ], + "id": "1b7097396bfbb1f3" + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "- Creating Some Mock Files for stage-out including STAC metadata file and a catalog\n", + "- Note that this is usually not done here as the application will take care of it.\n", + "- How each stac metadata file is created can be used as an example though" + ], + "id": "f71cb9161be6325d" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T19:51:39.435978Z", + "start_time": "2025-02-10T19:51:39.425621Z" + } + }, + "cell_type": "code", + "source": [ + "import json\n", + "from mdps_ds_lib.lib.utils.time_utils import TimeUtils\n", + "from pystac import Item, Asset, Catalog, Link\n", + "\n", + "granules_dir = os.path.join('normal-stage-out', 'some_granules')\n", + "FileUtils.mk_dir_p(granules_dir) # base directory\n", + "total_files = 3 # Uploading 3 mock granules\n", + "\n", + "catalog = Catalog( # Creating a STAC catalog (an example from previous cell)\n", + " id='NA', # we don't need to know the ID\n", + " description='NA')\n", + "catalog.set_self_href(os.environ['CATALOG_FILE'])\n", + "\n", + "for i in range(1, total_files+1):\n", + " filename = f'test_file{i:02d}'\n", + " with open(os.path.join(granules_dir, f'{filename}.nc'), 'w') as ff: # Creating Data File\n", + " ff.write('sample_file')\n", + " with open(os.path.join(granules_dir, f'{filename}.nc.cas'), 'w') as ff: # Creating native metadata file\n", + " ff.write('''\n", + " \n", + " \n", + " AggregateDir\n", + " snppatmsl1a\n", + " \n", + " \n", + " AutomaticQualityFlag\n", + " Passed\n", + " \n", + " \n", + " BuildId\n", + " v01.43.00\n", + " \n", + " \n", + " CollectionLabel\n", + " L1AMw_nominal2\n", + " \n", + " \n", + " DataGroup\n", + " sndr\n", + " \n", + " \n", + " EndDateTime\n", + " 2016-01-14T10:06:00.000Z\n", + " \n", + " \n", + " EndTAI93\n", + " 726919569.000\n", + " \n", + " \n", + " FileFormat\n", + " nc4\n", + " \n", + " \n", + " FileLocation\n", + " /pge/out\n", + " \n", + " \n", + " Filename\n", + " SNDR.SNPP.ATMS.L1A.nominal2.02.nc\n", + " \n", + " \n", + " GranuleNumber\n", + " 101\n", + " \n", + " \n", + " JobId\n", + " f163835c-9945-472f-bee2-2bc12673569f\n", + " \n", + " \n", + " ModelId\n", + " urn:npp:SnppAtmsL1a\n", + " \n", + " \n", + " NominalDate\n", + " 2016-01-14\n", + " \n", + " \n", + " ProductName\n", + " SNDR.SNPP.ATMS.20160114T1000.m06.g101.L1A.L1AMw_nominal2.v03_15_00.D.201214135000.nc\n", + " \n", + " \n", + " ProductType\n", + " SNDR_SNPP_ATMS_L1A\n", + " \n", + " \n", + " ProductionDateTime\n", + " 2020-12-14T13:50:00.000Z\n", + " \n", + " \n", + " ProductionLocation\n", + " Sounder SIPS: JPL/Caltech (Dev)\n", + " \n", + " \n", + " ProductionLocationCode\n", + " D\n", + " \n", + " \n", + " RequestId\n", + " 1215\n", + " \n", + " \n", + " StartDateTime\n", + " 2016-01-14T10:00:00.000Z\n", + " \n", + " \n", + " StartTAI93\n", + " 726919209.000\n", + " \n", + " \n", + " TaskId\n", + " 8c3ae101-8f7c-46c8-b5c6-63e7b6d3c8cd\n", + " \n", + " ''')\n", + " # Creating STAC metadata file object which is used by DS during stage-out and cataloging.\n", + " # pystac library is used to create a stac item file to have a standardized stac.\n", + " stac_item = Item(id=filename,\n", + " geometry={ # Set them if the algorithm knows what type of geometry is needed\n", + " \"type\": \"Point\",\n", + " \"coordinates\": [0.0, 0.0]\n", + " },\n", + " bbox=[-180, -90, 180, 90], # Set them if the algorithm knows what type of geometry is needed\n", + " datetime=TimeUtils().parse_from_unix(0, True).get_datetime_obj(), # Current metadata file creation datetime\n", + " properties={ # These 4 fields are mandatory\n", + " \"start_datetime\": \"2016-01-31T18:00:00.009057Z\",\n", + " \"end_datetime\": \"2016-01-31T19:59:59.991043Z\",\n", + " \"created\": \"2016-02-01T02:45:59.639000Z\",\n", + " \"updated\": \"2022-03-23T15:48:21.578000Z\",\n", + " },\n", + " href=os.path.join('some_granules', f'{filename}.nc.stac.json'),\n", + " collection='NA', # No need to find out what collection it belongs to DS will take care of that.\n", + " assets={ # Point to all relevant files including itself\n", + " f'{filename}.nc': Asset(os.path.join('.', f'{filename}.nc'), title='test_file01.nc', roles=['data']),\n", + " f'{filename}.nc.cas': Asset(os.path.join('.', f'{filename}.nc.cas'), title='test_file01.nc.cas', roles=['metadata']),\n", + " f'{filename}.nc.stac.json': Asset(os.path.join('.', f'{filename}.nc.stac.json'), title='test_file01.nc.stac.json', roles=['metadata']),\n", + " })\n", + " with open(os.path.join(granules_dir, f'{filename}.nc.stac.json'), 'w') as ff: # Creating STAC metadata file\n", + " ff.write(json.dumps(stac_item.to_dict(False, False)))\n", + " # Adding to the Catalog so that DS can find out the STAC file and finds out other related files from STAC\n", + " catalog.add_link(Link('item', os.path.join('some_granules', f'{filename}.nc.stac.json'), 'application/json'))\n", + "# Writing the main catalog file\n", + "with open(os.environ['CATALOG_FILE'], 'w') as ff:\n", + " ff.write(json.dumps(catalog.to_dict(False, False)))\n" + ], + "id": "cd2793837298e627", + "outputs": [], + "execution_count": 14 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "### Performing Stage-out as a dry-run to verify if things are setup correctly", + "id": "7cd0f6aaaa07ef8e" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T21:26:41.690596Z", + "start_time": "2025-02-10T21:26:38.269917Z" + } + }, + "cell_type": "code", + "source": [ + "os.environ['DRY_RUN'] = 'TRUE'\n", + "from mdps_ds_lib.stage_in_out.upoad_granules_factory import UploadGranulesFactory\n", + "\n", + "upload_result = UploadGranulesFactory().get_class(UploadGranulesFactory.UPLOAD_S3_BY_STAC_CATALOG).upload()\n", + "print(upload_result)" + ], + "id": "c46d384d19572817", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Result of dry-run\n", + "{\n", + " \"granule_file\": \"normal-stage-out/some_granules/test_file02.nc.stac.json\",\n", + " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file02/test_file02.nc\"\n", + "}\n", + "{\n", + " \"granule_file\": \"normal-stage-out/some_granules/test_file02.nc.stac.json\",\n", + " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file02/test_file02.nc.cas\"\n", + "}\n", + "{\n", + " \"granule_file\": \"normal-stage-out/some_granules/test_file02.nc.stac.json\",\n", + " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file02/test_file02.nc.stac.json\"\n", + "}\n", + "{\n", + " \"granule_file\": \"normal-stage-out/some_granules/test_file03.nc.stac.json\",\n", + " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file03/test_file03.nc\"\n", + "}\n", + "{\n", + " \"granule_file\": \"normal-stage-out/some_granules/test_file03.nc.stac.json\",\n", + " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file03/test_file03.nc.cas\"\n", + "}\n", + "{\n", + " \"granule_file\": \"normal-stage-out/some_granules/test_file03.nc.stac.json\",\n", + " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file03/test_file03.nc.stac.json\"\n", + "}\n", + "{\n", + " \"granule_file\": \"normal-stage-out/some_granules/test_file01.nc.stac.json\",\n", + " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file01/test_file01.nc\"\n", + "}\n", + "{\n", + " \"granule_file\": \"normal-stage-out/some_granules/test_file01.nc.stac.json\",\n", + " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file01/test_file01.nc.cas\"\n", + "}\n", + "{\n", + " \"granule_file\": \"normal-stage-out/some_granules/test_file01.nc.stac.json\",\n", + " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file01/test_file01.nc.stac.json\"\n", + "}\n", + "{}\n" + ] + } + ], + "execution_count": 25 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "### Performing Stage-out", + "id": "e87a8f50d52e90dc" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T21:29:21.314515Z", + "start_time": "2025-02-10T21:29:21.311438Z" + } + }, + "cell_type": "code", + "source": [ + "from glob import glob\n", + "\n", + "# Cleaning old result files.\n", + "\n", + "old_result_files = glob(os.path.join(os.environ['OUTPUT_DIRECTORY'], '*'))\n", + "for each_file in old_result_files:\n", + " FileUtils.remove_if_exists(each_file)" + ], + "id": "cda0ee7d594d1513", + "outputs": [], + "execution_count": 32 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-10T21:29:27.704310Z", + "start_time": "2025-02-10T21:29:23.201698Z" + } + }, + "cell_type": "code", + "source": [ + "\n", + "os.environ['DRY_RUN'] = 'FALSE'\n", + "\n", + "FileUtils.remove_if_exists(os.environ['OUTPUT_DIRECTORY'])\n", + "from mdps_ds_lib.stage_in_out.upoad_granules_factory import UploadGranulesFactory\n", + "\n", + "upload_result = UploadGranulesFactory().get_class(UploadGranulesFactory.UPLOAD_S3_BY_STAC_CATALOG).upload()\n", + "print(upload_result)" + ], + "id": "2e98b386bd09a655", + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-10 13:29:23,648 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=data, name=test_file01.nc, href=normal-stage-out/some_granules/./test_file01.nc\n", + "2025-02-10 13:29:23,648 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=data, name=test_file02.nc, href=normal-stage-out/some_granules/./test_file02.nc\n", + "2025-02-10 13:29:23,650 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=data, name=test_file03.nc, href=normal-stage-out/some_granules/./test_file03.nc\n", + "2025-02-10 13:29:24,041 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=metadata, name=test_file02.nc.cas, href=normal-stage-out/some_granules/./test_file02.nc.cas\n", + "2025-02-10 13:29:24,043 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=metadata, name=test_file03.nc.cas, href=normal-stage-out/some_granules/./test_file03.nc.cas\n", + "2025-02-10 13:29:24,053 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=metadata, name=test_file01.nc.cas, href=normal-stage-out/some_granules/./test_file01.nc.cas\n", + "2025-02-10 13:29:24,202 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=metadata, name=test_file02.nc.stac.json, href=normal-stage-out/some_granules/./test_file02.nc.stac.json\n", + "2025-02-10 13:29:24,207 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=metadata, name=test_file03.nc.stac.json, href=normal-stage-out/some_granules/./test_file03.nc.stac.json\n", + "2025-02-10 13:29:24,215 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=metadata, name=test_file01.nc.stac.json, href=normal-stage-out/some_granules/./test_file01.nc.stac.json\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\"type\": \"Catalog\", \"id\": \"NA\", \"stac_version\": \"1.0.0\", \"description\": \"NA\", \"links\": [{\"rel\": \"root\", \"href\": \"/Users/wphyo/Projects/unity/uds_lib/examples/normal-stage-out/catalog.json\", \"type\": \"application/json\"}, {\"rel\": \"item\", \"href\": \"normal-stage-out/output_dir/successful_features.json\", \"type\": \"application/json\"}, {\"rel\": \"item\", \"href\": \"normal-stage-out/output_dir/failed_features.json\", \"type\": \"application/json\"}]}\n" + ] + } + ], + "execution_count": 33 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "- Successful and failed items are written to 2 files into the directory set in `OUTPUT_DIRECTORY`", + "id": "61efcaa319b065b7" + }, + { + "metadata": {}, + "cell_type": "code", + "outputs": [], + "execution_count": null, + "source": "", + "id": "2987dc7d3c9d0898" + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 04fb9a91cdbe41f9aa7209bfeeac2ae1b7393baf Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Tue, 11 Feb 2025 11:57:22 -0800 Subject: [PATCH 08/11] chore: test execute --- examples/stage-in-example.ipynb | 56 +++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/examples/stage-in-example.ipynb b/examples/stage-in-example.ipynb index 7af72c4..77ee32e 100644 --- a/examples/stage-in-example.ipynb +++ b/examples/stage-in-example.ipynb @@ -22,8 +22,8 @@ { "metadata": { "ExecuteTime": { - "end_time": "2025-02-10T17:41:17.675227Z", - "start_time": "2025-02-10T17:41:16.781958Z" + "end_time": "2025-02-11T19:53:24.492999Z", + "start_time": "2025-02-11T19:53:24.010584Z" } }, "cell_type": "code", @@ -64,7 +64,7 @@ ] } ], - "execution_count": 3 + "execution_count": 2 }, { "metadata": {}, @@ -82,8 +82,8 @@ { "metadata": { "ExecuteTime": { - "end_time": "2025-02-10T17:41:24.968457Z", - "start_time": "2025-02-10T17:41:24.965823Z" + "end_time": "2025-02-11T19:53:26.256005Z", + "start_time": "2025-02-11T19:53:26.252858Z" } }, "cell_type": "code", @@ -94,7 +94,7 @@ ], "id": "d2aaaee1a57cb095", "outputs": [], - "execution_count": 4 + "execution_count": 3 }, { "metadata": {}, @@ -110,8 +110,8 @@ { "metadata": { "ExecuteTime": { - "end_time": "2025-02-10T23:09:05.795762Z", - "start_time": "2025-02-10T23:09:05.790161Z" + "end_time": "2025-02-11T19:54:20.621859Z", + "start_time": "2025-02-11T19:54:20.616769Z" } }, "cell_type": "code", @@ -149,7 +149,7 @@ ], "id": "abb5f945876a1a63", "outputs": [], - "execution_count": 43 + "execution_count": 5 }, { "metadata": {}, @@ -163,8 +163,8 @@ { "metadata": { "ExecuteTime": { - "end_time": "2025-02-10T17:41:44.530922Z", - "start_time": "2025-02-10T17:41:44.519332Z" + "end_time": "2025-02-11T19:54:23.479934Z", + "start_time": "2025-02-11T19:54:23.466763Z" } }, "cell_type": "code", @@ -459,12 +459,12 @@ " 'collection': 'C2408750690-LPCLOUD'}]}" ] }, - "execution_count": 7, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], - "execution_count": 7 + "execution_count": 6 }, { "metadata": {}, @@ -475,8 +475,8 @@ { "metadata": { "ExecuteTime": { - "end_time": "2025-02-10T23:09:49.537304Z", - "start_time": "2025-02-10T23:09:49.530650Z" + "end_time": "2025-02-11T19:54:33.736290Z", + "start_time": "2025-02-11T19:54:33.634548Z" } }, "cell_type": "code", @@ -680,7 +680,7 @@ ] } ], - "execution_count": 44 + "execution_count": 7 }, { "metadata": {}, @@ -696,8 +696,8 @@ { "metadata": { "ExecuteTime": { - "end_time": "2025-02-10T17:54:08.217050Z", - "start_time": "2025-02-10T17:48:17.006382Z" + "end_time": "2025-02-11T19:56:35.357697Z", + "start_time": "2025-02-11T19:54:37.689595Z" } }, "cell_type": "code", @@ -722,10 +722,10 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-02-10 09:48:19,170 [ERROR] [mdps_ds_lib.stage_in_out.download_granules_amalgamation::32] downloading: https://cmr.earthdata.nasa.gov:443/search/concepts/G2721699381-LPCLOUD.xml\n", - "2025-02-10 09:48:19,170 [ERROR] [mdps_ds_lib.stage_in_out.download_granules_amalgamation::32] downloading: https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.xml\n", - "2025-02-10 09:48:19,497 [ERROR] [mdps_ds_lib.stage_in_out.download_granules_amalgamation::32] downloading: https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL2ARFL.001/EMIT_L2A_RFL_001_20230620T084426_2317106_011/EMIT_L2A_RFL_001_20230620T084426_2317106_011.nc\n", - "2025-02-10 09:48:19,511 [ERROR] [mdps_ds_lib.stage_in_out.download_granules_amalgamation::32] downloading: https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc\n" + "2025-02-11 11:54:39,561 [ERROR] [mdps_ds_lib.stage_in_out.download_granules_amalgamation::32] downloading: https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.xml\n", + "2025-02-11 11:54:39,561 [ERROR] [mdps_ds_lib.stage_in_out.download_granules_amalgamation::32] downloading: https://cmr.earthdata.nasa.gov:443/search/concepts/G2721699381-LPCLOUD.xml\n", + "2025-02-11 11:54:39,845 [ERROR] [mdps_ds_lib.stage_in_out.download_granules_amalgamation::32] downloading: https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL2ARFL.001/EMIT_L2A_RFL_001_20230620T084426_2317106_011/EMIT_L2A_RFL_001_20230620T084426_2317106_011.nc\n", + "2025-02-11 11:54:39,848 [ERROR] [mdps_ds_lib.stage_in_out.download_granules_amalgamation::32] downloading: https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc\n" ] }, { @@ -733,11 +733,19 @@ "output_type": "stream", "text": [ "done\n", - "['/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/error.log', '/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/G2721699381-LPCLOUD.xml', '/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/G2721220118-LPCLOUD.xml', '/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/G2721220118-LPCLOUD.stac.json', '/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/G2721699381-LPCLOUD.stac.json', '/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/catalog.json', '/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc', '/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/EMIT_L2A_RFL_001_20230620T084426_2317106_011.nc']\n" + "['/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/G2721699381-LPCLOUD.xml', '/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/G2721220118-LPCLOUD.xml', '/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/G2721220118-LPCLOUD.stac.json', '/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/G2721699381-LPCLOUD.stac.json', '/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/catalog.json', '/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc', '/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/EMIT_L2A_RFL_001_20230620T084426_2317106_011.nc']\n" ] } ], - "execution_count": 11 + "execution_count": 8 + }, + { + "metadata": {}, + "cell_type": "code", + "outputs": [], + "execution_count": null, + "source": "", + "id": "4c294d528124a3ec" } ], "metadata": { From 8aff8fae44c5a69cbb15485338f7a9e3728e974f Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Tue, 11 Feb 2025 11:58:28 -0800 Subject: [PATCH 09/11] chore: re-execute --- examples/stage-out-ancillary-example.ipynb | 42 +++++++++++----------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/examples/stage-out-ancillary-example.ipynb b/examples/stage-out-ancillary-example.ipynb index d082231..4af29dc 100644 --- a/examples/stage-out-ancillary-example.ipynb +++ b/examples/stage-out-ancillary-example.ipynb @@ -22,8 +22,8 @@ { "metadata": { "ExecuteTime": { - "end_time": "2025-02-10T17:41:17.675227Z", - "start_time": "2025-02-10T17:41:16.781958Z" + "end_time": "2025-02-11T19:57:35.450042Z", + "start_time": "2025-02-11T19:57:34.844022Z" } }, "cell_type": "code", @@ -64,7 +64,7 @@ ] } ], - "execution_count": 3 + "execution_count": 1 }, { "metadata": {}, @@ -82,8 +82,8 @@ { "metadata": { "ExecuteTime": { - "end_time": "2025-02-10T17:41:24.968457Z", - "start_time": "2025-02-10T17:41:24.965823Z" + "end_time": "2025-02-11T19:57:38.535713Z", + "start_time": "2025-02-11T19:57:38.532166Z" } }, "cell_type": "code", @@ -94,7 +94,7 @@ ], "id": "d2aaaee1a57cb095", "outputs": [], - "execution_count": 4 + "execution_count": 2 }, { "metadata": {}, @@ -109,8 +109,8 @@ { "metadata": { "ExecuteTime": { - "end_time": "2025-02-10T23:11:49.193312Z", - "start_time": "2025-02-10T23:11:49.189141Z" + "end_time": "2025-02-11T19:57:45.072584Z", + "start_time": "2025-02-11T19:57:45.036781Z" } }, "cell_type": "code", @@ -142,7 +142,7 @@ ], "id": "29f786b4ed0929b5", "outputs": [], - "execution_count": 45 + "execution_count": 3 }, { "metadata": {}, @@ -153,8 +153,8 @@ { "metadata": { "ExecuteTime": { - "end_time": "2025-02-10T22:09:55.801596Z", - "start_time": "2025-02-10T22:09:55.795144Z" + "end_time": "2025-02-11T19:57:47.378396Z", + "start_time": "2025-02-11T19:57:47.372003Z" } }, "cell_type": "code", @@ -170,7 +170,7 @@ ], "id": "9b97566830b4afc", "outputs": [], - "execution_count": 34 + "execution_count": 4 }, { "metadata": {}, @@ -184,8 +184,8 @@ { "metadata": { "ExecuteTime": { - "end_time": "2025-02-10T22:17:28.437985Z", - "start_time": "2025-02-10T22:17:28.433764Z" + "end_time": "2025-02-11T19:57:54.140366Z", + "start_time": "2025-02-11T19:57:54.136673Z" } }, "cell_type": "code", @@ -200,13 +200,13 @@ ], "id": "c61692dd202c889c", "outputs": [], - "execution_count": 40 + "execution_count": 5 }, { "metadata": { "ExecuteTime": { - "end_time": "2025-02-10T22:17:35.511806Z", - "start_time": "2025-02-10T22:17:31.251498Z" + "end_time": "2025-02-11T19:57:59.320173Z", + "start_time": "2025-02-11T19:57:55.038070Z" } }, "cell_type": "code", @@ -226,9 +226,9 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-02-10 14:17:31,715 [AUDIT] [mdps_ds_lib.stage_in_out.upload_arbitrary_files_as_granules::11] uploading auxiliary file: auxiliary_files/mocked_general_file01\n", - "2025-02-10 14:17:31,715 [AUDIT] [mdps_ds_lib.stage_in_out.upload_arbitrary_files_as_granules::11] uploading auxiliary file: auxiliary_files/mocked_general_file03\n", - "2025-02-10 14:17:31,714 [AUDIT] [mdps_ds_lib.stage_in_out.upload_arbitrary_files_as_granules::11] uploading auxiliary file: auxiliary_files/mocked_general_file02\n" + "2025-02-11 11:57:55,489 [AUDIT] [mdps_ds_lib.stage_in_out.upload_arbitrary_files_as_granules::11] uploading auxiliary file: auxiliary_files/mocked_general_file02\n", + "2025-02-11 11:57:55,489 [AUDIT] [mdps_ds_lib.stage_in_out.upload_arbitrary_files_as_granules::11] uploading auxiliary file: auxiliary_files/mocked_general_file03\n", + "2025-02-11 11:57:55,489 [AUDIT] [mdps_ds_lib.stage_in_out.upload_arbitrary_files_as_granules::11] uploading auxiliary file: auxiliary_files/mocked_general_file01\n" ] }, { @@ -239,7 +239,7 @@ ] } ], - "execution_count": 41 + "execution_count": 6 }, { "metadata": {}, From c0aad1dd5aaae411f0ad9f37f7f943c653ef8d66 Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Tue, 11 Feb 2025 11:59:44 -0800 Subject: [PATCH 10/11] chore: re-execute --- examples/stage-out-example.ipynb | 144 +++++++++++++++++++++---------- 1 file changed, 98 insertions(+), 46 deletions(-) diff --git a/examples/stage-out-example.ipynb b/examples/stage-out-example.ipynb index 204eb81..aa77aeb 100644 --- a/examples/stage-out-example.ipynb +++ b/examples/stage-out-example.ipynb @@ -23,8 +23,8 @@ { "metadata": { "ExecuteTime": { - "end_time": "2025-02-10T17:41:17.675227Z", - "start_time": "2025-02-10T17:41:16.781958Z" + "end_time": "2025-02-11T19:58:41.635059Z", + "start_time": "2025-02-11T19:58:41.158509Z" } }, "cell_type": "code", @@ -65,7 +65,7 @@ ] } ], - "execution_count": 3 + "execution_count": 1 }, { "metadata": {}, @@ -83,8 +83,8 @@ { "metadata": { "ExecuteTime": { - "end_time": "2025-02-10T17:41:24.968457Z", - "start_time": "2025-02-10T17:41:24.965823Z" + "end_time": "2025-02-11T19:58:43.276301Z", + "start_time": "2025-02-11T19:58:43.274300Z" } }, "cell_type": "code", @@ -95,7 +95,7 @@ ], "id": "d2aaaee1a57cb095", "outputs": [], - "execution_count": 4 + "execution_count": 2 }, { "metadata": {}, @@ -110,8 +110,8 @@ { "metadata": { "ExecuteTime": { - "end_time": "2025-02-10T21:25:22.300759Z", - "start_time": "2025-02-10T21:25:22.297419Z" + "end_time": "2025-02-11T19:58:48.771462Z", + "start_time": "2025-02-11T19:58:48.748302Z" } }, "cell_type": "code", @@ -140,7 +140,7 @@ ], "id": "518885efaf032c92", "outputs": [], - "execution_count": 23 + "execution_count": 3 }, { "metadata": {}, @@ -149,10 +149,13 @@ "id": "c4db172f84e3e689" }, { - "metadata": {}, + "metadata": { + "ExecuteTime": { + "end_time": "2025-02-11T19:58:53.565054Z", + "start_time": "2025-02-11T19:58:53.549031Z" + } + }, "cell_type": "code", - "outputs": [], - "execution_count": null, "source": [ "{\n", " \"type\": \"Catalog\",\n", @@ -218,7 +221,56 @@ " ]\n", "}" ], - "id": "1b7097396bfbb1f3" + "id": "1b7097396bfbb1f3", + "outputs": [ + { + "data": { + "text/plain": [ + "{'type': 'Catalog',\n", + " 'id': 'NA',\n", + " 'stac_version': '1.0.0',\n", + " 'description': 'NA',\n", + " 'links': [{'rel': 'root',\n", + " 'href': '/tmp/normal-stage-out/catalog.json',\n", + " 'type': 'application/json'},\n", + " {'rel': 'item',\n", + " 'href': 'some_granules/test_file01.nc.stac.json',\n", + " 'type': 'application/json'},\n", + " {'rel': 'item',\n", + " 'href': 'some_granules/test_file02.nc.stac.json',\n", + " 'type': 'application/json'},\n", + " {'rel': 'item',\n", + " 'href': 'some_granules/test_file03.nc.stac.json',\n", + " 'type': 'application/json'},\n", + " {'rel': 'item',\n", + " 'href': 'some_granules/test_file04.nc.stac.json',\n", + " 'type': 'application/json'},\n", + " {'rel': 'item',\n", + " 'href': 'some_granules/test_file05.nc.stac.json',\n", + " 'type': 'application/json'},\n", + " {'rel': 'item',\n", + " 'href': 'some_granules/test_file06.nc.stac.json',\n", + " 'type': 'application/json'},\n", + " {'rel': 'item',\n", + " 'href': 'some_granules/test_file07.nc.stac.json',\n", + " 'type': 'application/json'},\n", + " {'rel': 'item',\n", + " 'href': 'some_granules/test_file08.nc.stac.json',\n", + " 'type': 'application/json'},\n", + " {'rel': 'item',\n", + " 'href': 'some_granules/test_file09.nc.stac.json',\n", + " 'type': 'application/json'},\n", + " {'rel': 'item',\n", + " 'href': 'some_granules/test_file10.nc.stac.json',\n", + " 'type': 'application/json'}]}" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 4 }, { "metadata": {}, @@ -233,8 +285,8 @@ { "metadata": { "ExecuteTime": { - "end_time": "2025-02-10T19:51:39.435978Z", - "start_time": "2025-02-10T19:51:39.425621Z" + "end_time": "2025-02-11T19:58:56.118450Z", + "start_time": "2025-02-11T19:58:56.040011Z" } }, "cell_type": "code", @@ -384,7 +436,7 @@ ], "id": "cd2793837298e627", "outputs": [], - "execution_count": 14 + "execution_count": 5 }, { "metadata": {}, @@ -395,8 +447,8 @@ { "metadata": { "ExecuteTime": { - "end_time": "2025-02-10T21:26:41.690596Z", - "start_time": "2025-02-10T21:26:38.269917Z" + "end_time": "2025-02-11T19:59:02.826840Z", + "start_time": "2025-02-11T19:58:58.480377Z" } }, "cell_type": "code", @@ -427,18 +479,6 @@ " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file02/test_file02.nc.stac.json\"\n", "}\n", "{\n", - " \"granule_file\": \"normal-stage-out/some_granules/test_file03.nc.stac.json\",\n", - " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file03/test_file03.nc\"\n", - "}\n", - "{\n", - " \"granule_file\": \"normal-stage-out/some_granules/test_file03.nc.stac.json\",\n", - " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file03/test_file03.nc.cas\"\n", - "}\n", - "{\n", - " \"granule_file\": \"normal-stage-out/some_granules/test_file03.nc.stac.json\",\n", - " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file03/test_file03.nc.stac.json\"\n", - "}\n", - "{\n", " \"granule_file\": \"normal-stage-out/some_granules/test_file01.nc.stac.json\",\n", " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file01/test_file01.nc\"\n", "}\n", @@ -450,11 +490,23 @@ " \"granule_file\": \"normal-stage-out/some_granules/test_file01.nc.stac.json\",\n", " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file01/test_file01.nc.stac.json\"\n", "}\n", + "{\n", + " \"granule_file\": \"normal-stage-out/some_granules/test_file03.nc.stac.json\",\n", + " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file03/test_file03.nc\"\n", + "}\n", + "{\n", + " \"granule_file\": \"normal-stage-out/some_granules/test_file03.nc.stac.json\",\n", + " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file03/test_file03.nc.cas\"\n", + "}\n", + "{\n", + " \"granule_file\": \"normal-stage-out/some_granules/test_file03.nc.stac.json\",\n", + " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file03/test_file03.nc.stac.json\"\n", + "}\n", "{}\n" ] } ], - "execution_count": 25 + "execution_count": 6 }, { "metadata": {}, @@ -465,8 +517,8 @@ { "metadata": { "ExecuteTime": { - "end_time": "2025-02-10T21:29:21.314515Z", - "start_time": "2025-02-10T21:29:21.311438Z" + "end_time": "2025-02-11T19:59:11.252189Z", + "start_time": "2025-02-11T19:59:11.248397Z" } }, "cell_type": "code", @@ -481,13 +533,13 @@ ], "id": "cda0ee7d594d1513", "outputs": [], - "execution_count": 32 + "execution_count": 7 }, { "metadata": { "ExecuteTime": { - "end_time": "2025-02-10T21:29:27.704310Z", - "start_time": "2025-02-10T21:29:23.201698Z" + "end_time": "2025-02-11T19:59:16.813028Z", + "start_time": "2025-02-11T19:59:12.409849Z" } }, "cell_type": "code", @@ -507,15 +559,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-02-10 13:29:23,648 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=data, name=test_file01.nc, href=normal-stage-out/some_granules/./test_file01.nc\n", - "2025-02-10 13:29:23,648 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=data, name=test_file02.nc, href=normal-stage-out/some_granules/./test_file02.nc\n", - "2025-02-10 13:29:23,650 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=data, name=test_file03.nc, href=normal-stage-out/some_granules/./test_file03.nc\n", - "2025-02-10 13:29:24,041 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=metadata, name=test_file02.nc.cas, href=normal-stage-out/some_granules/./test_file02.nc.cas\n", - "2025-02-10 13:29:24,043 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=metadata, name=test_file03.nc.cas, href=normal-stage-out/some_granules/./test_file03.nc.cas\n", - "2025-02-10 13:29:24,053 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=metadata, name=test_file01.nc.cas, href=normal-stage-out/some_granules/./test_file01.nc.cas\n", - "2025-02-10 13:29:24,202 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=metadata, name=test_file02.nc.stac.json, href=normal-stage-out/some_granules/./test_file02.nc.stac.json\n", - "2025-02-10 13:29:24,207 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=metadata, name=test_file03.nc.stac.json, href=normal-stage-out/some_granules/./test_file03.nc.stac.json\n", - "2025-02-10 13:29:24,215 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=metadata, name=test_file01.nc.stac.json, href=normal-stage-out/some_granules/./test_file01.nc.stac.json\n" + "2025-02-11 11:59:12,790 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=data, name=test_file01.nc, href=normal-stage-out/some_granules/./test_file01.nc\n", + "2025-02-11 11:59:12,790 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=data, name=test_file02.nc, href=normal-stage-out/some_granules/./test_file02.nc\n", + "2025-02-11 11:59:12,790 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=data, name=test_file03.nc, href=normal-stage-out/some_granules/./test_file03.nc\n", + "2025-02-11 11:59:13,181 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=metadata, name=test_file01.nc.cas, href=normal-stage-out/some_granules/./test_file01.nc.cas\n", + "2025-02-11 11:59:13,186 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=metadata, name=test_file03.nc.cas, href=normal-stage-out/some_granules/./test_file03.nc.cas\n", + "2025-02-11 11:59:13,186 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=metadata, name=test_file02.nc.cas, href=normal-stage-out/some_granules/./test_file02.nc.cas\n", + "2025-02-11 11:59:13,328 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=metadata, name=test_file02.nc.stac.json, href=normal-stage-out/some_granules/./test_file02.nc.stac.json\n", + "2025-02-11 11:59:13,329 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=metadata, name=test_file01.nc.stac.json, href=normal-stage-out/some_granules/./test_file01.nc.stac.json\n", + "2025-02-11 11:59:13,346 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=metadata, name=test_file03.nc.stac.json, href=normal-stage-out/some_granules/./test_file03.nc.stac.json\n" ] }, { @@ -526,7 +578,7 @@ ] } ], - "execution_count": 33 + "execution_count": 8 }, { "metadata": {}, From 2dd836af00cd6d5c1d3098fded5de32d279a5b0a Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Tue, 11 Feb 2025 12:00:08 -0800 Subject: [PATCH 11/11] chore: remove old notebook --- examples/stage-in-stage-out-example.ipynb | 1348 --------------------- 1 file changed, 1348 deletions(-) delete mode 100644 examples/stage-in-stage-out-example.ipynb diff --git a/examples/stage-in-stage-out-example.ipynb b/examples/stage-in-stage-out-example.ipynb deleted file mode 100644 index eb67c17..0000000 --- a/examples/stage-in-stage-out-example.ipynb +++ /dev/null @@ -1,1348 +0,0 @@ -{ - "cells": [ - { - "metadata": {}, - "cell_type": "markdown", - "source": "## Stage In Stage Out Example", - "id": "30b6016bc9435c51" - }, - { - "metadata": {}, - "cell_type": "markdown", - "source": [ - "Steps\n", - "- Install Data Service (DS) Client Library\n", - "- Set Log Level\n", - "- Set Environment variables for Stage-In\n", - "- Try Stage-In\n", - "- Try Stage-Out (Dry-Run Only)\n", - "- Set Environment variables for Stage-In\n", - "- Try Stage-Out\n", - "- Try Stage-Out Ancillary Files\n", - "- Check the results" - ], - "id": "41307b0225e83dcc" - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-02-10T17:41:17.675227Z", - "start_time": "2025-02-10T17:41:16.781958Z" - } - }, - "cell_type": "code", - "source": [ - "from anyio.streams import file\n", - "%pip install mdps-ds-lib" - ], - "id": "fe80dd452ebd717b", - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: mdps-ds-lib in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (1.1.1)\r\n", - "Requirement already satisfied: boto3<2.0.0,>=1.26.51 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (1.36.15)\r\n", - "Requirement already satisfied: elasticsearch==7.13.4 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (7.13.4)\r\n", - "Requirement already satisfied: fastjsonschema<3.0.0,>=2.19.1 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (2.21.1)\r\n", - "Requirement already satisfied: jsonschema<5.0.0,>=4.23.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (4.23.0)\r\n", - "Requirement already satisfied: pystac==1.9.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (1.9.0)\r\n", - "Requirement already satisfied: requests==2.31.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (2.31.0)\r\n", - "Requirement already satisfied: requests-aws4auth==1.2.3 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (1.2.3)\r\n", - "Requirement already satisfied: tenacity==8.2.3 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (8.2.3)\r\n", - "Requirement already satisfied: xmltodict==0.13.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from mdps-ds-lib) (0.13.0)\r\n", - "Requirement already satisfied: urllib3<2,>=1.21.1 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from elasticsearch==7.13.4->mdps-ds-lib) (1.26.20)\r\n", - "Requirement already satisfied: certifi in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from elasticsearch==7.13.4->mdps-ds-lib) (2024.12.14)\r\n", - "Requirement already satisfied: python-dateutil>=2.7.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from pystac==1.9.0->mdps-ds-lib) (2.9.0.post0)\r\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from requests==2.31.0->mdps-ds-lib) (3.4.1)\r\n", - "Requirement already satisfied: idna<4,>=2.5 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from requests==2.31.0->mdps-ds-lib) (3.10)\r\n", - "Requirement already satisfied: six in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from requests-aws4auth==1.2.3->mdps-ds-lib) (1.17.0)\r\n", - "Requirement already satisfied: botocore<1.37.0,>=1.36.15 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from boto3<2.0.0,>=1.26.51->mdps-ds-lib) (1.36.15)\r\n", - "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from boto3<2.0.0,>=1.26.51->mdps-ds-lib) (1.0.1)\r\n", - "Requirement already satisfied: s3transfer<0.12.0,>=0.11.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from boto3<2.0.0,>=1.26.51->mdps-ds-lib) (0.11.2)\r\n", - "Requirement already satisfied: attrs>=22.2.0 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.23.0->mdps-ds-lib) (24.3.0)\r\n", - "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.23.0->mdps-ds-lib) (2024.10.1)\r\n", - "Requirement already satisfied: referencing>=0.28.4 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.23.0->mdps-ds-lib) (0.35.1)\r\n", - "Requirement already satisfied: rpds-py>=0.7.1 in /Users/wphyo/anaconda3/envs/cs.7650/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.23.0->mdps-ds-lib) (0.22.3)\r\n", - "Note: you may need to restart the kernel to use updated packages.\n" - ] - } - ], - "execution_count": 3 - }, - { - "metadata": {}, - "cell_type": "markdown", - "source": [ - "### Setting the Log Level\n", - "- Log level mappings:\n", - "- 10 = debug\n", - "- 20 = info\n", - "- 30 = warning\n", - "- 40 = error" - ], - "id": "e6bf69a910209c6c" - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-02-10T17:41:24.968457Z", - "start_time": "2025-02-10T17:41:24.965823Z" - } - }, - "cell_type": "code", - "source": [ - "import logging\n", - "log_level = 30\n", - "logging.basicConfig(level=log_level, format=\"%(asctime)s [%(levelname)s] [%(name)s::%(lineno)d] %(message)s\")" - ], - "id": "d2aaaee1a57cb095", - "outputs": [], - "execution_count": 4 - }, - { - "metadata": {}, - "cell_type": "markdown", - "source": [ - "#### Stage-in Only Environment Variables\n", - "\n", - "Follow this link for more information: https://app.gitbook.com/o/xZRqGQeQXJ0RP4VMj7Lq/s/UMIRhLdbRQTvMWop8Il9/developer-docs/data/docs/users-guide/stage-in\n", - "\n" - ], - "id": "8a87921572f1578d" - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-02-10T23:09:05.795762Z", - "start_time": "2025-02-10T23:09:05.790161Z" - } - }, - "cell_type": "code", - "source": [ - "import os\n", - "# The following environment variables are needed to \"stage-in\" from AWS S3\n", - "# They will also be needed if \"EDL\" settings and \"STAC_AUTH_TYPE\" settings are coming from parameter store\n", - "# Note that this may not be needed if JupyterNotebook can take care of the access.\n", - "os.environ['AWS_ACCESS_KEY_ID'] = 'xxx'\n", - "os.environ['AWS_SECRET_ACCESS_KEY'] = 'xxx'\n", - "os.environ['AWS_SESSION_TOKEN'] = 'xxx'\n", - "\n", - "# The following environment variables are ONLY needed if \"stage-in\" requires Earth data Login\n", - "os.environ['EDL_USERNAME'] = '/unity/uds/user/wphyo/edl_username' # Parameter Store Key for EarthData Login Username\n", - "os.environ['EDL_PASSWORD'] = '/unity/uds/user/wphyo/edl_dwssap' # Parameter Store Key for EarthData Login Password\n", - "os.environ['EDL_PASSWORD_TYPE'] = 'PARAM_STORE' # Can hardcode it to PARAM_STORE if that is used.\n", - "os.environ['EDL_BASE_URL'] = 'urs.earthdata.nasa.gov' # Earthdata Login URL to get Earthdata token to download files\n", - "\n", - "# The following environment variables are ONLY needed if \"stage-in\" requires STAC_AUTH_TYPE is \"UNITY\".\n", - "os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' # Look above links for all available options\n", - "os.environ['USERNAME'] = '/unity/uds/user/wphyo/username'\n", - "os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap'\n", - "os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c'\n", - "os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com'\n", - "\n", - "\n", - "# Others environment variables\n", - "os.environ['GRANULES_DOWNLOAD_TYPE'] = 'AMALGAMATION' # Download type to choose download class. AMALGAMATION, S3, DAAC, HTTP, and so on.\n", - "os.environ['DOWNLOADING_KEYS'] = 'data,metadata' # Which asset keys to download\n", - "os.environ['STAC_JSON'] = 'https://raw.githubusercontent.com/GodwinShen/emit-ghg/main/test/catalog.json' # URL to direct which granules + assets to download\n", - "# Other options\n", - "# os.environ['STAC_JSON'] = os.path.join(os.getcwd(), 'stage_in.json') # Alternatively, you can store the file locally, and point it as a path\n", - "os.environ['DOWNLOAD_DIR'] = os.path.join(os.getcwd(), 'downloaded_files') # Base directory where files will be downloaded\n", - "os.environ['OUTPUT_FILE'] = os.path.join(os.getcwd(), 'stage_in_result.json') # file path where the result is written locally for review\n" - ], - "id": "abb5f945876a1a63", - "outputs": [], - "execution_count": 43 - }, - { - "metadata": {}, - "cell_type": "markdown", - "source": [ - "- This is an example STAC_JSON file (or URL contents) used as a guide for stage-in.\n", - "- It is a STAC \"FeatureCollection\", a wrapper of an array of [Stac items](https://github.com/radiantearth/stac-spec/blob/master/item-spec/item-spec.md)" - ], - "id": "6c17a162ab168924" - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-02-10T17:41:44.530922Z", - "start_time": "2025-02-10T17:41:44.519332Z" - } - }, - "cell_type": "code", - "source": [ - "{\n", - " \"type\": \"FeatureCollection\",\n", - " \"features\": [\n", - " {\n", - " \"type\": \"Feature\",\n", - " \"stac_version\": \"1.0.0\",\n", - " \"id\": \"G2721220118-LPCLOUD\",\n", - " \"properties\": {\n", - " \"datetime\": \"2023-06-20T08:44:26Z\",\n", - " \"start_datetime\": \"2023-06-20T08:44:26.000Z\",\n", - " \"end_datetime\": \"2023-06-20T08:44:38.000Z\",\n", - " \"eo:cloud_cover\": 41\n", - " },\n", - " \"geometry\": {\n", - " \"type\": \"Polygon\",\n", - " \"coordinates\": [\n", - " [\n", - " [\n", - " 54.235199,\n", - " 39.2283897\n", - " ],\n", - " [\n", - " 53.7996902,\n", - " 38.5576591\n", - " ],\n", - " [\n", - " 54.6095352,\n", - " 38.031826\n", - " ],\n", - " [\n", - " 55.0450439,\n", - " 38.7025566\n", - " ],\n", - " [\n", - " 54.235199,\n", - " 39.2283897\n", - " ]\n", - " ]\n", - " ]\n", - " },\n", - " \"links\": [\n", - " {\n", - " \"rel\": \"self\",\n", - " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.stac\"\n", - " },\n", - " {\n", - " \"rel\": \"parent\",\n", - " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/C2408009906-LPCLOUD.stac\"\n", - " },\n", - " {\n", - " \"rel\": \"collection\",\n", - " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/C2408009906-LPCLOUD.stac\"\n", - " },\n", - " {\n", - " \"rel\": \"root\",\n", - " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/\"\n", - " },\n", - " {\n", - " \"rel\": \"via\",\n", - " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.json\"\n", - " },\n", - " {\n", - " \"rel\": \"via\",\n", - " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.umm_json\"\n", - " }\n", - " ],\n", - " \"assets\": {\n", - " \"metadata\": {\n", - " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.xml\",\n", - " \"type\": \"application/xml\"\n", - " },\n", - " \"browse\": {\n", - " \"href\": \"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-public/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_RAD_001_20230620T084426_2317106_011.png\",\n", - " \"type\": \"image/png\",\n", - " \"title\": \"Download EMIT_L1B_RAD_001_20230620T084426_2317106_011.png\"\n", - " },\n", - " \"opendap\": {\n", - " \"href\": \"https://opendap.earthdata.nasa.gov/collections/C2408009906-LPCLOUD/granules/EMIT_L1B_RAD_001_20230620T084426_2317106_011\",\n", - " \"title\": \"OPeNDAP request URL\"\n", - " },\n", - " \"data\": {\n", - " \"href\": \"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc\",\n", - " \"title\": \"Download EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc\"\n", - " },\n", - " \"data1\": {\n", - " \"href\": \"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc\",\n", - " \"title\": \"Download EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc\"\n", - " }\n", - " },\n", - " \"bbox\": [\n", - " 53.7996902,\n", - " 38.031826,\n", - " 55.0450439,\n", - " 39.2283897\n", - " ],\n", - " \"stac_extensions\": [\n", - " \"https://stac-extensions.github.io/eo/v1.0.0/schema.json\"\n", - " ],\n", - " \"collection\": \"C2408009906-LPCLOUD\"\n", - " },\n", - " {\n", - " \"type\": \"Feature\",\n", - " \"stac_version\": \"1.0.0\",\n", - " \"id\": \"G2721699381-LPCLOUD\",\n", - " \"properties\": {\n", - " \"datetime\": \"2023-06-20T08:44:26Z\",\n", - " \"start_datetime\": \"2023-06-20T08:44:26.000Z\",\n", - " \"end_datetime\": \"2023-06-20T08:44:38.000Z\",\n", - " \"eo:cloud_cover\": 41\n", - " },\n", - " \"geometry\": {\n", - " \"type\": \"Polygon\",\n", - " \"coordinates\": [\n", - " [\n", - " [\n", - " 54.235199,\n", - " 39.2283897\n", - " ],\n", - " [\n", - " 53.7996902,\n", - " 38.5576591\n", - " ],\n", - " [\n", - " 54.6095352,\n", - " 38.031826\n", - " ],\n", - " [\n", - " 55.0450439,\n", - " 38.7025566\n", - " ],\n", - " [\n", - " 54.235199,\n", - " 39.2283897\n", - " ]\n", - " ]\n", - " ]\n", - " },\n", - " \"links\": [\n", - " {\n", - " \"rel\": \"self\",\n", - " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721699381-LPCLOUD.stac\"\n", - " },\n", - " {\n", - " \"rel\": \"parent\",\n", - " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/C2408750690-LPCLOUD.stac\"\n", - " },\n", - " {\n", - " \"rel\": \"collection\",\n", - " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/C2408750690-LPCLOUD.stac\"\n", - " },\n", - " {\n", - " \"rel\": \"root\",\n", - " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/\"\n", - " },\n", - " {\n", - " \"rel\": \"via\",\n", - " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721699381-LPCLOUD.json\"\n", - " },\n", - " {\n", - " \"rel\": \"via\",\n", - " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721699381-LPCLOUD.umm_json\"\n", - " }\n", - " ],\n", - " \"assets\": {\n", - " \"metadata\": {\n", - " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721699381-LPCLOUD.xml\",\n", - " \"type\": \"application/xml\"\n", - " },\n", - " \"browse\": {\n", - " \"href\": \"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-public/EMITL2ARFL.001/EMIT_L2A_RFL_001_20230620T084426_2317106_011/EMIT_L2A_RFL_001_20230620T084426_2317106_011.png\",\n", - " \"type\": \"image/png\",\n", - " \"title\": \"Download EMIT_L2A_RFL_001_20230620T084426_2317106_011.png\"\n", - " },\n", - " \"opendap\": {\n", - " \"href\": \"https://opendap.earthdata.nasa.gov/collections/C2408750690-LPCLOUD/granules/EMIT_L2A_RFL_001_20230620T084426_2317106_011\",\n", - " \"title\": \"OPeNDAP request URL\"\n", - " },\n", - " \"data\": {\n", - " \"href\": \"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL2ARFL.001/EMIT_L2A_RFL_001_20230620T084426_2317106_011/EMIT_L2A_RFL_001_20230620T084426_2317106_011.nc\",\n", - " \"title\": \"Download EMIT_L2A_RFL_001_20230620T084426_2317106_011.nc\"\n", - " },\n", - " \"data1\": {\n", - " \"href\": \"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL2ARFL.001/EMIT_L2A_RFL_001_20230620T084426_2317106_011/EMIT_L2A_RFLUNCERT_001_20230620T084426_2317106_011.nc\",\n", - " \"title\": \"Download EMIT_L2A_RFLUNCERT_001_20230620T084426_2317106_011.nc\"\n", - " },\n", - " \"data2\": {\n", - " \"href\": \"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL2ARFL.001/EMIT_L2A_RFL_001_20230620T084426_2317106_011/EMIT_L2A_MASK_001_20230620T084426_2317106_011.nc\",\n", - " \"title\": \"Download EMIT_L2A_MASK_001_20230620T084426_2317106_011.nc\"\n", - " }\n", - " },\n", - " \"bbox\": [\n", - " 53.7996902,\n", - " 38.031826,\n", - " 55.0450439,\n", - " 39.2283897\n", - " ],\n", - " \"stac_extensions\": [\n", - " \"https://stac-extensions.github.io/eo/v1.0.0/schema.json\"\n", - " ],\n", - " \"collection\": \"C2408750690-LPCLOUD\"\n", - " }\n", - " ]\n", - "}" - ], - "id": "8a3c3ceaf481e2fe", - "outputs": [ - { - "data": { - "text/plain": [ - "{'type': 'FeatureCollection',\n", - " 'features': [{'type': 'Feature',\n", - " 'stac_version': '1.0.0',\n", - " 'id': 'G2721220118-LPCLOUD',\n", - " 'properties': {'datetime': '2023-06-20T08:44:26Z',\n", - " 'start_datetime': '2023-06-20T08:44:26.000Z',\n", - " 'end_datetime': '2023-06-20T08:44:38.000Z',\n", - " 'eo:cloud_cover': 41},\n", - " 'geometry': {'type': 'Polygon',\n", - " 'coordinates': [[[54.235199, 39.2283897],\n", - " [53.7996902, 38.5576591],\n", - " [54.6095352, 38.031826],\n", - " [55.0450439, 38.7025566],\n", - " [54.235199, 39.2283897]]]},\n", - " 'links': [{'rel': 'self',\n", - " 'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.stac'},\n", - " {'rel': 'parent',\n", - " 'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/C2408009906-LPCLOUD.stac'},\n", - " {'rel': 'collection',\n", - " 'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/C2408009906-LPCLOUD.stac'},\n", - " {'rel': 'root', 'href': 'https://cmr.earthdata.nasa.gov:443/search/'},\n", - " {'rel': 'via',\n", - " 'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.json'},\n", - " {'rel': 'via',\n", - " 'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.umm_json'}],\n", - " 'assets': {'metadata': {'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.xml',\n", - " 'type': 'application/xml'},\n", - " 'browse': {'href': 'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-public/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_RAD_001_20230620T084426_2317106_011.png',\n", - " 'type': 'image/png',\n", - " 'title': 'Download EMIT_L1B_RAD_001_20230620T084426_2317106_011.png'},\n", - " 'opendap': {'href': 'https://opendap.earthdata.nasa.gov/collections/C2408009906-LPCLOUD/granules/EMIT_L1B_RAD_001_20230620T084426_2317106_011',\n", - " 'title': 'OPeNDAP request URL'},\n", - " 'data': {'href': 'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc',\n", - " 'title': 'Download EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc'},\n", - " 'data1': {'href': 'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc',\n", - " 'title': 'Download EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc'}},\n", - " 'bbox': [53.7996902, 38.031826, 55.0450439, 39.2283897],\n", - " 'stac_extensions': ['https://stac-extensions.github.io/eo/v1.0.0/schema.json'],\n", - " 'collection': 'C2408009906-LPCLOUD'},\n", - " {'type': 'Feature',\n", - " 'stac_version': '1.0.0',\n", - " 'id': 'G2721699381-LPCLOUD',\n", - " 'properties': {'datetime': '2023-06-20T08:44:26Z',\n", - " 'start_datetime': '2023-06-20T08:44:26.000Z',\n", - " 'end_datetime': '2023-06-20T08:44:38.000Z',\n", - " 'eo:cloud_cover': 41},\n", - " 'geometry': {'type': 'Polygon',\n", - " 'coordinates': [[[54.235199, 39.2283897],\n", - " [53.7996902, 38.5576591],\n", - " [54.6095352, 38.031826],\n", - " [55.0450439, 38.7025566],\n", - " [54.235199, 39.2283897]]]},\n", - " 'links': [{'rel': 'self',\n", - " 'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/G2721699381-LPCLOUD.stac'},\n", - " {'rel': 'parent',\n", - " 'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/C2408750690-LPCLOUD.stac'},\n", - " {'rel': 'collection',\n", - " 'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/C2408750690-LPCLOUD.stac'},\n", - " {'rel': 'root', 'href': 'https://cmr.earthdata.nasa.gov:443/search/'},\n", - " {'rel': 'via',\n", - " 'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/G2721699381-LPCLOUD.json'},\n", - " {'rel': 'via',\n", - " 'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/G2721699381-LPCLOUD.umm_json'}],\n", - " 'assets': {'metadata': {'href': 'https://cmr.earthdata.nasa.gov:443/search/concepts/G2721699381-LPCLOUD.xml',\n", - " 'type': 'application/xml'},\n", - " 'browse': {'href': 'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-public/EMITL2ARFL.001/EMIT_L2A_RFL_001_20230620T084426_2317106_011/EMIT_L2A_RFL_001_20230620T084426_2317106_011.png',\n", - " 'type': 'image/png',\n", - " 'title': 'Download EMIT_L2A_RFL_001_20230620T084426_2317106_011.png'},\n", - " 'opendap': {'href': 'https://opendap.earthdata.nasa.gov/collections/C2408750690-LPCLOUD/granules/EMIT_L2A_RFL_001_20230620T084426_2317106_011',\n", - " 'title': 'OPeNDAP request URL'},\n", - " 'data': {'href': 'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL2ARFL.001/EMIT_L2A_RFL_001_20230620T084426_2317106_011/EMIT_L2A_RFL_001_20230620T084426_2317106_011.nc',\n", - " 'title': 'Download EMIT_L2A_RFL_001_20230620T084426_2317106_011.nc'},\n", - " 'data1': {'href': 'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL2ARFL.001/EMIT_L2A_RFL_001_20230620T084426_2317106_011/EMIT_L2A_RFLUNCERT_001_20230620T084426_2317106_011.nc',\n", - " 'title': 'Download EMIT_L2A_RFLUNCERT_001_20230620T084426_2317106_011.nc'},\n", - " 'data2': {'href': 'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL2ARFL.001/EMIT_L2A_RFL_001_20230620T084426_2317106_011/EMIT_L2A_MASK_001_20230620T084426_2317106_011.nc',\n", - " 'title': 'Download EMIT_L2A_MASK_001_20230620T084426_2317106_011.nc'}},\n", - " 'bbox': [53.7996902, 38.031826, 55.0450439, 39.2283897],\n", - " 'stac_extensions': ['https://stac-extensions.github.io/eo/v1.0.0/schema.json'],\n", - " 'collection': 'C2408750690-LPCLOUD'}]}" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "execution_count": 7 - }, - { - "metadata": {}, - "cell_type": "markdown", - "source": "#### How to create a Stac Item and a Feature Collection", - "id": "e0d932f5366ff630" - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-02-10T23:09:49.537304Z", - "start_time": "2025-02-10T23:09:49.530650Z" - } - }, - "cell_type": "code", - "source": [ - "import json\n", - "\n", - "from pystac import ItemCollection, Item, Asset\n", - "\n", - "from mdps_ds_lib.lib.utils.time_utils import TimeUtils\n", - "\n", - "test1 = ItemCollection(items=[\n", - " Item(id='G2721220118-LPCLOUD',\n", - " geometry={ # Set them if the algorithm knows what type of geometry is needed\n", - " \"type\": \"Point\",\n", - " \"coordinates\": [0.0, 0.0]\n", - " },\n", - " bbox=[53.7996902,\n", - " 38.031826,\n", - " 55.0450439,\n", - " 39.2283897], # Set them if the algorithm knows what type of geometry is needed\n", - " datetime=TimeUtils().parse_from_str('2023-06-20T08:44:26Z').get_datetime_obj(),\n", - " # Current metadata file creation datetime\n", - " properties={ # These 4 fields are mandatory\n", - " \"start_datetime\": \"2016-01-31T18:00:00.009057Z\",\n", - " \"end_datetime\": \"2016-01-31T19:59:59.991043Z\",\n", - " \"created\": \"2016-02-01T02:45:59.639000Z\",\n", - " \"updated\": \"2022-03-23T15:48:21.578000Z\",\n", - " },\n", - " href='https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.stac',\n", - " collection='NA', # No need to find out what collection it belongs to DS will take care of that.\n", - " assets={ # Point to all relevant files including itself\n", - " f'EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc': Asset(\n", - " 'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc',\n", - " title='EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc', roles=['data']),\n", - " f'EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc': Asset(\n", - " 'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc',\n", - " title='EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc', roles=['data']),\n", - " f'EMIT_L1B_RAD_001_20230620T084426_2317106_011.xml': Asset(\n", - " 'https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.xml',\n", - " title='EMIT_L1B_RAD_001_20230620T084426_2317106_011.xml', roles=['metadata']),\n", - " }),\n", - " Item(id='G2721220118-LPCLOUD',\n", - " geometry={ # Set them if the algorithm knows what type of geometry is needed\n", - " \"type\": \"Point\",\n", - " \"coordinates\": [0.0, 0.0]\n", - " },\n", - " bbox=[53.7996902,\n", - " 38.031826,\n", - " 55.0450439,\n", - " 39.2283897], # Set them if the algorithm knows what type of geometry is needed\n", - " datetime=TimeUtils().parse_from_str('2023-06-20T08:44:26Z').get_datetime_obj(),\n", - " # Current metadata file creation datetime\n", - " properties={ # These 4 fields are mandatory\n", - " \"start_datetime\": \"2016-01-31T18:00:00.009057Z\",\n", - " \"end_datetime\": \"2016-01-31T19:59:59.991043Z\",\n", - " \"created\": \"2016-02-01T02:45:59.639000Z\",\n", - " \"updated\": \"2022-03-23T15:48:21.578000Z\",\n", - " },\n", - " href='https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.stac',\n", - " collection='NA', # No need to find out what collection it belongs to DS will take care of that.\n", - " assets={ # Point to all relevant files including itself\n", - " f'EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc': Asset(\n", - " 'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc',\n", - " title='EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc', roles=['data']),\n", - " f'EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc': Asset(\n", - " 'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc',\n", - " title='EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc', roles=['data']),\n", - " f'EMIT_L1B_RAD_001_20230620T084426_2317106_011.xml': Asset(\n", - " 'https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.xml',\n", - " title='EMIT_L1B_RAD_001_20230620T084426_2317106_011.xml', roles=['metadata']),\n", - " })\n", - "])\n", - "\n", - "print(json.dumps(test1.to_dict(False), indent=4))\n" - ], - "id": "1429c01da795b287", - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"type\": \"FeatureCollection\",\n", - " \"features\": [\n", - " {\n", - " \"type\": \"Feature\",\n", - " \"stac_version\": \"1.0.0\",\n", - " \"id\": \"G2721220118-LPCLOUD\",\n", - " \"properties\": {\n", - " \"start_datetime\": \"2016-01-31T18:00:00.009057Z\",\n", - " \"end_datetime\": \"2016-01-31T19:59:59.991043Z\",\n", - " \"created\": \"2016-02-01T02:45:59.639000Z\",\n", - " \"updated\": \"2022-03-23T15:48:21.578000Z\",\n", - " \"datetime\": \"2023-06-20T08:44:26Z\"\n", - " },\n", - " \"geometry\": {\n", - " \"type\": \"Point\",\n", - " \"coordinates\": [\n", - " 0.0,\n", - " 0.0\n", - " ]\n", - " },\n", - " \"links\": [\n", - " {\n", - " \"rel\": \"self\",\n", - " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.stac\",\n", - " \"type\": \"application/json\"\n", - " }\n", - " ],\n", - " \"assets\": {\n", - " \"EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc\": {\n", - " \"href\": \"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc\",\n", - " \"title\": \"EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc\",\n", - " \"roles\": [\n", - " \"data\"\n", - " ]\n", - " },\n", - " \"EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc\": {\n", - " \"href\": \"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc\",\n", - " \"title\": \"EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc\",\n", - " \"roles\": [\n", - " \"data\"\n", - " ]\n", - " },\n", - " \"EMIT_L1B_RAD_001_20230620T084426_2317106_011.xml\": {\n", - " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.xml\",\n", - " \"title\": \"EMIT_L1B_RAD_001_20230620T084426_2317106_011.xml\",\n", - " \"roles\": [\n", - " \"metadata\"\n", - " ]\n", - " }\n", - " },\n", - " \"bbox\": [\n", - " 53.7996902,\n", - " 38.031826,\n", - " 55.0450439,\n", - " 39.2283897\n", - " ],\n", - " \"stac_extensions\": [],\n", - " \"collection\": \"NA\"\n", - " },\n", - " {\n", - " \"type\": \"Feature\",\n", - " \"stac_version\": \"1.0.0\",\n", - " \"id\": \"G2721220118-LPCLOUD\",\n", - " \"properties\": {\n", - " \"start_datetime\": \"2016-01-31T18:00:00.009057Z\",\n", - " \"end_datetime\": \"2016-01-31T19:59:59.991043Z\",\n", - " \"created\": \"2016-02-01T02:45:59.639000Z\",\n", - " \"updated\": \"2022-03-23T15:48:21.578000Z\",\n", - " \"datetime\": \"2023-06-20T08:44:26Z\"\n", - " },\n", - " \"geometry\": {\n", - " \"type\": \"Point\",\n", - " \"coordinates\": [\n", - " 0.0,\n", - " 0.0\n", - " ]\n", - " },\n", - " \"links\": [\n", - " {\n", - " \"rel\": \"self\",\n", - " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.stac\",\n", - " \"type\": \"application/json\"\n", - " }\n", - " ],\n", - " \"assets\": {\n", - " \"EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc\": {\n", - " \"href\": \"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc\",\n", - " \"title\": \"EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc\",\n", - " \"roles\": [\n", - " \"data\"\n", - " ]\n", - " },\n", - " \"EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc\": {\n", - " \"href\": \"https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc\",\n", - " \"title\": \"EMIT_L1B_OBS_001_20230620T084426_2317106_011.nc\",\n", - " \"roles\": [\n", - " \"data\"\n", - " ]\n", - " },\n", - " \"EMIT_L1B_RAD_001_20230620T084426_2317106_011.xml\": {\n", - " \"href\": \"https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.xml\",\n", - " \"title\": \"EMIT_L1B_RAD_001_20230620T084426_2317106_011.xml\",\n", - " \"roles\": [\n", - " \"metadata\"\n", - " ]\n", - " }\n", - " },\n", - " \"bbox\": [\n", - " 53.7996902,\n", - " 38.031826,\n", - " 55.0450439,\n", - " 39.2283897\n", - " ],\n", - " \"stac_extensions\": [],\n", - " \"collection\": \"NA\"\n", - " }\n", - " ]\n", - "}\n" - ] - } - ], - "execution_count": 44 - }, - { - "metadata": {}, - "cell_type": "markdown", - "source": [ - "#### Performing Stage-in\n", - "- Creating Stage-in Directory\n", - "- Calling Stage-in method to perform stage-in.\n", - "- Checking the result." - ], - "id": "f7756b2cea8a3f" - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-02-10T17:54:08.217050Z", - "start_time": "2025-02-10T17:48:17.006382Z" - } - }, - "cell_type": "code", - "source": [ - "from mdps_ds_lib.stage_in_out.download_granules_factory import DownloadGranulesFactory\n", - "from mdps_ds_lib.stage_in_out.stage_in_out_utils import StageInOutUtils\n", - "from mdps_ds_lib.lib.utils.file_utils import FileUtils\n", - "from glob import glob\n", - "\n", - "FileUtils.mk_dir_p(os.environ['DOWNLOAD_DIR']) # Creating a base directory if not created. They can be created manually w/o calling this.\n", - "\n", - "# Hardcoded method call. All params are set via environment previously\n", - "result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download()\n", - "StageInOutUtils.write_output_to_file(result_str)\n", - "print('done')\n", - "\n", - "print(list(glob(os.path.join(os.environ['DOWNLOAD_DIR'], '*')))) # Checking if files are downloaded." - ], - "id": "1353836ca421355e", - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-10 09:48:19,170 [ERROR] [mdps_ds_lib.stage_in_out.download_granules_amalgamation::32] downloading: https://cmr.earthdata.nasa.gov:443/search/concepts/G2721699381-LPCLOUD.xml\n", - "2025-02-10 09:48:19,170 [ERROR] [mdps_ds_lib.stage_in_out.download_granules_amalgamation::32] downloading: https://cmr.earthdata.nasa.gov:443/search/concepts/G2721220118-LPCLOUD.xml\n", - "2025-02-10 09:48:19,497 [ERROR] [mdps_ds_lib.stage_in_out.download_granules_amalgamation::32] downloading: https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL2ARFL.001/EMIT_L2A_RFL_001_20230620T084426_2317106_011/EMIT_L2A_RFL_001_20230620T084426_2317106_011.nc\n", - "2025-02-10 09:48:19,511 [ERROR] [mdps_ds_lib.stage_in_out.download_granules_amalgamation::32] downloading: https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.001/EMIT_L1B_RAD_001_20230620T084426_2317106_011/EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "done\n", - "['/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/error.log', '/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/G2721699381-LPCLOUD.xml', '/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/G2721220118-LPCLOUD.xml', '/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/G2721220118-LPCLOUD.stac.json', '/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/G2721699381-LPCLOUD.stac.json', '/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/catalog.json', '/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/EMIT_L1B_RAD_001_20230620T084426_2317106_011.nc', '/Users/wphyo/Projects/unity/uds_lib/examples/downloaded_files/EMIT_L2A_RFL_001_20230620T084426_2317106_011.nc']\n" - ] - } - ], - "execution_count": 11 - }, - { - "metadata": {}, - "cell_type": "markdown", - "source": [ - "### Stage-out Only Environment Variables\n", - "\n", - "Follow this link for more information: https://app.gitbook.com/o/xZRqGQeQXJ0RP4VMj7Lq/s/UMIRhLdbRQTvMWop8Il9/developer-docs/data/docs/users-guide/stage-out" - ], - "id": "44c5455db73d82c0" - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-02-10T21:25:22.300759Z", - "start_time": "2025-02-10T21:25:22.297419Z" - } - }, - "cell_type": "code", - "source": [ - "from mdps_ds_lib.lib.utils.file_utils import FileUtils\n", - "\n", - "# The following environment variables are needed to \"stage-in\" from AWS S3\n", - "# They will also be needed if \"EDL\" settings and \"STAC_AUTH_TYPE\" settings are coming from parameter store\n", - "# Note that this may not be needed if JupyterNotebook can take care of the access.\n", - "os.environ['AWS_ACCESS_KEY_ID'] = 'xxx'\n", - "os.environ['AWS_SECRET_ACCESS_KEY'] = 'xxx'\n", - "os.environ['AWS_SESSION_TOKEN'] = 'xxx'\n", - "\n", - "os.environ['PROJECT'] = 'DEMO'\n", - "os.environ['VENUE'] = 'DEV43'\n", - "\n", - "os.environ['VERIFY_SSL'] = 'FALSE'\n", - "os.environ['RESULT_PATH_PREFIX'] = '' # We can usually ignore this\n", - "os.environ['STAGING_BUCKET'] = 'uds-sbx-cumulus-staging'\n", - "os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY'\n", - "os.environ['OUTPUT_FILE'] = 'normal-stage-out/some_output/output.json'\n", - "os.environ['OUTPUT_DIRECTORY'] = 'normal-stage-out/output_dir'\n", - "FileUtils.mk_dir_p(os.environ.get('OUTPUT_DIRECTORY'))\n", - "os.environ['CATALOG_FILE'] = 'normal-stage-out/catalog.json'" - ], - "id": "518885efaf032c92", - "outputs": [], - "execution_count": 23 - }, - { - "metadata": {}, - "cell_type": "markdown", - "source": "#### Sample catalog.json used as a guide for stage-out", - "id": "c4db172f84e3e689" - }, - { - "metadata": {}, - "cell_type": "code", - "outputs": [], - "execution_count": null, - "source": [ - "{\n", - " \"type\": \"Catalog\",\n", - " \"id\": \"NA\",\n", - " \"stac_version\": \"1.0.0\",\n", - " \"description\": \"NA\",\n", - " \"links\": [\n", - " {\n", - " \"rel\": \"root\",\n", - " \"href\": \"/tmp/normal-stage-out/catalog.json\",\n", - " \"type\": \"application/json\"\n", - " },\n", - " {\n", - " \"rel\": \"item\",\n", - " \"href\": \"some_granules/test_file01.nc.stac.json\",\n", - " \"type\": \"application/json\"\n", - " },\n", - " {\n", - " \"rel\": \"item\",\n", - " \"href\": \"some_granules/test_file02.nc.stac.json\",\n", - " \"type\": \"application/json\"\n", - " },\n", - " {\n", - " \"rel\": \"item\",\n", - " \"href\": \"some_granules/test_file03.nc.stac.json\",\n", - " \"type\": \"application/json\"\n", - " },\n", - " {\n", - " \"rel\": \"item\",\n", - " \"href\": \"some_granules/test_file04.nc.stac.json\",\n", - " \"type\": \"application/json\"\n", - " },\n", - " {\n", - " \"rel\": \"item\",\n", - " \"href\": \"some_granules/test_file05.nc.stac.json\",\n", - " \"type\": \"application/json\"\n", - " },\n", - " {\n", - " \"rel\": \"item\",\n", - " \"href\": \"some_granules/test_file06.nc.stac.json\",\n", - " \"type\": \"application/json\"\n", - " },\n", - " {\n", - " \"rel\": \"item\",\n", - " \"href\": \"some_granules/test_file07.nc.stac.json\",\n", - " \"type\": \"application/json\"\n", - " },\n", - " {\n", - " \"rel\": \"item\",\n", - " \"href\": \"some_granules/test_file08.nc.stac.json\",\n", - " \"type\": \"application/json\"\n", - " },\n", - " {\n", - " \"rel\": \"item\",\n", - " \"href\": \"some_granules/test_file09.nc.stac.json\",\n", - " \"type\": \"application/json\"\n", - " },\n", - " {\n", - " \"rel\": \"item\",\n", - " \"href\": \"some_granules/test_file10.nc.stac.json\",\n", - " \"type\": \"application/json\"\n", - " }\n", - " ]\n", - "}" - ], - "id": "1b7097396bfbb1f3" - }, - { - "metadata": {}, - "cell_type": "markdown", - "source": [ - "- Creating Some Mock Files for stage-out including STAC metadata file and a catalog\n", - "- Note that this is usually not done here as the application will take care of it.\n", - "- How each stac metadata file is created can be used as an example though" - ], - "id": "f71cb9161be6325d" - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-02-10T19:51:39.435978Z", - "start_time": "2025-02-10T19:51:39.425621Z" - } - }, - "cell_type": "code", - "source": [ - "import json\n", - "from mdps_ds_lib.lib.utils.time_utils import TimeUtils\n", - "from pystac import Item, Asset, Catalog, Link\n", - "\n", - "granules_dir = os.path.join('normal-stage-out', 'some_granules')\n", - "FileUtils.mk_dir_p(granules_dir) # base directory\n", - "total_files = 3 # Uploading 3 mock granules\n", - "\n", - "catalog = Catalog( # Creating a STAC catalog (an example from previous cell)\n", - " id='NA', # we don't need to know the ID\n", - " description='NA')\n", - "catalog.set_self_href(os.environ['CATALOG_FILE'])\n", - "\n", - "for i in range(1, total_files+1):\n", - " filename = f'test_file{i:02d}'\n", - " with open(os.path.join(granules_dir, f'{filename}.nc'), 'w') as ff: # Creating Data File\n", - " ff.write('sample_file')\n", - " with open(os.path.join(granules_dir, f'{filename}.nc.cas'), 'w') as ff: # Creating native metadata file\n", - " ff.write('''\n", - " \n", - " \n", - " AggregateDir\n", - " snppatmsl1a\n", - " \n", - " \n", - " AutomaticQualityFlag\n", - " Passed\n", - " \n", - " \n", - " BuildId\n", - " v01.43.00\n", - " \n", - " \n", - " CollectionLabel\n", - " L1AMw_nominal2\n", - " \n", - " \n", - " DataGroup\n", - " sndr\n", - " \n", - " \n", - " EndDateTime\n", - " 2016-01-14T10:06:00.000Z\n", - " \n", - " \n", - " EndTAI93\n", - " 726919569.000\n", - " \n", - " \n", - " FileFormat\n", - " nc4\n", - " \n", - " \n", - " FileLocation\n", - " /pge/out\n", - " \n", - " \n", - " Filename\n", - " SNDR.SNPP.ATMS.L1A.nominal2.02.nc\n", - " \n", - " \n", - " GranuleNumber\n", - " 101\n", - " \n", - " \n", - " JobId\n", - " f163835c-9945-472f-bee2-2bc12673569f\n", - " \n", - " \n", - " ModelId\n", - " urn:npp:SnppAtmsL1a\n", - " \n", - " \n", - " NominalDate\n", - " 2016-01-14\n", - " \n", - " \n", - " ProductName\n", - " SNDR.SNPP.ATMS.20160114T1000.m06.g101.L1A.L1AMw_nominal2.v03_15_00.D.201214135000.nc\n", - " \n", - " \n", - " ProductType\n", - " SNDR_SNPP_ATMS_L1A\n", - " \n", - " \n", - " ProductionDateTime\n", - " 2020-12-14T13:50:00.000Z\n", - " \n", - " \n", - " ProductionLocation\n", - " Sounder SIPS: JPL/Caltech (Dev)\n", - " \n", - " \n", - " ProductionLocationCode\n", - " D\n", - " \n", - " \n", - " RequestId\n", - " 1215\n", - " \n", - " \n", - " StartDateTime\n", - " 2016-01-14T10:00:00.000Z\n", - " \n", - " \n", - " StartTAI93\n", - " 726919209.000\n", - " \n", - " \n", - " TaskId\n", - " 8c3ae101-8f7c-46c8-b5c6-63e7b6d3c8cd\n", - " \n", - " ''')\n", - " # Creating STAC metadata file object which is used by DS during stage-out and cataloging.\n", - " # pystac library is used to create a stac item file to have a standardized stac.\n", - " stac_item = Item(id=filename,\n", - " geometry={ # Set them if the algorithm knows what type of geometry is needed\n", - " \"type\": \"Point\",\n", - " \"coordinates\": [0.0, 0.0]\n", - " },\n", - " bbox=[-180, -90, 180, 90], # Set them if the algorithm knows what type of geometry is needed\n", - " datetime=TimeUtils().parse_from_unix(0, True).get_datetime_obj(), # Current metadata file creation datetime\n", - " properties={ # These 4 fields are mandatory\n", - " \"start_datetime\": \"2016-01-31T18:00:00.009057Z\",\n", - " \"end_datetime\": \"2016-01-31T19:59:59.991043Z\",\n", - " \"created\": \"2016-02-01T02:45:59.639000Z\",\n", - " \"updated\": \"2022-03-23T15:48:21.578000Z\",\n", - " },\n", - " href=os.path.join('some_granules', f'{filename}.nc.stac.json'),\n", - " collection='NA', # No need to find out what collection it belongs to DS will take care of that.\n", - " assets={ # Point to all relevant files including itself\n", - " f'{filename}.nc': Asset(os.path.join('.', f'{filename}.nc'), title='test_file01.nc', roles=['data']),\n", - " f'{filename}.nc.cas': Asset(os.path.join('.', f'{filename}.nc.cas'), title='test_file01.nc.cas', roles=['metadata']),\n", - " f'{filename}.nc.stac.json': Asset(os.path.join('.', f'{filename}.nc.stac.json'), title='test_file01.nc.stac.json', roles=['metadata']),\n", - " })\n", - " with open(os.path.join(granules_dir, f'{filename}.nc.stac.json'), 'w') as ff: # Creating STAC metadata file\n", - " ff.write(json.dumps(stac_item.to_dict(False, False)))\n", - " # Adding to the Catalog so that DS can find out the STAC file and finds out other related files from STAC\n", - " catalog.add_link(Link('item', os.path.join('some_granules', f'{filename}.nc.stac.json'), 'application/json'))\n", - "# Writing the main catalog file\n", - "with open(os.environ['CATALOG_FILE'], 'w') as ff:\n", - " ff.write(json.dumps(catalog.to_dict(False, False)))\n" - ], - "id": "cd2793837298e627", - "outputs": [], - "execution_count": 14 - }, - { - "metadata": {}, - "cell_type": "markdown", - "source": "### Performing Stage-out as a dry-run to verify if things are setup correctly", - "id": "7cd0f6aaaa07ef8e" - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-02-10T21:26:41.690596Z", - "start_time": "2025-02-10T21:26:38.269917Z" - } - }, - "cell_type": "code", - "source": [ - "os.environ['DRY_RUN'] = 'TRUE'\n", - "from mdps_ds_lib.stage_in_out.upoad_granules_factory import UploadGranulesFactory\n", - "\n", - "upload_result = UploadGranulesFactory().get_class(UploadGranulesFactory.UPLOAD_S3_BY_STAC_CATALOG).upload()\n", - "print(upload_result)" - ], - "id": "c46d384d19572817", - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Result of dry-run\n", - "{\n", - " \"granule_file\": \"normal-stage-out/some_granules/test_file02.nc.stac.json\",\n", - " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file02/test_file02.nc\"\n", - "}\n", - "{\n", - " \"granule_file\": \"normal-stage-out/some_granules/test_file02.nc.stac.json\",\n", - " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file02/test_file02.nc.cas\"\n", - "}\n", - "{\n", - " \"granule_file\": \"normal-stage-out/some_granules/test_file02.nc.stac.json\",\n", - " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file02/test_file02.nc.stac.json\"\n", - "}\n", - "{\n", - " \"granule_file\": \"normal-stage-out/some_granules/test_file03.nc.stac.json\",\n", - " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file03/test_file03.nc\"\n", - "}\n", - "{\n", - " \"granule_file\": \"normal-stage-out/some_granules/test_file03.nc.stac.json\",\n", - " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file03/test_file03.nc.cas\"\n", - "}\n", - "{\n", - " \"granule_file\": \"normal-stage-out/some_granules/test_file03.nc.stac.json\",\n", - " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file03/test_file03.nc.stac.json\"\n", - "}\n", - "{\n", - " \"granule_file\": \"normal-stage-out/some_granules/test_file01.nc.stac.json\",\n", - " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file01/test_file01.nc\"\n", - "}\n", - "{\n", - " \"granule_file\": \"normal-stage-out/some_granules/test_file01.nc.stac.json\",\n", - " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file01/test_file01.nc.cas\"\n", - "}\n", - "{\n", - " \"granule_file\": \"normal-stage-out/some_granules/test_file01.nc.stac.json\",\n", - " \"s3_url\": \"s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:DEMO:DEV43:NA___001/URN:NASA:UNITY:DEMO:DEV43:NA___001:test_file01/test_file01.nc.stac.json\"\n", - "}\n", - "{}\n" - ] - } - ], - "execution_count": 25 - }, - { - "metadata": {}, - "cell_type": "markdown", - "source": "### Performing Stage-out", - "id": "e87a8f50d52e90dc" - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-02-10T21:29:21.314515Z", - "start_time": "2025-02-10T21:29:21.311438Z" - } - }, - "cell_type": "code", - "source": [ - "# Cleaning old result files.\n", - "\n", - "old_result_files = glob(os.path.join(os.environ['OUTPUT_DIRECTORY'], '*'))\n", - "for each_file in old_result_files:\n", - " FileUtils.remove_if_exists(each_file)" - ], - "id": "cda0ee7d594d1513", - "outputs": [], - "execution_count": 32 - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-02-10T21:29:27.704310Z", - "start_time": "2025-02-10T21:29:23.201698Z" - } - }, - "cell_type": "code", - "source": [ - "\n", - "os.environ['DRY_RUN'] = 'FALSE'\n", - "\n", - "FileUtils.remove_if_exists(os.environ['OUTPUT_DIRECTORY'])\n", - "from mdps_ds_lib.stage_in_out.upoad_granules_factory import UploadGranulesFactory\n", - "\n", - "upload_result = UploadGranulesFactory().get_class(UploadGranulesFactory.UPLOAD_S3_BY_STAC_CATALOG).upload()\n", - "print(upload_result)" - ], - "id": "2e98b386bd09a655", - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-10 13:29:23,648 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=data, name=test_file01.nc, href=normal-stage-out/some_granules/./test_file01.nc\n", - "2025-02-10 13:29:23,648 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=data, name=test_file02.nc, href=normal-stage-out/some_granules/./test_file02.nc\n", - "2025-02-10 13:29:23,650 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=data, name=test_file03.nc, href=normal-stage-out/some_granules/./test_file03.nc\n", - "2025-02-10 13:29:24,041 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=metadata, name=test_file02.nc.cas, href=normal-stage-out/some_granules/./test_file02.nc.cas\n", - "2025-02-10 13:29:24,043 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=metadata, name=test_file03.nc.cas, href=normal-stage-out/some_granules/./test_file03.nc.cas\n", - "2025-02-10 13:29:24,053 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=metadata, name=test_file01.nc.cas, href=normal-stage-out/some_granules/./test_file01.nc.cas\n", - "2025-02-10 13:29:24,202 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=metadata, name=test_file02.nc.stac.json, href=normal-stage-out/some_granules/./test_file02.nc.stac.json\n", - "2025-02-10 13:29:24,207 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=metadata, name=test_file03.nc.stac.json, href=normal-stage-out/some_granules/./test_file03.nc.stac.json\n", - "2025-02-10 13:29:24,215 [AUDIT] [mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3::11] uploading type=metadata, name=test_file01.nc.stac.json, href=normal-stage-out/some_granules/./test_file01.nc.stac.json\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\"type\": \"Catalog\", \"id\": \"NA\", \"stac_version\": \"1.0.0\", \"description\": \"NA\", \"links\": [{\"rel\": \"root\", \"href\": \"/Users/wphyo/Projects/unity/uds_lib/examples/normal-stage-out/catalog.json\", \"type\": \"application/json\"}, {\"rel\": \"item\", \"href\": \"normal-stage-out/output_dir/successful_features.json\", \"type\": \"application/json\"}, {\"rel\": \"item\", \"href\": \"normal-stage-out/output_dir/failed_features.json\", \"type\": \"application/json\"}]}\n" - ] - } - ], - "execution_count": 33 - }, - { - "metadata": {}, - "cell_type": "markdown", - "source": "- Successful and failed items are written to 2 files into the directory set in `OUTPUT_DIRECTORY`", - "id": "61efcaa319b065b7" - }, - { - "metadata": {}, - "cell_type": "markdown", - "source": "### Ancillary Stage-out Environment Variables", - "id": "c111762146892c41" - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-02-10T23:11:49.193312Z", - "start_time": "2025-02-10T23:11:49.189141Z" - } - }, - "cell_type": "code", - "source": [ - "import os\n", - "\n", - "os.environ['AWS_ACCESS_KEY_ID'] = 'xxx'\n", - "os.environ['AWS_SECRET_ACCESS_KEY'] = 'xxx'\n", - "os.environ['AWS_SESSION_TOKEN'] = 'xxx'\n", - "\n", - "os.environ['GRANULES_UPLOAD_TYPE'] = 'UPLOAD_AUXILIARY_FILE_AS_GRANULE' # Setting uploading as auxiliary\n", - "os.environ['STAGING_BUCKET'] = 'uds-sbx-cumulus-staging' # S3 bucket where they will reside\n", - "os.environ['VERIFY_SSL'] = 'FALSE' # Optional param.\n", - "os.environ['BASE_DIRECTORY'] = 'auxiliary_files' # Base folder to upload\n", - "\n", - "os.environ['OUTPUT_DIRECTORY'] = 'auxiliary_files/output_dir' # the success / failure results to be stored locally for rewview\n", - "os.environ['OUTPUT_FILE'] = 'auxiliary_files/output_dir/stage_out_result.json' # file path where the result overview is written locally for\n", - "\n", - "tenant = 'UDS_DEMO'\n", - "tenant_venue = 'TEST'\n", - "collection_name = 'UDS_UNIT_COLLECTION'\n", - "collection_version = '24.09.10.11.00'.replace('.', '')\n", - "temp_collection_id = f'URN:NASA:UNITY:{tenant}:{tenant_venue}:{collection_name}___{collection_version}'\n", - "os.environ['COLLECTION_ID'] = temp_collection_id # Setting Collection ID.\n", - "# Note that Collection ID needs to follow the standard.\n", - "\n", - "FileUtils.mk_dir_p(os.environ.get('OUTPUT_DIRECTORY'))\n" - ], - "id": "29f786b4ed0929b5", - "outputs": [], - "execution_count": 45 - }, - { - "metadata": {}, - "cell_type": "markdown", - "source": "#### Mocking some ancillary files", - "id": "f40f81242b3930f7" - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-02-10T22:09:55.801596Z", - "start_time": "2025-02-10T22:09:55.795144Z" - } - }, - "cell_type": "code", - "source": [ - "granules_dir = os.path.join('auxiliary_files')\n", - "FileUtils.mk_dir_p(granules_dir) # base directory\n", - "total_files = 3 # Uploading 3 mock files\n", - "\n", - "for i in range(1, total_files+1):\n", - " filename = f'mocked_general_file{i:02d}'\n", - " with open(os.path.join(granules_dir, f'{filename}'), 'w') as ff: # Creating Data File\n", - " ff.write('mocked_general_file')" - ], - "id": "9b97566830b4afc", - "outputs": [], - "execution_count": 34 - }, - { - "metadata": {}, - "cell_type": "markdown", - "source": [ - "#### Performing actual stage-out\n", - "- Note that there is no dry-run on ancillary stage-out since there are no settings that client setup." - ], - "id": "d516f0a9fffbdc2a" - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-02-10T22:17:28.437985Z", - "start_time": "2025-02-10T22:17:28.433764Z" - } - }, - "cell_type": "code", - "source": [ - "# Cleaning old result files.\n", - "\n", - "old_result_files = glob(os.path.join(os.environ['OUTPUT_DIRECTORY'], '*'))\n", - "for each_file in old_result_files:\n", - " FileUtils.remove_if_exists(each_file)" - ], - "id": "c61692dd202c889c", - "outputs": [], - "execution_count": 40 - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-02-10T22:17:35.511806Z", - "start_time": "2025-02-10T22:17:31.251498Z" - } - }, - "cell_type": "code", - "source": [ - "\n", - "from mdps_ds_lib.stage_in_out.upoad_granules_factory import UploadGranulesFactory\n", - "from mdps_ds_lib.stage_in_out.stage_in_out_utils import StageInOutUtils\n", - "\n", - "# Hardcoded method call. All params are set via environment previously\n", - "upload_result_str = UploadGranulesFactory().get_class(os.getenv('GRANULES_UPLOAD_TYPE', UploadGranulesFactory.UPLOAD_S3_BY_STAC_CATALOG)).upload()\n", - "StageInOutUtils.write_output_to_file(upload_result_str)\n", - "print('done')" - ], - "id": "3aef5ceb43098a07", - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-10 14:17:31,715 [AUDIT] [mdps_ds_lib.stage_in_out.upload_arbitrary_files_as_granules::11] uploading auxiliary file: auxiliary_files/mocked_general_file01\n", - "2025-02-10 14:17:31,715 [AUDIT] [mdps_ds_lib.stage_in_out.upload_arbitrary_files_as_granules::11] uploading auxiliary file: auxiliary_files/mocked_general_file03\n", - "2025-02-10 14:17:31,714 [AUDIT] [mdps_ds_lib.stage_in_out.upload_arbitrary_files_as_granules::11] uploading auxiliary file: auxiliary_files/mocked_general_file02\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "done\n" - ] - } - ], - "execution_count": 41 - }, - { - "metadata": {}, - "cell_type": "code", - "outputs": [], - "execution_count": null, - "source": "", - "id": "2987dc7d3c9d0898" - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 2 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.6" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -}