From b00bf3bdd9913d24722655a833b532f271b07f36 Mon Sep 17 00:00:00 2001 From: SpicyGarlicAlbacoreRoll Date: Tue, 27 May 2025 17:07:09 -0800 Subject: [PATCH 01/13] add multipart form data and octet-stream as accepted binary media formats --- cdk/cdk/cdk_stack.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cdk/cdk/cdk_stack.py b/cdk/cdk/cdk_stack.py index 9694fbd..4493961 100644 --- a/cdk/cdk/cdk_stack.py +++ b/cdk/cdk/cdk_stack.py @@ -70,6 +70,7 @@ def __init__(self, scope: Construct, construct_id: str, staging: bool = False, * id=api_id, handler=search_api_lambda, proxy=True, + binary_media_types=['multipart/form-data', 'application/octet-stream'], default_cors_preflight_options=apigateway.CorsOptions( allow_origins=apigateway.Cors.ALL_ORIGINS, allow_methods=apigateway.Cors.ALL_METHODS ), From 4cd437729eb6840303c955e2668436851fd167c5 Mon Sep 17 00:00:00 2001 From: SpicyGarlicAlbacoreRoll Date: Tue, 27 May 2025 17:14:17 -0800 Subject: [PATCH 02/13] add file name to error --- src/SearchAPI/application/application.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/SearchAPI/application/application.py b/src/SearchAPI/application/application.py index 9744e6c..98fa7bf 100644 --- a/src/SearchAPI/application/application.py +++ b/src/SearchAPI/application/application.py @@ -220,6 +220,7 @@ async def file_to_wkt(files: list[UploadFile]): data = FilesToWKT([file.file for file in files]).getWKT() + data['error'] = [file.file.filename for file in files] return JSONResponse(content={ ** data, ** validate_wkt(data["parsed wkt"])}, From 5b2b8775bf970cad2084136d7721fc946161f22c Mon Sep 17 00:00:00 2001 From: SpicyGarlicAlbacoreRoll Date: Wed, 28 May 2025 15:49:52 -0800 Subject: [PATCH 03/13] deserialize aws context object in log routing preflight/teardown --- src/SearchAPI/application/log_router.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/SearchAPI/application/log_router.py b/src/SearchAPI/application/log_router.py index e232b66..ad4c8cc 100644 --- a/src/SearchAPI/application/log_router.py +++ b/src/SearchAPI/application/log_router.py @@ -6,7 +6,7 @@ from fastapi.routing import APIRoute from .logger import api_logger - +import json class LoggingRoute(APIRoute): """ @@ -39,9 +39,11 @@ def get_route_handler(self) -> Callable: async def custom_route_handler(request: Request) -> Response: # Grab the AWS UUID and set it for every log: - context = request.scope.get("aws.context") - if context is not None: - self.aws_request_id = context.aws_request_id + + if context := request.headers.get('x-amzn-request-context'): + context_object = json.loads(context) + self.aws_request_id = context_object.get('aws_request_id') + logging.setLogRecordFactory(self.record_factory) # Time the request itself: before = time.time() From 2d819a377e01f469196f58d3fa365532118abcb2 Mon Sep 17 00:00:00 2001 From: SpicyGarlicAlbacoreRoll Date: Wed, 28 May 2025 15:58:16 -0800 Subject: [PATCH 04/13] grab the correct json key for requestID --- src/SearchAPI/application/log_router.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SearchAPI/application/log_router.py b/src/SearchAPI/application/log_router.py index ad4c8cc..ef44f8e 100644 --- a/src/SearchAPI/application/log_router.py +++ b/src/SearchAPI/application/log_router.py @@ -42,7 +42,7 @@ async def custom_route_handler(request: Request) -> Response: if context := request.headers.get('x-amzn-request-context'): context_object = json.loads(context) - self.aws_request_id = context_object.get('aws_request_id') + self.aws_request_id = context_object.get('requestId') logging.setLogRecordFactory(self.record_factory) # Time the request itself: From dd3d9e5cf24bc458fd34f7b1240ec8d07e21b61f Mon Sep 17 00:00:00 2001 From: SpicyGarlicAlbacoreRoll Date: Wed, 28 May 2025 16:25:22 -0800 Subject: [PATCH 05/13] feat: include queryBody in logging. Remove extra logging in application.py --- src/SearchAPI/application/application.py | 14 -------------- src/SearchAPI/application/log_router.py | 1 + 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/src/SearchAPI/application/application.py b/src/SearchAPI/application/application.py index 98fa7bf..b783a10 100644 --- a/src/SearchAPI/application/application.py +++ b/src/SearchAPI/application/application.py @@ -54,9 +54,7 @@ async def query_params(searchOptions: SearchOptsModel = Depends(process_search_r raise HTTPException(detail=repr(exc), status_code=400) from exc if output.lower() == 'count': - start = time.perf_counter() count=asf.search_count(opts=opts) - api_logger.info(f'/services/search/param count query time {time.perf_counter()-start}') return Response( content=str(count), status_code=200, @@ -65,10 +63,8 @@ async def query_params(searchOptions: SearchOptsModel = Depends(process_search_r ) if output.lower() == 'python': - start = time.perf_counter() file_name, search_script = get_asf_search_script(opts) - api_logger.info(f'/services/search/param count query time {time.perf_counter()-start}') return Response( content=search_script, status_code=200, @@ -79,9 +75,7 @@ async def query_params(searchOptions: SearchOptsModel = Depends(process_search_r } ) try: - start = time.perf_counter() results = asf.search(opts=opts) - api_logger.info(f'/services/search/param query time {time.perf_counter()-start}') response_info = as_output(results, output) return Response(**response_info) @@ -102,10 +96,8 @@ async def query_baseline(searchOptions: BaselineSearchOptsModel = Depends(proces # Load the reference scene: if output.lower() == 'python': - start = time.perf_counter() file_name, search_script = get_asf_search_script(opts, reference=reference, search_endpoint='baseline') - api_logger.info(f'/services/search/param count query time {time.perf_counter()-start}') return Response( content=search_script, status_code=200, @@ -116,9 +108,7 @@ async def query_baseline(searchOptions: BaselineSearchOptsModel = Depends(proces } ) try: - start = time.perf_counter() reference_product = asf.granule_search(granule_list=[reference], opts=opts)[0] - api_logger.info(f'/services/search/baseline reference query time {time.perf_counter()-start}') except (KeyError, IndexError, ValueError) as exc: raise HTTPException(detail=f"Reference scene not found: {reference}", status_code=400) from exc @@ -148,9 +138,7 @@ async def query_baseline(searchOptions: BaselineSearchOptsModel = Depends(proces # Figure out the response params: if output.lower() == 'count': stack_opts = reference_product.get_stack_opts() - start = time.perf_counter() count = asf.search_count(opts=stack_opts) - api_logger.info(f'/services/search/baseline count stack query time {time.perf_counter()-start}') return Response( content=str(count), @@ -161,9 +149,7 @@ async def query_baseline(searchOptions: BaselineSearchOptsModel = Depends(proces # Finally stream everything back: try: - start = time.perf_counter() stack = reference_product.stack(opts=opts) - api_logger.info(f'/services/search/baseline stack query time {time.perf_counter()-start}') response_info = as_output(stack, output) return Response(**response_info) diff --git a/src/SearchAPI/application/log_router.py b/src/SearchAPI/application/log_router.py index ef44f8e..9124a6d 100644 --- a/src/SearchAPI/application/log_router.py +++ b/src/SearchAPI/application/log_router.py @@ -57,6 +57,7 @@ async def custom_route_handler(request: Request) -> Response: extra={ "QueryTime": duration, "QueryParams": dict(request.query_params), + "QueryBody": dict(await request.json()), "Endpoint": request.scope['path'], } ) From 4da42a3e6a100360c1fffba9b45f9f7d1019c5cf Mon Sep 17 00:00:00 2001 From: SpicyGarlicAlbacoreRoll Date: Thu, 29 May 2025 14:55:17 -0800 Subject: [PATCH 06/13] bump asf-search version to 8.3.3 --- CHANGELOG.md | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d29f68..3f81b83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,7 +41,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - dev -> test -> prod-staging -> prod ### Changed -- pin `asf-search` to v8.3.1, All basic Vertex dataset searches working +- pin `asf-search` to v8.3.3, All basic Vertex dataset searches working ## [1.0.0](https://github.com/asfadmin/Discovery-SearchAPI-v3/compare/v0.1.0...v1.0.0) diff --git a/requirements.txt b/requirements.txt index 578dba9..f2032f6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,7 @@ ujson==5.7.0 uvicorn==0.21.1 watchfiles==0.19.0 -asf_search==8.3.1 +asf_search==8.3.3 python-json-logger==2.0.7 pyshp==2.1.3 From 2755c8e7857f8fd8291c6e8f652da660f288597b Mon Sep 17 00:00:00 2001 From: SpicyGarlicAlbacoreRoll Date: Thu, 29 May 2025 15:16:10 -0800 Subject: [PATCH 07/13] test: file upload integration tests --- CHANGELOG.md | 2 ++ cdk/cdk/cdk_stack.py | 3 ++- tests/integration/test_stack.py | 24 ++++++++++++++++++++---- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f81b83..7ae399e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,11 +34,13 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Changed - Include wkt in error when raising in `validate_wkt()` +- Specify which binary file types are allowed to be passed to lambda ### Added - Add dedicated dev branch for test-staging deployment - Intended Dev->Release workflow - dev -> test -> prod-staging -> prod +- Added more files to integration testing endpoint ### Changed - pin `asf-search` to v8.3.3, All basic Vertex dataset searches working diff --git a/cdk/cdk/cdk_stack.py b/cdk/cdk/cdk_stack.py index 4493961..7010160 100644 --- a/cdk/cdk/cdk_stack.py +++ b/cdk/cdk/cdk_stack.py @@ -59,7 +59,8 @@ def __init__(self, scope: Construct, construct_id: str, staging: bool = False, * timeout=Duration.seconds(30), memory_size=5308, code=lambda_.DockerImageCode.from_image_asset( - directory='..' + directory='..', + build_args={'MATURITY': } ), **lambda_vpc_kwargs, ) diff --git a/tests/integration/test_stack.py b/tests/integration/test_stack.py index 7ac5807..1efcc05 100644 --- a/tests/integration/test_stack.py +++ b/tests/integration/test_stack.py @@ -9,7 +9,10 @@ session = asf.ASFSession() cwd = os.getcwd() -test_file_path= os.path.join(cwd, 'tests/integration/', 'elvey.geojson') +geojson_test_file_path= os.path.join(cwd, 'tests/integration/', 'elvey.geojson') +kml_test_file_path = os.path.join(cwd, 'tests/yml_tests/Resources/kmls_valid/', '3D_coords.kml') +shp_test_file_path = os.path.join(cwd, 'tests/yml_tests/Resources/shps_valid/', 'NED1_F.shp') +zip_test_file_path = os.path.join(cwd, 'tests/yml_tests/Resources/zips_valid/', 'NED1_F.zip') basic_search_params = { 'maxResults': 250, @@ -102,9 +105,22 @@ def test_wkt_endpoint_post_json(): assert response.status_code == 200, f'Non-200 status code from baseline POST endpoint (data): \nstatus code: {response.status_code}\nresponse: {response.text}' ### WKT FILE UPLOAD TEST -def test_wkt_file_upload_endpoint(): - files = {'files': open(test_file_path,'rb')} +def test_wkt_file_upload_endpoint_geojson(): + _wkt_file_upload_endpoint(geojson_test_file_path) + +def test_wkt_file_upload_endpoint_kml(): + _wkt_file_upload_endpoint(kml_test_file_path) + +def test_wkt_file_upload_endpoint_shp(): + _wkt_file_upload_endpoint(shp_test_file_path) + +def test_wkt_file_upload_endpoint_zip(): + _wkt_file_upload_endpoint(zip_test_file_path) + + +def _wkt_file_upload_endpoint(file: str): + files = {'files': open(file,'rb')} response = session.post(files_wkt_endpoint, files=files) response.raise_for_status() - assert response.status_code == 200, f'Non-200 status code from baseline POST endpoint (data): \nstatus code: {response.status_code}\nresponse: {response.text}' + assert response.status_code == 200, f'Non-200 status code from files_to_wkt endpoint for file {file.split("/")[-1]}: \nstatus code: {response.status_code}\nresponse: {response.text}' From ade4fe93cf24e2745d6693f32ca612d88c74305b Mon Sep 17 00:00:00 2001 From: SpicyGarlicAlbacoreRoll Date: Thu, 29 May 2025 15:20:09 -0800 Subject: [PATCH 08/13] remove erroneous build_args from cdk_stack definition --- cdk/cdk/cdk_stack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdk/cdk/cdk_stack.py b/cdk/cdk/cdk_stack.py index 7010160..bd46d7a 100644 --- a/cdk/cdk/cdk_stack.py +++ b/cdk/cdk/cdk_stack.py @@ -60,7 +60,7 @@ def __init__(self, scope: Construct, construct_id: str, staging: bool = False, * memory_size=5308, code=lambda_.DockerImageCode.from_image_asset( directory='..', - build_args={'MATURITY': } + # build_args={'MATURITY': } ), **lambda_vpc_kwargs, ) From 88f664b2f14a79c7a0061c6fc52aee21b250c255 Mon Sep 17 00:00:00 2001 From: SpicyGarlicAlbacoreRoll Date: Thu, 29 May 2025 16:08:05 -0800 Subject: [PATCH 09/13] fix: disable binary_media_types flag --- cdk/cdk/cdk_stack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdk/cdk/cdk_stack.py b/cdk/cdk/cdk_stack.py index bd46d7a..b183d2b 100644 --- a/cdk/cdk/cdk_stack.py +++ b/cdk/cdk/cdk_stack.py @@ -71,7 +71,7 @@ def __init__(self, scope: Construct, construct_id: str, staging: bool = False, * id=api_id, handler=search_api_lambda, proxy=True, - binary_media_types=['multipart/form-data', 'application/octet-stream'], + # binary_media_types=['multipart/form-data', 'application/octet-stream'], default_cors_preflight_options=apigateway.CorsOptions( allow_origins=apigateway.Cors.ALL_ORIGINS, allow_methods=apigateway.Cors.ALL_METHODS ), From 819db5fb8044e3bded00ddf53af6030414c80785 Mon Sep 17 00:00:00 2001 From: SpicyGarlicAlbacoreRoll Date: Thu, 29 May 2025 16:17:30 -0800 Subject: [PATCH 10/13] re-arrange log_router --- src/SearchAPI/application/log_router.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/SearchAPI/application/log_router.py b/src/SearchAPI/application/log_router.py index 9124a6d..e069eb7 100644 --- a/src/SearchAPI/application/log_router.py +++ b/src/SearchAPI/application/log_router.py @@ -50,6 +50,10 @@ async def custom_route_handler(request: Request) -> Response: try: response: Response = await original_route_handler(request) finally: + queryBody = {} + if (content_type := request.headers.get('content-type')) is not None: + if content_type == 'application/json': + queryBody = await request.json() # What to ALWAYS log: duration = time.time() - before api_logger.info( @@ -57,7 +61,7 @@ async def custom_route_handler(request: Request) -> Response: extra={ "QueryTime": duration, "QueryParams": dict(request.query_params), - "QueryBody": dict(await request.json()), + "QueryBody": queryBody, "Endpoint": request.scope['path'], } ) From a67c52862c7a992ab34578695210f0e195483e1c Mon Sep 17 00:00:00 2001 From: SpicyGarlicAlbacoreRoll Date: Thu, 29 May 2025 16:26:08 -0800 Subject: [PATCH 11/13] re-add binary_media_types list --- cdk/cdk/cdk_stack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdk/cdk/cdk_stack.py b/cdk/cdk/cdk_stack.py index b183d2b..bd46d7a 100644 --- a/cdk/cdk/cdk_stack.py +++ b/cdk/cdk/cdk_stack.py @@ -71,7 +71,7 @@ def __init__(self, scope: Construct, construct_id: str, staging: bool = False, * id=api_id, handler=search_api_lambda, proxy=True, - # binary_media_types=['multipart/form-data', 'application/octet-stream'], + binary_media_types=['multipart/form-data', 'application/octet-stream'], default_cors_preflight_options=apigateway.CorsOptions( allow_origins=apigateway.Cors.ALL_ORIGINS, allow_methods=apigateway.Cors.ALL_METHODS ), From 50483fc292a075a1294df71fe5be4e8f25a2c7d9 Mon Sep 17 00:00:00 2001 From: SpicyGarlicAlbacoreRoll Date: Thu, 29 May 2025 16:37:28 -0800 Subject: [PATCH 12/13] chore: remove debug error from files_to_wkt --- src/SearchAPI/application/application.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/SearchAPI/application/application.py b/src/SearchAPI/application/application.py index b783a10..c6ec258 100644 --- a/src/SearchAPI/application/application.py +++ b/src/SearchAPI/application/application.py @@ -206,7 +206,6 @@ async def file_to_wkt(files: list[UploadFile]): data = FilesToWKT([file.file for file in files]).getWKT() - data['error'] = [file.file.filename for file in files] return JSONResponse(content={ ** data, ** validate_wkt(data["parsed wkt"])}, From 0a47f0cb9335bab12628a039c80570779dd17669 Mon Sep 17 00:00:00 2001 From: SpicyGarlicAlbacoreRoll Date: Thu, 29 May 2025 16:38:29 -0800 Subject: [PATCH 13/13] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ae399e..f8894b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Intended Dev->Release workflow - dev -> test -> prod-staging -> prod - Added more files to integration testing endpoint +- Add remaining file upload support for .zip and .shp files. All previous file formats now supported ### Changed - pin `asf-search` to v8.3.3, All basic Vertex dataset searches working