From fc5350af42d95c36675474eb43b7d61a10cee0f1 Mon Sep 17 00:00:00 2001 From: SpicyGarlicAlbacoreRoll Date: Mon, 19 May 2025 13:37:10 -0800 Subject: [PATCH 1/7] feat + logging: pin asf-search to 8.2.1, basic vertex dataset searches work --- CHANGELOG.md | 11 +++++++++++ cdk/cdk/cdk_stack.py | 26 +++++++++++++++++++++++++- requirements.txt | 2 +- src/SearchAPI/application/logger.py | 2 +- 4 files changed, 38 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9af499f..eed2775 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,17 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). --> ------ +## [1.0.1](https://github.com/asfadmin/Discovery-SearchAPI-v3/compare/v1.0.0...v1.0.1) + +### Added +- Add log group +- Add dedicated dev branch for test-staging deployment + - Intended Dev->Release workflow + - dev -> test -> prod-staging -> prod + +### Changed +- pin `asf-search` to v8.2.1, All basic Vertex dataset searches working + ## [1.0.0](https://github.com/asfadmin/Discovery-SearchAPI-v3/compare/v0.1.0...v1.0.0) ### Added diff --git a/cdk/cdk/cdk_stack.py b/cdk/cdk/cdk_stack.py index c0581cc..538687d 100644 --- a/cdk/cdk/cdk_stack.py +++ b/cdk/cdk/cdk_stack.py @@ -64,7 +64,6 @@ def __init__(self, scope: Construct, construct_id: str, staging: bool = False, * **lambda_vpc_kwargs, ) - staging api_id = f'SearchAPI-V3{"-Staging" if staging else ""}-RestAPI' api = apigateway.LambdaRestApi( self, @@ -74,6 +73,31 @@ def __init__(self, scope: Construct, construct_id: str, staging: bool = False, * default_cors_preflight_options=apigateway.CorsOptions( allow_origins=apigateway.Cors.ALL_ORIGINS, allow_methods=apigateway.Cors.ALL_METHODS ), + deploy_options=apigateway.StageOptions( + access_log_destination=apigateway.LogGroupLogDestination( + logs.LogGroup( + self, + f'{api_id}-LogGroup', + retention=logs.RetentionDays.THREE_MONTHS, + ) + ), # type: ignore + access_log_format=apigateway.AccessLogFormat.custom( + json.dumps( + { + 'sourceIp': '$context.identity.sourceIp', + 'httpMethod': '$context.httpMethod', + 'path': '$context.path', + 'status': '$context.status', + 'responseLength': '$context.responseLength', + 'responseLatency': '$context.responseLatency', + 'requestTime': '$context.requestTime', + 'protocol': '$context.protocol', + 'userAgent': '$context.identity.userAgent', + 'requestId': '$context.requestId', + } + ) + ), + ), **apigateway_kwargs, # endpoint_configuration=apigateway.EndpointConfiguration( # # https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_apigateway.EndpointConfiguration.html diff --git a/requirements.txt b/requirements.txt index 9c8c796..bbdbf5f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,7 @@ ujson==5.7.0 uvicorn==0.21.1 watchfiles==0.19.0 -asf_search==8.1.4 +asf_search==8.2.1 python-json-logger==2.0.7 pyshp diff --git a/src/SearchAPI/application/logger.py b/src/SearchAPI/application/logger.py index 94d885f..ba5f7d7 100644 --- a/src/SearchAPI/application/logger.py +++ b/src/SearchAPI/application/logger.py @@ -108,4 +108,4 @@ def get_logger(name: str, level: int=logging.DEBUG) -> logging.Logger: return logger -api_logger = get_logger(__name__, logging.DEBUG) +api_logger = get_logger(__name__, logging.INFO) From 4b204e0a35bd2e6b0647bfa70b4e6c1c41214d13 Mon Sep 17 00:00:00 2001 From: SpicyGarlicAlbacoreRoll Date: Mon, 19 May 2025 13:46:28 -0800 Subject: [PATCH 2/7] add separate test-staging workflow, remove test-staging integration tests from general pytest suite workflow --- .github/workflows/deploy-prod-staging.yml | 2 +- .github/workflows/deploy-test-staging.yml | 115 ++++++++++++++++++++++ .github/workflows/run-pytest.yml | 109 -------------------- 3 files changed, 116 insertions(+), 110 deletions(-) create mode 100644 .github/workflows/deploy-test-staging.yml diff --git a/.github/workflows/deploy-prod-staging.yml b/.github/workflows/deploy-prod-staging.yml index 41b888a..badaf0a 100644 --- a/.github/workflows/deploy-prod-staging.yml +++ b/.github/workflows/deploy-prod-staging.yml @@ -1,4 +1,4 @@ -name: SearchAPI Integration Deployment and Testing Suite +name: SearchAPI Prod-Staging Integration Deployment and Testing Suite permissions: contents: read on: diff --git a/.github/workflows/deploy-test-staging.yml b/.github/workflows/deploy-test-staging.yml new file mode 100644 index 0000000..912b4d8 --- /dev/null +++ b/.github/workflows/deploy-test-staging.yml @@ -0,0 +1,115 @@ +name: SearchAPI Test-Staging Integration Deployment and Testing Suite +permissions: + contents: read +on: + push: + branches: + - dev + +jobs: + # Deploy Staging + deploy-edc-test-staging: + runs-on: ubuntu-latest + environment: edc-test + permissions: + id-token: write + contents: read + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: build + uses: ./.github/workflows/search-api-composite + with: + aws-account-id: ${{ secrets.AWS_ACCOUNT_ID }} + vpc-id: ${{ secrets.VPC_ID }} + subnet-ids: ${{ secrets.SUBNET_IDS }} + security-group: ${{ secrets.SECURITY_GROUP }} + staging: true + + run-edc-test-staging-integration-tests: + needs: [deploy-edc-test-staging] + runs-on: ubuntu-latest + environment: edc-test + permissions: + id-token: write + contents: read + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Configure AWS OIDC credentials + uses: aws-actions/configure-aws-credentials@v3 + with: + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/SearchAPIActionsOIDCRole + aws-region: us-west-2 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Install dependencies + shell: bash + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + python -m pip install -r tests/requirements.txt + python -m pip install . + + - name: run tests + shell: bash + run: | + pytest tests/integration/test_stack.py \ + --reruns 3 --reruns-delay 10 + + deploy-test-staging: + runs-on: ubuntu-latest + environment: test + permissions: + id-token: write + contents: read + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: build + uses: ./.github/workflows/search-api-composite + with: + aws-account-id: ${{ secrets.AWS_ACCOUNT_ID }} + staging: true + + run-test-staging-integration-tests: + needs: [deploy-test-staging] + runs-on: ubuntu-latest + environment: test + permissions: + id-token: write + contents: read + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Configure AWS OIDC credentials + uses: aws-actions/configure-aws-credentials@v3 + with: + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/SearchAPIActionsOIDCRole + aws-region: us-west-2 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Install dependencies + shell: bash + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + python -m pip install -r tests/requirements.txt + python -m pip install . + + - name: run tests + shell: bash + run: | + pytest tests/integration/test_stack.py \ + --reruns 3 --reruns-delay 10 diff --git a/.github/workflows/run-pytest.yml b/.github/workflows/run-pytest.yml index 62a161e..953fab7 100644 --- a/.github/workflows/run-pytest.yml +++ b/.github/workflows/run-pytest.yml @@ -51,112 +51,3 @@ jobs: flags: unittests name: asf_admin searchAPI-v3 pytest verbose: true - - # Deploy Staging - deploy-edc-test-staging: - needs: [run-tests] - runs-on: ubuntu-latest - environment: edc-test - permissions: - id-token: write - contents: read - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: build - uses: ./.github/workflows/search-api-composite - with: - aws-account-id: ${{ secrets.AWS_ACCOUNT_ID }} - vpc-id: ${{ secrets.VPC_ID }} - subnet-ids: ${{ secrets.SUBNET_IDS }} - security-group: ${{ secrets.SECURITY_GROUP }} - staging: true - - run-tests-edc-staging: - needs: [deploy-edc-test-staging] - runs-on: ubuntu-latest - environment: edc-test - permissions: - id-token: write - contents: read - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Configure AWS OIDC credentials - uses: aws-actions/configure-aws-credentials@v3 - with: - role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/SearchAPIActionsOIDCRole - aws-region: us-west-2 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.11" - - - name: Install dependencies - shell: bash - run: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - python -m pip install -r tests/requirements.txt - python -m pip install . - - - name: run tests - shell: bash - run: | - pytest tests/integration/test_stack.py \ - --reruns 3 --reruns-delay 10 - - deploy-test-staging: - needs: [run-tests] - runs-on: ubuntu-latest - environment: test - permissions: - id-token: write - contents: read - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: build - uses: ./.github/workflows/search-api-composite - with: - aws-account-id: ${{ secrets.AWS_ACCOUNT_ID }} - staging: true - - run-tests-staging: - needs: [deploy-test-staging] - runs-on: ubuntu-latest - environment: test - permissions: - id-token: write - contents: read - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Configure AWS OIDC credentials - uses: aws-actions/configure-aws-credentials@v3 - with: - role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/SearchAPIActionsOIDCRole - aws-region: us-west-2 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.11" - - - name: Install dependencies - shell: bash - run: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - python -m pip install -r tests/requirements.txt - python -m pip install . - - - name: run tests - shell: bash - run: | - pytest tests/integration/test_stack.py \ - --reruns 3 --reruns-delay 10 From ff739002ce90cb313ca1df58b43f16984b8be35a Mon Sep 17 00:00:00 2001 From: SpicyGarlicAlbacoreRoll Date: Mon, 19 May 2025 13:53:46 -0800 Subject: [PATCH 3/7] bug: temp disable lambda logging group attachement --- cdk/cdk/cdk_stack.py | 50 ++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/cdk/cdk/cdk_stack.py b/cdk/cdk/cdk_stack.py index 538687d..9694fbd 100644 --- a/cdk/cdk/cdk_stack.py +++ b/cdk/cdk/cdk_stack.py @@ -73,31 +73,31 @@ def __init__(self, scope: Construct, construct_id: str, staging: bool = False, * default_cors_preflight_options=apigateway.CorsOptions( allow_origins=apigateway.Cors.ALL_ORIGINS, allow_methods=apigateway.Cors.ALL_METHODS ), - deploy_options=apigateway.StageOptions( - access_log_destination=apigateway.LogGroupLogDestination( - logs.LogGroup( - self, - f'{api_id}-LogGroup', - retention=logs.RetentionDays.THREE_MONTHS, - ) - ), # type: ignore - access_log_format=apigateway.AccessLogFormat.custom( - json.dumps( - { - 'sourceIp': '$context.identity.sourceIp', - 'httpMethod': '$context.httpMethod', - 'path': '$context.path', - 'status': '$context.status', - 'responseLength': '$context.responseLength', - 'responseLatency': '$context.responseLatency', - 'requestTime': '$context.requestTime', - 'protocol': '$context.protocol', - 'userAgent': '$context.identity.userAgent', - 'requestId': '$context.requestId', - } - ) - ), - ), + # deploy_options=apigateway.StageOptions( + # access_log_destination=apigateway.LogGroupLogDestination( + # logs.LogGroup( + # self, + # f'{api_id}-LogGroup', + # retention=logs.RetentionDays.THREE_MONTHS, + # ) + # ), # type: ignore + # access_log_format=apigateway.AccessLogFormat.custom( + # json.dumps( + # { + # 'sourceIp': '$context.identity.sourceIp', + # 'httpMethod': '$context.httpMethod', + # 'path': '$context.path', + # 'status': '$context.status', + # 'responseLength': '$context.responseLength', + # 'responseLatency': '$context.responseLatency', + # 'requestTime': '$context.requestTime', + # 'protocol': '$context.protocol', + # 'userAgent': '$context.identity.userAgent', + # 'requestId': '$context.requestId', + # } + # ) + # ), + # ), **apigateway_kwargs, # endpoint_configuration=apigateway.EndpointConfiguration( # # https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_apigateway.EndpointConfiguration.html From f1d17396edc47ef7523353224f9976f25841fe34 Mon Sep 17 00:00:00 2001 From: Tyler Chase Date: Tue, 20 May 2025 13:14:03 -0400 Subject: [PATCH 4/7] chore: update asf_search version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bbdbf5f..2796f05 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,7 @@ ujson==5.7.0 uvicorn==0.21.1 watchfiles==0.19.0 -asf_search==8.2.1 +asf_search==8.2.2 python-json-logger==2.0.7 pyshp From 5c28def9afd85e4a8c488fa9ef7fb4ee07ffedcb Mon Sep 17 00:00:00 2001 From: SpicyGarlicAlbacoreRoll Date: Tue, 20 May 2025 10:49:25 -0800 Subject: [PATCH 5/7] requirements: pin asf-search version 8.2.2 (opera-disp) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bbdbf5f..2796f05 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,7 @@ ujson==5.7.0 uvicorn==0.21.1 watchfiles==0.19.0 -asf_search==8.2.1 +asf_search==8.2.2 python-json-logger==2.0.7 pyshp From 43be2cd7d473a0d3c2eb2431b334c7a5d0ac4cea Mon Sep 17 00:00:00 2001 From: SpicyGarlicAlbacoreRoll Date: Wed, 21 May 2025 11:00:41 -0800 Subject: [PATCH 6/7] requirements: update asf-search to 8.2.3 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2796f05..8aacbdc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,7 @@ ujson==5.7.0 uvicorn==0.21.1 watchfiles==0.19.0 -asf_search==8.2.2 +asf_search==8.2.3 python-json-logger==2.0.7 pyshp From b0b8cc60cb33b88c297051083bf02effb20a2c0f Mon Sep 17 00:00:00 2001 From: Kim <33294735+SpicyGarlicAlbacoreRoll@users.noreply.github.com> Date: Wed, 21 May 2025 11:33:05 -0800 Subject: [PATCH 7/7] Update CHANGELOG.md Fix changelog --- CHANGELOG.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eed2775..98a95c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,13 +29,12 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [1.0.1](https://github.com/asfadmin/Discovery-SearchAPI-v3/compare/v1.0.0...v1.0.1) ### Added -- Add log group - Add dedicated dev branch for test-staging deployment - Intended Dev->Release workflow - dev -> test -> prod-staging -> prod ### Changed -- pin `asf-search` to v8.2.1, All basic Vertex dataset searches working +- pin `asf-search` to v8.2.3, All basic Vertex dataset searches working ## [1.0.0](https://github.com/asfadmin/Discovery-SearchAPI-v3/compare/v0.1.0...v1.0.0)