From a4609c3e398062ea28e117389730dd5fcb49c074 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Dec 2025 21:29:18 +0000 Subject: [PATCH 1/7] Bump the pip-deps group with 5 updates Bumps the pip-deps group with 5 updates: | Package | From | To | | --- | --- | --- | | [boto3](https://github.com/boto/boto3) | `1.41.2` | `1.42.0` | | [moto[dynamodb]](https://github.com/getmoto/moto) | `5.1.17` | `5.1.18` | | [ruff](https://github.com/astral-sh/ruff) | `0.14.6` | `0.14.7` | | [mypy](https://github.com/python/mypy) | `1.18.2` | `1.19.0` | | [cfn-lint](https://github.com/aws-cloudformation/cfn-lint) | `1.41.0` | `1.42.0` | Updates `boto3` from 1.41.2 to 1.42.0 - [Release notes](https://github.com/boto/boto3/releases) - [Commits](https://github.com/boto/boto3/compare/1.41.2...1.42.0) Updates `moto[dynamodb]` from 5.1.17 to 5.1.18 - [Release notes](https://github.com/getmoto/moto/releases) - [Changelog](https://github.com/getmoto/moto/blob/master/CHANGELOG.md) - [Commits](https://github.com/getmoto/moto/compare/5.1.17...5.1.18) Updates `ruff` from 0.14.6 to 0.14.7 - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/ruff/compare/0.14.6...0.14.7) Updates `mypy` from 1.18.2 to 1.19.0 - [Changelog](https://github.com/python/mypy/blob/master/CHANGELOG.md) - [Commits](https://github.com/python/mypy/compare/v1.18.2...v1.19.0) Updates `cfn-lint` from 1.41.0 to 1.42.0 - [Release notes](https://github.com/aws-cloudformation/cfn-lint/releases) - [Changelog](https://github.com/aws-cloudformation/cfn-lint/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws-cloudformation/cfn-lint/compare/v1.41.0...v1.42.0) --- updated-dependencies: - dependency-name: boto3 dependency-version: 1.42.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: pip-deps - dependency-name: moto[dynamodb] dependency-version: 5.1.18 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: pip-deps - dependency-name: ruff dependency-version: 0.14.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: pip-deps - dependency-name: mypy dependency-version: 1.19.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: pip-deps - dependency-name: cfn-lint dependency-version: 1.42.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: pip-deps ... Signed-off-by: dependabot[bot] --- requirements-all.txt | 10 +++++----- requirements-apps-disable-private-dns.txt | 2 +- requirements-apps-get-files.txt | 2 +- requirements-apps-start-execution.txt | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/requirements-all.txt b/requirements-all.txt index d02812124..5f0592a40 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -5,15 +5,15 @@ -r requirements-apps-disable-private-dns.txt -r requirements-apps-search-archive.txt -r requirements-apps-update-db.txt -boto3==1.41.2 +boto3==1.42.0 jinja2==3.1.6 -moto[dynamodb]==5.1.17 +moto[dynamodb]==5.1.18 pytest==9.0.1 PyYAML==6.0.3 responses==0.25.8 -ruff==0.14.6 -mypy==1.18.2 +ruff==0.14.7 +mypy==1.19.0 setuptools==80.9.0 setuptools_scm==9.2.2 openapi-spec-validator==0.7.2 -cfn-lint==1.41.0 +cfn-lint==1.42.0 diff --git a/requirements-apps-disable-private-dns.txt b/requirements-apps-disable-private-dns.txt index b382190bd..1eadc932c 100644 --- a/requirements-apps-disable-private-dns.txt +++ b/requirements-apps-disable-private-dns.txt @@ -1 +1 @@ -boto3==1.41.2 +boto3==1.42.0 diff --git a/requirements-apps-get-files.txt b/requirements-apps-get-files.txt index cdd6df691..a7850cac9 100644 --- a/requirements-apps-get-files.txt +++ b/requirements-apps-get-files.txt @@ -1,2 +1,2 @@ -boto3==1.41.2 +boto3==1.42.0 ./lib/dynamo/ diff --git a/requirements-apps-start-execution.txt b/requirements-apps-start-execution.txt index d12e7df6a..20438025c 100644 --- a/requirements-apps-start-execution.txt +++ b/requirements-apps-start-execution.txt @@ -1,3 +1,3 @@ -boto3==1.41.2 +boto3==1.42.0 ./lib/dynamo/ ./lib/lambda_logging/ From c24bccfea023de4a2cc97c5fe05eaaa647372b85 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Mon, 8 Dec 2025 14:53:27 -0900 Subject: [PATCH 2/7] Use 16G memory for >=13-burst 20x4 insar jobs --- apps/set-batch-overrides/src/set_batch_overrides.py | 2 +- tests/test_set_batch_overrides.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/set-batch-overrides/src/set_batch_overrides.py b/apps/set-batch-overrides/src/set_batch_overrides.py index 717714668..9329536fb 100644 --- a/apps/set-batch-overrides/src/set_batch_overrides.py +++ b/apps/set-batch-overrides/src/set_batch_overrides.py @@ -66,7 +66,7 @@ def get_insar_isce_burst_memory(job_parameters: dict) -> str: if bursts < 31: return INSAR_ISCE_BURST_MEMORY_32G if looks == '20x4': - if bursts < 16: + if bursts < 13: return INSAR_ISCE_BURST_MEMORY_8G if bursts < 31: return INSAR_ISCE_BURST_MEMORY_16G diff --git a/tests/test_set_batch_overrides.py b/tests/test_set_batch_overrides.py index 82fc5440e..3b950ffcc 100644 --- a/tests/test_set_batch_overrides.py +++ b/tests/test_set_batch_overrides.py @@ -126,7 +126,7 @@ def test_set_batch_overrides_insar_isce_burst_10x2(): def test_set_batch_overrides_insar_isce_burst_20x4(): - for n in range(1, 16): + for n in range(1, 13): assert lambda_handler(mock_insar_isce_burst_job('20x4', bursts=n), None) == { 'ResourceRequirements': [ { @@ -136,7 +136,7 @@ def test_set_batch_overrides_insar_isce_burst_20x4(): ], 'Environment': [{'Name': 'OMP_NUM_THREADS', 'Value': '1'}], } - for n in range(16, 31): + for n in range(13, 31): assert lambda_handler(mock_insar_isce_burst_job('20x4', bursts=n), None) == { 'ResourceRequirements': [ { From cd538dc80fc7b642e01add467c18f2ab1e81ce10 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Mon, 8 Dec 2025 15:40:26 -0900 Subject: [PATCH 3/7] double credit cost for 20x4 >=13-burst insar --- job_spec/INSAR_ISCE_MULTI_BURST.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/job_spec/INSAR_ISCE_MULTI_BURST.yml b/job_spec/INSAR_ISCE_MULTI_BURST.yml index e3767ab28..e05e52e64 100644 --- a/job_spec/INSAR_ISCE_MULTI_BURST.yml +++ b/job_spec/INSAR_ISCE_MULTI_BURST.yml @@ -60,9 +60,9 @@ INSAR_ISCE_MULTI_BURST: 10: 5.0 11: 5.0 12: 5.0 - 13: 5.0 - 14: 5.0 - 15: 5.0 + 13: 10.0 + 14: 10.0 + 15: 10.0 10x2: 1: 1.0 2: 1.0 From 3815d6c491976ab6eb2d938c7fe767876b73f8c8 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Mon, 8 Dec 2025 15:43:32 -0900 Subject: [PATCH 4/7] changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index de635d3ff..144429ca8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [10.12.1] + +### Fixed +- Doubled the memory requirement for `INSAR_ISCE_MULTI_BURST` jobs at 20x4 looks and >= 13 burst pairs. As a result, the credit cost for such jobs as also doubled from 5 to 10. Fixes https://github.com/ASFHyP3/hyp3/issues/2933 + ## [10.12.0] ### Added From 72dc3fb0f4fb0e89c3e6b1630fe716db6ed78ab9 Mon Sep 17 00:00:00 2001 From: Andrew Johnston Date: Tue, 9 Dec 2025 14:16:53 -0900 Subject: [PATCH 5/7] implement new fix for #2965 in scale_cluster.py --- CHANGELOG.md | 5 ++++ apps/scale-cluster/src/scale_cluster.py | 11 +++++--- tests/test_scale_cluster.py | 34 ++++++++----------------- 3 files changed, 23 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de635d3ff..e77b5b0a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [10.12.1] + +### Fixed +- ScaleCluster now takes no action when desired vCPUs only marginally exceeds taget vCPUs. Fixes https://github.com/ASFHyP3/hyp3/issues/2965 + ## [10.12.0] ### Added diff --git a/apps/scale-cluster/src/scale_cluster.py b/apps/scale-cluster/src/scale_cluster.py index d2e4eac76..60da33c09 100644 --- a/apps/scale-cluster/src/scale_cluster.py +++ b/apps/scale-cluster/src/scale_cluster.py @@ -5,7 +5,6 @@ import boto3 import dateutil.relativedelta - BATCH = boto3.client('batch') COST_EXPLORER = boto3.client('ce') @@ -26,8 +25,7 @@ def get_month_to_date_spending(today: date) -> float: def get_current_desired_vcpus(compute_environment_arn: str) -> int: response = BATCH.describe_compute_environments(computeEnvironments=[compute_environment_arn]) - compute_resources = response['computeEnvironments'][0]['computeResources'] - return min(compute_resources['desiredvCpus'], compute_resources['maxvCpus']) + return response['computeEnvironments'][0]['computeResources']['desiredvCpus'] def set_max_vcpus(compute_environment_arn: str, target_max_vcpus: int, current_desired_vcpus: int) -> None: @@ -39,7 +37,7 @@ def set_max_vcpus(compute_environment_arn: str, target_max_vcpus: int, current_d computeResources=compute_resources, state='ENABLED', ) - else: + elif current_desired_vcpus > target_max_vcpus * 1.05: print( f'Disabling {compute_environment_arn}. Current desiredvCpus {current_desired_vcpus} is larger than ' f'target maxvCpus {target_max_vcpus}' @@ -48,6 +46,11 @@ def set_max_vcpus(compute_environment_arn: str, target_max_vcpus: int, current_d computeEnvironment=compute_environment_arn, state='DISABLED', ) + else: + print( + f'Doing nothing with {compute_environment_arn}. Current desiredvCpus {current_desired_vcpus} only ' + f'marginally exceeds target maxvCpus {target_max_vcpus}' + ) def get_target_max_vcpus( diff --git a/tests/test_scale_cluster.py b/tests/test_scale_cluster.py index bcf6d2a3c..fd91f5a3d 100644 --- a/tests/test_scale_cluster.py +++ b/tests/test_scale_cluster.py @@ -128,40 +128,28 @@ def test_get_current_desired_vcpus(batch_stubber): ) assert scale_cluster.get_current_desired_vcpus('foo') == 5 - expected_params = {'computeEnvironments': ['bar']} - service_response = { - 'computeEnvironments': [ - { - 'computeEnvironmentName': 'environment name', - 'computeEnvironmentArn': 'environment arn', - 'ecsClusterArn': 'cluster arn', - 'computeResources': { - 'type': 'MANAGED', - 'desiredvCpus': 8, - 'maxvCpus': 7, - 'subnets': ['subnet1', 'subnet2'], - }, - }, - ] - } - batch_stubber.add_response( - method='describe_compute_environments', expected_params=expected_params, service_response=service_response - ) - assert scale_cluster.get_current_desired_vcpus('bar') == 7 - def test_set_max_vcpus(batch_stubber): expected_params = {'computeEnvironment': 'foo', 'computeResources': {'maxvCpus': 10}, 'state': 'ENABLED'} batch_stubber.add_response( method='update_compute_environment', expected_params=expected_params, service_response={} ) - scale_cluster.set_max_vcpus(compute_environment_arn='foo', target_max_vcpus=10, current_desired_vcpus=10) + scale_cluster.set_max_vcpus(compute_environment_arn='foo', target_max_vcpus=10, current_desired_vcpus=0) + + expected_params = {'computeEnvironment': 'bar', 'computeResources': {'maxvCpus': 10}, 'state': 'ENABLED'} + batch_stubber.add_response( + method='update_compute_environment', expected_params=expected_params, service_response={} + ) + scale_cluster.set_max_vcpus(compute_environment_arn='bar', target_max_vcpus=10, current_desired_vcpus=10) expected_params = {'computeEnvironment': 'foo', 'state': 'DISABLED'} batch_stubber.add_response( method='update_compute_environment', expected_params=expected_params, service_response={} ) - scale_cluster.set_max_vcpus(compute_environment_arn='foo', target_max_vcpus=10, current_desired_vcpus=11) + scale_cluster.set_max_vcpus(compute_environment_arn='foo', target_max_vcpus=100, current_desired_vcpus=106) + + scale_cluster.set_max_vcpus(compute_environment_arn='foo', target_max_vcpus=100, current_desired_vcpus=101) + scale_cluster.set_max_vcpus(compute_environment_arn='foo', target_max_vcpus=100, current_desired_vcpus=105) def test_get_month_to_date_spending(cost_explorer_stubber): From 13fab18c49ca1065f5a9aa50a5a52042951ea64f Mon Sep 17 00:00:00 2001 From: Andrew Johnston Date: Tue, 9 Dec 2025 14:21:50 -0900 Subject: [PATCH 6/7] revert erroneous whitespace change --- apps/scale-cluster/src/scale_cluster.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/scale-cluster/src/scale_cluster.py b/apps/scale-cluster/src/scale_cluster.py index 60da33c09..bfb41e28b 100644 --- a/apps/scale-cluster/src/scale_cluster.py +++ b/apps/scale-cluster/src/scale_cluster.py @@ -5,6 +5,7 @@ import boto3 import dateutil.relativedelta + BATCH = boto3.client('batch') COST_EXPLORER = boto3.client('ce') From 15f229ca334cc0a35aa77e9dbb52426330e5222b Mon Sep 17 00:00:00 2001 From: Andrew Johnston Date: Tue, 9 Dec 2025 14:51:07 -0900 Subject: [PATCH 7/7] fix typo in CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45b06a004..d6c430477 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Doubled the memory requirement for `INSAR_ISCE_MULTI_BURST` jobs at 20x4 looks and >= 13 burst pairs. As a result, the credit cost for such jobs as also doubled from 5 to 10. Fixes https://github.com/ASFHyP3/hyp3/issues/2933 -- ScaleCluster now takes no action when desired vCPUs only marginally exceeds taget vCPUs. Fixes https://github.com/ASFHyP3/hyp3/issues/2965 +- ScaleCluster now takes no action when desired vCPUs only marginally exceeds target vCPUs. Fixes https://github.com/ASFHyP3/hyp3/issues/2965 ## [10.12.0]