diff --git a/CHANGELOG.md b/CHANGELOG.md index de635d3ff..d6c430477 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [10.12.1] + +### Fixed +- Doubled the memory requirement for `INSAR_ISCE_MULTI_BURST` jobs at 20x4 looks and >= 13 burst pairs. As a result, the credit cost for such jobs as also doubled from 5 to 10. Fixes https://github.com/ASFHyP3/hyp3/issues/2933 +- ScaleCluster now takes no action when desired vCPUs only marginally exceeds target vCPUs. Fixes https://github.com/ASFHyP3/hyp3/issues/2965 + ## [10.12.0] ### Added diff --git a/apps/scale-cluster/src/scale_cluster.py b/apps/scale-cluster/src/scale_cluster.py index d2e4eac76..bfb41e28b 100644 --- a/apps/scale-cluster/src/scale_cluster.py +++ b/apps/scale-cluster/src/scale_cluster.py @@ -26,8 +26,7 @@ def get_month_to_date_spending(today: date) -> float: def get_current_desired_vcpus(compute_environment_arn: str) -> int: response = BATCH.describe_compute_environments(computeEnvironments=[compute_environment_arn]) - compute_resources = response['computeEnvironments'][0]['computeResources'] - return min(compute_resources['desiredvCpus'], compute_resources['maxvCpus']) + return response['computeEnvironments'][0]['computeResources']['desiredvCpus'] def set_max_vcpus(compute_environment_arn: str, target_max_vcpus: int, current_desired_vcpus: int) -> None: @@ -39,7 +38,7 @@ def set_max_vcpus(compute_environment_arn: str, target_max_vcpus: int, current_d computeResources=compute_resources, state='ENABLED', ) - else: + elif current_desired_vcpus > target_max_vcpus * 1.05: print( f'Disabling {compute_environment_arn}. Current desiredvCpus {current_desired_vcpus} is larger than ' f'target maxvCpus {target_max_vcpus}' @@ -48,6 +47,11 @@ def set_max_vcpus(compute_environment_arn: str, target_max_vcpus: int, current_d computeEnvironment=compute_environment_arn, state='DISABLED', ) + else: + print( + f'Doing nothing with {compute_environment_arn}. Current desiredvCpus {current_desired_vcpus} only ' + f'marginally exceeds target maxvCpus {target_max_vcpus}' + ) def get_target_max_vcpus( diff --git a/apps/set-batch-overrides/src/set_batch_overrides.py b/apps/set-batch-overrides/src/set_batch_overrides.py index 717714668..9329536fb 100644 --- a/apps/set-batch-overrides/src/set_batch_overrides.py +++ b/apps/set-batch-overrides/src/set_batch_overrides.py @@ -66,7 +66,7 @@ def get_insar_isce_burst_memory(job_parameters: dict) -> str: if bursts < 31: return INSAR_ISCE_BURST_MEMORY_32G if looks == '20x4': - if bursts < 16: + if bursts < 13: return INSAR_ISCE_BURST_MEMORY_8G if bursts < 31: return INSAR_ISCE_BURST_MEMORY_16G diff --git a/job_spec/INSAR_ISCE_MULTI_BURST.yml b/job_spec/INSAR_ISCE_MULTI_BURST.yml index e3767ab28..e05e52e64 100644 --- a/job_spec/INSAR_ISCE_MULTI_BURST.yml +++ b/job_spec/INSAR_ISCE_MULTI_BURST.yml @@ -60,9 +60,9 @@ INSAR_ISCE_MULTI_BURST: 10: 5.0 11: 5.0 12: 5.0 - 13: 5.0 - 14: 5.0 - 15: 5.0 + 13: 10.0 + 14: 10.0 + 15: 10.0 10x2: 1: 1.0 2: 1.0 diff --git a/requirements-all.txt b/requirements-all.txt index d02812124..5f0592a40 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -5,15 +5,15 @@ -r requirements-apps-disable-private-dns.txt -r requirements-apps-search-archive.txt -r requirements-apps-update-db.txt -boto3==1.41.2 +boto3==1.42.0 jinja2==3.1.6 -moto[dynamodb]==5.1.17 +moto[dynamodb]==5.1.18 pytest==9.0.1 PyYAML==6.0.3 responses==0.25.8 -ruff==0.14.6 -mypy==1.18.2 +ruff==0.14.7 +mypy==1.19.0 setuptools==80.9.0 setuptools_scm==9.2.2 openapi-spec-validator==0.7.2 -cfn-lint==1.41.0 +cfn-lint==1.42.0 diff --git a/requirements-apps-disable-private-dns.txt b/requirements-apps-disable-private-dns.txt index b382190bd..1eadc932c 100644 --- a/requirements-apps-disable-private-dns.txt +++ b/requirements-apps-disable-private-dns.txt @@ -1 +1 @@ -boto3==1.41.2 +boto3==1.42.0 diff --git a/requirements-apps-get-files.txt b/requirements-apps-get-files.txt index cdd6df691..a7850cac9 100644 --- a/requirements-apps-get-files.txt +++ b/requirements-apps-get-files.txt @@ -1,2 +1,2 @@ -boto3==1.41.2 +boto3==1.42.0 ./lib/dynamo/ diff --git a/requirements-apps-start-execution.txt b/requirements-apps-start-execution.txt index d12e7df6a..20438025c 100644 --- a/requirements-apps-start-execution.txt +++ b/requirements-apps-start-execution.txt @@ -1,3 +1,3 @@ -boto3==1.41.2 +boto3==1.42.0 ./lib/dynamo/ ./lib/lambda_logging/ diff --git a/tests/test_scale_cluster.py b/tests/test_scale_cluster.py index bcf6d2a3c..fd91f5a3d 100644 --- a/tests/test_scale_cluster.py +++ b/tests/test_scale_cluster.py @@ -128,40 +128,28 @@ def test_get_current_desired_vcpus(batch_stubber): ) assert scale_cluster.get_current_desired_vcpus('foo') == 5 - expected_params = {'computeEnvironments': ['bar']} - service_response = { - 'computeEnvironments': [ - { - 'computeEnvironmentName': 'environment name', - 'computeEnvironmentArn': 'environment arn', - 'ecsClusterArn': 'cluster arn', - 'computeResources': { - 'type': 'MANAGED', - 'desiredvCpus': 8, - 'maxvCpus': 7, - 'subnets': ['subnet1', 'subnet2'], - }, - }, - ] - } - batch_stubber.add_response( - method='describe_compute_environments', expected_params=expected_params, service_response=service_response - ) - assert scale_cluster.get_current_desired_vcpus('bar') == 7 - def test_set_max_vcpus(batch_stubber): expected_params = {'computeEnvironment': 'foo', 'computeResources': {'maxvCpus': 10}, 'state': 'ENABLED'} batch_stubber.add_response( method='update_compute_environment', expected_params=expected_params, service_response={} ) - scale_cluster.set_max_vcpus(compute_environment_arn='foo', target_max_vcpus=10, current_desired_vcpus=10) + scale_cluster.set_max_vcpus(compute_environment_arn='foo', target_max_vcpus=10, current_desired_vcpus=0) + + expected_params = {'computeEnvironment': 'bar', 'computeResources': {'maxvCpus': 10}, 'state': 'ENABLED'} + batch_stubber.add_response( + method='update_compute_environment', expected_params=expected_params, service_response={} + ) + scale_cluster.set_max_vcpus(compute_environment_arn='bar', target_max_vcpus=10, current_desired_vcpus=10) expected_params = {'computeEnvironment': 'foo', 'state': 'DISABLED'} batch_stubber.add_response( method='update_compute_environment', expected_params=expected_params, service_response={} ) - scale_cluster.set_max_vcpus(compute_environment_arn='foo', target_max_vcpus=10, current_desired_vcpus=11) + scale_cluster.set_max_vcpus(compute_environment_arn='foo', target_max_vcpus=100, current_desired_vcpus=106) + + scale_cluster.set_max_vcpus(compute_environment_arn='foo', target_max_vcpus=100, current_desired_vcpus=101) + scale_cluster.set_max_vcpus(compute_environment_arn='foo', target_max_vcpus=100, current_desired_vcpus=105) def test_get_month_to_date_spending(cost_explorer_stubber): diff --git a/tests/test_set_batch_overrides.py b/tests/test_set_batch_overrides.py index 82fc5440e..3b950ffcc 100644 --- a/tests/test_set_batch_overrides.py +++ b/tests/test_set_batch_overrides.py @@ -126,7 +126,7 @@ def test_set_batch_overrides_insar_isce_burst_10x2(): def test_set_batch_overrides_insar_isce_burst_20x4(): - for n in range(1, 16): + for n in range(1, 13): assert lambda_handler(mock_insar_isce_burst_job('20x4', bursts=n), None) == { 'ResourceRequirements': [ { @@ -136,7 +136,7 @@ def test_set_batch_overrides_insar_isce_burst_20x4(): ], 'Environment': [{'Name': 'OMP_NUM_THREADS', 'Value': '1'}], } - for n in range(16, 31): + for n in range(13, 31): assert lambda_handler(mock_insar_isce_burst_job('20x4', bursts=n), None) == { 'ResourceRequirements': [ {