Scenario Failure: peakvalley #400
Description
Benchmark scenario ID: peakvalley
Benchmark scenario definition: https://github.com/ESA-APEx/apex_algorithms/blob/ef4ce84a7f3490d0850430d4355c0d40eaf5fe72/algorithm_catalog/vito/peakvalley/benchmark_scenarios/peakvalley.json
openEO backend: openeofed.dataspace.copernicus.eu
GitHub Actions workflow run: https://github.com/ESA-APEx/apex_algorithms/actions/runs/22985007240
Workflow artifacts: https://github.com/ESA-APEx/apex_algorithms/actions/runs/22985007240#artifacts
Test start: 2026-03-12 03:18:16.151007+00:00
Test duration: 0:07:16.986697
Test outcome: ❌ failed
Last successful test phase: download-reference
Failure in test phase: compare:derived_from-change
Contact Information
| Name | Organization | Contact |
|---|---|---|
| Bram Janssen | VITO | Contact via VITO (VITO Website, GitHub) |
Process Graph
{
"peakvalley1": {
"arguments": {
"drop_threshold": 0.15,
"recovery_ratio": 1,
"slope_threshold": 0.007,
"spatial_extent": {
"crs": 4326,
"east": 15.185336903822831,
"north": 45.81302555710934,
"south": 45.80924633589998,
"west": 15.179421073198585
},
"temporal_extent": [
"2021-01-01",
"2021-12-31"
]
},
"namespace": "https://raw.githubusercontent.com/VITObelgium/openeo_algorithm_catalog/refs/heads/main/peakvalley/openeo_udp/peakvalley.json",
"process_id": "peakvalley"
},
"saveresult1": {
"arguments": {
"data": {
"from_node": "peakvalley1"
},
"format": "NetCDF",
"options": {}
},
"process_id": "save_result",
"result": true
}
}Error Logs
scenario = BenchmarkScenario(id='peakvalley', description='Detects peaks and valleys in a time series by analysing amplitude chan...me/runner/work/apex_algorithms/apex_algorithms/algorithm_catalog/vito/peakvalley/benchmark_scenarios/peakvalley.json'))
connection_factory = <function connection_factory.<locals>.get_connection at 0x7fe8371a5d00>
tmp_path = PosixPath('/home/runner/work/apex_algorithms/apex_algorithms/qa/benchmarks/tmp_path_root/test_run_benchmark_peakvalley_0')
track_metric = <function track_metric.<locals>.track at 0x7fe8371a5e40>
track_phase = <function track_phase.<locals>.track at 0x7fe8371a5f80>
upload_assets_on_fail = <function upload_assets_on_fail.<locals>.collect at 0x7fe8371a6020>
request = <FixtureRequest for <Function test_run_benchmark[peakvalley]>>
@pytest.mark.parametrize(
"scenario",
[
# Use scenario id as parameterization id to give nicer test names.
pytest.param(uc, id=uc.id)
for uc in get_benchmark_scenarios()
],
)
def test_run_benchmark(
scenario: BenchmarkScenario,
connection_factory,
tmp_path: Path,
track_metric,
track_phase,
upload_assets_on_fail,
request,
):
track_metric("scenario_id", scenario.id)
with track_phase(phase="connect"):
# Check if a backend override has been provided via cli options.
override_backend = request.config.getoption("--override-backend")
backend_filter = request.config.getoption("--backend-filter")
if backend_filter and not re.match(backend_filter, scenario.backend):
# TODO apply filter during scenario retrieval, but seems to be hard to retrieve cli param
pytest.skip(
f"skipping scenario {scenario.id} because backend {scenario.backend} does not match filter {backend_filter!r}"
)
backend = scenario.backend
if override_backend:
_log.info(f"Overriding backend URL with {override_backend!r}")
backend = override_backend
connection: openeo.Connection = connection_factory(url=backend)
report_path = None
with track_phase(phase="create-job"):
# TODO #14 scenario option to use synchronous instead of batch job mode?
job = connection.create_job(
process_graph=scenario.process_graph,
title=f"APEx benchmark {scenario.id}",
additional=scenario.job_options,
)
track_metric("job_id", job.job_id)
if request.config.getoption("--upload-benchmark-report"):
report_path = tmp_path / "benchmark_report.json"
report_path.write_text(json.dumps({
"job_id": job.job_id,
"scenario_id": scenario.id,
"scenario_description": scenario.description,
"scenario_backend": scenario.backend,
"scenario_source": str(scenario.source) if scenario.source else None,
"reference_data": scenario.reference_data,
"reference_options": scenario.reference_options,
}, indent=2))
upload_assets_on_fail(report_path)
with track_phase(phase="run-job"):
# TODO: monitor timing and progress
# TODO: separate "job started" and run phases?
max_minutes = request.config.getoption("--maximum-job-time-in-minutes")
if max_minutes:
def _timeout_handler(signum, frame):
raise TimeoutError(
f"Batch job {job.job_id} exceeded maximum allowed time of {max_minutes} minutes"
)
old_handler = signal.signal(signal.SIGALRM, _timeout_handler)
signal.alarm(max_minutes * 60)
try:
job.start_and_wait()
finally:
if max_minutes:
signal.alarm(0)
signal.signal(signal.SIGALRM, old_handler)
with track_phase(phase="collect-metadata"):
collect_metrics_from_job_metadata(job, track_metric=track_metric)
results = job.get_results()
collect_metrics_from_results_metadata(results, track_metric=track_metric)
with track_phase(phase="download-actual"):
# Download actual results
actual_dir = tmp_path / "actual"
paths = results.download_files(target=actual_dir, include_stac_metadata=True)
# Upload assets on failure
upload_assets_on_fail(*paths)
with track_phase(phase="download-reference"):
reference_dir = download_reference_data(
scenario=scenario, reference_dir=tmp_path / "reference"
)
if report_path is not None:
report = json.loads(report_path.read_text())
report["actual_files"] = {
str(p.relative_to(actual_dir)): f"{p.stat().st_size / 1024:.1f} kb"
for p in sorted(actual_dir.rglob("*")) if p.is_file()
}
ref_files = {}
for p in sorted(reference_dir.rglob("*")):
if not p.is_file():
continue
rel = p.relative_to(reference_dir)
size_str = f"{p.stat().st_size / 1024:.1f} kb"
actual_counterpart = actual_dir / rel
if not actual_counterpart.exists():
size_str += " (missing in actual)"
elif actual_counterpart.stat().st_size != p.stat().st_size:
size_str += f" (actual: {actual_counterpart.stat().st_size / 1024:.1f} kb)"
ref_files[str(rel)] = size_str
report["reference_files"] = ref_files
report_path.write_text(json.dumps(report, indent=2))
with track_phase(
phase="compare", describe_exception=analyse_results_comparison_exception
):
# Compare actual results with reference data
> assert_job_results_allclose(
actual=actual_dir,
expected=reference_dir,
tmp_path=tmp_path,
rtol=scenario.reference_options.get("rtol", 1e-3),
atol=scenario.reference_options.get("atol", 1),
pixel_tolerance=scenario.reference_options.get("pixel_tolerance", 1),
)
tests/test_benchmarks.py:146:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
actual = PosixPath('/home/runner/work/apex_algorithms/apex_algorithms/qa/benchmarks/tmp_path_root/test_run_benchmark_peakvalley_0/actual')
expected = PosixPath('/home/runner/work/apex_algorithms/apex_algorithms/qa/benchmarks/tmp_path_root/test_run_benchmark_peakvalley_0/reference')
def assert_job_results_allclose(
actual: Union[BatchJob, JobResults, str, Path],
expected: Union[BatchJob, JobResults, str, Path],
*,
rtol: float = _DEFAULT_RTOL,
atol: float = _DEFAULT_ATOL,
pixel_tolerance: float = _DEFAULT_PIXELTOL,
tmp_path: Optional[Path] = None,
):
"""
Assert that two job results sets are equal (with tolerance).
:param actual: actual job results, provided as :py:class:`~openeo.rest.job.BatchJob` object,
:py:meth:`~openeo.rest.job.JobResults` object or path to directory with downloaded assets.
:param expected: expected job results, provided as :py:class:`~openeo.rest.job.BatchJob` object,
:py:meth:`~openeo.rest.job.JobResults` object or path to directory with downloaded assets.
:param rtol: relative tolerance
:param atol: absolute tolerance
:param pixel_tolerance: maximum fraction of pixels (in percent)
that is allowed to be significantly different (considering ``atol`` and ``rtol``)
:param tmp_path: root temp path to download results if needed.
It's recommended to pass pytest's `tmp_path` fixture here
:raises AssertionError: if not equal within the given tolerance
.. versionadded:: 0.31.0
.. warning::
This function is experimental and subject to change.
"""
issues = _compare_job_results(
actual, expected, rtol=rtol, atol=atol, pixel_tolerance=pixel_tolerance, tmp_path=tmp_path
)
if issues:
> raise AssertionError("\n".join(issues))
E AssertionError: Issues for metadata file 'job-results.json':
E Differing 'derived_from' links (0 common, 145 only in actual, 145 only in expected):
E only in actual: {'S2A_MSIL2A_20210907T100031_N0500_R122_T33TWL_20230119T110128', 'S2A_MSIL2A_20210110T100401_N0500_R122_T33TWL_20230603T062203', 'S2A_MSIL2A_20210420T100021_N0500_R122_T33TWL_20230523T130212', 'S2A_MSIL2A_20210507T095031_N0500_R079_T33TWL_20230303T004903', 'S2A_MSIL2A_20210716T095031_N0500_R079_T33TWL_20230213T175646', 'S2A_MSIL2A_20210117T095341_N0500_R079_T33TWL_20230528T081930', 'S2A_MSIL2A_20211004T095031_N0500_R079_T33TWL_20230111T081656', 'S2A_MSIL2A_20211024T095101_N0500_R079_T33TWL_20230102T174907', 'S2B_MSIL2A_20210512T095029_N0500_R079_T33TWL_20230331T214427', 'S2A_MSIL2A_20211017T100031_N0500_R122_T33TWL_20230115T185426', 'S2A_MSIL2A_20211206T100411_N0500_R122_T33TWL_20221229T224450', 'S2A_MSIL2A_20210301T100031_N0500_R122_T33TWL_20230525T121257', 'S2B_MSIL2A_20210714T100029_N0500_R122_T33TWL_20230224T182455', 'S2A_MSIL2A_20210609T100031_N0500_R122_T33TWL_20230131T152530', 'S2A_MSIL2A_20210226T095031_N0500_R079_T33TWL_20230603T080427', 'S2B_MSIL2A_20210803T100029_N0500_R1...
E only in expected: {'/eodata/Sentinel-2/MSI/L2A_N0500/2021/01/10/S2A_MSIL2A_20210110T100401_N0500_R122_T33TWL_20230603T062203.SAFE', '/eodata/Sentinel-2/MSI/L2A_N0500/2021/04/30/S2A_MSIL2A_20210430T100021_N0500_R122_T33TWL_20230215T005505.SAFE', '/eodata/Sentinel-2/MSI/L2A_N0500/2021/12/13/S2A_MSIL2A_20211213T095411_N0500_R079_T33TWL_20221225T055452.SAFE', '/eodata/Sentinel-2/MSI/L2A_N0500/2021/08/15/S2A_MSIL2A_20210815T095031_N0500_R079_T33TWL_20230213T093926.SAFE', '/eodata/Sentinel-2/MSI/L2A_N0500/2021/04/02/S2B_MSIL2A_20210402T095029_N0500_R079_T33TWL_20230510T203809.SAFE', '/eodata/Sentinel-2/MSI/L2A_N0500/2021/03/11/S2A_MSIL2A_20210311T100031_N0500_R122_T33TWL_20230528T215553.SAFE', '/eodata/Sentinel-2/MSI/L2A_N0500/2021/03/21/S2A_MSIL2A_20210321T100031_N0500_R122_T33TWL_20230521T144013.SAFE', '/eodata/Sentinel-2/MSI/L2A_N0500/2021/11/28/S2B_MSIL2A_20211128T095249_N0500_R079_T33TWL_20230106T144251.SAFE', '/eodata/Sentinel-2/MSI/L2A_N0500/2021/12/16/S2A_MSIL2A_20211216T100421_N0500_R122_T33TWL_20....
E Issues for file 'openEO.nc':
E Issues for variable 'var':
E Coordinates mismatch for dimension 't': ['2021-02-14T00:00:00.000000000' '2021-02-24T00:00:00.000000000'
E '2021-02-26T00:00:00.000000000' '2021-03-01T00:00:00.000000000'
E '2021-03-03T00:00:00.000000000' '2021-03-06T00:00:00.000000000'
E '2021-03-08T00:00:00.000000000' '2021-03-21T00:00:00.000000000'
E '2021-03-26T00:00:00.000000000' '2021-03-31T00:00:00.000000000'
E '2021-04-05T00:00:00.000000000' '2021-04-15T00:00:00.000000000'
E '2021-04-22T00:00:00.000000000' '2021-05-05T00:00:00.000000000'
E '2021-05-10T00:00:00.000000000' '2021-05-15T00:00:00.000000000'
E '2021-05-20T00:00:00.000000000' '2021-05-27T00:00:00.000000000'
E '2021-05-30T00:00:00.000000000' '2021-06-01T00:00:00.000000000'
E '2021-06-04T00:00:00.000000000' '2021-06-09T00:00:00.000000000'
E '2021-06-14T00:00:00.000000000' '2021-06-19T00:00:00.000000000'
E '2021-06-21T00:00:00.000000000' '2021-06-24T00:00:00.000000000'
E '2021-06-26T00:00:00.000000000' '2021-06-29T00:00:00.000000000'
E '2021-07-06T00:00:00.000000000' '2021-07-09T00:00:00.000000000'
E '2021-07-11T00:00:00.000000000' '2021-07-14T00:00:00.000000000'
E '2021-07-21T00:00:00.000000000' '2021-07-24T00:00:00.000000000'
E '2021-07-29T00:00:00.000000000' '2021-08-03T00:00:00.000000000'
E '2021-08-08T00:00:00.000000000' '2021-08-10T00:00:00.000000000'
E '2021-08-15T00:00:00.000000000' '2021-08-18T00:00:00.000000000'
E '2021-08-20T00:00:00.000000000' '2021-08-25T00:00:00.000000000'
E '2021-08-30T00:00:00.000000000' '2021-09-02T00:00:00.000000000'
E '2021-09-04T00:00:00.000000000' '2021-09-09T00:00:00.000000000'
E '2021-09-12T00:00:00.000000000' '2021-09-14T00:00:00.000000000'
E '2021-09-24T00:00:00.000000000' '2021-09-29T00:00:00.000000000'
E '2021-10-04T00:00:00.000000000' '2021-10-14T00:00:00.000000000'
E '2021-10-17T00:00:00.000000000' '2021-10-24T00:00:00.000000000'
E '2021-10-27T00:00:00.000000000' '2021-11-11T00:00:00.000000000'
E '2021-11-23T00:00:00.000000000' '2021-12-01T00:00:00.000000000'
E '2021-12-06T00:00:00.000000000' '2021-12-08T00:00:00.000000000'
E '2021-12-18T00:00:00.000000000' '2021-12-21T00:00:00.000000000'] != ['2021-02-14T00:00:00.000000000' '2021-02-24T00:00:00.000000000'
E '2021-02-26T00:00:00.000000000' '2021-03-01T00:00:00.000000000'
E '2021-03-03T00:00:00.000000000' '2021-03-06T00:00:00.000000000'
E '2021-03-08T00:00:00.000000000' '2021-03-21T00:00:00.000000000'
E '2021-03-26T00:00:00.000000000' '2021-03-31T00:00:00.000000000'
E '2021-04-05T00:00:00.000000000' '2021-04-15T00:00:00.000000000'
E '2021-04-22T00:00:00.000000000' '2021-05-05T00:00:00.000000000'
E '2021-05-10T00:00:00.000000000' '2021-05-15T00:00:00.000000000'
E '2021-05-20T00:00:00.000000000' '2021-05-25T00:00:00.000000000'
E '2021-05-27T00:00:00.000000000' '2021-05-30T00:00:00.000000000'
E '2021-06-01T00:00:00.000000000' '2021-06-04T00:00:00.000000000'
E '2021-06-09T00:00:00.000000000' '2021-06-14T00:00:00.000000000'
E '2021-06-19T00:00:00.000000000' '2021-06-21T00:00:00.000000000'
E '2021-06-24T00:00:00.000000000' '2021-06-26T00:00:00.000000000'
E '2021-06-29T00:00:00.000000000' '2021-07-01T00:00:00.000000000'
E '2021-07-06T00:00:00.000000000' '2021-07-09T00:00:00.000000000'
E '2021-07-11T00:00:00.000000000' '2021-07-14T00:00:00.000000000'
E '2021-07-21T00:00:00.000000000' '2021-07-24T00:00:00.000000000'
E '2021-07-29T00:00:00.000000000' '2021-08-03T00:00:00.000000000'
E '2021-08-08T00:00:00.000000000' '2021-08-10T00:00:00.000000000'
E '2021-08-15T00:00:00.000000000' '2021-08-18T00:00:00.000000000'
E '2021-08-20T00:00:00.000000000' '2021-08-25T00:00:00.000000000'
E '2021-08-30T00:00:00.000000000' '2021-09-02T00:00:00.000000000'
E '2021-09-04T00:00:00.000000000' '2021-09-09T00:00:00.000000000'
E '2021-09-12T00:00:00.000000000' '2021-09-14T00:00:00.000000000'
E '2021-09-24T00:00:00.000000000' '2021-09-29T00:00:00.000000000'
E '2021-10-04T00:00:00.000000000' '2021-10-14T00:00:00.000000000'
E '2021-10-17T00:00:00.000000000' '2021-10-24T00:00:00.000000000'
E '2021-10-27T00:00:00.000000000' '2021-11-11T00:00:00.000000000'
E '2021-11-23T00:00:00.000000000' '2021-12-01T00:00:00.000000000'
E '2021-12-06T00:00:00.000000000' '2021-12-08T00:00:00.000000000'
E '2021-12-18T00:00:00.000000000' '2021-12-21T00:00:00.000000000']
E Shape mismatch: (62, 44, 48) != (64, 44, 48)
E Left and right DataArray objects are not close
E Differing dimensions:
E (t: 62, y: 44, x: 48) != (t: 64, y: 44, x: 48)
E Differing values:
E L
E array([[[nan, nan, ..., 1., nan],
E [nan, nan, ..., 1., 1.],
E ...,
E [nan, nan, ..., nan, nan],
E [nan, nan, ..., nan, nan]],
E
E [[nan, nan, ..., -1., nan],
E [nan, nan, ..., -1., -1.],
E ...,
E [nan, nan, ..., nan, nan],
E [nan, nan, ..., nan, nan]],
E
E ...,
E
E [[nan, nan, ..., nan, nan],
E [nan, nan, ..., nan, nan],
E ...,
E [nan, nan, ..., nan, nan],
E [nan, nan, ..., nan, nan]],
E
E [[nan, nan, ..., nan, nan],
E [nan, nan, ..., nan, nan],
E ...,
E [nan, nan, ..., nan, nan],
E [nan, nan, ..., nan, nan]]], shape=(62, 44, 48))
E R
E array([[[nan, nan, ..., 1., nan],
E [nan, nan, ..., 1., 1.],
E ...,
E [nan, nan, ..., nan, nan],
E [nan, nan, ..., nan, nan]],
E
E [[nan, nan, ..., -1., nan],
E [nan, nan, ..., -1., -1.],
E ...,
E [nan, nan, ..., nan, nan],
E [nan, nan, ..., nan, nan]],
E
E ...,
E
E [[nan, nan, ..., nan, nan],
E [nan, nan, ..., nan, nan],
E ...,
E [nan, nan, ..., nan, nan],
E [nan, nan, ..., nan, nan]],
E
E [[nan, nan, ..., nan, nan],
E [nan, nan, ..., nan, nan],
E ...,
E [nan, nan, ..., nan, nan],
E [nan, nan, ..., nan, nan]]], shape=(64, 44, 48))
E Differing coordinates:
E L * t (t) datetime64[ns] 496B 2021-02-14 2021-02-24 ... 2021-12-21
E R * t (t) datetime64[ns] 512B 2021-02-14 2021-02-24 ... 2021-12-21
/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/site-packages/openeo/testing/results.py:521: AssertionError
----------------------------- Captured stdout call -----------------------------
0:00:00 Job 'cdse-j-2603120318204370adec2a86f6726150': send 'start'
0:00:18 Job 'cdse-j-2603120318204370adec2a86f6726150': created (progress 0%)
0:00:25 Job 'cdse-j-2603120318204370adec2a86f6726150': created (progress 0%)
0:00:32 Job 'cdse-j-2603120318204370adec2a86f6726150': created (progress 0%)
0:00:40 Job 'cdse-j-2603120318204370adec2a86f6726150': created (progress 0%)
0:00:50 Job 'cdse-j-2603120318204370adec2a86f6726150': queued (progress 0%)
0:01:03 Job 'cdse-j-2603120318204370adec2a86f6726150': queued (progress 0%)
0:01:21 Job 'cdse-j-2603120318204370adec2a86f6726150': queued (progress 0%)
0:01:41 Job 'cdse-j-2603120318204370adec2a86f6726150': queued (progress 0%)
0:02:05 Job 'cdse-j-2603120318204370adec2a86f6726150': running (progress N/A)
0:02:36 Job 'cdse-j-2603120318204370adec2a86f6726150': running (progress N/A)
0:03:14 Job 'cdse-j-2603120318204370adec2a86f6726150': running (progress N/A)
0:04:01 Job 'cdse-j-2603120318204370adec2a86f6726150': running (progress N/A)
0:05:00 Job 'cdse-j-2603120318204370adec2a86f6726150': running (progress N/A)
0:06:00 Job 'cdse-j-2603120318204370adec2a86f6726150': running (progress N/A)
0:07:00 Job 'cdse-j-2603120318204370adec2a86f6726150': finished (progress 100%)
------------------------------ Captured log call -------------------------------
INFO conftest:conftest.py:145 Connecting to 'openeofed.dataspace.copernicus.eu'
INFO openeo.config:config.py:193 Loaded openEO client config from sources: []
INFO conftest:conftest.py:158 Checking for auth_env_var='OPENEO_AUTH_CLIENT_CREDENTIALS_CDSEFED' to drive auth against url='openeofed.dataspace.copernicus.eu'.
INFO conftest:conftest.py:162 Extracted provider_id='CDSE' client_id='openeo-apex-benchmarks-service-account' from auth_env_var='OPENEO_AUTH_CLIENT_CREDENTIALS_CDSEFED'
INFO openeo.rest.connection:connection.py:302 Found OIDC providers: ['CDSE']
INFO openeo.rest.auth.oidc:oidc.py:404 Doing 'client_credentials' token request 'https://identity.dataspace.copernicus.eu/auth/realms/CDSE/protocol/openid-connect/token' with post data fields ['grant_type', 'client_id', 'client_secret', 'scope'] (client_id 'openeo-apex-benchmarks-service-account')
INFO openeo.rest.connection:connection.py:401 Obtained tokens: ['access_token', 'id_token']
INFO openeo.rest.job:job.py:436 Downloading Job result asset 'openEO.nc' from https://s3.waw3-1.openeo.v1.dataspace.copernicus.eu/openeo-data-prod-waw4-1/batch_jobs/j-2603120318204370adec2a86f6726150/openEO.nc?X-Proxy-Head-As-Get=true&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=0afe22bd51d84f97ba716304abe09484%2F20260312%2Fwaw4-1%2Fs3%2Faws4_request&X-Amz-Date=20260312T032527Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Security-Token=eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlX2FybiI6ImFybjpvcGVuZW93czppYW06Ojpyb2xlL29wZW5lby1kYXRhLXByb2Qtd2F3NC0xLXdvcmtzcGFjZSIsImluaXRpYWxfaXNzdWVyIjoib3BlbmVvLnByb2Qud2F3My0xLm9wZW5lby1pbnQudjEuZGF0YXNwYWNlLmNvcGVybmljdXMuZXUiLCJodHRwczovL2F3cy5hbWF6b24uY29tL3RhZ3MiOnsicHJpbmNpcGFsX3RhZ3MiOnsiam9iX2lkIjpbImotMjYwMzEyMDMxODIwNDM3MGFkZWMyYTg2ZjY3MjYxNTAiXSwidXNlcl9pZCI6WyI2YTc3ZmNkMS05YzA4LTQ2ZTktYjg3NS01NGZiOTk5YWIyMDAiXX0sInRyYW5zaXRpdmVfdGFnX2tleXMiOlsidXNlcl9pZCIsImpvYl9pZCJdfSwiaXNzIjoic3RzLndhdzMtMS5vcGVuZW8udjEuZGF0YXNwYWNlLmNvcGVybmljdXMuZXUiLCJzdWIiOiJvcGVuZW8tZHJpdmVyIiwiZXhwIjoxNzczMzI5MTI3LCJuYmYiOjE3NzMyODU5MjcsImlhdCI6MTc3MzI4NTkyNywianRpIjoiZGI2Zjg1MGMtNmY0My00ZmNhLThhMjEtN2MyZjA1YWNlYzUyIiwiYWNjZXNzX2tleV9pZCI6IjBhZmUyMmJkNTFkODRmOTdiYTcxNjMwNGFiZTA5NDg0In0.WRmA-JCdrrku1OtlgGQd3mCo3hxJtbb35qXka7k5z2B_A87TiuCj--Q7aFTxnqqiL9Zd4o1VtXj89QuET2gl85spCQwTBbtiuqlkFaSMfqmh3E-72U54eFOzMU5eWB8UN2LUFnslHsmqnkxys8CJJ6LUMUzSuPqH8VwWiUIuX46hyPDmjcdnImOHbe-H61FiylWn9UNWtWhK0w4ZeZqHvcI4cS5x8UBsIDE8Zq3DZfwe5eKK4RAHT6115E_gOJDP--6pGeJLYMKFNpjMLOp6PHXNtOh3DqHwSyencs2jznxCTqFnDvmeDvcPqYMwY8pGnotbAtaJ5QB_WBSa2mtpCQ&X-Amz-Signature=cc2a5dd35f9d9df5dedcefd29e6d1030426dfee10d275d41047eed3df17c9488 to /home/runner/work/apex_algorithms/apex_algorithms/qa/benchmarks/tmp_path_root/test_run_benchmark_peakvalley_0/actual/openEO.nc
INFO apex_algorithm_qa_tools.scenarios:util.py:345 Downloading reference data for scenario.id='peakvalley' to reference_dir=PosixPath('/home/runner/work/apex_algorithms/apex_algorithms/qa/benchmarks/tmp_path_root/test_run_benchmark_peakvalley_0/reference'): start 2026-03-12 03:25:30.621510
INFO apex_algorithm_qa_tools.scenarios:util.py:345 Downloading source='https://s3.waw3-1.cloudferro.com/apex-benchmarks/gh-20735970267!tests_test_benchmarks.py__test_run_benchmark_peakvalley_!actual/job-results.json' to path=PosixPath('/home/runner/work/apex_algorithms/apex_algorithms/qa/benchmarks/tmp_path_root/test_run_benchmark_peakvalley_0/reference/job-results.json'): start 2026-03-12 03:25:30.621837
INFO apex_algorithm_qa_tools.scenarios:util.py:351 Downloading source='https://s3.waw3-1.cloudferro.com/apex-benchmarks/gh-20735970267!tests_test_benchmarks.py__test_run_benchmark_peakvalley_!actual/job-results.json' to path=PosixPath('/home/runner/work/apex_algorithms/apex_algorithms/qa/benchmarks/tmp_path_root/test_run_benchmark_peakvalley_0/reference/job-results.json'): end 2026-03-12 03:25:31.968579, elapsed 0:00:01.346742
INFO apex_algorithm_qa_tools.scenarios:util.py:345 Downloading source='https://s3.waw3-1.cloudferro.com/swift/v1/apex-examples/peakvalley/openEO.nc' to path=PosixPath('/home/runner/work/apex_algorithms/apex_algorithms/qa/benchmarks/tmp_path_root/test_run_benchmark_peakvalley_0/reference/openEO.nc'): start 2026-03-12 03:25:31.968930
INFO apex_algorithm_qa_tools.scenarios:util.py:351 Downloading source='https://s3.waw3-1.cloudferro.com/swift/v1/apex-examples/peakvalley/openEO.nc' to path=PosixPath('/home/runner/work/apex_algorithms/apex_algorithms/qa/benchmarks/tmp_path_root/test_run_benchmark_peakvalley_0/reference/openEO.nc'): end 2026-03-12 03:25:33.030585, elapsed 0:00:01.061655
INFO apex_algorithm_qa_tools.scenarios:util.py:351 Downloading reference data for scenario.id='peakvalley' to reference_dir=PosixPath('/home/runner/work/apex_algorithms/apex_algorithms/qa/benchmarks/tmp_path_root/test_run_benchmark_peakvalley_0/reference'): end 2026-03-12 03:25:33.030775, elapsed 0:00:02.409265
INFO openeo.testing.results:results.py:422 Comparing job results: PosixPath('/home/runner/work/apex_algorithms/apex_algorithms/qa/benchmarks/tmp_path_root/test_run_benchmark_peakvalley_0/actual') vs PosixPath('/home/runner/work/apex_algorithms/apex_algorithms/qa/benchmarks/tmp_path_root/test_run_benchmark_peakvalley_0/reference')