From 16516e124a0b35059ad2077d0b568e39bf22e661 Mon Sep 17 00:00:00 2001 From: Ji Hwan Date: Wed, 25 Mar 2026 10:52:31 +0900 Subject: [PATCH 01/18] feat: ethereum execution specs rpc tests Signed-off-by: Ji Hwan --- .github/workflows/pos-e2e.yml | 215 +++++++++++++++++++++++++++++++++- 1 file changed, 213 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pos-e2e.yml b/.github/workflows/pos-e2e.yml index 6edf1f82..d83b56c9 100644 --- a/.github/workflows/pos-e2e.yml +++ b/.github/workflows/pos-e2e.yml @@ -38,6 +38,11 @@ on: required: false type: string default: "feat/execution-specs" + exec-specs-ref: + description: "Branch or tag of ethereum/execution-specs to use for remote execution tests." + required: false + type: string + default: "forks/amsterdam" force-run: description: "Run tests even if no new bor release is detected" required: false @@ -57,6 +62,7 @@ env: ENCLAVE_NAME: pos POLYCLI_VERSION: v0.1.103 KURTOSIS_POS_REF: ${{ inputs.kurtosis-pos-ref || 'feat/execution-specs' }} + EXEC_SPECS_REF: ${{ inputs.exec-specs-ref || 'forks/amsterdam' }} jobs: # --------------------------------------------------------------------------- @@ -879,18 +885,223 @@ jobs: kurtosis enclave stop "${{ env.ENCLAVE_NAME }}" || true kurtosis clean || true + # --------------------------------------------------------------------------- + # Job 3: Execution-specs remote — run ethereum/execution-specs against latest + # bor release via `execute remote`. Runs after all matrix jobs to maximise + # available runner resources. + # --------------------------------------------------------------------------- + pos-exec-specs-remote: + needs: [check-new-release, pos-execution-specs, pos-fork-transition] + if: >- + always() && + needs.check-new-release.outputs.should-run == 'true' + runs-on: ubuntu-latest + timeout-minutes: 120 + name: exec-specs-remote (latest bor) + steps: + - name: Checkout agglayer/e2e + uses: actions/checkout@v6 + with: + ref: ${{ github.ref }} + + - name: Install kurtosis + run: | + echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list + sudo apt update + sudo apt install -y kurtosis-cli jq + kurtosis analytics disable + + - name: Install foundry + uses: foundry-rs/foundry-toolchain@v1.3.1 + + - name: Install uv + uses: astral-sh/setup-uv@v6 + + - name: Checkout kurtosis-pos + uses: actions/checkout@v6 + with: + repository: 0xPolygon/kurtosis-pos + ref: ${{ env.KURTOSIS_POS_REF }} + path: kurtosis-pos + + - name: Checkout execution-specs + uses: actions/checkout@v6 + with: + repository: ethereum/execution-specs + ref: ${{ env.EXEC_SPECS_REF }} + path: execution-specs + + - name: Install execution-specs dependencies + run: | + cd execution-specs + uv sync + + - name: Patch constants.star (all post-Rio forks at 256) + run: | + CONSTANTS="kurtosis-pos/src/config/constants.star" + for fork in madhugiri madhugiriPro dandeli lisovo lisovoPro giugliano; do + sed -i -E '/^\s+"'"${fork}"'":/s/[0-9]+/256/' "$CONSTANTS" + done + echo "--- Patched EL_HARD_FORK_BLOCKS ---" + grep -A 20 '^EL_HARD_FORK_BLOCKS' "$CONSTANTS" + + - name: Generate kurtosis params (single-version latest bor) + run: | + BOR_LATEST="${{ needs.check-new-release.outputs.bor-latest-image }}" + HV2_IMAGE="${{ needs.check-new-release.outputs.heimdall-v2-image }}" + + echo "Bor (latest): ${BOR_LATEST} | Heimdall-v2: ${HV2_IMAGE}" + + cat > /tmp/pos-params.yml </dev/null) || block=0 + [[ "$block" -ge "$TARGET_BLOCK" ]] && echo "Block ${block} — all forks active." && exit 0 + echo "[${elapsed}s] Block: ${block} / ${TARGET_BLOCK}" + sleep "$POLL_INTERVAL" + elapsed=$(( elapsed + POLL_INTERVAL )) + done + echo "::error::Chain stuck at block ${block}." + exit 1 + + - name: Run execution-specs remote tests + run: | + set -eo pipefail + L2_RPC_RAW="$(kurtosis port print "${{ env.ENCLAVE_NAME }}" l2-el-4-bor-heimdall-v2-rpc rpc)" + L2_RPC_RAW="${L2_RPC_RAW#http://}" ; L2_RPC_RAW="${L2_RPC_RAW#https://}" + L2_RPC_URL="http://${L2_RPC_RAW}" + L2_PRIVATE_KEY="0xd40311b5a5ca5eaeb48dfba5403bde4993ece8eccf4190e98e19fcd4754260ea" + CHAIN_ID="$(cast chain-id --rpc-url "${L2_RPC_URL}")" + + echo "RPC: ${L2_RPC_URL} | Chain ID: ${CHAIN_ID}" + + cd execution-specs + uv run execute remote \ + --fork=Prague \ + --rpc-endpoint="${L2_RPC_URL}" \ + --rpc-seed-key="${L2_PRIVATE_KEY}" \ + --chain-id="${CHAIN_ID}" \ + --default-max-fee-per-blob-gas=1 \ + --default-max-fee-per-gas=100000000000 \ + --default-max-priority-fee-per-gas=100000000000 \ + --default-gas-price=100000000000 \ + --sender-funding-txs-gas-price=100000000000 \ + --seed-account-sweep-amount=800000000000000000000 \ + --max-tx-per-batch=75 \ + -n 8 \ + -s \ + --tx-wait-timeout=60 \ + -k "not blob and not 4844 and not beacon_root and not mainnet and not genesis_hash_available and not test_selfdestruct_to_precompile and not test_selfdestruct_to_system_contract and not test_dynamic_create2_selfdestruct_collision_multi_tx and not test_extcodehash_of_empty and not test_call_memory_expands_on_early_revert and not type_3 and not address_0x000000000000000000000000000000000000000a and not address_0x000000000000000000000000000000000000000b and not address_0x000000000000000000000000000000000000000c and not address_0x000000000000000000000000000000000000000d and not address_0x000000000000000000000000000000000000000e and not address_0x000000000000000000000000000000000000000f and not address_0x0000000000000000000000000000000000000010 and not address_0x0000000000000000000000000000000000000011" \ + --ignore=tests/frontier/opcodes/test_calldatacopy.py \ + --ignore=tests/frontier/opcodes/test_dup.py \ + --ignore=tests/frontier/opcodes/test_all_opcodes.py \ + --ignore=tests/frontier/opcodes/test_blockhash.py \ + --ignore=tests/frontier/scenarios/test_scenarios.py \ + --ignore=tests/frontier/validation/test_transaction.py \ + --ignore=tests/frontier/create/test_create_one_byte.py \ + --ignore=tests/frontier/precompiles/test_precompile_absence.py \ + --ignore=tests/homestead/coverage/test_coverage.py \ + --ignore=tests/byzantium/eip197_ec_pairing/test_gas.py \ + --ignore=tests/byzantium/eip196_ec_add_mul/test_gas.py \ + --ignore=tests/london/eip1559_fee_market_change/test_tx_type.py \ + --ignore=tests/london/validation/test_header.py \ + --ignore=tests/cancun/eip4788_beacon_root \ + --ignore=tests/cancun/create/test_create_oog_from_eoa_refunds.py \ + --ignore=tests/cancun/eip1153_tstore/test_basic_tload.py \ + --ignore=tests/cancun/eip1153_tstore/test_tload_calls.py \ + --ignore=tests/cancun/eip1153_tstore/test_tstore_reentrancy.py \ + --ignore=tests/cancun/eip1153_tstore/test_tload_reentrancy.py \ + --ignore=tests/constantinople/eip145_bitwise_shift/test_shift_combinations.py \ + --ignore=tests/prague/eip6110_deposits \ + --ignore=tests/prague/eip7002_el_triggerable_withdrawals \ + --ignore=tests/prague/eip7251_consolidations \ + --ignore=tests/prague/eip7685_general_purpose_el_requests \ + --ignore=tests/prague/eip2537_bls_12_381_precompiles \ + --ignore=tests/prague/eip2935_historical_block_hashes_from_state \ + --ignore=tests/prague/eip7702_set_code_tx \ + --ignore=tests/shanghai/eip4895_withdrawals \ + --ignore=tests/istanbul/eip1344_chainid/test_chainid.py \ + tests/frontier/ \ + tests/homestead/ \ + tests/tangerine_whistle/ \ + tests/byzantium/ \ + tests/constantinople/ \ + tests/istanbul/ \ + tests/berlin/ \ + tests/london/ \ + tests/paris/ \ + tests/shanghai/ \ + tests/cancun/ \ + tests/prague/ + + - name: Dump enclave logs + if: failure() + run: kurtosis dump ./dump + + - name: Upload logs + if: failure() + uses: actions/upload-artifact@v4 + with: + name: pos-e2e-dump-execspecs-remote-${{ github.run_id }} + path: ./dump + + - name: Clean up enclave + if: always() + run: | + kurtosis enclave stop "${{ env.ENCLAVE_NAME }}" || true + kurtosis clean || true + # --------------------------------------------------------------------------- # Mark release as tested (write cache entry). # Only runs after all test phases succeed. # --------------------------------------------------------------------------- mark-tested: - needs: [check-new-release, pos-execution-specs, pos-fork-transition] + needs: [check-new-release, pos-execution-specs, pos-fork-transition, pos-exec-specs-remote] if: >- always() && needs.check-new-release.outputs.bor-tag != 'manual' && needs.check-new-release.outputs.bor-tag != '' && needs.pos-execution-specs.result == 'success' && - needs.pos-fork-transition.result == 'success' + needs.pos-fork-transition.result == 'success' && + needs.pos-exec-specs-remote.result == 'success' runs-on: ubuntu-latest steps: - name: Write cache marker From 2c2c03d59afa9f52b0bbb1d31a8790463265abb0 Mon Sep 17 00:00:00 2001 From: Ji Hwan Date: Wed, 25 Mar 2026 12:57:36 +0900 Subject: [PATCH 02/18] feat: ignore eip6780 in ethereum execution specs Signed-off-by: Ji Hwan --- .github/workflows/pos-e2e.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pos-e2e.yml b/.github/workflows/pos-e2e.yml index d83b56c9..14544fc6 100644 --- a/.github/workflows/pos-e2e.yml +++ b/.github/workflows/pos-e2e.yml @@ -1044,6 +1044,7 @@ jobs: --ignore=tests/london/eip1559_fee_market_change/test_tx_type.py \ --ignore=tests/london/validation/test_header.py \ --ignore=tests/cancun/eip4788_beacon_root \ + --ignore=tests/cancun/eip6780_selfdestruct \ --ignore=tests/cancun/create/test_create_oog_from_eoa_refunds.py \ --ignore=tests/cancun/eip1153_tstore/test_basic_tload.py \ --ignore=tests/cancun/eip1153_tstore/test_tload_calls.py \ From d170bab8d0ee937fbd7073f9626e227869ba2470 Mon Sep 17 00:00:00 2001 From: Ji Hwan Date: Wed, 25 Mar 2026 13:47:08 +0900 Subject: [PATCH 03/18] feat: use existing gh workflows for bot PRs Signed-off-by: Ji Hwan --- .../pos-update-compat-versions-matrix.yml | 39 ++++++------------- 1 file changed, 11 insertions(+), 28 deletions(-) diff --git a/.github/workflows/pos-update-compat-versions-matrix.yml b/.github/workflows/pos-update-compat-versions-matrix.yml index 9d07385e..9df1d0d9 100644 --- a/.github/workflows/pos-update-compat-versions-matrix.yml +++ b/.github/workflows/pos-update-compat-versions-matrix.yml @@ -42,31 +42,14 @@ jobs: - name: Create pull request if: steps.check.outputs.changed == 'true' - env: - GH_TOKEN: ${{ github.token }} - run: | - BRANCH="auto/update-compat-versions" - - # Configure git for the github-actions bot. - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - - # Create or update the branch. - git checkout -B "$BRANCH" - git add scripts/pos-version-matrix/compat-versions.yml - git commit -m "chore: update compat-versions.yml with new releases" - git push -f origin "$BRANCH" - - # Create PR if one doesn't already exist. - existing_pr=$(gh pr list --head "$BRANCH" --state open --json number --jq '.[0].number' 2>/dev/null || echo "") - if [[ -n "$existing_pr" ]]; then - echo "PR #${existing_pr} already exists, updated branch." - else - PR_BODY="New bor/erigon releases detected. This PR adds them to compat-versions.yml." - PR_BODY="${PR_BODY}\n\n**Review before merging** — remove any versions that should not be tested and verify the pair count is reasonable." - gh pr create \ - --title "chore: update PoS compat versions with new releases" \ - --body "$(echo -e "$PR_BODY")" \ - --head "$BRANCH" \ - --base main - fi + uses: peter-evans/create-pull-request@v7 + with: + token: ${{ github.token }} + branch: auto/update-compat-versions + commit-message: "chore: update compat-versions.yml with new releases" + title: "chore: update PoS compat versions with new releases" + body: | + New bor/erigon releases detected. This PR adds them to compat-versions.yml. + + **Review before merging** — remove any versions that should not be tested and verify the pair count is reasonable. + base: main From 9468c63f8b1c70d71e5f75582771ddf6e56f413c Mon Sep 17 00:00:00 2001 From: Ji Hwan Date: Wed, 25 Mar 2026 14:33:38 +0900 Subject: [PATCH 04/18] feat: change bor version tracking to be checked in relevant PRs Signed-off-by: Ji Hwan --- .../pos-update-compat-versions-matrix.yml | 54 ++++++++----------- 1 file changed, 23 insertions(+), 31 deletions(-) diff --git a/.github/workflows/pos-update-compat-versions-matrix.yml b/.github/workflows/pos-update-compat-versions-matrix.yml index 9df1d0d9..1fc111c8 100644 --- a/.github/workflows/pos-update-compat-versions-matrix.yml +++ b/.github/workflows/pos-update-compat-versions-matrix.yml @@ -1,19 +1,21 @@ -name: Update PoS Compat Versions -run-name: Check for new bor/erigon releases +name: PoS Compat Versions Check on: - schedule: - # Check daily at 04:00 UTC (before the pos-e2e run at 06:00). - - cron: "0 4 * * *" - + pull_request: + branches: + - main + paths: + - "scripts/pos-version-matrix/**" + - "scenarios/pos/**" + - "tests/pos/**" + - ".github/workflows/pos-*.yml" workflow_dispatch: {} permissions: - contents: write - pull-requests: write + contents: read jobs: - update-compat-versions: + check-compat-versions: runs-on: ubuntu-latest timeout-minutes: 5 steps: @@ -21,35 +23,25 @@ jobs: uses: actions/checkout@v6 - name: Install dependencies - run: pip install -q requests PyYAML + run: pip install -q -r scripts/pos-version-matrix/requirements.txt - name: Check for new releases - id: check env: - GH_TOKEN: ${{ github.token }} GITHUB_TOKEN: ${{ github.token }} run: | - output=$(python3 scripts/pos-version-matrix/update-compat-versions.py 2>&1) - echo "$output" + python3 scripts/pos-version-matrix/update-compat-versions.py if git diff --quiet scripts/pos-version-matrix/compat-versions.yml; then - echo "changed=false" >> "$GITHUB_OUTPUT" + echo "compat-versions.yml is up to date." else - echo "changed=true" >> "$GITHUB_OUTPUT" - echo "--- Diff ---" + echo "New bor/erigon releases are available that are not in compat-versions.yml." + echo "" + echo "To update, run:" + echo " make update-pos-compat-versions" + echo "" + echo "then commit the result." + echo "" + echo "Diff:" git diff scripts/pos-version-matrix/compat-versions.yml + exit 1 fi - - - name: Create pull request - if: steps.check.outputs.changed == 'true' - uses: peter-evans/create-pull-request@v7 - with: - token: ${{ github.token }} - branch: auto/update-compat-versions - commit-message: "chore: update compat-versions.yml with new releases" - title: "chore: update PoS compat versions with new releases" - body: | - New bor/erigon releases detected. This PR adds them to compat-versions.yml. - - **Review before merging** — remove any versions that should not be tested and verify the pair count is reasonable. - base: main From 2df17f0221e417578b1192bb03dd1cf239f3a68d Mon Sep 17 00:00:00 2001 From: Ji Hwan Date: Wed, 25 Mar 2026 14:34:53 +0900 Subject: [PATCH 05/18] feat: bump makefile + pos versions matrix Signed-off-by: Ji Hwan --- Makefile | 19 ++++++- .../pos-version-matrix/compat-versions.yml | 54 +++++++++---------- 2 files changed, 44 insertions(+), 29 deletions(-) diff --git a/Makefile b/Makefile index 59220b60..13811e41 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,24 @@ -.PHONY: check-dependencies compile-contracts +.PHONY: check-dependencies compile-contracts update-tests-inventory update-pos-compat-versions check-dependencies: ./scripts/check-dependencies.sh compile-contracts: - find core/contracts/ -type f | grep -E '(yul|sol)' | while read contract; do echo "$$contract"; forge build -C "$$contract" ; done + find core/contracts/ -type f -name '*.sol' | while read contract; do echo "$$contract"; forge build -C "$$contract" ; done + # .yul files: use --root so forge picks up the per-directory foundry.toml which sets + # lint_on_build = false. Solar (forge's linter) does not support Yul syntax. + find core/contracts/ -type f -name '*.yul' | while read f; do dir=$$(dirname "$$f"); echo "$$f"; forge build --root "$$dir" ; done ./core/helpers/scripts/postprocess_contracts.sh +## Run before opening a PR that touches test files. +## Updates TESTSINVENTORY.md and the test_tags section of README.md. +update-tests-inventory: + ./scripts/update-tests-inventory.sh + +## Run before opening a PR that touches PoS-related files. +## Fetches the latest bor/erigon releases and updates scripts/pos-version-matrix/compat-versions.yml. +## Requires a GITHUB_TOKEN env var to avoid rate limits: GITHUB_TOKEN= make update-pos-compat-versions +update-pos-compat-versions: + pip install -q -r scripts/pos-version-matrix/requirements.txt + python3 scripts/pos-version-matrix/update-compat-versions.py + diff --git a/scripts/pos-version-matrix/compat-versions.yml b/scripts/pos-version-matrix/compat-versions.yml index c6018e70..7a8912cf 100644 --- a/scripts/pos-version-matrix/compat-versions.yml +++ b/scripts/pos-version-matrix/compat-versions.yml @@ -1,31 +1,31 @@ -# PoS version compatibility matrix — curated version lists for compat testing. +# PoS version compatibility matrix — curated list of EL versions to test. # -# The pos-e2e workflow generates pairwise bor combinations and deploys -# mixed devnets with validators/RPCs split across bor versions, plus -# erigon RPCs as cross-client controls. +# The pos-e2e workflow reads this file to generate pairwise compat pairs. +# Only versions listed under `versions` are tested. Keep this list focused: +# - Versions actively running in production +# - Release candidates being validated for rollout +# - Remove versions once fully deprecated and no longer in use # -# Each devnet topology (per bor pair): -# - 2 validators on bor version_a -# - 1 validator on bor version_b -# - 1 RPC on bor version_a -# - 1 RPC on bor version_b -# - 1 RPC on erigon (latest from erigon_versions) +# Each entry must have: image, el_type (bor or erigon), reason. +# Pairwise pairs are generated across ALL entries (bor-bor, erigon-erigon, +# and bor-erigon cross-client combinations). # -# Tests run: fork-transition (staggered forks) + execution-specs (after all forks active). +# If this file is absent or `versions` is empty, the workflow falls back +# to auto-detecting the latest bor release from each major.minor line. -bor_versions: - - image: "0xpolygon/bor:2.7.0-beta3" - reason: "latest pre-release, validating for production" - - - image: "0xpolygon/bor:2.6.5" - reason: "latest 2.6.x stable, active in production" - - - image: "0xpolygon/bor:2.5.9" - reason: "latest 2.5.x stable, still active on some nodes" - -erigon_versions: - - image: "0xpolygon/erigon:v3.5.0-beta2" - reason: "latest pre-release, validating for production" - - - image: "0xpolygon/erigon:v3.4.0" - reason: "current stable, active in production" +versions: +- image: 0xpolygon/bor:2.7.0-beta5 + el_type: bor + reason: new pre-release (2.7 line), auto-detected +- image: 0xpolygon/bor:2.6.5 + el_type: bor + reason: new stable release (2.6 line), auto-detected +- image: 0xpolygon/erigon:v3.5.0-beta2 + el_type: erigon + reason: new pre-release (3.5 line), auto-detected +- image: 0xpolygon/erigon:v3.4.0 + el_type: erigon + reason: new stable release (3.4 line), auto-detected +- image: 0xpolygon/erigon:v3.3.7 + el_type: erigon + reason: new stable release (3.3 line), auto-detected From 41a2bce6b6cf321a478b6dd2e4a97bc9e846335d Mon Sep 17 00:00:00 2001 From: Ji Hwan Date: Wed, 25 Mar 2026 15:15:14 +0900 Subject: [PATCH 06/18] chore: rename jobs for verbosity Signed-off-by: Ji Hwan --- .github/workflows/pos-e2e.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pos-e2e.yml b/.github/workflows/pos-e2e.yml index 14544fc6..cbad57dd 100644 --- a/.github/workflows/pos-e2e.yml +++ b/.github/workflows/pos-e2e.yml @@ -38,7 +38,7 @@ on: required: false type: string default: "feat/execution-specs" - exec-specs-ref: + ethereum-exec-specs-ref: description: "Branch or tag of ethereum/execution-specs to use for remote execution tests." required: false type: string @@ -62,7 +62,7 @@ env: ENCLAVE_NAME: pos POLYCLI_VERSION: v0.1.103 KURTOSIS_POS_REF: ${{ inputs.kurtosis-pos-ref || 'feat/execution-specs' }} - EXEC_SPECS_REF: ${{ inputs.exec-specs-ref || 'forks/amsterdam' }} + EXEC_SPECS_REF: ${{ inputs.ethereum-exec-specs-ref || 'forks/amsterdam' }} jobs: # --------------------------------------------------------------------------- @@ -348,7 +348,7 @@ jobs: max-parallel: 5 matrix: include: ${{ fromJson(needs.check-new-release.outputs.compat-pairs) }} - name: exec-specs (${{ matrix.pair_label }}) + name: pos-exec-specs (${{ matrix.pair_label }}) steps: - name: Checkout agglayer/e2e uses: actions/checkout@v6 @@ -541,7 +541,7 @@ jobs: - name: Upload logs if: failure() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: pos-e2e-dump-execspecs-${{ matrix.pair_label }}-${{ github.run_id }} path: ./dump @@ -874,7 +874,7 @@ jobs: - name: Upload logs if: failure() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: pos-e2e-dump-forktrans-${{ matrix.pair_label }}-${{ github.run_id }} path: ./dump @@ -890,14 +890,14 @@ jobs: # bor release via `execute remote`. Runs after all matrix jobs to maximise # available runner resources. # --------------------------------------------------------------------------- - pos-exec-specs-remote: + ethereum-exec-specs-remote: needs: [check-new-release, pos-execution-specs, pos-fork-transition] if: >- always() && needs.check-new-release.outputs.should-run == 'true' runs-on: ubuntu-latest timeout-minutes: 120 - name: exec-specs-remote (latest bor) + name: ethereum-exec-specs-remote (against latest bor) steps: - name: Checkout agglayer/e2e uses: actions/checkout@v6 @@ -1079,7 +1079,7 @@ jobs: - name: Upload logs if: failure() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: pos-e2e-dump-execspecs-remote-${{ github.run_id }} path: ./dump @@ -1095,14 +1095,14 @@ jobs: # Only runs after all test phases succeed. # --------------------------------------------------------------------------- mark-tested: - needs: [check-new-release, pos-execution-specs, pos-fork-transition, pos-exec-specs-remote] + needs: [check-new-release, pos-execution-specs, pos-fork-transition, ethereum-exec-specs-remote] if: >- always() && needs.check-new-release.outputs.bor-tag != 'manual' && needs.check-new-release.outputs.bor-tag != '' && needs.pos-execution-specs.result == 'success' && needs.pos-fork-transition.result == 'success' && - needs.pos-exec-specs-remote.result == 'success' + needs.ethereum-exec-specs-remote.result == 'success' runs-on: ubuntu-latest steps: - name: Write cache marker @@ -1112,7 +1112,7 @@ jobs: echo "Marked ${{ needs.check-new-release.outputs.bor-tag }} as tested." - name: Save cache - uses: actions/cache/save@v4 + uses: actions/cache/save@v5 with: path: /tmp/pos-release-cache key: pos-e2e-bor-release-${{ needs.check-new-release.outputs.bor-tag }} From 284be03afb9ecdbc9cdcdf452b2d6700569bf74f Mon Sep 17 00:00:00 2001 From: Ji Hwan Date: Wed, 25 Mar 2026 15:19:39 +0900 Subject: [PATCH 07/18] feat: option to select specific step in pos-e2e ci Signed-off-by: Ji Hwan --- .github/workflows/pos-e2e.yml | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pos-e2e.yml b/.github/workflows/pos-e2e.yml index cbad57dd..c786b406 100644 --- a/.github/workflows/pos-e2e.yml +++ b/.github/workflows/pos-e2e.yml @@ -43,6 +43,21 @@ on: required: false type: string default: "forks/amsterdam" + run-exec-specs: + description: "Manually trigger exec-specs matrix tests (bypasses release detection)." + required: false + type: boolean + default: false + run-fork-trans: + description: "Manually trigger fork-transition matrix tests (bypasses release detection)." + required: false + type: boolean + default: false + run-ethereum-exec-specs-remote: + description: "Manually trigger ethereum execution-specs remote tests (bypasses release detection)." + required: false + type: boolean + default: false force-run: description: "Run tests even if no new bor release is detected" required: false @@ -339,7 +354,7 @@ jobs: pos-execution-specs: needs: check-new-release if: >- - needs.check-new-release.outputs.should-run == 'true' && + (needs.check-new-release.outputs.should-run == 'true' || inputs.run-exec-specs == true) && needs.check-new-release.outputs.compat-pairs != '[]' runs-on: ubuntu-latest timeout-minutes: 60 @@ -558,7 +573,7 @@ jobs: pos-fork-transition: needs: check-new-release if: >- - needs.check-new-release.outputs.should-run == 'true' && + (needs.check-new-release.outputs.should-run == 'true' || inputs.run-fork-trans == true) && needs.check-new-release.outputs.compat-pairs != '[]' runs-on: ubuntu-latest timeout-minutes: 60 @@ -894,7 +909,7 @@ jobs: needs: [check-new-release, pos-execution-specs, pos-fork-transition] if: >- always() && - needs.check-new-release.outputs.should-run == 'true' + (needs.check-new-release.outputs.should-run == 'true' || inputs.run-ethereum-exec-specs-remote == true) runs-on: ubuntu-latest timeout-minutes: 120 name: ethereum-exec-specs-remote (against latest bor) From af213d605bd47441c25a99594026052e72d67a6c Mon Sep 17 00:00:00 2001 From: Ji Hwan Date: Wed, 25 Mar 2026 15:22:56 +0900 Subject: [PATCH 08/18] refactor: use dropdown instead of boolean Signed-off-by: Ji Hwan --- .github/workflows/pos-e2e.yml | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/.github/workflows/pos-e2e.yml b/.github/workflows/pos-e2e.yml index c786b406..0583c076 100644 --- a/.github/workflows/pos-e2e.yml +++ b/.github/workflows/pos-e2e.yml @@ -43,21 +43,17 @@ on: required: false type: string default: "forks/amsterdam" - run-exec-specs: - description: "Manually trigger exec-specs matrix tests (bypasses release detection)." + run-tests: + description: "Select which test jobs to run (bypasses release detection for the selected jobs)." required: false - type: boolean - default: false - run-fork-trans: - description: "Manually trigger fork-transition matrix tests (bypasses release detection)." - required: false - type: boolean - default: false - run-ethereum-exec-specs-remote: - description: "Manually trigger ethereum execution-specs remote tests (bypasses release detection)." - required: false - type: boolean - default: false + type: choice + default: "all" + options: + - all + - exec-specs + - fork-trans + - matrix + - ethereum-exec-specs-remote force-run: description: "Run tests even if no new bor release is detected" required: false @@ -354,7 +350,8 @@ jobs: pos-execution-specs: needs: check-new-release if: >- - (needs.check-new-release.outputs.should-run == 'true' || inputs.run-exec-specs == true) && + (needs.check-new-release.outputs.should-run == 'true' || + inputs.run-tests == 'all' || inputs.run-tests == 'exec-specs' || inputs.run-tests == 'matrix') && needs.check-new-release.outputs.compat-pairs != '[]' runs-on: ubuntu-latest timeout-minutes: 60 @@ -573,7 +570,8 @@ jobs: pos-fork-transition: needs: check-new-release if: >- - (needs.check-new-release.outputs.should-run == 'true' || inputs.run-fork-trans == true) && + (needs.check-new-release.outputs.should-run == 'true' || + inputs.run-tests == 'all' || inputs.run-tests == 'fork-trans' || inputs.run-tests == 'matrix') && needs.check-new-release.outputs.compat-pairs != '[]' runs-on: ubuntu-latest timeout-minutes: 60 @@ -909,7 +907,8 @@ jobs: needs: [check-new-release, pos-execution-specs, pos-fork-transition] if: >- always() && - (needs.check-new-release.outputs.should-run == 'true' || inputs.run-ethereum-exec-specs-remote == true) + (needs.check-new-release.outputs.should-run == 'true' || + inputs.run-tests == 'all' || inputs.run-tests == 'ethereum-exec-specs-remote') runs-on: ubuntu-latest timeout-minutes: 120 name: ethereum-exec-specs-remote (against latest bor) From 03c3eb9586056b07c3c869e8a4d769c97932b884 Mon Sep 17 00:00:00 2001 From: Ji Hwan Date: Wed, 25 Mar 2026 15:34:22 +0900 Subject: [PATCH 09/18] chore: verbosity Signed-off-by: Ji Hwan --- .github/workflows/pos-e2e.yml | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pos-e2e.yml b/.github/workflows/pos-e2e.yml index 0583c076..89232a40 100644 --- a/.github/workflows/pos-e2e.yml +++ b/.github/workflows/pos-e2e.yml @@ -50,7 +50,7 @@ on: default: "all" options: - all - - exec-specs + - pos-exec-specs - fork-trans - matrix - ethereum-exec-specs-remote @@ -350,9 +350,10 @@ jobs: pos-execution-specs: needs: check-new-release if: >- - (needs.check-new-release.outputs.should-run == 'true' || - inputs.run-tests == 'all' || inputs.run-tests == 'exec-specs' || inputs.run-tests == 'matrix') && - needs.check-new-release.outputs.compat-pairs != '[]' + needs.check-new-release.outputs.compat-pairs != '[]' && ( + (github.event_name == 'schedule' && needs.check-new-release.outputs.should-run == 'true') || + (github.event_name == 'workflow_dispatch' && (inputs.run-tests == 'all' || inputs.run-tests == 'pos-exec-specs' || inputs.run-tests == 'matrix')) + ) runs-on: ubuntu-latest timeout-minutes: 60 strategy: @@ -570,9 +571,10 @@ jobs: pos-fork-transition: needs: check-new-release if: >- - (needs.check-new-release.outputs.should-run == 'true' || - inputs.run-tests == 'all' || inputs.run-tests == 'fork-trans' || inputs.run-tests == 'matrix') && - needs.check-new-release.outputs.compat-pairs != '[]' + needs.check-new-release.outputs.compat-pairs != '[]' && ( + (github.event_name == 'schedule' && needs.check-new-release.outputs.should-run == 'true') || + (github.event_name == 'workflow_dispatch' && (inputs.run-tests == 'all' || inputs.run-tests == 'fork-trans' || inputs.run-tests == 'matrix')) + ) runs-on: ubuntu-latest timeout-minutes: 60 strategy: @@ -906,9 +908,10 @@ jobs: ethereum-exec-specs-remote: needs: [check-new-release, pos-execution-specs, pos-fork-transition] if: >- - always() && - (needs.check-new-release.outputs.should-run == 'true' || - inputs.run-tests == 'all' || inputs.run-tests == 'ethereum-exec-specs-remote') + always() && ( + (github.event_name == 'schedule' && needs.check-new-release.outputs.should-run == 'true') || + (github.event_name == 'workflow_dispatch' && (inputs.run-tests == 'all' || inputs.run-tests == 'ethereum-exec-specs-remote')) + ) runs-on: ubuntu-latest timeout-minutes: 120 name: ethereum-exec-specs-remote (against latest bor) From 5e5e00cef570dca97b4b3d723d638b8393befe0a Mon Sep 17 00:00:00 2001 From: Ji Hwan Date: Wed, 25 Mar 2026 16:51:37 +0900 Subject: [PATCH 10/18] feat: add ignore flags for ethereum-exeuction-specs tests Signed-off-by: Ji Hwan --- .github/workflows/pos-e2e.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pos-e2e.yml b/.github/workflows/pos-e2e.yml index 89232a40..d70d97fd 100644 --- a/.github/workflows/pos-e2e.yml +++ b/.github/workflows/pos-e2e.yml @@ -1046,7 +1046,7 @@ jobs: -n 8 \ -s \ --tx-wait-timeout=60 \ - -k "not blob and not 4844 and not beacon_root and not mainnet and not genesis_hash_available and not test_selfdestruct_to_precompile and not test_selfdestruct_to_system_contract and not test_dynamic_create2_selfdestruct_collision_multi_tx and not test_extcodehash_of_empty and not test_call_memory_expands_on_early_revert and not type_3 and not address_0x000000000000000000000000000000000000000a and not address_0x000000000000000000000000000000000000000b and not address_0x000000000000000000000000000000000000000c and not address_0x000000000000000000000000000000000000000d and not address_0x000000000000000000000000000000000000000e and not address_0x000000000000000000000000000000000000000f and not address_0x0000000000000000000000000000000000000010 and not address_0x0000000000000000000000000000000000000011" \ + -k "not blob and not 4844 and not beacon_root and not mainnet and not genesis_hash_available and not test_selfdestruct_to_precompile and not test_selfdestruct_to_system_contract and not test_dynamic_create2_selfdestruct_collision_multi_tx and not test_extcodehash_of_empty and not test_call_memory_expands_on_early_revert and not type_3 and not address_0x000000000000000000000000000000000000000a and not address_0x000000000000000000000000000000000000000b and not address_0x000000000000000000000000000000000000000c and not address_0x000000000000000000000000000000000000000d and not address_0x000000000000000000000000000000000000000e and not address_0x000000000000000000000000000000000000000f and not address_0x0000000000000000000000000000000000000010 and not address_0x0000000000000000000000000000000000000011 and not test_create_nonce_overflow" \ --ignore=tests/frontier/opcodes/test_calldatacopy.py \ --ignore=tests/frontier/opcodes/test_dup.py \ --ignore=tests/frontier/opcodes/test_all_opcodes.py \ @@ -1061,12 +1061,15 @@ jobs: --ignore=tests/london/eip1559_fee_market_change/test_tx_type.py \ --ignore=tests/london/validation/test_header.py \ --ignore=tests/cancun/eip4788_beacon_root \ + --ignore=tests/cancun/eip5656_mcopy/test_mcopy_memory_expansion.py \ --ignore=tests/cancun/eip6780_selfdestruct \ --ignore=tests/cancun/create/test_create_oog_from_eoa_refunds.py \ --ignore=tests/cancun/eip1153_tstore/test_basic_tload.py \ --ignore=tests/cancun/eip1153_tstore/test_tload_calls.py \ --ignore=tests/cancun/eip1153_tstore/test_tstore_reentrancy.py \ --ignore=tests/cancun/eip1153_tstore/test_tload_reentrancy.py \ + --ignore=tests/cancun/eip1153_tstore/test_tstorage_create_contexts.py \ + --ignore=tests/constantinople/eip1052_extcodehash/test_extcodehash.py \ --ignore=tests/constantinople/eip145_bitwise_shift/test_shift_combinations.py \ --ignore=tests/prague/eip6110_deposits \ --ignore=tests/prague/eip7002_el_triggerable_withdrawals \ @@ -1074,6 +1077,7 @@ jobs: --ignore=tests/prague/eip7685_general_purpose_el_requests \ --ignore=tests/prague/eip2537_bls_12_381_precompiles \ --ignore=tests/prague/eip2935_historical_block_hashes_from_state \ + --ignore=tests/prague/eip7623_increase_calldata_cost \ --ignore=tests/prague/eip7702_set_code_tx \ --ignore=tests/shanghai/eip4895_withdrawals \ --ignore=tests/istanbul/eip1344_chainid/test_chainid.py \ From 08ef3e88a76ddb77d56178e700f68ae8d636869a Mon Sep 17 00:00:00 2001 From: Ji Hwan Date: Wed, 25 Mar 2026 17:53:57 +0900 Subject: [PATCH 11/18] fix: enable pairwise tests + fix scripts Signed-off-by: Ji Hwan --- .github/workflows/pos-e2e.yml | 20 +++++++++---------- .../pos-version-matrix/compat-versions.yml | 5 ++++- .../update-compat-versions.py | 4 ++-- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/.github/workflows/pos-e2e.yml b/.github/workflows/pos-e2e.yml index d70d97fd..eef72b03 100644 --- a/.github/workflows/pos-e2e.yml +++ b/.github/workflows/pos-e2e.yml @@ -250,11 +250,10 @@ jobs: - name: Generate version compatibility matrix id: gen-compat-matrix run: | - COMPAT_CSV="${{ steps.resolve.outputs.compat-versions }}" COMPAT_YML="scripts/pos-version-matrix/compat-versions.yml" # --- Collect bor and erigon version lists --- - # YAML has separate bor_versions / erigon_versions sections. + # YAML has flat `versions:` list with `el_type: bor|erigon` discriminator. declare -a bor_images=() declare -a erigon_images=() OVERRIDE="${{ steps.resolve.outputs.compat-versions-override }}" @@ -264,15 +263,16 @@ jobs: IFS=',' read -ra bor_images <<< "$OVERRIDE" elif [[ -f "$COMPAT_YML" ]]; then echo "Reading versions from ${COMPAT_YML}..." - in_bor=false; in_erigon=false + pending_img="" while IFS= read -r line; do - [[ "$line" =~ ^bor_versions: ]] && { in_bor=true; in_erigon=false; continue; } - [[ "$line" =~ ^erigon_versions: ]] && { in_erigon=true; in_bor=false; continue; } - [[ "$line" =~ ^[a-z] ]] && { in_bor=false; in_erigon=false; continue; } - if [[ "$line" =~ image:\ *\"?([^\"]+)\"? ]]; then - img="${BASH_REMATCH[1]}" - $in_bor && bor_images+=("$img") - $in_erigon && erigon_images+=("$img") + if [[ "$line" =~ image:\ *\"?([^\"[:space:]]+)\"? ]]; then + [[ -n "$pending_img" ]] && echo "::warning::Skipped image without el_type: ${pending_img}" + pending_img="${BASH_REMATCH[1]}" + elif [[ -n "$pending_img" && "$line" =~ el_type:\ *(bor|erigon) ]]; then + el_type="${BASH_REMATCH[1]}" + [[ "$el_type" == "bor" ]] && bor_images+=("$pending_img") + [[ "$el_type" == "erigon" ]] && erigon_images+=("$pending_img") + pending_img="" fi done < "$COMPAT_YML" fi diff --git a/scripts/pos-version-matrix/compat-versions.yml b/scripts/pos-version-matrix/compat-versions.yml index 7a8912cf..b051f00f 100644 --- a/scripts/pos-version-matrix/compat-versions.yml +++ b/scripts/pos-version-matrix/compat-versions.yml @@ -19,7 +19,10 @@ versions: reason: new pre-release (2.7 line), auto-detected - image: 0xpolygon/bor:2.6.5 el_type: bor - reason: new stable release (2.6 line), auto-detected + reason: latest stable release (2.6 line) +- image: 0xpolygon/bor:2.5.9 + el_type: bor + reason: latest stable release (2.5 line) - image: 0xpolygon/erigon:v3.5.0-beta2 el_type: erigon reason: new pre-release (3.5 line), auto-detected diff --git a/scripts/pos-version-matrix/update-compat-versions.py b/scripts/pos-version-matrix/update-compat-versions.py index d9ebe4a8..d44d3ab6 100644 --- a/scripts/pos-version-matrix/update-compat-versions.py +++ b/scripts/pos-version-matrix/update-compat-versions.py @@ -40,9 +40,9 @@ def get_headers() -> dict: return {} -def fetch_latest_releases(repo: str, max_results: int = 10) -> list: +def fetch_latest_releases(repo: str, max_results: int = 30) -> list: """Fetch recent non-draft releases with valid semver tags.""" - url = f"https://api.github.com/repos/{repo}/releases?per_page=30" + url = f"https://api.github.com/repos/{repo}/releases?per_page=50" resp = requests.get(url, timeout=15, headers=get_headers()) if resp.status_code != 200: print(f"Warning: failed to fetch releases for {repo} (HTTP {resp.status_code})") From c67d5ef2600121708bc1243924f9a418e9f036ae Mon Sep 17 00:00:00 2001 From: Ji Hwan Date: Wed, 25 Mar 2026 17:59:09 +0900 Subject: [PATCH 12/18] feat: cap 3 last major releases for erigon detection Signed-off-by: Ji Hwan --- scripts/pos-version-matrix/update-compat-versions.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/scripts/pos-version-matrix/update-compat-versions.py b/scripts/pos-version-matrix/update-compat-versions.py index d44d3ab6..dcf7557c 100644 --- a/scripts/pos-version-matrix/update-compat-versions.py +++ b/scripts/pos-version-matrix/update-compat-versions.py @@ -23,12 +23,14 @@ "el_type": "bor", "image_prefix": "0xpolygon/bor:", "strip_v": True, + "max_lines": 3, # Track 3 major.minor lines (e.g. 2.7, 2.6, 2.5) }, "erigon": { "repo": "0xPolygon/erigon", "el_type": "erigon", "image_prefix": "0xpolygon/erigon:", "strip_v": False, + "max_lines": 2, # Only latest lines — used as RPC endpoint, not pairwise tested }, } @@ -106,15 +108,19 @@ def update_compat_versions(compat_path: Path) -> list: if not releases: continue - # Group by major.minor line. + # Group by major.minor line (insertion order = newest first from API). lines: dict = {} for r in releases: mm = version_major_minor(r["tag"]) if mm not in lines: lines[mm] = r - # Check each major.minor line's latest release. - for mm, release in lines.items(): + # Limit to the N most recent major.minor lines. + max_lines = comp_config.get("max_lines", 3) + tracked_lines = dict(list(lines.items())[:max_lines]) + + # Check each tracked major.minor line's latest release. + for mm, release in tracked_lines.items(): image = make_image(release["tag"], comp_config) if image in current_images: continue From 50da62fa0cb790fb30b8739759a8465e517804da Mon Sep 17 00:00:00 2001 From: Ji Hwan Date: Fri, 27 Mar 2026 09:00:52 +0900 Subject: [PATCH 13/18] feat: skip bad pairs + clean up Signed-off-by: Ji Hwan --- .github/workflows/pos-e2e.yml | 38 ++++++++++++++++++- .../pos-version-matrix/compat-versions.yml | 26 +++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pos-e2e.yml b/.github/workflows/pos-e2e.yml index eef72b03..2e685f1e 100644 --- a/.github/workflows/pos-e2e.yml +++ b/.github/workflows/pos-e2e.yml @@ -306,10 +306,46 @@ jobs: '. + [{"bor_a":$va,"bor_b":$vb,"erigon_rpc":$eri,"pair_label":$lbl}]') fi - # Pairwise bor mixes. + # --- Load excluded pairs from compat-versions.yml --- + declare -A excluded_set + if [[ -f "$COMPAT_YML" ]]; then + # Parse excluded_pairs: collect image pairs as "imageA|imageB" keys. + # Both orderings are stored so lookup works regardless of pair order. + in_excluded=false + img_buf=() + while IFS= read -r line; do + if [[ "$line" =~ ^excluded_pairs: ]]; then + in_excluded=true; continue + fi + [[ "$in_excluded" != "true" ]] && continue + # Stop at next top-level key (non-indented, non-comment, non-blank). + if [[ "$line" =~ ^[a-zA-Z] ]]; then break; fi + if [[ "$line" =~ ^-\ +images: ]]; then + img_buf=(); continue + fi + if [[ "$line" =~ ^[[:space:]]+-\ +(.+)$ ]]; then + img="${BASH_REMATCH[1]}" + img=$(echo "$img" | sed 's/^["'"'"']//; s/["'"'"']$//') + img_buf+=("$img") + if [[ ${#img_buf[@]} -eq 2 ]]; then + excluded_set["${img_buf[0]}|${img_buf[1]}"]=1 + excluded_set["${img_buf[1]}|${img_buf[0]}"]=1 + echo " Excluded pair: ${img_buf[0]} <-> ${img_buf[1]}" + img_buf=() + fi + fi + done < "$COMPAT_YML" + echo "Loaded ${#excluded_set[@]} excluded pair entries." + fi + + # Pairwise bor mixes (skip excluded pairs). for ((i=0; i<${#bor_images[@]}; i++)); do for ((j=i+1; j<${#bor_images[@]}; j++)); do va="${bor_images[$i]}"; vb="${bor_images[$j]}" + if [[ -n "${excluded_set["${va}|${vb}"]:-}" ]]; then + echo " Skipping excluded pair: ${va} + ${vb}" + continue + fi la=$(echo "$va" | sed 's|.*/||; s/:/-/g'); lb=$(echo "$vb" | sed 's|.*/||; s/:/-/g') idx=$(( (i + j) % ${#erigon_images[@]} )) eri="${erigon_images[$idx]:-$erigon_latest}" diff --git a/scripts/pos-version-matrix/compat-versions.yml b/scripts/pos-version-matrix/compat-versions.yml index b051f00f..1960be9e 100644 --- a/scripts/pos-version-matrix/compat-versions.yml +++ b/scripts/pos-version-matrix/compat-versions.yml @@ -32,3 +32,29 @@ versions: - image: 0xpolygon/erigon:v3.3.7 el_type: erigon reason: new stable release (3.3 line), auto-detected + +# Excluded version pairs — combinations known to fail due to precompile or EVM +# behavior differences across major versions. These are skipped during pairwise +# matrix generation to avoid wasting CI resources on expected failures. +# +# Rule: always keep latest + second-latest release pairs — rolling upgrades +# across adjacent major versions must always work. +# +# Each entry needs: images (list of 2), reason, and link (CI run with failure). +excluded_pairs: +- images: + - 0xpolygon/bor:2.7.0-beta5 + - 0xpolygon/bor:2.5.9 + reason: >- + EVM behavior mismatch across 2 major versions (2.7 vs 2.5). Execution-specs + tests fail due to precompile and fork activation differences when validators + run incompatible versions. + link: https://github.com/agglayer/e2e/actions/runs/23532924942/job/68627382271 +- images: + - 0xpolygon/bor:2.6.5 + - 0xpolygon/bor:2.5.9 + reason: >- + EVM behavior mismatch (2.6 vs 2.5). Fork-transition tests hit BAD BLOCK + errors on archive sync; execution-specs tests fail due to inconsistent + precompile behavior across validators. + link: https://github.com/agglayer/e2e/actions/runs/23532924942/job/68627382263 From ef2b96889aa49708baf6ed3bb77ba0b4009b6947 Mon Sep 17 00:00:00 2001 From: Ji Hwan Date: Fri, 27 Mar 2026 08:55:18 +0900 Subject: [PATCH 14/18] fix: preserve excluded_pairs in compat-versions auto-updater The _write_compat_versions() function was hardcoded to write only the `versions` key, silently dropping any other top-level keys like `excluded_pairs`. This fixes it to carry over `excluded_pairs` when present, and documents the section in the file header comment. Co-Authored-By: Claude Opus 4.6 (1M context) --- scripts/pos-version-matrix/update-compat-versions.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/pos-version-matrix/update-compat-versions.py b/scripts/pos-version-matrix/update-compat-versions.py index dcf7557c..c9ae78ac 100644 --- a/scripts/pos-version-matrix/update-compat-versions.py +++ b/scripts/pos-version-matrix/update-compat-versions.py @@ -164,12 +164,18 @@ def _write_compat_versions(path: Path, data: dict): # # If this file is absent or `versions` is empty, the workflow falls back # to auto-detecting the latest bor release from each major.minor line. +# +# `excluded_pairs` (optional): version pairs to skip in pairwise testing. +# Each entry needs: images (list of 2), reason, and link. """ with open(path, "w") as f: f.write(header) + output = {"versions": data["versions"]} + if data.get("excluded_pairs"): + output["excluded_pairs"] = data["excluded_pairs"] yaml.dump( - {"versions": data["versions"]}, + output, f, default_flow_style=False, sort_keys=False, From cf7820cdfe7e7fdc9eeb94e5794853f855d40d53 Mon Sep 17 00:00:00 2001 From: Ji Hwan Date: Fri, 27 Mar 2026 08:58:27 +0900 Subject: [PATCH 15/18] fix: add excluded pairs parsing + division-by-zero guard in pair gen Add excluded_pairs parsing from compat-versions.yml with warnings for incomplete image buffers and trailing whitespace stripping. Guard erigon_images modulo against division by zero when the array is empty. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/pos-e2e.yml | 60 ++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/.github/workflows/pos-e2e.yml b/.github/workflows/pos-e2e.yml index 2e685f1e..0c132295 100644 --- a/.github/workflows/pos-e2e.yml +++ b/.github/workflows/pos-e2e.yml @@ -291,26 +291,9 @@ jobs: echo "Erigon versions (${#erigon_images[@]}):" printf ' %s\n' "${erigon_images[@]}" - # --- Generate test scenarios --- - # 1) Single-version baseline: all latest bor + latest erigon RPC - # 2) Pairwise bor mixes: validators split + erigon cross-client RPC - erigon_latest="${erigon_images[0]:-}" - scenarios="[]" - - # Single-version baseline (latest bor + latest erigon). - if [[ ${#bor_images[@]} -gt 0 ]]; then - la=$(echo "${bor_images[0]}" | sed 's|.*/||; s/:/-/g') - scenarios=$(echo "$scenarios" | jq -c \ - --arg va "${bor_images[0]}" --arg vb "${bor_images[0]}" \ - --arg eri "$erigon_latest" --arg lbl "${la}-single" \ - '. + [{"bor_a":$va,"bor_b":$vb,"erigon_rpc":$eri,"pair_label":$lbl}]') - fi - - # --- Load excluded pairs from compat-versions.yml --- + # --- Parse excluded pairs from compat-versions.yml --- declare -A excluded_set if [[ -f "$COMPAT_YML" ]]; then - # Parse excluded_pairs: collect image pairs as "imageA|imageB" keys. - # Both orderings are stored so lookup works regardless of pair order. in_excluded=false img_buf=() while IFS= read -r line; do @@ -318,14 +301,15 @@ jobs: in_excluded=true; continue fi [[ "$in_excluded" != "true" ]] && continue - # Stop at next top-level key (non-indented, non-comment, non-blank). if [[ "$line" =~ ^[a-zA-Z] ]]; then break; fi if [[ "$line" =~ ^-\ +images: ]]; then + if [[ ${#img_buf[@]} -gt 0 ]]; then + echo "::warning::Incomplete excluded pair (only ${#img_buf[@]} image(s)): ${img_buf[*]}" + fi img_buf=(); continue fi if [[ "$line" =~ ^[[:space:]]+-\ +(.+)$ ]]; then - img="${BASH_REMATCH[1]}" - img=$(echo "$img" | sed 's/^["'"'"']//; s/["'"'"']$//') + img=$(echo "${BASH_REMATCH[1]}" | sed 's/^["'"'"']//; s/["'"'"']$//; s/[[:space:]]*$//') img_buf+=("$img") if [[ ${#img_buf[@]} -eq 2 ]]; then excluded_set["${img_buf[0]}|${img_buf[1]}"]=1 @@ -335,20 +319,44 @@ jobs: fi fi done < "$COMPAT_YML" - echo "Loaded ${#excluded_set[@]} excluded pair entries." + if [[ ${#img_buf[@]} -gt 0 ]]; then + echo "::warning::Incomplete excluded pair at end of file (${#img_buf[@]} image(s)): ${img_buf[*]}" + fi + echo "Loaded $(( ${#excluded_set[@]} / 2 )) excluded pairs." fi - # Pairwise bor mixes (skip excluded pairs). + # --- Generate test scenarios --- + # 1) Single-version baseline: all latest bor + latest erigon RPC + # 2) Pairwise bor mixes: validators split + erigon cross-client RPC + erigon_latest="${erigon_images[0]:-}" + scenarios="[]" + + # Single-version baseline (latest bor + latest erigon). + if [[ ${#bor_images[@]} -gt 0 ]]; then + la=$(echo "${bor_images[0]}" | sed 's|.*/||; s/:/-/g') + scenarios=$(echo "$scenarios" | jq -c \ + --arg va "${bor_images[0]}" --arg vb "${bor_images[0]}" \ + --arg eri "$erigon_latest" --arg lbl "${la}-single" \ + '. + [{"bor_a":$va,"bor_b":$vb,"erigon_rpc":$eri,"pair_label":$lbl}]') + fi + + # Pairwise bor-bor mixes (skip excluded pairs). + # NOTE: Exclusion only applies to bor-bor pairs; cross-client pairs + # are not subject to excluded_pairs filtering. for ((i=0; i<${#bor_images[@]}; i++)); do for ((j=i+1; j<${#bor_images[@]}; j++)); do va="${bor_images[$i]}"; vb="${bor_images[$j]}" if [[ -n "${excluded_set["${va}|${vb}"]:-}" ]]; then - echo " Skipping excluded pair: ${va} + ${vb}" + echo " Skipping excluded pair: ${va} <-> ${vb}" continue fi la=$(echo "$va" | sed 's|.*/||; s/:/-/g'); lb=$(echo "$vb" | sed 's|.*/||; s/:/-/g') - idx=$(( (i + j) % ${#erigon_images[@]} )) - eri="${erigon_images[$idx]:-$erigon_latest}" + if [[ ${#erigon_images[@]} -gt 0 ]]; then + idx=$(( (i + j) % ${#erigon_images[@]} )) + eri="${erigon_images[$idx]}" + else + eri="$erigon_latest" + fi scenarios=$(echo "$scenarios" | jq -c \ --arg va "$va" --arg vb "$vb" --arg eri "$eri" --arg lbl "${la}-vs-${lb}" \ '. + [{"bor_a":$va,"bor_b":$vb,"erigon_rpc":$eri,"pair_label":$lbl}]') From 2dc130921075c3f3f839e0cf605e1d8bac792171 Mon Sep 17 00:00:00 2001 From: Ji Hwan Date: Fri, 27 Mar 2026 08:56:19 +0900 Subject: [PATCH 16/18] fix: make ethereum-exec-specs-remote job respect workflow cancellation Replace always() with !cancelled() in the job-level if condition so the job still runs after upstream jobs regardless of pass/fail, but correctly stops when the workflow is cancelled. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/pos-e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pos-e2e.yml b/.github/workflows/pos-e2e.yml index 0c132295..6c3fe0b0 100644 --- a/.github/workflows/pos-e2e.yml +++ b/.github/workflows/pos-e2e.yml @@ -952,7 +952,7 @@ jobs: ethereum-exec-specs-remote: needs: [check-new-release, pos-execution-specs, pos-fork-transition] if: >- - always() && ( + !cancelled() && ( (github.event_name == 'schedule' && needs.check-new-release.outputs.should-run == 'true') || (github.event_name == 'workflow_dispatch' && (inputs.run-tests == 'all' || inputs.run-tests == 'ethereum-exec-specs-remote')) ) From bd674a8c83d91e25e6c18f1ef1352de70a70900d Mon Sep 17 00:00:00 2001 From: Ji Hwan Date: Fri, 27 Mar 2026 09:18:00 +0900 Subject: [PATCH 17/18] chore: clean up Signed-off-by: Ji Hwan --- .../update-compat-versions.py | 41 +++++++++++++++---- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/scripts/pos-version-matrix/update-compat-versions.py b/scripts/pos-version-matrix/update-compat-versions.py index c9ae78ac..9cb40835 100644 --- a/scripts/pos-version-matrix/update-compat-versions.py +++ b/scripts/pos-version-matrix/update-compat-versions.py @@ -147,8 +147,34 @@ def update_compat_versions(compat_path: Path) -> list: return changes +def _read_excluded_pairs_tail(path: Path) -> str: + """Return everything from the first blank/comment line before + ``excluded_pairs:`` to end-of-file, preserving exact formatting. + + If there is no ``excluded_pairs`` section, returns ``""``.""" + if not path.exists(): + return "" + lines = path.read_text().splitlines(keepends=True) + ep_idx: Optional[int] = None + for i, line in enumerate(lines): + if line.rstrip() == "excluded_pairs:": + ep_idx = i + break + if ep_idx is None: + return "" + # Walk backwards to include the comment block above excluded_pairs. + start = ep_idx + while start > 0 and (lines[start - 1].startswith("#") or lines[start - 1].strip() == ""): + start -= 1 + return "".join(lines[start:]) + + def _write_compat_versions(path: Path, data: dict): - """Write compat-versions.yml preserving the header comment.""" + """Write compat-versions.yml, only touching the ``versions`` section. + + The ``excluded_pairs`` section (and its comment block) is preserved + verbatim from the original file — it is never serialised through + yaml.dump and must only be edited by hand.""" header = """\ # PoS version compatibility matrix — curated list of EL versions to test. # @@ -164,22 +190,21 @@ def _write_compat_versions(path: Path, data: dict): # # If this file is absent or `versions` is empty, the workflow falls back # to auto-detecting the latest bor release from each major.minor line. -# -# `excluded_pairs` (optional): version pairs to skip in pairwise testing. -# Each entry needs: images (list of 2), reason, and link. """ + # Capture the excluded_pairs tail before we overwrite the file. + tail = _read_excluded_pairs_tail(path) + with open(path, "w") as f: f.write(header) - output = {"versions": data["versions"]} - if data.get("excluded_pairs"): - output["excluded_pairs"] = data["excluded_pairs"] yaml.dump( - output, + {"versions": data["versions"]}, f, default_flow_style=False, sort_keys=False, ) + if tail: + f.write(tail) def main(): From dabb1ddf3936d1111769cd3096d7fd2882ce8ef2 Mon Sep 17 00:00:00 2001 From: Ji Hwan Date: Fri, 27 Mar 2026 09:21:53 +0900 Subject: [PATCH 18/18] chore: bump compat-versions Signed-off-by: Ji Hwan --- scripts/pos-version-matrix/compat-versions.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/pos-version-matrix/compat-versions.yml b/scripts/pos-version-matrix/compat-versions.yml index 1960be9e..8d6c4e4f 100644 --- a/scripts/pos-version-matrix/compat-versions.yml +++ b/scripts/pos-version-matrix/compat-versions.yml @@ -32,6 +32,12 @@ versions: - image: 0xpolygon/erigon:v3.3.7 el_type: erigon reason: new stable release (3.3 line), auto-detected +- image: 0xpolygon/bor:2.7.0 + el_type: bor + reason: new stable release (2.7 line), auto-detected +- image: 0xpolygon/erigon:v3.5.0 + el_type: erigon + reason: new stable release (3.5 line), auto-detected # Excluded version pairs — combinations known to fail due to precompile or EVM # behavior differences across major versions. These are skipped during pairwise