From eaee2fba5339ac71b474acfa5970bb5ae0d6d88c Mon Sep 17 00:00:00 2001 From: Kai Graeper Date: Fri, 13 Mar 2026 16:04:50 +0100 Subject: [PATCH 1/2] add unified workflow --- .github/workflows/build_and_test_autosd.yml | 15 +- .github/workflows/build_and_test_ebclfsa.yml | 15 +- .github/workflows/build_and_test_linux.yml | 12 +- .github/workflows/build_and_test_qnx.yml | 15 +- .github/workflows/nightly_integration.yml | 28 +++ .github/workflows/reusable_smoke-test-bak.yml | 185 +++++++++++++++++ .github/workflows/reusable_smoke-test.yml | 192 ++++++++++++------ .github/workflows/test_and_docs.yml | 15 +- .github/workflows/test_integration.yml | 16 +- scripts/integration_test.py | 12 +- .../known_good_to_workspace_metadata.py | 7 +- .../known_good/override_known_good_repo.py | 8 +- .../update_module_from_known_good.py | 27 ++- scripts/known_good/update_module_latest.py | 69 ++++--- 14 files changed, 461 insertions(+), 155 deletions(-) create mode 100644 .github/workflows/nightly_integration.yml create mode 100644 .github/workflows/reusable_smoke-test-bak.yml diff --git a/.github/workflows/build_and_test_autosd.yml b/.github/workflows/build_and_test_autosd.yml index 293b8eb21e4..4f62e37c5f3 100644 --- a/.github/workflows/build_and_test_autosd.yml +++ b/.github/workflows/build_and_test_autosd.yml @@ -12,13 +12,14 @@ # ******************************************************************************* name: AutoSD - Build & Integration Test on: - pull_request: - types: [opened, reopened, synchronize] - merge_group: - types: [checks_requested] - push: - branches: - - main + workflow_dispatch: + # pull_request: + # types: [opened, reopened, synchronize] + # merge_group: + # types: [checks_requested] + # push: + # branches: + # - main # Do not flood CI with unneeded previous runs in PR concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number }} diff --git a/.github/workflows/build_and_test_ebclfsa.yml b/.github/workflows/build_and_test_ebclfsa.yml index c0390351546..3944eb9cc32 100644 --- a/.github/workflows/build_and_test_ebclfsa.yml +++ b/.github/workflows/build_and_test_ebclfsa.yml @@ -12,13 +12,14 @@ # ******************************************************************************* name: EB corbos Linux for Safety Applications - Build & Integration Test on: - pull_request: - types: [opened, reopened, synchronize] - merge_group: - types: [checks_requested] - push: - branches: - - main + workflow_dispatch: + # pull_request: + # types: [opened, reopened, synchronize] + # merge_group: + # types: [checks_requested] + # push: + # branches: + # - main # Do not flood CI with unneeded previous runs in PR concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number }} diff --git a/.github/workflows/build_and_test_linux.yml b/.github/workflows/build_and_test_linux.yml index 39f9fa2527f..a0ede4647be 100644 --- a/.github/workflows/build_and_test_linux.yml +++ b/.github/workflows/build_and_test_linux.yml @@ -13,12 +13,12 @@ name: Linux - Build & Integration Test on: workflow_dispatch: - pull_request: - release: - types: [created] - push: - branches: - - main + # pull_request: + # release: + # types: [created] + # push: + # branches: + # - main # Do not flood CI with unneeded previous runs in PR concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number }} diff --git a/.github/workflows/build_and_test_qnx.yml b/.github/workflows/build_and_test_qnx.yml index b45b09e1420..06292e2b33c 100644 --- a/.github/workflows/build_and_test_qnx.yml +++ b/.github/workflows/build_and_test_qnx.yml @@ -12,13 +12,14 @@ # ******************************************************************************* name: QNX8 - Build & Integration Test on: - pull_request_target: - types: [opened, reopened, synchronize] - merge_group: - types: [checks_requested] - push: - branches: - - main + workflow_dispatch: + # pull_request_target: + # types: [opened, reopened, synchronize] + # merge_group: + # types: [checks_requested] + # push: + # branches: + # - main # Do not flood CI with unneeded previous runs in PR concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number }} diff --git a/.github/workflows/nightly_integration.yml b/.github/workflows/nightly_integration.yml new file mode 100644 index 00000000000..66efc79adc3 --- /dev/null +++ b/.github/workflows/nightly_integration.yml @@ -0,0 +1,28 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +# Nightly / manual integration tests: updates all modules to latest HEAD on their branch. + +name: Integration Tests (Nightly) +on: + workflow_dispatch: + schedule: + - cron: '30 2 * * *' # Every night at 02:30 UTC on main-etas branch +jobs: + integration_test: + uses: ./.github/workflows/reusable_smoke-test.yml + secrets: inherit + with: + repo_runner_labels: 'standard' + update_to_latest: true + target_branch: '${{ github.ref }}' diff --git a/.github/workflows/reusable_smoke-test-bak.yml b/.github/workflows/reusable_smoke-test-bak.yml new file mode 100644 index 00000000000..afc095df06d --- /dev/null +++ b/.github/workflows/reusable_smoke-test-bak.yml @@ -0,0 +1,185 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +# Module Smoke Test Workflow +# +# Summary: +# Orchestrates a fast validation of a single module by producing an +# updated known_good.json (optionally overriding one module to the current PR) +# and then invoking the integration build workflow to compile and test. +# +# What it does: +# - Checks out the reference integration repository +# - Generates known_good.updated.json: +# * If testing an external module PR: overrides `module_name` to the PR SHA +# * If testing this repo: updates modules to latest branches +# - Uploads known_good.updated.json as an artifact +# - Calls the Module Integration Build workflow with a matrix of configs +# - Publishes a summary to the GitHub Step Summary +# +# Inputs: +# - repo_runner_labels (string, required, default: ubuntu-latest): Runner label. +# - module_name (string, required): Module to override (e.g., score_baselibs). +# - target_branch (string, required, default: main): +# The ref to checkout via actions/checkout — can be a branch name, tag, or +# commit SHA. This ensures the workflow uses the exact version of the +# integration files you intend. +# +# Repository Variables: +# - reference_integration_repo (optional): Repository providing integration +# workflows and tools (format: owner/repo). Supports private forks. +# Default: eclipse-score/reference_integration +# +# Secrets: +# - REPO_READ_TOKEN (optional): Token for reading private repos; falls back to +# github.token when not provided. +# +# Usage: +# This workflow is reusable and triggered via workflow_call from other workflows. +# Example: +# jobs: +# smoke: +# uses: eclipse-score/reference_integration/.github/workflows/module-smoke-test.yml@main +# with: +# repo_runner_labels: ubuntu-latest +# module_name: score_baselibs +# target_branch: main +# secrets: +# REPO_READ_TOKEN: ${{ secrets.REPO_READ_TOKEN }} +# +# Note: Set the 'reference_integration_repo' repository variable to use a +# private fork (e.g., my-org/reference_integration). +# +# Notes: +# - Extend the matrix in `integration-test` to cover additional configs. +name: Module Smoke Test +on: + workflow_call: + inputs: + repo_runner_labels: + description: 'The runner tag to use for the job' + required: true + type: string + default: 'ubuntu-latest' + module_name: + description: 'Name of the module to override (e.g., score_baselibs).' + required: true + type: string + target_branch: + description: 'Ref to checkout (branch, tag, or commit SHA).' + required: true + type: string + default: 'main' + secrets: + REPO_READ_TOKEN: + description: 'Token for reading repositories' + required: false +env: + REFERENCE_INTEGRATION_REPO: ${{ vars.reference_integration_repo != '' && vars.reference_integration_repo || 'eclipse-score/reference_integration' }} +jobs: + preparation: + name: Preparation + runs-on: ubuntu-latest + outputs: + known_good_updated: ${{ steps.set_known_good.outputs.known_good_updated }} + steps: + - name: Checkout repository + uses: actions/checkout@v4.2.2 + with: + repository: ${{ env.REFERENCE_INTEGRATION_REPO }} + ref: ${{ inputs.target_branch }} + token: ${{ secrets.REPO_READ_TOKEN != '' && secrets.REPO_READ_TOKEN || github.token }} + - name: Create updated known_good.json with PR commit + id: set_known_good + run: | + if [ "${{ github.repository }}" != "${{ env.REFERENCE_INTEGRATION_REPO }}" ]; then + echo "Overriding ${{ inputs.module_name }} with current PR" + python3 scripts/known_good/override_known_good_repo.py \ + --known known_good.json \ + --output known_good.updated.json \ + --module-override ${{ inputs.module_name }}@${{ github.event.repository.clone_url }}@${{ github.sha }} + else + echo "Testing reference integration repository itself - updating to latest commits" + echo "::group::get latest commits from module branches" + python3 scripts/known_good/update_module_latest.py --output known_good.updated.json + cat known_good.updated.json + echo "::endgroup::" + fi + + # Output the content as a JSON string + echo "known_good_updated<> $GITHUB_OUTPUT + cat known_good.updated.json >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + env: + GITHUB_TOKEN: ${{ secrets.REPO_READ_TOKEN != '' && secrets.REPO_READ_TOKEN || github.token }} + - name: Show updated known_good.json + run: | + echo "Updated known_good.json:" + cat known_good.updated.json + - name: Upload updated known_good.json artifact + uses: actions/upload-artifact@v4 + with: + name: known_good.updated.json + path: known_good.updated.json + docs: + name: Generate Documentation + runs-on: ubuntu-latest + needs: preparation + steps: + - name: not implemented + run: echo "Documentation generation not yet implemented here." + integration-test: + name: Integration Testing (${{ matrix.config }}) + needs: preparation + strategy: + fail-fast: false + matrix: + config: + - x86_64-linux + # - arm64-linux + # Add more configs here as needed + # - arm64-qnx8_0 + # - x86_64-qnx8_0 + uses: ./.github/workflows/reusable_integration-build.yml + secrets: inherit + with: + known_good: ${{ needs.preparation.outputs.known_good_updated }} + config: ${{ matrix.config }} + repo_runner_labels: ${{ inputs.repo_runner_labels }} + target_branch: ${{ inputs.target_branch }} + summary: + name: Publish Summary + runs-on: ubuntu-latest + needs: [integration-test, docs] + if: always() + steps: + - name: Checkout repository + uses: actions/checkout@v4.2.2 + with: + repository: ${{ env.REFERENCE_INTEGRATION_REPO }} + ref: ${{ inputs.target_branch }} + token: ${{ secrets.REPO_READ_TOKEN != '' && secrets.REPO_READ_TOKEN || github.token }} + # get all artefacts from integration-test job with name bazel-build-logs-* + - name: Download Integration Test Artifacts + uses: actions/download-artifact@v4 + with: + pattern: bazel-build-logs-* + path: _logs/integration_test_logs + merge-multiple: true + - name: Publish Integration Test Summary + run: | + ls -l _logs/integration_test_logs || true + python3 scripts/publish_integration_summary.py \ + --integration-result "${{ needs.integration-test.result }}" \ + --docs-result "${{ needs.docs.result }}" \ + --logs-dir "_logs/integration_test_logs" \ + >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/reusable_smoke-test.yml b/.github/workflows/reusable_smoke-test.yml index afc095df06d..aaf3714ebe3 100644 --- a/.github/workflows/reusable_smoke-test.yml +++ b/.github/workflows/reusable_smoke-test.yml @@ -1,34 +1,30 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* # Module Smoke Test Workflow # # Summary: -# Orchestrates a fast validation of a single module by producing an -# updated known_good.json (optionally overriding one module to the current PR) +# Orchestrates validation of modules by producing a known_good.updated.json # and then invoking the integration build workflow to compile and test. # +# Modes of operation (controlled by inputs): +# 1. External module PR (github.repository != reference_integration_repo): +# Overrides `module_name` to the PR commit SHA. +# 2. Nightly / latest (update_to_latest=true): +# Updates all modules to latest HEAD on their branch +# (via update_module_latest.py). +# 3. PR / pinned (update_to_latest=false, default): +# Uses the repository's known_good.json as-is. +# # What it does: # - Checks out the reference integration repository -# - Generates known_good.updated.json: -# * If testing an external module PR: overrides `module_name` to the PR SHA -# * If testing this repo: updates modules to latest branches +# - Generates known_good.updated.json (see modes above) # - Uploads known_good.updated.json as an artifact # - Calls the Module Integration Build workflow with a matrix of configs # - Publishes a summary to the GitHub Step Summary # # Inputs: # - repo_runner_labels (string, required, default: ubuntu-latest): Runner label. -# - module_name (string, required): Module to override (e.g., score_baselibs). +# - module_name (string, optional): Module to override for external PRs. +# - update_to_latest (boolean, optional, default: false): When true, updates +# all modules to the latest commit on their branch. Used for nightly builds. # - target_branch (string, required, default: main): # The ref to checkout via actions/checkout — can be a branch name, tag, or # commit SHA. This ensures the workflow uses the exact version of the @@ -45,23 +41,40 @@ # # Usage: # This workflow is reusable and triggered via workflow_call from other workflows. -# Example: +# +# External module PR: # jobs: # smoke: -# uses: eclipse-score/reference_integration/.github/workflows/module-smoke-test.yml@main +# uses: eclipse-score/reference_integration/.github/workflows/reusable_smoke-test.yml@main # with: -# repo_runner_labels: ubuntu-latest # module_name: score_baselibs # target_branch: main # secrets: # REPO_READ_TOKEN: ${{ secrets.REPO_READ_TOKEN }} # +# Nightly (latest HEAD): +# jobs: +# nightly: +# uses: ./.github/workflows/reusable_smoke-test.yml +# with: +# update_to_latest: true +# target_branch: main +# +# PR (pinned known_good.json): +# jobs: +# pr: +# uses: ./.github/workflows/reusable_smoke-test.yml +# with: +# target_branch: ${{ github.ref }} +# # Note: Set the 'reference_integration_repo' repository variable to use a # private fork (e.g., my-org/reference_integration). # # Notes: # - Extend the matrix in `integration-test` to cover additional configs. + name: Module Smoke Test + on: workflow_call: inputs: @@ -71,9 +84,15 @@ on: type: string default: 'ubuntu-latest' module_name: - description: 'Name of the module to override (e.g., score_baselibs).' - required: true + description: 'Name of the module to override (e.g., score_baselibs). Required for external module PRs.' + required: false type: string + default: '' + update_to_latest: + description: 'When true, update all modules to latest HEAD on their branch (nightly mode).' + required: false + type: boolean + default: false target_branch: description: 'Ref to checkout (branch, tag, or commit SHA).' required: true @@ -83,8 +102,10 @@ on: REPO_READ_TOKEN: description: 'Token for reading repositories' required: false + env: REFERENCE_INTEGRATION_REPO: ${{ vars.reference_integration_repo != '' && vars.reference_integration_repo || 'eclipse-score/reference_integration' }} + jobs: preparation: name: Preparation @@ -98,23 +119,29 @@ jobs: repository: ${{ env.REFERENCE_INTEGRATION_REPO }} ref: ${{ inputs.target_branch }} token: ${{ secrets.REPO_READ_TOKEN != '' && secrets.REPO_READ_TOKEN || github.token }} - - name: Create updated known_good.json with PR commit + - name: Create updated known_good.json id: set_known_good run: | if [ "${{ github.repository }}" != "${{ env.REFERENCE_INTEGRATION_REPO }}" ]; then + # External module PR: override the specific module to the PR commit echo "Overriding ${{ inputs.module_name }} with current PR" python3 scripts/known_good/override_known_good_repo.py \ --known known_good.json \ --output known_good.updated.json \ --module-override ${{ inputs.module_name }}@${{ github.event.repository.clone_url }}@${{ github.sha }} - else - echo "Testing reference integration repository itself - updating to latest commits" + elif [ "${{ inputs.update_to_latest }}" = "true" ]; then + # Nightly mode: update all modules to latest HEAD on their branch + echo "Updating all modules to latest commits" echo "::group::get latest commits from module branches" python3 scripts/known_good/update_module_latest.py --output known_good.updated.json cat known_good.updated.json echo "::endgroup::" + else + # PR mode: use the pinned known_good.json from the repository + echo "Using pinned known_good.json from repository" + cp known_good.json known_good.updated.json fi - + # Output the content as a JSON string echo "known_good_updated<> $GITHUB_OUTPUT cat known_good.updated.json >> $GITHUB_OUTPUT @@ -130,56 +157,87 @@ jobs: with: name: known_good.updated.json path: known_good.updated.json - docs: - name: Generate Documentation + + integration-build: + name: build S-CORE (${{ matrix.config }}) runs-on: ubuntu-latest needs: preparation - steps: - - name: not implemented - run: echo "Documentation generation not yet implemented here." - integration-test: - name: Integration Testing (${{ matrix.config }}) - needs: preparation strategy: fail-fast: false matrix: config: - - x86_64-linux - # - arm64-linux + - linux-x86_64 + # - linux-arm64 # Add more configs here as needed - # - arm64-qnx8_0 - # - x86_64-qnx8_0 - uses: ./.github/workflows/reusable_integration-build.yml - secrets: inherit - with: - known_good: ${{ needs.preparation.outputs.known_good_updated }} - config: ${{ matrix.config }} - repo_runner_labels: ${{ inputs.repo_runner_labels }} - target_branch: ${{ inputs.target_branch }} + - qnx-x86_64 + - qnx-arm64 + steps: + - name: utest ${{ matrix.config }} + run: echo "run reusable_integration-build.yml" + + utest: + name: Integration Testing (${{ matrix.config }}) + runs-on: ubuntu-latest + needs: integration-build + steps: + - name: build ${{ matrix.config }} + run: echo "run test_and_docs.yml" + # maybe should be included in integration-build to make use of the cache + + docs: + name: Generate Documentation + runs-on: ubuntu-latest + needs: utest + steps: + - name: not implemented + run: echo "separate doc workflow if we have a matrix build for utest " + + build_and_test_autosd: + name: build_and_test_autosd + runs-on: ubuntu-latest + needs: docs + steps: + - name: not implemented + run: echo "run build_and_test_autosd.yml" + + build_and_test_ebclfsa: + name: build_and_test_ebclfsa + runs-on: ubuntu-latest + needs: docs + steps: + - name: not implemented + run: echo "run build_and_test_ebclfsa.yml" + + build_and_test_linux: + name: build_and_test_linux + runs-on: ubuntu-latest + needs: docs + steps: + - name: not implemented + run: echo "run build_and_test_linux.yml" + + build_and_test_qnx: + name: build_and_test_qnx + runs-on: ubuntu-latest + needs: docs + steps: + - name: not implemented + run: echo "run build_and_test_qnx.yml" + + update_known_good: + name: update_known_good + runs-on: ubuntu-latest + needs: [build_and_test_autosd, build_and_test_ebclfsa, build_and_test_linux, build_and_test_qnx] + steps: + - name: not implemented + run: echo "run update_known_good" + summary: name: Publish Summary runs-on: ubuntu-latest - needs: [integration-test, docs] + needs: update_known_good if: always() steps: - - name: Checkout repository - uses: actions/checkout@v4.2.2 - with: - repository: ${{ env.REFERENCE_INTEGRATION_REPO }} - ref: ${{ inputs.target_branch }} - token: ${{ secrets.REPO_READ_TOKEN != '' && secrets.REPO_READ_TOKEN || github.token }} - # get all artefacts from integration-test job with name bazel-build-logs-* - - name: Download Integration Test Artifacts - uses: actions/download-artifact@v4 - with: - pattern: bazel-build-logs-* - path: _logs/integration_test_logs - merge-multiple: true - - name: Publish Integration Test Summary + - name: summary run: | - ls -l _logs/integration_test_logs || true - python3 scripts/publish_integration_summary.py \ - --integration-result "${{ needs.integration-test.result }}" \ - --docs-result "${{ needs.docs.result }}" \ - --logs-dir "_logs/integration_test_logs" \ - >> "$GITHUB_STEP_SUMMARY" + echo "run publish_integration_summary.py" diff --git a/.github/workflows/test_and_docs.yml b/.github/workflows/test_and_docs.yml index ad27a1fcb00..41626c57372 100644 --- a/.github/workflows/test_and_docs.yml +++ b/.github/workflows/test_and_docs.yml @@ -17,13 +17,14 @@ permissions: pull-requests: write id-token: write on: - pull_request_target: # Allows forks to trigger the docs build - types: [opened, reopened, synchronize] - push: - branches: - - main - merge_group: - types: [checks_requested] + workflow_dispatch: + # pull_request_target: # Allows forks to trigger the docs build + # types: [opened, reopened, synchronize] + # push: + # branches: + # - main + # merge_group: + # types: [checks_requested] release: types: [created] # Do not flood CI with unneeded previous runs in PR diff --git a/.github/workflows/test_integration.yml b/.github/workflows/test_integration.yml index 34a961e529f..04e987e1517 100644 --- a/.github/workflows/test_integration.yml +++ b/.github/workflows/test_integration.yml @@ -12,19 +12,19 @@ # ******************************************************************************* # Workflow configuration for S-CORE CI - Bazel Build & Test baselibs # This workflow runs Bazel build and test when triggered by specific pull request events. -name: build latest mains + +name: Integration Tests (PR) on: workflow_dispatch: - pull_request: + pull_request_target: + types: [opened, reopened, synchronize] push: - # schedule: - # - cron: '30 2 * * *' # Every night at 02:30 UTC on main branch + # branches: + # - main-etas jobs: integration_test: - if: false # Temporarily disabled until we get things sorted out uses: ./.github/workflows/reusable_smoke-test.yml secrets: inherit with: - repo_runner_labels: 'ubuntu-latest' - module_name: 'reference_integration' - target_branch: ${{ github.ref }} + repo_runner_labels: 'standard' + target_branch: '${{ github.ref }}' \ No newline at end of file diff --git a/scripts/integration_test.py b/scripts/integration_test.py index 219905689c7..0bb8964f6a4 100755 --- a/scripts/integration_test.py +++ b/scripts/integration_test.py @@ -300,14 +300,20 @@ def main(): log_dir.mkdir(parents=True, exist_ok=True) summary_file.parent.mkdir(parents=True, exist_ok=True) - # Load modules from known_good files + # Load modules from known_good files (flatten grouped structure to flat module dict) + def _flatten_modules(grouped: Dict[str, Dict[str, Module]]) -> Dict[str, Module]: + flat: Dict[str, Module] = {} + for group_modules in grouped.values(): + flat.update(group_modules) + return flat + try: - old_modules = load_known_good(Path("known_good.json")).modules if Path("known_good.json").exists() else {} + old_modules = _flatten_modules(load_known_good(Path("known_good.json")).modules) if Path("known_good.json").exists() else {} except FileNotFoundError: old_modules = {} try: - new_modules = load_known_good(known_good_file).modules if known_good_file else {} + new_modules = _flatten_modules(load_known_good(known_good_file).modules) if known_good_file else {} except FileNotFoundError as e: raise SystemExit(f"ERROR: {e}") diff --git a/scripts/known_good/known_good_to_workspace_metadata.py b/scripts/known_good/known_good_to_workspace_metadata.py index bb933c8dcca..af7a9d75eb4 100644 --- a/scripts/known_good/known_good_to_workspace_metadata.py +++ b/scripts/known_good/known_good_to_workspace_metadata.py @@ -42,10 +42,13 @@ def main(): except ValueError as e: raise SystemExit(f"ERROR: {e}") - modules = list(known_good.modules.values()) + # Flatten modules from all groups + all_modules = [] + for group_modules in known_good.modules.values(): + all_modules.extend(group_modules.values()) gita_metadata = [] - for module in modules: + for module in all_modules: if not module.repo: raise RuntimeError(f"Module {module.name}: repo must not be empty") diff --git a/scripts/known_good/override_known_good_repo.py b/scripts/known_good/override_known_good_repo.py index 218d360d0a3..1b678ef42d2 100755 --- a/scripts/known_good/override_known_good_repo.py +++ b/scripts/known_good/override_known_good_repo.py @@ -142,8 +142,14 @@ def apply_overrides(known_good: KnownGood, repo_overrides: List[str]) -> KnownGo Returns: Updated KnownGood instance """ + # Flatten grouped modules into flat dict for override lookup + # (Module instances are shared by reference, so changes propagate back to known_good) + flat_modules: Dict[str, Module] = {} + for group_modules in known_good.modules.values(): + flat_modules.update(group_modules) + # Parse and apply overrides - overrides_applied = parse_and_apply_overrides(known_good.modules, repo_overrides) + overrides_applied = parse_and_apply_overrides(flat_modules, repo_overrides) if overrides_applied == 0: logging.warning("No overrides were applied to any modules") diff --git a/scripts/known_good/update_module_from_known_good.py b/scripts/known_good/update_module_from_known_good.py index e6c82fe3cb3..c6836d69df9 100755 --- a/scripts/known_good/update_module_from_known_good.py +++ b/scripts/known_good/update_module_from_known_good.py @@ -42,7 +42,9 @@ logging.basicConfig(level=logging.WARNING, format="%(levelname)s: %(message)s") -def generate_git_override_blocks(modules: List[Module], repo_commit_dict: Dict[str, str]) -> List[str]: +def generate_git_override_blocks( + modules: List[Module], repo_commit_dict: Dict[str, str] +) -> List[str]: """Generate bazel_dep and git_override blocks for each module.""" blocks = [] @@ -59,8 +61,7 @@ def generate_git_override_blocks(modules: List[Module], repo_commit_dict: Dict[s patches_lines = " patches = [\n" for patch in module.bazel_patches: patches_lines += f' "{patch}",\n' - patches_lines += " ],\n" - patch_strip_line = " patch_strip = 1,\n" if patches_lines else "" + patches_lines += " ],\n patch_strip = 1,\n" if module.version: # If version is provided, use bazel_dep with single_version_override @@ -68,9 +69,8 @@ def generate_git_override_blocks(modules: List[Module], repo_commit_dict: Dict[s f'bazel_dep(name = "{module.name}")\n' "single_version_override(\n" f' module_name = "{module.name}",\n' - f"{patch_strip_line}" - f"{patches_lines}" f' version = "{module.version}",\n' + f"{patches_lines}" ")\n" ) else: @@ -93,17 +93,16 @@ def generate_git_override_blocks(modules: List[Module], repo_commit_dict: Dict[s continue # If no version, use bazel_dep with git_override - # Only include patch_strip if there are patches to apply block = ( f'bazel_dep(name = "{module.name}")\n' "git_override(\n" f' module_name = "{module.name}",\n' + f' remote = "{module.repo}",\n' f' commit = "{commit}",\n' - f"{patch_strip_line}" f"{patches_lines}" - f' remote = "{module.repo}",\n' ")\n" ) + blocks.append(block) return blocks @@ -129,7 +128,7 @@ def generate_local_override_blocks(modules: List[Module]) -> List[str]: def generate_coverage_blocks(modules: List[Module]) -> List[str]: """Generate rust_coverage_report blocks for each module with rust impl.""" - blocks = ["""load("@score_tooling//:defs.bzl", "rust_coverage_report")"""] + blocks = ["""load("@score_tooling//:defs.bzl", "rust_coverage_report")\n\n"""] for module in modules: if "rust" not in module.metadata.langs: @@ -251,7 +250,11 @@ def main() -> None: action="store_true", help="Print generated content instead of writing to file", ) - parser.add_argument("-v", "--verbose", action="store_true", help="Enable verbose logging") + parser.add_argument( + "-v", "--verbose", + action="store_true", + help="Enable verbose logging" + ) parser.add_argument( "--repo-override", action="append", @@ -277,7 +280,9 @@ def main() -> None: # Parse repo overrides repo_commit_dict = {} if args.repo_override: - repo_pattern = re.compile(r"https://[a-zA-Z0-9.-]+/[a-zA-Z0-9._/-]+\.git@[a-fA-F0-9]{7,40}$") + repo_pattern = re.compile( + r"https://[a-zA-Z0-9.-]+/[a-zA-Z0-9._/-]+\.git@[a-fA-F0-9]{7,40}$" + ) for entry in args.repo_override: if not repo_pattern.match(entry): raise SystemExit( diff --git a/scripts/known_good/update_module_latest.py b/scripts/known_good/update_module_latest.py index 33e238c4f17..4b42de4e3c7 100755 --- a/scripts/known_good/update_module_latest.py +++ b/scripts/known_good/update_module_latest.py @@ -22,8 +22,8 @@ Usage: python tools/update_module_latest.py \ - --known-good score_reference_integration/known_good.json \ - [--branch main] [--output updated_known_good.json] + --known-good score_reference_integration/known_good.json \ + [--branch main] [--output updated_known_good.json] Environment: Optionally set GITHUB_TOKEN to increase rate limits / access private repos. @@ -48,7 +48,6 @@ try: from github import Github, GithubException - HAS_PYGITHUB = True except ImportError: HAS_PYGITHUB = False @@ -150,33 +149,45 @@ def main(argv: list[str]) -> int: if args.no_gh and shutil.which("gh") is not None: print("INFO: --no-gh specified; ignoring installed 'gh' CLI", file=sys.stderr) - for mod in known_good.modules.values(): - if mod.pin_version: - print(f"{mod.name}: pinned, skipping") - continue - - try: - branch = mod.branch if mod.branch else args.branch - if use_gh: - latest = fetch_latest_commit_gh(mod.owner_repo, branch) - else: - latest = fetch_latest_commit(mod.owner_repo, branch, token) - - old_hash = mod.hash - if latest != old_hash: - mod.hash = latest - mod.version = None # Clear version when hash changes - if mod.version: - print(f"{mod.name}: {mod.version} -> {latest[:8]} (branch {branch})") + for group_name, group_modules in known_good.modules.items(): + for mod in group_modules.values(): + if mod.pin_version: + print(f"{mod.name}: pinned, skipping") + continue + + # Skip version-only modules (single_version_override, no git hash) + if mod.version and not mod.hash: + print(f"{mod.name}: version override ({mod.version}), skipping") + continue + + # Skip modules without a repo URL + if not mod.repo: + print(f"{mod.name}: no repo URL, skipping") + continue + + try: + branch = mod.branch if mod.branch else args.branch + if use_gh: + latest = fetch_latest_commit_gh(mod.owner_repo, branch) + else: + latest = fetch_latest_commit(mod.owner_repo, branch, token) + + old_hash = mod.hash + if latest != old_hash: + old_version = mod.version + mod.hash = latest + mod.version = None # Clear version when hash changes + if old_version: + print(f"{mod.name}: {old_version} -> {latest[:8]} (branch {branch})") + else: + print(f"{mod.name}: {old_hash[:8]} -> {latest[:8]} (branch {branch})") else: - print(f"{mod.name}: {old_hash[:8]} -> {latest[:8]} (branch {branch})") - else: - print(f"{mod.name}: {old_hash[:8]} (no update)") - except Exception as e: # noqa: BLE001 - failures += 1 - print(f"ERROR {mod.name}: {e}", file=sys.stderr) - if args.fail_fast: - break + print(f"{mod.name}: {old_hash[:8]} (no update)") + except Exception as e: # noqa: BLE001 + failures += 1 + print(f"ERROR {mod.name}: {e}", file=sys.stderr) + if args.fail_fast: + break if args.output: try: From 47ab654ebf728e20551f3ffff2b848372f5fa2f6 Mon Sep 17 00:00:00 2001 From: Kai Graeper Date: Fri, 13 Mar 2026 16:21:53 +0100 Subject: [PATCH 2/2] test workflow --- .github/workflows/test_integration.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_integration.yml b/.github/workflows/test_integration.yml index 04e987e1517..73a61d7cf6b 100644 --- a/.github/workflows/test_integration.yml +++ b/.github/workflows/test_integration.yml @@ -16,7 +16,8 @@ name: Integration Tests (PR) on: workflow_dispatch: - pull_request_target: + # pull_request_target: + pull_request: types: [opened, reopened, synchronize] push: # branches: