From e9c86f7c595f2e67f26c72b8fe086566c0b389ab Mon Sep 17 00:00:00 2001 From: Saumya-R Date: Wed, 11 Mar 2026 11:19:56 +0530 Subject: [PATCH 1/4] codeql: removing the coding-standard repo adding conditional jq chore: extend codeowners list (#175) * chore: extend codeowners list * fix: no commas Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Alexander Lanin --------- Signed-off-by: Alexander Lanin Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Integrates datarouter (#178) * Integrates datarouter - Deploys datarouetr in linux x86_64 and qnx_x86_64 image - Multicast route added to qnx guest in network_setup_dhcp.sh required for capturing DLTs. - dlt_config added for x86_64 and qnx x86_64 targets that includes the vlan_address updated for dlt_receive to capture untagged ethernet frames on tap0 interface * Adds datarouter test_remote_logging integration test - Adds test_remote_logging integration test that verifies remote logging of datarouter and network config. of the linux x86_64 and qnx qemu image by capturing DLT logs on the host. Use py_itf_test rule with select support (#177) - Update score_itf to 44c75de and score_bazel_platforms to b72a5c3 in known_good.json and MODULE.bazel overrides - Add explicit bazel_dep on platforms 1.0.0 (required by new config_settings) - Merge the two py_itf_test targets (linux_x86_64, qnx_x86_64) in feature_integration_tests/itf/BUILD into a single 'itf' target that uses config_setting + select() to switch args/data/plugins per platform; keep backward-compatible aliases for the old target names - Upgrade rules_oci from 1.8.0 to 2.2.7; adapt docker images to use oci_load (replacing the removed oci_tarball) and add the platform-specific repo variants revert the filters to test adding jd check and filter regex matches relative path removing the filter to test codeql: removing the coding-standard repo adding conditional jq chore: extend codeowners list (#175) * chore: extend codeowners list * fix: no commas Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Alexander Lanin --------- Signed-off-by: Alexander Lanin Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Integrates datarouter (#178) * Integrates datarouter - Deploys datarouetr in linux x86_64 and qnx_x86_64 image - Multicast route added to qnx guest in network_setup_dhcp.sh required for capturing DLTs. - dlt_config added for x86_64 and qnx x86_64 targets that includes the vlan_address updated for dlt_receive to capture untagged ethernet frames on tap0 interface * Adds datarouter test_remote_logging integration test - Adds test_remote_logging integration test that verifies remote logging of datarouter and network config. of the linux x86_64 and qnx qemu image by capturing DLT logs on the host. Use py_itf_test rule with select support (#177) - Update score_itf to 44c75de and score_bazel_platforms to b72a5c3 in known_good.json and MODULE.bazel overrides - Add explicit bazel_dep on platforms 1.0.0 (required by new config_settings) - Merge the two py_itf_test targets (linux_x86_64, qnx_x86_64) in feature_integration_tests/itf/BUILD into a single 'itf' target that uses config_setting + select() to switch args/data/plugins per platform; keep backward-compatible aliases for the old target names - Upgrade rules_oci from 1.8.0 to 2.2.7; adapt docker images to use oci_load (replacing the removed oci_tarball) and add the platform-specific repo variants revert the filters to test adding jd check and filter regex matches relative path removing the filter to test adding the filter --- scripts/workflow/recategorize_guidelines.sh | 29 ++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/scripts/workflow/recategorize_guidelines.sh b/scripts/workflow/recategorize_guidelines.sh index 8fa4b736020..ae5b6637421 100755 --- a/scripts/workflow/recategorize_guidelines.sh +++ b/scripts/workflow/recategorize_guidelines.sh @@ -15,8 +15,9 @@ RECATEGORIZE_SCRIPT="codeql-coding-standards-repo/scripts/guideline_recategoriza CODING_STANDARDS_CONFIG="./.github/codeql/coding-standards.yml" CODING_STANDARDS_SCHEMA="codeql-coding-standards-repo/schemas/coding-standards-schema-1.0.0.json" SARIF_SCHEMA="codeql-coding-standards-repo/schemas/sarif-schema-2.1.0.json" -SARIF_FILE="sarif-results/cpp.sarif" +SARIF_FILE="sarif-results/cpp.sarif" mkdir -p sarif-results-recategorized + echo "Processing $SARIF_FILE for recategorization..." python3 "$RECATEGORIZE_SCRIPT" \ --coding-standards-schema-file "$CODING_STANDARDS_SCHEMA" \ @@ -24,5 +25,27 @@ python3 "$RECATEGORIZE_SCRIPT" \ "$CODING_STANDARDS_CONFIG" \ "$SARIF_FILE" \ "sarif-results-recategorized/$(basename "$SARIF_FILE")" - rm "$SARIF_FILE" - mv "sarif-results-recategorized/$(basename "$SARIF_FILE")" "$SARIF_FILE" +PY_EXIT=$? +if [ $PY_EXIT -ne 0 ]; then + echo "Recategorization failed (exit code $PY_EXIT). SARIF file not updated." >&2 + exit $PY_EXIT +fi +rm "$SARIF_FILE" +mv "sarif-results-recategorized/$(basename "$SARIF_FILE")" "$SARIF_FILE" + +# Ensure jq is available +if ! command -v jq >/dev/null 2>&1; then + echo "Error: jq is required but not installed. Please install jq and rerun this script." >&2 + exit 1 +fi + +# Filter SARIF to only include results from repos/* (relative or absolute) +echo "Filtering SARIF results to only include entries with paths matching (^|/)repos/ ..." +jq '(.runs) |= map(.results |= map(select((.locations // [] | length > 0) and ((.locations[0].physicalLocation.artifactLocation.uri // "") | test("(^|/)repos/")))) )' "$SARIF_FILE" > "${SARIF_FILE}.filtered" +if [ $? -eq 0 ]; then + mv "${SARIF_FILE}.filtered" "$SARIF_FILE" +else + echo "jq filtering failed. SARIF file was not modified." >&2 + rm -f "${SARIF_FILE}.filtered" + exit 1 +fi From 22fb1179f96b1b844bf95d1561c87b6405df580b Mon Sep 17 00:00:00 2001 From: Saumya-R Date: Fri, 13 Mar 2026 16:57:52 +0530 Subject: [PATCH 2/4] use old script --- .../workflows/codeql-multiple-repo-scan.yml | 221 +++++++++++++----- 1 file changed, 165 insertions(+), 56 deletions(-) diff --git a/.github/workflows/codeql-multiple-repo-scan.yml b/.github/workflows/codeql-multiple-repo-scan.yml index a22531153b2..279284124b4 100644 --- a/.github/workflows/codeql-multiple-repo-scan.yml +++ b/.github/workflows/codeql-multiple-repo-scan.yml @@ -10,7 +10,9 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* + name: "CodeQL - Multi-Repo Source Scan" + on: pull_request: types: [opened, reopened, synchronize] @@ -22,12 +24,10 @@ on: release: types: [created] workflow_dispatch: + permissions: contents: write -# Do not flood CI with unneeded previous runs in PR -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number }} - cancel-in-progress: ${{ github.ref_name != 'main' && !startsWith(github.ref_name, 'release/') }} + jobs: analyze-repos: name: Analyze Multiple Repositories @@ -37,56 +37,165 @@ jobs: packages: read actions: read contents: read + steps: - - name: Checkout central repository - uses: actions/checkout@v4 - - name: Checkout CodeQL Coding Standards scripts - uses: actions/checkout@v4 - with: - repository: github/codeql-coding-standards - path: codeql-coding-standards-repo # Klonen in diesen Ordner - ref: main # Oder eine spezifische Release-Version, z.B. 'v2.53.0-dev' - # Add coding standard packages and dependencies - - name: Install Python dependencies for Coding Standards scripts - run: | - python3 -m pip install --upgrade pip - pip3 install pyyaml jsonpath-ng jsonschema jsonpatch jsonpointer pytest sarif-tools - - name: Parse known_good.json and create repos.json - id: parse-repos - run: | - scripts/workflow/parse_repos.sh - - name: Checkout all pinned repositories - id: checkout-repos - run: | - scripts/workflow/checkout_repos.sh - - name: Initialize CodeQL for all repositories - uses: github/codeql-action/init@v4 - with: - languages: cpp - build-mode: none - packs: codeql/misra-cpp-coding-standards - config-file: ./.github/codeql/codeql-config.yml - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v4 - with: - upload-database: false # Don't upload databases for each repo - output: sarif-results/ - category: "multi-repo-scan" - - name: Recategorize Guidelines - if: always() - run: | - scripts/workflow/recategorize_guidelines.sh - - name: Generate HTML Report from SARIF - run: | - SARIF_FILE="sarif-results/cpp.sarif" - sarif html "$SARIF_FILE" --output codeql-report.html - - name: Upload SARIF results as artifact - uses: actions/upload-artifact@v4 - with: - name: codeql-sarif-results - path: sarif-results/ - - name: Upload HTML Report as artifact - uses: actions/upload-artifact@v4 - with: - name: codeql-html-report - path: codeql-report.html + - name: Checkout central repository + uses: actions/checkout@v4 + + - name: Checkout CodeQL Coding Standards scripts + uses: actions/checkout@v4 + with: + repository: github/codeql-coding-standards + path: codeql-coding-standards-repo # Klonen in diesen Ordner + ref: main # Oder eine spezifische Release-Version, z.B. 'v2.53.0-dev' + + # Add coding standard packages and dependencies + - name: Install Python dependencies for Coding Standards scripts + run: | + python3 -m pip install --upgrade pip + pip3 install pyyaml jsonpath-ng jsonschema jsonpatch jsonpointer pytest sarif-tools + - name: Parse known_good.json and create repos.json + id: parse-repos + run: | + sudo apt-get update && sudo apt-get install -y jq + JSON_FILE="./known_good.json" + # Check if the file exists + if [ ! -f "$JSON_FILE" ]; then + echo "Error file not found '$JSON_FILE' " + ls -la . + exit 1 + fi + # Create repos.json from known_good.json + # This jq command transforms the 'modules' object into an array of repository objects + # with 'name', 'url', 'version' (branch/tag/hash), and 'path'. + jq '[.modules.target_sw | to_entries[] | { + name: .key, + url: .value.repo, + version: (.value.branch // .value.hash // .value.version), + path: ("repos/" + .key) + }]' "$JSON_FILE" > repos.json + + echo "Generated repos.json:" + cat repos.json + echo "" # Add a newline for better readability + # The following GITHUB_OUTPUT variables are set for each module. + # These might be useful for other steps, but are not directly used by the 'checkout-repos' step + # which now reads 'repos.json' directly. + echo "MODULE_COUNT=$(jq '.modules.target_sw | length' "$JSON_FILE")" >> $GITHUB_OUTPUT + jq -c '.modules.target_sw | to_entries[]' "$JSON_FILE" | while read -r module_entry; do + module_name=$(echo "$module_entry" | jq -r '.key') + repo_url=$(echo "$module_entry" | jq -r '.value.repo // empty') + version=$(echo "$module_entry" | jq -r '.value.version // empty') + branch=$(echo "$module_entry" | jq -r '.value.branch // empty') + hash=$(echo "$module_entry" | jq -r '.value.hash // empty') + echo "${module_name}_url=$repo_url" >> $GITHUB_OUTPUT + + if [ -n "$version" ]; then + echo "${module_name}_version=$version" >> $GITHUB_OUTPUT + fi + + if [ -n "$branch" ]; then + echo "${module_name}_branch=$branch" >> $GITHUB_OUTPUT + fi + + if [ -n "$hash" ]; then + echo "${module_name}_hash=$hash" >> $GITHUB_OUTPUT + fi + done + + - name: Checkout all pinned repositories + id: checkout-repos + run: | + # jq is already installed by the previous step. + + # Read repositories from the repos.json file created by the previous step + repos=$(cat repos.json) + repo_count=$(echo "$repos" | jq length) + + # Initialize an empty string for paths to be outputted + repo_paths_output="" + for i in $(seq 0 $((repo_count-1))); do + name=$(echo "$repos" | jq -r ".[$i].name") + url=$(echo "$repos" | jq -r ".[$i].url") + ref=$(echo "$repos" | jq -r ".[$i].version") # This can be a branch, tag, or commit hash + path=$(echo "$repos" | jq -r ".[$i].path") # e.g., "repos/score_baselibs" + + echo "Checking out $name ($ref) to $path" + + # Create the parent directory if it doesn't exist + mkdir -p "$(dirname "$path")" + # Check if 'ref' looks like a commit hash (e.g., 40 hex characters) + # This is a heuristic; a more robust check might involve fetching refs first. + if [[ "$ref" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo " Detected commit hash. Cloning and then checking out." + git clone "$url" "$path" + (cd "$path" && git checkout "$ref") + else + echo " Detected branch/tag. Cloning with --branch." + git clone --depth 1 --branch v"$ref" "$url" "$path" + fi + + # Append the path to the list, separated by commas + if [ -z "$repo_paths_output" ]; then + repo_paths_output="$path" + else + repo_paths_output="$repo_paths_output,$path" + fi + done + + # Output all paths as a single variable + echo "repo_paths=$repo_paths_output" >> $GITHUB_OUTPUT + - name: Initialize CodeQL for all repositories + uses: github/codeql-action/init@v4 + with: + languages: cpp + build-mode: none + packs: codeql/misra-cpp-coding-standards + config-file: ./.github/codeql/codeql-config.yml + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v4 + with: + upload-database: false # Don't upload databases for each repo + output: sarif-results/ + category: "multi-repo-scan" + + - name: Recategorize Guidelines + if: always() + run: | + RECATEGORIZE_SCRIPT="codeql-coding-standards-repo/scripts/guideline_recategorization/recategorize.py" + CODING_STANDARDS_CONFIG="./.github/codeql/coding-standards.yml" + + CODING_STANDARDS_SCHEMA="codeql-coding-standards-repo/schemas/coding-standards-schema-1.0.0.json" + SARIF_SCHEMA="codeql-coding-standards-repo/schemas/sarif-schema-2.1.0.json" + + + SARIF_FILE="sarif-results/cpp.sarif" + + mkdir -p sarif-results-recategorized + echo "Processing $SARIF_FILE for recategorization..." + python3 "$RECATEGORIZE_SCRIPT" \ + --coding-standards-schema-file "$CODING_STANDARDS_SCHEMA" \ + --sarif-schema-file "$SARIF_SCHEMA" \ + "$CODING_STANDARDS_CONFIG" \ + "$SARIF_FILE" \ + "sarif-results-recategorized/$(basename "$SARIF_FILE")" + + rm "$SARIF_FILE" + mv "sarif-results-recategorized/$(basename "$SARIF_FILE")" "$SARIF_FILE" + - name: Generate HTML Report from SARIF + run: | + SARIF_FILE="sarif-results/cpp.sarif" + sarif html "$SARIF_FILE" --output codeql-report.html + - name: Upload SARIF results as artifact + uses: actions/upload-artifact@v4 + with: + name: codeql-sarif-results + path: sarif-results/ + + + - name: Upload HTML Report as artifact + uses: actions/upload-artifact@v4 + with: + name: codeql-html-report + path: codeql-report.html From a11b8ce651722efac2ee21ae7852e3802f9e01da Mon Sep 17 00:00:00 2001 From: Saumya-R Date: Fri, 13 Mar 2026 23:46:52 +0530 Subject: [PATCH 3/4] adding release 0.6 yml --- .github/workflows/codeql-multiple-repo-scan.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/codeql-multiple-repo-scan.yml b/.github/workflows/codeql-multiple-repo-scan.yml index 279284124b4..2873d12b8fb 100644 --- a/.github/workflows/codeql-multiple-repo-scan.yml +++ b/.github/workflows/codeql-multiple-repo-scan.yml @@ -54,17 +54,20 @@ jobs: run: | python3 -m pip install --upgrade pip pip3 install pyyaml jsonpath-ng jsonschema jsonpatch jsonpointer pytest sarif-tools + - name: Parse known_good.json and create repos.json id: parse-repos run: | sudo apt-get update && sudo apt-get install -y jq JSON_FILE="./known_good.json" + # Check if the file exists if [ ! -f "$JSON_FILE" ]; then echo "Error file not found '$JSON_FILE' " ls -la . exit 1 fi + # Create repos.json from known_good.json # This jq command transforms the 'modules' object into an array of repository objects # with 'name', 'url', 'version' (branch/tag/hash), and 'path'. @@ -78,16 +81,19 @@ jobs: echo "Generated repos.json:" cat repos.json echo "" # Add a newline for better readability + # The following GITHUB_OUTPUT variables are set for each module. # These might be useful for other steps, but are not directly used by the 'checkout-repos' step # which now reads 'repos.json' directly. echo "MODULE_COUNT=$(jq '.modules.target_sw | length' "$JSON_FILE")" >> $GITHUB_OUTPUT + jq -c '.modules.target_sw | to_entries[]' "$JSON_FILE" | while read -r module_entry; do module_name=$(echo "$module_entry" | jq -r '.key') repo_url=$(echo "$module_entry" | jq -r '.value.repo // empty') version=$(echo "$module_entry" | jq -r '.value.version // empty') branch=$(echo "$module_entry" | jq -r '.value.branch // empty') hash=$(echo "$module_entry" | jq -r '.value.hash // empty') + echo "${module_name}_url=$repo_url" >> $GITHUB_OUTPUT if [ -n "$version" ]; then @@ -114,6 +120,7 @@ jobs: # Initialize an empty string for paths to be outputted repo_paths_output="" + for i in $(seq 0 $((repo_count-1))); do name=$(echo "$repos" | jq -r ".[$i].name") url=$(echo "$repos" | jq -r ".[$i].url") @@ -124,6 +131,7 @@ jobs: # Create the parent directory if it doesn't exist mkdir -p "$(dirname "$path")" + # Check if 'ref' looks like a commit hash (e.g., 40 hex characters) # This is a heuristic; a more robust check might involve fetching refs first. if [[ "$ref" =~ ^[0-9a-fA-F]{40}$ ]]; then @@ -145,6 +153,7 @@ jobs: # Output all paths as a single variable echo "repo_paths=$repo_paths_output" >> $GITHUB_OUTPUT + - name: Initialize CodeQL for all repositories uses: github/codeql-action/init@v4 with: @@ -183,10 +192,12 @@ jobs: rm "$SARIF_FILE" mv "sarif-results-recategorized/$(basename "$SARIF_FILE")" "$SARIF_FILE" + - name: Generate HTML Report from SARIF run: | SARIF_FILE="sarif-results/cpp.sarif" sarif html "$SARIF_FILE" --output codeql-report.html + - name: Upload SARIF results as artifact uses: actions/upload-artifact@v4 with: From 70cd15b84c523f278b1343cd417eb19e89cc2648 Mon Sep 17 00:00:00 2001 From: Saumya-R Date: Fri, 13 Mar 2026 23:58:26 +0530 Subject: [PATCH 4/4] removing the filters --- .../workflows/codeql-multiple-repo-scan.yml | 332 +++++++++--------- scripts/workflow/recategorize_guidelines.sh | 30 +- 2 files changed, 173 insertions(+), 189 deletions(-) diff --git a/.github/workflows/codeql-multiple-repo-scan.yml b/.github/workflows/codeql-multiple-repo-scan.yml index 2873d12b8fb..cfdb7969791 100644 --- a/.github/workflows/codeql-multiple-repo-scan.yml +++ b/.github/workflows/codeql-multiple-repo-scan.yml @@ -10,9 +10,7 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* - name: "CodeQL - Multi-Repo Source Scan" - on: pull_request: types: [opened, reopened, synchronize] @@ -24,10 +22,8 @@ on: release: types: [created] workflow_dispatch: - permissions: contents: write - jobs: analyze-repos: name: Analyze Multiple Repositories @@ -37,176 +33,164 @@ jobs: packages: read actions: read contents: read - steps: - - name: Checkout central repository - uses: actions/checkout@v4 - - - name: Checkout CodeQL Coding Standards scripts - uses: actions/checkout@v4 - with: - repository: github/codeql-coding-standards - path: codeql-coding-standards-repo # Klonen in diesen Ordner - ref: main # Oder eine spezifische Release-Version, z.B. 'v2.53.0-dev' - - # Add coding standard packages and dependencies - - name: Install Python dependencies for Coding Standards scripts - run: | - python3 -m pip install --upgrade pip - pip3 install pyyaml jsonpath-ng jsonschema jsonpatch jsonpointer pytest sarif-tools - - - name: Parse known_good.json and create repos.json - id: parse-repos - run: | - sudo apt-get update && sudo apt-get install -y jq - JSON_FILE="./known_good.json" - - # Check if the file exists - if [ ! -f "$JSON_FILE" ]; then - echo "Error file not found '$JSON_FILE' " - ls -la . - exit 1 - fi - - # Create repos.json from known_good.json - # This jq command transforms the 'modules' object into an array of repository objects - # with 'name', 'url', 'version' (branch/tag/hash), and 'path'. - jq '[.modules.target_sw | to_entries[] | { - name: .key, - url: .value.repo, - version: (.value.branch // .value.hash // .value.version), - path: ("repos/" + .key) - }]' "$JSON_FILE" > repos.json - - echo "Generated repos.json:" - cat repos.json - echo "" # Add a newline for better readability - - # The following GITHUB_OUTPUT variables are set for each module. - # These might be useful for other steps, but are not directly used by the 'checkout-repos' step - # which now reads 'repos.json' directly. - echo "MODULE_COUNT=$(jq '.modules.target_sw | length' "$JSON_FILE")" >> $GITHUB_OUTPUT - - jq -c '.modules.target_sw | to_entries[]' "$JSON_FILE" | while read -r module_entry; do - module_name=$(echo "$module_entry" | jq -r '.key') - repo_url=$(echo "$module_entry" | jq -r '.value.repo // empty') - version=$(echo "$module_entry" | jq -r '.value.version // empty') - branch=$(echo "$module_entry" | jq -r '.value.branch // empty') - hash=$(echo "$module_entry" | jq -r '.value.hash // empty') - - echo "${module_name}_url=$repo_url" >> $GITHUB_OUTPUT - - if [ -n "$version" ]; then - echo "${module_name}_version=$version" >> $GITHUB_OUTPUT + - name: Checkout central repository + uses: actions/checkout@v4 + - name: Checkout CodeQL Coding Standards scripts + uses: actions/checkout@v4 + with: + repository: github/codeql-coding-standards + path: codeql-coding-standards-repo # Klonen in diesen Ordner + ref: main # Oder eine spezifische Release-Version, z.B. 'v2.53.0-dev' + # Add coding standard packages and dependencies + - name: Install Python dependencies for Coding Standards scripts + run: | + python3 -m pip install --upgrade pip + pip3 install pyyaml jsonpath-ng jsonschema jsonpatch jsonpointer pytest sarif-tools + - name: Parse known_good.json and create repos.json + id: parse-repos + run: | + sudo apt-get update && sudo apt-get install -y jq + JSON_FILE="./known_good.json" + + # Check if the file exists + if [ ! -f "$JSON_FILE" ]; then + echo "Error file not found '$JSON_FILE' " + ls -la . + exit 1 fi - if [ -n "$branch" ]; then - echo "${module_name}_branch=$branch" >> $GITHUB_OUTPUT - fi - - if [ -n "$hash" ]; then - echo "${module_name}_hash=$hash" >> $GITHUB_OUTPUT - fi - done - - - name: Checkout all pinned repositories - id: checkout-repos - run: | - # jq is already installed by the previous step. - - # Read repositories from the repos.json file created by the previous step - repos=$(cat repos.json) - repo_count=$(echo "$repos" | jq length) - - # Initialize an empty string for paths to be outputted - repo_paths_output="" - - for i in $(seq 0 $((repo_count-1))); do - name=$(echo "$repos" | jq -r ".[$i].name") - url=$(echo "$repos" | jq -r ".[$i].url") - ref=$(echo "$repos" | jq -r ".[$i].version") # This can be a branch, tag, or commit hash - path=$(echo "$repos" | jq -r ".[$i].path") # e.g., "repos/score_baselibs" - - echo "Checking out $name ($ref) to $path" - - # Create the parent directory if it doesn't exist - mkdir -p "$(dirname "$path")" - - # Check if 'ref' looks like a commit hash (e.g., 40 hex characters) - # This is a heuristic; a more robust check might involve fetching refs first. - if [[ "$ref" =~ ^[0-9a-fA-F]{40}$ ]]; then - echo " Detected commit hash. Cloning and then checking out." - git clone "$url" "$path" - (cd "$path" && git checkout "$ref") - else - echo " Detected branch/tag. Cloning with --branch." - git clone --depth 1 --branch v"$ref" "$url" "$path" - fi - - # Append the path to the list, separated by commas - if [ -z "$repo_paths_output" ]; then - repo_paths_output="$path" - else - repo_paths_output="$repo_paths_output,$path" - fi - done - - # Output all paths as a single variable - echo "repo_paths=$repo_paths_output" >> $GITHUB_OUTPUT - - - name: Initialize CodeQL for all repositories - uses: github/codeql-action/init@v4 - with: - languages: cpp - build-mode: none - packs: codeql/misra-cpp-coding-standards - config-file: ./.github/codeql/codeql-config.yml - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v4 - with: - upload-database: false # Don't upload databases for each repo - output: sarif-results/ - category: "multi-repo-scan" - - - name: Recategorize Guidelines - if: always() - run: | - RECATEGORIZE_SCRIPT="codeql-coding-standards-repo/scripts/guideline_recategorization/recategorize.py" - CODING_STANDARDS_CONFIG="./.github/codeql/coding-standards.yml" - - CODING_STANDARDS_SCHEMA="codeql-coding-standards-repo/schemas/coding-standards-schema-1.0.0.json" - SARIF_SCHEMA="codeql-coding-standards-repo/schemas/sarif-schema-2.1.0.json" - - - SARIF_FILE="sarif-results/cpp.sarif" - - mkdir -p sarif-results-recategorized - echo "Processing $SARIF_FILE for recategorization..." - python3 "$RECATEGORIZE_SCRIPT" \ - --coding-standards-schema-file "$CODING_STANDARDS_SCHEMA" \ - --sarif-schema-file "$SARIF_SCHEMA" \ - "$CODING_STANDARDS_CONFIG" \ - "$SARIF_FILE" \ - "sarif-results-recategorized/$(basename "$SARIF_FILE")" - - rm "$SARIF_FILE" - mv "sarif-results-recategorized/$(basename "$SARIF_FILE")" "$SARIF_FILE" - - - name: Generate HTML Report from SARIF - run: | - SARIF_FILE="sarif-results/cpp.sarif" - sarif html "$SARIF_FILE" --output codeql-report.html - - - name: Upload SARIF results as artifact - uses: actions/upload-artifact@v4 - with: - name: codeql-sarif-results - path: sarif-results/ - - - - name: Upload HTML Report as artifact - uses: actions/upload-artifact@v4 - with: - name: codeql-html-report - path: codeql-report.html + # Create repos.json from known_good.json + # This jq command transforms the 'modules' object into an array of repository objects + # with 'name', 'url', 'version' (branch/tag/hash), and 'path'. + jq '[.modules.target_sw | to_entries[] | { + name: .key, + url: .value.repo, + version: (.value.branch // .value.hash // .value.version), + path: ("repos/" + .key) + }]' "$JSON_FILE" > repos.json + + echo "Generated repos.json:" + cat repos.json + echo "" # Add a newline for better readability + + # The following GITHUB_OUTPUT variables are set for each module. + # These might be useful for other steps, but are not directly used by the 'checkout-repos' step + # which now reads 'repos.json' directly. + echo "MODULE_COUNT=$(jq '.modules.target_sw | length' "$JSON_FILE")" >> $GITHUB_OUTPUT + + jq -c '.modules.target_sw | to_entries[]' "$JSON_FILE" | while read -r module_entry; do + module_name=$(echo "$module_entry" | jq -r '.key') + repo_url=$(echo "$module_entry" | jq -r '.value.repo // empty') + version=$(echo "$module_entry" | jq -r '.value.version // empty') + branch=$(echo "$module_entry" | jq -r '.value.branch // empty') + hash=$(echo "$module_entry" | jq -r '.value.hash // empty') + + echo "${module_name}_url=$repo_url" >> $GITHUB_OUTPUT + + if [ -n "$version" ]; then + echo "${module_name}_version=$version" >> $GITHUB_OUTPUT + fi + + if [ -n "$branch" ]; then + echo "${module_name}_branch=$branch" >> $GITHUB_OUTPUT + fi + + if [ -n "$hash" ]; then + echo "${module_name}_hash=$hash" >> $GITHUB_OUTPUT + fi + done + - name: Checkout all pinned repositories + id: checkout-repos + run: | + # jq is already installed by the previous step. + + # Read repositories from the repos.json file created by the previous step + repos=$(cat repos.json) + repo_count=$(echo "$repos" | jq length) + + # Initialize an empty string for paths to be outputted + repo_paths_output="" + + for i in $(seq 0 $((repo_count-1))); do + name=$(echo "$repos" | jq -r ".[$i].name") + url=$(echo "$repos" | jq -r ".[$i].url") + ref=$(echo "$repos" | jq -r ".[$i].version") # This can be a branch, tag, or commit hash + path=$(echo "$repos" | jq -r ".[$i].path") # e.g., "repos/score_baselibs" + + echo "Checking out $name ($ref) to $path" + + # Create the parent directory if it doesn't exist + mkdir -p "$(dirname "$path")" + + # Check if 'ref' looks like a commit hash (e.g., 40 hex characters) + # This is a heuristic; a more robust check might involve fetching refs first. + if [[ "$ref" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo " Detected commit hash. Cloning and then checking out." + git clone "$url" "$path" + (cd "$path" && git checkout "$ref") + else + echo " Detected branch/tag. Cloning with --branch." + git clone --depth 1 --branch "$ref" "$url" "$path" + fi + + # Append the path to the list, separated by commas + if [ -z "$repo_paths_output" ]; then + repo_paths_output="$path" + else + repo_paths_output="$repo_paths_output,$path" + fi + done + + # Output all paths as a single variable + echo "repo_paths=$repo_paths_output" >> $GITHUB_OUTPUT + - name: Initialize CodeQL for all repositories + uses: github/codeql-action/init@v4 + with: + languages: cpp + build-mode: none + packs: codeql/misra-cpp-coding-standards + config-file: ./.github/codeql/codeql-config.yml + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v4 + with: + upload-database: false # Don't upload databases for each repo + output: sarif-results/ + category: "multi-repo-scan" + - name: Recategorize Guidelines + if: always() + run: | + RECATEGORIZE_SCRIPT="codeql-coding-standards-repo/scripts/guideline_recategorization/recategorize.py" + CODING_STANDARDS_CONFIG="./.github/codeql/coding-standards.yml" + + CODING_STANDARDS_SCHEMA="codeql-coding-standards-repo/schemas/coding-standards-schema-1.0.0.json" + SARIF_SCHEMA="codeql-coding-standards-repo/schemas/sarif-schema-2.1.0.json" + + + SARIF_FILE="sarif-results/cpp.sarif" + + mkdir -p sarif-results-recategorized + echo "Processing $SARIF_FILE for recategorization..." + python3 "$RECATEGORIZE_SCRIPT" \ + --coding-standards-schema-file "$CODING_STANDARDS_SCHEMA" \ + --sarif-schema-file "$SARIF_SCHEMA" \ + "$CODING_STANDARDS_CONFIG" \ + "$SARIF_FILE" \ + "sarif-results-recategorized/$(basename "$SARIF_FILE")" + + rm "$SARIF_FILE" + mv "sarif-results-recategorized/$(basename "$SARIF_FILE")" "$SARIF_FILE" + - name: Generate HTML Report from SARIF + run: | + SARIF_FILE="sarif-results/cpp.sarif" + sarif html "$SARIF_FILE" --output codeql-report.html + - name: Upload SARIF results as artifact + uses: actions/upload-artifact@v4 + with: + name: codeql-sarif-results + path: sarif-results/ + - name: Upload HTML Report as artifact + uses: actions/upload-artifact@v4 + with: + name: codeql-html-report + path: codeql-report.html diff --git a/scripts/workflow/recategorize_guidelines.sh b/scripts/workflow/recategorize_guidelines.sh index ae5b6637421..83d612f70ce 100755 --- a/scripts/workflow/recategorize_guidelines.sh +++ b/scripts/workflow/recategorize_guidelines.sh @@ -33,19 +33,19 @@ fi rm "$SARIF_FILE" mv "sarif-results-recategorized/$(basename "$SARIF_FILE")" "$SARIF_FILE" -# Ensure jq is available -if ! command -v jq >/dev/null 2>&1; then - echo "Error: jq is required but not installed. Please install jq and rerun this script." >&2 - exit 1 -fi +# # Ensure jq is available +# if ! command -v jq >/dev/null 2>&1; then +# echo "Error: jq is required but not installed. Please install jq and rerun this script." >&2 +# exit 1 +# fi -# Filter SARIF to only include results from repos/* (relative or absolute) -echo "Filtering SARIF results to only include entries with paths matching (^|/)repos/ ..." -jq '(.runs) |= map(.results |= map(select((.locations // [] | length > 0) and ((.locations[0].physicalLocation.artifactLocation.uri // "") | test("(^|/)repos/")))) )' "$SARIF_FILE" > "${SARIF_FILE}.filtered" -if [ $? -eq 0 ]; then - mv "${SARIF_FILE}.filtered" "$SARIF_FILE" -else - echo "jq filtering failed. SARIF file was not modified." >&2 - rm -f "${SARIF_FILE}.filtered" - exit 1 -fi +# # Filter SARIF to only include results from repos/* (relative or absolute) +# echo "Filtering SARIF results to only include entries with paths matching (^|/)repos/ ..." +# jq '(.runs) |= map(.results |= map(select((.locations // [] | length > 0) and ((.locations[0].physicalLocation.artifactLocation.uri // "") | test("(^|/)repos/")))) )' "$SARIF_FILE" > "${SARIF_FILE}.filtered" +# if [ $? -eq 0 ]; then +# mv "${SARIF_FILE}.filtered" "$SARIF_FILE" +# else +# echo "jq filtering failed. SARIF file was not modified." >&2 +# rm -f "${SARIF_FILE}.filtered" +# exit 1 +# fi