Draft: automate weekly cross-repo dependency alert sync

.github/scripts/security_sync.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+import re
+import subprocess
+from pathlib import Path
+
+
+def run(command, cwd=None):
+    subprocess.run(command, cwd=cwd, check=True)
+
+
+def capture(command, cwd=None):
+    result = subprocess.run(command, cwd=cwd, check=True, text=True, capture_output=True)
+    return result.stdout
+
+
+def normalize_package(name):
+    return name.strip().lower()
+
+
+def update_pipfile(pipfile_path, patched_versions):
+    content = pipfile_path.read_text()
+    original_content = content
+
+    for package, patched in patched_versions.items():
+        if not patched:
+            continue
+
+        pattern = re.compile(
+            rf'^(\s*"?{re.escape(package)}"?\s*=\s*)"[^"]*"\s*$',
+            re.IGNORECASE | re.MULTILINE,
+        )
+
+        def replacement(match):
+            return f'{match.group(1)}"=={patched}"'
+
+        content = pattern.sub(replacement, content)
+
+    if content != original_content:
+        pipfile_path.write_text(content)
+
+
+def write_requirements(the_process_root, base_requirements, dev_requirements, citus_sha):
+    base_header = (
+        f"# generated from Citus's Pipfile.lock (in src/test/regress) as of {citus_sha}\n"
+        "# using `pipenv requirements > requirements.txt`, so as to avoid the\n"
+        "# need for pipenv/pyenv in this image\n\n"
+    )
+    dev_header = (
+        f"# generated from Citus's Pipfile.lock (in src/test/regress) as of {citus_sha}\n"
+        "# using `pipenv requirements --dev > requirements.txt`, so as to avoid the\n"
+        "# need for pipenv/pyenv in this image\n\n"
+    )
+
+    base_targets = [
+        the_process_root / "circleci/images/citusupgradetester/files/etc/requirements.txt",
+        the_process_root / "circleci/images/failtester/files/etc/requirements.txt",
+        the_process_root / "circleci/images/pgupgradetester/files/etc/requirements.txt",
+    ]
+    for target in base_targets:
+        target.write_text(base_header + base_requirements)
+
+    dev_target = the_process_root / "circleci/images/stylechecker/files/etc/requirements.txt"
+    dev_target.write_text(dev_header + dev_requirements)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--alerts", required=True)
+    parser.add_argument("--citus-root", required=True)
+    parser.add_argument("--the-process-root", required=True)
+    args = parser.parse_args()
+
+    citus_root = Path(args.citus_root)
+    the_process_root = Path(args.the_process_root)
+
+    alerts = json.loads(Path(args.alerts).read_text())
+    patched_versions = {}
+    for alert in alerts:
+        package = normalize_package(alert["dependency"]["package"]["name"])
+        patched = (alert.get("security_vulnerability", {}).get("first_patched_version") or {}).get("identifier")
+        if patched:
+            patched_versions[package] = patched
+
+    update_pipfile(citus_root / "src/test/regress/Pipfile", patched_versions)
+    update_pipfile(citus_root / ".devcontainer/src/test/regress/Pipfile", patched_versions)
+
+    run(["pipenv", "lock"], cwd=citus_root / "src/test/regress")
+    run(["pipenv", "lock"], cwd=citus_root / ".devcontainer/src/test/regress")
+
+    base_requirements = capture(["pipenv", "requirements"], cwd=citus_root / "src/test/regress")
+    dev_requirements = capture(["pipenv", "requirements", "--dev"], cwd=citus_root / "src/test/regress")
+
+    citus_sha = capture(["git", "rev-parse", "--short", "HEAD"], cwd=citus_root).strip()
+    write_requirements(the_process_root, base_requirements, dev_requirements, f"citusdata/citus@{citus_sha}")
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/dependency-security-post-merge.yml

@@ -0,0 +1,62 @@
+name: dependency-security-post-merge
+
+on:
+  pull_request:
+    types: [closed]
+    branches: [master]
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  notify-citus:
+    if: github.event.pull_request.merged == true && github.event.pull_request.head.ref == 'automation/dependency-security-sync'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Create GitHub App token
+        id: app-token
+        uses: actions/create-github-app-token@v1
+        with:
+          app-id: ${{ secrets.PACKAGING_APP_ID }}
+          private-key: ${{ secrets.PACKAGING_APP_PRIVATE_KEY }}
+          owner: citusdata
+
+      - name: Checkout citus sync branch
+        uses: actions/checkout@v4
+        with:
+          repository: citusdata/citus
+          ref: automation/dependency-security-sync
+          token: ${{ steps.app-token.outputs.token }}
+          path: citus
+
+      - name: Update citus image_suffix with merged the-process SHA
+        env:
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
+          MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha }}
+        run: |
+          cd citus
+          short_sha=$(echo "$MERGE_SHA" | cut -c1-7)
+          postfix="-v${short_sha}"
+
+          citus_pr=$(gh pr list --repo citusdata/citus --head automation/dependency-security-sync --state open --json number --jq '.[0].number // empty')
+          if [ -z "$citus_pr" ]; then
+            echo "No open citus sync PR found; skipping."
+            exit 0
+          fi
+
+          sed -i -E "s|(image_suffix:\s*)\"-v[0-9a-f]+\"|\1\"${postfix}\"|" .github/workflows/build_and_test.yml
+
+          if git diff --quiet -- .github/workflows/build_and_test.yml; then
+            echo "No build_and_test.yml image_suffix change needed."
+            gh pr comment "$citus_pr" --repo citusdata/citus --body "the-process sync merged at ${MERGE_SHA}. image_suffix already matches ${postfix}."
+            exit 0
+          fi
+
+          git config user.name "packagingApp[bot]"
+          git config user.email "packagingApp[bot]@users.noreply.github.com"
+          git add .github/workflows/build_and_test.yml
+          git commit -m "Update image_suffix from merged the-process commit"
+          git push origin automation/dependency-security-sync
+
+          gh pr comment "$citus_pr" --repo citusdata/citus --body "the-process sync merged at ${MERGE_SHA}; updated build_and_test.yml image_suffix to ${postfix}."

.github/workflows/dependency-security-sync.yml

@@ -0,0 +1,164 @@
+name: dependency-security-sync
+
+on:
+  schedule:
+    - cron: '0 2 * * 0'
+  workflow_dispatch:
+
+concurrency:
+  group: dependency-security-sync
+  cancel-in-progress: false
+
+permissions:
+  contents: write
+  pull-requests: write
+  issues: write
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Create GitHub App token
+        id: app-token
+        uses: actions/create-github-app-token@v1
+        with:
+          app-id: ${{ secrets.PACKAGING_APP_ID }}
+          private-key: ${{ secrets.PACKAGING_APP_PRIVATE_KEY }}
+          owner: citusdata
+
+      - name: Checkout the-process
+        uses: actions/checkout@v4
+        with:
+          token: ${{ steps.app-token.outputs.token }}
+          fetch-depth: 0
+
+      - name: Checkout citus
+        uses: actions/checkout@v4
+        with:
+          repository: citusdata/citus
+          path: citus
+          token: ${{ steps.app-token.outputs.token }}
+          fetch-depth: 0
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install pipenv and gh auth
+        env:
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
+        run: |
+          python -m pip install --upgrade pip pipenv
+          gh auth setup-git
+
+      - name: Fetch open Dependabot alerts from citus
+        id: alerts
+        env:
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
+        run: |
+          gh api -H "Accept: application/vnd.github+json" \
+            "/repos/citusdata/citus/dependabot/alerts?state=open&per_page=100" > alerts.json
+          count=$(jq 'length' alerts.json)
+          echo "count=$count" >> "$GITHUB_OUTPUT"
+          if [ "$count" -eq 0 ]; then
+            echo "No open alerts found. Exiting." >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+      - name: Run cross-repo sync
+        if: steps.alerts.outputs.count != '0'
+        run: |
+          python .github/scripts/security_sync.py \
+            --alerts alerts.json \
+            --citus-root "$PWD/citus" \
+            --the-process-root "$PWD"
+
+      - name: Create or update citus PR
+        if: steps.alerts.outputs.count != '0'
+        env:
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
+        run: |
+          cd citus
+          git config user.name "packagingApp[bot]"
+          git config user.email "packagingApp[bot]@users.noreply.github.com"
+          git checkout -B automation/dependency-security-sync
+          git add src/test/regress/Pipfile src/test/regress/Pipfile.lock .devcontainer/src/test/regress/Pipfile .devcontainer/src/test/regress/Pipfile.lock || true
+          if git diff --cached --quiet; then
+            echo "No citus dependency changes to commit."
+          else
+            git commit -m "Automate Dependabot alert security dependency sync"
+            git push -f origin automation/dependency-security-sync
+          fi
+
+          pr_number=$(gh pr list --repo citusdata/citus --head automation/dependency-security-sync --state open --json number --jq '.[0].number // empty')
+          if [ -z "$pr_number" ]; then
+            gh pr create \
+              --repo citusdata/citus \
+              --base main \
+              --head automation/dependency-security-sync \
+              --title "Automated security dependency sync from Dependabot alerts" \
+              --body "Automated weekly security sync based on open Dependabot alerts.\n\nThis PR is managed by dependency-security-sync workflow." \
+              --label dependencies
+            pr_number=$(gh pr list --repo citusdata/citus --head automation/dependency-security-sync --state open --json number --jq '.[0].number')
+          fi
+          echo "CITUS_PR=$pr_number" >> "$GITHUB_ENV"
+
+      - name: Create or update the-process PR
+        if: steps.alerts.outputs.count != '0'
+        env:
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
+        run: |
+          git config user.name "packagingApp[bot]"
+          git config user.email "packagingApp[bot]@users.noreply.github.com"
+          git checkout -B automation/dependency-security-sync
+          git add circleci/images/citusupgradetester/files/etc/requirements.txt circleci/images/failtester/files/etc/requirements.txt circleci/images/pgupgradetester/files/etc/requirements.txt circleci/images/stylechecker/files/etc/requirements.txt || true
+          if git diff --cached --quiet; then
+            echo "No the-process dependency changes to commit."
+          else
+            git commit -m "Automate security requirements sync from citus alerts"
+            git push -f origin automation/dependency-security-sync
+          fi
+
+          pr_number=$(gh pr list --repo citusdata/the-process --head automation/dependency-security-sync --state open --json number --jq '.[0].number // empty')
+          if [ -z "$pr_number" ]; then
+            gh pr create \
+              --repo citusdata/the-process \
+              --base master \
+              --head automation/dependency-security-sync \
+              --title "Automated requirements sync for Dependabot security alerts" \
+              --body "Automated weekly requirements refresh based on open Dependabot alerts from citus.\n\nThis PR is managed by dependency-security-sync workflow." \
+              --label dependencies
+            pr_number=$(gh pr list --repo citusdata/the-process --head automation/dependency-security-sync --state open --json number --jq '.[0].number')
+          fi
+          echo "THE_PROCESS_PR=$pr_number" >> "$GITHUB_ENV"
+
+      - name: Close superseded Dependabot PRs in citus
+        if: steps.alerts.outputs.count != '0'
+        env:
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
+        run: |
+          cd citus
+          for n in $(gh pr list --repo citusdata/citus --state open --json number,author --jq '.[] | select(.author.login=="app/dependabot") | .number'); do
+            gh pr close "$n" --repo citusdata/citus --comment "Closing in favor of consolidated automated security sync PR #${CITUS_PR}."
+          done
+          gh pr comment "$CITUS_PR" --repo citusdata/citus --body "Supersedes open Dependabot PRs with this consolidated automated security sync."
+
+      - name: Close superseded Dependabot PRs in the-process
+        if: steps.alerts.outputs.count != '0'
+        env:
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
+        run: |
+          for n in $(gh pr list --repo citusdata/the-process --state open --json number,author --jq '.[] | select(.author.login=="app/dependabot") | .number'); do
+            gh pr close "$n" --repo citusdata/the-process --comment "Closing in favor of consolidated automated security sync PR #${THE_PROCESS_PR}."
+          done
+          gh pr comment "$THE_PROCESS_PR" --repo citusdata/the-process --body "Supersedes open Dependabot PRs with this consolidated automated requirements sync."
+
+      - name: Summary
+        if: steps.alerts.outputs.count != '0'
+        run: |
+          {
+            echo "## Dependency security sync"
+            echo "- Alerts processed: ${{ steps.alerts.outputs.count }}"
+            echo "- the-process PR: #${THE_PROCESS_PR}"
+            echo "- citus PR: #${CITUS_PR}"
+          } >> "$GITHUB_STEP_SUMMARY"