diff --git a/.github/scripts/security_sync.py b/.github/scripts/security_sync.py new file mode 100644 index 0000000..3f0cbbb --- /dev/null +++ b/.github/scripts/security_sync.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 + +import argparse +import json +import re +import subprocess +from pathlib import Path + + +def run(command, cwd=None): + subprocess.run(command, cwd=cwd, check=True) + + +def capture(command, cwd=None): + result = subprocess.run(command, cwd=cwd, check=True, text=True, capture_output=True) + return result.stdout + + +def normalize_package(name): + return name.strip().lower() + + +def update_pipfile(pipfile_path, patched_versions): + content = pipfile_path.read_text() + original_content = content + + for package, patched in patched_versions.items(): + if not patched: + continue + + pattern = re.compile( + rf'^(\s*"?{re.escape(package)}"?\s*=\s*)"[^"]*"\s*$', + re.IGNORECASE | re.MULTILINE, + ) + + def replacement(match): + return f'{match.group(1)}"=={patched}"' + + content = pattern.sub(replacement, content) + + if content != original_content: + pipfile_path.write_text(content) + + +def write_requirements(the_process_root, base_requirements, dev_requirements, citus_sha): + base_header = ( + f"# generated from Citus's Pipfile.lock (in src/test/regress) as of {citus_sha}\n" + "# using `pipenv requirements > requirements.txt`, so as to avoid the\n" + "# need for pipenv/pyenv in this image\n\n" + ) + dev_header = ( + f"# generated from Citus's Pipfile.lock (in src/test/regress) as of {citus_sha}\n" + "# using `pipenv requirements --dev > requirements.txt`, so as to avoid the\n" + "# need for pipenv/pyenv in this image\n\n" + ) + + base_targets = [ + the_process_root / "circleci/images/citusupgradetester/files/etc/requirements.txt", + the_process_root / "circleci/images/failtester/files/etc/requirements.txt", + the_process_root / "circleci/images/pgupgradetester/files/etc/requirements.txt", + ] + for target in base_targets: + target.write_text(base_header + base_requirements) + + dev_target = the_process_root / "circleci/images/stylechecker/files/etc/requirements.txt" + dev_target.write_text(dev_header + dev_requirements) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--alerts", required=True) + parser.add_argument("--citus-root", required=True) + parser.add_argument("--the-process-root", required=True) + args = parser.parse_args() + + citus_root = Path(args.citus_root) + the_process_root = Path(args.the_process_root) + + alerts = json.loads(Path(args.alerts).read_text()) + patched_versions = {} + for alert in alerts: + package = normalize_package(alert["dependency"]["package"]["name"]) + patched = (alert.get("security_vulnerability", {}).get("first_patched_version") or {}).get("identifier") + if patched: + patched_versions[package] = patched + + update_pipfile(citus_root / "src/test/regress/Pipfile", patched_versions) + update_pipfile(citus_root / ".devcontainer/src/test/regress/Pipfile", patched_versions) + + run(["pipenv", "lock"], cwd=citus_root / "src/test/regress") + run(["pipenv", "lock"], cwd=citus_root / ".devcontainer/src/test/regress") + + base_requirements = capture(["pipenv", "requirements"], cwd=citus_root / "src/test/regress") + dev_requirements = capture(["pipenv", "requirements", "--dev"], cwd=citus_root / "src/test/regress") + + citus_sha = capture(["git", "rev-parse", "--short", "HEAD"], cwd=citus_root).strip() + write_requirements(the_process_root, base_requirements, dev_requirements, f"citusdata/citus@{citus_sha}") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/dependency-security-post-merge.yml b/.github/workflows/dependency-security-post-merge.yml new file mode 100644 index 0000000..58ac3d1 --- /dev/null +++ b/.github/workflows/dependency-security-post-merge.yml @@ -0,0 +1,62 @@ +name: dependency-security-post-merge + +on: + pull_request: + types: [closed] + branches: [master] + +permissions: + contents: write + pull-requests: write + +jobs: + notify-citus: + if: github.event.pull_request.merged == true && github.event.pull_request.head.ref == 'automation/dependency-security-sync' + runs-on: ubuntu-latest + steps: + - name: Create GitHub App token + id: app-token + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ secrets.PACKAGING_APP_ID }} + private-key: ${{ secrets.PACKAGING_APP_PRIVATE_KEY }} + owner: citusdata + + - name: Checkout citus sync branch + uses: actions/checkout@v4 + with: + repository: citusdata/citus + ref: automation/dependency-security-sync + token: ${{ steps.app-token.outputs.token }} + path: citus + + - name: Update citus image_suffix with merged the-process SHA + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha }} + run: | + cd citus + short_sha=$(echo "$MERGE_SHA" | cut -c1-7) + postfix="-v${short_sha}" + + citus_pr=$(gh pr list --repo citusdata/citus --head automation/dependency-security-sync --state open --json number --jq '.[0].number // empty') + if [ -z "$citus_pr" ]; then + echo "No open citus sync PR found; skipping." + exit 0 + fi + + sed -i -E "s|(image_suffix:\s*)\"-v[0-9a-f]+\"|\1\"${postfix}\"|" .github/workflows/build_and_test.yml + + if git diff --quiet -- .github/workflows/build_and_test.yml; then + echo "No build_and_test.yml image_suffix change needed." + gh pr comment "$citus_pr" --repo citusdata/citus --body "the-process sync merged at ${MERGE_SHA}. image_suffix already matches ${postfix}." + exit 0 + fi + + git config user.name "packagingApp[bot]" + git config user.email "packagingApp[bot]@users.noreply.github.com" + git add .github/workflows/build_and_test.yml + git commit -m "Update image_suffix from merged the-process commit" + git push origin automation/dependency-security-sync + + gh pr comment "$citus_pr" --repo citusdata/citus --body "the-process sync merged at ${MERGE_SHA}; updated build_and_test.yml image_suffix to ${postfix}." diff --git a/.github/workflows/dependency-security-sync.yml b/.github/workflows/dependency-security-sync.yml new file mode 100644 index 0000000..9f429ba --- /dev/null +++ b/.github/workflows/dependency-security-sync.yml @@ -0,0 +1,164 @@ +name: dependency-security-sync + +on: + schedule: + - cron: '0 2 * * 0' + workflow_dispatch: + +concurrency: + group: dependency-security-sync + cancel-in-progress: false + +permissions: + contents: write + pull-requests: write + issues: write + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - name: Create GitHub App token + id: app-token + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ secrets.PACKAGING_APP_ID }} + private-key: ${{ secrets.PACKAGING_APP_PRIVATE_KEY }} + owner: citusdata + + - name: Checkout the-process + uses: actions/checkout@v4 + with: + token: ${{ steps.app-token.outputs.token }} + fetch-depth: 0 + + - name: Checkout citus + uses: actions/checkout@v4 + with: + repository: citusdata/citus + path: citus + token: ${{ steps.app-token.outputs.token }} + fetch-depth: 0 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install pipenv and gh auth + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + python -m pip install --upgrade pip pipenv + gh auth setup-git + + - name: Fetch open Dependabot alerts from citus + id: alerts + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + gh api -H "Accept: application/vnd.github+json" \ + "/repos/citusdata/citus/dependabot/alerts?state=open&per_page=100" > alerts.json + count=$(jq 'length' alerts.json) + echo "count=$count" >> "$GITHUB_OUTPUT" + if [ "$count" -eq 0 ]; then + echo "No open alerts found. Exiting." >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Run cross-repo sync + if: steps.alerts.outputs.count != '0' + run: | + python .github/scripts/security_sync.py \ + --alerts alerts.json \ + --citus-root "$PWD/citus" \ + --the-process-root "$PWD" + + - name: Create or update citus PR + if: steps.alerts.outputs.count != '0' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + cd citus + git config user.name "packagingApp[bot]" + git config user.email "packagingApp[bot]@users.noreply.github.com" + git checkout -B automation/dependency-security-sync + git add src/test/regress/Pipfile src/test/regress/Pipfile.lock .devcontainer/src/test/regress/Pipfile .devcontainer/src/test/regress/Pipfile.lock || true + if git diff --cached --quiet; then + echo "No citus dependency changes to commit." + else + git commit -m "Automate Dependabot alert security dependency sync" + git push -f origin automation/dependency-security-sync + fi + + pr_number=$(gh pr list --repo citusdata/citus --head automation/dependency-security-sync --state open --json number --jq '.[0].number // empty') + if [ -z "$pr_number" ]; then + gh pr create \ + --repo citusdata/citus \ + --base main \ + --head automation/dependency-security-sync \ + --title "Automated security dependency sync from Dependabot alerts" \ + --body "Automated weekly security sync based on open Dependabot alerts.\n\nThis PR is managed by dependency-security-sync workflow." \ + --label dependencies + pr_number=$(gh pr list --repo citusdata/citus --head automation/dependency-security-sync --state open --json number --jq '.[0].number') + fi + echo "CITUS_PR=$pr_number" >> "$GITHUB_ENV" + + - name: Create or update the-process PR + if: steps.alerts.outputs.count != '0' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + git config user.name "packagingApp[bot]" + git config user.email "packagingApp[bot]@users.noreply.github.com" + git checkout -B automation/dependency-security-sync + git add circleci/images/citusupgradetester/files/etc/requirements.txt circleci/images/failtester/files/etc/requirements.txt circleci/images/pgupgradetester/files/etc/requirements.txt circleci/images/stylechecker/files/etc/requirements.txt || true + if git diff --cached --quiet; then + echo "No the-process dependency changes to commit." + else + git commit -m "Automate security requirements sync from citus alerts" + git push -f origin automation/dependency-security-sync + fi + + pr_number=$(gh pr list --repo citusdata/the-process --head automation/dependency-security-sync --state open --json number --jq '.[0].number // empty') + if [ -z "$pr_number" ]; then + gh pr create \ + --repo citusdata/the-process \ + --base master \ + --head automation/dependency-security-sync \ + --title "Automated requirements sync for Dependabot security alerts" \ + --body "Automated weekly requirements refresh based on open Dependabot alerts from citus.\n\nThis PR is managed by dependency-security-sync workflow." \ + --label dependencies + pr_number=$(gh pr list --repo citusdata/the-process --head automation/dependency-security-sync --state open --json number --jq '.[0].number') + fi + echo "THE_PROCESS_PR=$pr_number" >> "$GITHUB_ENV" + + - name: Close superseded Dependabot PRs in citus + if: steps.alerts.outputs.count != '0' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + cd citus + for n in $(gh pr list --repo citusdata/citus --state open --json number,author --jq '.[] | select(.author.login=="app/dependabot") | .number'); do + gh pr close "$n" --repo citusdata/citus --comment "Closing in favor of consolidated automated security sync PR #${CITUS_PR}." + done + gh pr comment "$CITUS_PR" --repo citusdata/citus --body "Supersedes open Dependabot PRs with this consolidated automated security sync." + + - name: Close superseded Dependabot PRs in the-process + if: steps.alerts.outputs.count != '0' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + for n in $(gh pr list --repo citusdata/the-process --state open --json number,author --jq '.[] | select(.author.login=="app/dependabot") | .number'); do + gh pr close "$n" --repo citusdata/the-process --comment "Closing in favor of consolidated automated security sync PR #${THE_PROCESS_PR}." + done + gh pr comment "$THE_PROCESS_PR" --repo citusdata/the-process --body "Supersedes open Dependabot PRs with this consolidated automated requirements sync." + + - name: Summary + if: steps.alerts.outputs.count != '0' + run: | + { + echo "## Dependency security sync" + echo "- Alerts processed: ${{ steps.alerts.outputs.count }}" + echo "- the-process PR: #${THE_PROCESS_PR}" + echo "- citus PR: #${CITUS_PR}" + } >> "$GITHUB_STEP_SUMMARY"