From f1d81927152c83fb3c046a362984e48475ad4915 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 14 Oct 2025 13:03:27 +0200 Subject: [PATCH 1/5] feat(ci): skip qemu image build if version already exists in ECR Add check to prevent rebuilding existing images in ECR repository. The workflow now verifies if an image with the current version tag exists before proceeding with the build and publish process. --- .github/workflows/qemu-image-build.yml | 27 +++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index 1be4caa15..81b260491 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -32,8 +32,33 @@ jobs: VERSIONS=$(yq '.postgres_major[1]' ansible/vars.yml | jq -R -s -c 'split("\n")[:-1]') echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT - build: + check-existing-image: needs: prepare + runs-on: ubuntu-latest + outputs: + image_exists: ${{ steps.check-image.outputs.image_exists }} + steps: + - name: Checkout Repo + uses: supabase/postgres/.github/actions/shared-checkout@HEAD + + - name: Check if image already exists in ECR + id: check-image + env: + AWS_REGION: us-east-1 + REPOSITORY: postgres-vm-image + run: | + VERSION=$(yq '.postgres_release["postgres'${{ fromJson(needs.prepare.outputs.postgres_versions) }}'"]' ansible/vars.yml | tr -d '"') + if aws ecr describe-images --repository-name $REPOSITORY --image-ids imageTag=$VERSION --region $AWS_REGION 2>/dev/null; then + echo "::notice title=Qemu image::Image with tag $VERSION already exists. Skipping build. Please update the version in ansible/vars.yml if you want to upload a new image." + echo "image_exists=true" >> $GITHUB_OUTPUT + else + echo "Image with tag $VERSION does not exist. Proceeding with build." + echo "image_exists=false" >> $GITHUB_OUTPUT + fi + + build: + needs: [prepare, check-existing-image] + if: ${{ needs.check-existing-image.outputs.image_exists == 'false' }} strategy: matrix: postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} From b40b06abefc5a0ed35af1bcfad381ed5b27bfc01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 6 Oct 2025 15:05:41 +0200 Subject: [PATCH 2/5] feat: run actionlint on GitHub Actions workflows Starting to lint GitHub Actions workflows with actionlint. --- nix/hooks.nix | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/nix/hooks.nix b/nix/hooks.nix index 896c262ba..782cde403 100644 --- a/nix/hooks.nix +++ b/nix/hooks.nix @@ -1,4 +1,8 @@ { inputs, ... }: +let + ghWorkflows = builtins.attrNames (builtins.readDir ../.github/workflows); + lintedWorkflows = [ "qemu-image-build.yml" ]; +in { imports = [ inputs.git-hooks.flakeModule ]; perSystem = @@ -8,9 +12,17 @@ check.enable = true; settings = { hooks = { + actionlint = { + enable = true; + excludes = builtins.filter (name: !builtins.elem name lintedWorkflows) ghWorkflows; + verbose = true; + }; + treefmt = { enable = true; package = config.treefmt.build.wrapper; + pass_filenames = false; + verbose = true; }; }; }; From 7aaa8e055279c988ed9a1478069e2b7af44d4d55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 14 Oct 2025 13:20:23 +0200 Subject: [PATCH 3/5] chore: fix lint for qemu-image-build workflow --- .github/actionlint.yaml | 3 +++ .github/workflows/qemu-image-build.yml | 22 +++++++++++----------- 2 files changed, 14 insertions(+), 11 deletions(-) create mode 100644 .github/actionlint.yaml diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 000000000..b663451aa --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,3 @@ +self-hosted-runner: + labels: + - arm-native-runner diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index 81b260491..ed7ca993b 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -30,7 +30,7 @@ jobs: id: set-versions run: | VERSIONS=$(yq '.postgres_major[1]' ansible/vars.yml | jq -R -s -c 'split("\n")[:-1]') - echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT + echo "postgres_versions=$VERSIONS" >> "$GITHUB_OUTPUT" check-existing-image: needs: prepare @@ -48,12 +48,12 @@ jobs: REPOSITORY: postgres-vm-image run: | VERSION=$(yq '.postgres_release["postgres'${{ fromJson(needs.prepare.outputs.postgres_versions) }}'"]' ansible/vars.yml | tr -d '"') - if aws ecr describe-images --repository-name $REPOSITORY --image-ids imageTag=$VERSION --region $AWS_REGION 2>/dev/null; then + if aws ecr describe-images --repository-name "$REPOSITORY" --image-ids imageTag="$VERSION" --region "$AWS_REGION" 2>/dev/null; then echo "::notice title=Qemu image::Image with tag $VERSION already exists. Skipping build. Please update the version in ansible/vars.yml if you want to upload a new image." - echo "image_exists=true" >> $GITHUB_OUTPUT + echo "image_exists=true" >> "$GITHUB_OUTPUT" else echo "Image with tag $VERSION does not exist. Proceeding with build." - echo "image_exists=false" >> $GITHUB_OUTPUT + echo "image_exists=false" >> "$GITHUB_OUTPUT" fi build: @@ -91,16 +91,16 @@ jobs: - name: Set PostgreSQL version environment variable run: | - echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV - echo "EXECUTION_ID=${{ github.run_id }}-${{ matrix.postgres_version }}" >> $GITHUB_ENV + echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> "$GITHUB_ENV" + echo "EXECUTION_ID=${{ github.run_id }}-${{ matrix.postgres_version }}" >> "$GITHUB_ENV" - name: Generate common-nix.vars.pkr.hcl run: | curl -L https://github.com/mikefarah/yq/releases/download/v4.45.1/yq_linux_arm64 -o yq && chmod +x yq - PG_VERSION=$(./yq '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) + PG_VERSION=$(./yq '.postgres_release["postgres${{ matrix.postgres_version }}"]' ansible/vars.yml) PG_VERSION=$(echo "$PG_VERSION" | tr -d '"') # Remove any surrounding quotes - echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl - echo 'postgres-major-version = "'$POSTGRES_MAJOR_VERSION'"' >> common-nix.vars.pkr.hcl + echo "postgres-version = \"${PG_VERSION}\"" > common-nix.vars.pkr.hcl + echo "postgres-major-version = \"${POSTGRES_MAJOR_VERSION}\"" >> common-nix.vars.pkr.hcl # Ensure there's a newline at the end of the file echo "" >> common-nix.vars.pkr.hcl @@ -121,8 +121,8 @@ jobs: - name: Grab release version id: process_release_version run: | - VERSION=$(cat common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g') - echo "version=$VERSION" >> $GITHUB_OUTPUT + VERSION=$(sed -e 's/postgres-version = "\(.*\)"/\1/g' common-nix.vars.pkr.hcl) + echo "version=$VERSION" >> "$GITHUB_OUTPUT" - name: configure aws credentials - staging uses: aws-actions/configure-aws-credentials@v4 From 73c767ee8161da93e2a207de867c8a9aef959aba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 6 Oct 2025 15:05:41 +0200 Subject: [PATCH 4/5] fix: disable treefmt flake check treefmt is already included in the pre-commit hooks check. --- nix/fmt.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nix/fmt.nix b/nix/fmt.nix index 562c3b3c5..08763e5b8 100644 --- a/nix/fmt.nix +++ b/nix/fmt.nix @@ -4,6 +4,7 @@ perSystem = { pkgs, ... }: { + treefmt.flakeCheck = false; treefmt.programs = { deadnix.enable = true; nixfmt = { From 91b446da4407c10f61bbae96a87fa6177ddeba5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 14 Oct 2025 14:16:49 +0200 Subject: [PATCH 5/5] refactor: simplify qemu image build workflow by removing prepare job Run image existence check into build job and use environment variable for PostgreSQL versions instead of separate prepare step. --- .github/workflows/qemu-image-build.yml | 78 +++++++++++--------------- 1 file changed, 33 insertions(+), 45 deletions(-) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index ed7ca993b..b4d34f971 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -18,65 +18,47 @@ permissions: id-token: write jobs: - prepare: - runs-on: ubuntu-latest - outputs: - postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }} - steps: - - name: Checkout Repo - uses: supabase/postgres/.github/actions/shared-checkout@HEAD + build: + strategy: + matrix: + postgres_version: [17] + runs-on: arm-native-runner + timeout-minutes: 150 + permissions: + contents: write + packages: write + id-token: write - - name: Set PostgreSQL versions - only builds pg17 atm - id: set-versions - run: | - VERSIONS=$(yq '.postgres_major[1]' ansible/vars.yml | jq -R -s -c 'split("\n")[:-1]') - echo "postgres_versions=$VERSIONS" >> "$GITHUB_OUTPUT" - - check-existing-image: - needs: prepare - runs-on: ubuntu-latest - outputs: - image_exists: ${{ steps.check-image.outputs.image_exists }} steps: - name: Checkout Repo uses: supabase/postgres/.github/actions/shared-checkout@HEAD + - name: Configure AWS credentials for image check + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.CONTROL_PLANE_DEV_ROLE }} + aws-region: "us-east-1" + - name: Check if image already exists in ECR id: check-image env: AWS_REGION: us-east-1 REPOSITORY: postgres-vm-image run: | - VERSION=$(yq '.postgres_release["postgres'${{ fromJson(needs.prepare.outputs.postgres_versions) }}'"]' ansible/vars.yml | tr -d '"') + VERSION=$(yq '.postgres_release["postgres${{ matrix.postgres_version }}"]' ansible/vars.yml | tr -d '"') if aws ecr describe-images --repository-name "$REPOSITORY" --image-ids imageTag="$VERSION" --region "$AWS_REGION" 2>/dev/null; then echo "::notice title=Qemu image::Image with tag $VERSION already exists. Skipping build. Please update the version in ansible/vars.yml if you want to upload a new image." - echo "image_exists=true" >> "$GITHUB_OUTPUT" + echo "skip=true" >> "$GITHUB_OUTPUT" else echo "Image with tag $VERSION does not exist. Proceeding with build." - echo "image_exists=false" >> "$GITHUB_OUTPUT" + echo "skip=false" >> "$GITHUB_OUTPUT" fi - build: - needs: [prepare, check-existing-image] - if: ${{ needs.check-existing-image.outputs.image_exists == 'false' }} - strategy: - matrix: - postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} - runs-on: arm-native-runner - timeout-minutes: 150 - permissions: - contents: write - packages: write - id-token: write - - steps: - - name: Checkout Repo - uses: supabase/postgres/.github/actions/shared-checkout@HEAD - - uses: DeterminateSystems/nix-installer-action@main + if: steps.check-image.outputs.skip == 'false' - name: Run checks if triggered manually - if: ${{ github.event_name == 'workflow_dispatch' }} + if: ${{ github.event_name == 'workflow_dispatch' && steps.check-image.outputs.skip == 'false' }} run: | SUFFIX=$(yq ".postgres_release[\"postgres${{ matrix.postgres_version }}\"]" ansible/vars.yml | sed -E 's/[0-9\.]+(.*)$/\1/') if [[ -z $SUFFIX ]] ; then @@ -85,16 +67,19 @@ jobs: fi - name: enable KVM support + if: steps.check-image.outputs.skip == 'false' run: | sudo chown runner /dev/kvm sudo chmod 666 /dev/kvm - name: Set PostgreSQL version environment variable + if: steps.check-image.outputs.skip == 'false' run: | echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> "$GITHUB_ENV" echo "EXECUTION_ID=${{ github.run_id }}-${{ matrix.postgres_version }}" >> "$GITHUB_ENV" - name: Generate common-nix.vars.pkr.hcl + if: steps.check-image.outputs.skip == 'false' run: | curl -L https://github.com/mikefarah/yq/releases/download/v4.45.1/yq_linux_arm64 -o yq && chmod +x yq PG_VERSION=$(./yq '.postgres_release["postgres${{ matrix.postgres_version }}"]' ansible/vars.yml) @@ -106,12 +91,14 @@ jobs: # TODO (darora): not quite sure why I'm having to uninstall and re-install these deps, but the build fails w/o this - name: Install dependencies + if: steps.check-image.outputs.skip == 'false' run: | sudo apt-get update sudo apt-get remove -y qemu-efi-aarch64 cloud-image-utils qemu-system-arm qemu-utils sudo apt-get install -y qemu-efi-aarch64 cloud-image-utils qemu-system-arm qemu-utils - name: Build QEMU artifact + if: steps.check-image.outputs.skip == 'false' run: | make init GIT_SHA=${{github.sha}} @@ -119,28 +106,26 @@ jobs: packer build -var "git_sha=${GIT_SHA}" -var-file="common-nix.vars.pkr.hcl" qemu-arm64-nix.pkr.hcl - name: Grab release version + if: steps.check-image.outputs.skip == 'false' id: process_release_version run: | VERSION=$(sed -e 's/postgres-version = "\(.*\)"/\1/g' common-nix.vars.pkr.hcl) echo "version=$VERSION" >> "$GITHUB_OUTPUT" - - name: configure aws credentials - staging - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.CONTROL_PLANE_DEV_ROLE }} - aws-region: "us-east-1" - - name: Login to Amazon ECR + if: steps.check-image.outputs.skip == 'false' id: login-ecr-private-dev uses: aws-actions/amazon-ecr-login@v2 - name: Build image + if: steps.check-image.outputs.skip == 'false' env: IMAGE_TAG: ${{ steps.process_release_version.outputs.version }} run: | docker build -f Dockerfile-kubernetes -t "postgres:$IMAGE_TAG" . - name: Push docker image to Amazon ECR + if: steps.check-image.outputs.skip == 'false' env: REGISTRY: 812073016711.dkr.ecr.us-east-1.amazonaws.com REPOSITORY: postgres-vm-image @@ -151,16 +136,19 @@ jobs: # TODO (darora): temporarily also push to prod account from here - add a guard to only publish proper tagged releases to prod? - name: configure aws credentials - prod + if: steps.check-image.outputs.skip == 'false' uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.CONTROL_PLANE_PROD_ROLE }} aws-region: "us-east-1" - name: Login to Amazon ECR + if: steps.check-image.outputs.skip == 'false' id: login-ecr-private-prod uses: aws-actions/amazon-ecr-login@v2 - name: Push docker image to Amazon ECR + if: steps.check-image.outputs.skip == 'false' env: REGISTRY: 156470330064.dkr.ecr.us-east-1.amazonaws.com REPOSITORY: postgres-vm-image