From e6be07b4c503fae47491c1883dc64db1f6afc357 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 1 Oct 2025 17:46:44 +1000 Subject: [PATCH 001/135] images.json --- .github/actions/manifest/action.yml | 75 +++++ .github/workflows/newbuild.yml | 446 ++++++++++++++++++++++++++++ .scripts/images-additional-tests | 13 + .scripts/images-deps | 11 + .scripts/images-with-dep-ids | 24 ++ .scripts/images-with-dep-shas | 37 +++ Dockerfile | 18 +- Dockerfile.core | 8 +- Dockerfile.friendbot | 3 +- Dockerfile.horizon | 3 +- Dockerfile.lab | 10 +- Dockerfile.rpc | 3 +- images.json | 82 +++++ 13 files changed, 714 insertions(+), 19 deletions(-) create mode 100644 .github/actions/manifest/action.yml create mode 100644 .github/workflows/newbuild.yml create mode 100755 .scripts/images-additional-tests create mode 100755 .scripts/images-deps create mode 100755 .scripts/images-with-dep-ids create mode 100755 .scripts/images-with-dep-shas create mode 100644 images.json diff --git a/.github/actions/manifest/action.yml b/.github/actions/manifest/action.yml new file mode 100644 index 000000000..3d2f985b4 --- /dev/null +++ b/.github/actions/manifest/action.yml @@ -0,0 +1,75 @@ +name: 'Manifest' +inputs: + head_sha: + required: true + artifact_name: + required: true + artifact_image_file: + required: true + artifact_image_name: + required: true + arch: + required: true + image: + required: true + default: ghcr.io/${{ github.repository }}:latest + registry: + required: true + default: ghcr.io + username: + required: true + default: ${{ github.actor }} + password: + required: true + default: ${{ github.token }} +runs: + using: "composite" + steps: + - + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.artifact_name }} + path: /tmp/ + - + shell: bash + run: docker load -i /tmp/${{ inputs.artifact_image_file }} + - + id: image_parts + shell: bash + run: | + IMAGE_TAGLESS=$(echo ${{ inputs.image }} | cut -d':' -f1) + IMAGE_REPO=$(echo $IMAGE_TAGLESS | cut -d'/' -f2,3) + IMAGE_TAG=$(echo ${{ inputs.image }} | cut -d':' -f2) + echo "::set-output name=repo::$IMAGE_REPO" + echo "::set-output name=tag::$IMAGE_TAG" + - + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + registry: ${{ inputs.registry }} + username: ${{ inputs.username }} + password: ${{ inputs.password }} + - + shell: bash + run: echo "IMAGE_URL=https://${{ inputs.image }}" >> $GITHUB_ENV + - + if: ${{ inputs.registry == 'docker.io' }} + shell: bash + run: | + echo "IMAGE_URL=https://hub.docker.com/r/${{ steps.image_parts.outputs.repo }}/tags?name=${{ steps.image_parts.outputs.tag }}" >> $GITHUB_ENV + - + shell: bash + run: | + docker push ${{ inputs.image }} + - + uses: actions/github-script@v5 + with: + script: | + github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: '${{ inputs.head_sha }}', + state: 'success', + context: `${{ inputs.image }}`, + target_url: '${{ env.IMAGE_URL }}', + description: 'Available', + }); diff --git a/.github/workflows/newbuild.yml b/.github/workflows/newbuild.yml new file mode 100644 index 000000000..b28cf70e6 --- /dev/null +++ b/.github/workflows/newbuild.yml @@ -0,0 +1,446 @@ +name: Build + +on: + push: + branches: + - main + pull_request: + +# Prevent more than one build of this workflow for a branch to be running at the +# same time, and if multiple are queued, only run the latest, cancelling any +# already running build. The exception being any protected branch, such as +# main, where a build for every commit will run. +concurrency: + group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }} + cancel-in-progress: true + +env: + sha: ${{ github.event.pull_request.head.sha || github.sha }} + image_repo: ${{ format('{0}/{1}', secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io', github.repository) }} + # Cache ID is a value inserted into cache keys. Whenever changing the build + # in a way that needs to use entirely new fresh builds, increment the number + # by one so that all the keys become new. + cache_id: 5 + artifact_retention_days_for_image: 7 + artifact_retention_days_for_tombstone: 7 + artifact_retention_days_for_logs: 60 + +jobs: + + complete: + if: always() + needs: [setup, load, build, test, push, manifest] + runs-on: ubuntu-latest + steps: + - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') + run: exit 1 + + setup: + runs-on: ubuntu-latest + outputs: + tag-prefix: ${{ steps.tag-prefix.outputs.tag-prefix }} + tag-alias-prefix: ${{ steps.tag-prefix.outputs.tag-alias-prefix }} + images: ${{ steps.images.outputs.images }} + deps: ${{ steps.deps.outputs.deps }} + additional-tests: ${{ steps.tests.outputs.additional-tests }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 # Get all history for the sha count below. + ref: ${{ env.sha }} + - name: Tag Prefix + id: tag-prefix + run: | + pr_prefix="${{ github.event_name == 'pull_request' && format('pr{0}-', github.event.pull_request.number) || '' }}" + count="$(git rev-list HEAD --count --first-parent)" + echo "tag-prefix=${pr_prefix}v${count}-" | tee -a $GITHUB_OUTPUT + echo "tag-alias-prefix=${pr_prefix}" | tee -a $GITHUB_OUTPUT + - name: Images + id: images + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + images="$(> $GITHUB_OUTPUT + echo "images=$images" >> $GITHUB_ENV + - name: Deps + id: deps + run: | + deps="$(<<< $images ./.scripts/images-deps)" + <<< $deps jq + echo "deps=$deps" >> $GITHUB_OUTPUT + - name: Additional Tests + id: tests + run: | + tests="$(<<< $images ./.scripts/images-additional-tests)" + <<< $tests jq + echo "additional-tests=$tests" >> $GITHUB_OUTPUT + + load: + needs: [setup] + strategy: + matrix: + dep: ${{ fromJSON(needs.setup.outputs.deps) }} + arch: ["amd64", "arm64"] + fail-fast: false + name: load ${{ matrix.dep.name }} ${{ matrix.dep.repo }} ${{ matrix.dep.ref }} ${{ matrix.arch }} ${{ matrix.dep.options && toJSON(matrix.dep.options) }} + runs-on: ubuntu-latest + env: + dep_json: ${{ toJSON(matrix.dep) }} + image_filename: image-${{ matrix.dep.name }}-${{ matrix.dep.id }}-${{ matrix.arch }}.tar + json_filename: image-${{ matrix.dep.name }}-${{ matrix.dep.id }}-${{ matrix.arch }}.json + missing_filename: missing-${{ matrix.dep.name }}-${{ matrix.dep.id }}-${{ matrix.arch }}.json + steps: + - name: Create Dep Details JSON (with arch) + run: > + echo "${dep_json}" + | jq --arg arch ${{ matrix.arch }} '.arch = $arch' + | tee /tmp/${{ env.json_filename }} + - name: Upload Dep Details JSON + uses: actions/upload-artifact@v4 + with: + name: ${{ env.json_filename }} + path: /tmp/${{ env.json_filename }} + retention-days: ${{ env.artifact_retention_days_for_tombstone }} + - name: Find Image in Cache + id: cache + uses: actions/cache/restore@v3 + with: + key: ${{ env.cache_id }}-${{ env.image_filename }} + path: /tmp/${{ env.image_filename }} + - name: Upload Image + if: steps.cache.outputs.cache-hit == 'true' + uses: actions/upload-artifact@v4 + with: + name: ${{ env.image_filename }} + path: /tmp/${{ env.image_filename }} + retention-days: ${{ env.artifact_retention_days_for_image }} + - name: Upload Dep Details as Missing Marker Due to Cache Miss + if: steps.cache.outputs.cache-hit != 'true' + uses: actions/upload-artifact@v4 + with: + name: ${{ env.missing_filename }} + path: /tmp/${{ env.json_filename }} + retention-days: ${{ env.artifact_retention_days_for_tombstone }} + + prepare: + needs: [load] + name: prepare + runs-on: ubuntu-latest + outputs: + deps-to-build: ${{ steps.deps-to-build.outputs.deps }} + steps: + - name: Download Missing Markers + uses: actions/download-artifact@v4 + with: + pattern: missing-* + merge-multiple: true + path: /tmp/missing + - name: Collect Deps-to-Build from Missing Markers + id: deps-to-build + run: | + deps="$(find /tmp/missing -name "*.json" -exec cat {} \; | jq -c -s '.')" + echo "deps=$deps" | tee -a $GITHUB_OUTPUT + + build-dep: + needs: [setup, prepare] + if: needs.prepare.outputs.deps-to-build != '[]' + strategy: + matrix: + dep: ${{ fromJSON(needs.prepare.outputs.deps-to-build) }} + fail-fast: false + name: build ${{ matrix.dep.name }} ${{ matrix.dep.repo }} ${{ matrix.dep.ref }} ${{ matrix.dep.arch }} ${{ matrix.dep.options && toJSON(matrix.dep.options) }} + runs-on: ${{ matrix.dep.name == 'core' && (matrix.dep.arch == 'arm64' && 'ubuntu-jammy-16-cores-amd64' || 'ubuntu-latest-16-cores') || (matrix.dep.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest') }} + env: + image_filename: image-${{ matrix.dep.name }}-${{ matrix.dep.id }}-${{ matrix.dep.arch }}.tar + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ env.sha }} + - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f + - name: Build Image + env: + options_json: ${{ toJSON(matrix.dep.options) }} + run: > + docker buildx build + --platform linux/${{ matrix.dep.arch }} + -f Dockerfile.${{ matrix.dep.name }} + -t stellar-${{ matrix.dep.name }}:${{ matrix.dep.arch }} + -o type=docker,dest=/tmp/${image_filename} + --build-arg REPO="${{ matrix.dep.repo }}" + --build-arg REF="${{ matrix.dep.sha }}" + --build-arg OPTIONS="${options_json}" + . + - name: Upload Image to Cache + uses: actions/cache/save@v3 + id: cache + with: + key: ${{ env.cache_id }}-${{env.image_filename }} + path: /tmp/${{ env.image_filename }} + - name: Upload Image to Artifacts + uses: actions/upload-artifact@v4 + with: + name: ${{ env.image_filename }} + path: /tmp/${{ env.image_filename }} + retention-days: ${{ env.artifact_retention_days_for_image }} + + build: + needs: [setup, build-dep] + if: always() && !failure() && !cancelled() + strategy: + matrix: + image: ${{ fromJSON(needs.setup.outputs.images) }} + arch: ["amd64", "arm64"] + fail-fast: false + name: build quickstart ${{ matrix.image.tag }} ${{ matrix.arch }} + runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ env.sha }} + - name: Download Images + uses: actions/download-artifact@v4 + with: + pattern: image-*-${{ matrix.arch }}.* + merge-multiple: true + path: /tmp/images + - name: Load Image into Docker + run: | + ls -lah /tmp/images/ + for image in /tmp/images/*.tar; do + echo Loading image $image + < "${image/%.tar/.json}" jq + docker load -i $image + done + - name: Create Tag + id: tag + run: echo "tag=${{ needs.setup.outputs.tag-prefix }}${{ matrix.image.tag }}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + - name: Pull Base Image + run: docker pull --platform linux/${{ matrix.arch }} ubuntu:22.04 + # Docker buildx cannot be used to build the dev quickstart image because + # buildx does not yet support importing existing images, like the core and + # horizon images above, into a buildx builder's cache. Buildx would be + # preferred because it can output a smaller image file faster than docker + # save can. Once buildx supports it we can update. + # https://github.com/docker/buildx/issues/847 + - name: Build Image + run: > + docker build + --platform linux/${{ matrix.arch }} + -f Dockerfile + -t ${{ env.image_repo }}:${{ steps.tag.outputs.tag }} + --label org.opencontainers.image.revision="${{ env.sha }}" + --build-arg REVISION="${{ env.sha }}" + --build-arg PROTOCOL_VERSION_DEFAULT="${{ matrix.image.config.protocol_version_default }}" + --build-arg XDR_IMAGE_REF=stellar-xdr:${{ matrix.arch }} + --build-arg CORE_IMAGE_REF=stellar-core:${{ matrix.arch }} + --build-arg RPC_IMAGE_REF=stellar-rpc:${{ matrix.arch }} + --build-arg HORIZON_IMAGE_REF=stellar-horizon:${{ matrix.arch }} + --build-arg FRIENDBOT_IMAGE_REF=stellar-friendbot:${{ matrix.arch }} + --build-arg LAB_IMAGE_REF=stellar-lab:${{ matrix.arch }} + . + - name: Save Quickstart Image + run: docker save ${{ env.image_repo }}:${{ steps.tag.outputs.tag }} -o /tmp/image + - name: Upload Quickstart Image + uses: actions/upload-artifact@v4 + with: + name: image-quickstart-${{ matrix.image.tag }}-${{ matrix.arch }}.tar + path: /tmp/image + retention-days: ${{ env.ARTIFACT_RETENTION_DAYS_FOR_IMAGE }} + + test: + needs: [setup, build] + if: always() && !failure() && !cancelled() + strategy: + matrix: + tag: ${{ fromJSON(needs.setup.outputs.images).*.tag }} + arch: ["amd64", "arm64"] + network: ["local"] + core: [true, false] + horizon: [true, false] + rpc: [true, false] + options: [""] + include: ${{ fromJSON(needs.setup.outputs.additional-tests) }} + fail-fast: false + name: test ${{ matrix.tag }} ${{ matrix.arch }} ${{ matrix.network }} ${{ matrix.core && 'core' || '' }} ${{ matrix.rpc && 'rpc' || '' }} ${{ matrix.horizon && 'horizon' || '' }} ${{ matrix.options || '' }} + runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} + steps: + - name: Free up disk space + if: matrix.network == 'pubnet' + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + sudo rm -rf /opt/hostedtoolcache/CodeQL + df -h + - uses: actions/checkout@v2 + with: + ref: ${{ env.sha }} + - name: Download Quickstart Image + uses: actions/download-artifact@v4 + with: + name: image-quickstart-${{ matrix.tag }}-${{ matrix.arch }}.tar + path: /tmp/ + - name: Load Quickstart Image + run: docker load -i /tmp/image + - name: Create Tag + id: tag + run: echo "tag=${{ needs.setup.outputs.tag-prefix }}${{ matrix.tag }}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + - name: Prepare Logs Directory + run: mkdir -p logs + - name: Run Quickstart Image + run: > + docker run + --platform linux/${{ matrix.arch }} + -d + -p + "8000:8000" + -p "11626:11626" + --name stellar + ${{ env.image_repo }}:${{ steps.tag.outputs.tag }} + --${{ matrix.network }} + --enable ${{ matrix.core && 'core' }},${{ matrix.rpc && 'rpc' }},${{ matrix.horizon && 'horizon' }} + ${{ matrix.options }} + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: ^1 + - name: Sleep until supervisor is up + run: sleep 10 + - name: Run core test + if: ${{ matrix.core }} + run: | + docker logs stellar -f & + echo "supervisorctl tail -f stellar-core" | docker exec -i stellar sh & + go run tests/test_core.go + curl http://localhost:11626/info + - name: Run horizon up test + if: ${{ matrix.horizon }} + run: | + docker logs stellar -f & + echo "supervisorctl tail -f horizon" | docker exec -i stellar sh & + go run tests/test_horizon_up.go + curl http://localhost:8000 + - name: Run horizon core up test + if: ${{ matrix.horizon && matrix.network != 'pubnet' }} + run: | + docker logs stellar -f & + echo "supervisorctl tail -f horizon" | docker exec -i stellar sh & + go run tests/test_horizon_core_up.go + curl http://localhost:8000 + - name: Run horizon ingesting test + if: ${{ matrix.horizon && matrix.network != 'pubnet' }} + run: | + docker logs stellar -f & + echo "supervisorctl tail -f stellar-core" | docker exec -i stellar sh & + echo "supervisorctl tail -f horizon" | docker exec -i stellar sh & + go run tests/test_horizon_ingesting.go + curl http://localhost:8000 + - name: Run friendbot test + if: ${{ matrix.horizon && matrix.network == 'local' }} + run: | + docker logs stellar -f & + echo "supervisorctl tail -f friendbot" | docker exec -i stellar sh & + echo "supervisorctl tail -f horizon" | docker exec -i stellar sh & + go run tests/test_friendbot.go + - name: Run stellar rpc up test + if: ${{ matrix.rpc }} + run: | + docker logs stellar -f & + echo "supervisorctl tail -f stellar-rpc" | docker exec -i stellar sh & + go run tests/test_stellar_rpc_up.go + - name: Run stellar rpc healthy test + if: ${{ matrix.rpc && matrix.network != 'pubnet' }} + run: | + docker logs stellar -f & + echo "supervisorctl tail -f stellar-rpc" | docker exec -i stellar sh & + go run tests/test_stellar_rpc_healthy.go + - name: Prepare Test Logs + if: always() + run: docker cp stellar:/var/log logs + - name: Upload Test Logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: logs-${{ matrix.tag }}-${{ matrix.arch }}-test-${{ strategy.job-index }} + path: logs + retention-days: ${{ env.artifact_retention_days_for_logs }} + + push: + needs: test + if: always() && !failure() && !cancelled() && (github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository)) + strategy: + matrix: + tag: ${{ fromJSON(needs.setup.outputs.images).*.tag }} + arch: ["amd64", "arm64"] + fail-fast: false + permissions: + packages: write + statuses: write + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ inputs.sha }} + - name: Create Tag + id: tag + run: echo "tag=${{ needs.setup.outputs.tag-prefix }}${{ matrix.tag }}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + - uses: ./.github/actions/push + with: + head_sha: ${{ env.sha }} + artifact_name: image-quickstart-${{ matrix.tag }}-${{ matrix.arch }} + artifact_image_file: image + arch: ${{ matrix.arch }} + image: ${{ env.image_repo }}:${{ steps.tag.outputs.tag }} + registry: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} + username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} + password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} + + manifest: + needs: push + if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) + strategy: + matrix: + tag: ${{ fromJSON(needs.setup.outputs.images).*.tag }} + fail-fast: false + permissions: + packages: write + statuses: write + runs-on: ubuntu-latest + steps: + - name: Create Tag + id: tag + run: | + echo "tag=${{ needs.setup.outputs.tag-prefix }}${{ matrix.tag }}" >> $GITHUB_OUTPUT + echo "tag-alias=${{ needs.setup.outputs.tag-alias-prefix }}${{ matrix.tag }}" >> $GITHUB_OUTPUT + - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} + password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} + - run: > + docker manifest create + ${{ env.image_repo }}:${{ steps.tag.outputs.tag }} + ${{ env.image_repo }}:${{ steps.tag.outputs.tag }}-amd64 + ${{ env.image_repo }}:${{ steps.tag.outputs.tag }}-arm64 + - run: > + docker manifest push + ${{ env.image_repo }}:${{ steps.tag.outputs.tag }} + - run: > + docker buildx imagetools create -t + ${{ env.image_repo }}:${{ steps.tag.outputs.tag-alias }} + ${{ env.image_repo }}:${{ steps.tag.outputs.tag }} + - uses: actions/github-script@v5 + with: + script: | + github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: '${{ env.sha }}', + state: 'success', + context: `${{ env.image_repo }}:${{ steps.tag.outputs.tag }}`, + description: 'Available', + }); diff --git a/.scripts/images-additional-tests b/.scripts/images-additional-tests new file mode 100755 index 000000000..02dee2200 --- /dev/null +++ b/.scripts/images-additional-tests @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 + +import json +import sys + +images = json.load(sys.stdin) +tests = [] +for image in images: + tag = image['tag'] + for test in image['additional-tests']: + tests.append({'tag': tag, **test}) + +print(json.dumps(tests)) diff --git a/.scripts/images-deps b/.scripts/images-deps new file mode 100755 index 000000000..8d1d5e776 --- /dev/null +++ b/.scripts/images-deps @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -e +set -u +set -o pipefail + +# Accepts as stdin a JSON object in the format of images.json. Outputs an array +# of all dependencies that need to be built across all the images. +# Usage: < images.json ./.scripts/images-deps + +jq -c '[ .[] | .deps[] ] | unique' diff --git a/.scripts/images-with-dep-ids b/.scripts/images-with-dep-ids new file mode 100755 index 000000000..6963f0051 --- /dev/null +++ b/.scripts/images-with-dep-ids @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 + +import json +import sys +import subprocess +import hashlib + +# Accepts as stdin a JSON object in the format of images.json. Hashes the +# entire dep details and injects the hash as an id into the deps details. The +# id can be used to uniquely identify a dep configuration. +# Usage: < images.json ./.scripts/images-with-dep-ids + +images = json.load(sys.stdin) + +cache = {} + +for image in images: + for dep in image["deps"]: + dep_str = json.dumps(dep, separators=(',', ':')) + hash = hashlib.sha256(dep_str.encode()).hexdigest() + dep["id"] = hash + dep["id_short"] = hash[:5] + +print(json.dumps(images, separators=(',', ':'))) diff --git a/.scripts/images-with-dep-shas b/.scripts/images-with-dep-shas new file mode 100755 index 000000000..34183ceb0 --- /dev/null +++ b/.scripts/images-with-dep-shas @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 + +import json +import sys +import subprocess + +# Accepts as stdin a JSON object in the format of images.json. +# Resolves any 'ref' values in the JSON to a commit sha. +# Outputs the original JSON modified with the shas. +# Usage: < images.json ./.scripts/images-with-dep-shas + +images = json.load(sys.stdin) + +cache = {} + +for image in images: + tag = image["tag"] + for dep in image["deps"]: + name = dep["name"] + repo = dep["repo"] + ref = dep["ref"] + print(f"{tag} {name} {repo} {ref} ...", end=" ", flush=True, file=sys.stderr) + key = (name, repo, ref) + if key in cache: + sha = cache[key] + else: + sha = subprocess.run( + ["gh", "api", f"repos/{repo}/commits/{ref}", "--jq", ".sha"], + capture_output=True, + text=True, + check=True + ).stdout.strip() + cache[key] = sha + print(sha, file=sys.stderr) + dep["sha"] = sha + +print(json.dumps(images, separators=(',', ':'))) diff --git a/Dockerfile b/Dockerfile index 1474ca979..843b72d6f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,15 +1,15 @@ -ARG STELLAR_XDR_IMAGE_REF -ARG STELLAR_CORE_IMAGE_REF +ARG XDR_IMAGE_REF +ARG CORE_IMAGE_REF ARG HORIZON_IMAGE_REF ARG FRIENDBOT_IMAGE_REF -ARG STELLAR_RPC_IMAGE_REF +ARG RPC_IMAGE_REF ARG LAB_IMAGE_REF -FROM $STELLAR_XDR_IMAGE_REF AS stellar-xdr -FROM $STELLAR_CORE_IMAGE_REF AS stellar-core +FROM $XDR_IMAGE_REF AS xdr +FROM $CORE_IMAGE_REF AS core FROM $HORIZON_IMAGE_REF AS horizon FROM $FRIENDBOT_IMAGE_REF AS friendbot -FROM $STELLAR_RPC_IMAGE_REF AS stellar-rpc +FROM $RPC_IMAGE_REF AS rpc FROM $LAB_IMAGE_REF AS lab FROM ubuntu:22.04 @@ -29,11 +29,11 @@ EXPOSE 11626 ADD dependencies / RUN /dependencies -COPY --from=stellar-xdr /stellar-xdr /usr/local/bin/stellar-xdr -COPY --from=stellar-core /stellar-core /usr/bin/stellar-core +COPY --from=xdr /stellar-xdr /usr/local/bin/stellar-xdr +COPY --from=core /stellar-core /usr/bin/stellar-core COPY --from=horizon /horizon /usr/bin/stellar-horizon COPY --from=friendbot /friendbot /usr/local/bin/friendbot -COPY --from=stellar-rpc /stellar-rpc /usr/bin/stellar-rpc +COPY --from=rpc /stellar-rpc /usr/bin/stellar-rpc COPY --from=lab /lab /opt/stellar/lab COPY --from=lab /node /usr/bin/ diff --git a/Dockerfile.core b/Dockerfile.core index 491a1ff83..5acd57fbe 100644 --- a/Dockerfile.core +++ b/Dockerfile.core @@ -6,10 +6,13 @@ RUN apt-get update && \ git build-essential pkg-config autoconf automake libtool \ bison flex sed perl libpq-dev parallel libunwind-dev \ clang-12 libc++abi-12-dev libc++-12-dev \ - postgresql curl + postgresql curl jq ARG REPO ARG REF +ARG OPTIONS +RUN echo "$OPTIONS" | jq -r '.configure_flags // ""' > /tmp/arg_configure_flags + WORKDIR /wd RUN git clone https://github.com/${REPO} /wd RUN git fetch origin ${REF} @@ -22,14 +25,13 @@ ARG CC=clang-12 ARG CXX=clang++-12 ARG CFLAGS='-O3 -g1 -fno-omit-frame-pointer' ARG CXXFLAGS='-O3 -g1 -fno-omit-frame-pointer -stdlib=libc++' -ARG CONFIGURE_FLAGS='' RUN sysctl vm.mmap_rnd_bits=28 RUN ./autogen.sh RUN ./install-rust.sh ENV PATH "/root/.cargo/bin:$PATH" -RUN ./configure CC="${CC}" CXX="${CXX}" CFLAGS="${CFLAGS}" CXXFLAGS="${CXXFLAGS}" ${CONFIGURE_FLAGS} +RUN sh -c './configure CC="${CC}" CXX="${CXX}" CFLAGS="${CFLAGS}" CXXFLAGS="${CXXFLAGS}" $( Date: Fri, 3 Oct 2025 21:39:53 +1000 Subject: [PATCH 002/135] add action job and update workflow conditions --- .github/workflows/newbuild.yml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/newbuild.yml b/.github/workflows/newbuild.yml index b28cf70e6..0cb9bb8a3 100644 --- a/.github/workflows/newbuild.yml +++ b/.github/workflows/newbuild.yml @@ -29,7 +29,7 @@ jobs: complete: if: always() - needs: [setup, load, build, test, push, manifest] + needs: [setup, load, build, test, push, manifest, action] runs-on: ubuntu-latest steps: - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') @@ -371,7 +371,7 @@ jobs: push: needs: test - if: always() && !failure() && !cancelled() && (github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository)) + if: always() && !failure() && !cancelled() && ((github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository))) strategy: matrix: tag: ${{ fromJSON(needs.setup.outputs.images).*.tag }} @@ -401,7 +401,7 @@ jobs: manifest: needs: push - if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) + if: (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) strategy: matrix: tag: ${{ fromJSON(needs.setup.outputs.images).*.tag }} @@ -444,3 +444,14 @@ jobs: context: `${{ env.image_repo }}:${{ steps.tag.outputs.tag }}`, description: 'Available', }); + + action: + needs: [setup, manifest] + if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event.pull_request.head.repo.full_name == github.repository + strategy: + matrix: + tag: ${{ fromJSON(needs.setup.outputs.images).*.tag }} + fail-fast: false + uses: ./.github/workflows/action-test.yml + with: + tag: ${{ needs.setup.outputs.tag-prefix }}${{ matrix.tag }} From 918968e973bd4d472ab1d31d8283400bcf117db8 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Fri, 3 Oct 2025 21:40:03 +1000 Subject: [PATCH 003/135] remove old workflows --- .github/workflows/build-future.yml | 87 ---- .github/workflows/build-latest.yml | 95 ----- .github/workflows/build-start.yml | 93 ---- .github/workflows/build-testing.yml | 98 ----- .github/workflows/build.yml | 638 ---------------------------- .github/workflows/manifest.yml | 90 ---- 6 files changed, 1101 deletions(-) delete mode 100644 .github/workflows/build-future.yml delete mode 100644 .github/workflows/build-latest.yml delete mode 100644 .github/workflows/build-start.yml delete mode 100644 .github/workflows/build-testing.yml delete mode 100644 .github/workflows/build.yml delete mode 100644 .github/workflows/manifest.yml diff --git a/.github/workflows/build-future.yml b/.github/workflows/build-future.yml deleted file mode 100644 index 31d629e68..000000000 --- a/.github/workflows/build-future.yml +++ /dev/null @@ -1,87 +0,0 @@ -name: Future - -# The `:future` tag points to a build containing unreleased versions of -# software that have been informally released to the futurenet network. - -on: - workflow_call: - secrets: - DOCKERHUB_USERNAME: - required: false - DOCKERHUB_TOKEN: - required: false - inputs: - sha: - description: "Sha to build" - type: "string" - required: true - tag-prefix: - description: "Prefix for the tag name" - type: "string" - default: "" - tag-alias-prefix: - description: 'Prefix for the alias tag name' - type: 'string' - default: '' - -jobs: - amd64: - uses: ./.github/workflows/build.yml - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - with: - sha: ${{ inputs.sha }} - arch: amd64 - tag: ${{ inputs.tag-prefix }}future-amd64 - protocol_version_default: 23 - xdr_ref: v23.0.0 - core_ref: v23.0.1 - horizon_ref: horizon-v23.0.0 - stellar_rpc_ref: v23.0.1 - friendbot_ref: horizon-v23.0.0 - lab_ref: main - test_matrix: | - { - "network": ["local"], - "core": ["core", null], - "horizon": ["horizon", null], - "rpc": ["rpc", null], - "options": [""] - } - - arm64: - uses: ./.github/workflows/build.yml - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - with: - sha: ${{ inputs.sha }} - arch: arm64 - tag: ${{ inputs.tag-prefix }}future-arm64 - protocol_version_default: 23 - xdr_ref: v23.0.0 - core_ref: v23.0.1 - horizon_ref: horizon-v23.0.0 - stellar_rpc_ref: v23.0.4 - friendbot_ref: horizon-v23.0.0 - lab_ref: main - test_matrix: | - { - "network": ["local"], - "core": ["core", null], - "horizon": ["horizon", null], - "rpc": ["rpc", null], - "options": [""] - } - - manifest: - needs: [amd64, arm64] - uses: ./.github/workflows/manifest.yml - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - with: - tag: ${{ inputs.tag-prefix }}future - tag-alias: ${{ inputs.tag-alias-prefix }}future - images: ${{ needs.amd64.outputs.image }} ${{ needs.arm64.outputs.image }} diff --git a/.github/workflows/build-latest.yml b/.github/workflows/build-latest.yml deleted file mode 100644 index 3b3b3fd6d..000000000 --- a/.github/workflows/build-latest.yml +++ /dev/null @@ -1,95 +0,0 @@ -name: Latest - -# The `:latest` tag should only ever be formally released versions of -# software. That means each ref should only ever be specified as a version -# tag for software that's been released which is not a release candidate. - -on: - workflow_call: - secrets: - DOCKERHUB_USERNAME: - required: false - DOCKERHUB_TOKEN: - required: false - inputs: - sha: - description: 'Sha to build' - type: 'string' - required: true - tag-prefix: - description: 'Prefix for the tag name' - type: 'string' - default: '' - tag-alias-prefix: - description: 'Prefix for the alias tag name' - type: 'string' - default: '' - -jobs: - - amd64: - uses: ./.github/workflows/build.yml - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - with: - sha: ${{ inputs.sha }} - arch: amd64 - tag: ${{ inputs.tag-prefix }}latest-amd64 - protocol_version_default: 23 - xdr_ref: v23.0.0 - core_ref: v23.0.1 - horizon_ref: horizon-v23.0.0 - stellar_rpc_ref: v23.0.4 - friendbot_ref: horizon-v23.0.0 - lab_ref: main - test_matrix: | - { - "network": ["local"], - "core": ["core", null], - "horizon": ["horizon", null], - "rpc": ["rpc", null], - "options": [""], - "include": [ - { "network": "pubnet", "core": "core", "horizon": "horizon", "rpc": "rpc" } - ] - } - - arm64: - uses: ./.github/workflows/build.yml - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - with: - sha: ${{ inputs.sha }} - arch: arm64 - tag: ${{ inputs.tag-prefix }}latest-arm64 - protocol_version_default: 23 - xdr_ref: v23.0.0 - core_ref: v23.0.1 - horizon_ref: horizon-v23.0.0 - stellar_rpc_ref: v23.0.1 - friendbot_ref: horizon-v23.0.0 - lab_ref: main - test_matrix: | - { - "network": ["local"], - "core": ["core", null], - "horizon": ["horizon", null], - "rpc": ["rpc", null], - "options": [""], - "include": [ - { "network": "pubnet", "core": "core", "horizon": "horizon", "rpc": "rpc" } - ] - } - - manifest: - needs: [amd64, arm64] - uses: ./.github/workflows/manifest.yml - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - with: - tag: ${{ inputs.tag-prefix }}latest - tag-alias: ${{ inputs.tag-alias-prefix }}latest - images: ${{ needs.amd64.outputs.image }} ${{ needs.arm64.outputs.image }} diff --git a/.github/workflows/build-start.yml b/.github/workflows/build-start.yml deleted file mode 100644 index 2ebe14f01..000000000 --- a/.github/workflows/build-start.yml +++ /dev/null @@ -1,93 +0,0 @@ -name: Build - -on: - push: - branches: - - main - pull_request: - -# Prevent more than one build of this workflow for a branch to be running at the -# same time, and if multiple are queued, only run the latest, cancelling any -# already running build. The exception being any protected branch, such as -# main, where a build for every commit will run. -concurrency: - group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }} - cancel-in-progress: true - -jobs: - - complete: - if: always() - needs: [latest, testing, future, latest-action-test, testing-action-test] - runs-on: ubuntu-latest - steps: - - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') - run: exit 1 - - setup: - runs-on: ubuntu-latest - outputs: - tag-prefix: ${{ steps.tag-prefix.outputs.tag-prefix }} - tag-alias-prefix: ${{ steps.tag-prefix.outputs.tag-alias-prefix }} - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 # Get all history for the sha count below. - ref: ${{ github.event.pull_request.head.sha || github.sha }} - - id: tag-prefix - run: | - pr_prefix="${{ github.event_name == 'pull_request' && format('pr{0}-', github.event.pull_request.number) }}" - count="$(git rev-list HEAD --count --first-parent)" - echo "tag-prefix=${pr_prefix}v${count}-" >> $GITHUB_OUTPUT - echo "tag-alias-prefix=${pr_prefix}" >> $GITHUB_OUTPUT - - latest: - needs: [setup] - uses: ./.github/workflows/build-latest.yml - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - with: - sha: ${{ github.event.pull_request.head.sha || github.sha }} - tag-prefix: ${{ needs.setup.outputs.tag-prefix }} - tag-alias-prefix: ${{ needs.setup.outputs.tag-alias-prefix }} - - latest-action-test: - needs: [setup, latest] - if: | - github.event_name == 'push' && github.ref == 'refs/heads/main' - || github.event.pull_request.head.repo.full_name == github.repository - uses: ./.github/workflows/action-test.yml - with: - tag: ${{ needs.setup.outputs.tag-prefix }}latest - - testing: - needs: [setup] - uses: ./.github/workflows/build-testing.yml - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - with: - sha: ${{ github.event.pull_request.head.sha || github.sha }} - tag-prefix: ${{ needs.setup.outputs.tag-prefix }} - tag-alias-prefix: ${{ needs.setup.outputs.tag-alias-prefix }} - - testing-action-test: - needs: [setup, testing] - if: | - github.event_name == 'push' && github.ref == 'refs/heads/main' - || github.event.pull_request.head.repo.full_name == github.repository - uses: ./.github/workflows/action-test.yml - with: - tag: ${{ needs.setup.outputs.tag-prefix }}testing - - future: - needs: [setup] - uses: ./.github/workflows/build-future.yml - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - with: - sha: ${{ github.event.pull_request.head.sha || github.sha }} - tag-prefix: ${{ needs.setup.outputs.tag-prefix }} - tag-alias-prefix: ${{ needs.setup.outputs.tag-alias-prefix }} diff --git a/.github/workflows/build-testing.yml b/.github/workflows/build-testing.yml deleted file mode 100644 index ba2e11f0a..000000000 --- a/.github/workflows/build-testing.yml +++ /dev/null @@ -1,98 +0,0 @@ -name: Testing - -# The `:testing` tag should only ever be formally released or release candidate -# versions of software. That means each ref should only ever be specified as a -# version tag for software that's either the latest release candidate or latest -# release. - -on: - workflow_call: - secrets: - DOCKERHUB_USERNAME: - required: false - DOCKERHUB_TOKEN: - required: false - inputs: - sha: - description: 'Sha to build' - type: 'string' - required: true - tag-prefix: - description: 'Prefix for the tag name' - type: 'string' - default: '' - tag-alias-prefix: - description: 'Prefix for the alias tag name' - type: 'string' - default: '' - -jobs: - - amd64: - uses: ./.github/workflows/build.yml - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - with: - sha: ${{ inputs.sha }} - arch: amd64 - tag: ${{ inputs.tag-prefix }}testing-amd64 - protocol_version_default: 23 - xdr_ref: v23.0.0 - core_ref: v23.0.1 - horizon_ref: horizon-v23.0.0 - stellar_rpc_ref: v23.0.4 - friendbot_ref: horizon-v23.0.0 - lab_ref: main - test_matrix: | - { - "network": ["local"], - "core": ["core", null], - "horizon": ["horizon", null], - "rpc": ["rpc", null], - "options": [""], - "include": [ - { "network": "testnet", "core": "core", "horizon": "horizon", "rpc": "rpc" }, - { "network": "pubnet", "core": "core", "horizon": "horizon", "rpc": "rpc" } - ] - } - - arm64: - uses: ./.github/workflows/build.yml - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - with: - sha: ${{ inputs.sha }} - arch: arm64 - tag: ${{ inputs.tag-prefix }}testing-arm64 - protocol_version_default: 23 - xdr_ref: v23.0.0 - core_ref: v23.0.1 - horizon_ref: horizon-v23.0.0 - stellar_rpc_ref: v23.0.1 - friendbot_ref: horizon-v23.0.0 - lab_ref: main - test_matrix: | - { - "network": ["local"], - "core": ["core", null], - "horizon": ["horizon", null], - "rpc": ["rpc", null], - "options": [""], - "include": [ - { "network": "testnet", "core": "core", "horizon": "horizon", "rpc": "rpc" }, - { "network": "pubnet", "core": "core", "horizon": "horizon", "rpc": "rpc" } - ] - } - - manifest: - needs: [amd64, arm64] - uses: ./.github/workflows/manifest.yml - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - with: - tag: ${{ inputs.tag-prefix }}testing - tag-alias: ${{ inputs.tag-alias-prefix }}testing - images: ${{ needs.amd64.outputs.image }} ${{ needs.arm64.outputs.image }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 22bd555cb..000000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,638 +0,0 @@ -on: - workflow_call: - secrets: - DOCKERHUB_USERNAME: - required: false - DOCKERHUB_TOKEN: - required: false - inputs: - sha: - description: 'Sha to build' - type: 'string' - required: true - arch: - description: 'Architecture to build the image for (amd64, arm64)' - type: 'string' - required: true - tag: - description: 'Tag to use on the image name' - type: 'string' - required: true - protocol_version_default: - description: 'Default protocol version to use on local networks' - type: 'number' - required: true - core_repo: - description: 'Git repo for stellar-core' - type: 'string' - default: 'stellar/stellar-core' - core_ref: - description: 'Git ref for the stellar-core repo' - type: 'string' - required: true - core_configure_flags: - description: 'CONFIGURE_FLAGS used when building stellar-core' - type: 'string' - default: '--disable-tests' - horizon_ref: - description: 'Git ref for the stellar/go repo (horizon)' - type: 'string' - required: true - xdr_ref: - description: 'Git ref for the stellar/rs-stellar-xdr repo' - type: 'string' - required: false - stellar_rpc_ref: - description: 'Git ref for the stellar/stellar-rpc repo (stellar-rpc)' - type: 'string' - required: true - friendbot_ref: - description: 'Git ref for the stellar/go repo (friendbot)' - type: 'string' - required: true - lab_ref: - description: 'Git ref for the stellar/laboratory (lab)' - type: 'string' - required: true - test_matrix: - description: 'JSON matrix for the test job' - type: 'string' - required: true - outputs: - image: - description: 'Image pushed as a result of this build' - value: ${{ jobs.build.outputs.image }} - -env: - IMAGE: ${{ format('{0}/{1}:{2}', secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io', github.repository, inputs.tag) }} - # Cache ID is a value inserted into cache keys. Whenever changing the build - # in a way that needs to use entirely new fresh builds, increment the number - # by one so that all the keys become new. - CACHE_ID: 5 - ARTIFACT_RETENTION_DAYS_FOR_IMAGE: 7 - ARTIFACT_RETENTION_DAYS_FOR_LOGS: 60 - -jobs: - - shas: - runs-on: ubuntu-latest - outputs: - xdr: ${{ steps.xdr.outputs.sha }} - core: ${{ steps.core.outputs.sha }} - rpc: ${{ steps.rpc.outputs.sha }} - horizon: ${{ steps.horizon.outputs.sha }} - friendbot: ${{ steps.friendbot.outputs.sha }} - lab: ${{ steps.lab.outputs.sha }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - id: xdr - run: echo "sha=$(gh api repos/stellar/rs-stellar-xdr/commits/"${{inputs.xdr_ref}}" --jq '.sha')" | tee -a $GITHUB_OUTPUT - - id: core - run: echo "sha=$(gh api repos/${{inputs.core_repo}}/commits/"${{inputs.core_ref}}" --jq '.sha')" | tee -a $GITHUB_OUTPUT - - id: rpc - run: echo "sha=$(gh api repos/stellar/stellar-rpc/commits/"${{inputs.stellar_rpc_ref}}" --jq '.sha')" | tee -a $GITHUB_OUTPUT - - id: horizon - run: echo "sha=$(gh api repos/stellar/go/commits/"${{inputs.horizon_ref}}" --jq '.sha')" | tee -a $GITHUB_OUTPUT - - id: friendbot - run: echo "sha=$(gh api repos/stellar/go/commits/"${{inputs.friendbot_ref}}" --jq '.sha')" | tee -a $GITHUB_OUTPUT - - id: lab - run: echo "sha=$(gh api repos/stellar/laboratory/commits/"${{inputs.lab_ref}}" --jq '.sha')" | tee -a $GITHUB_OUTPUT - - load-stellar-core-from-cache: - needs: [shas] - runs-on: ubuntu-latest - outputs: - cache-hit: ${{ steps.cache.outputs.cache-hit }} - steps: - - id: cache - uses: actions/cache@v3 - with: - path: /tmp/image - key: image-${{ env.CACHE_ID }}-stellar-core-${{ inputs.arch }}-${{ needs.shas.outputs.core }}-${{ inputs.core_configure_flags }} - - name: Upload Stellar-Core Image - if: steps.cache.outputs.cache-hit == 'true' - uses: actions/upload-artifact@v4 - with: - name: image-stellar-core-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/image - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS_FOR_IMAGE }} - - build-stellar-core: - needs: [shas, load-stellar-core-from-cache] - if: ${{ needs.load-stellar-core-from-cache.outputs.cache-hit != 'true' }} - runs-on: ${{ inputs.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} - steps: - - name: Checkout Quickstart for Core docker file - uses: actions/checkout@v3 - with: - ref: ${{ inputs.sha }} - - id: cache - uses: actions/cache@v3 - with: - path: /tmp/image - key: image-${{ env.CACHE_ID }}-stellar-core-${{ inputs.arch }}-${{ needs.shas.outputs.core }}-${{ inputs.core_configure_flags }} - - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - - name: Build Stellar-Core Image - run: > - docker buildx build --platform linux/${{ inputs.arch }} - -f Dockerfile.core - -t stellar-core:${{ inputs.arch }} - -o type=docker,dest=/tmp/image - --build-arg REPO="${{ inputs.core_repo }}" - --build-arg REF="${{ needs.shas.outputs.core }}" - --build-arg CONFIGURE_FLAGS='${{ inputs.core_configure_flags }}' . - - name: Upload Stellar-Core Image - uses: actions/upload-artifact@v4 - with: - name: image-stellar-core-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/image - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS_FOR_IMAGE }} - - load-stellar-horizon-from-cache: - needs: [shas] - runs-on: ubuntu-latest - outputs: - cache-hit: ${{ steps.cache.outputs.cache-hit }} - steps: - - id: cache - uses: actions/cache@v3 - with: - path: /tmp/image - key: image-${{ env.CACHE_ID }}-stellar-horizon-${{ inputs.arch }}-${{ needs.shas.outputs.horizon }} - - name: Upload Stellar-Horizon Image - if: steps.cache.outputs.cache-hit == 'true' - uses: actions/upload-artifact@v4 - with: - name: image-stellar-horizon-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/image - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS_FOR_IMAGE }} - - build-stellar-horizon: - needs: [shas, load-stellar-horizon-from-cache] - if: ${{ needs.load-stellar-horizon-from-cache.outputs.cache-hit != 'true' }} - runs-on: ${{ inputs.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} - steps: - - id: cache - uses: actions/cache@v3 - with: - path: /tmp/image - key: image-${{ env.CACHE_ID }}-stellar-horizon-${{ inputs.arch }}-${{ needs.shas.outputs.horizon }} - - name: Checkout Quickstart for Horizon docker file - uses: actions/checkout@v3 - with: - ref: ${{ inputs.sha }} - - name: Setup buildx - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - - name: Build Stellar-Horizon Image - run: > - docker buildx build --platform linux/${{ inputs.arch }} - -f Dockerfile.horizon - -t stellar-horizon:${{ inputs.arch }} -o type=docker,dest=/tmp/image - --build-arg REF="${{ needs.shas.outputs.horizon }}" . - - name: Upload Stellar-Horizon Image - uses: actions/upload-artifact@v4 - with: - name: image-stellar-horizon-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/image - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS_FOR_IMAGE }} - - load-stellar-friendbot-from-cache: - needs: [shas] - runs-on: ubuntu-latest - outputs: - cache-hit: ${{ steps.cache.outputs.cache-hit }} - steps: - - id: cache - uses: actions/cache@v3 - with: - path: /tmp/image - key: image-${{ env.CACHE_ID }}-stellar-friendbot-${{ inputs.arch }}-${{ needs.shas.outputs.friendbot }} - - name: Upload Stellar-Friendbot Image - if: steps.cache.outputs.cache-hit == 'true' - uses: actions/upload-artifact@v4 - with: - name: image-stellar-friendbot-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/image - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS_FOR_IMAGE }} - - build-stellar-friendbot: - needs: [shas, load-stellar-friendbot-from-cache] - if: ${{ needs.load-stellar-friendbot-from-cache.outputs.cache-hit != 'true' }} - runs-on: ${{ inputs.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} - steps: - - id: cache - uses: actions/cache@v3 - with: - path: /tmp/image - key: image-${{ env.CACHE_ID }}-stellar-friendbot-${{ inputs.arch }}-${{ needs.shas.outputs.friendbot }} - - name: Checkout Quickstart for Friendbot docker file - uses: actions/checkout@v3 - with: - ref: ${{ inputs.sha }} - - name: Setup buildx - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - - name: Build Stellar-Friendbot Image - run: > - docker buildx build --platform linux/${{ inputs.arch }} - -f Dockerfile.friendbot - -t stellar-friendbot:${{ inputs.arch }} - -o type=docker,dest=/tmp/image - --build-arg REF="${{ needs.shas.outputs.friendbot }}" . - - name: Upload Stellar-Friendbot Image - uses: actions/upload-artifact@v4 - with: - name: image-stellar-friendbot-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/image - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS_FOR_IMAGE }} - - load-stellar-rpc-from-cache: - needs: [shas] - runs-on: ubuntu-latest - outputs: - cache-hit: ${{ steps.cache.outputs.cache-hit }} - steps: - - id: cache - uses: actions/cache@v3 - with: - path: /tmp/image - key: image-${{ env.CACHE_ID }}-stellar-rpc-${{ inputs.arch }}-${{ needs.shas.outputs.rpc }} - - name: Upload Stellar-Core Image - if: steps.cache.outputs.cache-hit == 'true' - uses: actions/upload-artifact@v4 - with: - name: image-stellar-rpc-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/image - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS_FOR_IMAGE }} - - build-stellar-rpc: - needs: [shas, load-stellar-rpc-from-cache] - if: ${{ needs.load-stellar-rpc-from-cache.outputs.cache-hit != 'true' }} - runs-on: ${{ inputs.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} - steps: - - id: cache - uses: actions/cache@v3 - with: - path: /tmp/image - key: image-${{ env.CACHE_ID }}-stellar-rpc-${{ inputs.arch }}-${{ needs.shas.outputs.rpc }} - - name: Checkout Quickstart for RPC docker file - uses: actions/checkout@v3 - with: - ref: ${{ inputs.sha }} - - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - - name: Build Stellar-rpc Image - run: > - docker buildx build --platform linux/${{ inputs.arch }} - -f Dockerfile.rpc - -t stellar-rpc:${{ inputs.arch }} - -o type=docker,dest=/tmp/image - --build-arg REF="${{ needs.shas.outputs.rpc }}" . - - name: Upload Stellar-rpc Image - uses: actions/upload-artifact@v4 - with: - name: image-stellar-rpc-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/image - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS_FOR_IMAGE }} - - load-stellar-lab-from-cache: - needs: [shas] - runs-on: ubuntu-latest - outputs: - cache-hit: ${{ steps.cache.outputs.cache-hit }} - steps: - - id: cache - uses: actions/cache@v3 - with: - path: /tmp/image - key: image-${{ env.CACHE_ID }}-stellar-lab-${{ inputs.arch }}-${{ needs.shas.outputs.lab }} - - name: Upload Stellar-Lab Image - if: steps.cache.outputs.cache-hit == 'true' - uses: actions/upload-artifact@v4 - with: - name: image-stellar-lab-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/image - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS_FOR_IMAGE }} - - build-stellar-lab: - needs: [shas, load-stellar-lab-from-cache] - if: ${{ needs.load-stellar-lab-from-cache.outputs.cache-hit != 'true' }} - runs-on: ${{ inputs.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} - steps: - - id: cache - uses: actions/cache@v3 - with: - path: /tmp/image - key: image-${{ env.CACHE_ID }}-stellar-lab-${{ inputs.arch }}-${{ needs.shas.outputs.lab }} - - name: Checkout Quickstart for Horizon docker file - uses: actions/checkout@v3 - with: - ref: ${{ inputs.sha }} - - name: Setup buildx - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - - name: Build Stellar-lab Image - run: > - docker buildx build --platform linux/${{ inputs.arch }} - -f Dockerfile.lab - -t stellar-lab:${{ inputs.arch }} -o type=docker,dest=/tmp/image - --build-arg NEXT_PUBLIC_COMMIT_HASH=${{ needs.shas.outputs.lab }} . - - name: Upload Stellar-lab Image - uses: actions/upload-artifact@v4 - with: - name: image-stellar-lab-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/image - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS_FOR_IMAGE }} - - load-rs-stellar-xdr-from-cache: - needs: [shas] - runs-on: ubuntu-latest - outputs: - cache-hit: ${{ steps.cache.outputs.cache-hit }} - steps: - - id: cache - uses: actions/cache@v3 - with: - path: /tmp/image - key: image-${{ env.CACHE_ID }}-rs-stellar-xdr-${{ inputs.arch }}-${{ needs.shas.outputs.xdr }} - - name: Upload Stellar-Core Image - if: steps.cache.outputs.cache-hit == 'true' - uses: actions/upload-artifact@v4 - with: - name: image-rs-stellar-xdr-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/image - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS_FOR_IMAGE }} - - build-rs-stellar-xdr: - needs: [shas, load-rs-stellar-xdr-from-cache] - if: ${{ needs.load-rs-stellar-xdr-from-cache.outputs.cache-hit != 'true' }} - runs-on: ${{ inputs.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} - steps: - - name: Checkout Quickstart for Horizon docker file - uses: actions/checkout@v3 - with: - ref: ${{ inputs.sha }} - - id: cache - uses: actions/cache@v3 - with: - path: /tmp/image - key: image-${{ env.CACHE_ID }}-rs-stellar-xdr-${{ inputs.arch }}-${{ needs.shas.outputs.xdr }} - - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - - name: Build Stellar-Rs-Xdr Image - run: > - docker buildx build --platform linux/${{ inputs.arch }} - -f Dockerfile.xdr - -t stellar-rs-xdr:${{ inputs.arch }} - -o type=docker,dest=/tmp/image - --build-arg REPO=stellar/rs-stellar-xdr - --build-arg REF="${{ needs.shas.outputs.xdr }}" . - - name: Upload Stellar-Rs-Xdr Image - uses: actions/upload-artifact@v4 - with: - name: image-rs-stellar-xdr-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/image - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS_FOR_IMAGE }} - - build: - needs: [build-stellar-core, build-stellar-horizon, build-rs-stellar-xdr, build-stellar-friendbot, build-stellar-rpc, build-stellar-lab] - if: always() && !failure() && !cancelled() - outputs: - image: ${{ steps.image.outputs.name }} - runs-on: ${{ inputs.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ inputs.sha }} - - name: Download Stellar XDR - uses: actions/download-artifact@v4 - with: - name: image-rs-stellar-xdr-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/stellar-xdr - - name: Download Stellar-Core Image - uses: actions/download-artifact@v4 - with: - name: image-stellar-core-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/stellar-core - - name: Download Stellar-Horizon Image - uses: actions/download-artifact@v4 - with: - name: image-stellar-horizon-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/stellar-horizon - - name: Download Stellar-Friendbot Image - uses: actions/download-artifact@v4 - with: - name: image-stellar-friendbot-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/stellar-friendbot - - name: Download Stellar-Lab Image - uses: actions/download-artifact@v4 - with: - name: image-stellar-lab-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/stellar-lab - - name: Download Stellar-rpc Image - uses: actions/download-artifact@v4 - with: - name: image-stellar-rpc-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/stellar-rpc - - name: Load Stellar-Core Image - run: docker load -i /tmp/stellar-core/image - - name: Load Stellar-Horizon Image - run: docker load -i /tmp/stellar-horizon/image - - name: Load Stellar-Friendbot Image - run: docker load -i /tmp/stellar-friendbot/image - - name: Load Stellar-Lab Image - run: docker load -i /tmp/stellar-lab/image - - name: Load Stellar-rpc Image - run: docker load -i /tmp/stellar-rpc/image - - name: Load Stellar-Rs-Xdr Image - run: docker load -i /tmp/stellar-xdr/image - - id: image - name: Image Name - run: echo "name=$IMAGE" >> $GITHUB_OUTPUT - - name: Pull Base Image - run: docker pull --platform linux/${{ inputs.arch }} ubuntu:22.04 - # Docker buildx cannot be used to build the dev quickstart image because - # buildx does not yet support importing existing images, like the core and - # horizon images above, into a buildx builder's cache. Buildx would be - # preferred because it can output a smaller image file faster than docker - # save can. Once buildx supports it we can update. - # https://github.com/docker/buildx/issues/847 - - name: Build Quickstart Image - run: > - docker build - --platform linux/${{ inputs.arch }} - -f Dockerfile - -t $IMAGE - --label org.opencontainers.image.revision="${{ inputs.sha }}" - . - --build-arg REVISION="${{ inputs.sha }}" - --build-arg PROTOCOL_VERSION_DEFAULT="${{ inputs.protocol_version_default }}" - --build-arg STELLAR_XDR_IMAGE_REF=stellar-rs-xdr:${{ inputs.arch }} - --build-arg STELLAR_CORE_IMAGE_REF=stellar-core:${{ inputs.arch }} - --build-arg HORIZON_IMAGE_REF=stellar-horizon:${{ inputs.arch }} - --build-arg FRIENDBOT_IMAGE_REF=stellar-friendbot:${{ inputs.arch }} - --build-arg STELLAR_RPC_IMAGE_REF=stellar-rpc:${{ inputs.arch }} - --build-arg LAB_IMAGE_REF=stellar-lab:${{ inputs.arch }} - - name: Save Quickstart Image - run: docker save $IMAGE -o /tmp/image - - name: Upload Quickstart Image - uses: actions/upload-artifact@v4 - with: - name: image-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/image - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS_FOR_IMAGE }} - - test: - needs: build - if: always() && !failure() && !cancelled() - strategy: - matrix: ${{ fromJSON(inputs.test_matrix) }} - fail-fast: false - runs-on: ${{ inputs.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} - steps: - - name: Free up disk space - if: matrix.network == 'pubnet' - run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf /usr/local/lib/android - sudo rm -rf /opt/ghc - sudo rm -rf /opt/hostedtoolcache/CodeQL - df -h - - uses: actions/checkout@v2 - with: - ref: ${{ inputs.sha }} - - name: Download Quickstart Image - uses: actions/download-artifact@v4 - with: - name: image-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/ - - name: Load Quickstart Image - run: docker load -i /tmp/image - - if: inputs.arch == 'arm64' - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - with: - platforms: arm64 - - name: Prepare Logs Directory - run: mkdir -p logs - - name: Run Quickstart Image - run: > - docker run - --platform linux/${{ inputs.arch }} - -d - -p - "8000:8000" - -p "11626:11626" - --name stellar - $IMAGE - --${{ matrix.network }} - --enable ${{ matrix.core }},${{ matrix.horizon }},${{ matrix.rpc }} - ${{ matrix.options }} - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: ^1 - - name: Sleep until supervisor is up - run: sleep 10 - - name: Run core test - if: ${{ matrix.core }} - run: | - docker logs stellar -f & - echo "supervisorctl tail -f stellar-core" | docker exec -i stellar sh & - go run tests/test_core.go - curl http://localhost:11626/info - - name: Run horizon up test - if: ${{ matrix.horizon }} - run: | - docker logs stellar -f & - echo "supervisorctl tail -f horizon" | docker exec -i stellar sh & - go run tests/test_horizon_up.go - curl http://localhost:8000 - - name: Run horizon core up test - if: ${{ matrix.horizon && matrix.network != 'pubnet' }} - run: | - docker logs stellar -f & - echo "supervisorctl tail -f horizon" | docker exec -i stellar sh & - go run tests/test_horizon_core_up.go - curl http://localhost:8000 - - name: Run horizon ingesting test - if: ${{ matrix.horizon && matrix.network != 'pubnet' }} - run: | - docker logs stellar -f & - echo "supervisorctl tail -f stellar-core" | docker exec -i stellar sh & - echo "supervisorctl tail -f horizon" | docker exec -i stellar sh & - go run tests/test_horizon_ingesting.go - curl http://localhost:8000 - - name: Run friendbot test - if: ${{ matrix.horizon && matrix.network == 'local' }} - run: | - docker logs stellar -f & - echo "supervisorctl tail -f friendbot" | docker exec -i stellar sh & - echo "supervisorctl tail -f horizon" | docker exec -i stellar sh & - go run tests/test_friendbot.go - - name: Run stellar rpc up test - if: ${{ matrix.rpc }} - run: | - docker logs stellar -f & - echo "supervisorctl tail -f stellar-rpc" | docker exec -i stellar sh & - go run tests/test_stellar_rpc_up.go - - name: Run stellar rpc healthy test - if: ${{ matrix.rpc && matrix.network != 'pubnet' }} - run: | - docker logs stellar -f & - echo "supervisorctl tail -f stellar-rpc" | docker exec -i stellar sh & - go run tests/test_stellar_rpc_healthy.go - - name: Prepare Test Logs - if: always() - run: docker cp stellar:/var/log logs - - name: Upload Test Logs - if: always() - uses: actions/upload-artifact@v4 - with: - name: logs-${{ inputs.tag }}-${{ inputs.arch }}-test-${{ strategy.job-index }} - path: logs - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS_FOR_LOGS }} - - push-pr: - # Push image to registry after build for pull requests from a local branch. - if: always() && !failure() && !cancelled() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository - needs: build - permissions: - packages: write - statuses: write - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - ref: ${{ inputs.sha }} - - id: push - uses: ./.github/actions/push - with: - head_sha: ${{ inputs.sha }} - artifact_name: image-${{ inputs.tag }}-${{ inputs.arch }} - artifact_image_file: image - arch: ${{ inputs.arch }} - image: ${{ env.IMAGE }} - registry: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} - username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} - password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} - - push-release: - # Push image to registry after test for main. - if: always() && !failure() && !cancelled() && github.event_name == 'push' && github.ref == 'refs/heads/main' - needs: [build, test] - permissions: - packages: write - statuses: write - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - ref: ${{ inputs.sha }} - - id: push - uses: ./.github/actions/push - with: - head_sha: ${{ inputs.sha }} - artifact_name: image-${{ inputs.tag }}-${{ inputs.arch }} - artifact_image_file: image - arch: ${{ inputs.arch }} - image: ${{ env.IMAGE }} - registry: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} - username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} - password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} diff --git a/.github/workflows/manifest.yml b/.github/workflows/manifest.yml deleted file mode 100644 index 752ae89a1..000000000 --- a/.github/workflows/manifest.yml +++ /dev/null @@ -1,90 +0,0 @@ -on: - workflow_call: - secrets: - DOCKERHUB_USERNAME: - required: false - DOCKERHUB_TOKEN: - required: false - inputs: - tag: - description: 'Tag to use as the manifest list image name' - type: 'string' - required: true - tag-alias: - description: 'Tag to alias to the tag of the manifest, e.g. "latest"' - type: 'string' - required: true - images: - description: 'Space separated list of images to include in the manifest list' - type: 'string' - required: true - outputs: - image: - description: 'Image pushed as a result of this build' - value: ${{ jobs.build.outputs.image }} - -env: - HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - IMAGE: ${{ format('{0}/{1}:{2}', secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io', github.repository, inputs.tag) }} - IMAGE_ALIAS: ${{ format('{0}/{1}:{2}', secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io', github.repository, inputs.tag-alias) }} - REGISTRY: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} - -jobs: - - input-validations: - runs-on: ubuntu-latest - steps: - - name: Check tag and tag-alias format has pr prefix for pr builds - if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main') }} - run: | - if ! [[ "${{ inputs.tag }}" =~ ^pr[0-9]+- ]]; then - echo "Error: 'tag' input must start with 'prN-'" - exit 1 - fi - if ! [[ "${{ inputs.tag-alias }}" =~ ^pr[0-9]+- ]]; then - echo "Error: 'tag-alias' input must start with 'prN-'" - exit 1 - fi - - push: - needs: input-validations - if: ${{ github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) }} - permissions: - packages: write - statuses: write - runs-on: ubuntu-latest - steps: - - id: image_parts - run: | - IMAGE_TAGLESS=$(echo ${{ env.IMAGE }} | cut -d':' -f1) - IMAGE_REPO=$(echo $IMAGE_TAGLESS | cut -d'/' -f2,3) - IMAGE_TAG=$(echo ${{ env.IMAGE }} | cut -d':' -f2) - echo "::set-output name=repo::$IMAGE_REPO" - echo "::set-output name=tag::$IMAGE_TAG" - - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 - with: - registry: ${{ env.REGISTRY }} - username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} - password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} - - run: echo "IMAGE_URL=https://${{ env.IMAGE }}" >> $GITHUB_ENV - - if: ${{ env.REGISTRY == 'docker.io' }} - run: | - echo "IMAGE_URL=https://hub.docker.com/r/${{ steps.image_parts.outputs.repo }}/tags?name=${{ steps.image_parts.outputs.tag }}" >> $GITHUB_ENV - - run: | - docker manifest create ${{ env.IMAGE }} ${{ inputs.images }} - - run: | - docker manifest push ${{ env.IMAGE }} - - run: | - docker buildx imagetools create -t ${{ env.IMAGE_ALIAS }} ${{ env.IMAGE }} - - uses: actions/github-script@v5 - with: - script: | - github.rest.repos.createCommitStatus({ - owner: context.repo.owner, - repo: context.repo.repo, - sha: '${{ env.HEAD_SHA }}', - state: 'success', - context: `${{ env.IMAGE }}`, - target_url: '${{ env.IMAGE_URL }}', - description: 'Available', - }); From 4a3cd970c9298cd2a02cec075ada032eb6f0c575 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Fri, 3 Oct 2025 21:41:25 +1000 Subject: [PATCH 004/135] update github workflow condition --- .github/workflows/newbuild.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/newbuild.yml b/.github/workflows/newbuild.yml index 0cb9bb8a3..e24c8e45b 100644 --- a/.github/workflows/newbuild.yml +++ b/.github/workflows/newbuild.yml @@ -371,7 +371,7 @@ jobs: push: needs: test - if: always() && !failure() && !cancelled() && ((github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository))) + if: always() && !failure() && !cancelled() && ((github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository)) strategy: matrix: tag: ${{ fromJSON(needs.setup.outputs.images).*.tag }} From 73cf864976617ffb2bc23a5dcfa01d5f8262cbe6 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Fri, 3 Oct 2025 21:43:31 +1000 Subject: [PATCH 005/135] update stellar-core ref from main to master in images.json --- images.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images.json b/images.json index b96dde80e..b8da23a16 100644 --- a/images.json +++ b/images.json @@ -56,7 +56,7 @@ }, "deps": [ { "name": "xdr", "repo": "stellar/rs-stellar-xdr", "ref": "main" }, - { "name": "core", "repo": "stellar/stellar-core", "ref": "main", "options": { "configure_flags": "--disable-tests" } }, + { "name": "core", "repo": "stellar/stellar-core", "ref": "master", "options": { "configure_flags": "--disable-tests" } }, { "name": "rpc", "repo": "stellar/stellar-rpc", "ref": "main" }, { "name": "horizon", "repo": "stellar/go", "ref": "master" }, { "name": "friendbot", "repo": "stellar/go", "ref": "master" }, @@ -71,7 +71,7 @@ }, "deps": [ { "name": "xdr", "repo": "stellar/rs-stellar-xdr", "ref": "main" }, - { "name": "core", "repo": "stellar/stellar-core", "ref": "main", "options": { "configure_flags": "--disable-tests --enable-next-protocol-version-unsafe-for-production" } }, + { "name": "core", "repo": "stellar/stellar-core", "ref": "master", "options": { "configure_flags": "--disable-tests --enable-next-protocol-version-unsafe-for-production" } }, { "name": "rpc", "repo": "stellar/stellar-rpc", "ref": "main" }, { "name": "horizon", "repo": "stellar/go", "ref": "master" }, { "name": "friendbot", "repo": "stellar/go", "ref": "master" }, From 0671faf630aaa0b2dd02b07acdd7a51e95008a6e Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Fri, 3 Oct 2025 21:46:34 +1000 Subject: [PATCH 006/135] rename build workflow file --- .github/workflows/{newbuild.yml => build.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{newbuild.yml => build.yml} (100%) diff --git a/.github/workflows/newbuild.yml b/.github/workflows/build.yml similarity index 100% rename from .github/workflows/newbuild.yml rename to .github/workflows/build.yml From 5d1a48158d8d46d32bbd6da31d73607dcf2d4d93 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Fri, 3 Oct 2025 21:59:40 +1000 Subject: [PATCH 007/135] remove repo from build workflow names and simplify runs-on condition --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e24c8e45b..092f4bc42 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -86,7 +86,7 @@ jobs: dep: ${{ fromJSON(needs.setup.outputs.deps) }} arch: ["amd64", "arm64"] fail-fast: false - name: load ${{ matrix.dep.name }} ${{ matrix.dep.repo }} ${{ matrix.dep.ref }} ${{ matrix.arch }} ${{ matrix.dep.options && toJSON(matrix.dep.options) }} + name: load ${{ matrix.dep.name }} ${{ matrix.dep.ref }} ${{ matrix.arch }} ${{ matrix.dep.options && toJSON(matrix.dep.options) }} runs-on: ubuntu-latest env: dep_json: ${{ toJSON(matrix.dep) }} @@ -152,8 +152,8 @@ jobs: matrix: dep: ${{ fromJSON(needs.prepare.outputs.deps-to-build) }} fail-fast: false - name: build ${{ matrix.dep.name }} ${{ matrix.dep.repo }} ${{ matrix.dep.ref }} ${{ matrix.dep.arch }} ${{ matrix.dep.options && toJSON(matrix.dep.options) }} - runs-on: ${{ matrix.dep.name == 'core' && (matrix.dep.arch == 'arm64' && 'ubuntu-jammy-16-cores-amd64' || 'ubuntu-latest-16-cores') || (matrix.dep.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest') }} + name: build ${{ matrix.dep.name }} ${{ matrix.dep.ref }} ${{ matrix.dep.arch }} ${{ matrix.dep.options && toJSON(matrix.dep.options) }} + runs-on: ${{ matrix.dep.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} env: image_filename: image-${{ matrix.dep.name }}-${{ matrix.dep.id }}-${{ matrix.dep.arch }}.tar steps: From 231e4e99b0eb5b1e3ab54d61809c92a4ae19ac06 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Fri, 3 Oct 2025 22:02:43 +1000 Subject: [PATCH 008/135] update configure flags file name --- Dockerfile.core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.core b/Dockerfile.core index 5acd57fbe..1e1f053de 100644 --- a/Dockerfile.core +++ b/Dockerfile.core @@ -31,7 +31,7 @@ RUN sysctl vm.mmap_rnd_bits=28 RUN ./autogen.sh RUN ./install-rust.sh ENV PATH "/root/.cargo/bin:$PATH" -RUN sh -c './configure CC="${CC}" CXX="${CXX}" CFLAGS="${CFLAGS}" CXXFLAGS="${CXXFLAGS}" $( Date: Fri, 3 Oct 2025 23:03:46 +1000 Subject: [PATCH 009/135] add push alias action and update manifest workflow --- .github/actions/push-alias/action.yml | 41 +++++++ .../{manifest => push-manifest}/action.yml | 42 +++++--- .github/actions/push/action.yml | 4 +- .github/workflows/build.yml | 101 +++++++++++------- 4 files changed, 134 insertions(+), 54 deletions(-) create mode 100644 .github/actions/push-alias/action.yml rename .github/actions/{manifest => push-manifest}/action.yml (61%) diff --git a/.github/actions/push-alias/action.yml b/.github/actions/push-alias/action.yml new file mode 100644 index 000000000..da2968830 --- /dev/null +++ b/.github/actions/push-alias/action.yml @@ -0,0 +1,41 @@ +name: 'Push Alias' +inputs: + head_sha: + required: true + image: + required: true + image-alias: + required: true + registry: + required: true + default: ghcr.io + username: + required: true + default: ${{ github.actor }} + password: + required: true + default: ${{ github.token }} +runs: + using: "composite" + steps: + - + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + registry: ${{ inputs.registry }} + username: ${{ inputs.username }} + password: ${{ inputs.password }} + - + run: | + docker buildx imagetools create -t ${{ inputs.image-alias }} ${{ inputs.image }} + - + uses: actions/github-script@v5 + with: + script: | + github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: '${{ inputs.head_sha }}', + state: 'success', + context: `${{ inputs.image-alias }}`, + description: 'Available', + }); diff --git a/.github/actions/manifest/action.yml b/.github/actions/push-manifest/action.yml similarity index 61% rename from .github/actions/manifest/action.yml rename to .github/actions/push-manifest/action.yml index 3d2f985b4..dc9ecd63f 100644 --- a/.github/actions/manifest/action.yml +++ b/.github/actions/push-manifest/action.yml @@ -1,18 +1,7 @@ -name: 'Manifest' +name: 'Push Manifest' inputs: - head_sha: - required: true - artifact_name: - required: true - artifact_image_file: - required: true - artifact_image_name: - required: true - arch: - required: true image: required: true - default: ghcr.io/${{ github.repository }}:latest registry: required: true default: ghcr.io @@ -73,3 +62,32 @@ runs: target_url: '${{ env.IMAGE_URL }}', description: 'Available', }); + + - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} + password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} + - run: > + docker manifest create + ${{ env.image_repo }}:${{ steps.tag.outputs.tag }} + ${{ env.image_repo }}:${{ steps.tag.outputs.tag }}-amd64 + ${{ env.image_repo }}:${{ steps.tag.outputs.tag }}-arm64 + - run: > + docker manifest push + ${{ env.image_repo }}:${{ steps.tag.outputs.tag }} + - run: > + docker buildx imagetools create -t + ${{ env.image_repo }}:${{ steps.tag.outputs.tag-alias }} + ${{ env.image_repo }}:${{ steps.tag.outputs.tag }} + - uses: actions/github-script@v5 + with: + script: | + github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: '${{ env.sha }}', + state: 'success', + context: `${{ env.image_repo }}:${{ steps.tag.outputs.tag }}`, + description: 'Available', + }); diff --git a/.github/actions/push/action.yml b/.github/actions/push/action.yml index 9ec398fa0..ba4870cb2 100644 --- a/.github/actions/push/action.yml +++ b/.github/actions/push/action.yml @@ -40,8 +40,8 @@ runs: IMAGE_TAGLESS=$(echo ${{ inputs.image }} | cut -d':' -f1) IMAGE_REPO=$(echo $IMAGE_TAGLESS | cut -d'/' -f2,3) IMAGE_TAG=$(echo ${{ inputs.image }} | cut -d':' -f2) - echo "::set-output name=repo::$IMAGE_REPO" - echo "::set-output name=tag::$IMAGE_TAG" + echo "repo=$IMAGE_REPO" | tee -a $GITHUB_OUTPUTS + echo "tag=$IMAGE_TAG" | tee -a $GITHUB_OUTPUTS - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 with: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 092f4bc42..73c40e6e3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -178,7 +178,7 @@ jobs: uses: actions/cache/save@v3 id: cache with: - key: ${{ env.cache_id }}-${{env.image_filename }} + key: ${{ env.cache_id }}-${{ env.image_filename }} path: /tmp/${{ env.image_filename }} - name: Upload Image to Artifacts uses: actions/upload-artifact@v4 @@ -197,14 +197,50 @@ jobs: fail-fast: false name: build quickstart ${{ matrix.image.tag }} ${{ matrix.arch }} runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} + env: + image_json: ${{ matrix.image }} steps: - uses: actions/checkout@v3 with: ref: ${{ env.sha }} - - name: Download Images + - name: Collect Dep IDs + id: ids + run: + echo "$(<<< $image_json jq -r '.deps[] | "\(.name)=\(.id)"')" | tee -a $GITHUB_OUTPUT + - name: Download Image XDR + uses: actions/download-artifact@v4 + with: + pattern: image-xdr-${{ steps.ids.outputs.xdr }}-${{ matrix.arch }}.* + merge-multiple: true + path: /tmp/images + - name: Download Image Core + uses: actions/download-artifact@v4 + with: + pattern: image-core-${{ steps.ids.outputs.core }}-${{ matrix.arch }}.* + merge-multiple: true + path: /tmp/images + - name: Download Image RPC + uses: actions/download-artifact@v4 + with: + pattern: image-rpc-${{ steps.ids.outputs.rpc }}-${{ matrix.arch }}.* + merge-multiple: true + path: /tmp/images + - name: Download Image Horizon + uses: actions/download-artifact@v4 + with: + pattern: image-horizon-${{ steps.ids.outputs.horizon }}-${{ matrix.arch }}.* + merge-multiple: true + path: /tmp/images + - name: Download Image Friendbot + uses: actions/download-artifact@v4 + with: + pattern: image-friendbot-${{ steps.ids.outputs.friendbot }}-${{ matrix.arch }}.* + merge-multiple: true + path: /tmp/images + - name: Download Image Lab uses: actions/download-artifact@v4 with: - pattern: image-*-${{ matrix.arch }}.* + pattern: image-lab-${{ steps.ids.outputs.lab }}-${{ matrix.arch }}.* merge-multiple: true path: /tmp/images - name: Load Image into Docker @@ -217,7 +253,7 @@ jobs: done - name: Create Tag id: tag - run: echo "tag=${{ needs.setup.outputs.tag-prefix }}${{ matrix.image.tag }}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + run: echo "tag=${{ needs.setup.outputs.tag-prefix }}${{ matrix.image.tag }}-${{ matrix.arch }}" | tee -a $GITHUB_OUTPUT - name: Pull Base Image run: docker pull --platform linux/${{ matrix.arch }} ubuntu:22.04 # Docker buildx cannot be used to build the dev quickstart image because @@ -288,7 +324,7 @@ jobs: run: docker load -i /tmp/image - name: Create Tag id: tag - run: echo "tag=${{ needs.setup.outputs.tag-prefix }}${{ matrix.tag }}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + run: echo "tag=${{ needs.setup.outputs.tag-prefix }}${{ matrix.tag }}-${{ matrix.arch }}" | tee -a $GITHUB_OUTPUT - name: Prepare Logs Directory run: mkdir -p logs - name: Run Quickstart Image @@ -382,12 +418,9 @@ jobs: statuses: write runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - with: - ref: ${{ inputs.sha }} - name: Create Tag id: tag - run: echo "tag=${{ needs.setup.outputs.tag-prefix }}${{ matrix.tag }}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + run: echo "tag=${{ needs.setup.outputs.tag-prefix }}${{ matrix.tag }}-${{ matrix.arch }}" | tee -a $GITHUB_OUTPUT - uses: ./.github/actions/push with: head_sha: ${{ env.sha }} @@ -399,9 +432,9 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} - manifest: + push-manifest: needs: push - if: (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) + if: always() && !failure() && !cancelled() && ((github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository)) strategy: matrix: tag: ${{ fromJSON(needs.setup.outputs.images).*.tag }} @@ -414,40 +447,28 @@ jobs: - name: Create Tag id: tag run: | - echo "tag=${{ needs.setup.outputs.tag-prefix }}${{ matrix.tag }}" >> $GITHUB_OUTPUT - echo "tag-alias=${{ needs.setup.outputs.tag-alias-prefix }}${{ matrix.tag }}" >> $GITHUB_OUTPUT - - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + echo "tag=${{ needs.setup.outputs.tag-prefix }}${{ matrix.tag }}" | tee -a $GITHUB_OUTPUT + echo "tag-alias=${{ needs.setup.outputs.tag-alias-prefix }}${{ matrix.tag }}" | tee -a $GITHUB_OUTPUT + - uses: ./.github/actions/push-manifest with: - registry: ${{ env.REGISTRY }} + head_sha: ${{ env.sha }} + image: ${{ env.image_repo }}:${{ steps.tag.outputs.tag }} + images: ${{ env.image_repo }}:${{ steps.tag.outputs.tag }}-amd64 ${{ env.image_repo }}:${{ steps.tag.outputs.tag }}-arm64 + registry: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} - - run: > - docker manifest create - ${{ env.image_repo }}:${{ steps.tag.outputs.tag }} - ${{ env.image_repo }}:${{ steps.tag.outputs.tag }}-amd64 - ${{ env.image_repo }}:${{ steps.tag.outputs.tag }}-arm64 - - run: > - docker manifest push - ${{ env.image_repo }}:${{ steps.tag.outputs.tag }} - - run: > - docker buildx imagetools create -t - ${{ env.image_repo }}:${{ steps.tag.outputs.tag-alias }} - ${{ env.image_repo }}:${{ steps.tag.outputs.tag }} - - uses: actions/github-script@v5 + - uses: ./.github/actions/push-alias with: - script: | - github.rest.repos.createCommitStatus({ - owner: context.repo.owner, - repo: context.repo.repo, - sha: '${{ env.sha }}', - state: 'success', - context: `${{ env.image_repo }}:${{ steps.tag.outputs.tag }}`, - description: 'Available', - }); - + head_sha: ${{ env.sha }} + image: ${{ env.image_repo }}:${{ steps.tag.outputs.tag }} + image-alias: ${{ env.image_repo }}:${{ steps.tag.outputs.tag-alias }} + registry: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} + username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} + password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} + action: - needs: [setup, manifest] - if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event.pull_request.head.repo.full_name == github.repository + needs: [setup, push-manifest] + if: always() && !failure() && !cancelled() && ((github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository)) strategy: matrix: tag: ${{ fromJSON(needs.setup.outputs.images).*.tag }} From 08df62fa5cd17732b23435df20b070a0d30a87ca Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Fri, 3 Oct 2025 23:04:17 +1000 Subject: [PATCH 010/135] update build workflow to use push-manifest instead of manifest --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 73c40e6e3..a58a0f16c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,7 +29,7 @@ jobs: complete: if: always() - needs: [setup, load, build, test, push, manifest, action] + needs: [setup, load, build, test, push, push-manifest, action] runs-on: ubuntu-latest steps: - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') From 729899193f1ad6b831ef6c487dea6ebdbc75c70b Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Fri, 3 Oct 2025 23:08:33 +1000 Subject: [PATCH 011/135] use toJSON to serialize matrix image in build workflow --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a58a0f16c..c0842a4ab 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -198,7 +198,7 @@ jobs: name: build quickstart ${{ matrix.image.tag }} ${{ matrix.arch }} runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} env: - image_json: ${{ matrix.image }} + image_json: ${{ toJSON(matrix.image) }} steps: - uses: actions/checkout@v3 with: From 75cbd0004acf3805c7015e6bc69b1b15c2f0e1ac Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Fri, 3 Oct 2025 23:20:10 +1000 Subject: [PATCH 012/135] remove nightly-next image configuration --- images.json | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/images.json b/images.json index b8da23a16..e54c549af 100644 --- a/images.json +++ b/images.json @@ -63,20 +63,5 @@ { "name": "lab", "repo": "stellar/laboratory", "ref": "main" } ], "additional-tests": [] - }, - { - "tag": "nightly-next", - "config": { - "protocol_version_default": 24 - }, - "deps": [ - { "name": "xdr", "repo": "stellar/rs-stellar-xdr", "ref": "main" }, - { "name": "core", "repo": "stellar/stellar-core", "ref": "master", "options": { "configure_flags": "--disable-tests --enable-next-protocol-version-unsafe-for-production" } }, - { "name": "rpc", "repo": "stellar/stellar-rpc", "ref": "main" }, - { "name": "horizon", "repo": "stellar/go", "ref": "master" }, - { "name": "friendbot", "repo": "stellar/go", "ref": "master" }, - { "name": "lab", "repo": "stellar/laboratory", "ref": "main" } - ], - "additional-tests": [] } ] From 5bc9ff7e1a4304fd5d97556d7c83ce05c0ff5bc0 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Fri, 3 Oct 2025 23:39:21 +1000 Subject: [PATCH 013/135] update workflow dependencies --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c0842a4ab..e4b16edd9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -406,7 +406,7 @@ jobs: retention-days: ${{ env.artifact_retention_days_for_logs }} push: - needs: test + needs: [setup, test] if: always() && !failure() && !cancelled() && ((github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository)) strategy: matrix: @@ -433,7 +433,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} push-manifest: - needs: push + needs: [setup, push] if: always() && !failure() && !cancelled() && ((github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository)) strategy: matrix: From 9189d7ddfdc51d1dea04e737ec46ddebfa4b1b05 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Fri, 3 Oct 2025 23:54:54 +1000 Subject: [PATCH 014/135] add checkout step with sha ref to build workflows --- .github/workflows/build.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e4b16edd9..6afa6a28d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -418,6 +418,9 @@ jobs: statuses: write runs-on: ubuntu-latest steps: + - uses: actions/checkout@v2 + with: + ref: ${{ env.sha }} - name: Create Tag id: tag run: echo "tag=${{ needs.setup.outputs.tag-prefix }}${{ matrix.tag }}-${{ matrix.arch }}" | tee -a $GITHUB_OUTPUT @@ -444,6 +447,9 @@ jobs: statuses: write runs-on: ubuntu-latest steps: + - uses: actions/checkout@v2 + with: + ref: ${{ env.sha }} - name: Create Tag id: tag run: | From 3858e8626e964a7940c526581aba36b2d6b91fd5 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Sat, 4 Oct 2025 00:42:27 +1000 Subject: [PATCH 015/135] update artifact name with tar extension --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6afa6a28d..737b3f814 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -427,7 +427,7 @@ jobs: - uses: ./.github/actions/push with: head_sha: ${{ env.sha }} - artifact_name: image-quickstart-${{ matrix.tag }}-${{ matrix.arch }} + artifact_name: image-quickstart-${{ matrix.tag }}-${{ matrix.arch }}.tar artifact_image_file: image arch: ${{ matrix.arch }} image: ${{ env.image_repo }}:${{ steps.tag.outputs.tag }} From 0af571fab6bc33f63fcb588352b99b9d303d7224 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Sat, 4 Oct 2025 01:43:04 +1000 Subject: [PATCH 016/135] update push manifest action to use images input and simplify docker manifest steps --- .github/actions/push-manifest/action.yml | 75 +++++------------------- 1 file changed, 14 insertions(+), 61 deletions(-) diff --git a/.github/actions/push-manifest/action.yml b/.github/actions/push-manifest/action.yml index dc9ecd63f..0af196f2e 100644 --- a/.github/actions/push-manifest/action.yml +++ b/.github/actions/push-manifest/action.yml @@ -2,6 +2,8 @@ name: 'Push Manifest' inputs: image: required: true + images: + required: true registry: required: true default: ghcr.io @@ -14,73 +16,24 @@ inputs: runs: using: "composite" steps: - - - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.artifact_name }} - path: /tmp/ - - - shell: bash - run: docker load -i /tmp/${{ inputs.artifact_image_file }} - - - id: image_parts - shell: bash - run: | - IMAGE_TAGLESS=$(echo ${{ inputs.image }} | cut -d':' -f1) - IMAGE_REPO=$(echo $IMAGE_TAGLESS | cut -d'/' -f2,3) - IMAGE_TAG=$(echo ${{ inputs.image }} | cut -d':' -f2) - echo "::set-output name=repo::$IMAGE_REPO" - echo "::set-output name=tag::$IMAGE_TAG" - - + - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 with: registry: ${{ inputs.registry }} username: ${{ inputs.username }} password: ${{ inputs.password }} - - - shell: bash - run: echo "IMAGE_URL=https://${{ inputs.image }}" >> $GITHUB_ENV - - - if: ${{ inputs.registry == 'docker.io' }} - shell: bash - run: | - echo "IMAGE_URL=https://hub.docker.com/r/${{ steps.image_parts.outputs.repo }}/tags?name=${{ steps.image_parts.outputs.tag }}" >> $GITHUB_ENV - - - shell: bash - run: | - docker push ${{ inputs.image }} - - - uses: actions/github-script@v5 - with: - script: | - github.rest.repos.createCommitStatus({ - owner: context.repo.owner, - repo: context.repo.repo, - sha: '${{ inputs.head_sha }}', - state: 'success', - context: `${{ inputs.image }}`, - target_url: '${{ env.IMAGE_URL }}', - description: 'Available', - }); - - - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 - with: - registry: ${{ env.REGISTRY }} - username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} - password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} - - run: > + - + run: > docker manifest create - ${{ env.image_repo }}:${{ steps.tag.outputs.tag }} - ${{ env.image_repo }}:${{ steps.tag.outputs.tag }}-amd64 - ${{ env.image_repo }}:${{ steps.tag.outputs.tag }}-arm64 - - run: > + ${{ inputs.image }} + ${{ inputs.image }}-amd64 + ${{ inputs.image }}-arm64 + - + run: > docker manifest push - ${{ env.image_repo }}:${{ steps.tag.outputs.tag }} - - run: > - docker buildx imagetools create -t - ${{ env.image_repo }}:${{ steps.tag.outputs.tag-alias }} - ${{ env.image_repo }}:${{ steps.tag.outputs.tag }} - - uses: actions/github-script@v5 + ${{ inputs.image }} + - + uses: actions/github-script@v5 with: script: | github.rest.repos.createCommitStatus({ @@ -88,6 +41,6 @@ runs: repo: context.repo.repo, sha: '${{ env.sha }}', state: 'success', - context: `${{ env.image_repo }}:${{ steps.tag.outputs.tag }}`, + context: `${{ inputs.image }}`, description: 'Available', }); From b190df3cdb2ecdd600236ce3446721ef387f0e50 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Sat, 4 Oct 2025 01:43:42 +1000 Subject: [PATCH 017/135] update manifest action to use input sha --- .github/actions/push-manifest/action.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/actions/push-manifest/action.yml b/.github/actions/push-manifest/action.yml index 0af196f2e..b88543682 100644 --- a/.github/actions/push-manifest/action.yml +++ b/.github/actions/push-manifest/action.yml @@ -1,5 +1,7 @@ name: 'Push Manifest' inputs: + head_sha: + required: true image: required: true images: @@ -39,7 +41,7 @@ runs: github.rest.repos.createCommitStatus({ owner: context.repo.owner, repo: context.repo.repo, - sha: '${{ env.sha }}', + sha: '${{ inputs.head_sha }}', state: 'success', context: `${{ inputs.image }}`, description: 'Available', From bbab6352ed15c23bad18a0b8e8cf132bb229bbed Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Sat, 4 Oct 2025 02:01:32 +1000 Subject: [PATCH 018/135] add shell: bash to docker commands --- .github/actions/push-alias/action.yml | 1 + .github/actions/push-manifest/action.yml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.github/actions/push-alias/action.yml b/.github/actions/push-alias/action.yml index da2968830..a98a55414 100644 --- a/.github/actions/push-alias/action.yml +++ b/.github/actions/push-alias/action.yml @@ -25,6 +25,7 @@ runs: username: ${{ inputs.username }} password: ${{ inputs.password }} - + shell: bash run: | docker buildx imagetools create -t ${{ inputs.image-alias }} ${{ inputs.image }} - diff --git a/.github/actions/push-manifest/action.yml b/.github/actions/push-manifest/action.yml index b88543682..08e67de91 100644 --- a/.github/actions/push-manifest/action.yml +++ b/.github/actions/push-manifest/action.yml @@ -25,12 +25,14 @@ runs: username: ${{ inputs.username }} password: ${{ inputs.password }} - + shell: bash run: > docker manifest create ${{ inputs.image }} ${{ inputs.image }}-amd64 ${{ inputs.image }}-arm64 - + shell: bash run: > docker manifest push ${{ inputs.image }} From 4bf51b0bc81602d4454ac06f87cf2bd51510d2a5 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Sat, 4 Oct 2025 02:02:44 +1000 Subject: [PATCH 019/135] remove image url generation and output steps --- .github/actions/push/action.yml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/.github/actions/push/action.yml b/.github/actions/push/action.yml index ba4870cb2..df0037583 100644 --- a/.github/actions/push/action.yml +++ b/.github/actions/push/action.yml @@ -33,29 +33,12 @@ runs: - shell: bash run: docker load -i /tmp/${{ inputs.artifact_image_file }} - - - id: image_parts - shell: bash - run: | - IMAGE_TAGLESS=$(echo ${{ inputs.image }} | cut -d':' -f1) - IMAGE_REPO=$(echo $IMAGE_TAGLESS | cut -d'/' -f2,3) - IMAGE_TAG=$(echo ${{ inputs.image }} | cut -d':' -f2) - echo "repo=$IMAGE_REPO" | tee -a $GITHUB_OUTPUTS - echo "tag=$IMAGE_TAG" | tee -a $GITHUB_OUTPUTS - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 with: registry: ${{ inputs.registry }} username: ${{ inputs.username }} password: ${{ inputs.password }} - - - shell: bash - run: echo "IMAGE_URL=https://${{ inputs.image }}" >> $GITHUB_ENV - - - if: ${{ inputs.registry == 'docker.io' }} - shell: bash - run: | - echo "IMAGE_URL=https://hub.docker.com/r/${{ steps.image_parts.outputs.repo }}/tags?name=${{ steps.image_parts.outputs.tag }}" >> $GITHUB_ENV - shell: bash run: | @@ -70,6 +53,5 @@ runs: sha: '${{ inputs.head_sha }}', state: 'success', context: `${{ inputs.image }}`, - target_url: '${{ env.IMAGE_URL }}', description: 'Available', }); From f864c87a6bea93e0ee05544ce5d61337487d4124 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Sat, 4 Oct 2025 02:03:15 +1000 Subject: [PATCH 020/135] remove redundant artifact image name input --- .github/actions/push/action.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/actions/push/action.yml b/.github/actions/push/action.yml index df0037583..2f9d72cc2 100644 --- a/.github/actions/push/action.yml +++ b/.github/actions/push/action.yml @@ -6,8 +6,6 @@ inputs: required: true artifact_image_file: required: true - artifact_image_name: - required: true arch: required: true image: From 668abf25178dad94a36a55b84c1b17c271501c51 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Sat, 4 Oct 2025 08:26:16 +1000 Subject: [PATCH 021/135] update workflow job names with numbered prefixes and consistent formatting --- .github/workflows/build.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 737b3f814..35ad05e94 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,6 +29,7 @@ jobs: complete: if: always() + name: 10 complete needs: [setup, load, build, test, push, push-manifest, action] runs-on: ubuntu-latest steps: @@ -36,6 +37,7 @@ jobs: run: exit 1 setup: + name: 1 setup runs-on: ubuntu-latest outputs: tag-prefix: ${{ steps.tag-prefix.outputs.tag-prefix }} @@ -86,7 +88,7 @@ jobs: dep: ${{ fromJSON(needs.setup.outputs.deps) }} arch: ["amd64", "arm64"] fail-fast: false - name: load ${{ matrix.dep.name }} ${{ matrix.dep.ref }} ${{ matrix.arch }} ${{ matrix.dep.options && toJSON(matrix.dep.options) }} + name: 2 load (${{ matrix.dep.name }}, ${{ matrix.dep.ref }}, ${{ matrix.arch }}, ${{ matrix.dep.options && toJSON(matrix.dep.options) || '-' }}) runs-on: ubuntu-latest env: dep_json: ${{ toJSON(matrix.dep) }} @@ -128,7 +130,7 @@ jobs: prepare: needs: [load] - name: prepare + name: 3 prepare runs-on: ubuntu-latest outputs: deps-to-build: ${{ steps.deps-to-build.outputs.deps }} @@ -152,7 +154,7 @@ jobs: matrix: dep: ${{ fromJSON(needs.prepare.outputs.deps-to-build) }} fail-fast: false - name: build ${{ matrix.dep.name }} ${{ matrix.dep.ref }} ${{ matrix.dep.arch }} ${{ matrix.dep.options && toJSON(matrix.dep.options) }} + name: 4 build (${{ matrix.dep.name }}, ${{ matrix.dep.ref }}, ${{ matrix.dep.arch }}, ${{ matrix.dep.options && toJSON(matrix.dep.options) || '-' }}) runs-on: ${{ matrix.dep.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} env: image_filename: image-${{ matrix.dep.name }}-${{ matrix.dep.id }}-${{ matrix.dep.arch }}.tar @@ -195,7 +197,7 @@ jobs: image: ${{ fromJSON(needs.setup.outputs.images) }} arch: ["amd64", "arm64"] fail-fast: false - name: build quickstart ${{ matrix.image.tag }} ${{ matrix.arch }} + name: 5 build quickstart (${{ matrix.image.tag }}, ${{ matrix.arch }}) runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} env: image_json: ${{ toJSON(matrix.image) }} @@ -301,7 +303,7 @@ jobs: options: [""] include: ${{ fromJSON(needs.setup.outputs.additional-tests) }} fail-fast: false - name: test ${{ matrix.tag }} ${{ matrix.arch }} ${{ matrix.network }} ${{ matrix.core && 'core' || '' }} ${{ matrix.rpc && 'rpc' || '' }} ${{ matrix.horizon && 'horizon' || '' }} ${{ matrix.options || '' }} + name: 6 test (${{ matrix.tag }}, ${{ matrix.arch }}, ${{ matrix.network }}, ${{ matrix.core && 'core' || '' }} ${{ matrix.rpc && 'rpc' || '' }} ${{ matrix.horizon && 'horizon' || '' }} ${{ matrix.options || '' }}) runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} steps: - name: Free up disk space @@ -413,6 +415,7 @@ jobs: tag: ${{ fromJSON(needs.setup.outputs.images).*.tag }} arch: ["amd64", "arm64"] fail-fast: false + name: 7 push (${{ matrix.tag }}, ${{ matrix.arch }}) permissions: packages: write statuses: write @@ -442,6 +445,7 @@ jobs: matrix: tag: ${{ fromJSON(needs.setup.outputs.images).*.tag }} fail-fast: false + name: 8 push manifest (${{ matrix.tag }}) permissions: packages: write statuses: write @@ -479,6 +483,7 @@ jobs: matrix: tag: ${{ fromJSON(needs.setup.outputs.images).*.tag }} fail-fast: false + name: 9 test action (${{ matrix.tag }}) uses: ./.github/workflows/action-test.yml with: tag: ${{ needs.setup.outputs.tag-prefix }}${{ matrix.tag }} From 9486e23711cbda72360af8cd1b5776c13d311ce9 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Sat, 4 Oct 2025 08:26:47 +1000 Subject: [PATCH 022/135] update cache id to 6 --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 35ad05e94..dcfb61687 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,7 +20,7 @@ env: # Cache ID is a value inserted into cache keys. Whenever changing the build # in a way that needs to use entirely new fresh builds, increment the number # by one so that all the keys become new. - cache_id: 5 + cache_id: 6 artifact_retention_days_for_image: 7 artifact_retention_days_for_tombstone: 7 artifact_retention_days_for_logs: 60 From 9065c7dd6cfc9aa852b20905ef97d5a29933c19c Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Sat, 4 Oct 2025 08:28:17 +1000 Subject: [PATCH 023/135] update build workflow name format --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dcfb61687..a3e52ab79 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -197,7 +197,7 @@ jobs: image: ${{ fromJSON(needs.setup.outputs.images) }} arch: ["amd64", "arm64"] fail-fast: false - name: 5 build quickstart (${{ matrix.image.tag }}, ${{ matrix.arch }}) + name: 5 build (quickstart, ${{ matrix.image.tag }}, ${{ matrix.arch }}) runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} env: image_json: ${{ toJSON(matrix.image) }} From 0a579be28da13df073653d3163b7ee03992a44e4 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Sat, 4 Oct 2025 15:32:24 +1000 Subject: [PATCH 024/135] add load step to build job dependencies --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a3e52ab79..66796ca5e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -190,7 +190,7 @@ jobs: retention-days: ${{ env.artifact_retention_days_for_image }} build: - needs: [setup, build-dep] + needs: [setup, load, build-dep] if: always() && !failure() && !cancelled() strategy: matrix: From 00f89751fe12ad6f19d7a0cd1e9de0d67d9ab7e2 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Mon, 6 Oct 2025 00:15:18 +1000 Subject: [PATCH 025/135] update workflow to use images-with-extras script and remove obsolete scripts --- .github/workflows/build.yml | 3 +-- .scripts/images-with-dep-ids | 24 ------------------- ...mages-with-dep-shas => images-with-extras} | 23 ++++++++++++++---- 3 files changed, 19 insertions(+), 31 deletions(-) delete mode 100755 .scripts/images-with-dep-ids rename .scripts/{images-with-dep-shas => images-with-extras} (50%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 66796ca5e..ed78f800c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -63,8 +63,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | images="$(> $GITHUB_OUTPUT echo "images=$images" >> $GITHUB_ENV diff --git a/.scripts/images-with-dep-ids b/.scripts/images-with-dep-ids deleted file mode 100755 index 6963f0051..000000000 --- a/.scripts/images-with-dep-ids +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python3 - -import json -import sys -import subprocess -import hashlib - -# Accepts as stdin a JSON object in the format of images.json. Hashes the -# entire dep details and injects the hash as an id into the deps details. The -# id can be used to uniquely identify a dep configuration. -# Usage: < images.json ./.scripts/images-with-dep-ids - -images = json.load(sys.stdin) - -cache = {} - -for image in images: - for dep in image["deps"]: - dep_str = json.dumps(dep, separators=(',', ':')) - hash = hashlib.sha256(dep_str.encode()).hexdigest() - dep["id"] = hash - dep["id_short"] = hash[:5] - -print(json.dumps(images, separators=(',', ':'))) diff --git a/.scripts/images-with-dep-shas b/.scripts/images-with-extras similarity index 50% rename from .scripts/images-with-dep-shas rename to .scripts/images-with-extras index 34183ceb0..900357460 100755 --- a/.scripts/images-with-dep-shas +++ b/.scripts/images-with-extras @@ -3,11 +3,18 @@ import json import sys import subprocess +import hashlib # Accepts as stdin a JSON object in the format of images.json. -# Resolves any 'ref' values in the JSON to a commit sha. -# Outputs the original JSON modified with the shas. -# Usage: < images.json ./.scripts/images-with-dep-shas +# And adds some calculatble elements used during the build. +# +# 1. Resolves any 'ref' values in the JSON to a revision sha. +# 2. Hashes the entire dep details and injects the hash as an id into the deps +# details. The id can be used to uniquely identify a dep configuration. +# +# Outputs the original JSON modified. +# +# Usage: < images.json ./.scripts/images-with-extras images = json.load(sys.stdin) @@ -19,7 +26,7 @@ for image in images: name = dep["name"] repo = dep["repo"] ref = dep["ref"] - print(f"{tag} {name} {repo} {ref} ...", end=" ", flush=True, file=sys.stderr) + print(f"{tag} {name} {repo} {ref} ...", file=sys.stderr) key = (name, repo, ref) if key in cache: sha = cache[key] @@ -31,7 +38,13 @@ for image in images: check=True ).stdout.strip() cache[key] = sha - print(sha, file=sys.stderr) + print(f" • revision sha = {sha}", file=sys.stderr) dep["sha"] = sha + dep_str = json.dumps(dep, separators=(',', ':')) + id = hashlib.sha256(dep_str.encode()).hexdigest() + dep["id"] = id + dep["id_short"] = id[:5] + print(f" • id = {id}", file=sys.stderr) + print(json.dumps(images, separators=(',', ':'))) From 7b332bf75b6c74e34be09cb04f895d11d79ff5cb Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 06:46:39 +1000 Subject: [PATCH 026/135] add build to push job dependencies --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ed78f800c..2abd1c724 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -407,7 +407,7 @@ jobs: retention-days: ${{ env.artifact_retention_days_for_logs }} push: - needs: [setup, test] + needs: [setup, build, test] if: always() && !failure() && !cancelled() && ((github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository)) strategy: matrix: From d215612d65e96c6ed7e550bd123dc6550b238ebd Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 08:52:23 +1000 Subject: [PATCH 027/135] update build workflow to include scheduled runs and image filtering --- .github/workflows/build.yml | 9 ++++++++- images.json | 4 ++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2abd1c724..3d63c30f0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -5,6 +5,8 @@ on: branches: - main pull_request: + schedule: + - cron: '0 0 * * *' # Prevent more than one build of this workflow for a branch to be running at the # same time, and if multiple are queued, only run the latest, cancelling any @@ -58,11 +60,16 @@ jobs: echo "tag-prefix=${pr_prefix}v${count}-" | tee -a $GITHUB_OUTPUT echo "tag-alias-prefix=${pr_prefix}" | tee -a $GITHUB_OUTPUT - name: Images + run: | + images="$(> $GITHUB_ENV + - name: Images with Extras id: images env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - images="$(> $GITHUB_OUTPUT diff --git a/images.json b/images.json index e54c549af..a07fd570b 100644 --- a/images.json +++ b/images.json @@ -1,6 +1,7 @@ [ { "tag": "latest", + "events": ["pull_request", "push"], "config": { "protocol_version_default": 23 }, @@ -18,6 +19,7 @@ }, { "tag": "testing", + "events": ["pull_request", "push"], "config": { "protocol_version_default": 23 }, @@ -36,6 +38,7 @@ }, { "tag": "future", + "events": ["pull_request", "push"], "config": { "protocol_version_default": 23 }, @@ -51,6 +54,7 @@ }, { "tag": "nightly", + "events": ["push", "schedule"], "config": { "protocol_version_default": 23 }, From 446e5ae58b2b0ac5724150ba858573dcf60e52bd Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 09:26:57 +1000 Subject: [PATCH 028/135] update json processing to preserve formatting --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3d63c30f0..61c4605ef 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -62,7 +62,7 @@ jobs: - name: Images run: | images="$(> $GITHUB_ENV - name: Images with Extras From cb1f537571d10d7c0db66750c98f71ca5383cf84 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 09:55:36 +1000 Subject: [PATCH 029/135] rename complete job to remove numeric prefix --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 61c4605ef..d3321bcc1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,7 +31,7 @@ jobs: complete: if: always() - name: 10 complete + name: complete needs: [setup, load, build, test, push, push-manifest, action] runs-on: ubuntu-latest steps: From e007bd64326c9fe45f735b6679eac93371d07fb8 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 13:49:26 +1000 Subject: [PATCH 030/135] write image config to json file and add to docker build --- .github/workflows/build.yml | 3 +++ Dockerfile | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d3321bcc1..e4571eca7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -262,6 +262,9 @@ jobs: - name: Create Tag id: tag run: echo "tag=${{ needs.setup.outputs.tag-prefix }}${{ matrix.image.tag }}-${{ matrix.arch }}" | tee -a $GITHUB_OUTPUT + - name: Write Image Config + run: | + echo "[${{ toJSON(matrix.image) }}]" > image.json - name: Pull Base Image run: docker pull --platform linux/${{ matrix.arch }} ubuntu:22.04 # Docker buildx cannot be used to build the dev quickstart image because diff --git a/Dockerfile b/Dockerfile index 843b72d6f..097b07e41 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,6 +42,8 @@ RUN adduser --system --group --quiet --home /var/lib/stellar --disabled-password RUN ["mkdir", "-p", "/opt/stellar"] RUN ["touch", "/opt/stellar/.docker-ephemeral"] +ADD image.json /opt/stellar/image.json + RUN ["rm", "-fr", "/etc/supervisor"] RUN ["ln", "-sT", "/opt/stellar/supervisor/etc", "/etc/supervisor"] @@ -57,6 +59,7 @@ ADD futurenet /opt/stellar-default/futurenet ADD start / RUN ["chmod", "+x", "start"] + ARG PROTOCOL_VERSION_DEFAULT RUN test -n "$PROTOCOL_VERSION_DEFAULT" || (echo "Image build arg PROTOCOL_VERSION_DEFAULT required and not set" && false) ENV PROTOCOL_VERSION_DEFAULT $PROTOCOL_VERSION_DEFAULT From 1dc383c8c9fdb7727662cb4670a4fe7aa521a916 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 13:49:52 +1000 Subject: [PATCH 031/135] remove array wrapper from json output --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e4571eca7..de13d24ac 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -264,7 +264,7 @@ jobs: run: echo "tag=${{ needs.setup.outputs.tag-prefix }}${{ matrix.image.tag }}-${{ matrix.arch }}" | tee -a $GITHUB_OUTPUT - name: Write Image Config run: | - echo "[${{ toJSON(matrix.image) }}]" > image.json + echo "${{ toJSON(matrix.image) }}" > image.json - name: Pull Base Image run: docker pull --platform linux/${{ matrix.arch }} ubuntu:22.04 # Docker buildx cannot be used to build the dev quickstart image because From f0c71f3b43ed4c3c3ca36500d5772bb95da7a7ba Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 13:50:47 +1000 Subject: [PATCH 032/135] write image config to file in build workflow --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index de13d24ac..c346000ae 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -215,6 +215,9 @@ jobs: id: ids run: echo "$(<<< $image_json jq -r '.deps[] | "\(.name)=\(.id)"')" | tee -a $GITHUB_OUTPUT + - name: Write Image Config + run: | + echo "$image_json" > image.json - name: Download Image XDR uses: actions/download-artifact@v4 with: @@ -262,9 +265,6 @@ jobs: - name: Create Tag id: tag run: echo "tag=${{ needs.setup.outputs.tag-prefix }}${{ matrix.image.tag }}-${{ matrix.arch }}" | tee -a $GITHUB_OUTPUT - - name: Write Image Config - run: | - echo "${{ toJSON(matrix.image) }}" > image.json - name: Pull Base Image run: docker pull --platform linux/${{ matrix.arch }} ubuntu:22.04 # Docker buildx cannot be used to build the dev quickstart image because From b262f7ddc2a91635b4df82fc0889c7613bc4c0d7 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 14:25:48 +1000 Subject: [PATCH 033/135] update build workflow to include build number in tag prefix --- .github/workflows/build.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c346000ae..37ca4b7a0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -56,8 +56,9 @@ jobs: id: tag-prefix run: | pr_prefix="${{ github.event_name == 'pull_request' && format('pr{0}-', github.event.pull_request.number) || '' }}" - count="$(git rev-list HEAD --count --first-parent)" - echo "tag-prefix=${pr_prefix}v${count}-" | tee -a $GITHUB_OUTPUT + commit_count="$(git rev-list HEAD --count --first-parent)" + build_number="${{ github.run_number }}.${{ github.run_attempt }}" + echo "tag-prefix=${pr_prefix}-v${commit_count}-b${build_number}-" | tee -a $GITHUB_OUTPUT echo "tag-alias-prefix=${pr_prefix}" | tee -a $GITHUB_OUTPUT - name: Images run: | From 4aa51153edcdba2e18899441603133ce70f5f26d Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 14:42:41 +1000 Subject: [PATCH 034/135] add json version files to docker images and show them in start script --- Dockerfile | 2 ++ Dockerfile.friendbot | 2 ++ Dockerfile.lab | 2 ++ start | 11 +++++++++-- 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 097b07e41..f56bdfa01 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,8 +33,10 @@ COPY --from=xdr /stellar-xdr /usr/local/bin/stellar-xdr COPY --from=core /stellar-core /usr/bin/stellar-core COPY --from=horizon /horizon /usr/bin/stellar-horizon COPY --from=friendbot /friendbot /usr/local/bin/friendbot +COPY --from=friendbot /friendbot.json /usr/local/bin/friendbot.json COPY --from=rpc /stellar-rpc /usr/bin/stellar-rpc COPY --from=lab /lab /opt/stellar/lab +COPY --from=lab /lab.json /opt/stellar/lab.json COPY --from=lab /node /usr/bin/ RUN adduser --system --group --quiet --home /var/lib/stellar --disabled-password --shell /bin/bash stellar; diff --git a/Dockerfile.friendbot b/Dockerfile.friendbot index 04353715d..5fd4c48c8 100644 --- a/Dockerfile.friendbot +++ b/Dockerfile.friendbot @@ -9,7 +9,9 @@ RUN git checkout ${REF} ENV CGO_ENABLED=0 ENV GOFLAGS="-ldflags=-X=github.com/stellar/go/support/app.version=${REF}-(built-from-source)" RUN go install github.com/stellar/go/services/friendbot +RUN go version -json -m /go/bin/friendbot > /go/bin/friendbot.json FROM scratch AS artifacts COPY --from=builder /go/bin/friendbot /friendbot +COPY --from=builder /go/bin/friendbot.json /friendbot.json diff --git a/Dockerfile.lab b/Dockerfile.lab index 5eac9bb1a..16ce9598d 100644 --- a/Dockerfile.lab +++ b/Dockerfile.lab @@ -18,10 +18,12 @@ ENV NEXT_PUBLIC_DEFAULT_NETWORK=custom ENV NEXT_PUBLIC_RESOURCE_PATH=/lab ENV NEXT_BASE_PATH=/lab RUN pnpm build +RUN echo '{"version":"'${REF}'"}' > lab.json FROM scratch AS artifacts COPY --from=builder /lab/build/standalone /lab COPY --from=builder /lab/public /lab/public COPY --from=builder /lab/build/static /lab/public/_next/static +COPY --from=builder /lab/lab.json /lab.json COPY --from=builder /usr/local/bin/node /node diff --git a/start b/start index edc6c9bec..d5bcebf9a 100755 --- a/start +++ b/start @@ -80,12 +80,19 @@ function start() { echo "versions:" echo " quickstart: $REVISION" + echo " stellar-xdr:" + echo "$(stellar-xdr version | sed 's/^/ /')" echo " stellar-core:" echo "$(stellar-core version 2>/dev/null | sed 's/^/ /')" - echo " horizon:" - echo "$(stellar-horizon version | sed 's/^/ /')" echo " stellar-rpc:" echo "$(stellar-rpc version | sed 's/^/ /')" + echo " horizon:" + echo "$(stellar-horizon version | sed 's/^/ /')" + echo " friendbot:" + echo "$(< "$(which stellar-friendbot).json" jq -r '.Main.Version' | sed 's/^/ /')" + echo "$(< "$(which stellar-friendbot).json" jq -r '.GoVersion' | sed 's/^/ /')" + echo " lab:" + echo "$(< /opt/stellar/lab/lab.json jq -r '.version' | sed 's/^/ /')" echo "mode: $STELLAR_MODE" echo "network: $NETWORK" From 3b4a4738db025db1c10c13f18fbda4f6f732dc94 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 14:43:32 +1000 Subject: [PATCH 035/135] remove dash from tag prefix --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 37ca4b7a0..891cbe846 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -58,7 +58,7 @@ jobs: pr_prefix="${{ github.event_name == 'pull_request' && format('pr{0}-', github.event.pull_request.number) || '' }}" commit_count="$(git rev-list HEAD --count --first-parent)" build_number="${{ github.run_number }}.${{ github.run_attempt }}" - echo "tag-prefix=${pr_prefix}-v${commit_count}-b${build_number}-" | tee -a $GITHUB_OUTPUT + echo "tag-prefix=${pr_prefix}v${commit_count}-b${build_number}-" | tee -a $GITHUB_OUTPUT echo "tag-alias-prefix=${pr_prefix}" | tee -a $GITHUB_OUTPUT - name: Images run: | From 019ad00d91f7058d89a262d1a488ba32e526f27c Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 15:47:41 +1000 Subject: [PATCH 036/135] remove friendbot.json and lab.json copies and update version display to use images.json --- Dockerfile | 4 +--- Dockerfile.friendbot | 2 -- Dockerfile.lab | 2 -- start | 7 ++++--- 4 files changed, 5 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index f56bdfa01..b0e5eb26a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,10 +33,8 @@ COPY --from=xdr /stellar-xdr /usr/local/bin/stellar-xdr COPY --from=core /stellar-core /usr/bin/stellar-core COPY --from=horizon /horizon /usr/bin/stellar-horizon COPY --from=friendbot /friendbot /usr/local/bin/friendbot -COPY --from=friendbot /friendbot.json /usr/local/bin/friendbot.json COPY --from=rpc /stellar-rpc /usr/bin/stellar-rpc COPY --from=lab /lab /opt/stellar/lab -COPY --from=lab /lab.json /opt/stellar/lab.json COPY --from=lab /node /usr/bin/ RUN adduser --system --group --quiet --home /var/lib/stellar --disabled-password --shell /bin/bash stellar; @@ -44,7 +42,7 @@ RUN adduser --system --group --quiet --home /var/lib/stellar --disabled-password RUN ["mkdir", "-p", "/opt/stellar"] RUN ["touch", "/opt/stellar/.docker-ephemeral"] -ADD image.json /opt/stellar/image.json +ADD image.json /image.json RUN ["rm", "-fr", "/etc/supervisor"] RUN ["ln", "-sT", "/opt/stellar/supervisor/etc", "/etc/supervisor"] diff --git a/Dockerfile.friendbot b/Dockerfile.friendbot index 5fd4c48c8..04353715d 100644 --- a/Dockerfile.friendbot +++ b/Dockerfile.friendbot @@ -9,9 +9,7 @@ RUN git checkout ${REF} ENV CGO_ENABLED=0 ENV GOFLAGS="-ldflags=-X=github.com/stellar/go/support/app.version=${REF}-(built-from-source)" RUN go install github.com/stellar/go/services/friendbot -RUN go version -json -m /go/bin/friendbot > /go/bin/friendbot.json FROM scratch AS artifacts COPY --from=builder /go/bin/friendbot /friendbot -COPY --from=builder /go/bin/friendbot.json /friendbot.json diff --git a/Dockerfile.lab b/Dockerfile.lab index 16ce9598d..5eac9bb1a 100644 --- a/Dockerfile.lab +++ b/Dockerfile.lab @@ -18,12 +18,10 @@ ENV NEXT_PUBLIC_DEFAULT_NETWORK=custom ENV NEXT_PUBLIC_RESOURCE_PATH=/lab ENV NEXT_BASE_PATH=/lab RUN pnpm build -RUN echo '{"version":"'${REF}'"}' > lab.json FROM scratch AS artifacts COPY --from=builder /lab/build/standalone /lab COPY --from=builder /lab/public /lab/public COPY --from=builder /lab/build/static /lab/public/_next/static -COPY --from=builder /lab/lab.json /lab.json COPY --from=builder /usr/local/bin/node /node diff --git a/start b/start index d5bcebf9a..a94e3185f 100755 --- a/start +++ b/start @@ -89,10 +89,11 @@ function start() { echo " horizon:" echo "$(stellar-horizon version | sed 's/^/ /')" echo " friendbot:" - echo "$(< "$(which stellar-friendbot).json" jq -r '.Main.Version' | sed 's/^/ /')" - echo "$(< "$(which stellar-friendbot).json" jq -r '.GoVersion' | sed 's/^/ /')" + echo "$(< images.json jq -r '.deps[] | select(.name == "friendbot") | .ref' | sed 's/^/ /')" + echo "$(< images.json jq -r '.deps[] | select(.name == "friendbot") | .sha' | sed 's/^/ /')" echo " lab:" - echo "$(< /opt/stellar/lab/lab.json jq -r '.version' | sed 's/^/ /')" + echo "$(< images.json jq -r '.deps[] | select(.name == "lab") | .ref' | sed 's/^/ /')" + echo "$(< images.json jq -r '.deps[] | select(.name == "lab") | .sha' | sed 's/^/ /')" echo "mode: $STELLAR_MODE" echo "network: $NETWORK" From 6c7fd366a2729d40454392471fe8f9a10a6f4940 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 15:47:51 +1000 Subject: [PATCH 037/135] use image.json instead of images.json in start script --- start | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/start b/start index a94e3185f..7953125dd 100755 --- a/start +++ b/start @@ -89,11 +89,11 @@ function start() { echo " horizon:" echo "$(stellar-horizon version | sed 's/^/ /')" echo " friendbot:" - echo "$(< images.json jq -r '.deps[] | select(.name == "friendbot") | .ref' | sed 's/^/ /')" - echo "$(< images.json jq -r '.deps[] | select(.name == "friendbot") | .sha' | sed 's/^/ /')" + echo "$(< image.json jq -r '.deps[] | select(.name == "friendbot") | .ref' | sed 's/^/ /')" + echo "$(< image.json jq -r '.deps[] | select(.name == "friendbot") | .sha' | sed 's/^/ /')" echo " lab:" - echo "$(< images.json jq -r '.deps[] | select(.name == "lab") | .ref' | sed 's/^/ /')" - echo "$(< images.json jq -r '.deps[] | select(.name == "lab") | .sha' | sed 's/^/ /')" + echo "$(< image.json jq -r '.deps[] | select(.name == "lab") | .ref' | sed 's/^/ /')" + echo "$(< image.json jq -r '.deps[] | select(.name == "lab") | .sha' | sed 's/^/ /')" echo "mode: $STELLAR_MODE" echo "network: $NETWORK" From b0b630730212bb2b9c7c56c9c80c9dd7d3d63fec Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 15:48:12 +1000 Subject: [PATCH 038/135] shorten service names in version output --- start | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/start b/start index 7953125dd..1a6273710 100755 --- a/start +++ b/start @@ -80,11 +80,11 @@ function start() { echo "versions:" echo " quickstart: $REVISION" - echo " stellar-xdr:" + echo " xdr:" echo "$(stellar-xdr version | sed 's/^/ /')" - echo " stellar-core:" + echo " core:" echo "$(stellar-core version 2>/dev/null | sed 's/^/ /')" - echo " stellar-rpc:" + echo " rpc:" echo "$(stellar-rpc version | sed 's/^/ /')" echo " horizon:" echo "$(stellar-horizon version | sed 's/^/ /')" From ab06f6c68f7a2824682dd2e2e60916fde7cb28a1 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 15:55:02 +1000 Subject: [PATCH 039/135] update json configuration to use enable array instead of individual flags --- .github/workflows/build.yml | 6 ++---- images.json | 6 +++--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 891cbe846..4e22a8012 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -307,9 +307,7 @@ jobs: tag: ${{ fromJSON(needs.setup.outputs.images).*.tag }} arch: ["amd64", "arm64"] network: ["local"] - core: [true, false] - horizon: [true, false] - rpc: [true, false] + enable: ["core", "rpc", "core,rpc,horizon"] options: [""] include: ${{ fromJSON(needs.setup.outputs.additional-tests) }} fail-fast: false @@ -350,7 +348,7 @@ jobs: --name stellar ${{ env.image_repo }}:${{ steps.tag.outputs.tag }} --${{ matrix.network }} - --enable ${{ matrix.core && 'core' }},${{ matrix.rpc && 'rpc' }},${{ matrix.horizon && 'horizon' }} + --enable ${{ matrix.enable }} ${{ matrix.options }} - name: Set up Go uses: actions/setup-go@v2 diff --git a/images.json b/images.json index a07fd570b..26306eebf 100644 --- a/images.json +++ b/images.json @@ -14,7 +14,7 @@ { "name": "lab", "repo": "stellar/laboratory", "ref": "main" } ], "additional-tests": [ - { "arch": "amd64", "network": "pubnet", "core": true, "horizon": true, "rpc": true } + { "arch": "amd64", "network": "pubnet", "enable": ["core,rpc,horizon"] } ] }, { @@ -32,8 +32,8 @@ { "name": "lab", "repo": "stellar/laboratory", "ref": "main" } ], "additional-tests": [ - { "arch": "amd64", "network": "testnet", "core": true, "horizon": true, "rpc": true }, - { "arch": "amd64", "network": "pubnet", "core": true, "horizon": true, "rpc": true } + { "arch": "amd64", "network": "testnet", "enable": ["core,rpc,horizon"] }, + { "arch": "amd64", "network": "pubnet", "enable": ["core,rpc,horizon"] } ] }, { From d456c9717ca94512a8cbfc228186cfc9547a317e Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 16:02:06 +1000 Subject: [PATCH 040/135] update test job name to use enable matrix field instead of core rpc horizon fields --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4e22a8012..97c584860 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -311,7 +311,7 @@ jobs: options: [""] include: ${{ fromJSON(needs.setup.outputs.additional-tests) }} fail-fast: false - name: 6 test (${{ matrix.tag }}, ${{ matrix.arch }}, ${{ matrix.network }}, ${{ matrix.core && 'core' || '' }} ${{ matrix.rpc && 'rpc' || '' }} ${{ matrix.horizon && 'horizon' || '' }} ${{ matrix.options || '' }}) + name: 6 test (${{ matrix.tag }}, ${{ matrix.arch }}, ${{ matrix.network }}, ${{ matrix.enable }} ${{ matrix.options || '' }}) runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} steps: - name: Free up disk space From edfb955bc612bf6ebed2821f4c9672af462f13bf Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 16:04:46 +1000 Subject: [PATCH 041/135] update workflow conditions to use enable matrix --- .github/workflows/build.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 97c584860..68e3fb6f4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -357,28 +357,28 @@ jobs: - name: Sleep until supervisor is up run: sleep 10 - name: Run core test - if: ${{ matrix.core }} + if: ${{ contains(matrix.enable, 'core') }} run: | docker logs stellar -f & echo "supervisorctl tail -f stellar-core" | docker exec -i stellar sh & go run tests/test_core.go curl http://localhost:11626/info - name: Run horizon up test - if: ${{ matrix.horizon }} + if: ${{ contains(matrix.enable, 'horizon') }} run: | docker logs stellar -f & echo "supervisorctl tail -f horizon" | docker exec -i stellar sh & go run tests/test_horizon_up.go curl http://localhost:8000 - name: Run horizon core up test - if: ${{ matrix.horizon && matrix.network != 'pubnet' }} + if: ${{ contains(matrix.enable, 'horizon') && matrix.network != 'pubnet' }} run: | docker logs stellar -f & echo "supervisorctl tail -f horizon" | docker exec -i stellar sh & go run tests/test_horizon_core_up.go curl http://localhost:8000 - name: Run horizon ingesting test - if: ${{ matrix.horizon && matrix.network != 'pubnet' }} + if: ${{ contains(matrix.enable, 'horizon') && matrix.network != 'pubnet' }} run: | docker logs stellar -f & echo "supervisorctl tail -f stellar-core" | docker exec -i stellar sh & @@ -386,20 +386,20 @@ jobs: go run tests/test_horizon_ingesting.go curl http://localhost:8000 - name: Run friendbot test - if: ${{ matrix.horizon && matrix.network == 'local' }} + if: ${{ contains(matrix.enable, 'horizon') && matrix.network == 'local' }} run: | docker logs stellar -f & echo "supervisorctl tail -f friendbot" | docker exec -i stellar sh & echo "supervisorctl tail -f horizon" | docker exec -i stellar sh & go run tests/test_friendbot.go - name: Run stellar rpc up test - if: ${{ matrix.rpc }} + if: ${{ contains(matrix.enable, 'rpc') }} run: | docker logs stellar -f & echo "supervisorctl tail -f stellar-rpc" | docker exec -i stellar sh & go run tests/test_stellar_rpc_up.go - name: Run stellar rpc healthy test - if: ${{ matrix.rpc && matrix.network != 'pubnet' }} + if: ${{ contains(matrix.enable, 'rpc') && matrix.network != 'pubnet' }} run: | docker logs stellar -f & echo "supervisorctl tail -f stellar-rpc" | docker exec -i stellar sh & From ae2cbfe3f6e31ba6c1d44a7d4017566fff5bb4e4 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 16:09:10 +1000 Subject: [PATCH 042/135] change enable field from array to string in images.json --- images.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/images.json b/images.json index 26306eebf..dbd0250ae 100644 --- a/images.json +++ b/images.json @@ -14,7 +14,7 @@ { "name": "lab", "repo": "stellar/laboratory", "ref": "main" } ], "additional-tests": [ - { "arch": "amd64", "network": "pubnet", "enable": ["core,rpc,horizon"] } + { "arch": "amd64", "network": "pubnet", "enable": "core,rpc,horizon" } ] }, { @@ -32,8 +32,8 @@ { "name": "lab", "repo": "stellar/laboratory", "ref": "main" } ], "additional-tests": [ - { "arch": "amd64", "network": "testnet", "enable": ["core,rpc,horizon"] }, - { "arch": "amd64", "network": "pubnet", "enable": ["core,rpc,horizon"] } + { "arch": "amd64", "network": "testnet", "enable": "core,rpc,horizon" }, + { "arch": "amd64", "network": "pubnet", "enable": "core,rpc,horizon" } ] }, { From c459795f323a551919edf5aca575917adb1c6c15 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 17:04:39 +1000 Subject: [PATCH 043/135] update docker login action to v3.6.0 in push workflows --- .github/actions/push-alias/action.yml | 2 +- .github/actions/push-manifest/action.yml | 2 +- .github/actions/push/action.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/push-alias/action.yml b/.github/actions/push-alias/action.yml index a98a55414..fbf3443db 100644 --- a/.github/actions/push-alias/action.yml +++ b/.github/actions/push-alias/action.yml @@ -19,7 +19,7 @@ runs: using: "composite" steps: - - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: registry: ${{ inputs.registry }} username: ${{ inputs.username }} diff --git a/.github/actions/push-manifest/action.yml b/.github/actions/push-manifest/action.yml index 08e67de91..4e66ee325 100644 --- a/.github/actions/push-manifest/action.yml +++ b/.github/actions/push-manifest/action.yml @@ -19,7 +19,7 @@ runs: using: "composite" steps: - - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: registry: ${{ inputs.registry }} username: ${{ inputs.username }} diff --git a/.github/actions/push/action.yml b/.github/actions/push/action.yml index 2f9d72cc2..d897164e6 100644 --- a/.github/actions/push/action.yml +++ b/.github/actions/push/action.yml @@ -32,7 +32,7 @@ runs: shell: bash run: docker load -i /tmp/${{ inputs.artifact_image_file }} - - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: registry: ${{ inputs.registry }} username: ${{ inputs.username }} From eabf4980c4728ca966365cdbbbe0aebc2d5f384d Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 17:08:02 +1000 Subject: [PATCH 044/135] update json processing scripts to accept stdin and remove id_short field --- .scripts/images-additional-tests | 4 ++++ .scripts/images-with-extras | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.scripts/images-additional-tests b/.scripts/images-additional-tests index 02dee2200..eabfd80a3 100755 --- a/.scripts/images-additional-tests +++ b/.scripts/images-additional-tests @@ -3,6 +3,10 @@ import json import sys +# Accepts as stdin a JSON object in the format of images.json. Outputs an array +# of all the additional test cases defined in the images merged together. +# Usage: < images.json ./.scripts/images-additional-tests + images = json.load(sys.stdin) tests = [] for image in images: diff --git a/.scripts/images-with-extras b/.scripts/images-with-extras index 900357460..9b9ed3b8b 100755 --- a/.scripts/images-with-extras +++ b/.scripts/images-with-extras @@ -44,7 +44,6 @@ for image in images: dep_str = json.dumps(dep, separators=(',', ':')) id = hashlib.sha256(dep_str.encode()).hexdigest() dep["id"] = id - dep["id_short"] = id[:5] print(f" • id = {id}", file=sys.stderr) print(json.dumps(images, separators=(',', ':'))) From 3402ef0169f3dcdcf6fa8e8405a59e8677fc7ce5 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 17:11:57 +1000 Subject: [PATCH 045/135] remove tombstone artifact retention and update image upload names --- .github/workflows/build.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 68e3fb6f4..17e081c34 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,7 +24,6 @@ env: # by one so that all the keys become new. cache_id: 6 artifact_retention_days_for_image: 7 - artifact_retention_days_for_tombstone: 7 artifact_retention_days_for_logs: 60 jobs: @@ -113,14 +112,14 @@ jobs: with: name: ${{ env.json_filename }} path: /tmp/${{ env.json_filename }} - retention-days: ${{ env.artifact_retention_days_for_tombstone }} + retention-days: ${{ env.artifact_retention_days_for_image }} - name: Find Image in Cache id: cache uses: actions/cache/restore@v3 with: key: ${{ env.cache_id }}-${{ env.image_filename }} path: /tmp/${{ env.image_filename }} - - name: Upload Image + - name: Upload Image to Artifacts if: steps.cache.outputs.cache-hit == 'true' uses: actions/upload-artifact@v4 with: @@ -133,7 +132,7 @@ jobs: with: name: ${{ env.missing_filename }} path: /tmp/${{ env.json_filename }} - retention-days: ${{ env.artifact_retention_days_for_tombstone }} + retention-days: ${{ env.artifact_retention_days_for_image }} prepare: needs: [load] @@ -292,12 +291,12 @@ jobs: . - name: Save Quickstart Image run: docker save ${{ env.image_repo }}:${{ steps.tag.outputs.tag }} -o /tmp/image - - name: Upload Quickstart Image + - name: Upload Quickstart Image to Artifacts uses: actions/upload-artifact@v4 with: name: image-quickstart-${{ matrix.image.tag }}-${{ matrix.arch }}.tar path: /tmp/image - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS_FOR_IMAGE }} + retention-days: ${{ env.artifact_retention_days_for_image }} test: needs: [setup, build] From d47ee4c3a8bd2e5039e26f01e81dfc6cc5990dd4 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 17:15:14 +1000 Subject: [PATCH 046/135] format multi-line images field in build workflow --- .github/workflows/build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 17e081c34..dcc1f9524 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -470,7 +470,9 @@ jobs: with: head_sha: ${{ env.sha }} image: ${{ env.image_repo }}:${{ steps.tag.outputs.tag }} - images: ${{ env.image_repo }}:${{ steps.tag.outputs.tag }}-amd64 ${{ env.image_repo }}:${{ steps.tag.outputs.tag }}-arm64 + images: > + ${{ env.image_repo }}:${{ steps.tag.outputs.tag }}-amd64 + ${{ env.image_repo }}:${{ steps.tag.outputs.tag }}-arm64 registry: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} From c0728e6292407cf3634aa3c4ba09085adebe2b75 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 17:36:51 +1000 Subject: [PATCH 047/135] update build workflow to support custom image json and workflow calls --- .github/workflows/build.yml | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dcc1f9524..413522e0d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,6 +7,22 @@ on: pull_request: schedule: - cron: '0 0 * * *' + workflow_call: + inputs: + sha: + description: "Quickstart sha/ref to build (use main for latest)" + type: "string" + default: "main" + image_json: + description: "A custom image.json (a single image from the same format as images.json)" + required: true + test: + description: "Whether the general image tests should run" + default: false + outputs: + image_artifact: + description: "Name of the artifact containing the image built" + type: "string" # Prevent more than one build of this workflow for a branch to be running at the # same time, and if multiple are queued, only run the latest, cancelling any @@ -54,17 +70,25 @@ jobs: - name: Tag Prefix id: tag-prefix run: | - pr_prefix="${{ github.event_name == 'pull_request' && format('pr{0}-', github.event.pull_request.number) || '' }}" + pr_prefix="${{ github.event_name == 'workflow_call' && 'custom-' || (github.event_name == 'pull_request' && format('pr{0}-', github.event.pull_request.number) || '') }}" commit_count="$(git rev-list HEAD --count --first-parent)" build_number="${{ github.run_number }}.${{ github.run_attempt }}" echo "tag-prefix=${pr_prefix}v${commit_count}-b${build_number}-" | tee -a $GITHUB_OUTPUT echo "tag-alias-prefix=${pr_prefix}" | tee -a $GITHUB_OUTPUT - name: Images + if: github.event_name != 'workflow_call' run: | images="$(> $GITHUB_ENV + - name: Images (for workflow_call) + if: github.event_name == 'workflow_call' + run: | + images="${{ inputs.image_json }}" + images="$(<<< $images jq -c '[ . ]')" + <<< $images jq + echo "images=$images" >> $GITHUB_ENV - name: Images with Extras id: images env: @@ -300,7 +324,7 @@ jobs: test: needs: [setup, build] - if: always() && !failure() && !cancelled() + if: always() && !failure() && !cancelled() && inputs.test !== false strategy: matrix: tag: ${{ fromJSON(needs.setup.outputs.images).*.tag }} From 7b75533b0d95f8a50b8e7a238a976d1e397e2cc5 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 17:42:09 +1000 Subject: [PATCH 048/135] update docker image tagging and workflow configuration --- .github/actions/push/action.yml | 8 +++++++- .github/workflows/build.yml | 15 +++++---------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/.github/actions/push/action.yml b/.github/actions/push/action.yml index d897164e6..76701cbe7 100644 --- a/.github/actions/push/action.yml +++ b/.github/actions/push/action.yml @@ -6,9 +6,12 @@ inputs: required: true artifact_image_file: required: true + image: + required: true + default: quickstart arch: required: true - image: + name: required: true default: ghcr.io/${{ github.repository }}:latest registry: @@ -31,6 +34,9 @@ runs: - shell: bash run: docker load -i /tmp/${{ inputs.artifact_image_file }} + - + shell: bash + run: docker tag ${{ inputs.image }} ${{ inputs.name }} - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 413522e0d..e858f6470 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -286,9 +286,6 @@ jobs: < "${image/%.tar/.json}" jq docker load -i $image done - - name: Create Tag - id: tag - run: echo "tag=${{ needs.setup.outputs.tag-prefix }}${{ matrix.image.tag }}-${{ matrix.arch }}" | tee -a $GITHUB_OUTPUT - name: Pull Base Image run: docker pull --platform linux/${{ matrix.arch }} ubuntu:22.04 # Docker buildx cannot be used to build the dev quickstart image because @@ -302,7 +299,7 @@ jobs: docker build --platform linux/${{ matrix.arch }} -f Dockerfile - -t ${{ env.image_repo }}:${{ steps.tag.outputs.tag }} + -t quickstart --label org.opencontainers.image.revision="${{ env.sha }}" --build-arg REVISION="${{ env.sha }}" --build-arg PROTOCOL_VERSION_DEFAULT="${{ matrix.image.config.protocol_version_default }}" @@ -314,7 +311,7 @@ jobs: --build-arg LAB_IMAGE_REF=stellar-lab:${{ matrix.arch }} . - name: Save Quickstart Image - run: docker save ${{ env.image_repo }}:${{ steps.tag.outputs.tag }} -o /tmp/image + run: docker save quickstart -o /tmp/image - name: Upload Quickstart Image to Artifacts uses: actions/upload-artifact@v4 with: @@ -355,9 +352,6 @@ jobs: path: /tmp/ - name: Load Quickstart Image run: docker load -i /tmp/image - - name: Create Tag - id: tag - run: echo "tag=${{ needs.setup.outputs.tag-prefix }}${{ matrix.tag }}-${{ matrix.arch }}" | tee -a $GITHUB_OUTPUT - name: Prepare Logs Directory run: mkdir -p logs - name: Run Quickstart Image @@ -369,7 +363,7 @@ jobs: "8000:8000" -p "11626:11626" --name stellar - ${{ env.image_repo }}:${{ steps.tag.outputs.tag }} + quickstart --${{ matrix.network }} --enable ${{ matrix.enable }} ${{ matrix.options }} @@ -463,8 +457,9 @@ jobs: head_sha: ${{ env.sha }} artifact_name: image-quickstart-${{ matrix.tag }}-${{ matrix.arch }}.tar artifact_image_file: image + image: quickstart arch: ${{ matrix.arch }} - image: ${{ env.image_repo }}:${{ steps.tag.outputs.tag }} + name: ${{ env.image_repo }}:${{ steps.tag.outputs.tag }} registry: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} From 6681a8fa89b1c33541a4bdaa659343bd4412eafd Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 17:44:04 +1000 Subject: [PATCH 049/135] update build workflow to include string and boolean types for image_json and test parameters --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e858f6470..59b4aa5ab 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,9 +15,11 @@ on: default: "main" image_json: description: "A custom image.json (a single image from the same format as images.json)" + type: "string" required: true test: description: "Whether the general image tests should run" + type: "boolean" default: false outputs: image_artifact: From 0fb78ee3c710ab33ea814f4a9ada3df2842173bd Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 17:47:32 +1000 Subject: [PATCH 050/135] update build workflow to conditional arch matrix and add artifact output --- .github/workflows/build.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 59b4aa5ab..f45af682a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,7 +24,7 @@ on: outputs: image_artifact: description: "Name of the artifact containing the image built" - type: "string" + value: ${{ jobs.build.outputs.artifact }} # Prevent more than one build of this workflow for a branch to be running at the # same time, and if multiple are queued, only run the latest, cancelling any @@ -118,7 +118,7 @@ jobs: strategy: matrix: dep: ${{ fromJSON(needs.setup.outputs.deps) }} - arch: ["amd64", "arm64"] + arch: ${{ github.event_name == 'workflow_call' && ["amd64"] || ["amd64", "arm64"] }} fail-fast: false name: 2 load (${{ matrix.dep.name }}, ${{ matrix.dep.ref }}, ${{ matrix.arch }}, ${{ matrix.dep.options && toJSON(matrix.dep.options) || '-' }}) runs-on: ubuntu-latest @@ -227,10 +227,12 @@ jobs: strategy: matrix: image: ${{ fromJSON(needs.setup.outputs.images) }} - arch: ["amd64", "arm64"] + arch: ${{ github.event_name == 'workflow_call' && ["amd64"] || ["amd64", "arm64"] }} fail-fast: false name: 5 build (quickstart, ${{ matrix.image.tag }}, ${{ matrix.arch }}) runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} + outputs: + artifact: image-quickstart-${{ matrix.image.tag }}-${{ matrix.arch }}.tar env: image_json: ${{ toJSON(matrix.image) }} steps: From bc7796bee1bd5284488125f7c59f365d07f27906 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 17:48:31 +1000 Subject: [PATCH 051/135] update arch matrix to use single quotes for consistency --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f45af682a..428326363 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -118,7 +118,7 @@ jobs: strategy: matrix: dep: ${{ fromJSON(needs.setup.outputs.deps) }} - arch: ${{ github.event_name == 'workflow_call' && ["amd64"] || ["amd64", "arm64"] }} + arch: ${{ github.event_name == 'workflow_call' && ['amd64'] || ['amd64', 'arm64'] }} fail-fast: false name: 2 load (${{ matrix.dep.name }}, ${{ matrix.dep.ref }}, ${{ matrix.arch }}, ${{ matrix.dep.options && toJSON(matrix.dep.options) || '-' }}) runs-on: ubuntu-latest @@ -227,7 +227,7 @@ jobs: strategy: matrix: image: ${{ fromJSON(needs.setup.outputs.images) }} - arch: ${{ github.event_name == 'workflow_call' && ["amd64"] || ["amd64", "arm64"] }} + arch: ${{ github.event_name == 'workflow_call' && ['amd64'] || ['amd64', 'arm64'] }} fail-fast: false name: 5 build (quickstart, ${{ matrix.image.tag }}, ${{ matrix.arch }}) runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} @@ -329,7 +329,7 @@ jobs: strategy: matrix: tag: ${{ fromJSON(needs.setup.outputs.images).*.tag }} - arch: ["amd64", "arm64"] + arch: ${{ github.event_name == 'workflow_call' && ['amd64'] || ['amd64', 'arm64'] }} network: ["local"] enable: ["core", "rpc", "core,rpc,horizon"] options: [""] From 0844313292706c3c821021f40e45d4b2ab41c6f0 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 17:49:55 +1000 Subject: [PATCH 052/135] update json parsing in workflow arch matrix --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 428326363..f3336f0ac 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -118,7 +118,7 @@ jobs: strategy: matrix: dep: ${{ fromJSON(needs.setup.outputs.deps) }} - arch: ${{ github.event_name == 'workflow_call' && ['amd64'] || ['amd64', 'arm64'] }} + arch: ${{ github.event_name == 'workflow_call' && fromJSON('["amd64"]') || fromJSON('["amd64", "arm64"]') }} fail-fast: false name: 2 load (${{ matrix.dep.name }}, ${{ matrix.dep.ref }}, ${{ matrix.arch }}, ${{ matrix.dep.options && toJSON(matrix.dep.options) || '-' }}) runs-on: ubuntu-latest @@ -227,7 +227,7 @@ jobs: strategy: matrix: image: ${{ fromJSON(needs.setup.outputs.images) }} - arch: ${{ github.event_name == 'workflow_call' && ['amd64'] || ['amd64', 'arm64'] }} + arch: ${{ github.event_name == 'workflow_call' && fromJSON('["amd64"]') || fromJSON('["amd64", "arm64"]') }} fail-fast: false name: 5 build (quickstart, ${{ matrix.image.tag }}, ${{ matrix.arch }}) runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} @@ -329,7 +329,7 @@ jobs: strategy: matrix: tag: ${{ fromJSON(needs.setup.outputs.images).*.tag }} - arch: ${{ github.event_name == 'workflow_call' && ['amd64'] || ['amd64', 'arm64'] }} + arch: ${{ github.event_name == 'workflow_call' && fromJSON('["amd64"]') || fromJSON('["amd64", "arm64"]') }} network: ["local"] enable: ["core", "rpc", "core,rpc,horizon"] options: [""] From df5b7fe1b3ccdc4d48d75598fb0ab41e0069c456 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 17:50:25 +1000 Subject: [PATCH 053/135] change not equal operator in test condition --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f3336f0ac..353350641 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -325,7 +325,7 @@ jobs: test: needs: [setup, build] - if: always() && !failure() && !cancelled() && inputs.test !== false + if: always() && !failure() && !cancelled() && inputs.test != false strategy: matrix: tag: ${{ fromJSON(needs.setup.outputs.images).*.tag }} From 6998c686df7c2e71a16790c61c7678dcc7970188 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 18:13:25 +1000 Subject: [PATCH 054/135] update docker push and context to use inputs.name instead of inputs.image --- .github/actions/push/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/push/action.yml b/.github/actions/push/action.yml index 76701cbe7..7fd496e23 100644 --- a/.github/actions/push/action.yml +++ b/.github/actions/push/action.yml @@ -46,7 +46,7 @@ runs: - shell: bash run: | - docker push ${{ inputs.image }} + docker push ${{ inputs.name }} - uses: actions/github-script@v5 with: @@ -56,6 +56,6 @@ runs: repo: context.repo.repo, sha: '${{ inputs.head_sha }}', state: 'success', - context: `${{ inputs.image }}`, + context: `${{ inputs.name }}`, description: 'Available', }); From 7a999a7f5cd2def0bfc5d83272812b1f73834801 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 18:15:41 +1000 Subject: [PATCH 055/135] update test condition to handle workflow_call event --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 353350641..fc5e39c6f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -325,7 +325,7 @@ jobs: test: needs: [setup, build] - if: always() && !failure() && !cancelled() && inputs.test != false + if: always() && !failure() && !cancelled() && (github.event_name != 'workflow_call' || inputs.test == true) strategy: matrix: tag: ${{ fromJSON(needs.setup.outputs.images).*.tag }} From 3313555321d1ff4cc4465c9b2e3c9eed837baa89 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 20:31:31 +1000 Subject: [PATCH 056/135] update makefile to use json config and add rpc dependency --- .gitignore | 1 + Makefile | 84 +++++++++++++++++++++--------------------------------- 2 files changed, 33 insertions(+), 52 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..e73a5f795 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/image.json diff --git a/Makefile b/Makefile index e5191fab8..9734b8caf 100644 --- a/Makefile +++ b/Makefile @@ -1,20 +1,26 @@ -__PHONY__: run logs build build-deps build-deps-core build-deps-horizon build-deps-friendbot build-deps-stellar-rpc +__PHONY__: run logs console build build-deps build-deps-xdr build-deps-core build-deps-horizon build-deps-friendbot build-deps-rpc build-deps-lab REVISION=$(shell git -c core.abbrev=no describe --always --exclude='*' --long --dirty) -TAG?=dev -PROTOCOL_VERSION_DEFAULT?=22 -XDR_REPO?=stellar/rs-stellar-xdr -XDR_REF?=main -CORE_REPO?=stellar/stellar-core -CORE_REF?=master -CORE_CONFIGURE_FLAGS?=--disable-tests -STELLAR_RPC_REF?=main -HORIZON_REF?=master -FRIENDBOT_REF?=$(HORIZON_REF) -LAB_REF?=main +TAG?=latest + +# Extract configuration from images.json +PROTOCOL_VERSION_DEFAULT = $(shell jq -r '.[] | select(.tag == "$(TAG)") | .config.protocol_version_default' images.json) +XDR_REPO = $(shell jq -r '.[] | select(.tag == "$(TAG)") | .deps[] | select(.name == "xdr") | .repo' images.json) +XDR_REF = $(shell jq -r '.[] | select(.tag == "$(TAG)") | .deps[] | select(.name == "xdr") | .ref' images.json) +CORE_REPO = $(shell jq -r '.[] | select(.tag == "$(TAG)") | .deps[] | select(.name == "core") | .repo' images.json) +CORE_REF = $(shell jq -r '.[] | select(.tag == "$(TAG)") | .deps[] | select(.name == "core") | .ref' images.json) +CORE_OPTIONS = $(shell jq -c '.[] | select(.tag == "$(TAG)") | .deps[] | select(.name == "core") | .options // {}' images.json) +RPC_REPO = $(shell jq -r '.[] | select(.tag == "$(TAG)") | .deps[] | select(.name == "rpc") | .repo' images.json) +RPC_REF = $(shell jq -r '.[] | select(.tag == "$(TAG)") | .deps[] | select(.name == "rpc") | .ref' images.json) +HORIZON_REPO = $(shell jq -r '.[] | select(.tag == "$(TAG)") | .deps[] | select(.name == "horizon") | .repo' images.json) +HORIZON_REF = $(shell jq -r '.[] | select(.tag == "$(TAG)") | .deps[] | select(.name == "horizon") | .ref' images.json) +FRIENDBOT_REPO = $(shell jq -r '.[] | select(.tag == "$(TAG)") | .deps[] | select(.name == "friendbot") | .repo' images.json) +FRIENDBOT_REF = $(shell jq -r '.[] | select(.tag == "$(TAG)") | .deps[] | select(.name == "friendbot") | .ref' images.json) +LAB_REPO = $(shell jq -r '.[] | select(.tag == "$(TAG)") | .deps[] | select(.name == "lab") | .repo' images.json) +LAB_REF = $(shell jq -r '.[] | select(.tag == "$(TAG)") | .deps[] | select(.name == "lab") | .ref' images.json) run: - docker run --rm --name stellar -p 8000:8000 stellar/quickstart:$(TAG) --local --enable-stellar-rpc + docker run --rm --name stellar -p 8000:8000 stellar/quickstart:$(TAG) --local logs: docker exec stellar /bin/sh -c 'tail -F /var/log/supervisor/*' @@ -22,61 +28,35 @@ logs: console: docker exec -it stellar /bin/bash -build-latest: - $(MAKE) build TAG=latest \ - PROTOCOL_VERSION_DEFAULT=23 \ - XDR_REF=v23.0.0 \ - CORE_REF=v23.0.1 \ - HORIZON_REF=horizon-v23.0.0 \ - STELLAR_RPC_REF=v23.0.4 \ - FRIENDBOT_REF=horizon-v23.0.0 - -build-testing: - $(MAKE) build TAG=testing \ - PROTOCOL_VERSION_DEFAULT=23 \ - XDR_REF=v23.0.0 \ - CORE_REF=v23.0.1 \ - HORIZON_REF=horizon-v23.0.0 \ - STELLAR_RPC_REF=v23.0.4 \ - FRIENDBOT_REF=horizon-v23.0.0 - -build-future: - $(MAKE) build TAG=future \ - PROTOCOL_VERSION_DEFAULT=23 \ - XDR_REF=v23.0.0 \ - CORE_REF=v23.0.1 \ - HORIZON_REF=horizon-v23.0.0 \ - STELLAR_RPC_REF=v23.0.4 \ - FRIENDBOT_REF=horizon-v23.0.0 - build: - $(MAKE) -j 4 build-deps + < images.json jq -c --arg tag '$(TAG)' '.[] | select(.tag == $$tag)' > image.json + $(MAKE) build-deps docker build -t stellar/quickstart:$(TAG) -f Dockerfile . \ --build-arg REVISION=$(REVISION) \ --build-arg PROTOCOL_VERSION_DEFAULT=$(PROTOCOL_VERSION_DEFAULT) \ - --build-arg STELLAR_XDR_IMAGE_REF=stellar-xdr:$(XDR_REF) \ - --build-arg STELLAR_CORE_IMAGE_REF=stellar-core:$(CORE_REF) \ + --build-arg XDR_IMAGE_REF=stellar-xdr:$(XDR_REF) \ + --build-arg CORE_IMAGE_REF=stellar-core:$(CORE_REF) \ + --build-arg RPC_IMAGE_REF=stellar-rpc:$(RPC_REF) \ --build-arg HORIZON_IMAGE_REF=stellar-horizon:$(HORIZON_REF) \ --build-arg FRIENDBOT_IMAGE_REF=stellar-friendbot:$(FRIENDBOT_REF) \ - --build-arg STELLAR_RPC_IMAGE_REF=stellar-rpc:$(STELLAR_RPC_REF) \ --build-arg LAB_IMAGE_REF=stellar-lab:$(LAB_REF) -build-deps: build-deps-xdr build-deps-core build-deps-horizon build-deps-friendbot build-deps-stellar-rpc build-deps-lab +build-deps: build-deps-xdr build-deps-rpc build-deps-horizon build-deps-friendbot build-deps-lab build-deps-core build-deps-xdr: docker build -t stellar-xdr:$(XDR_REF) -f Dockerfile.xdr . --build-arg REPO="$(XDR_REPO)" --build-arg REF="$(XDR_REF)" build-deps-core: - docker build -t stellar-core:$(CORE_REF) -f Dockerfile.core . --build-arg REPO="$(CORE_REPO)" --build-arg REF="$(CORE_REF)" --build-arg CONFIGURE_FLAGS="$(CORE_CONFIGURE_FLAGS)" + docker build -t stellar-core:$(CORE_REF) -f Dockerfile.core . --build-arg REPO="$(CORE_REPO)" --build-arg REF="$(CORE_REF)" --build-arg OPTIONS='$(CORE_OPTIONS)' + +build-deps-rpc: + docker build -t stellar-rpc:$(RPC_REF) -f Dockerfile.rpc . --build-arg=REPO="$(RPC_REPO)" --build-arg REF="$(RPC_REF)" build-deps-horizon: - docker build -t stellar-horizon:$(HORIZON_REF) -f Dockerfile.horizon . --build-arg REF="$(HORIZON_REF)" + docker build -t stellar-horizon:$(HORIZON_REF) -f Dockerfile.horizon . --build-arg REPO="$(HORIZON_REPO)" --build-arg REF="$(HORIZON_REF)" build-deps-friendbot: - docker build -t stellar-friendbot:$(FRIENDBOT_REF) -f Dockerfile.friendbot . --build-arg REF="$(FRIENDBOT_REF)" - -build-deps-stellar-rpc: - docker build -t stellar-rpc:$(STELLAR_RPC_REF) -f Dockerfile.rpc . --build-arg REF="$(STELLAR_RPC_REF)" + docker build -t stellar-friendbot:$(FRIENDBOT_REF) -f Dockerfile.friendbot . --build-arg REPO="$(FRIENDBOT_REPO)" --build-arg REF="$(FRIENDBOT_REF)" build-deps-lab: - docker build -t stellar-lab:$(LAB_REF) -f Dockerfile.lab . --build-arg NEXT_PUBLIC_COMMIT_HASH=$(LAB_REF) + docker build -t stellar-lab:$(LAB_REF) -f Dockerfile.lab . --build-arg REPO="$(LAB_REPO)" --build-arg NEXT_PUBLIC_COMMIT_HASH=$(LAB_REF) From 7d93b60cad778efa8722c30dc987c68bea96de79 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 20:46:12 +1000 Subject: [PATCH 057/135] update makefile to use images-with-extras script and sha references --- .github/workflows/build.yml | 2 +- .gitignore | 2 +- Dockerfile | 2 +- Makefile | 74 +++++++++++++++++++------------------ 4 files changed, 42 insertions(+), 38 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fc5e39c6f..629a26cbb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -245,7 +245,7 @@ jobs: echo "$(<<< $image_json jq -r '.deps[] | "\(.name)=\(.id)"')" | tee -a $GITHUB_OUTPUT - name: Write Image Config run: | - echo "$image_json" > image.json + echo "$image_json" > .image.json - name: Download Image XDR uses: actions/download-artifact@v4 with: diff --git a/.gitignore b/.gitignore index e73a5f795..4070854d4 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1 @@ -/image.json +/.image.json diff --git a/Dockerfile b/Dockerfile index b0e5eb26a..984bc5c12 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,7 +42,7 @@ RUN adduser --system --group --quiet --home /var/lib/stellar --disabled-password RUN ["mkdir", "-p", "/opt/stellar"] RUN ["touch", "/opt/stellar/.docker-ephemeral"] -ADD image.json /image.json +ADD .image.json /image.json RUN ["rm", "-fr", "/etc/supervisor"] RUN ["ln", "-sT", "/opt/stellar/supervisor/etc", "/etc/supervisor"] diff --git a/Makefile b/Makefile index 9734b8caf..e54dc8f4f 100644 --- a/Makefile +++ b/Makefile @@ -3,21 +3,26 @@ __PHONY__: run logs console build build-deps build-deps-xdr build-deps-core buil REVISION=$(shell git -c core.abbrev=no describe --always --exclude='*' --long --dirty) TAG?=latest -# Extract configuration from images.json -PROTOCOL_VERSION_DEFAULT = $(shell jq -r '.[] | select(.tag == "$(TAG)") | .config.protocol_version_default' images.json) -XDR_REPO = $(shell jq -r '.[] | select(.tag == "$(TAG)") | .deps[] | select(.name == "xdr") | .repo' images.json) -XDR_REF = $(shell jq -r '.[] | select(.tag == "$(TAG)") | .deps[] | select(.name == "xdr") | .ref' images.json) -CORE_REPO = $(shell jq -r '.[] | select(.tag == "$(TAG)") | .deps[] | select(.name == "core") | .repo' images.json) -CORE_REF = $(shell jq -r '.[] | select(.tag == "$(TAG)") | .deps[] | select(.name == "core") | .ref' images.json) -CORE_OPTIONS = $(shell jq -c '.[] | select(.tag == "$(TAG)") | .deps[] | select(.name == "core") | .options // {}' images.json) -RPC_REPO = $(shell jq -r '.[] | select(.tag == "$(TAG)") | .deps[] | select(.name == "rpc") | .repo' images.json) -RPC_REF = $(shell jq -r '.[] | select(.tag == "$(TAG)") | .deps[] | select(.name == "rpc") | .ref' images.json) -HORIZON_REPO = $(shell jq -r '.[] | select(.tag == "$(TAG)") | .deps[] | select(.name == "horizon") | .repo' images.json) -HORIZON_REF = $(shell jq -r '.[] | select(.tag == "$(TAG)") | .deps[] | select(.name == "horizon") | .ref' images.json) -FRIENDBOT_REPO = $(shell jq -r '.[] | select(.tag == "$(TAG)") | .deps[] | select(.name == "friendbot") | .repo' images.json) -FRIENDBOT_REF = $(shell jq -r '.[] | select(.tag == "$(TAG)") | .deps[] | select(.name == "friendbot") | .ref' images.json) -LAB_REPO = $(shell jq -r '.[] | select(.tag == "$(TAG)") | .deps[] | select(.name == "lab") | .repo' images.json) -LAB_REF = $(shell jq -r '.[] | select(.tag == "$(TAG)") | .deps[] | select(.name == "lab") | .ref' images.json) +# Process images.json through the images-with-extras script +IMAGE_JSON=.image.json +.image.json: images.json .scripts/images-with-extras + < images.json .scripts/images-with-extras | jq '.[] | select(.tag == "$(TAG)")' > $@ + +# Extract configuration from selected image +PROTOCOL_VERSION_DEFAULT = $(shell < $(IMAGE_JSON) jq -r '.config.protocol_version_default') +XDR_REPO = $(shell < $(IMAGE_JSON) jq -r '.deps[] | select(.name == "xdr") | .repo') +XDR_SHA = $(shell < $(IMAGE_JSON) jq -r '.deps[] | select(.name == "xdr") | .sha') +CORE_REPO = $(shell < $(IMAGE_JSON) jq -r '.deps[] | select(.name == "core") | .repo') +CORE_SHA = $(shell < $(IMAGE_JSON) jq -r '.deps[] | select(.name == "core") | .sha') +CORE_OPTIONS = $(shell < $(IMAGE_JSON) jq -c '.deps[] | select(.name == "core") | .options // {}') +RPC_REPO = $(shell < $(IMAGE_JSON) jq -r '.deps[] | select(.name == "rpc") | .repo') +RPC_SHA = $(shell < $(IMAGE_JSON) jq -r '.deps[] | select(.name == "rpc") | .sha') +HORIZON_REPO = $(shell < $(IMAGE_JSON) jq -r '.deps[] | select(.name == "horizon") | .repo') +HORIZON_SHA = $(shell < $(IMAGE_JSON) jq -r '.deps[] | select(.name == "horizon") | .sha') +FRIENDBOT_REPO = $(shell < $(IMAGE_JSON) jq -r '.deps[] | select(.name == "friendbot") | .repo') +FRIENDBOT_SHA = $(shell < $(IMAGE_JSON) jq -r '.deps[] | select(.name == "friendbot") | .sha') +LAB_REPO = $(shell < $(IMAGE_JSON) jq -r '.deps[] | select(.name == "lab") | .repo') +LAB_SHA = $(shell < $(IMAGE_JSON) jq -r '.deps[] | select(.name == "lab") | .sha') run: docker run --rm --name stellar -p 8000:8000 stellar/quickstart:$(TAG) --local @@ -28,35 +33,34 @@ logs: console: docker exec -it stellar /bin/bash -build: - < images.json jq -c --arg tag '$(TAG)' '.[] | select(.tag == $$tag)' > image.json +build: $(IMAGE_JSON) $(MAKE) build-deps docker build -t stellar/quickstart:$(TAG) -f Dockerfile . \ --build-arg REVISION=$(REVISION) \ --build-arg PROTOCOL_VERSION_DEFAULT=$(PROTOCOL_VERSION_DEFAULT) \ - --build-arg XDR_IMAGE_REF=stellar-xdr:$(XDR_REF) \ - --build-arg CORE_IMAGE_REF=stellar-core:$(CORE_REF) \ - --build-arg RPC_IMAGE_REF=stellar-rpc:$(RPC_REF) \ - --build-arg HORIZON_IMAGE_REF=stellar-horizon:$(HORIZON_REF) \ - --build-arg FRIENDBOT_IMAGE_REF=stellar-friendbot:$(FRIENDBOT_REF) \ - --build-arg LAB_IMAGE_REF=stellar-lab:$(LAB_REF) + --build-arg XDR_IMAGE_REF=stellar-xdr:$(XDR_SHA) \ + --build-arg CORE_IMAGE_REF=stellar-core:$(CORE_SHA) \ + --build-arg RPC_IMAGE_REF=stellar-rpc:$(RPC_SHA) \ + --build-arg HORIZON_IMAGE_REF=stellar-horizon:$(HORIZON_SHA) \ + --build-arg FRIENDBOT_IMAGE_REF=stellar-friendbot:$(FRIENDBOT_SHA) \ + --build-arg LAB_IMAGE_REF=stellar-lab:$(LAB_SHA) build-deps: build-deps-xdr build-deps-rpc build-deps-horizon build-deps-friendbot build-deps-lab build-deps-core -build-deps-xdr: - docker build -t stellar-xdr:$(XDR_REF) -f Dockerfile.xdr . --build-arg REPO="$(XDR_REPO)" --build-arg REF="$(XDR_REF)" +build-deps-xdr: $(IMAGE_JSON) + docker build -t stellar-xdr:$(XDR_SHA) -f Dockerfile.xdr . --build-arg REPO="$(XDR_REPO)" --build-arg REF="$(XDR_SHA)" -build-deps-core: - docker build -t stellar-core:$(CORE_REF) -f Dockerfile.core . --build-arg REPO="$(CORE_REPO)" --build-arg REF="$(CORE_REF)" --build-arg OPTIONS='$(CORE_OPTIONS)' +build-deps-core: $(IMAGE_JSON) + docker build -t stellar-core:$(CORE_SHA) -f Dockerfile.core . --build-arg REPO="$(CORE_REPO)" --build-arg REF="$(CORE_SHA)" --build-arg OPTIONS='$(CORE_OPTIONS)' -build-deps-rpc: - docker build -t stellar-rpc:$(RPC_REF) -f Dockerfile.rpc . --build-arg=REPO="$(RPC_REPO)" --build-arg REF="$(RPC_REF)" +build-deps-rpc: $(IMAGE_JSON) + docker build -t stellar-rpc:$(RPC_SHA) -f Dockerfile.rpc . --build-arg=REPO="$(RPC_REPO)" --build-arg REF="$(RPC_SHA)" -build-deps-horizon: - docker build -t stellar-horizon:$(HORIZON_REF) -f Dockerfile.horizon . --build-arg REPO="$(HORIZON_REPO)" --build-arg REF="$(HORIZON_REF)" +build-deps-horizon: $(IMAGE_JSON) + docker build -t stellar-horizon:$(HORIZON_SHA) -f Dockerfile.horizon . --build-arg REPO="$(HORIZON_REPO)" --build-arg REF="$(HORIZON_SHA)" -build-deps-friendbot: - docker build -t stellar-friendbot:$(FRIENDBOT_REF) -f Dockerfile.friendbot . --build-arg REPO="$(FRIENDBOT_REPO)" --build-arg REF="$(FRIENDBOT_REF)" +build-deps-friendbot: $(IMAGE_JSON) + docker build -t stellar-friendbot:$(FRIENDBOT_SHA) -f Dockerfile.friendbot . --build-arg REPO="$(FRIENDBOT_REPO)" --build-arg REF="$(FRIENDBOT_SHA)" -build-deps-lab: - docker build -t stellar-lab:$(LAB_REF) -f Dockerfile.lab . --build-arg REPO="$(LAB_REPO)" --build-arg NEXT_PUBLIC_COMMIT_HASH=$(LAB_REF) +build-deps-lab: $(IMAGE_JSON) + docker build -t stellar-lab:$(LAB_SHA) -f Dockerfile.lab . --build-arg REPO="$(LAB_REPO)" --build-arg NEXT_PUBLIC_COMMIT_HASH=$(LAB_SHA) From eb8762246b809a3f1662d656f975cd6f2d0f9589 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 22:20:41 +1000 Subject: [PATCH 058/135] update dockerfile to use equals syntax for environment variables --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 984bc5c12..395176cfe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ FROM $LAB_IMAGE_REF AS lab FROM ubuntu:22.04 ARG REVISION -ENV REVISION $REVISION +ENV REVISION=$REVISION EXPOSE 5432 EXPOSE 6060 @@ -62,6 +62,6 @@ RUN ["chmod", "+x", "start"] ARG PROTOCOL_VERSION_DEFAULT RUN test -n "$PROTOCOL_VERSION_DEFAULT" || (echo "Image build arg PROTOCOL_VERSION_DEFAULT required and not set" && false) -ENV PROTOCOL_VERSION_DEFAULT $PROTOCOL_VERSION_DEFAULT +ENV PROTOCOL_VERSION_DEFAULT=$PROTOCOL_VERSION_DEFAULT ENTRYPOINT ["/start"] From 879cf0407d8c4959a59148eb0a513a4d5e3329b1 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 23:12:02 +1000 Subject: [PATCH 059/135] update workflow to require sha input and add test workflow --- .github/workflows/build.yml | 6 +++--- .github/workflows/workflow-test.yml | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/workflow-test.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 629a26cbb..8f98c1528 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,9 +10,9 @@ on: workflow_call: inputs: sha: - description: "Quickstart sha/ref to build (use main for latest)" + description: "Quickstart sha to build" type: "string" - default: "main" + required: true image_json: description: "A custom image.json (a single image from the same format as images.json)" type: "string" @@ -35,7 +35,7 @@ concurrency: cancel-in-progress: true env: - sha: ${{ github.event.pull_request.head.sha || github.sha }} + sha: ${{ inputs.sha || github.event.pull_request.head.sha || github.sha }} image_repo: ${{ format('{0}/{1}', secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io', github.repository) }} # Cache ID is a value inserted into cache keys. Whenever changing the build # in a way that needs to use entirely new fresh builds, increment the number diff --git a/.github/workflows/workflow-test.yml b/.github/workflows/workflow-test.yml new file mode 100644 index 000000000..cd22fa35c --- /dev/null +++ b/.github/workflows/workflow-test.yml @@ -0,0 +1,28 @@ +name: Workflow Call Test + +on: + push: + +jobs: + + build-custom: + uses: ./.github/workflows/build.yml + with: + sha: ${{ github.event.pull_request.head.sha || github.sha }} + test: true + image_json: | + { + "tag": "custom", + "config": { + "protocol_version_default": 23 + }, + "deps": [ + { "name": "xdr", "repo": "stellar/rs-stellar-xdr", "ref": "v23.0.0" }, + { "name": "core", "repo": "stellar/stellar-core", "ref": "v23.0.1", "options": { "configure_flags": "--disable-tests" } }, + { "name": "rpc", "repo": "stellar/stellar-rpc", "ref": "v23.0.1" }, + { "name": "horizon", "repo": "stellar/go", "ref": "horizon-v23.0.0" }, + { "name": "friendbot", "repo": "stellar/go", "ref": "horizon-v23.0.0" }, + { "name": "lab", "repo": "stellar/laboratory", "ref": "main" } + ], + "additional-tests": [] + } From f1c8a9d6ce8a60ddb8442d63116947139d9d37eb Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 23:32:42 +1000 Subject: [PATCH 060/135] update workflow to use image_json input for conditional logic --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8f98c1528..c3083835c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -72,20 +72,20 @@ jobs: - name: Tag Prefix id: tag-prefix run: | - pr_prefix="${{ github.event_name == 'workflow_call' && 'custom-' || (github.event_name == 'pull_request' && format('pr{0}-', github.event.pull_request.number) || '') }}" + pr_prefix="${{ inputs.image_json && 'custom-' || (github.event_name == 'pull_request' && format('pr{0}-', github.event.pull_request.number) || '') }}" commit_count="$(git rev-list HEAD --count --first-parent)" build_number="${{ github.run_number }}.${{ github.run_attempt }}" echo "tag-prefix=${pr_prefix}v${commit_count}-b${build_number}-" | tee -a $GITHUB_OUTPUT echo "tag-alias-prefix=${pr_prefix}" | tee -a $GITHUB_OUTPUT - name: Images - if: github.event_name != 'workflow_call' + if: '!inputs.image_json' run: | images="$(> $GITHUB_ENV - name: Images (for workflow_call) - if: github.event_name == 'workflow_call' + if: inputs.image_json run: | images="${{ inputs.image_json }}" images="$(<<< $images jq -c '[ . ]')" @@ -118,7 +118,7 @@ jobs: strategy: matrix: dep: ${{ fromJSON(needs.setup.outputs.deps) }} - arch: ${{ github.event_name == 'workflow_call' && fromJSON('["amd64"]') || fromJSON('["amd64", "arm64"]') }} + arch: ${{ inputs.image_json && fromJSON('["amd64"]') || fromJSON('["amd64", "arm64"]') }} fail-fast: false name: 2 load (${{ matrix.dep.name }}, ${{ matrix.dep.ref }}, ${{ matrix.arch }}, ${{ matrix.dep.options && toJSON(matrix.dep.options) || '-' }}) runs-on: ubuntu-latest From 12d5d8afacd7b010719a9470599374406d475d5d Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 23:32:48 +1000 Subject: [PATCH 061/135] add custom workflow job for local testing --- .github/workflows/workflow-test.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/workflow-test.yml b/.github/workflows/workflow-test.yml index cd22fa35c..e274ce0bd 100644 --- a/.github/workflows/workflow-test.yml +++ b/.github/workflows/workflow-test.yml @@ -26,3 +26,22 @@ jobs: ], "additional-tests": [] } + + use-custom: + needs: build-custom + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v4 + with: + name: ${{ needs.build-custom.outputs.image_artifact }} + path: /tmp/ + - run: docker load -i /tmp/image + - run: > + docker run + --platform linux/amd64 + -i + -p + "8000:8000" + --name stellar + quickstart + --local From 220c8b546e9865bbd8e022706e306ae99da193ea Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 7 Oct 2025 23:35:58 +1000 Subject: [PATCH 062/135] update workflow to use environment variable for image json --- .github/workflows/build.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c3083835c..b6ba6b42c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -86,9 +86,10 @@ jobs: echo "images=$images" >> $GITHUB_ENV - name: Images (for workflow_call) if: inputs.image_json + env: + image: ${{ inputs.image_json }} run: | - images="${{ inputs.image_json }}" - images="$(<<< $images jq -c '[ . ]')" + images="$(<<< $image jq -c '[ . ]')" <<< $images jq echo "images=$images" >> $GITHUB_ENV - name: Images with Extras From 81b48b5575f8fb7bb90cfa24a31ac7fffc04874c Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 00:50:58 +1000 Subject: [PATCH 063/135] update build workflow to use image_json input for arch selection --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b6ba6b42c..9ee02b474 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -228,7 +228,7 @@ jobs: strategy: matrix: image: ${{ fromJSON(needs.setup.outputs.images) }} - arch: ${{ github.event_name == 'workflow_call' && fromJSON('["amd64"]') || fromJSON('["amd64", "arm64"]') }} + arch: ${{ inputs.image_json && fromJSON('["amd64"]') || fromJSON('["amd64", "arm64"]') }} fail-fast: false name: 5 build (quickstart, ${{ matrix.image.tag }}, ${{ matrix.arch }}) runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} From 58545ae2e9ee5aa5712dcd6c79303e6693adb84b Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 00:55:56 +1000 Subject: [PATCH 064/135] update build workflow to use image_json input for arch selection --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9ee02b474..4f55ff43d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -330,7 +330,7 @@ jobs: strategy: matrix: tag: ${{ fromJSON(needs.setup.outputs.images).*.tag }} - arch: ${{ github.event_name == 'workflow_call' && fromJSON('["amd64"]') || fromJSON('["amd64", "arm64"]') }} + arch: ${{ inputs.image_json && fromJSON('["amd64"]') || fromJSON('["amd64", "arm64"]') }} network: ["local"] enable: ["core", "rpc", "core,rpc,horizon"] options: [""] From 6a678005224f26bdbe3525735ad0a4232081fca9 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 01:00:25 +1000 Subject: [PATCH 065/135] update build workflow to conditionally enable services based on image_json input --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4f55ff43d..f6c5c15f0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -332,7 +332,7 @@ jobs: tag: ${{ fromJSON(needs.setup.outputs.images).*.tag }} arch: ${{ inputs.image_json && fromJSON('["amd64"]') || fromJSON('["amd64", "arm64"]') }} network: ["local"] - enable: ["core", "rpc", "core,rpc,horizon"] + enable: ${{ inputs.image_json && fromJSON('["core,rpc,horizon"]') || fromJSON('["core","rpc","core,rpc,horizon"]') }} options: [""] include: ${{ fromJSON(needs.setup.outputs.additional-tests) }} fail-fast: false From 275468e0168b768084940b232455929e11438b5e Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 01:08:26 +1000 Subject: [PATCH 066/135] update build workflow with multiplatform image support and documentation --- .github/workflows/build.yml | 24 ++++++++++++++++++++++++ .github/workflows/workflow-test.yml | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f6c5c15f0..3eeac8386 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,5 +1,29 @@ name: Build +# This workflow builds multiple quickstart images as defined in the images.json +# file. +# +# The dependencies (xdr, core, rpc, horizon, friendbot, lab) are first +# deduplicated across all images, and then built. Dependencies are cached and +# so only rebuilt when needed. Dependencies are defined by a tag or branch, but +# when building those git refs are resolved to a sha to ensure stability of the +# sha throughout the full build process. For all dependencies and the final +# image, amd64 and arm64 variants are built and the final image is a +# multiplatform image. +# +# The images defined in the images.json file can specify what events the images +# are built on. Most of the images will be built on push and pull requests, but +# this workflow also runs on a schedule an so images that need updating on a +# schedule, such as a nightly-like image, can specify running additionally or +# only on the schedule. +# +# This workflow is also triggerable via a workflow call from another workflow. +# When used that way, the image only builds a single amd64 image and it is not +# pushed. The workflow returns an output which is an artifact name that can be +# downloaded and loaded into docker for use in another job. The workflow can be +# improved to support multiple images in the workflow call case, it just +# requires more work to do so. + on: push: branches: diff --git a/.github/workflows/workflow-test.yml b/.github/workflows/workflow-test.yml index e274ce0bd..83065aea2 100644 --- a/.github/workflows/workflow-test.yml +++ b/.github/workflows/workflow-test.yml @@ -33,7 +33,7 @@ jobs: steps: - uses: actions/download-artifact@v4 with: - name: ${{ needs.build-custom.outputs.image_artifact }} + name: ${{ needs.build-custom.outputs.image_artifact path: /tmp/ - run: docker load -i /tmp/image - run: > From f794603e0ca0e60da4adcd2bba5dac1f662332b1 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 01:09:04 +1000 Subject: [PATCH 067/135] update workflow to run container in detached mode and add health check --- .github/workflows/workflow-test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/workflow-test.yml b/.github/workflows/workflow-test.yml index 83065aea2..c630005fe 100644 --- a/.github/workflows/workflow-test.yml +++ b/.github/workflows/workflow-test.yml @@ -39,9 +39,11 @@ jobs: - run: > docker run --platform linux/amd64 - -i + -d -p "8000:8000" --name stellar quickstart --local + - run: sleep 10 + - run: curl http://localhost:8000 From 337c5f4b0a394942048b438f1302fd001be8eb43 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 01:10:07 +1000 Subject: [PATCH 068/135] fix workflow-test.yml syntax error --- .github/workflows/workflow-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/workflow-test.yml b/.github/workflows/workflow-test.yml index c630005fe..ee80c8fb2 100644 --- a/.github/workflows/workflow-test.yml +++ b/.github/workflows/workflow-test.yml @@ -33,7 +33,7 @@ jobs: steps: - uses: actions/download-artifact@v4 with: - name: ${{ needs.build-custom.outputs.image_artifact + name: ${{ needs.build-custom.outputs.image_artifact }} path: /tmp/ - run: docker load -i /tmp/image - run: > From 28eea1358489423817da94382915fb2da67305c0 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 01:48:22 +1000 Subject: [PATCH 069/135] update README to use images.json instead of makefile parameters for building custom images --- README.md | 25 +++---------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 4e608762e..6c71cf703 100644 --- a/README.md +++ b/README.md @@ -243,30 +243,11 @@ _Disclaimer_: The DigitalOcean server is publicly accessible on the Internet. Do ### Building Custom Images To build a quickstart image with custom or specific versions of stellar-core, -horizon, etc, use the `Makefile`. The following parameters can be specified to -customize the version of each component, and for stellar-core the features it is -built with. - -- `TAG`: The docker tag to assign to the build. Default `dev`. -- `CORE_REF`: The git reference of stellar-core to build. -- `CORE_CONFIGURE_FLAGS`: The `CONFIGURE_FLAGS` to configure the stellar-core - build with. Typically include `--disable-tests`, and to enable the next protocol - version that is still in development, add - `--enable-next-protocol-version-unsafe-for-production`. -- `HORIZON_REF`: The git reference of stellar-horizon to build. -- `FRIENDBOT_REF`: The git reference of stellar-friendbot to build. -- `STELLAR_RPC_REF`: The git reference of stellar-rpc to build. - -For example: +horizon, etc, use the `Makefile`. Edit the `images.json` file, adding a new +image then build that image specifying its tag name: ``` -make build \ - TAG=future \ - CORE_REF=... \ - CORE_CONFIGURE_FLAGS=... \ - HORIZON_REF=... \ - FRIENDBOT_REF=... \ - STELLAR_RPC_REF=... +make build TAG=mytag ``` ### Background vs. Interactive containers From e7f519cce6bab9879d8b4ecbbec6ffbe5f9e9e62 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 01:52:19 +1000 Subject: [PATCH 070/135] update workflow path to use quickstart repository --- .github/workflows/workflow-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/workflow-test.yml b/.github/workflows/workflow-test.yml index ee80c8fb2..0565d2859 100644 --- a/.github/workflows/workflow-test.yml +++ b/.github/workflows/workflow-test.yml @@ -6,7 +6,7 @@ on: jobs: build-custom: - uses: ./.github/workflows/build.yml + uses: stellar/quickstart/.github/workflows/build.yml with: sha: ${{ github.event.pull_request.head.sha || github.sha }} test: true From 14a26375cb8ead8a7f86d56a4f40dc09f09979e9 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 01:55:11 +1000 Subject: [PATCH 071/135] remove workflow test file --- .github/workflows/workflow-test.yml | 49 ----------------------------- 1 file changed, 49 deletions(-) delete mode 100644 .github/workflows/workflow-test.yml diff --git a/.github/workflows/workflow-test.yml b/.github/workflows/workflow-test.yml deleted file mode 100644 index 0565d2859..000000000 --- a/.github/workflows/workflow-test.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: Workflow Call Test - -on: - push: - -jobs: - - build-custom: - uses: stellar/quickstart/.github/workflows/build.yml - with: - sha: ${{ github.event.pull_request.head.sha || github.sha }} - test: true - image_json: | - { - "tag": "custom", - "config": { - "protocol_version_default": 23 - }, - "deps": [ - { "name": "xdr", "repo": "stellar/rs-stellar-xdr", "ref": "v23.0.0" }, - { "name": "core", "repo": "stellar/stellar-core", "ref": "v23.0.1", "options": { "configure_flags": "--disable-tests" } }, - { "name": "rpc", "repo": "stellar/stellar-rpc", "ref": "v23.0.1" }, - { "name": "horizon", "repo": "stellar/go", "ref": "horizon-v23.0.0" }, - { "name": "friendbot", "repo": "stellar/go", "ref": "horizon-v23.0.0" }, - { "name": "lab", "repo": "stellar/laboratory", "ref": "main" } - ], - "additional-tests": [] - } - - use-custom: - needs: build-custom - runs-on: ubuntu-latest - steps: - - uses: actions/download-artifact@v4 - with: - name: ${{ needs.build-custom.outputs.image_artifact }} - path: /tmp/ - - run: docker load -i /tmp/image - - run: > - docker run - --platform linux/amd64 - -d - -p - "8000:8000" - --name stellar - quickstart - --local - - run: sleep 10 - - run: curl http://localhost:8000 From 82a9b89b0764e182e06b1b91adf10d1be2992078 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 01:58:19 +1000 Subject: [PATCH 072/135] adjust workflow permissions based on image_json input --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3eeac8386..74e8a92ee 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -471,8 +471,8 @@ jobs: fail-fast: false name: 7 push (${{ matrix.tag }}, ${{ matrix.arch }}) permissions: - packages: write - statuses: write + packages: ${{ inputs.image_json && 'read' || 'write' }} + statuses: ${{ inputs.image_json && 'none' || 'write' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -502,8 +502,8 @@ jobs: fail-fast: false name: 8 push manifest (${{ matrix.tag }}) permissions: - packages: write - statuses: write + packages: ${{ inputs.image_json && 'read' || 'write' }} + statuses: ${{ inputs.image_json && 'none' || 'write' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 From af80aa5d03b4d6b33e927f3baa44c4ade5c6ae7c Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 06:32:42 +1000 Subject: [PATCH 073/135] simplify workflow permissions for json branch --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 74e8a92ee..25c403a35 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -471,8 +471,8 @@ jobs: fail-fast: false name: 7 push (${{ matrix.tag }}, ${{ matrix.arch }}) permissions: - packages: ${{ inputs.image_json && 'read' || 'write' }} - statuses: ${{ inputs.image_json && 'none' || 'write' }} + packages: read + statuses: none runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -502,8 +502,8 @@ jobs: fail-fast: false name: 8 push manifest (${{ matrix.tag }}) permissions: - packages: ${{ inputs.image_json && 'read' || 'write' }} - statuses: ${{ inputs.image_json && 'none' || 'write' }} + packages: read + statuses: none runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 From 7251d923773871b273fae9fe39456f9669ab6f84 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 08:51:06 +1000 Subject: [PATCH 074/135] update checkout actions to use github workflow repository and sha --- .github/workflows/build.yml | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 25c403a35..ea1b35048 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -89,10 +89,16 @@ jobs: deps: ${{ steps.deps.outputs.deps }} additional-tests: ${{ steps.tests.outputs.additional-tests }} steps: + - run: | + echo 1 ${{ github.workflow_repository }} + echo 2 ${{ github.repository }} + echo 3 ${{ github.workflow_sha }} + echo 4 ${{ env.sha }} - uses: actions/checkout@v2 with: fetch-depth: 0 # Get all history for the sha count below. - ref: ${{ env.sha }} + repository: ${{ github.workflow_repository || github.repository }} + ref: ${{ github.workflow_sha || env.sha }} - name: Tag Prefix id: tag-prefix run: | @@ -218,7 +224,8 @@ jobs: steps: - uses: actions/checkout@v3 with: - ref: ${{ env.sha }} + repository: ${{ github.workflow_repository || github.repository }} + ref: ${{ github.workflow_sha || env.sha }} - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - name: Build Image env: @@ -263,7 +270,8 @@ jobs: steps: - uses: actions/checkout@v3 with: - ref: ${{ env.sha }} + repository: ${{ github.workflow_repository || github.repository }} + ref: ${{ github.workflow_sha || env.sha }} - name: Collect Dep IDs id: ids run: @@ -373,7 +381,8 @@ jobs: df -h - uses: actions/checkout@v2 with: - ref: ${{ env.sha }} + repository: ${{ github.workflow_repository || github.repository }} + ref: ${{ github.workflow_sha || env.sha }} - name: Download Quickstart Image uses: actions/download-artifact@v4 with: @@ -477,7 +486,8 @@ jobs: steps: - uses: actions/checkout@v2 with: - ref: ${{ env.sha }} + repository: ${{ github.workflow_repository || github.repository }} + ref: ${{ github.workflow_sha || env.sha }} - name: Create Tag id: tag run: echo "tag=${{ needs.setup.outputs.tag-prefix }}${{ matrix.tag }}-${{ matrix.arch }}" | tee -a $GITHUB_OUTPUT @@ -508,7 +518,8 @@ jobs: steps: - uses: actions/checkout@v2 with: - ref: ${{ env.sha }} + repository: ${{ github.workflow_repository || github.repository }} + ref: ${{ github.workflow_sha || env.sha }} - name: Create Tag id: tag run: | From c648bca38c996adff78091e97286d7bd02f007df Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 08:53:10 +1000 Subject: [PATCH 075/135] update workflow reference to use github.workflow_ref instead of github.workflow_repository --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ea1b35048..bdba5b901 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -90,7 +90,7 @@ jobs: additional-tests: ${{ steps.tests.outputs.additional-tests }} steps: - run: | - echo 1 ${{ github.workflow_repository }} + echo 1 ${{ github.workflow_ref }} echo 2 ${{ github.repository }} echo 3 ${{ github.workflow_sha }} echo 4 ${{ env.sha }} From b1179a337d75a9ed64c006a2712dd38e62d26a5a Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 08:57:20 +1000 Subject: [PATCH 076/135] add github action repository and ref to build workflow --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bdba5b901..ba5990506 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -94,6 +94,8 @@ jobs: echo 2 ${{ github.repository }} echo 3 ${{ github.workflow_sha }} echo 4 ${{ env.sha }} + echo 5 ${{ github.action_repository }} + echo 6 ${{ github.action_ref }} - uses: actions/checkout@v2 with: fetch-depth: 0 # Get all history for the sha count below. From 85073e75514197ab09fabec335713742134b5414 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 08:59:01 +1000 Subject: [PATCH 077/135] add debug output for github event --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ba5990506..c89b02540 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -96,6 +96,7 @@ jobs: echo 4 ${{ env.sha }} echo 5 ${{ github.action_repository }} echo 6 ${{ github.action_ref }} + echo 7 ${{ toJSON(github.event) }} - uses: actions/checkout@v2 with: fetch-depth: 0 # Get all history for the sha count below. From f3638017bab08614d25b692d3d149b5c07528165 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 08:59:45 +1000 Subject: [PATCH 078/135] wrap github event json in quotes --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c89b02540..346a432a2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -96,7 +96,7 @@ jobs: echo 4 ${{ env.sha }} echo 5 ${{ github.action_repository }} echo 6 ${{ github.action_ref }} - echo 7 ${{ toJSON(github.event) }} + echo 7 "${{ toJSON(github.event) }}" - uses: actions/checkout@v2 with: fetch-depth: 0 # Get all history for the sha count below. From d14bbcce8617bda526550981fedbc611cb4a535c Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 09:26:15 +1000 Subject: [PATCH 079/135] update build workflow to use inputs for repo and sha --- .github/workflows/build.yml | 44 ++++++++++++++----------------------- 1 file changed, 17 insertions(+), 27 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 346a432a2..c38a4416d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,14 +25,12 @@ name: Build # requires more work to do so. on: - push: - branches: - - main - pull_request: - schedule: - - cron: '0 0 * * *' workflow_call: inputs: + repo: + description: "Quickstart repo where quickstart is hosted" + type: "string" + default: "stellar/quickstart" sha: description: "Quickstart sha to build" type: "string" @@ -42,7 +40,7 @@ on: type: "string" required: true test: - description: "Whether the general image tests should run" + description: "Whether the image tests should run" type: "boolean" default: false outputs: @@ -89,19 +87,11 @@ jobs: deps: ${{ steps.deps.outputs.deps }} additional-tests: ${{ steps.tests.outputs.additional-tests }} steps: - - run: | - echo 1 ${{ github.workflow_ref }} - echo 2 ${{ github.repository }} - echo 3 ${{ github.workflow_sha }} - echo 4 ${{ env.sha }} - echo 5 ${{ github.action_repository }} - echo 6 ${{ github.action_ref }} - echo 7 "${{ toJSON(github.event) }}" - uses: actions/checkout@v2 with: fetch-depth: 0 # Get all history for the sha count below. - repository: ${{ github.workflow_repository || github.repository }} - ref: ${{ github.workflow_sha || env.sha }} + repository: ${{ inputs.repo || github.repository }} + ref: ${{ inputs.sha || env.sha }} - name: Tag Prefix id: tag-prefix run: | @@ -227,8 +217,8 @@ jobs: steps: - uses: actions/checkout@v3 with: - repository: ${{ github.workflow_repository || github.repository }} - ref: ${{ github.workflow_sha || env.sha }} + repository: ${{ inputs.repo || github.repository }} + ref: ${{ inputs.sha || env.sha }} - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - name: Build Image env: @@ -273,8 +263,8 @@ jobs: steps: - uses: actions/checkout@v3 with: - repository: ${{ github.workflow_repository || github.repository }} - ref: ${{ github.workflow_sha || env.sha }} + repository: ${{ inputs.repo || github.repository }} + ref: ${{ inputs.sha || env.sha }} - name: Collect Dep IDs id: ids run: @@ -384,8 +374,8 @@ jobs: df -h - uses: actions/checkout@v2 with: - repository: ${{ github.workflow_repository || github.repository }} - ref: ${{ github.workflow_sha || env.sha }} + repository: ${{ inputs.repo || github.repository }} + ref: ${{ inputs.sha || env.sha }} - name: Download Quickstart Image uses: actions/download-artifact@v4 with: @@ -489,8 +479,8 @@ jobs: steps: - uses: actions/checkout@v2 with: - repository: ${{ github.workflow_repository || github.repository }} - ref: ${{ github.workflow_sha || env.sha }} + repository: ${{ inputs.repo || github.repository }} + ref: ${{ inputs.sha || env.sha }} - name: Create Tag id: tag run: echo "tag=${{ needs.setup.outputs.tag-prefix }}${{ matrix.tag }}-${{ matrix.arch }}" | tee -a $GITHUB_OUTPUT @@ -521,8 +511,8 @@ jobs: steps: - uses: actions/checkout@v2 with: - repository: ${{ github.workflow_repository || github.repository }} - ref: ${{ github.workflow_sha || env.sha }} + repository: ${{ inputs.repo || github.repository }} + ref: ${{ inputs.sha || env.sha }} - name: Create Tag id: tag run: | From 8f42f9e369a585659b1e21888568015a3aabb8c3 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 09:34:35 +1000 Subject: [PATCH 080/135] update workflow to use ref input and env sha --- .github/workflows/build.yml | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c38a4416d..9f400506f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,10 +31,10 @@ on: description: "Quickstart repo where quickstart is hosted" type: "string" default: "stellar/quickstart" - sha: - description: "Quickstart sha to build" + ref: + description: "Quickstart ref to build (sha, branch, tag)" type: "string" - required: true + default: "main" image_json: description: "A custom image.json (a single image from the same format as images.json)" type: "string" @@ -57,8 +57,7 @@ concurrency: cancel-in-progress: true env: - sha: ${{ inputs.sha || github.event.pull_request.head.sha || github.sha }} - image_repo: ${{ format('{0}/{1}', secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io', github.repository) }} + image_repo: ${{ format('{0}/{1}', secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io', inputs.repo) }} # Cache ID is a value inserted into cache keys. Whenever changing the build # in a way that needs to use entirely new fresh builds, increment the number # by one so that all the keys become new. @@ -90,8 +89,11 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 # Get all history for the sha count below. - repository: ${{ inputs.repo || github.repository }} - ref: ${{ inputs.sha || env.sha }} + repository: ${{ inputs.repo }} + ref: ${{ inputs.ref }} + - name: Sha + run: | + echo "sha=$(git rev-parse HEAD)" | tee -a $GITHUB_ENV - name: Tag Prefix id: tag-prefix run: | @@ -217,8 +219,8 @@ jobs: steps: - uses: actions/checkout@v3 with: - repository: ${{ inputs.repo || github.repository }} - ref: ${{ inputs.sha || env.sha }} + repository: ${{ inputs.repo }} + ref: ${{ env.sha }} - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - name: Build Image env: @@ -263,8 +265,8 @@ jobs: steps: - uses: actions/checkout@v3 with: - repository: ${{ inputs.repo || github.repository }} - ref: ${{ inputs.sha || env.sha }} + repository: ${{ inputs.repo }} + ref: ${{ env.sha }} - name: Collect Dep IDs id: ids run: @@ -374,8 +376,8 @@ jobs: df -h - uses: actions/checkout@v2 with: - repository: ${{ inputs.repo || github.repository }} - ref: ${{ inputs.sha || env.sha }} + repository: ${{ inputs.repo }} + ref: ${{ env.sha }} - name: Download Quickstart Image uses: actions/download-artifact@v4 with: @@ -479,8 +481,8 @@ jobs: steps: - uses: actions/checkout@v2 with: - repository: ${{ inputs.repo || github.repository }} - ref: ${{ inputs.sha || env.sha }} + repository: ${{ inputs.repo }} + ref: ${{ env.sha }} - name: Create Tag id: tag run: echo "tag=${{ needs.setup.outputs.tag-prefix }}${{ matrix.tag }}-${{ matrix.arch }}" | tee -a $GITHUB_OUTPUT @@ -511,8 +513,8 @@ jobs: steps: - uses: actions/checkout@v2 with: - repository: ${{ inputs.repo || github.repository }} - ref: ${{ inputs.sha || env.sha }} + repository: ${{ inputs.repo }} + ref: ${{ env.sha }} - name: Create Tag id: tag run: | From c560f37ce1194f7e4c968faa479e11f856643af2 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 09:45:07 +1000 Subject: [PATCH 081/135] use sha from setup job outputs instead of env --- .github/workflows/build.yml | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9f400506f..aec60e85b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -80,6 +80,7 @@ jobs: name: 1 setup runs-on: ubuntu-latest outputs: + sha: ${{ steps.sha.outputs.sha }} tag-prefix: ${{ steps.tag-prefix.outputs.tag-prefix }} tag-alias-prefix: ${{ steps.tag-prefix.outputs.tag-alias-prefix }} images: ${{ steps.images.outputs.images }} @@ -92,8 +93,9 @@ jobs: repository: ${{ inputs.repo }} ref: ${{ inputs.ref }} - name: Sha + id: sha run: | - echo "sha=$(git rev-parse HEAD)" | tee -a $GITHUB_ENV + echo "sha=$(git rev-parse HEAD)" | tee -a $GITHUB_OUTPUT - name: Tag Prefix id: tag-prefix run: | @@ -220,7 +222,7 @@ jobs: - uses: actions/checkout@v3 with: repository: ${{ inputs.repo }} - ref: ${{ env.sha }} + ref: ${{ needs.setup.outputs.sha }} - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - name: Build Image env: @@ -266,7 +268,7 @@ jobs: - uses: actions/checkout@v3 with: repository: ${{ inputs.repo }} - ref: ${{ env.sha }} + ref: ${{ needs.setup.outputs.sha }} - name: Collect Dep IDs id: ids run: @@ -332,8 +334,8 @@ jobs: --platform linux/${{ matrix.arch }} -f Dockerfile -t quickstart - --label org.opencontainers.image.revision="${{ env.sha }}" - --build-arg REVISION="${{ env.sha }}" + --label org.opencontainers.image.revision="${{ needs.setup.outputs.sha }}" + --build-arg REVISION="${{ needs.setup.outputs.sha }}" --build-arg PROTOCOL_VERSION_DEFAULT="${{ matrix.image.config.protocol_version_default }}" --build-arg XDR_IMAGE_REF=stellar-xdr:${{ matrix.arch }} --build-arg CORE_IMAGE_REF=stellar-core:${{ matrix.arch }} @@ -377,7 +379,7 @@ jobs: - uses: actions/checkout@v2 with: repository: ${{ inputs.repo }} - ref: ${{ env.sha }} + ref: ${{ needs.setup.outputs.sha }} - name: Download Quickstart Image uses: actions/download-artifact@v4 with: @@ -482,13 +484,13 @@ jobs: - uses: actions/checkout@v2 with: repository: ${{ inputs.repo }} - ref: ${{ env.sha }} + ref: ${{ needs.setup.outputs.sha }} - name: Create Tag id: tag run: echo "tag=${{ needs.setup.outputs.tag-prefix }}${{ matrix.tag }}-${{ matrix.arch }}" | tee -a $GITHUB_OUTPUT - uses: ./.github/actions/push with: - head_sha: ${{ env.sha }} + head_sha: ${{ needs.setup.outputs.sha }} artifact_name: image-quickstart-${{ matrix.tag }}-${{ matrix.arch }}.tar artifact_image_file: image image: quickstart @@ -514,7 +516,7 @@ jobs: - uses: actions/checkout@v2 with: repository: ${{ inputs.repo }} - ref: ${{ env.sha }} + ref: ${{ needs.setup.outputs.sha }} - name: Create Tag id: tag run: | @@ -522,7 +524,7 @@ jobs: echo "tag-alias=${{ needs.setup.outputs.tag-alias-prefix }}${{ matrix.tag }}" | tee -a $GITHUB_OUTPUT - uses: ./.github/actions/push-manifest with: - head_sha: ${{ env.sha }} + head_sha: ${{ needs.setup.outputs.sha }} image: ${{ env.image_repo }}:${{ steps.tag.outputs.tag }} images: > ${{ env.image_repo }}:${{ steps.tag.outputs.tag }}-amd64 @@ -532,7 +534,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} - uses: ./.github/actions/push-alias with: - head_sha: ${{ env.sha }} + head_sha: ${{ needs.setup.outputs.sha }} image: ${{ env.image_repo }}:${{ steps.tag.outputs.tag }} image-alias: ${{ env.image_repo }}:${{ steps.tag.outputs.tag-alias }} registry: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} From 4afc22a87e7d074ed26a10915dc3295ea16fba7c Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 11:01:07 +1000 Subject: [PATCH 082/135] split across workflows --- .github/actions/push-alias/action.yml | 6 +- .github/workflows/build.yml | 133 ++++++++++++-------------- .github/workflows/ci.yml | 85 ++++++++++++++++ 3 files changed, 147 insertions(+), 77 deletions(-) create mode 100644 .github/workflows/ci.yml diff --git a/.github/actions/push-alias/action.yml b/.github/actions/push-alias/action.yml index fbf3443db..980938a68 100644 --- a/.github/actions/push-alias/action.yml +++ b/.github/actions/push-alias/action.yml @@ -4,7 +4,7 @@ inputs: required: true image: required: true - image-alias: + alias: required: true registry: required: true @@ -27,7 +27,7 @@ runs: - shell: bash run: | - docker buildx imagetools create -t ${{ inputs.image-alias }} ${{ inputs.image }} + docker buildx imagetools create -t ${{ inputs.alias }} ${{ inputs.image }} - uses: actions/github-script@v5 with: @@ -37,6 +37,6 @@ runs: repo: context.repo.repo, sha: '${{ inputs.head_sha }}', state: 'success', - context: `${{ inputs.image-alias }}`, + context: `${{ inputs.alias }}`, description: 'Available', }); diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index aec60e85b..54ac4e930 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,18 +35,38 @@ on: description: "Quickstart ref to build (sha, branch, tag)" type: "string" default: "main" - image_json: - description: "A custom image.json (a single image from the same format as images.json)" + images: + description: "A custom image.json (a single image from the same format as images.json), if not provided the full images.json is run" type: "string" required: true + archs: + description: "Architectures to build for as an array (amd64, arm64)" + type: "string" + default: '["amd64"]' test: description: "Whether the image tests should run" type: "boolean" + default: true + push: + description: "Whether the image should be pushed to the registry" + type: "boolean" default: false - outputs: - image_artifact: - description: "Name of the artifact containing the image built" - value: ${{ jobs.build.outputs.artifact }} + cache_id: + description: "A value insert into cache keys to namespace cache usage, or invalidate it by incrementing" + type: "string" + default: 6 + registry: + description: "Registry to push to" + type: "string" + registry_repo: + description: "Repo at the registry to push to" + type: "string" + registry_username: + description: "Username to auth with the registry" + type: "string" + registry_password: + description: "Password to auth with the registry" + type: "string" # Prevent more than one build of this workflow for a branch to be running at the # same time, and if multiple are queued, only run the latest, cancelling any @@ -57,11 +77,6 @@ concurrency: cancel-in-progress: true env: - image_repo: ${{ format('{0}/{1}', secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io', inputs.repo) }} - # Cache ID is a value inserted into cache keys. Whenever changing the build - # in a way that needs to use entirely new fresh builds, increment the number - # by one so that all the keys become new. - cache_id: 6 artifact_retention_days_for_image: 7 artifact_retention_days_for_logs: 60 @@ -81,48 +96,23 @@ jobs: runs-on: ubuntu-latest outputs: sha: ${{ steps.sha.outputs.sha }} - tag-prefix: ${{ steps.tag-prefix.outputs.tag-prefix }} - tag-alias-prefix: ${{ steps.tag-prefix.outputs.tag-alias-prefix }} images: ${{ steps.images.outputs.images }} deps: ${{ steps.deps.outputs.deps }} additional-tests: ${{ steps.tests.outputs.additional-tests }} steps: - uses: actions/checkout@v2 with: - fetch-depth: 0 # Get all history for the sha count below. repository: ${{ inputs.repo }} ref: ${{ inputs.ref }} - name: Sha id: sha run: | echo "sha=$(git rev-parse HEAD)" | tee -a $GITHUB_OUTPUT - - name: Tag Prefix - id: tag-prefix - run: | - pr_prefix="${{ inputs.image_json && 'custom-' || (github.event_name == 'pull_request' && format('pr{0}-', github.event.pull_request.number) || '') }}" - commit_count="$(git rev-list HEAD --count --first-parent)" - build_number="${{ github.run_number }}.${{ github.run_attempt }}" - echo "tag-prefix=${pr_prefix}v${commit_count}-b${build_number}-" | tee -a $GITHUB_OUTPUT - echo "tag-alias-prefix=${pr_prefix}" | tee -a $GITHUB_OUTPUT - - name: Images - if: '!inputs.image_json' - run: | - images="$(> $GITHUB_ENV - - name: Images (for workflow_call) - if: inputs.image_json - env: - image: ${{ inputs.image_json }} - run: | - images="$(<<< $image jq -c '[ . ]')" - <<< $images jq - echo "images=$images" >> $GITHUB_ENV - name: Images with Extras id: images env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + images: ${{ inputs.images }} run: | images="$(<<< $images ./.scripts/images-with-extras)" <<< $images jq @@ -146,7 +136,7 @@ jobs: strategy: matrix: dep: ${{ fromJSON(needs.setup.outputs.deps) }} - arch: ${{ inputs.image_json && fromJSON('["amd64"]') || fromJSON('["amd64", "arm64"]') }} + arch: ${{ fromJSON(inputs.archs) }} fail-fast: false name: 2 load (${{ matrix.dep.name }}, ${{ matrix.dep.ref }}, ${{ matrix.arch }}, ${{ matrix.dep.options && toJSON(matrix.dep.options) || '-' }}) runs-on: ubuntu-latest @@ -171,7 +161,7 @@ jobs: id: cache uses: actions/cache/restore@v3 with: - key: ${{ env.cache_id }}-${{ env.image_filename }} + key: ${{ inputs.cache_id }}-${{ env.image_filename }} path: /tmp/${{ env.image_filename }} - name: Upload Image to Artifacts if: steps.cache.outputs.cache-hit == 'true' @@ -241,7 +231,7 @@ jobs: uses: actions/cache/save@v3 id: cache with: - key: ${{ env.cache_id }}-${{ env.image_filename }} + key: ${{ inputs.cache_id }}-${{ env.image_filename }} path: /tmp/${{ env.image_filename }} - name: Upload Image to Artifacts uses: actions/upload-artifact@v4 @@ -256,12 +246,10 @@ jobs: strategy: matrix: image: ${{ fromJSON(needs.setup.outputs.images) }} - arch: ${{ inputs.image_json && fromJSON('["amd64"]') || fromJSON('["amd64", "arm64"]') }} + arch: ${{ fromJSON(inputs.archs) }} fail-fast: false name: 5 build (quickstart, ${{ matrix.image.tag }}, ${{ matrix.arch }}) runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} - outputs: - artifact: image-quickstart-${{ matrix.image.tag }}-${{ matrix.arch }}.tar env: image_json: ${{ toJSON(matrix.image) }} steps: @@ -359,7 +347,7 @@ jobs: strategy: matrix: tag: ${{ fromJSON(needs.setup.outputs.images).*.tag }} - arch: ${{ inputs.image_json && fromJSON('["amd64"]') || fromJSON('["amd64", "arm64"]') }} + arch: ${{ fromJSON(inputs.archs) }} network: ["local"] enable: ${{ inputs.image_json && fromJSON('["core,rpc,horizon"]') || fromJSON('["core","rpc","core,rpc,horizon"]') }} options: [""] @@ -469,11 +457,11 @@ jobs: push: needs: [setup, build, test] - if: always() && !failure() && !cancelled() && ((github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository)) + if: always() && !failure() && !cancelled() && inputs.push strategy: matrix: tag: ${{ fromJSON(needs.setup.outputs.images).*.tag }} - arch: ["amd64", "arm64"] + arch: ${{ fromJSON(inputs.archs) }} fail-fast: false name: 7 push (${{ matrix.tag }}, ${{ matrix.arch }}) permissions: @@ -481,28 +469,24 @@ jobs: statuses: none runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - with: - repository: ${{ inputs.repo }} - ref: ${{ needs.setup.outputs.sha }} - name: Create Tag id: tag run: echo "tag=${{ needs.setup.outputs.tag-prefix }}${{ matrix.tag }}-${{ matrix.arch }}" | tee -a $GITHUB_OUTPUT - uses: ./.github/actions/push with: - head_sha: ${{ needs.setup.outputs.sha }} + head_sha: ${{ github.event.pull_request.head.sha || github.sha }} artifact_name: image-quickstart-${{ matrix.tag }}-${{ matrix.arch }}.tar artifact_image_file: image image: quickstart arch: ${{ matrix.arch }} - name: ${{ env.image_repo }}:${{ steps.tag.outputs.tag }} - registry: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} - username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} - password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} + name: ${{ inputs.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }} + registry: ${{ inputs.registry }} + username: ${{ inputs.registry_username }} + password: ${{ inputs.registry_password }} push-manifest: needs: [setup, push] - if: always() && !failure() && !cancelled() && ((github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository)) + if: always() && !failure() && !cancelled() && inputs.push strategy: matrix: tag: ${{ fromJSON(needs.setup.outputs.images).*.tag }} @@ -513,37 +497,38 @@ jobs: statuses: none runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - with: - repository: ${{ inputs.repo }} - ref: ${{ needs.setup.outputs.sha }} - name: Create Tag id: tag run: | echo "tag=${{ needs.setup.outputs.tag-prefix }}${{ matrix.tag }}" | tee -a $GITHUB_OUTPUT echo "tag-alias=${{ needs.setup.outputs.tag-alias-prefix }}${{ matrix.tag }}" | tee -a $GITHUB_OUTPUT + - name: Prepare Image List + id: images + env: + archs: ${{ inputs.archs }} + run: | + images="$(<<< $archs jq 'map("${{ inputs.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }}-\(.)")')" + echo "images=$images" | tee -a $GITHUB_OUTPUT - uses: ./.github/actions/push-manifest with: - head_sha: ${{ needs.setup.outputs.sha }} - image: ${{ env.image_repo }}:${{ steps.tag.outputs.tag }} - images: > - ${{ env.image_repo }}:${{ steps.tag.outputs.tag }}-amd64 - ${{ env.image_repo }}:${{ steps.tag.outputs.tag }}-arm64 - registry: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} - username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} - password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} + head_sha: ${{ github.event.pull_request.head.sha || github.sha }} + image: ${{ inputs.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }} + images: ${{ steps.images.outputs.images }} + registry: ${{ inputs.registry }} + username: ${{ inputs.registry_username }} + password: ${{ inputs.registry_password }} - uses: ./.github/actions/push-alias with: head_sha: ${{ needs.setup.outputs.sha }} - image: ${{ env.image_repo }}:${{ steps.tag.outputs.tag }} - image-alias: ${{ env.image_repo }}:${{ steps.tag.outputs.tag-alias }} - registry: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} - username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} - password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} + image: ${{ inputs.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }} + alias: ${{ inputs.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag-alias }} + registry: ${{ inputs.registry }} + username: ${{ inputs.registry_username }} + password: ${{ inputs.registry_password }} action: needs: [setup, push-manifest] - if: always() && !failure() && !cancelled() && ((github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository)) + if: always() && !failure() && !cancelled() && inputs.push strategy: matrix: tag: ${{ fromJSON(needs.setup.outputs.images).*.tag }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..8eaa88650 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,85 @@ +me: CI + +on: + push: + branches: + - main + pull_request: + schedule: + - cron: '0 0 * * *' + +jobs: + + complete: + if: always() + name: complete + needs: [build] + runs-on: ubuntu-latest + steps: + - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') + run: exit 1 + + setup: + name: 1 setup + runs-on: ubuntu-latest + outputs: + tag-prefix: ${{ steps.tag-prefix.outputs.tag-prefix }} + tag-alias-prefix: ${{ steps.tag-prefix.outputs.tag-alias-prefix }} + images: ${{ steps.images.outputs.images }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 # Get all history for the sha count below. + ref: ${{ github.event.pull_request.head.sha || github.sha }} + - name: Tag Prefix + id: tag-prefix + run: | + pr_prefix="${{ github.event_name == 'pull_request' && format('pr{0}-', github.event.pull_request.number) || '' }}" + commit_count="$(git rev-list HEAD --count --first-parent)" + build_number="${{ github.run_number }}.${{ github.run_attempt }}" + echo "tag-prefix=${pr_prefix}v${commit_count}-b${build_number}-" | tee -a $GITHUB_OUTPUT + echo "tag-alias-prefix=${pr_prefix}" | tee -a $GITHUB_OUTPUT + - name: Images + run: | + images="$(> $GITHUB_OUTPUT + + build: + name: 2 build + needs: images + uses: ./.github/workflows/build.yml + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + images: ${{ needs.setup.outputs.images }} + archs: '["amd64","arm64"]' + push: ${{ (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) }} + registry: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} + registry_repo: ${{ github.repository) }} + registry_username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} + registry_password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} + tag-prefix: ${{ needs.setup.outputs.tag-prefix }} + tag-alias-prefix: ${{ needs.setup.outputs.tag-alias-prefix }} + + use: + name: 3 use + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v4 + with: + name: image-quickstart-latest-amd64.tar + path: /tmp/ + - run: docker load -i /tmp/image + - run: > + docker run + --platform linux/amd64 + -d + -p + "8000:8000" + --name stellar + quickstart + --local + - run: sleep 10 + - run: curl http://localhost:8000 From 89450a327359904c42442d21bd52e9082755b586 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 11:01:59 +1000 Subject: [PATCH 083/135] rename ci workflow name to CI --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8eaa88650..bd28c32bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -me: CI +name: CI on: push: From 2ed0259a8471240a13cf6f9fe7905340b923341f Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 11:04:36 +1000 Subject: [PATCH 084/135] add secrets to build workflow --- .github/workflows/build.yml | 1 + .github/workflows/ci.yml | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 54ac4e930..de144fcc9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -55,6 +55,7 @@ on: description: "A value insert into cache keys to namespace cache usage, or invalidate it by incrementing" type: "string" default: 6 + secrets: registry: description: "Registry to push to" type: "string" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd28c32bb..68b968c8c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,12 +55,13 @@ jobs: images: ${{ needs.setup.outputs.images }} archs: '["amd64","arm64"]' push: ${{ (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) }} + tag-prefix: ${{ needs.setup.outputs.tag-prefix }} + tag-alias-prefix: ${{ needs.setup.outputs.tag-alias-prefix }} + secrets: registry: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} registry_repo: ${{ github.repository) }} registry_username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} registry_password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} - tag-prefix: ${{ needs.setup.outputs.tag-prefix }} - tag-alias-prefix: ${{ needs.setup.outputs.tag-alias-prefix }} use: name: 3 use From 37a15569699f30273220738be21aa6a231d18d32 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 11:05:13 +1000 Subject: [PATCH 085/135] use secrets instead of inputs for registry credentials --- .github/workflows/build.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index de144fcc9..aebf9544e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -480,10 +480,10 @@ jobs: artifact_image_file: image image: quickstart arch: ${{ matrix.arch }} - name: ${{ inputs.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }} - registry: ${{ inputs.registry }} - username: ${{ inputs.registry_username }} - password: ${{ inputs.registry_password }} + name: ${{ secrets.registry }}/${{ secrets.registry_repo }}:${{ steps.tag.outputs.tag }} + registry: ${{ secrets.registry }} + username: ${{ secrets.registry_username }} + password: ${{ secrets.registry_password }} push-manifest: needs: [setup, push] @@ -508,24 +508,24 @@ jobs: env: archs: ${{ inputs.archs }} run: | - images="$(<<< $archs jq 'map("${{ inputs.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }}-\(.)")')" + images="$(<<< $archs jq 'map("${{ secrets.registry }}/${{ secrets.registry_repo }}:${{ steps.tag.outputs.tag }}-\(.)")')" echo "images=$images" | tee -a $GITHUB_OUTPUT - uses: ./.github/actions/push-manifest with: head_sha: ${{ github.event.pull_request.head.sha || github.sha }} - image: ${{ inputs.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }} + image: ${{ secrets.registry }}/${{ secrets.registry_repo }}:${{ steps.tag.outputs.tag }} images: ${{ steps.images.outputs.images }} - registry: ${{ inputs.registry }} - username: ${{ inputs.registry_username }} - password: ${{ inputs.registry_password }} + registry: ${{ secrets.registry }} + username: ${{ secrets.registry_username }} + password: ${{ secrets.registry_password }} - uses: ./.github/actions/push-alias with: head_sha: ${{ needs.setup.outputs.sha }} - image: ${{ inputs.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }} - alias: ${{ inputs.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag-alias }} - registry: ${{ inputs.registry }} - username: ${{ inputs.registry_username }} - password: ${{ inputs.registry_password }} + image: ${{ secrets.registry }}/${{ secrets.registry_repo }}:${{ steps.tag.outputs.tag }} + alias: ${{ secrets.registry }}/${{ secrets.registry_repo }}:${{ steps.tag.outputs.tag-alias }} + registry: ${{ secrets.registry }} + username: ${{ secrets.registry_username }} + password: ${{ secrets.registry_password }} action: needs: [setup, push-manifest] From 9c317eca41936717cc2b3624c830e212c323f1a6 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 11:05:38 +1000 Subject: [PATCH 086/135] fix typo in workflow file --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 68b968c8c..2e1b3da46 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ jobs: tag-alias-prefix: ${{ needs.setup.outputs.tag-alias-prefix }} secrets: registry: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} - registry_repo: ${{ github.repository) }} + registry_repo: ${{ github.repository }} registry_username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} registry_password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} From 42c619ff99028c5d57483fb195df350be0767667 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 11:06:13 +1000 Subject: [PATCH 087/135] remove type declarations from workflow secrets --- .github/workflows/build.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index aebf9544e..fcb330000 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -58,16 +58,12 @@ on: secrets: registry: description: "Registry to push to" - type: "string" registry_repo: description: "Repo at the registry to push to" - type: "string" registry_username: description: "Username to auth with the registry" - type: "string" registry_password: description: "Password to auth with the registry" - type: "string" # Prevent more than one build of this workflow for a branch to be running at the # same time, and if multiple are queued, only run the latest, cancelling any From d51d7096e73af5281cf81f5e7ad62ad80c7f252b Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 11:06:36 +1000 Subject: [PATCH 088/135] change build job dependency to setup --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e1b3da46..1c53f2506 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,7 +48,7 @@ jobs: build: name: 2 build - needs: images + needs: setup uses: ./.github/workflows/build.yml with: ref: ${{ github.event.pull_request.head.sha || github.sha }} From 4c8e757b2db6b3128c96d8d15089fae2b88ec9d5 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 11:07:51 +1000 Subject: [PATCH 089/135] add tag-prefix and tag-alias-prefix inputs to build workflow --- .github/workflows/build.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fcb330000..0ba6db0ca 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,6 +43,14 @@ on: description: "Architectures to build for as an array (amd64, arm64)" type: "string" default: '["amd64"]' + tag-prefix: + description: "Tag prefix for the image when pushed" + type: "string" + default: '' + tag-alias-prefix: + description: "A second tag prefix for the image when pushed that'll be pushed in addition as an alias" + type: "string" + default: '' test: description: "Whether the image tests should run" type: "boolean" From cad26741d81a6d402f4f32384fb47c9034c8b406 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 11:08:35 +1000 Subject: [PATCH 090/135] use inputs instead of needs.setup.outputs for tag prefix --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0ba6db0ca..8bc521d90 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -476,7 +476,7 @@ jobs: steps: - name: Create Tag id: tag - run: echo "tag=${{ needs.setup.outputs.tag-prefix }}${{ matrix.tag }}-${{ matrix.arch }}" | tee -a $GITHUB_OUTPUT + run: echo "tag=${{ inputs.tag-prefix }}${{ matrix.tag }}-${{ matrix.arch }}" | tee -a $GITHUB_OUTPUT - uses: ./.github/actions/push with: head_sha: ${{ github.event.pull_request.head.sha || github.sha }} @@ -505,8 +505,8 @@ jobs: - name: Create Tag id: tag run: | - echo "tag=${{ needs.setup.outputs.tag-prefix }}${{ matrix.tag }}" | tee -a $GITHUB_OUTPUT - echo "tag-alias=${{ needs.setup.outputs.tag-alias-prefix }}${{ matrix.tag }}" | tee -a $GITHUB_OUTPUT + echo "tag=${{ inputs.tag-prefix }}${{ matrix.tag }}" | tee -a $GITHUB_OUTPUT + echo "tag-alias=${{ inputs.tag-alias-prefix }}${{ matrix.tag }}" | tee -a $GITHUB_OUTPUT - name: Prepare Image List id: images env: @@ -541,4 +541,4 @@ jobs: name: 9 test action (${{ matrix.tag }}) uses: ./.github/workflows/action-test.yml with: - tag: ${{ needs.setup.outputs.tag-prefix }}${{ matrix.tag }} + tag: ${{ inputs.tag-prefix }}${{ matrix.tag }} From ac4b0e1d4860f6f20c34eca1ae7360fb3216b919 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 11:12:08 +1000 Subject: [PATCH 091/135] add id field to images step in ci workflow --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1c53f2506..fb9a04ada 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,6 +40,7 @@ jobs: echo "tag-prefix=${pr_prefix}v${commit_count}-b${build_number}-" | tee -a $GITHUB_OUTPUT echo "tag-alias-prefix=${pr_prefix}" | tee -a $GITHUB_OUTPUT - name: Images + id: images run: | images="$( Date: Wed, 8 Oct 2025 11:39:56 +1000 Subject: [PATCH 092/135] update build workflow to require archs input and add checkout steps --- .github/workflows/build.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8bc521d90..5de58e22b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -40,9 +40,9 @@ on: type: "string" required: true archs: - description: "Architectures to build for as an array (amd64, arm64)" + description: 'Architectures to build for as an array (e.g. ["amd64", "arm64"])' type: "string" - default: '["amd64"]' + required: true tag-prefix: description: "Tag prefix for the image when pushed" type: "string" @@ -474,6 +474,10 @@ jobs: statuses: none runs-on: ubuntu-latest steps: + - uses: actions/checkout@v3 + with: + repository: ${{ inputs.repo }} + ref: ${{ needs.setup.outputs.sha }} - name: Create Tag id: tag run: echo "tag=${{ inputs.tag-prefix }}${{ matrix.tag }}-${{ matrix.arch }}" | tee -a $GITHUB_OUTPUT @@ -502,6 +506,10 @@ jobs: statuses: none runs-on: ubuntu-latest steps: + - uses: actions/checkout@v3 + with: + repository: ${{ inputs.repo }} + ref: ${{ needs.setup.outputs.sha }} - name: Create Tag id: tag run: | From b35105b4e966d626a784b1fea3cab13589a870d9 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 12:00:20 +1000 Subject: [PATCH 093/135] split build and push --- .github/workflows/build.yml | 120 +-------------------------- .github/workflows/ci.yml | 27 +++++- .github/workflows/push.yml | 160 ++++++++++++++++++++++++++++++++++++ 3 files changed, 187 insertions(+), 120 deletions(-) create mode 100644 .github/workflows/push.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5de58e22b..82d73f8ca 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,7 +32,7 @@ on: type: "string" default: "stellar/quickstart" ref: - description: "Quickstart ref to build (sha, branch, tag)" + description: "Quickstart ref to build should match workflow (sha, branch, tag)" type: "string" default: "main" images: @@ -43,35 +43,14 @@ on: description: 'Architectures to build for as an array (e.g. ["amd64", "arm64"])' type: "string" required: true - tag-prefix: - description: "Tag prefix for the image when pushed" - type: "string" - default: '' - tag-alias-prefix: - description: "A second tag prefix for the image when pushed that'll be pushed in addition as an alias" - type: "string" - default: '' test: description: "Whether the image tests should run" type: "boolean" default: true - push: - description: "Whether the image should be pushed to the registry" - type: "boolean" - default: false cache_id: description: "A value insert into cache keys to namespace cache usage, or invalidate it by incrementing" type: "string" default: 6 - secrets: - registry: - description: "Registry to push to" - registry_repo: - description: "Repo at the registry to push to" - registry_username: - description: "Username to auth with the registry" - registry_password: - description: "Password to auth with the registry" # Prevent more than one build of this workflow for a branch to be running at the # same time, and if multiple are queued, only run the latest, cancelling any @@ -90,7 +69,7 @@ jobs: complete: if: always() name: complete - needs: [setup, load, build, test, push, push-manifest, action] + needs: [setup, load, build, test] runs-on: ubuntu-latest steps: - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') @@ -326,7 +305,7 @@ jobs: docker build --platform linux/${{ matrix.arch }} -f Dockerfile - -t quickstart + -t quickstart:${{ matrix.image.tag }}-${{ matrix.arch }} --label org.opencontainers.image.revision="${{ needs.setup.outputs.sha }}" --build-arg REVISION="${{ needs.setup.outputs.sha }}" --build-arg PROTOCOL_VERSION_DEFAULT="${{ matrix.image.config.protocol_version_default }}" @@ -391,7 +370,7 @@ jobs: "8000:8000" -p "11626:11626" --name stellar - quickstart + quickstart:${{ matrix.tag }}-${{ matrix.arch }} --${{ matrix.network }} --enable ${{ matrix.enable }} ${{ matrix.options }} @@ -459,94 +438,3 @@ jobs: name: logs-${{ matrix.tag }}-${{ matrix.arch }}-test-${{ strategy.job-index }} path: logs retention-days: ${{ env.artifact_retention_days_for_logs }} - - push: - needs: [setup, build, test] - if: always() && !failure() && !cancelled() && inputs.push - strategy: - matrix: - tag: ${{ fromJSON(needs.setup.outputs.images).*.tag }} - arch: ${{ fromJSON(inputs.archs) }} - fail-fast: false - name: 7 push (${{ matrix.tag }}, ${{ matrix.arch }}) - permissions: - packages: read - statuses: none - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - repository: ${{ inputs.repo }} - ref: ${{ needs.setup.outputs.sha }} - - name: Create Tag - id: tag - run: echo "tag=${{ inputs.tag-prefix }}${{ matrix.tag }}-${{ matrix.arch }}" | tee -a $GITHUB_OUTPUT - - uses: ./.github/actions/push - with: - head_sha: ${{ github.event.pull_request.head.sha || github.sha }} - artifact_name: image-quickstart-${{ matrix.tag }}-${{ matrix.arch }}.tar - artifact_image_file: image - image: quickstart - arch: ${{ matrix.arch }} - name: ${{ secrets.registry }}/${{ secrets.registry_repo }}:${{ steps.tag.outputs.tag }} - registry: ${{ secrets.registry }} - username: ${{ secrets.registry_username }} - password: ${{ secrets.registry_password }} - - push-manifest: - needs: [setup, push] - if: always() && !failure() && !cancelled() && inputs.push - strategy: - matrix: - tag: ${{ fromJSON(needs.setup.outputs.images).*.tag }} - fail-fast: false - name: 8 push manifest (${{ matrix.tag }}) - permissions: - packages: read - statuses: none - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - repository: ${{ inputs.repo }} - ref: ${{ needs.setup.outputs.sha }} - - name: Create Tag - id: tag - run: | - echo "tag=${{ inputs.tag-prefix }}${{ matrix.tag }}" | tee -a $GITHUB_OUTPUT - echo "tag-alias=${{ inputs.tag-alias-prefix }}${{ matrix.tag }}" | tee -a $GITHUB_OUTPUT - - name: Prepare Image List - id: images - env: - archs: ${{ inputs.archs }} - run: | - images="$(<<< $archs jq 'map("${{ secrets.registry }}/${{ secrets.registry_repo }}:${{ steps.tag.outputs.tag }}-\(.)")')" - echo "images=$images" | tee -a $GITHUB_OUTPUT - - uses: ./.github/actions/push-manifest - with: - head_sha: ${{ github.event.pull_request.head.sha || github.sha }} - image: ${{ secrets.registry }}/${{ secrets.registry_repo }}:${{ steps.tag.outputs.tag }} - images: ${{ steps.images.outputs.images }} - registry: ${{ secrets.registry }} - username: ${{ secrets.registry_username }} - password: ${{ secrets.registry_password }} - - uses: ./.github/actions/push-alias - with: - head_sha: ${{ needs.setup.outputs.sha }} - image: ${{ secrets.registry }}/${{ secrets.registry_repo }}:${{ steps.tag.outputs.tag }} - alias: ${{ secrets.registry }}/${{ secrets.registry_repo }}:${{ steps.tag.outputs.tag-alias }} - registry: ${{ secrets.registry }} - username: ${{ secrets.registry_username }} - password: ${{ secrets.registry_password }} - - action: - needs: [setup, push-manifest] - if: always() && !failure() && !cancelled() && inputs.push - strategy: - matrix: - tag: ${{ fromJSON(needs.setup.outputs.images).*.tag }} - fail-fast: false - name: 9 test action (${{ matrix.tag }}) - uses: ./.github/workflows/action-test.yml - with: - tag: ${{ inputs.tag-prefix }}${{ matrix.tag }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb9a04ada..1a60b2220 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,9 @@ on: schedule: - cron: '0 0 * * *' +env: + archs: '["amd64","arm64"]' + jobs: complete: @@ -26,6 +29,7 @@ jobs: tag-prefix: ${{ steps.tag-prefix.outputs.tag-prefix }} tag-alias-prefix: ${{ steps.tag-prefix.outputs.tag-alias-prefix }} images: ${{ steps.images.outputs.images }} + tags: ${{ steps.tags.outputs.tags }} steps: - uses: actions/checkout@v2 with: @@ -45,7 +49,14 @@ jobs: images="$(> $GITHUB_ENV echo "images=$images" >> $GITHUB_OUTPUT + - name: Tags + id: tags + run: | + tags="$(<<< $images jq -c '[.[].tag]')" + <<< $tags jq + echo "tags=$tags" >> $GITHUB_OUTPUT build: name: 2 build @@ -54,8 +65,16 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha || github.sha }} images: ${{ needs.setup.outputs.images }} - archs: '["amd64","arm64"]' - push: ${{ (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) }} + archs: ${{ env.archs }} + + push: + name: 3 push + needs: [setup, build] + uses: ./.github/workflows/push.yml + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + tags: ${{ needs.setup.outputs.tags }} + archs: ${{ env.archs }} tag-prefix: ${{ needs.setup.outputs.tag-prefix }} tag-alias-prefix: ${{ needs.setup.outputs.tag-alias-prefix }} secrets: @@ -65,8 +84,8 @@ jobs: registry_password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} use: - name: 3 use - needs: build + name: 4 use + needs: push runs-on: ubuntu-latest steps: - uses: actions/download-artifact@v4 diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml new file mode 100644 index 000000000..1347cd13a --- /dev/null +++ b/.github/workflows/push.yml @@ -0,0 +1,160 @@ +name: Push + +# This workflow builds multiple quickstart images as defined in the images.json +# file. +# +# The dependencies (xdr, core, rpc, horizon, friendbot, lab) are first +# deduplicated across all images, and then built. Dependencies are cached and +# so only rebuilt when needed. Dependencies are defined by a tag or branch, but +# when building those git refs are resolved to a sha to ensure stability of the +# sha throughout the full build process. For all dependencies and the final +# image, amd64 and arm64 variants are built and the final image is a +# multiplatform image. +# +# The images defined in the images.json file can specify what events the images +# are built on. Most of the images will be built on push and pull requests, but +# this workflow also runs on a schedule an so images that need updating on a +# schedule, such as a nightly-like image, can specify running additionally or +# only on the schedule. +# +# This workflow is also triggerable via a workflow call from another workflow. +# When used that way, the image only builds a single amd64 image and it is not +# pushed. The workflow returns an output which is an artifact name that can be +# downloaded and loaded into docker for use in another job. The workflow can be +# improved to support multiple images in the workflow call case, it just +# requires more work to do so. + +on: + workflow_call: + inputs: + repo: + description: "Quickstart repo where quickstart is hosted" + type: "string" + default: "stellar/quickstart" + ref: + description: "Quickstart ref to use for actions should match workflow (sha, branch, tag)" + type: "string" + default: "main" + tags: + description: 'Tags to push (e.g. ["latest", "testing", ...])' + type: "string" + required: true + archs: + description: 'Architectures to push for as an array (e.g. ["amd64", "arm64"])' + type: "string" + required: true + tag-prefix: + description: "Tag prefix for the image when pushed (e.g. pr877-v21-)" + type: "string" + default: '' + tag-alias-prefix: + description: "A second tag prefix for the image when pushed that'll be pushed in addition as an alias (e.g. pr877-)" + type: "string" + default: '' + secrets: + registry: + description: "Registry to push to" + registry_repo: + description: "Repo at the registry to push to" + registry_username: + description: "Username to auth with the registry" + registry_password: + description: "Password to auth with the registry" + +jobs: + + complete: + if: always() + name: complete + needs: [push, push-manifest, action] + runs-on: ubuntu-latest + steps: + - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') + run: exit 1 + + push: + strategy: + matrix: + tag: ${{ fromJSON(inputs.tags) }} + arch: ${{ fromJSON(inputs.archs) }} + fail-fast: false + name: 7 push (${{ matrix.tag }}, ${{ matrix.arch }}) + permissions: + packages: read + statuses: none + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + repository: ${{ inputs.repo }} + ref: ${{ inputs.ref }} + - name: Create Tag + id: tag + run: echo "tag=${{ inputs.tag-prefix }}${{ matrix.tag }}-${{ matrix.arch }}" | tee -a $GITHUB_OUTPUT + - uses: ./.github/actions/push + with: + head_sha: ${{ inputs.ref }} + artifact_name: image-quickstart-${{ matrix.tag }}-${{ matrix.arch }}.tar + artifact_image_file: image + image: quickstart + arch: ${{ matrix.arch }} + name: ${{ secrets.registry }}/${{ secrets.registry_repo }}:${{ steps.tag.outputs.tag }} + registry: ${{ secrets.registry }} + username: ${{ secrets.registry_username }} + password: ${{ secrets.registry_password }} + + push-manifest: + needs: [setup, push] + strategy: + matrix: + tag: ${{ fromJSON(inputs.tags) }} + fail-fast: false + name: 8 push manifest (${{ matrix.tag }}) + permissions: + packages: read + statuses: none + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + repository: ${{ inputs.repo }} + ref: ${{ inputs.ref }} + - name: Create Tag + id: tag + run: | + echo "tag=${{ inputs.tag-prefix }}${{ matrix.tag }}" | tee -a $GITHUB_OUTPUT + echo "tag-alias=${{ inputs.tag-alias-prefix }}${{ matrix.tag }}" | tee -a $GITHUB_OUTPUT + - name: Prepare Image List + id: images + env: + archs: ${{ inputs.archs }} + run: | + images="$(<<< $archs jq 'map("${{ secrets.registry }}/${{ secrets.registry_repo }}:${{ steps.tag.outputs.tag }}-\(.)")')" + echo "images=$images" | tee -a $GITHUB_OUTPUT + - uses: ./.github/actions/push-manifest + with: + head_sha: ${{ inputs.ref }} + image: ${{ secrets.registry }}/${{ secrets.registry_repo }}:${{ steps.tag.outputs.tag }} + images: ${{ steps.images.outputs.images }} + registry: ${{ secrets.registry }} + username: ${{ secrets.registry_username }} + password: ${{ secrets.registry_password }} + - uses: ./.github/actions/push-alias + with: + head_sha: ${{ inputs.ref }} + image: ${{ secrets.registry }}/${{ secrets.registry_repo }}:${{ steps.tag.outputs.tag }} + alias: ${{ secrets.registry }}/${{ secrets.registry_repo }}:${{ steps.tag.outputs.tag-alias }} + registry: ${{ secrets.registry }} + username: ${{ secrets.registry_username }} + password: ${{ secrets.registry_password }} + + action: + needs: [setup, push-manifest] + strategy: + matrix: + tag: ${{ fromJSON(inputs.tags) }} + fail-fast: false + name: 9 test action (${{ matrix.tag }}) + uses: ./.github/workflows/action-test.yml + with: + tag: ${{ inputs.tag-prefix }}${{ matrix.tag }} From 35b98a14d3318ade72de16e85bef939236ccb271 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 12:02:04 +1000 Subject: [PATCH 094/135] remove redundant setup dependency from push-manifest job --- .github/workflows/push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 1347cd13a..e96415b32 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -104,7 +104,7 @@ jobs: password: ${{ secrets.registry_password }} push-manifest: - needs: [setup, push] + needs: push strategy: matrix: tag: ${{ fromJSON(inputs.tags) }} From 0f1a36bf3aa5ff0dc93ba9ed781d0118a6da70f8 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 12:02:27 +1000 Subject: [PATCH 095/135] remove setup dependency from action job --- .github/workflows/push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index e96415b32..24a97e8df 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -149,7 +149,7 @@ jobs: password: ${{ secrets.registry_password }} action: - needs: [setup, push-manifest] + needs: push-manifest strategy: matrix: tag: ${{ fromJSON(inputs.tags) }} From 6c40c652d80cec7771144c481e94f5541b1d6331 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 12:03:19 +1000 Subject: [PATCH 096/135] remove archs environment variable and inline values in ci workflow --- .github/workflows/ci.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a60b2220..d4ea1bf54 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,9 +8,6 @@ on: schedule: - cron: '0 0 * * *' -env: - archs: '["amd64","arm64"]' - jobs: complete: @@ -65,7 +62,7 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha || github.sha }} images: ${{ needs.setup.outputs.images }} - archs: ${{ env.archs }} + archs: '["amd64","arm64"]' push: name: 3 push @@ -74,7 +71,7 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha || github.sha }} tags: ${{ needs.setup.outputs.tags }} - archs: ${{ env.archs }} + archs: '["amd64","arm64"]' tag-prefix: ${{ needs.setup.outputs.tag-prefix }} tag-alias-prefix: ${{ needs.setup.outputs.tag-alias-prefix }} secrets: From a73625f9b650fc33b3a182a3915e3cacb1ac7f9b Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 12:04:48 +1000 Subject: [PATCH 097/135] add archs to ci workflow --- .github/workflows/ci.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d4ea1bf54..a8e6c21a6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,6 +27,7 @@ jobs: tag-alias-prefix: ${{ steps.tag-prefix.outputs.tag-alias-prefix }} images: ${{ steps.images.outputs.images }} tags: ${{ steps.tags.outputs.tags }} + archs: ${{ steps.archs.outputs.archs }} steps: - uses: actions/checkout@v2 with: @@ -54,6 +55,12 @@ jobs: tags="$(<<< $images jq -c '[.[].tag]')" <<< $tags jq echo "tags=$tags" >> $GITHUB_OUTPUT + - name: Architectures + id: archs + run: | + archs=["amd64","arm64"] + <<< $archs jq + echo "archs=$archs" >> $GITHUB_OUTPUT build: name: 2 build @@ -62,7 +69,7 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha || github.sha }} images: ${{ needs.setup.outputs.images }} - archs: '["amd64","arm64"]' + archs: ${{ needs.setup.outputs.archs }} push: name: 3 push @@ -71,7 +78,7 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha || github.sha }} tags: ${{ needs.setup.outputs.tags }} - archs: '["amd64","arm64"]' + archs: ${{ needs.setup.outputs.archs }} tag-prefix: ${{ needs.setup.outputs.tag-prefix }} tag-alias-prefix: ${{ needs.setup.outputs.tag-alias-prefix }} secrets: From f920de2c36cde752885dea2e945d42ba401c6c1f Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 12:05:42 +1000 Subject: [PATCH 098/135] update ci workflow to properly format archs array --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a8e6c21a6..0a744e17b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,7 +58,7 @@ jobs: - name: Architectures id: archs run: | - archs=["amd64","arm64"] + archs='["amd64","arm64"]' <<< $archs jq echo "archs=$archs" >> $GITHUB_OUTPUT From 024ba4dc60da530e10f19313f075c320795426a2 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 12:07:06 +1000 Subject: [PATCH 099/135] remove concurrency from build workflow and add it to ci workflow --- .github/workflows/build.yml | 8 -------- .github/workflows/ci.yml | 8 ++++++++ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 82d73f8ca..e45794fd9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -52,14 +52,6 @@ on: type: "string" default: 6 -# Prevent more than one build of this workflow for a branch to be running at the -# same time, and if multiple are queued, only run the latest, cancelling any -# already running build. The exception being any protected branch, such as -# main, where a build for every commit will run. -concurrency: - group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }} - cancel-in-progress: true - env: artifact_retention_days_for_image: 7 artifact_retention_days_for_logs: 60 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0a744e17b..67a00b88b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,14 @@ on: schedule: - cron: '0 0 * * *' +# Prevent more than one build of this workflow for a branch to be running at the +# same time, and if multiple are queued, only run the latest, cancelling any +# already running build. The exception being any protected branch, such as +# main, where a build for every commit will run. +concurrency: + group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }} + cancel-in-progress: true + jobs: complete: From 671a7298b94616cc7ba6e1fcf1070bcbc3cf8a46 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 12:10:23 +1000 Subject: [PATCH 100/135] update quickstart image to latest-amd64 tag in ci workflow --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 67a00b88b..883d99969 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -112,7 +112,7 @@ jobs: -p "8000:8000" --name stellar - quickstart + quickstart:latest-amd64 --local - run: sleep 10 - run: curl http://localhost:8000 From da3e2b0815bedbff49e5e5bdcf3d7e5db781225e Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 12:10:56 +1000 Subject: [PATCH 101/135] update ci workflow to include push in needs --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 883d99969..2d099a919 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: complete: if: always() name: complete - needs: [build] + needs: [build, push] runs-on: ubuntu-latest steps: - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') From 71e1ef784c3a3815b055e49d04532cc3ab9adb60 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 12:11:06 +1000 Subject: [PATCH 102/135] remove use job from ci workflow --- .github/workflows/ci.yml | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d099a919..959dcbfba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,25 +94,3 @@ jobs: registry_repo: ${{ github.repository }} registry_username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} registry_password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} - - use: - name: 4 use - needs: push - runs-on: ubuntu-latest - steps: - - uses: actions/download-artifact@v4 - with: - name: image-quickstart-latest-amd64.tar - path: /tmp/ - - run: docker load -i /tmp/image - - run: > - docker run - --platform linux/amd64 - -d - -p - "8000:8000" - --name stellar - quickstart:latest-amd64 - --local - - run: sleep 10 - - run: curl http://localhost:8000 From e1a30028e91d04e5748e1111217dbfbb63c9485a Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 12:16:32 +1000 Subject: [PATCH 103/135] move action workflow to ci and remove from push --- .github/workflows/ci.yml | 11 +++++++++++ .github/workflows/push.yml | 11 ----------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 959dcbfba..b9d6d2476 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,3 +94,14 @@ jobs: registry_repo: ${{ github.repository }} registry_username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} registry_password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} + + action: + needs: push + strategy: + matrix: + tag: ${{ fromJSON(needs.setup.outputs.tags) }} + fail-fast: false + name: 4 test action (${{ matrix.tag }}) + uses: ./.github/workflows/action-test.yml + with: + tag: ${{ inputs.tag-prefix }}${{ matrix.tag }} diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 24a97e8df..062afee25 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -147,14 +147,3 @@ jobs: registry: ${{ secrets.registry }} username: ${{ secrets.registry_username }} password: ${{ secrets.registry_password }} - - action: - needs: push-manifest - strategy: - matrix: - tag: ${{ fromJSON(inputs.tags) }} - fail-fast: false - name: 9 test action (${{ matrix.tag }}) - uses: ./.github/workflows/action-test.yml - with: - tag: ${{ inputs.tag-prefix }}${{ matrix.tag }} From a0189cffde79533ae05d94ea4c34011e0abb3e77 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 12:16:41 +1000 Subject: [PATCH 104/135] add action dependency to ci workflow --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b9d6d2476..b7edcddb5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: complete: if: always() name: complete - needs: [build, push] + needs: [build, push, action] runs-on: ubuntu-latest steps: - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') From 3981a33fb77e9e8a173ac568a43782cb775cc2bf Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 12:33:04 +1000 Subject: [PATCH 105/135] remove old push and push-manifest actions and update workflow to use new approach --- .github/actions/push-alias/action.yml | 42 -------- .github/actions/push/action.yml | 61 ------------ .github/workflows/push.yml | 132 +++++++++++++++++--------- 3 files changed, 87 insertions(+), 148 deletions(-) delete mode 100644 .github/actions/push-alias/action.yml delete mode 100644 .github/actions/push/action.yml diff --git a/.github/actions/push-alias/action.yml b/.github/actions/push-alias/action.yml deleted file mode 100644 index 980938a68..000000000 --- a/.github/actions/push-alias/action.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: 'Push Alias' -inputs: - head_sha: - required: true - image: - required: true - alias: - required: true - registry: - required: true - default: ghcr.io - username: - required: true - default: ${{ github.actor }} - password: - required: true - default: ${{ github.token }} -runs: - using: "composite" - steps: - - - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 - with: - registry: ${{ inputs.registry }} - username: ${{ inputs.username }} - password: ${{ inputs.password }} - - - shell: bash - run: | - docker buildx imagetools create -t ${{ inputs.alias }} ${{ inputs.image }} - - - uses: actions/github-script@v5 - with: - script: | - github.rest.repos.createCommitStatus({ - owner: context.repo.owner, - repo: context.repo.repo, - sha: '${{ inputs.head_sha }}', - state: 'success', - context: `${{ inputs.alias }}`, - description: 'Available', - }); diff --git a/.github/actions/push/action.yml b/.github/actions/push/action.yml deleted file mode 100644 index 7fd496e23..000000000 --- a/.github/actions/push/action.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: 'Push' -inputs: - head_sha: - required: true - artifact_name: - required: true - artifact_image_file: - required: true - image: - required: true - default: quickstart - arch: - required: true - name: - required: true - default: ghcr.io/${{ github.repository }}:latest - registry: - required: true - default: ghcr.io - username: - required: true - default: ${{ github.actor }} - password: - required: true - default: ${{ github.token }} -runs: - using: "composite" - steps: - - - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.artifact_name }} - path: /tmp/ - - - shell: bash - run: docker load -i /tmp/${{ inputs.artifact_image_file }} - - - shell: bash - run: docker tag ${{ inputs.image }} ${{ inputs.name }} - - - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 - with: - registry: ${{ inputs.registry }} - username: ${{ inputs.username }} - password: ${{ inputs.password }} - - - shell: bash - run: | - docker push ${{ inputs.name }} - - - uses: actions/github-script@v5 - with: - script: | - github.rest.repos.createCommitStatus({ - owner: context.repo.owner, - repo: context.repo.repo, - sha: '${{ inputs.head_sha }}', - state: 'success', - context: `${{ inputs.name }}`, - description: 'Available', - }); diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 062afee25..d726041e4 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -27,14 +27,6 @@ name: Push on: workflow_call: inputs: - repo: - description: "Quickstart repo where quickstart is hosted" - type: "string" - default: "stellar/quickstart" - ref: - description: "Quickstart ref to use for actions should match workflow (sha, branch, tag)" - type: "string" - default: "main" tags: description: 'Tags to push (e.g. ["latest", "testing", ...])' type: "string" @@ -51,11 +43,11 @@ on: description: "A second tag prefix for the image when pushed that'll be pushed in addition as an alias (e.g. pr877-)" type: "string" default: '' - secrets: registry: description: "Registry to push to" registry_repo: description: "Repo at the registry to push to" + secrets: registry_username: description: "Username to auth with the registry" registry_password: @@ -66,7 +58,7 @@ jobs: complete: if: always() name: complete - needs: [push, push-manifest, action] + needs: [push, push-manifest, push-alias] runs-on: ubuntu-latest steps: - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') @@ -78,30 +70,47 @@ jobs: tag: ${{ fromJSON(inputs.tags) }} arch: ${{ fromJSON(inputs.archs) }} fail-fast: false - name: 7 push (${{ matrix.tag }}, ${{ matrix.arch }}) + name: 1 push (${{ matrix.tag }}, ${{ matrix.arch }}) permissions: packages: read statuses: none runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - repository: ${{ inputs.repo }} - ref: ${{ inputs.ref }} - name: Create Tag id: tag run: echo "tag=${{ inputs.tag-prefix }}${{ matrix.tag }}-${{ matrix.arch }}" | tee -a $GITHUB_OUTPUT - - uses: ./.github/actions/push + - name: Download Image from Artifacts + uses: actions/download-artifact@v4 + with: + name: image-quickstart-${{ matrix.tag }}-${{ matrix.arch }}.tar + path: /tmp/ + - name: Load Image + run: docker load -i /tmp/image + - name: Tag Image + run: > + docker tag + quickstart:${{ matrix.tag }}-${{ matrix.arch }} + ${{ inputs.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }} + - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: - head_sha: ${{ inputs.ref }} - artifact_name: image-quickstart-${{ matrix.tag }}-${{ matrix.arch }}.tar - artifact_image_file: image - image: quickstart - arch: ${{ matrix.arch }} - name: ${{ secrets.registry }}/${{ secrets.registry_repo }}:${{ steps.tag.outputs.tag }} - registry: ${{ secrets.registry }} - username: ${{ secrets.registry_username }} - password: ${{ secrets.registry_password }} + registry: ${{ inputs.registry }} + username: ${{ secrets.username }} + password: ${{ secrets.password }} + - name: Push Image + run: | + docker push ${{ inputs.name }} + - name: Post Status with Image Name + uses: actions/github-script@v5 + with: + script: | + github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: '${{ inputs.head_sha }}', + state: 'success', + context: `${{ inputs.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }}`, + description: 'Available', + }); push-manifest: needs: push @@ -109,21 +118,16 @@ jobs: matrix: tag: ${{ fromJSON(inputs.tags) }} fail-fast: false - name: 8 push manifest (${{ matrix.tag }}) + name: 2 push manifest (${{ matrix.tag }}) permissions: packages: read statuses: none runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - repository: ${{ inputs.repo }} - ref: ${{ inputs.ref }} - name: Create Tag id: tag run: | echo "tag=${{ inputs.tag-prefix }}${{ matrix.tag }}" | tee -a $GITHUB_OUTPUT - echo "tag-alias=${{ inputs.tag-alias-prefix }}${{ matrix.tag }}" | tee -a $GITHUB_OUTPUT - name: Prepare Image List id: images env: @@ -131,19 +135,57 @@ jobs: run: | images="$(<<< $archs jq 'map("${{ secrets.registry }}/${{ secrets.registry_repo }}:${{ steps.tag.outputs.tag }}-\(.)")')" echo "images=$images" | tee -a $GITHUB_OUTPUT - - uses: ./.github/actions/push-manifest + - name: Create Manifest + run: > + docker manifest create + ${{ inputs.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }} + ${{ steps.images.outputs.images }} + - name: Push Manifest + run: > + docker manifest push + ${{ inputs.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }} + - uses: actions/github-script@v5 with: - head_sha: ${{ inputs.ref }} - image: ${{ secrets.registry }}/${{ secrets.registry_repo }}:${{ steps.tag.outputs.tag }} - images: ${{ steps.images.outputs.images }} - registry: ${{ secrets.registry }} - username: ${{ secrets.registry_username }} - password: ${{ secrets.registry_password }} - - uses: ./.github/actions/push-alias + script: | + github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: '${{ github.event.pull_request.head.sha || github.sha }}', + state: 'success', + context: '${{ inputs.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }}', + description: 'Available', + }); + + push-alias: + needs: push-manifest + strategy: + matrix: + tag: ${{ fromJSON(inputs.tags) }} + fail-fast: false + name: 3 push alias (${{ matrix.tag }}) + permissions: + packages: read + statuses: none + runs-on: ubuntu-latest + steps: + - name: Create Tag + id: tag + run: | + echo "tag=${{ inputs.tag-prefix }}${{ matrix.tag }}" | tee -a $GITHUB_OUTPUT + echo "tag-alias=${{ inputs.tag-alias-prefix }}${{ matrix.tag }}" | tee -a $GITHUB_OUTPUT + - name: Push Alias + run: > + docker buildx imagetools create -t + ${{ inputs.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag-alias }} + ${{ inputs.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }} + - uses: actions/github-script@v5 with: - head_sha: ${{ inputs.ref }} - image: ${{ secrets.registry }}/${{ secrets.registry_repo }}:${{ steps.tag.outputs.tag }} - alias: ${{ secrets.registry }}/${{ secrets.registry_repo }}:${{ steps.tag.outputs.tag-alias }} - registry: ${{ secrets.registry }} - username: ${{ secrets.registry_username }} - password: ${{ secrets.registry_password }} + script: | + github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: '${{ github.event.pull_request.head.sha || github.sha }}', + state: 'success', + context: '${{ inputs.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag-alias }}', + description: 'Available', + }); From 7c25ce7107814df7addde317ca474b642071cd6d Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 12:33:42 +1000 Subject: [PATCH 106/135] update workflow inputs to specify string type --- .github/workflows/push.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index d726041e4..75bd2aa92 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -45,8 +45,10 @@ on: default: '' registry: description: "Registry to push to" + type: "string" registry_repo: description: "Repo at the registry to push to" + type: "string" secrets: registry_username: description: "Username to auth with the registry" From 7378593c2c3f22a02efe3891592c1e6858d0c9fe Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 12:34:34 +1000 Subject: [PATCH 107/135] remove ref override from ci workflow --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b7edcddb5..f9562d6a4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,7 +84,6 @@ jobs: needs: [setup, build] uses: ./.github/workflows/push.yml with: - ref: ${{ github.event.pull_request.head.sha || github.sha }} tags: ${{ needs.setup.outputs.tags }} archs: ${{ needs.setup.outputs.archs }} tag-prefix: ${{ needs.setup.outputs.tag-prefix }} From 34085077188d0717ac633b5ceec7d5d630234ab0 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 12:34:41 +1000 Subject: [PATCH 108/135] update ci workflow to use dockerhub credentials from secrets --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f9562d6a4..b3767b5ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,9 +88,9 @@ jobs: archs: ${{ needs.setup.outputs.archs }} tag-prefix: ${{ needs.setup.outputs.tag-prefix }} tag-alias-prefix: ${{ needs.setup.outputs.tag-alias-prefix }} - secrets: registry: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} registry_repo: ${{ github.repository }} + secrets: registry_username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} registry_password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} From 84976bb2b02b5d9bfb43bd04a323ce59c4836278 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 12:38:37 +1000 Subject: [PATCH 109/135] move registry configuration to secrets --- .github/workflows/ci.yml | 2 +- .github/workflows/push.yml | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b3767b5ba..507731866 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,9 +88,9 @@ jobs: archs: ${{ needs.setup.outputs.archs }} tag-prefix: ${{ needs.setup.outputs.tag-prefix }} tag-alias-prefix: ${{ needs.setup.outputs.tag-alias-prefix }} - registry: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} registry_repo: ${{ github.repository }} secrets: + registry: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} registry_username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} registry_password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 75bd2aa92..0f46eb1f0 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -43,13 +43,13 @@ on: description: "A second tag prefix for the image when pushed that'll be pushed in addition as an alias (e.g. pr877-)" type: "string" default: '' - registry: - description: "Registry to push to" - type: "string" registry_repo: description: "Repo at the registry to push to" type: "string" secrets: + registry: + description: "Registry to push to" + type: "string" registry_username: description: "Username to auth with the registry" registry_password: @@ -92,10 +92,10 @@ jobs: run: > docker tag quickstart:${{ matrix.tag }}-${{ matrix.arch }} - ${{ inputs.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }} + ${{ secrets.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }} - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: - registry: ${{ inputs.registry }} + registry: ${{ secrets.registry }} username: ${{ secrets.username }} password: ${{ secrets.password }} - name: Push Image @@ -110,7 +110,7 @@ jobs: repo: context.repo.repo, sha: '${{ inputs.head_sha }}', state: 'success', - context: `${{ inputs.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }}`, + context: `${{ secrets.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }}`, description: 'Available', }); @@ -140,12 +140,12 @@ jobs: - name: Create Manifest run: > docker manifest create - ${{ inputs.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }} + ${{ secrets.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }} ${{ steps.images.outputs.images }} - name: Push Manifest run: > docker manifest push - ${{ inputs.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }} + ${{ secrets.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }} - uses: actions/github-script@v5 with: script: | @@ -154,7 +154,7 @@ jobs: repo: context.repo.repo, sha: '${{ github.event.pull_request.head.sha || github.sha }}', state: 'success', - context: '${{ inputs.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }}', + context: '${{ secrets.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }}', description: 'Available', }); @@ -178,8 +178,8 @@ jobs: - name: Push Alias run: > docker buildx imagetools create -t - ${{ inputs.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag-alias }} - ${{ inputs.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }} + ${{ secrets.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag-alias }} + ${{ secrets.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }} - uses: actions/github-script@v5 with: script: | @@ -188,6 +188,6 @@ jobs: repo: context.repo.repo, sha: '${{ github.event.pull_request.head.sha || github.sha }}', state: 'success', - context: '${{ inputs.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag-alias }}', + context: '${{ secrets.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag-alias }}', description: 'Available', }); From 64346eaa68fe9011896766bc140702932dd7808a Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 12:38:44 +1000 Subject: [PATCH 110/135] remove unused registry type field --- .github/workflows/push.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 0f46eb1f0..e92e156e9 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -49,7 +49,6 @@ on: secrets: registry: description: "Registry to push to" - type: "string" registry_username: description: "Username to auth with the registry" registry_password: From 67a441b46a63ce65c0ada3180ae1f1ed921348a5 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 12:39:56 +1000 Subject: [PATCH 111/135] update ci workflow to depend on setup and push --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 507731866..c3046ba01 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,7 +95,7 @@ jobs: registry_password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} action: - needs: push + needs: [setup, push] strategy: matrix: tag: ${{ fromJSON(needs.setup.outputs.tags) }} From 76588a6c00945ad9ebefcf7aadf8b67effd69c6b Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 12:58:13 +1000 Subject: [PATCH 112/135] update action to support artifact and arch inputs --- action.yml | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/action.yml b/action.yml index c3e7bdbcd..6e493b803 100644 --- a/action.yml +++ b/action.yml @@ -5,10 +5,16 @@ inputs: description: "Image tag of quickstart image to use" required: true default: "latest" - image: - description: "Image for the quickstart image to use" + arch: + description: "Image architecture to use (default uses a multiplatform image)" required: true - default: "docker.io/stellar/quickstart" + default: "latest" + artifact: + description: "Artifact to collect image from (mutually exclusive with image)" + default: "" + image: + description: "Image for the quickstart image to use (mutually exclusive with artifact)" + default: "" enable: description: "Services to enable" default: "core,horizon,rpc" @@ -35,6 +41,17 @@ runs: steps: - name: Set up Docker uses: docker/setup-docker-action@b60f85385d03ac8acfca6d9996982511d8620a19 # v4 + + - if: inputs.artifact + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.artifact }} + path: /tmp/ + + - if: inputs.artifact + run: docker load -i /tmp/image + shell: bash + - run: > docker run -d --name stellar -p 8000:8000 @@ -53,8 +70,9 @@ runs: --health-interval ${{ inputs.health_interval }}s --health-timeout ${{ inputs.health_timeout }}s --health-retries ${{ inputs.health_retries }} - ${{ inputs.image }}:${{ inputs.tag }} + ${{ inputs.image || (inputs.artifact && 'quickstart' || 'docker.io/stellar/quickstart') }}:${{ inputs.tag }} shell: bash + - name: "Wait for container to be healthy" run: | i=0 From 65011f6e4e83e0ed23274e60f8b4b74ec9ca656f Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 13:05:37 +1000 Subject: [PATCH 113/135] update ci workflow to use artifact and registry actions --- .github/workflows/ci.yml | 18 +++++++++++++++--- action.yml | 11 ++++------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3046ba01..34ae89cad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,6 +79,18 @@ jobs: images: ${{ needs.setup.outputs.images }} archs: ${{ needs.setup.outputs.archs }} + action-using-artifact: + needs: [setup, build] + strategy: + matrix: + tag: ${{ fromJSON(needs.setup.outputs.tags) }} + fail-fast: false + name: 4 test action artifact (${{ matrix.tag }}) + uses: ./.github/workflows/action-test.yml + with: + artifact: image-quickstart-${{ matrix.tag }}-amd64 + tag: ${{ needs.setup.outputs.tag-prefix }}${{ matrix.tag }}-amd64 + push: name: 3 push needs: [setup, build] @@ -94,13 +106,13 @@ jobs: registry_username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} registry_password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} - action: + action-using-registry: needs: [setup, push] strategy: matrix: tag: ${{ fromJSON(needs.setup.outputs.tags) }} fail-fast: false - name: 4 test action (${{ matrix.tag }}) + name: 4 test action registry (${{ matrix.tag }}) uses: ./.github/workflows/action-test.yml with: - tag: ${{ inputs.tag-prefix }}${{ matrix.tag }} + tag: ${{ needs.setup.outputs.tag-prefix }}${{ matrix.tag }} diff --git a/action.yml b/action.yml index 6e493b803..73802618c 100644 --- a/action.yml +++ b/action.yml @@ -5,15 +5,12 @@ inputs: description: "Image tag of quickstart image to use" required: true default: "latest" - arch: - description: "Image architecture to use (default uses a multiplatform image)" - required: true - default: "latest" artifact: - description: "Artifact to collect image from (mutually exclusive with image)" - default: "" + description: "Artifact to collect image from" + type: boolean + default: false image: - description: "Image for the quickstart image to use (mutually exclusive with artifact)" + description: "Image for the quickstart image to use" default: "" enable: description: "Services to enable" From ec360ce88c3d62d2b942ccfa4f0075e06e80cdaf Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 13:07:35 +1000 Subject: [PATCH 114/135] update ci workflow to include action-using-artifact and action-using-registry dependencies --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 34ae89cad..7e66923e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: complete: if: always() name: complete - needs: [build, push, action] + needs: [build, action-using-artifact, push, action-using-registry] runs-on: ubuntu-latest steps: - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') From dc764dc8907edacd00550ff78ce9c10397bbcf89 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 13:08:52 +1000 Subject: [PATCH 115/135] add artifact input to action test workflow --- .github/workflows/action-test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/action-test.yml b/.github/workflows/action-test.yml index 36f534ffc..f6f2eb7eb 100644 --- a/.github/workflows/action-test.yml +++ b/.github/workflows/action-test.yml @@ -7,6 +7,10 @@ on: description: 'Tag to use' type: 'string' default: '' + artifact: + description: 'Artifact to use' + type: 'string' + default: '' jobs: action-setup: @@ -50,6 +54,7 @@ jobs: - uses: actions/checkout@v4 - uses: ./ with: + artifact: ${{ inputs.artifact }} tag: ${{ inputs.tag }} - name: "Run basic test making sure RPC and Horizon are available" run: > From e62f82a1f68f9ff3132c53dad8c8dc24f8182492 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 13:11:26 +1000 Subject: [PATCH 116/135] update workflow to conditionally run additional builds --- .github/workflows/action-test.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/action-test.yml b/.github/workflows/action-test.yml index f6f2eb7eb..87741837b 100644 --- a/.github/workflows/action-test.yml +++ b/.github/workflows/action-test.yml @@ -36,7 +36,12 @@ jobs: with: script: | const os = ['ubuntu-latest']; - if (process.env.ACTION_CHANGED === 'true') { + # Only run the additional builds if the action has changed, and this + # is not a test with an artifact image. The additional runs are time + # consuming so only do them if necessary. Don't run them for artifact + # action tests because the test won't be multiplatform since the + # artifact images contain only a single platform. + if (process.env.ACTION_CHANGED === 'true' && '${{ inputs.artifact }}' === '') { os.push('ubuntu-24.04-arm'); os.push('macos-13'); } From 71733ff05117a9f84d1d029558d81c2a99f4fbc3 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 13:20:41 +1000 Subject: [PATCH 117/135] change comment syntax from hash to slash --- .github/workflows/action-test.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/action-test.yml b/.github/workflows/action-test.yml index 87741837b..992fb1860 100644 --- a/.github/workflows/action-test.yml +++ b/.github/workflows/action-test.yml @@ -36,11 +36,11 @@ jobs: with: script: | const os = ['ubuntu-latest']; - # Only run the additional builds if the action has changed, and this - # is not a test with an artifact image. The additional runs are time - # consuming so only do them if necessary. Don't run them for artifact - # action tests because the test won't be multiplatform since the - # artifact images contain only a single platform. + // Only run the additional builds if the action has changed, and this + // is not a test with an artifact image. The additional runs are time + // consuming so only do them if necessary. Don't run them for artifact + // action tests because the test won't be multiplatform since the + // artifact images contain only a single platform. if (process.env.ACTION_CHANGED === 'true' && '${{ inputs.artifact }}' === '') { os.push('ubuntu-24.04-arm'); os.push('macos-13'); From a48883d17426fdb2c345a363786a2fda331fc9fe Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 13:23:27 +1000 Subject: [PATCH 118/135] update docker registry credentials to use registry_username and registry_password --- .github/workflows/push.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index e92e156e9..b8e4d394c 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -95,8 +95,8 @@ jobs: - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: registry: ${{ secrets.registry }} - username: ${{ secrets.username }} - password: ${{ secrets.password }} + username: ${{ secrets.registry_username }} + password: ${{ secrets.registry_password }} - name: Push Image run: | docker push ${{ inputs.name }} @@ -129,6 +129,11 @@ jobs: id: tag run: | echo "tag=${{ inputs.tag-prefix }}${{ matrix.tag }}" | tee -a $GITHUB_OUTPUT + - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + registry: ${{ secrets.registry }} + username: ${{ secrets.registry_username }} + password: ${{ secrets.registry_password }} - name: Prepare Image List id: images env: @@ -174,6 +179,11 @@ jobs: run: | echo "tag=${{ inputs.tag-prefix }}${{ matrix.tag }}" | tee -a $GITHUB_OUTPUT echo "tag-alias=${{ inputs.tag-alias-prefix }}${{ matrix.tag }}" | tee -a $GITHUB_OUTPUT + - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + registry: ${{ secrets.registry }} + username: ${{ secrets.registry_username }} + password: ${{ secrets.registry_password }} - name: Push Alias run: > docker buildx imagetools create -t From ccdf1029a7862cf8736161b6878501af9b255a5b Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 13:23:52 +1000 Subject: [PATCH 119/135] update workflow permissions to write access --- .github/workflows/push.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index b8e4d394c..6f74836e5 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -73,8 +73,8 @@ jobs: fail-fast: false name: 1 push (${{ matrix.tag }}, ${{ matrix.arch }}) permissions: - packages: read - statuses: none + packages: write + statuses: write runs-on: ubuntu-latest steps: - name: Create Tag @@ -121,8 +121,8 @@ jobs: fail-fast: false name: 2 push manifest (${{ matrix.tag }}) permissions: - packages: read - statuses: none + packages: write + statuses: write runs-on: ubuntu-latest steps: - name: Create Tag @@ -170,8 +170,8 @@ jobs: fail-fast: false name: 3 push alias (${{ matrix.tag }}) permissions: - packages: read - statuses: none + packages: write + statuses: write runs-on: ubuntu-latest steps: - name: Create Tag From e73842fb80af420a3965f473ca15f8272bc6a51e Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 13:40:26 +1000 Subject: [PATCH 120/135] update github action to use registry secret and enhance readme with custom build instructions --- .github/workflows/push.yml | 5 +++-- README.md | 42 +++++++++++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 6f74836e5..1f35570f7 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -98,8 +98,9 @@ jobs: username: ${{ secrets.registry_username }} password: ${{ secrets.registry_password }} - name: Push Image - run: | - docker push ${{ inputs.name }} + run: > + docker push + ${{ secrets.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }} - name: Post Status with Image Name uses: actions/github-script@v5 with: diff --git a/README.md b/README.md index 6c71cf703..c181f6c92 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Stellar Quickstart is the fastest way to spin up a complete Stellar blockchain d > uses: stellar/quickstart@main > ``` > -> See [Using in GitHub Actions] for more configuration options. +> See [Using in GitHub Actions] for more configuration options and how to build and run a custom configuration of quickstart. [`stellar-cli`]: https://github.com/stellar/stellar-cli @@ -217,6 +217,46 @@ jobs: # - Friendbot: http://localhost:8000/friendbot ``` +#### Custom Builds + +The quickstart image can also be built with custom software, such as custom versions of core, rpc, horizon, and so on. Use a workflow as follows to build a custom quickstart image and then run it using the action. + +```yaml +on: [push, pull_request] + +jobs: + build-custom: + uses: stellar/quickstart/.github/workflows/build.yml@main + with: + images: | + [ + { + "tag": "custom", + "config": { + "protocol_version_default": 23 + }, + "deps": [ + { "name": "xdr", "repo": "stellar/rs-stellar-xdr", "ref": "v23.0.0" }, + { "name": "core", "repo": "stellar/stellar-core", "ref": "v23.0.1", "options": { "configure_flags": "--disable-tests" } }, + { "name": "rpc", "repo": "stellar/stellar-rpc", "ref": "v23.0.1" }, + { "name": "horizon", "repo": "stellar/go", "ref": "horizon-v23.0.0" }, + { "name": "friendbot", "repo": "stellar/go", "ref": "horizon-v23.0.0" }, + { "name": "lab", "repo": "stellar/laboratory", "ref": "main" } + ], + "additional-tests": [] + } + ] + archs: '["amd64"]' + use-custom: + needs: build-custom + runs-on: ubuntu-latest + steps: + - uses: stellar/quickstart@main + with: + artifact: image-quickstart-custom-amd64.tar + tag: custom-amd64 +``` + ### Deploy to Digital Ocean From afdc0c8b327860bce3029caee028607da572fc46 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 13:55:54 +1000 Subject: [PATCH 121/135] add sha input to push workflow and pass it from ci --- .github/workflows/ci.yml | 1 + .github/workflows/push.yml | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7e66923e2..337c42d92 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,6 +96,7 @@ jobs: needs: [setup, build] uses: ./.github/workflows/push.yml with: + sha: ${{ github.event.pull_request.head.sha || github.sha }} tags: ${{ needs.setup.outputs.tags }} archs: ${{ needs.setup.outputs.archs }} tag-prefix: ${{ needs.setup.outputs.tag-prefix }} diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 1f35570f7..15e4c0ba4 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -27,6 +27,10 @@ name: Push on: workflow_call: inputs: + sha: + description: 'Sha to connect push status notifications to' + type: "string" + required: true tags: description: 'Tags to push (e.g. ["latest", "testing", ...])' type: "string" From 8a0196d01e4f7e627201b1446791ca47e9b39a6d Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 13:56:24 +1000 Subject: [PATCH 122/135] use inputs.sha consistently for commit status --- .github/workflows/push.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 15e4c0ba4..fab2cdb2d 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -112,7 +112,7 @@ jobs: github.rest.repos.createCommitStatus({ owner: context.repo.owner, repo: context.repo.repo, - sha: '${{ inputs.head_sha }}', + sha: '${{ inputs.sha }}', state: 'success', context: `${{ secrets.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }}`, description: 'Available', @@ -161,7 +161,7 @@ jobs: github.rest.repos.createCommitStatus({ owner: context.repo.owner, repo: context.repo.repo, - sha: '${{ github.event.pull_request.head.sha || github.sha }}', + sha: '${{ inputs.sha }}', state: 'success', context: '${{ secrets.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }}', description: 'Available', @@ -200,7 +200,7 @@ jobs: github.rest.repos.createCommitStatus({ owner: context.repo.owner, repo: context.repo.repo, - sha: '${{ github.event.pull_request.head.sha || github.sha }}', + sha: '${{ inputs.sha }}', state: 'success', context: '${{ secrets.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag-alias }}', description: 'Available', From 001f896aca34fb56b6ee8da2a45162d5f2684387 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 13:57:03 +1000 Subject: [PATCH 123/135] update artifact name with tar extension --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 337c42d92..7ed0572c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,7 +88,7 @@ jobs: name: 4 test action artifact (${{ matrix.tag }}) uses: ./.github/workflows/action-test.yml with: - artifact: image-quickstart-${{ matrix.tag }}-amd64 + artifact: image-quickstart-${{ matrix.tag }}-amd64.tar tag: ${{ needs.setup.outputs.tag-prefix }}${{ matrix.tag }}-amd64 push: From 5a5b1fbc0eae5e7bad313971ae4f60c2d0a477a5 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 14:10:23 +1000 Subject: [PATCH 124/135] update json processing to join image tags with spaces --- .github/workflows/push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index fab2cdb2d..bc12f08c0 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -144,7 +144,7 @@ jobs: env: archs: ${{ inputs.archs }} run: | - images="$(<<< $archs jq 'map("${{ secrets.registry }}/${{ secrets.registry_repo }}:${{ steps.tag.outputs.tag }}-\(.)")')" + images="$(<<< $archs jq -r 'map("${{ secrets.registry }}/${{ secrets.registry_repo }}:${{ steps.tag.outputs.tag }}-\(.)") | join(" ")')" echo "images=$images" | tee -a $GITHUB_OUTPUT - name: Create Manifest run: > From 138d1b594a6d6cbc5f3946755056a179f4fd96d5 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 14:11:13 +1000 Subject: [PATCH 125/135] remove tag-prefix from ci workflow --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7ed0572c5..7269062d2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,7 +89,7 @@ jobs: uses: ./.github/workflows/action-test.yml with: artifact: image-quickstart-${{ matrix.tag }}-amd64.tar - tag: ${{ needs.setup.outputs.tag-prefix }}${{ matrix.tag }}-amd64 + tag: ${{ matrix.tag }}-amd64 push: name: 3 push From 06f96b52bab6fbfb282bdc845a55391bb847b121 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 14:49:54 +1000 Subject: [PATCH 126/135] update registry repo reference in workflow --- .github/workflows/push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index bc12f08c0..12fbd4dc8 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -144,7 +144,7 @@ jobs: env: archs: ${{ inputs.archs }} run: | - images="$(<<< $archs jq -r 'map("${{ secrets.registry }}/${{ secrets.registry_repo }}:${{ steps.tag.outputs.tag }}-\(.)") | join(" ")')" + images="$(<<< $archs jq -r 'map("${{ secrets.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }}-\(.)") | join(" ")')" echo "images=$images" | tee -a $GITHUB_OUTPUT - name: Create Manifest run: > From 65a29ffc836d7b4c4943181415e80f88024d30cb Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 15:39:25 +1000 Subject: [PATCH 127/135] update workflow documentation and descriptions --- .github/workflows/build.yml | 34 ++++++++++++++-------------------- .github/workflows/ci.yml | 13 +++++++++++++ .github/workflows/push.yml | 27 ++++----------------------- 3 files changed, 31 insertions(+), 43 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e45794fd9..8a2d2e6ea 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,28 +1,22 @@ name: Build -# This workflow builds multiple quickstart images as defined in the images.json -# file. +# This workflow builds multiple quickstart images as defined in the images json +# passed to the workflow across the architectures defined in the archs input. # -# The dependencies (xdr, core, rpc, horizon, friendbot, lab) are first -# deduplicated across all images, and then built. Dependencies are cached and -# so only rebuilt when needed. Dependencies are defined by a tag or branch, but -# when building those git refs are resolved to a sha to ensure stability of the -# sha throughout the full build process. For all dependencies and the final -# image, amd64 and arm64 variants are built and the final image is a -# multiplatform image. +# See the images.json file in the repo for how to define the JSON for a set of +# images. # -# The images defined in the images.json file can specify what events the images -# are built on. Most of the images will be built on push and pull requests, but -# this workflow also runs on a schedule an so images that need updating on a -# schedule, such as a nightly-like image, can specify running additionally or -# only on the schedule. +# This workflow is intended to be called by third parties to build custom +# quickstart images. See the repository README for an example. # -# This workflow is also triggerable via a workflow call from another workflow. -# When used that way, the image only builds a single amd64 image and it is not -# pushed. The workflow returns an output which is an artifact name that can be -# downloaded and loaded into docker for use in another job. The workflow can be -# improved to support multiple images in the workflow call case, it just -# requires more work to do so. +# The build process first builds the dependencies (xdr, core, rpc, horizon, +# friendbot, lab). When doing so the dependencies needed as specified by the +# images json are deduplicated so that any software shared by the images is +# only built once. Dependencies are cached and so only rebuilt when needed. +# Dependencies are defined by a tag or branch, but when building those git refs +# are resolved to a sha to ensure stability of the sha throughout the full +# build process. For all dependencies and the final image, amd64 and arm64 +# variants may be built. on: workflow_call: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7269062d2..fee31d9b7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,18 @@ name: CI +# This workflow builds all the images defined in images.json with the specified +# dependencies, and runs a set of locally defined tests. +# +# Each image defined in images.json specifies the events the image should be +# rebuilt and pushed. Most images will do so for push and pull_request, but can +# also be specified to build on a schedule. +# +# The tests in this repo are designed to make sure that when quickstart starts +# up it is able to either sync with an existing network, or start a new network +# that is progressing. +# +# This workflow also tests that the quickstart action functions. + on: push: branches: diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 12fbd4dc8..d6cf9be3e 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -1,28 +1,9 @@ name: Push -# This workflow builds multiple quickstart images as defined in the images.json -# file. -# -# The dependencies (xdr, core, rpc, horizon, friendbot, lab) are first -# deduplicated across all images, and then built. Dependencies are cached and -# so only rebuilt when needed. Dependencies are defined by a tag or branch, but -# when building those git refs are resolved to a sha to ensure stability of the -# sha throughout the full build process. For all dependencies and the final -# image, amd64 and arm64 variants are built and the final image is a -# multiplatform image. -# -# The images defined in the images.json file can specify what events the images -# are built on. Most of the images will be built on push and pull requests, but -# this workflow also runs on a schedule an so images that need updating on a -# schedule, such as a nightly-like image, can specify running additionally or -# only on the schedule. -# -# This workflow is also triggerable via a workflow call from another workflow. -# When used that way, the image only builds a single amd64 image and it is not -# pushed. The workflow returns an output which is an artifact name that can be -# downloaded and loaded into docker for use in another job. The workflow can be -# improved to support multiple images in the workflow call case, it just -# requires more work to do so. +# This workflow pushes multiple quickstart images that have been built with the +# build.yml. This workflow and the build.yml workflow are coupled, and this +# workflow makes assumptions about what artifact names the images hae been +# saved under. This workflow is not intended to be called by third parties. on: workflow_call: From 00b11e21ca6b9445041b32d0d1cbe57680e202ad Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 20:21:26 +1000 Subject: [PATCH 128/135] remove push manifest action --- .github/actions/push-manifest/action.yml | 50 ------------------------ 1 file changed, 50 deletions(-) delete mode 100644 .github/actions/push-manifest/action.yml diff --git a/.github/actions/push-manifest/action.yml b/.github/actions/push-manifest/action.yml deleted file mode 100644 index 4e66ee325..000000000 --- a/.github/actions/push-manifest/action.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: 'Push Manifest' -inputs: - head_sha: - required: true - image: - required: true - images: - required: true - registry: - required: true - default: ghcr.io - username: - required: true - default: ${{ github.actor }} - password: - required: true - default: ${{ github.token }} -runs: - using: "composite" - steps: - - - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 - with: - registry: ${{ inputs.registry }} - username: ${{ inputs.username }} - password: ${{ inputs.password }} - - - shell: bash - run: > - docker manifest create - ${{ inputs.image }} - ${{ inputs.image }}-amd64 - ${{ inputs.image }}-arm64 - - - shell: bash - run: > - docker manifest push - ${{ inputs.image }} - - - uses: actions/github-script@v5 - with: - script: | - github.rest.repos.createCommitStatus({ - owner: context.repo.owner, - repo: context.repo.repo, - sha: '${{ inputs.head_sha }}', - state: 'success', - context: `${{ inputs.image }}`, - description: 'Available', - }); From 6d3edc74255ba3c0a7db468b93ce1c861cf21ed7 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 20:23:11 +1000 Subject: [PATCH 129/135] remove matrix strategy from action workflows and hardcode latest tag --- .github/workflows/ci.yml | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fee31d9b7..847bbb53b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,15 +94,11 @@ jobs: action-using-artifact: needs: [setup, build] - strategy: - matrix: - tag: ${{ fromJSON(needs.setup.outputs.tags) }} - fail-fast: false - name: 4 test action artifact (${{ matrix.tag }}) + name: 4 test action artifact uses: ./.github/workflows/action-test.yml with: - artifact: image-quickstart-${{ matrix.tag }}-amd64.tar - tag: ${{ matrix.tag }}-amd64 + artifact: image-quickstart-latest-amd64.tar + tag: latest-amd64 push: name: 3 push @@ -122,11 +118,7 @@ jobs: action-using-registry: needs: [setup, push] - strategy: - matrix: - tag: ${{ fromJSON(needs.setup.outputs.tags) }} - fail-fast: false - name: 4 test action registry (${{ matrix.tag }}) + name: 4 test action registry uses: ./.github/workflows/action-test.yml with: - tag: ${{ needs.setup.outputs.tag-prefix }}${{ matrix.tag }} + tag: ${{ needs.setup.outputs.tag-prefix }}latest From 275af80bb564baa2d6fb21597f9dcee94b3b1095 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Wed, 8 Oct 2025 20:26:11 +1000 Subject: [PATCH 130/135] update start script to display friendbot and lab dependencies with sha in a single line --- start | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/start b/start index 1a6273710..b58a4c8c5 100755 --- a/start +++ b/start @@ -89,11 +89,9 @@ function start() { echo " horizon:" echo "$(stellar-horizon version | sed 's/^/ /')" echo " friendbot:" - echo "$(< image.json jq -r '.deps[] | select(.name == "friendbot") | .ref' | sed 's/^/ /')" - echo "$(< image.json jq -r '.deps[] | select(.name == "friendbot") | .sha' | sed 's/^/ /')" + echo " $(< image.json jq -r '.deps[] | select(.name == "friendbot") | "\(.ref) (\(.sha))"')" echo " lab:" - echo "$(< image.json jq -r '.deps[] | select(.name == "lab") | .ref' | sed 's/^/ /')" - echo "$(< image.json jq -r '.deps[] | select(.name == "lab") | .sha' | sed 's/^/ /')" + echo " $(< image.json jq -r '.deps[] | select(.name == "lab") | "\(.ref) (\(.sha))"')" echo "mode: $STELLAR_MODE" echo "network: $NETWORK" From 2149b328fa238224301191b5fb22dd2fa3f5ca2e Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Thu, 9 Oct 2025 06:35:39 +1000 Subject: [PATCH 131/135] change artifact input type to string with empty default --- action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/action.yml b/action.yml index 73802618c..d7b7c2e4d 100644 --- a/action.yml +++ b/action.yml @@ -7,8 +7,8 @@ inputs: default: "latest" artifact: description: "Artifact to collect image from" - type: boolean - default: false + type: string + default: "" image: description: "Image for the quickstart image to use" default: "" From 3ecc3fe2d8ed9a6c1382079f03d064af71da4e60 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Thu, 9 Oct 2025 06:37:19 +1000 Subject: [PATCH 132/135] update build workflow to use static json array for enable field --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8a2d2e6ea..5877efa72 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -319,7 +319,7 @@ jobs: tag: ${{ fromJSON(needs.setup.outputs.images).*.tag }} arch: ${{ fromJSON(inputs.archs) }} network: ["local"] - enable: ${{ inputs.image_json && fromJSON('["core,rpc,horizon"]') || fromJSON('["core","rpc","core,rpc,horizon"]') }} + enable: ["core","rpc","core,rpc,horizon"] options: [""] include: ${{ fromJSON(needs.setup.outputs.additional-tests) }} fail-fast: false From 74e47e37b409f701da3e41813245c56d840a9917 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Thu, 9 Oct 2025 06:38:19 +1000 Subject: [PATCH 133/135] fix json path in start script --- start | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/start b/start index b58a4c8c5..f3a85e433 100755 --- a/start +++ b/start @@ -89,9 +89,9 @@ function start() { echo " horizon:" echo "$(stellar-horizon version | sed 's/^/ /')" echo " friendbot:" - echo " $(< image.json jq -r '.deps[] | select(.name == "friendbot") | "\(.ref) (\(.sha))"')" + echo " $(< /image.json jq -r '.deps[] | select(.name == "friendbot") | "\(.ref) (\(.sha))"')" echo " lab:" - echo " $(< image.json jq -r '.deps[] | select(.name == "lab") | "\(.ref) (\(.sha))"')" + echo " $(< /image.json jq -r '.deps[] | select(.name == "lab") | "\(.ref) (\(.sha))"')" echo "mode: $STELLAR_MODE" echo "network: $NETWORK" From 0f90e272f82bdfdeec057a1a8e2d1b0aaefb1d08 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Thu, 9 Oct 2025 06:39:05 +1000 Subject: [PATCH 134/135] rename ci workflow jobs --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 847bbb53b..324b2b1b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,14 +94,14 @@ jobs: action-using-artifact: needs: [setup, build] - name: 4 test action artifact + name: 3 test action artifact uses: ./.github/workflows/action-test.yml with: artifact: image-quickstart-latest-amd64.tar tag: latest-amd64 push: - name: 3 push + name: 4 push needs: [setup, build] uses: ./.github/workflows/push.yml with: @@ -118,7 +118,7 @@ jobs: action-using-registry: needs: [setup, push] - name: 4 test action registry + name: 5 test action registry uses: ./.github/workflows/action-test.yml with: tag: ${{ needs.setup.outputs.tag-prefix }}latest From db6204be3ce541da1594e6b3a44c2290f9e36467 Mon Sep 17 00:00:00 2001 From: Leigh <351529+leighmcculloch@users.noreply.github.com> Date: Thu, 9 Oct 2025 06:40:40 +1000 Subject: [PATCH 135/135] fix indentation in readme --- README.md | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index c181f6c92..b358111e4 100644 --- a/README.md +++ b/README.md @@ -227,26 +227,26 @@ on: [push, pull_request] jobs: build-custom: uses: stellar/quickstart/.github/workflows/build.yml@main - with: - images: | - [ - { - "tag": "custom", - "config": { - "protocol_version_default": 23 - }, - "deps": [ - { "name": "xdr", "repo": "stellar/rs-stellar-xdr", "ref": "v23.0.0" }, - { "name": "core", "repo": "stellar/stellar-core", "ref": "v23.0.1", "options": { "configure_flags": "--disable-tests" } }, - { "name": "rpc", "repo": "stellar/stellar-rpc", "ref": "v23.0.1" }, - { "name": "horizon", "repo": "stellar/go", "ref": "horizon-v23.0.0" }, - { "name": "friendbot", "repo": "stellar/go", "ref": "horizon-v23.0.0" }, - { "name": "lab", "repo": "stellar/laboratory", "ref": "main" } - ], - "additional-tests": [] - } - ] - archs: '["amd64"]' + with: + images: | + [ + { + "tag": "custom", + "config": { + "protocol_version_default": 23 + }, + "deps": [ + { "name": "xdr", "repo": "stellar/rs-stellar-xdr", "ref": "v23.0.0" }, + { "name": "core", "repo": "stellar/stellar-core", "ref": "v23.0.1", "options": { "configure_flags": "--disable-tests" } }, + { "name": "rpc", "repo": "stellar/stellar-rpc", "ref": "v23.0.1" }, + { "name": "horizon", "repo": "stellar/go", "ref": "horizon-v23.0.0" }, + { "name": "friendbot", "repo": "stellar/go", "ref": "horizon-v23.0.0" }, + { "name": "lab", "repo": "stellar/laboratory", "ref": "main" } + ], + "additional-tests": [] + } + ] + archs: '["amd64"]' use-custom: needs: build-custom runs-on: ubuntu-latest