From f371e94dbf3a7bc056aa6e4222be25602b51ba31 Mon Sep 17 00:00:00 2001 From: Luca Terracciano Date: Tue, 6 May 2025 13:22:44 +0200 Subject: [PATCH 1/3] build: update submodules --- extern/hicr | 2 +- extern/tracr | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extern/hicr b/extern/hicr index 8af1374..e31ef4e 160000 --- a/extern/hicr +++ b/extern/hicr @@ -1 +1 @@ -Subproject commit 8af13748fb6b9387014f3a938f2931bf2e13f89e +Subproject commit e31ef4ec287dde6c8c88f9977d5ae5772e788955 diff --git a/extern/tracr b/extern/tracr index ccf2a19..984f5d7 160000 --- a/extern/tracr +++ b/extern/tracr @@ -1 +1 @@ -Subproject commit ccf2a19119d4b90c2b1b8b34ae2d32628f4af648 +Subproject commit 984f5d750531199a5cbf13e54d10f45a0f28c107 From b98ee8b000660f8bcdec9a977d93081499aa5aa0 Mon Sep 17 00:00:00 2001 From: Luca Terracciano Date: Tue, 6 May 2025 13:23:00 +0200 Subject: [PATCH 2/3] ci: use hicr CI for taskr --- .build-tools/containers/build.sh | 19 +- .build-tools/containers/buildenv/Dockerfile | 3 +- .build-tools/containers/buildenv/run.sh | 2 +- .build-tools/containers/deploy.sh | 24 -- .build-tools/containers/remove.sh | 12 - .build-tools/containers/run.sh | 21 +- .github/PULL_REQUEST_TEMPLATE.md | 2 +- .github/workflows/master-build-workflow.yml | 99 +++++ .github/workflows/master-test-workflow.yml | 63 +++ .github/workflows/master.yml | 68 ++++ .github/workflows/pr-development-coverage.yml | 67 ++++ .github/workflows/pr-development-workflow.yml | 361 ++++++++++++++++++ .github/workflows/pr-development.yml | 25 ++ .github/workflows/style.yml | 2 +- .github/workflows/taskr.yml | 2 +- .gitlab-ci.yml | 4 +- README.md | 2 +- meson.build | 2 +- 18 files changed, 700 insertions(+), 78 deletions(-) delete mode 100755 .build-tools/containers/deploy.sh delete mode 100755 .build-tools/containers/remove.sh create mode 100644 .github/workflows/master-build-workflow.yml create mode 100644 .github/workflows/master-test-workflow.yml create mode 100644 .github/workflows/master.yml create mode 100644 .github/workflows/pr-development-coverage.yml create mode 100644 .github/workflows/pr-development-workflow.yml create mode 100644 .github/workflows/pr-development.yml diff --git a/.build-tools/containers/build.sh b/.build-tools/containers/build.sh index bc01183..2be02ce 100755 --- a/.build-tools/containers/build.sh +++ b/.build-tools/containers/build.sh @@ -1,22 +1,11 @@ #!/usr/bin/env bash -if command -v arch &>/dev/null; then - target_arch=$(arch) - if [ $target_arch == "aarch64" ]; then - target_arch="arm64v8" - else - target_arch="x86_64" - fi -else - if [[ $# -ne 2 ]]; then - echo "arch not installed. Please provice manually the target architecture. Usage: $0 " - exit 1 - else - target_arch=${2} - fi +if [[ $# -ne 1 ]]; then + echo "arch not installed. Please provide the folder to build. Usage: $0 " + exit 1 fi folder=${1} echo "Building $folder for arch $target_arch" -docker build -t "registry.gitlab.huaweirc.ch/zrc-von-neumann-lab/runtime-system-innovations/taskr/${folder}-${target_arch}:latest" --build-arg ARCH=${target_arch} ${folder} +docker build -t "ghcr.io/algebraic-programming/taskr/${folder}:latest" ${folder} diff --git a/.build-tools/containers/buildenv/Dockerfile b/.build-tools/containers/buildenv/Dockerfile index d710822..dacf918 100644 --- a/.build-tools/containers/buildenv/Dockerfile +++ b/.build-tools/containers/buildenv/Dockerfile @@ -1,5 +1,4 @@ -ARG ARCH= -FROM registry.gitlab.huaweirc.ch/zrc-von-neumann-lab/runtime-system-innovations/hicr/buildenv-${ARCH}:latest +FROM ghcr.io/algebraic-programming/hicr/buildenv:latest ENV USER=hicr ENV HOME=/home/$USER diff --git a/.build-tools/containers/buildenv/run.sh b/.build-tools/containers/buildenv/run.sh index 51ae1b5..0bfc235 100755 --- a/.build-tools/containers/buildenv/run.sh +++ b/.build-tools/containers/buildenv/run.sh @@ -8,4 +8,4 @@ fi folder=${1} arch=${2} -docker run --name taskr --shm-size=1024M --privileged -td "registry.gitlab.huaweirc.ch/zrc-von-neumann-lab/runtime-system-innovations/taskr/${folder}-${arch}:latest" bash +docker run --name taskr --shm-size=1024M --privileged -td "ghcr.io/algebraic-programming/taskr/${folder}:latest" bash diff --git a/.build-tools/containers/deploy.sh b/.build-tools/containers/deploy.sh deleted file mode 100755 index ca8f379..0000000 --- a/.build-tools/containers/deploy.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash - -if command -v arch &>/dev/null; then - target_arch=$(arch) - - if [ $target_arch == "aarch64" ]; then - target_arch="arm64v8" - else - target_arch="x86_64" - fi -else - if [[ $# -ne 2 ]]; then - echo "arch not installed. Please provice manually the target architecture. Usage: $0 " - exit 1 - else - arch=${2} - fi -fi - -folder=${1} -echo "Deploying $folder for arch $target_arch" - -docker login registry.gitlab.huaweirc.ch -docker push "registry.gitlab.huaweirc.ch/zrc-von-neumann-lab/runtime-system-innovations/taskr/${folder}-${target_arch}:latest" diff --git a/.build-tools/containers/remove.sh b/.build-tools/containers/remove.sh deleted file mode 100755 index 18f5471..0000000 --- a/.build-tools/containers/remove.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -if [[ $# -ne 2 ]]; then - echo "Usage: $0 " - exit 1 -fi - -folder=${1} -arch=${2} - -docker rmi -f "registry.gitlab.huaweirc.ch/zrc-von-neumann-lab/runtime-system-innovations/taskr/${folder}-${arch}:latest" - diff --git a/.build-tools/containers/run.sh b/.build-tools/containers/run.sh index 8ed89c1..9328bc2 100755 --- a/.build-tools/containers/run.sh +++ b/.build-tools/containers/run.sh @@ -1,24 +1,11 @@ #!/usr/bin/env bash -if command -v arch &>/dev/null; then - target_arch=$(arch) - - if [ $target_arch == "aarch64" ]; then - target_arch="arm64v8" - else - target_arch="x86_64" - fi -else - if [[ $# -ne 2 ]]; then - echo "arch not installed. Please provice manually the target architecture. Usage: $0 " - exit 1 - else - arch=${2} - fi +if [[ $# -ne 1 ]]; then + echo "arch not installed. Please provide the folder to build. Usage: $0 " + exit 1 fi folder=${1} -echo "Running $folder for arch $target_arch" build_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -$build_dir/$folder/run.sh ${folder} ${target_arch} +$build_dir/$folder/run.sh ${folder} diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 28e9254..9356185 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -5,7 +5,7 @@ If it fixes a bug or resolves a feature request, be sure to link to that issue._ ## Type of change -_What type of changes does your code introduce to HiCR? Put an `x` in the box that apply._ +_What type of changes does your code introduce to TaskR? Put an `x` in the box that apply._ - [ ] `CHANGE` (fix or feature that would cause existing functionality to not work as expected) - [ ] `FEATURE` (non-breaking change which adds functionality) diff --git a/.github/workflows/master-build-workflow.yml b/.github/workflows/master-build-workflow.yml new file mode 100644 index 0000000..2b6695e --- /dev/null +++ b/.github/workflows/master-build-workflow.yml @@ -0,0 +1,99 @@ +name: Merge into master - Build Workflow + +# if statements modified to avoid: https://stackoverflow.com/questions/69354003/github-action-job-fire-when-previous-job-skipped + +on: + workflow_call: + inputs: + os: + required: true + type: string + arch: + required: true + type: string + +permissions: + contents: read + +env: + REGISTRY: ghcr.io + DOCKERIMAGE: ghcr.io/algebraic-programming/taskr/buildenv + +defaults: + run: + shell: bash + +jobs: + check-dockerfile-modifications-with-last-commit: + runs-on: ${{ inputs.os }} + outputs: + dockerfile-modified: ${{ steps.check-dockerfile.outputs.dockerfile-modified }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 2 + + # Check the PR diff using the current branch and the base branch of the PR + - name: Run git diff + run: | + git diff --name-only HEAD^..HEAD > ${{ runner.temp }}/diff.txt + + - name: Check if Dockerfile was modified + id: check-dockerfile + env: + MODIFIED_FILES_PATH: ${{ runner.temp }}/diff.txt + run: | + cat $MODIFIED_FILES_PATH + if cat $MODIFIED_FILES_PATH | grep -q 'buildenv/Dockerfile' ; then + echo "Dockerfile was modified" + echo "dockerfile-modified=true" >> $GITHUB_OUTPUT + else + echo "Dockerfile was not modified" + echo "dockerfile-modified=false" >> $GITHUB_OUTPUT + fi + + build-image: + runs-on: ${{ inputs.os }} + needs: [check-dockerfile-modifications-with-last-commit] + if: ${{ needs.check-dockerfile-modifications-with-last-commit.outputs.dockerfile-modified == 'true' }} + permissions: + contents: read + packages: write + attestations: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Docker metadata + id: dockermeta + uses: docker/metadata-action@v4 + with: + images: ${{ env.DOCKERIMAGE }} + tags: ${{ inputs.arch }}-latest + + - name: Build and push docker image + uses: docker/build-push-action@v6 + with: + context: "{{defaultContext}}:.build-tools/containers/buildenv" + push: true + tags: ${{ steps.dockermeta.outputs.tags }} + labels: ${{ steps.dockermeta.outputs.labels }} + build-args: ARCH=${{ inputs.arch }} + cache-from: type=gha + cache-to: type=gha,mode=max \ No newline at end of file diff --git a/.github/workflows/master-test-workflow.yml b/.github/workflows/master-test-workflow.yml new file mode 100644 index 0000000..0fea4c5 --- /dev/null +++ b/.github/workflows/master-test-workflow.yml @@ -0,0 +1,63 @@ +name: Merge into master - Run Tests Workflow + +# if statements modified to avoid: https://stackoverflow.com/questions/69354003/github-action-job-fire-when-previous-job-skipped + +on: + workflow_call: + inputs: + os: + required: true + type: string + arch: + required: true + type: string + +permissions: + contents: read + +env: + REGISTRY: ghcr.io + DOCKERIMAGE: ghcr.io/algebraic-programming/taskr/buildenv + +defaults: + run: + shell: bash + +jobs: + # Build TaskR and run tests and the remote image + compile-and-test: + runs-on: ${{ inputs.os }} + container: + image: ghcr.io/algebraic-programming/taskr/buildenv:latest + options: --user taskr + credentials: + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: 'true' + + - name: Setup + run: source /home/hicr/.bashrc && meson setup build -Dbackends=hwloc,pthreads,mpi,lpf,nosv,boost,opencl -Dfrontends=channel,RPCEngine,tasking,objectStore -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true + + - name: Compile + run: source /home/hicr/.bashrc && meson compile -C build + + - name: Running tests and creating coverage report + shell: bash + run: | + echo "Running Tests..." + source /home/hicr/.bashrc + meson setup build --wipe -Db_coverage=true -Dbackends=hwloc,pthreads,mpi,nosv,boost -Dfrontends=channel,RPCEngine,tasking,objectStore -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true + meson compile -C build + meson test -C build + echo "Creating coverage report..." + ninja -C build coverage + - uses: actions/upload-artifact@v4 + if: always() + with: + name: meson-logs-${{ inputs.arch }} + path: build/meson-logs/ diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml new file mode 100644 index 0000000..83e649e --- /dev/null +++ b/.github/workflows/master.yml @@ -0,0 +1,68 @@ +name: Merge into master - Build and Run Tests + +# if statements modified to avoid: https://stackoverflow.com/questions/69354003/github-action-job-fire-when-previous-job-skipped + +on: + push: + branches: ["master"] + +env: + REGISTRY: ghcr.io + DOCKERIMAGE: ghcr.io/algebraic-programming/taskr/buildenv + +jobs: + build-docker-arm64: + uses: Algebraic-Programming/TaskR/.github/workflows/master-build-workflow.yml@master + with: + os: ubuntu-24.04-arm + arch: arm64 + + build-docker-amd64: + uses: Algebraic-Programming/TaskR/.github/workflows/master-build-workflow.yml@master + with: + os: ubuntu-24.04 + arch: amd64 + + push-buildenv-manifest: + runs-on: ubuntu-latest + needs: [ build-docker-amd64, build-docker-arm64 ] + if: | + always() && + (contains(needs.build-docker-amd64.result, 'success') || contains(needs.build-docker-amd64.result, 'skipped')) && + (contains(needs.build-docker-arm64.result, 'success') || contains(needs.build-docker-arm64.result, 'skipped')) + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Create and push manifest images + run: + docker buildx imagetools create --tag ${{ env.DOCKERIMAGE }}:latest ${{ env.DOCKERIMAGE }}:amd64-latest ${{ env.DOCKERIMAGE }}:arm64-latest + + compile-and-test-arm64: + needs: [ push-buildenv-manifest ] + if: | + always() && + (contains(needs.push-buildenv-manifest.result, 'success') || contains(needs.push-buildenv-manifest.result, 'skipped')) + uses: Algebraic-Programming/TaskR/.github/workflows/master-test-workflow.yml@master + with: + os: ubuntu-24.04-arm + arch: arm64 + + compile-and-test-amd64: + needs: [ push-buildenv-manifest ] + if: | + always() && + (contains(needs.push-buildenv-manifest.result, 'success') || contains(needs.push-buildenv-manifest.result, 'skipped')) + uses: Algebraic-Programming/TaskR/.github/workflows/master-test-workflow.yml@master + with: + os: ubuntu-24.04 + arch: amd64 + + diff --git a/.github/workflows/pr-development-coverage.yml b/.github/workflows/pr-development-coverage.yml new file mode 100644 index 0000000..84bfb02 --- /dev/null +++ b/.github/workflows/pr-development-coverage.yml @@ -0,0 +1,67 @@ +name: PR development - Build and Run Tests Workflow + +# if statements modified to avoid: https://stackoverflow.com/questions/69354003/github-action-job-fire-when-previous-job-skipped + +on: + workflow_run: + workflows: [ci-amd64, ci-arm64] + types: + - completed + +permissions: + contents: read + +env: + REGISTRY: ghcr.io + DOCKERIMAGE: ghcr.io/algebraic-programming/taskr/buildenv + +defaults: + run: + shell: bash + +jobs: + update-coverage-PR-amd64: + runs-on: ubuntu-latest + if: | + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.conclusion == 'success' + steps: + - name: Download standard coverage file + uses: actions/download-artifact@v4 + with: + name: meson-logs-amd64 + path: ${{ runner.temp }}/build + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Add Coverage PR Comment + uses: marocchino/sticky-pull-request-comment@v2 + if: github.event_name == 'pull_request' || github.event_name == 'pull_request' + with: + header: amd64 + message: | + Coverage for amd64 + recreate: true + path: code-coverage-results.md + + update-coverage-PR-arm64: + runs-on: ubuntu-latest + if: | + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.conclusion == 'success' + steps: + - name: Download standard coverage file + uses: actions/download-artifact@v4 + with: + name: meson-logs-arm64 + path: ${{ runner.temp }}/build + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Add Coverage PR Comment + uses: marocchino/sticky-pull-request-comment@v2 + if: github.event_name == 'pull_request' || github.event_name == 'pull_request' + with: + header: arm64 + message: | + Coverage for arm64 + recreate: true + path: code-coverage-results.md diff --git a/.github/workflows/pr-development-workflow.yml b/.github/workflows/pr-development-workflow.yml new file mode 100644 index 0000000..db91fa0 --- /dev/null +++ b/.github/workflows/pr-development-workflow.yml @@ -0,0 +1,361 @@ +name: PR development - Build and Run Tests Workflow + +# if statements modified to avoid: https://stackoverflow.com/questions/69354003/github-action-job-fire-when-previous-job-skipped + +on: + workflow_call: + inputs: + os: + required: true + type: string + arch: + required: true + type: string + +permissions: + contents: read + +env: + REGISTRY: ghcr.io + DOCKERIMAGE: ghcr.io/algebraic-programming/taskr/buildenv + +defaults: + run: + shell: bash + +jobs: + check-dockerfile-modifications-with-base: + runs-on: ${{ inputs.os }} + outputs: + dockerfile-modified: ${{ steps.check-dockerfile.outputs.dockerfile-modified }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # Check the PR diff using the current branch and the base branch of the PR + - name: Run git diff + run: | + git diff --name-only origin/master ${{ github.event.pull_request.head.sha }} > ${{ runner.temp }}/diff.txt + + - name: Check if Dockerfile was modified + id: check-dockerfile + env: + MODIFIED_FILES_PATH: ${{ runner.temp }}/diff.txt + run: | + echo "$(git rev-parse origin/master)" + echo "$(git rev-parse ${{ github.event.pull_request.head.sha }})" + cat $MODIFIED_FILES_PATH + if cat $MODIFIED_FILES_PATH | grep -q 'buildenv/Dockerfile' ; then + echo "Dockerfile was modified" + echo "dockerfile-modified=true" >> $GITHUB_OUTPUT + else + echo "Dockerfile was not modified" + echo "dockerfile-modified=false" >> $GITHUB_OUTPUT + fi + + check-dockerfile-modifications-with-last-commit: + runs-on: ${{ inputs.os }} + outputs: + dockerfile-modified: ${{ steps.check-dockerfile.outputs.dockerfile-modified }} + steps: + - name: Checkout repository + id: checkout-base + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + # Check the PR diff using the current branch and the base branch of the PR + - name: Run git diff + run: | + git diff --name-only HEAD^ HEAD > ${{ runner.temp }}/diff.txt + + - name: Check if Dockerfile was modified + id: check-dockerfile + env: + MODIFIED_FILES_PATH: ${{ runner.temp }}/diff.txt + run: | + echo "HEAD^: $(git rev-parse HEAD^)" + echo "HEAD: $(git rev-parse HEAD)" + cat $MODIFIED_FILES_PATH + if cat $MODIFIED_FILES_PATH | grep -q 'buildenv/Dockerfile' ; then + echo "Dockerfile was modified" + echo "dockerfile-modified=true" >> $GITHUB_OUTPUT + else + echo "Dockerfile was not modified" + echo "dockerfile-modified=false" >> $GITHUB_OUTPUT + fi + + check-docker-artifact-existence: + runs-on: ${{ inputs.os }} + # Trigger only if there are modification to the dockerfile and a temporary image should be used + outputs: + artifact-exists: ${{ steps.temporary-docker-exists.outputs.artifact-exists }} + artifact-run-id: ${{ steps.temporary-docker-exists.outputs.artifact-run-id }} + steps: + - name: Fetch artifacts JSON + run: | + curl https://api.github.com/repos/${{ github.repository }}/actions/artifacts?per_page=100&name=buildenv-${{ github.event.number }}-${{ inputs.arch }}.tar > artifacts.json + cat artifacts.json + + - name: Check artifact existence + id: temporary-docker-exists + run: | + echo "Running on ${{ inputs.arch }}" + + # count how many artifacts match our branch & name + COUNT=$(cat artifacts.json \ + | jq --arg branch "${{github.head_ref}}" --arg name "buildenv-${{ github.event.number }}-${{ inputs.arch }}.tar" \ + '[ .artifacts[] + | select(.workflow_run.head_branch==$branch and .name==$name) + ] | length') + + # export found=true/false (and the count if you like) + if [ "$COUNT" -gt 0 ]; then + echo "artifact-exists=true" >> $GITHUB_OUTPUT + echo "artifact-exists=true" + else + echo "artifact-exists=false" >> $GITHUB_OUTPUT + echo "artifact-run-id=none" >> $GITHUB_OUTPUT + echo "artifact-exists=false" + echo "artifact-run-id=none" + exit 0 + fi + + ARTIFACT_ID=$(cat artifacts.json | jq -r --arg branch "${{github.head_ref}}" --arg name "buildenv-${{ github.event.number }}-${{ inputs.arch }}.tar" \ + ' + .artifacts + | map(select(.workflow_run.head_branch == $branch and .name == $name)) + | max_by(.created_at) + | .workflow_run.id + ' ) + echo "artifact-run-id=$ARTIFACT_ID" + echo "artifact-run-id=$ARTIFACT_ID" >> $GITHUB_OUTPUT + + build-image: + runs-on: ${{ inputs.os }} + needs: [check-dockerfile-modifications-with-last-commit] + if: ${{ needs.check-dockerfile-modifications-with-last-commit.outputs.dockerfile-modified == 'true' }} + outputs: + new-artifact-run-id: ${{ steps.get-new-run-id.outputs.new-artifact-run-id }} + permissions: + contents: read + attestations: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Docker metadata + id: dockermeta + uses: docker/metadata-action@v4 + with: + images: ${{ env.DOCKERIMAGE }} + tags: ${{ inputs.arch }}-latest + + - name: Build docker image + uses: docker/build-push-action@v6 + with: + context: "{{defaultContext}}:.build-tools/containers/buildenv" + push: false + tags: ${{ steps.dockermeta.outputs.tags }} + labels: ${{ steps.dockermeta.outputs.labels }} + build-args: ARCH=${{ inputs.arch }} + outputs: type=docker,dest=/tmp/buildenv-${{ github.event.number }}-${{ inputs.arch }}.tar + + - name: Upload to artifacts + uses: actions/upload-artifact@v4 + with: + name: buildenv-${{ github.event.number }}-${{ inputs.arch }}.tar + path: /tmp/buildenv-${{ github.event.number }}-${{ inputs.arch }}.tar + compression-level: 9 + - name: Get the new run id + id: get-new-run-id + run: | + echo "new-artifact-run-id=${{ github.run_id }}" >> $GITHUB_OUTPUT + + # Build TaskR and run tests on the locally built image + compile-and-test-local-custom-image: + runs-on: ${{ inputs.os }} + permissions: + pull-requests: write + needs: + [ + build-image, + check-docker-artifact-existence, + check-dockerfile-modifications-with-base, + ] + if: | + always() && + (contains(needs.build-image.result, 'success') || contains(needs.build-image.result, 'skipped')) && + contains(needs.check-dockerfile-modifications-with-base.outputs.dockerfile-modified , 'true') + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: 'true' + + - name: Download new local docker image + if: ${{ needs.build-image.result == 'success' }} + uses: actions/download-artifact@v4 + with: + name: buildenv-${{ github.event.number }}-${{ inputs.arch }}.tar + path: ${{ runner.temp }} + run-id: ${{ needs.build-image.outputs.new-artifact-run-id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Download local docker image + if: ${{ needs.build-image.result == 'skipped' }} + uses: actions/download-artifact@v4 + with: + name: buildenv-${{ github.event.number }}-${{ inputs.arch }}.tar + path: ${{ runner.temp }} + run-id: ${{ needs.check-docker-artifact-existence.outputs.artifact-run-id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Load local image + run: | + docker load --input ${{ runner.temp }}/buildenv-${{ github.event.number }}-${{ inputs.arch }}.tar + docker image ls -a + + - name: Start local container + run: | + docker run --name taskr --shm-size=1024M --privileged -v $PWD:/home/hicr/taskr -w /home/hicr/taskr -td ${{ env.DOCKERIMAGE }}:${{ inputs.arch }}-latest bash + + - name: Setup + run: docker exec -u hicr taskr bash -c "meson setup build -Dbackends=hwloc,pthreads,mpi,lpf,nosv,boost,opencl -Dfrontends=channel,RPCEngine,tasking,objectStore -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true" + + - name: Compile + run: docker exec -u hicr taskr bash -c "meson compile -C build" + + - name: Running tests and creating coverage report + run: | + echo "Running Tests..." + docker exec -u hicr taskr bash -c "source /home/hicr/.bashrc && meson setup build --wipe -Db_coverage=true -Dbackends=hwloc,pthreads,mpi,nosv,boost -Dfrontends=channel,RPCEngine,tasking,objectStore -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true" + docker exec -u hicr taskr bash -c "source /home/hicr/.bashrc && meson compile -C build" + docker exec -u hicr taskr bash -c "source /home/hicr/.bashrc && meson test -C build" + echo "Creating coverage report..." + docker exec -u hicr taskr bash -c "ninja -C build coverage" + docker stop taskr + docker container rm taskr + - uses: actions/upload-artifact@v4 + with: + name: build-local-${{ inputs.arch }} + path: build/ + + + # Build TaskR and run tests and the remote image + compile-and-test-standard: + runs-on: ${{ inputs.os }} + needs: [check-dockerfile-modifications-with-base] + if: | + always() && + contains(needs.check-dockerfile-modifications-with-base.outputs.dockerfile-modified , 'false') + container: + image: ghcr.io/algebraic-programming/taskr/buildenv:latest + options: --user hicr + credentials: + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: 'true' + + - name: Setup + run: source /home/hicr/.bashrc && meson setup build -Dbackends=hwloc,pthreads,mpi,lpf,nosv,boost,opencl -Dfrontends=channel,RPCEngine,tasking,objectStore -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true + + - name: Compile + run: source /home/hicr/.bashrc && meson compile -C build + + - name: Running tests and creating coverage report + shell: bash + run: | + echo "Running Tests..." + source /home/hicr/.bashrc + meson setup build --wipe -Db_coverage=true -Dbackends=hwloc,pthreads,mpi,nosv,boost -Dfrontends=channel,RPCEngine,tasking,objectStore -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true + meson compile -C build + meson test -C build + echo "Creating coverage report..." + ninja -C build coverage + - uses: actions/upload-artifact@v4 + with: + name: build-standard-${{ inputs.arch }} + path: build/ + + + analyze-coverage-report: + runs-on: ubuntu-latest + needs: [compile-and-test-standard, compile-and-test-local-custom-image] + permissions: + pull-requests: write + if: | + always() && + (contains(needs.compile-and-test-standard.result, 'success') || contains(needs.compile-and-test-local-custom-image.result, 'success')) + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Create temp build folder + run: | + mkdir ${{ runner.temp }}/build + - name: Download standard coverage file + if: ${{ needs.compile-and-test-standard.result == 'success' }} + uses: actions/download-artifact@v4 + with: + name: build-standard-${{ inputs.arch }} + path: ${{ runner.temp }}/build + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Download local coverage file + if: ${{ needs.compile-and-test-local-custom-image.result == 'success' }} + uses: actions/download-artifact@v4 + with: + name: build-local-${{ inputs.arch }} + path: ${{ runner.temp }}/build + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Copy build folder in repo + run: | + mv ${{ runner.temp }}/build . + + - name: Code Coverage Report + uses: irongut/CodeCoverageSummary@v1.3.0 + with: + filename: build/meson-logs/coverage.xml + badge: true + fail_below_min: true + format: markdown + hide_branch_rate: false + hide_complexity: true + indicators: true + output: both + thresholds: "60 80" + + - name: Add arch to coverage file + run: | + sed -i '1i # Coverage ${{ inputs.arch }}' code-coverage-results.md + + - name: Add Coverage PR Comment + uses: marocchino/sticky-pull-request-comment@v2 + if: github.event_name == 'pull_request' || github.event_name == 'pull_request' + with: + header: ${{ inputs.arch }} + message: | + Coverage for ${{ inputs.arch }} + recreate: true + path: code-coverage-results.md + - uses: actions/upload-artifact@v4 + with: + name: meson-logs-${{ inputs.arch }} + path: build/meson-logs/ diff --git a/.github/workflows/pr-development.yml b/.github/workflows/pr-development.yml new file mode 100644 index 0000000..864424a --- /dev/null +++ b/.github/workflows/pr-development.yml @@ -0,0 +1,25 @@ +name: PR development - Build and Run Tests + +# if statements modified to avoid: https://stackoverflow.com/questions/69354003/github-action-job-fire-when-previous-job-skipped + +on: + pull_request: + branches: ["master"] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + + +jobs: + ci-arm64: + uses: algebraic-programming/TaskR/.github/workflows/pr-development-workflow.yml@master + with: + os: ubuntu-24.04-arm + arch: arm64 + + ci-amd64: + uses: algebraic-programming/TaskR/.github/workflows/pr-development-workflow.yml@master + with: + os: ubuntu-24.04 + arch: amd64 \ No newline at end of file diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index 1cfc56d..adb15a9 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -23,7 +23,7 @@ jobs: python3 -m pip install 'clang-format==18.1.0' - name: Checking style run: | - echo "Checking HiCR source and test formatting..." + echo "Checking TaskR source and test formatting..." .build-tools/style/check-style.sh check include .build-tools/style/check-style.sh check tests .build-tools/style/check-style.sh check examples \ No newline at end of file diff --git a/.github/workflows/taskr.yml b/.github/workflows/taskr.yml index 083359f..1a96c8f 100644 --- a/.github/workflows/taskr.yml +++ b/.github/workflows/taskr.yml @@ -12,7 +12,7 @@ concurrency: jobs: - # Build HiCR and run tests + # Build TaskR and run tests build: runs-on: ubuntu-latest steps: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cbd236e..aa7f19f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,5 @@ build: - image: registry.gitlab.huaweirc.ch/zrc-von-neumann-lab/runtime-system-innovations/taskr/buildenv-x86_64:latest + image: ghcr.io/algebraic-programming/taskr/buildenv-x86_64:latest variables: GIT_SUBMODULE_STRATEGY: recursive tags: @@ -36,7 +36,7 @@ build: - tags docs: - image: registry.gitlab.huaweirc.ch/zrc-von-neumann-lab/runtime-system-innovations/hicr/docs:latest + image: ghcr.io/algebraic-programming/hicr/docs:latest tags: - docker - tiny diff --git a/README.md b/README.md index 04bba0a..f64891b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Build and Run Tests](https://github.com/Algebraic-Programming/TaskR/actions/workflows/taskr.yml/badge.svg)](https://github.com/Algebraic-Programming/TaskR/actions/workflows/taskr.yml) +[![Build and Run Tests](https://github.com/Algebraic-Programming/TaskR/actions/workflows/master.yml/badge.svg)](https://github.com/Algebraic-Programming/TaskR/actions/workflows/master.yml) ``` ___________ __ __________ diff --git a/meson.build b/meson.build index d280099..a38cb0c 100644 --- a/meson.build +++ b/meson.build @@ -71,7 +71,7 @@ TaskRBuildDep = declare_dependency( dependencies: taskrDependencies ) -####### Build test / example targets only if HiCR is being loaded as a subproject +####### Build test / example targets only if TaskR is being loaded as a subproject if meson.is_subproject() == false From c502d12a9ae9729b96f62b7bb30c68e684dd398c Mon Sep 17 00:00:00 2001 From: Luca Terracciano Date: Tue, 6 May 2025 14:13:50 +0200 Subject: [PATCH 3/3] ci: fix ci trigger and gitlab image --- .github/workflows/master.yml | 2 +- .github/workflows/pr-development.yml | 2 +- .github/workflows/taskr.yml | 82 ---------------------------- .gitlab-ci.yml | 2 +- 4 files changed, 3 insertions(+), 85 deletions(-) delete mode 100644 .github/workflows/taskr.yml diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 83e649e..40b6d94 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -4,7 +4,7 @@ name: Merge into master - Build and Run Tests on: push: - branches: ["master"] + branches: ["main"] env: REGISTRY: ghcr.io diff --git a/.github/workflows/pr-development.yml b/.github/workflows/pr-development.yml index 864424a..7841297 100644 --- a/.github/workflows/pr-development.yml +++ b/.github/workflows/pr-development.yml @@ -4,7 +4,7 @@ name: PR development - Build and Run Tests on: pull_request: - branches: ["master"] + branches: ["main"] concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} diff --git a/.github/workflows/taskr.yml b/.github/workflows/taskr.yml deleted file mode 100644 index 1a96c8f..0000000 --- a/.github/workflows/taskr.yml +++ /dev/null @@ -1,82 +0,0 @@ -name: Build and Run Tests - -on: - pull_request: - branches: [ "main" ] - push: - branches: [ "main" ] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - - # Build TaskR and run tests - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Updating Apt - run: sudo apt update - - name: Installing apt packages - run: | - sudo apt install -y build-essential - sudo apt install -y git - sudo apt install -y libgtest-dev - sudo apt install -y libhwloc-dev - sudo apt install -y libopenblas-dev - sudo apt install -y liblapack-dev - sudo apt install -y liblapacke-dev - sudo apt install -y libboost-context-dev - sudo apt install -y curl - sudo apt install -y jq - sudo apt install -y pkgconf - sudo apt install -y wget - sudo apt install -y sudo - sudo apt install -y libopenmpi-dev - sudo apt install -y cmake - sudo apt install -y libstb-dev - sudo apt install -y libsfml-dev - sudo apt install -y libtclap-dev - - name: Installing meson, ninja and gcovr - run: python3 -m pip install meson ninja gcovr - - name: Install OVNI - run: | - git clone --recursive https://github.com/bsc-pm/ovni.git $HOME/ovni - cd $HOME/ovni - mkdir build - cd build - cmake .. -DCMAKE_INSTALL_PREFIX=/usr/local -DBUILD_TESTING=FALSE - make -j8 - sudo make install - sudo rm -rf $HOME/ovni - - name: Install NOS-V - run: | - git clone -b 3.1.0 --recursive https://github.com/bsc-pm/nos-v.git $HOME/nos-v - cd $HOME/nos-v - autoreconf -f -i -v - ./configure --prefix=/usr/local --with-ovni=/usr/local - make all - sudo make install - cd .. - rm -rf $HOME/nos-v - sudo sed -i '/^\[instrumentation\]/,/^\[/{s/version *= *"none"/version = "ovni"/}' /usr/local/share/nosv.toml - sudo sed -i '/^\[ovni\]/,/^\[/{s/level *= *2/level = 0/}' /usr/local/share/nosv.toml - - name: Updating submodules - run: git submodule update --init --recursive - - name: Building all modules - run: | - echo "Building..." - mkdir build - meson setup build -DdistributedEngine=mpi -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true - meson compile -C build - - name: Running tests - run: | - echo "Running Tests..." - meson test -C build - - uses: actions/upload-artifact@v4 - if: always() - with: - name: meson-logs - path: build/meson-logs/ \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index aa7f19f..baae1a7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,5 @@ build: - image: ghcr.io/algebraic-programming/taskr/buildenv-x86_64:latest + image: ghcr.io/algebraic-programming/taskr/buildenv:latest variables: GIT_SUBMODULE_STRATEGY: recursive tags: