Skip to content

Commit c6a01f3

Browse files
LegNeatoLegNeato
committed
Split llvm stuff into their own container
This makes the final containers smaller as they are too big still.
1 parent a0e1f91 commit c6a01f3

File tree

8 files changed

+404
-259
lines changed

8 files changed

+404
-259
lines changed

‎.github/workflows/container_images.yml‎

Lines changed: 180 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,106 @@ on:
1515
env:
1616
REGISTRY: ghcr.io
1717
jobs:
18+
build-toolchain:
19+
name: Build LLVM7 base (${{ matrix.variance.name }} / ${{ matrix.platform.arch }})
20+
runs-on: ${{ matrix.platform.runner }}
21+
permissions:
22+
contents: read
23+
packages: write
24+
strategy:
25+
fail-fast: false
26+
matrix:
27+
platform:
28+
- runner: ubuntu-latest
29+
arch: amd64
30+
- runner: ubuntu-24.04-arm
31+
arch: arm64
32+
variance:
33+
- name: Ubuntu-22.04
34+
toolchain_image: "rust-gpu/rust-cuda-ubuntu22-llvm7"
35+
toolchain_dockerfile: ./container/llvm7-ubuntu22/Dockerfile
36+
toolchain_artifact: llvm7-ubuntu22
37+
- name: Ubuntu-24.04
38+
toolchain_image: "rust-gpu/rust-cuda-ubuntu24-llvm7"
39+
toolchain_dockerfile: ./container/llvm7-ubuntu24/Dockerfile
40+
toolchain_artifact: llvm7-ubuntu24
41+
- name: RockyLinux-9
42+
toolchain_image: "rust-gpu/rust-cuda-rockylinux9-llvm7"
43+
toolchain_dockerfile: ./container/llvm7-rockylinux9/Dockerfile
44+
toolchain_artifact: llvm7-rockylinux9
45+
steps:
46+
- name: Checkout repository
47+
uses: actions/checkout@v4
48+
- name: Validate platform
49+
run: |
50+
ARCH=$(uname -m)
51+
if [[ "${{ matrix.platform.arch }}" == "amd64" && "$ARCH" != "x86_64" ]]; then
52+
echo "Error: Expected x86_64 but got $ARCH"
53+
exit 1
54+
fi
55+
if [[ "${{ matrix.platform.arch }}" == "arm64" && "$ARCH" != "aarch64" ]]; then
56+
echo "Error: Expected aarch64 but got $ARCH"
57+
exit 1
58+
fi
59+
echo "Platform validation passed: $ARCH matches ${{ matrix.platform.arch }}"
60+
- name: Log in to the Container registry
61+
uses: docker/login-action@v3
62+
with:
63+
registry: ${{ env.REGISTRY }}
64+
username: ${{ github.actor }}
65+
password: ${{ secrets.GITHUB_TOKEN }}
66+
- name: Extract metadata (tags, labels) for toolchain
67+
id: toolchain-meta
68+
uses: docker/metadata-action@v5
69+
with:
70+
images: ${{ env.REGISTRY }}/${{ matrix.variance.toolchain_image }}
71+
- name: Set up Docker Buildx
72+
uses: docker/setup-buildx-action@v3
73+
- name: Build and push toolchain
74+
id: toolchain-build
75+
uses: docker/build-push-action@v6
76+
with:
77+
context: .
78+
file: ${{ matrix.variance.toolchain_dockerfile }}
79+
platforms: linux/${{ matrix.platform.arch }}
80+
labels: ${{ steps.toolchain-meta.outputs.labels }}
81+
tags: |
82+
${{ steps.toolchain-meta.outputs.tags }}
83+
${{ env.REGISTRY }}/${{ matrix.variance.toolchain_image }}:ci-${{ github.run_id }}-${{ matrix.platform.arch }}
84+
outputs: |
85+
type=image,name=${{ env.REGISTRY }}/${{ matrix.variance.toolchain_image }},push-by-digest=true,name-canonical=true,push=${{ github.event_name != 'pull_request' }}
86+
type=docker,dest=/tmp/${{ matrix.variance.toolchain_artifact }}-${{ matrix.platform.arch }}.tar
87+
cache-from: type=gha
88+
- name: Export digest
89+
if: github.event_name != 'pull_request'
90+
run: |
91+
mkdir -p /tmp/digests
92+
digest="${{ steps.toolchain-build.outputs.digest }}"
93+
touch "/tmp/digests/${digest#sha256:}"
94+
- name: Upload digest
95+
if: github.event_name != 'pull_request'
96+
uses: actions/upload-artifact@v4
97+
with:
98+
name: digests-${{ matrix.variance.toolchain_artifact }}-${{ matrix.platform.arch }}
99+
path: /tmp/digests/*
100+
if-no-files-found: error
101+
retention-days: 1
102+
- name: Upload LLVM7 base layer
103+
uses: actions/upload-artifact@v4
104+
with:
105+
name: ${{ matrix.variance.toolchain_artifact }}-${{ matrix.platform.arch }}
106+
path: /tmp/${{ matrix.variance.toolchain_artifact }}-${{ matrix.platform.arch }}.tar
107+
if-no-files-found: error
108+
retention-days: 1
109+
- name: Clean up Docker cache
110+
if: always()
111+
run: |
112+
docker builder prune --all --force || true
113+
docker system prune --volumes --force || true
114+
18115
build-images:
19116
name: Build ${{ matrix.variance.name }} (${{ matrix.platform.arch }})
117+
needs: build-toolchain
20118
runs-on: ${{ matrix.platform.runner }}
21119
permissions:
22120
contents: read
@@ -33,18 +131,33 @@ jobs:
33131
- name: Ubuntu-22.04/CUDA-11.8.0
34132
image: "rust-gpu/rust-cuda-ubuntu22-cuda11"
35133
dockerfile: ./container/ubuntu22-cuda11/Dockerfile
134+
toolchain_image: "rust-gpu/rust-cuda-ubuntu22-llvm7"
135+
toolchain_artifact: llvm7-ubuntu22
36136
- name: Ubuntu-22.04/CUDA-12.8.1
37137
image: "rust-gpu/rust-cuda-ubuntu22-cuda12"
38138
dockerfile: ./container/ubuntu22-cuda12/Dockerfile
139+
toolchain_image: "rust-gpu/rust-cuda-ubuntu22-llvm7"
140+
toolchain_artifact: llvm7-ubuntu22
39141
- name: Ubuntu-24.04/CUDA-12.8.1
40142
image: "rust-gpu/rust-cuda-ubuntu24-cuda12"
41143
dockerfile: ./container/ubuntu24-cuda12/Dockerfile
144+
toolchain_image: "rust-gpu/rust-cuda-ubuntu24-llvm7"
145+
toolchain_artifact: llvm7-ubuntu24
42146
- name: RockyLinux-9/CUDA-12.8.1
43147
image: "rust-gpu/rust-cuda-rockylinux9-cuda12"
44148
dockerfile: ./container/rockylinux9-cuda12/Dockerfile
149+
toolchain_image: "rust-gpu/rust-cuda-rockylinux9-llvm7"
150+
toolchain_artifact: llvm7-rockylinux9
45151
steps:
46152
- name: Checkout repository
47153
uses: actions/checkout@v4
154+
- name: Download LLVM base image
155+
uses: actions/download-artifact@v4
156+
with:
157+
name: ${{ matrix.variance.toolchain_artifact }}-${{ matrix.platform.arch }}
158+
path: /tmp/toolchain
159+
- name: Load LLVM base image
160+
run: docker load --input /tmp/toolchain/${{ matrix.variance.toolchain_artifact }}-${{ matrix.platform.arch }}.tar
48161
- name: Validate platform
49162
run: |
50163
ARCH=$(uname -m)
@@ -80,7 +193,8 @@ jobs:
80193
labels: ${{ steps.meta.outputs.labels }}
81194
outputs: type=image,name=${{ env.REGISTRY }}/${{ matrix.variance.image }},push-by-digest=true,name-canonical=true,push=${{ github.event_name != 'pull_request' }}
82195
cache-from: type=gha
83-
cache-to: type=gha,mode=max
196+
build-args: |
197+
LLVM_BASE_IMAGE=${{ env.REGISTRY }}/${{ matrix.variance.toolchain_image }}:ci-${{ github.run_id }}-${{ matrix.platform.arch }}
84198
- name: Set artifact name
85199
if: github.event_name != 'pull_request'
86200
run: |
@@ -101,6 +215,71 @@ jobs:
101215
path: /tmp/digests/*
102216
if-no-files-found: error
103217
retention-days: 1
218+
- name: Clean up Docker cache
219+
if: always()
220+
run: |
221+
docker builder prune --all --force || true
222+
docker system prune --volumes --force || true
223+
224+
merge-toolchain:
225+
name: Create manifest for LLVM7 base (${{ matrix.variance.name }})
226+
runs-on: ubuntu-latest
227+
if: github.event_name != 'pull_request'
228+
needs: build-toolchain
229+
permissions:
230+
contents: read
231+
packages: write
232+
attestations: write
233+
id-token: write
234+
strategy:
235+
fail-fast: false
236+
matrix:
237+
variance:
238+
- name: Ubuntu-22.04
239+
toolchain_image: "rust-gpu/rust-cuda-ubuntu22-llvm7"
240+
toolchain_artifact: llvm7-ubuntu22
241+
- name: Ubuntu-24.04
242+
toolchain_image: "rust-gpu/rust-cuda-ubuntu24-llvm7"
243+
toolchain_artifact: llvm7-ubuntu24
244+
- name: RockyLinux-9
245+
toolchain_image: "rust-gpu/rust-cuda-rockylinux9-llvm7"
246+
toolchain_artifact: llvm7-rockylinux9
247+
steps:
248+
- name: Download digests
249+
uses: actions/download-artifact@v4
250+
with:
251+
path: /tmp/digests
252+
pattern: digests-${{ matrix.variance.toolchain_artifact }}-*
253+
merge-multiple: true
254+
- name: Set up Docker Buildx
255+
uses: docker/setup-buildx-action@v3
256+
- name: Docker meta
257+
id: toolchain-meta
258+
uses: docker/metadata-action@v5
259+
with:
260+
images: ${{ env.REGISTRY }}/${{ matrix.variance.toolchain_image }}
261+
tags: |
262+
type=ref,event=branch
263+
type=ref,event=pr
264+
type=semver,pattern={{version}}
265+
type=semver,pattern={{major}}.{{minor}}
266+
type=semver,pattern={{major}}
267+
type=sha,format=short
268+
type=raw,value=latest,enable={{is_default_branch}}
269+
- name: Login to Registry
270+
uses: docker/login-action@v3
271+
with:
272+
registry: ${{ env.REGISTRY }}
273+
username: ${{ github.actor }}
274+
password: ${{ secrets.GITHUB_TOKEN }}
275+
- name: Create manifest list and push
276+
working-directory: /tmp/digests
277+
run: |
278+
docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
279+
$(printf '${{ env.REGISTRY }}/${{ matrix.variance.toolchain_image }}@sha256:%s ' *)
280+
- name: Inspect image
281+
run: |
282+
docker buildx imagetools inspect ${{ env.REGISTRY }}/${{ matrix.variance.toolchain_image }}:${{ steps.toolchain-meta.outputs.version }}
104283
105284
merge-manifests:
106285
name: Create manifest for ${{ matrix.variance.name }}

‎container/llvm7-rockylinux9/Dockerfile‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
FROM nvcr.io/nvidia/cuda:12.8.1-cudnn-devel-rockylinux9 AS builder
2+
3+
RUN dnf -y update && \
4+
dnf -y install \
5+
clang \
6+
cmake \
7+
fontconfig-devel \
8+
libX11-devel \
9+
libXcursor-devel \
10+
libXi-devel \
11+
libXrandr-devel \
12+
libffi-devel \
13+
libxml2-devel \
14+
libedit-devel \
15+
ncurses-devel \
16+
openssl-devel \
17+
pkgconfig \
18+
python3 \
19+
redhat-rpm-config \
20+
which \
21+
xz \
22+
zlib-devel \
23+
make && \
24+
dnf clean all
25+
26+
WORKDIR /opt/build
27+
ARG LLVM_VERSION=7.1.0
28+
RUN curl -sSfL -O https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/llvm-${LLVM_VERSION}.src.tar.xz && \
29+
tar -xf llvm-${LLVM_VERSION}.src.tar.xz && \
30+
cd llvm-${LLVM_VERSION}.src && \
31+
mkdir build && cd build && \
32+
ARCH=$(uname -m) && \
33+
if [ "$ARCH" = "x86_64" ]; then \
34+
TARGETS="X86;NVPTX"; \
35+
else \
36+
TARGETS="AArch64;NVPTX"; \
37+
fi && \
38+
cmake \
39+
-DCMAKE_BUILD_TYPE=Release \
40+
-DLLVM_TARGETS_TO_BUILD="$TARGETS" \
41+
-DLLVM_BUILD_LLVM_DYLIB=ON \
42+
-DLLVM_LINK_LLVM_DYLIB=ON \
43+
-DLLVM_ENABLE_ASSERTIONS=OFF \
44+
-DLLVM_ENABLE_BINDINGS=OFF \
45+
-DLLVM_INCLUDE_EXAMPLES=OFF \
46+
-DLLVM_INCLUDE_TESTS=OFF \
47+
-DLLVM_INCLUDE_BENCHMARKS=OFF \
48+
-DLLVM_ENABLE_ZLIB=ON \
49+
-DLLVM_ENABLE_TERMINFO=ON \
50+
-DCMAKE_INSTALL_PREFIX=/opt/llvm-7 \
51+
.. && \
52+
make -j"$(nproc)" && \
53+
make install && \
54+
cd /opt && rm -rf /opt/build
55+
56+
RUN curl -sSfL https://sh.rustup.rs | bash -s -- -y --profile minimal && \
57+
/root/.cargo/bin/rustup component add rustfmt clippy && \
58+
rm -rf /root/.cargo/registry /root/.cargo/git /root/.rustup/tmp
59+
ENV PATH="/root/.cargo/bin:${PATH}"
60+
61+
FROM nvcr.io/nvidia/cuda:12.8.1-cudnn-devel-rockylinux9 AS toolchain
62+
COPY --from=builder /opt/llvm-7 /opt/llvm-7
63+
COPY --from=builder /root/.cargo /root/.cargo
64+
COPY --from=builder /root/.rustup /root/.rustup
65+
ENV PATH="/root/.cargo/bin:${PATH}"
66+
CMD ["bash"]

‎container/llvm7-ubuntu22/Dockerfile‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
FROM nvcr.io/nvidia/cuda:12.8.1-cudnn-devel-ubuntu22.04 AS builder
2+
3+
RUN apt-get update && \
4+
DEBIAN_FRONTEND=noninteractive apt-get -y install \
5+
build-essential \
6+
clang \
7+
cmake \
8+
curl \
9+
libffi-dev \
10+
libfontconfig-dev \
11+
libssl-dev \
12+
libtinfo-dev \
13+
libedit-dev \
14+
libncurses5-dev \
15+
libxml2-dev \
16+
libx11-xcb-dev \
17+
libxcursor-dev \
18+
libxi-dev \
19+
libxinerama-dev \
20+
libxrandr-dev \
21+
ninja-build \
22+
pkg-config \
23+
python3 \
24+
xz-utils \
25+
zlib1g-dev && \
26+
rm -rf /var/lib/apt/lists/*
27+
28+
WORKDIR /opt/build
29+
ARG LLVM_VERSION=7.1.0
30+
RUN curl -sSfL -O https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/llvm-${LLVM_VERSION}.src.tar.xz && \
31+
tar -xf llvm-${LLVM_VERSION}.src.tar.xz && \
32+
cd llvm-${LLVM_VERSION}.src && \
33+
mkdir build && cd build && \
34+
ARCH=$(dpkg --print-architecture) && \
35+
if [ "$ARCH" = "amd64" ]; then \
36+
TARGETS="X86;NVPTX"; \
37+
else \
38+
TARGETS="AArch64;NVPTX"; \
39+
fi && \
40+
cmake -G Ninja \
41+
-DCMAKE_BUILD_TYPE=Release \
42+
-DLLVM_TARGETS_TO_BUILD="$TARGETS" \
43+
-DLLVM_BUILD_LLVM_DYLIB=ON \
44+
-DLLVM_LINK_LLVM_DYLIB=ON \
45+
-DLLVM_ENABLE_ASSERTIONS=OFF \
46+
-DLLVM_ENABLE_BINDINGS=OFF \
47+
-DLLVM_INCLUDE_EXAMPLES=OFF \
48+
-DLLVM_INCLUDE_TESTS=OFF \
49+
-DLLVM_INCLUDE_BENCHMARKS=OFF \
50+
-DLLVM_ENABLE_ZLIB=ON \
51+
-DLLVM_ENABLE_TERMINFO=ON \
52+
-DCMAKE_INSTALL_PREFIX=/opt/llvm-7 \
53+
.. && \
54+
ninja -j"$(nproc)" && \
55+
ninja install && \
56+
cd /opt && rm -rf /opt/build
57+
58+
RUN curl -sSfL https://sh.rustup.rs | bash -s -- -y --profile minimal && \
59+
/root/.cargo/bin/rustup component add rustfmt clippy && \
60+
rm -rf /root/.cargo/registry /root/.cargo/git /root/.rustup/tmp
61+
ENV PATH="/root/.cargo/bin:${PATH}"
62+
63+
FROM nvcr.io/nvidia/cuda:12.8.1-cudnn-devel-ubuntu22.04 AS toolchain
64+
COPY --from=builder /opt/llvm-7 /opt/llvm-7
65+
COPY --from=builder /root/.cargo /root/.cargo
66+
COPY --from=builder /root/.rustup /root/.rustup
67+
ENV PATH="/root/.cargo/bin:${PATH}"
68+
CMD ["bash"]

0 commit comments

Comments
 (0)