From 808f77193d6699fbe3fd7e46d355408f4964aafc Mon Sep 17 00:00:00 2001 From: Brian Carp Date: Fri, 14 Nov 2025 02:36:14 -0500 Subject: [PATCH 01/11] Stabilize and automate the dev container build This commit resolves a series of build and runtime errors to create a stable, portable, and fully automated dev container environment that works on both `arm64` and `x86_64` architectures out-of-the-box. - Stabilize Dockerfile Build: - Upgrades the base image from `bullseye` to `bookworm`. - Consolidates all `apt-get` dependencies into a single, correctly ordered layer, installing necessary tools for cross-compilation (`gcc-x86-64-linux-gnu`, `libc6-dev-amd64-cross`). - Fixes `rustup` permission errors by installing the toolchain as `root` and granting ownership to the `vscode` user. - Adds `--break-system-packages` to the `pip install` command to comply with Debian `bookworm`'s package management policies. - Improve Architecture Portability: - Makes the `bin/build` and `bin/test` scripts architecture-aware, allowing them to run seamlessly on both `arm64` and `x86_64` hosts without manual configuration. - Fixes a bug that caused inconsistent naming of the shared library (`.so`) file between build and test runs. - Fix Container Startup on ARM64: - Centralizes QEMU and `binfmt` setup within the `Dockerfile` build, creating an architecture-aware initialization process. - This allows for the removal of legacy, conflicting setup methods that caused startup failures on `arm64` hosts: - Removes the privileged `docker run` command for `qemu-user-static` from the `postCreate` script. - Disables the redundant QEMU setup in the `docker-in-docker` feature by configuring `install-qemu: false` for the feature. --- .devcontainer/Dockerfile | 51 +++++++++++++++++++-------------- .devcontainer/devcontainer.json | 4 ++- .devcontainer/postCreate | 10 +++---- bin/build | 18 +++++++++++- bin/build-arch | 8 ++++-- bin/test | 28 +++++++++++++----- 6 files changed, 82 insertions(+), 37 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 3fda7a4..946a67c 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,30 +1,39 @@ -FROM mcr.microsoft.com/devcontainers/rust:1-1-bullseye +FROM mcr.microsoft.com/devcontainers/rust:2-1-bookworm -RUN sudo apt-get update -y \ - && sudo apt-get upgrade -y +RUN apt-get update -y \ + && apt-get upgrade -y \ + && apt-get install -y --fix-missing --no-install-recommends \ + zip \ + qemu-system \ + binfmt-support \ + qemu-user-static \ + nodejs \ + ruby \ + php \ + php-common \ + python3-pip \ + gcc-x86-64-linux-gnu \ + libc6-dev-amd64-cross \ + && rm -rf /var/lib/apt/lists/* \ + && update-binfmts --enable qemu-aarch64 -RUN sudo apt-get install -y --fix-missing zip +# Easy way to install Python. +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 -RUN sudo apt-get update -y \ - && sudo apt-get upgrade -y \ - && sudo apt-get install -y zip ltrace +# Switch to root to install rust targets and fix permissions +USER root -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +# Install rust targets for cross-compilation RUN rustup update \ - && rustup target add aarch64-unknown-linux-gnu + && rustup default stable \ + && rustup target add aarch64-unknown-linux-gnu \ + && rustup target add x86_64-unknown-linux-gnu -RUN rustup default stable -# x86_64 to arm64 support. -RUN sudo apt-get install -y \ - qemu \ - binfmt-support \ - qemu-user-static +# Grant vscode user ownership of rustup and cargo directories +RUN chown -R vscode:vscode /usr/local/rustup /usr/local/cargo -# Easy way to install node, ruby, and php -RUN apt-get -y install nodejs ruby php php-common - -# Easy way to install Python. -RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 +# Switch back to vscode user +USER vscode # Multi-platform SAM CLI. https://github.com/aws/aws-sam-cli/issues/3908 -RUN apt-get install -y pip && pip install aws-sam-cli +RUN pip install aws-sam-cli --break-system-packages diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 55620a3..1893e8b 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -5,7 +5,9 @@ }, "features": { "ghcr.io/devcontainers/features/aws-cli:latest": {}, - "ghcr.io/devcontainers/features/docker-in-docker:latest": {}, + "ghcr.io/devcontainers/features/docker-in-docker:latest": { + "install-qemu": false + }, "ghcr.io/customink/codespaces-features/docker-log-level": {}, "ghcr.io/devcontainers/features/sshd:latest": {} }, diff --git a/.devcontainer/postCreate b/.devcontainer/postCreate index 1415bf0..4b0762a 100755 --- a/.devcontainer/postCreate +++ b/.devcontainer/postCreate @@ -1,8 +1,8 @@ #!/bin/sh set -e -docker run \ - --rm \ - --privileged \ - multiarch/qemu-user-static \ - --reset -p yes +# Wait for docker to be ready +while ! docker info > /dev/null 2>&1; do + echo "Waiting for docker daemon..." + sleep 1 +done diff --git a/bin/build b/bin/build index ec7323c..3debfd3 100755 --- a/bin/build +++ b/bin/build @@ -3,7 +3,23 @@ set -e # Sync with bin/build, bin/test, & test/libcrypteia.sh. export CRYPTEIA_BUILD_OS="${CRYPTEIA_BUILD_OS:=debian}" -export CRYPTEIA_BUILD_TARGET="${CRYPTEIA_BUILD_TARGET:=x86_64-unknown-linux-gnu}" + +# Auto-detect build target if not already set +if [ -z "${CRYPTEIA_BUILD_TARGET}" ]; then + case "$(uname -m)" in + aarch64) + export CRYPTEIA_BUILD_TARGET="aarch64-unknown-linux-gnu" + ;; + x86_64) + export CRYPTEIA_BUILD_TARGET="x86_64-unknown-linux-gnu" + ;; + *) + echo "Unsupported architecture: $(uname -m)" + exit 1 + ;; + esac +fi + if [ "${CRYPTEIA_BUILD_TARGET}" = "aarch64-unknown-linux-gnu" ]; then export CRYPTEIA_BUILD_SUFFIX="-arm64" fi diff --git a/bin/build-arch b/bin/build-arch index 5c79a61..fe5a1c7 100755 --- a/bin/build-arch +++ b/bin/build-arch @@ -16,10 +16,14 @@ cargo build \ --target "${CRYPTEIA_BUILD_TARGET}" cp "./target/${CRYPTEIA_BUILD_TARGET}/release/crypteia" "./build/${BIN}" -cp ./target/${CRYPTEIA_BUILD_TARGET}/release/libcrypteia.so "./build/${LIB}" +cp "./target/${CRYPTEIA_BUILD_TARGET}/release/libcrypteia.so" "./build/${LIB}" cd ./build -strip "$BIN" +if [ "${CRYPTEIA_BUILD_TARGET}" = "aarch64-unknown-linux-gnu" ]; then + strip "$BIN" +else + x86_64-linux-gnu-strip "$BIN" +fi chmod +x "$BIN" zip -r "${BIN}.zip" "$BIN" zip -r "libcrypteia-${CRYPTEIA_BUILD_OS}${CRYPTEIA_BUILD_SUFFIX}.zip" "$LIB" diff --git a/bin/test b/bin/test index d08ba23..974f469 100755 --- a/bin/test +++ b/bin/test @@ -3,7 +3,23 @@ set -e # Sync with bin/build, bin/test, & test/libcrypteia.sh. export CRYPTEIA_BUILD_OS="${CRYPTEIA_BUILD_OS:=debian}" -export CRYPTEIA_BUILD_TARGET="${CRYPTEIA_BUILD_TARGET:=x86_64-unknown-linux-gnu}" + +# Auto-detect build target if not already set +if [ -z "${CRYPTEIA_BUILD_TARGET}" ]; then + case "$(uname -m)" in + aarch64) + export CRYPTEIA_BUILD_TARGET="aarch64-unknown-linux-gnu" + ;; + x86_64) + export CRYPTEIA_BUILD_TARGET="x86_64-unknown-linux-gnu" + ;; + *) + echo "Unsupported architecture: $(uname -m)" + exit 1 + ;; + esac +fi + if [ "${CRYPTEIA_BUILD_TARGET}" = "aarch64-unknown-linux-gnu" ]; then export CRYPTEIA_BUILD_SUFFIX="-arm64" fi @@ -12,9 +28,7 @@ if [ ! "${SKIP_CARGO_TEST}" = "1" ]; then cargo test --target "${CRYPTEIA_BUILD_TARGET}" --quiet fi -if [ ! "${CRYPTEIA_BUILD_TARGET}" = "aarch64-unknown-linux-gnu" ]; then - TEST_LANG=node ./test/libcrypteia.sh - TEST_LANG=ruby ./test/libcrypteia.sh - TEST_LANG=php ./test/libcrypteia.sh - TEST_LANG=python ./test/libcrypteia.sh -fi +TEST_LANG=node ./test/libcrypteia.sh +TEST_LANG=ruby ./test/libcrypteia.sh +TEST_LANG=php ./test/libcrypteia.sh +TEST_LANG=python ./test/libcrypteia.sh From e24a86396ce37a0601c7dc7fa2965184a06f5228 Mon Sep 17 00:00:00 2001 From: Brian Carp Date: Fri, 14 Nov 2025 03:08:34 -0500 Subject: [PATCH 02/11] Refactor to improve portability and robustness This commit implements a small refactor to make the dev container setup more resilient and truly multi-platform. - Installs `aarch64` cross-compilation packages (`gcc-aarch64-linux-gnu`, `libc6-dev-arm64-cross`) in the `Dockerfile` to enable building for ARM64 on x86_64 hosts. - Updates `bin/build-arch` to use the correct `strip` binary (native or cross-compile) by checking both the host and target architectures. - Adds a 30-second timeout to the `postCreate` script to prevent it from hanging if the Docker daemon fails to start. - Adds a comment to `bin/test` clarifying why language runtime tests are now enabled for all architectures. - Merges the `update-alternatives` command into the main `RUN` layer, reducing the total number of image layers. --- .devcontainer/Dockerfile | 18 ++++++++++++++---- .devcontainer/postCreate | 6 ++++++ bin/build-arch | 24 +++++++++++++++++++----- bin/test | 2 ++ 4 files changed, 41 insertions(+), 9 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 946a67c..589ae4e 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -3,22 +3,32 @@ FROM mcr.microsoft.com/devcontainers/rust:2-1-bookworm RUN apt-get update -y \ && apt-get upgrade -y \ && apt-get install -y --fix-missing --no-install-recommends \ + \ + # Zip for packaging zip \ + \ + # QEMU for multi-architecture support qemu-system \ binfmt-support \ qemu-user-static \ + \ + # Language runtimes for tests nodejs \ ruby \ php \ php-common \ python3-pip \ + \ + # Cross-compilation toolchains gcc-x86-64-linux-gnu \ libc6-dev-amd64-cross \ + gcc-aarch64-linux-gnu \ + libc6-dev-arm64-cross \ + \ + # Clean up, enable QEMU, and set Python alternative in a single layer && rm -rf /var/lib/apt/lists/* \ - && update-binfmts --enable qemu-aarch64 - -# Easy way to install Python. -RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 + && update-binfmts --enable qemu-aarch64 \ + && update-alternatives --install /usr/bin/python python /usr/bin/python3 1 # Switch to root to install rust targets and fix permissions USER root diff --git a/.devcontainer/postCreate b/.devcontainer/postCreate index 4b0762a..b84146d 100755 --- a/.devcontainer/postCreate +++ b/.devcontainer/postCreate @@ -2,7 +2,13 @@ set -e # Wait for docker to be ready +TIMEOUT=30 while ! docker info > /dev/null 2>&1; do echo "Waiting for docker daemon..." sleep 1 + TIMEOUT=$((TIMEOUT - 1)) + if [ $TIMEOUT -le 0 ]; then + echo "Docker daemon failed to start" + exit 1 + fi done diff --git a/bin/build-arch b/bin/build-arch index fe5a1c7..3b5a868 100755 --- a/bin/build-arch +++ b/bin/build-arch @@ -19,11 +19,25 @@ cp "./target/${CRYPTEIA_BUILD_TARGET}/release/crypteia" "./build/${BIN}" cp "./target/${CRYPTEIA_BUILD_TARGET}/release/libcrypteia.so" "./build/${LIB}" cd ./build -if [ "${CRYPTEIA_BUILD_TARGET}" = "aarch64-unknown-linux-gnu" ]; then - strip "$BIN" -else - x86_64-linux-gnu-strip "$BIN" -fi + +# Use appropriate strip command based on target +case "${CRYPTEIA_BUILD_TARGET}" in + aarch64-unknown-linux-gnu) + if [ "$(uname -m)" = "aarch64" ]; then + strip "$BIN" + else + aarch64-linux-gnu-strip "$BIN" 2>/dev/null || strip "$BIN" + fi + ;; + x86_64-unknown-linux-gnu) + if [ "$(uname -m)" = "x86_64" ]; then + strip "$BIN" + else + x86_64-linux-gnu-strip "$BIN" 2>/dev/null || strip "$BIN" + fi + ;; +esac + chmod +x "$BIN" zip -r "${BIN}.zip" "$BIN" zip -r "libcrypteia-${CRYPTEIA_BUILD_OS}${CRYPTEIA_BUILD_SUFFIX}.zip" "$LIB" diff --git a/bin/test b/bin/test index 974f469..a269cc5 100755 --- a/bin/test +++ b/bin/test @@ -28,6 +28,8 @@ if [ ! "${SKIP_CARGO_TEST}" = "1" ]; then cargo test --target "${CRYPTEIA_BUILD_TARGET}" --quiet fi +# Language runtime tests now work on both architectures thanks to proper +# LD_PRELOAD path matching (CRYPTEIA_BUILD_SUFFIX) and installed runtimes TEST_LANG=node ./test/libcrypteia.sh TEST_LANG=ruby ./test/libcrypteia.sh TEST_LANG=php ./test/libcrypteia.sh From 2189b08ba584c74d644de27d6461b82166eb5989 Mon Sep 17 00:00:00 2001 From: Brian Carp Date: Fri, 14 Nov 2025 03:15:30 -0500 Subject: [PATCH 03/11] Update `Test` workflow to use newer ubuntu runner image --- .github/workflows/test.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8a8b1f7..05d9357 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,7 +3,7 @@ on: [push, workflow_dispatch] jobs: image: name: Image - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - uses: docker/login-action@v2 @@ -19,7 +19,7 @@ jobs: runCmd: echo DONE! debian-x86-64: name: Debian x86_64 - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: image steps: - name: Checkout @@ -34,7 +34,7 @@ jobs: ./bin/test-local debian-arm64: name: Debian arm64 - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: image steps: - name: Checkout @@ -49,7 +49,7 @@ jobs: ./debian/test-arm64 amazon-x86-64: name: AmazonLinux2/x86_64 - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: image steps: - name: Checkout @@ -64,7 +64,7 @@ jobs: ./amzn/test amazon-arm64: name: AmazonLinux2 arm64 - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: image steps: - name: Checkout @@ -79,7 +79,7 @@ jobs: ./amzn/test-arm64 ubuntu-py27: name: Ubuntu x86_64 (Python27) - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: image steps: - name: Checkout From 40f350dcc2f5e6dcecb3b5c55363331b8d7e7f2c Mon Sep 17 00:00:00 2001 From: Brian Carp Date: Fri, 14 Nov 2025 03:37:43 -0500 Subject: [PATCH 04/11] Modernize Test workflow and Dockerfile images This commit updates the CI configuration to resolve build failures and align the test environments with modern, supported versions. - Replaces deprecated `ubuntu-20.04` runners with `ubuntu-22.04` in the GitHub Actions workflow, fixing the hanging jobs. - Adds QEMU and Docker Buildx to `arm64` jobs to enable cross-platform image builds. - Upgrades the Debian test environment from a Bullseye-based image to a Bookworm-based one, and updates Node.js from v18 to v22 (LTS). - Updates the Python 2.7 test environment to use an `ubuntu:22.04` base image and installs Python 2.7 via the `deadsnakes` PPA. --- .github/workflows/test.yml | 8 ++++++++ debian/Dockerfile-arm64 | 4 ++-- py27/Dockerfile-test | 11 +++++++---- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 05d9357..d837827 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,6 +39,10 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - name: Test uses: devcontainers/ci@v0.2 with: @@ -69,6 +73,10 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - name: Test uses: devcontainers/ci@v0.2 with: diff --git a/debian/Dockerfile-arm64 b/debian/Dockerfile-arm64 index cce64d5..7bffa13 100644 --- a/debian/Dockerfile-arm64 +++ b/debian/Dockerfile-arm64 @@ -1,6 +1,6 @@ -FROM mcr.microsoft.com/devcontainers/typescript-node:18 +FROM mcr.microsoft.com/devcontainers/typescript-node:22-bookworm -RUN apt update && apt-get install -y python3-pip +RUN apt update && apt-get install -y python3-pip && rm -rf /var/lib/apt/lists/* RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 RUN update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 diff --git a/py27/Dockerfile-test b/py27/Dockerfile-test index 779b163..b7e292a 100644 --- a/py27/Dockerfile-test +++ b/py27/Dockerfile-test @@ -1,10 +1,13 @@ -FROM ubuntu:20.04 +FROM ubuntu:22.04 ENV SHELL=/bin/sh -RUN apt update \ - && apt-get install -y curl \ - && apt-get install -y python2.7 python2.7-dev \ +RUN apt-get update && apt-get install -y software-properties-common && add-apt-repository ppa:deadsnakes/ppa && apt-get update +RUN apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + python2.7 \ + python2.7-dev \ && update-alternatives --install /usr/bin/python python /usr/bin/python2.7 2 RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py \ From f5351bef4f79f894b5e19151f76487eff1294caa Mon Sep 17 00:00:00 2001 From: Brian Carp Date: Fri, 14 Nov 2025 03:42:43 -0500 Subject: [PATCH 05/11] Add test environment for Amazon Linux 2023 --- .github/workflows/test.yml | 34 ++++++++++++++++++++++++++++++++++ amzn2023/Dockerfile | 14 ++++++++++++++ amzn2023/Dockerfile-arm64 | 14 ++++++++++++++ amzn2023/Dockerfile-test | 14 ++++++++++++++ amzn2023/setup | 13 +++++++++++++ amzn2023/setup-arm64 | 14 ++++++++++++++ amzn2023/test | 23 +++++++++++++++++++++++ amzn2023/test-arm64 | 24 ++++++++++++++++++++++++ 8 files changed, 150 insertions(+) create mode 100644 amzn2023/Dockerfile create mode 100644 amzn2023/Dockerfile-arm64 create mode 100644 amzn2023/Dockerfile-test create mode 100644 amzn2023/setup create mode 100644 amzn2023/setup-arm64 create mode 100644 amzn2023/test create mode 100644 amzn2023/test-arm64 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d837827..a6b963e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -100,3 +100,37 @@ jobs: runCmd: | ./py27/setup ./py27/test + amazonlinux2023-x86-64: + name: AmazonLinux2023/x86_64 + runs-on: ubuntu-22.04 + needs: image + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Test + uses: devcontainers/ci@v0.2 + with: + push: never + cacheFrom: ghcr.io/rails-lambda/crypteia-ci + runCmd: | + ./amzn2023/setup + ./amzn2023/test + amazonlinux2023-arm64: + name: AmazonLinux2023 arm64 + runs-on: ubuntu-22.04 + needs: image + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Test + uses: devcontainers/ci@v0.2 + with: + push: never + cacheFrom: ghcr.io/rails-lambda/crypteia-ci + runCmd: | + ./amzn2023/setup-arm64 + ./amzn2023/test-arm64 diff --git a/amzn2023/Dockerfile b/amzn2023/Dockerfile new file mode 100644 index 0000000..14e030e --- /dev/null +++ b/amzn2023/Dockerfile @@ -0,0 +1,14 @@ +FROM public.ecr.aws/amazonlinux/amazonlinux:2023 + +RUN dnf install -y gcc openssl-devel python3-pip +RUN dnf clean all + +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" + +RUN /root/.cargo/bin/rustup update \ + && /root/.cargo/bin/rustup target add aarch64-unknown-linux-gnu +RUN rustup default stable + +ENV CRYPTEIA_BUILD_OS=amzn +ENV CRYPTEIA_BUILD_TARGET=x86_64-unknown-linux-gnu diff --git a/amzn2023/Dockerfile-arm64 b/amzn2023/Dockerfile-arm64 new file mode 100644 index 0000000..f95c525 --- /dev/null +++ b/amzn2023/Dockerfile-arm64 @@ -0,0 +1,14 @@ +FROM public.ecr.aws/amazonlinux/amazonlinux:2023-arm64 + +RUN dnf install -y gcc openssl-devel python3-pip +RUN dnf clean all + +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" + +RUN /root/.cargo/bin/rustup update \ + && /root/.cargo/bin/rustup target add aarch64-unknown-linux-gnu +RUN rustup default stable + +ENV CRYPTEIA_BUILD_OS=amzn +ENV CRYPTEIA_BUILD_TARGET=aarch64-unknown-linux-gnu diff --git a/amzn2023/Dockerfile-test b/amzn2023/Dockerfile-test new file mode 100644 index 0000000..0ad9615 --- /dev/null +++ b/amzn2023/Dockerfile-test @@ -0,0 +1,14 @@ +FROM public.ecr.aws/lambda/provided:al2023 + +COPY build/crypteia-amzn /opt/extensions/crypteia +COPY build/libcrypteia-amzn.so /opt/lib/libcrypteia.so + +ENV CRYPTEIA_BUILD_OS=amzn +ENV SKIP_CARGO_TEST=1 + +ENV EXISTING=existingvalue +ENV LD_PRELOAD=/opt/lib/libcrypteia.so + +# For assert.sh support +RUN dnf install -y util-linux +RUN dnf clean all diff --git a/amzn2023/setup b/amzn2023/setup new file mode 100644 index 0000000..7204164 --- /dev/null +++ b/amzn2023/setup @@ -0,0 +1,13 @@ +#!/bin/sh +set -e + +echo "== [amzn2023/Dockerfile] building... ==" +docker build --tag crypteia-lambda-amzn2023 --file amzn2023/Dockerfile . + +echo "== [amzn2023/Dockerfile] bin/setup ==" +docker run \ + --rm \ + --user root \ + --entrypoint "./bin/setup" \ + --volume "${PWD}:/var/task" \ + crypteia-lambda-amzn2023 diff --git a/amzn2023/setup-arm64 b/amzn2023/setup-arm64 new file mode 100644 index 0000000..b43fe70 --- /dev/null +++ b/amzn2023/setup-arm64 @@ -0,0 +1,14 @@ +#!/bin/sh +set -e + +echo "== [amzn2023/Dockerfile-arm64] building... ==" +docker build --tag crypteia-lambda-amzn2023-arm64 --file amzn2023/Dockerfile-arm64 . + +echo "== [amzn2023/Dockerfile-arm64] bin/setup ==" +docker run \ + --rm \ + --user root \ + --entrypoint "./bin/setup" \ + --volume "${PWD}:/var/task" \ + --platform=linux/arm64 \ + crypteia-lambda-amzn2023-arm64 diff --git a/amzn2023/test b/amzn2023/test new file mode 100644 index 0000000..c932142 --- /dev/null +++ b/amzn2023/test @@ -0,0 +1,23 @@ +#!/bin/sh +set -e + +echo "== [amzn2023/Dockerfile] bin/test ==" +docker run \ + --rm \ + --user root \ + --entrypoint "./test/libcrypteia.sh" \ + --volume "${PWD}:/var/task" \ + --env TEST_LANG=node \ + crypteia-lambda-amzn2023 + +echo "== [amzn2023/Dockerfile-test] lambda/runtime building... ==" +docker build --tag crypteia-lambda-amzn2023-test --file amzn2023/Dockerfile-test . + +echo "== [amzn2023/Dockerfile-test] lambda/runtime bin/test ==" +docker run \ + --rm \ + --user root \ + --entrypoint "./test/libcrypteia.sh" \ + --volume "${PWD}:/var/task" \ + --env TEST_LANG=node \ + crypteia-lambda-amzn2023-test diff --git a/amzn2023/test-arm64 b/amzn2023/test-arm64 new file mode 100644 index 0000000..4e7ce15 --- /dev/null +++ b/amzn2023/test-arm64 @@ -0,0 +1,24 @@ +#!/bin/sh +set -e + +echo "== [amzn2023/Dockerfile-arm64] bin/test ==" +docker run \ + --rm \ + --user root \ + --entrypoint "./test/libcrypteia.sh" \ + --volume "${PWD}:/var/task" \ + --env TEST_LANG=node \ + --platform=linux/arm64 \ + crypteia-lambda-amzn202V3-arm64 + +echo "== [amzn2023/Dockerfile-test-arm64] lambda/runtime building... ==" +docker build --tag crypteia-lambda-amzn2023-test-arm64 --file amzn2023/Dockerfile-test-arm64 . + +echo "== [amzn2023/Dockerfile-test-arm64] lambda/runtime bin/test ==" +docker run \ + --rm \ + --user root \ + --entrypoint "./test/libcrypteia.sh" \ + --volume "${PWD}:/var/task" \ + --env TEST_LANG=node \ + crypteia-lambda-amzn2023-test-arm64 From dbb6c5502b8f52af43222b26971a27460cf242ca Mon Sep 17 00:00:00 2001 From: Brian Carp Date: Fri, 14 Nov 2025 08:36:36 -0500 Subject: [PATCH 06/11] Fix permissions/dependencies in tests --- amzn2023/setup | 0 amzn2023/setup-arm64 | 0 amzn2023/test | 0 amzn2023/test-arm64 | 0 py27/Dockerfile-test | 1 + 5 files changed, 1 insertion(+) mode change 100644 => 100755 amzn2023/setup mode change 100644 => 100755 amzn2023/setup-arm64 mode change 100644 => 100755 amzn2023/test mode change 100644 => 100755 amzn2023/test-arm64 diff --git a/amzn2023/setup b/amzn2023/setup old mode 100644 new mode 100755 diff --git a/amzn2023/setup-arm64 b/amzn2023/setup-arm64 old mode 100644 new mode 100755 diff --git a/amzn2023/test b/amzn2023/test old mode 100644 new mode 100755 diff --git a/amzn2023/test-arm64 b/amzn2023/test-arm64 old mode 100644 new mode 100755 diff --git a/py27/Dockerfile-test b/py27/Dockerfile-test index b7e292a..619c818 100644 --- a/py27/Dockerfile-test +++ b/py27/Dockerfile-test @@ -6,6 +6,7 @@ RUN apt-get update && apt-get install -y software-properties-common && add-apt-r RUN apt-get install -y --no-install-recommends \ ca-certificates \ curl \ + libffi-dev \ python2.7 \ python2.7-dev \ && update-alternatives --install /usr/bin/python python /usr/bin/python2.7 2 From fc98941e30286062e5a02311f0e255b56c3b1246 Mon Sep 17 00:00:00 2001 From: Brian Carp Date: Fri, 14 Nov 2025 08:53:13 -0500 Subject: [PATCH 07/11] Downgrade py27 Ubuntu, fix amzn2023 WORKDIR --- amzn2023/Dockerfile | 1 + amzn2023/Dockerfile-arm64 | 3 ++- amzn2023/setup-arm64 | 2 +- amzn2023/test-arm64 | 2 +- py27/Dockerfile-test | 12 ++++-------- 5 files changed, 9 insertions(+), 11 deletions(-) diff --git a/amzn2023/Dockerfile b/amzn2023/Dockerfile index 14e030e..babf48e 100644 --- a/amzn2023/Dockerfile +++ b/amzn2023/Dockerfile @@ -9,6 +9,7 @@ ENV PATH="/root/.cargo/bin:${PATH}" RUN /root/.cargo/bin/rustup update \ && /root/.cargo/bin/rustup target add aarch64-unknown-linux-gnu RUN rustup default stable +WORKDIR /var/task ENV CRYPTEIA_BUILD_OS=amzn ENV CRYPTEIA_BUILD_TARGET=x86_64-unknown-linux-gnu diff --git a/amzn2023/Dockerfile-arm64 b/amzn2023/Dockerfile-arm64 index f95c525..e575f47 100644 --- a/amzn2023/Dockerfile-arm64 +++ b/amzn2023/Dockerfile-arm64 @@ -1,4 +1,4 @@ -FROM public.ecr.aws/amazonlinux/amazonlinux:2023-arm64 +FROM public.ecr.aws/amazonlinux/amazonlinux:2023 RUN dnf install -y gcc openssl-devel python3-pip RUN dnf clean all @@ -9,6 +9,7 @@ ENV PATH="/root/.cargo/bin:${PATH}" RUN /root/.cargo/bin/rustup update \ && /root/.cargo/bin/rustup target add aarch64-unknown-linux-gnu RUN rustup default stable +WORKDIR /var/task ENV CRYPTEIA_BUILD_OS=amzn ENV CRYPTEIA_BUILD_TARGET=aarch64-unknown-linux-gnu diff --git a/amzn2023/setup-arm64 b/amzn2023/setup-arm64 index b43fe70..33460cc 100755 --- a/amzn2023/setup-arm64 +++ b/amzn2023/setup-arm64 @@ -2,7 +2,7 @@ set -e echo "== [amzn2023/Dockerfile-arm64] building... ==" -docker build --tag crypteia-lambda-amzn2023-arm64 --file amzn2023/Dockerfile-arm64 . +docker build --platform=linux/arm64 --tag crypteia-lambda-amzn2023-arm64 --file amzn2023/Dockerfile-arm64 . echo "== [amzn2023/Dockerfile-arm64] bin/setup ==" docker run \ diff --git a/amzn2023/test-arm64 b/amzn2023/test-arm64 index 4e7ce15..3fb26e3 100755 --- a/amzn2023/test-arm64 +++ b/amzn2023/test-arm64 @@ -9,7 +9,7 @@ docker run \ --volume "${PWD}:/var/task" \ --env TEST_LANG=node \ --platform=linux/arm64 \ - crypteia-lambda-amzn202V3-arm64 + crypteia-lambda-amzn2023-arm64 echo "== [amzn2023/Dockerfile-test-arm64] lambda/runtime building... ==" docker build --tag crypteia-lambda-amzn2023-test-arm64 --file amzn2023/Dockerfile-test-arm64 . diff --git a/py27/Dockerfile-test b/py27/Dockerfile-test index 619c818..779b163 100644 --- a/py27/Dockerfile-test +++ b/py27/Dockerfile-test @@ -1,14 +1,10 @@ -FROM ubuntu:22.04 +FROM ubuntu:20.04 ENV SHELL=/bin/sh -RUN apt-get update && apt-get install -y software-properties-common && add-apt-repository ppa:deadsnakes/ppa && apt-get update -RUN apt-get install -y --no-install-recommends \ - ca-certificates \ - curl \ - libffi-dev \ - python2.7 \ - python2.7-dev \ +RUN apt update \ + && apt-get install -y curl \ + && apt-get install -y python2.7 python2.7-dev \ && update-alternatives --install /usr/bin/python python /usr/bin/python2.7 2 RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py \ From af33155cb1b609da34636d2d904646fdffe2bf14 Mon Sep 17 00:00:00 2001 From: Brian Carp Date: Fri, 14 Nov 2025 16:19:36 -0500 Subject: [PATCH 08/11] Attempt fixes for amzn2023 in Test workflow --- amzn2023/Dockerfile | 17 ++++++++++------- amzn2023/Dockerfile-arm64 | 16 +++++++++------- amzn2023/Dockerfile-test | 5 ++--- amzn2023/Dockerfile-test-arm64 | 14 ++++++++++++++ amzn2023/setup | 1 + amzn2023/setup-arm64 | 4 ++-- amzn2023/test | 1 + bin/test | 2 +- 8 files changed, 40 insertions(+), 20 deletions(-) create mode 100644 amzn2023/Dockerfile-test-arm64 diff --git a/amzn2023/Dockerfile b/amzn2023/Dockerfile index babf48e..0a18451 100644 --- a/amzn2023/Dockerfile +++ b/amzn2023/Dockerfile @@ -1,15 +1,18 @@ FROM public.ecr.aws/amazonlinux/amazonlinux:2023 -RUN dnf install -y gcc openssl-devel python3-pip -RUN dnf clean all - RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y -ENV PATH="/root/.cargo/bin:${PATH}" +ENV PATH="~/.cargo/bin:${PATH}" + +RUN ~/.cargo/bin/rustup update \ + && ~/.cargo/bin/rustup target add aarch64-unknown-linux-gnu -RUN /root/.cargo/bin/rustup update \ - && /root/.cargo/bin/rustup target add aarch64-unknown-linux-gnu RUN rustup default stable -WORKDIR /var/task + +# Install required build dependencies +RUN dnf install -y gcc openssl-devel python3-pip util-linux && \ + pip3 install setuptools && \ + dnf clean all ENV CRYPTEIA_BUILD_OS=amzn ENV CRYPTEIA_BUILD_TARGET=x86_64-unknown-linux-gnu + diff --git a/amzn2023/Dockerfile-arm64 b/amzn2023/Dockerfile-arm64 index e575f47..40a940a 100644 --- a/amzn2023/Dockerfile-arm64 +++ b/amzn2023/Dockerfile-arm64 @@ -1,15 +1,17 @@ FROM public.ecr.aws/amazonlinux/amazonlinux:2023 -RUN dnf install -y gcc openssl-devel python3-pip -RUN dnf clean all - RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y -ENV PATH="/root/.cargo/bin:${PATH}" +ENV PATH="~/.cargo/bin:${PATH}" + +RUN ~/.cargo/bin/rustup update \ + && ~/.cargo/bin/rustup target add aarch64-unknown-linux-gnu -RUN /root/.cargo/bin/rustup update \ - && /root/.cargo/bin/rustup target add aarch64-unknown-linux-gnu RUN rustup default stable -WORKDIR /var/task + +# Install required build dependencies +RUN dnf install -y gcc openssl-devel python3-pip util-linux && \ + pip3 install setuptools && \ + dnf clean all ENV CRYPTEIA_BUILD_OS=amzn ENV CRYPTEIA_BUILD_TARGET=aarch64-unknown-linux-gnu diff --git a/amzn2023/Dockerfile-test b/amzn2023/Dockerfile-test index 0ad9615..9b0e879 100644 --- a/amzn2023/Dockerfile-test +++ b/amzn2023/Dockerfile-test @@ -1,4 +1,4 @@ -FROM public.ecr.aws/lambda/provided:al2023 +FROM public.ecr.aws/lambda/nodejs:20 COPY build/crypteia-amzn /opt/extensions/crypteia COPY build/libcrypteia-amzn.so /opt/lib/libcrypteia.so @@ -10,5 +10,4 @@ ENV EXISTING=existingvalue ENV LD_PRELOAD=/opt/lib/libcrypteia.so # For assert.sh support -RUN dnf install -y util-linux -RUN dnf clean all +RUN dnf install -y util-linux && dnf clean all diff --git a/amzn2023/Dockerfile-test-arm64 b/amzn2023/Dockerfile-test-arm64 new file mode 100644 index 0000000..698ef08 --- /dev/null +++ b/amzn2023/Dockerfile-test-arm64 @@ -0,0 +1,14 @@ +FROM public.ecr.aws/lambda/nodejs:20-arm64 + +COPY build/crypteia-amzn-arm64 /opt/extensions/crypteia +COPY build/libcrypteia-amzn-arm64.so /opt/lib/libcrypteia.so + +ENV CRYPTEIA_BUILD_OS=amzn +ENV SKIP_CARGO_TEST=1 + +ENV EXISTING=existingvalue +ENV LD_PRELOAD=/opt/lib/libcrypteia.so + +# For assert.sh support +RUN dnf install -y util-linux && dnf clean all + diff --git a/amzn2023/setup b/amzn2023/setup index 7204164..28c6f3d 100755 --- a/amzn2023/setup +++ b/amzn2023/setup @@ -11,3 +11,4 @@ docker run \ --entrypoint "./bin/setup" \ --volume "${PWD}:/var/task" \ crypteia-lambda-amzn2023 + diff --git a/amzn2023/setup-arm64 b/amzn2023/setup-arm64 index 33460cc..0d2c0ac 100755 --- a/amzn2023/setup-arm64 +++ b/amzn2023/setup-arm64 @@ -2,13 +2,13 @@ set -e echo "== [amzn2023/Dockerfile-arm64] building... ==" -docker build --platform=linux/arm64 --tag crypteia-lambda-amzn2023-arm64 --file amzn2023/Dockerfile-arm64 . +docker build --platform linux/arm64 --tag crypteia-lambda-amzn2023-arm64 --file amzn2023/Dockerfile-arm64 . echo "== [amzn2023/Dockerfile-arm64] bin/setup ==" docker run \ + --platform linux/arm64 \ --rm \ --user root \ --entrypoint "./bin/setup" \ --volume "${PWD}:/var/task" \ - --platform=linux/arm64 \ crypteia-lambda-amzn2023-arm64 diff --git a/amzn2023/test b/amzn2023/test index c932142..6a60c82 100755 --- a/amzn2023/test +++ b/amzn2023/test @@ -21,3 +21,4 @@ docker run \ --volume "${PWD}:/var/task" \ --env TEST_LANG=node \ crypteia-lambda-amzn2023-test + diff --git a/bin/test b/bin/test index a269cc5..6ede9ab 100755 --- a/bin/test +++ b/bin/test @@ -14,7 +14,7 @@ if [ -z "${CRYPTEIA_BUILD_TARGET}" ]; then export CRYPTEIA_BUILD_TARGET="x86_64-unknown-linux-gnu" ;; *) - echo "Unsupported architecture: $(uname -m)" + echo "Unsupported architecture: $(uname -m)" >&2 exit 1 ;; esac From 805922e6a308d97f885d1923c83fa46a3e6fa223 Mon Sep 17 00:00:00 2001 From: Brian Carp Date: Fri, 14 Nov 2025 16:29:53 -0500 Subject: [PATCH 09/11] Fix py27 in Test workflow --- py27/Dockerfile | 25 +++++++++++++++++++++++++ py27/Dockerfile-test | 4 +++- py27/setup | 11 ++++++++++- 3 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 py27/Dockerfile diff --git a/py27/Dockerfile b/py27/Dockerfile new file mode 100644 index 0000000..29ba187 --- /dev/null +++ b/py27/Dockerfile @@ -0,0 +1,25 @@ +FROM ubuntu:20.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt update && apt-get install -y \ + curl \ + gcc \ + libssl-dev \ + python3-pip \ + pkg-config \ + && rm -rf /var/lib/apt/lists/* + +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" + +RUN /root/.cargo/bin/rustup update \ + && /root/.cargo/bin/rustup target add aarch64-unknown-linux-gnu + +RUN rustup default stable + +WORKDIR /var/task + +ENV CRYPTEIA_BUILD_OS=debian +ENV CRYPTEIA_BUILD_TARGET=x86_64-unknown-linux-gnu + diff --git a/py27/Dockerfile-test b/py27/Dockerfile-test index 779b163..c3628ba 100644 --- a/py27/Dockerfile-test +++ b/py27/Dockerfile-test @@ -13,7 +13,9 @@ RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -COPY ./package/opt /opt +COPY build/crypteia-debian /opt/extensions/crypteia +COPY build/libcrypteia-debian.so /opt/lib/libcrypteia.so +COPY package/opt/crypteia /opt/crypteia ENV CRYPTEIA_BUILD_OS=debian ENV SKIP_CARGO_TEST=1 diff --git a/py27/setup b/py27/setup index 29c823e..5b22893 100755 --- a/py27/setup +++ b/py27/setup @@ -1,7 +1,16 @@ #!/bin/sh set -e -./bin/setup +echo "== [py27/Dockerfile] building... ==" +docker build --tag crypteia-debian-py27 --file py27/Dockerfile . + +echo "== [py27/Dockerfile] bin/setup ==" +docker run \ + --rm \ + --user root \ + --entrypoint "./bin/setup" \ + --volume "${PWD}:/var/task" \ + crypteia-debian-py27 echo "== [py27/Dockerfile-test] building... ==" docker build --tag crypteia-debian-py27-test --file py27/Dockerfile-test . From 7eb7d352764b2532b119ce97610eb9c90d7d597c Mon Sep 17 00:00:00 2001 From: Brian Carp Date: Fri, 14 Nov 2025 16:35:52 -0500 Subject: [PATCH 10/11] Use absolute path instead of tilde --- amzn2023/Dockerfile | 15 +++++++-------- amzn2023/Dockerfile-arm64 | 15 +++++++-------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/amzn2023/Dockerfile b/amzn2023/Dockerfile index 0a18451..432ebcc 100644 --- a/amzn2023/Dockerfile +++ b/amzn2023/Dockerfile @@ -1,18 +1,17 @@ FROM public.ecr.aws/amazonlinux/amazonlinux:2023 -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y -ENV PATH="~/.cargo/bin:${PATH}" - -RUN ~/.cargo/bin/rustup update \ - && ~/.cargo/bin/rustup target add aarch64-unknown-linux-gnu - -RUN rustup default stable - # Install required build dependencies RUN dnf install -y gcc openssl-devel python3-pip util-linux && \ pip3 install setuptools && \ dnf clean all +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" + +RUN /root/.cargo/bin/rustup update \ + && /root/.cargo/bin/rustup target add aarch64-unknown-linux-gnu \ + && /root/.cargo/bin/rustup default stable + ENV CRYPTEIA_BUILD_OS=amzn ENV CRYPTEIA_BUILD_TARGET=x86_64-unknown-linux-gnu diff --git a/amzn2023/Dockerfile-arm64 b/amzn2023/Dockerfile-arm64 index 40a940a..6267f07 100644 --- a/amzn2023/Dockerfile-arm64 +++ b/amzn2023/Dockerfile-arm64 @@ -1,17 +1,16 @@ FROM public.ecr.aws/amazonlinux/amazonlinux:2023 -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y -ENV PATH="~/.cargo/bin:${PATH}" - -RUN ~/.cargo/bin/rustup update \ - && ~/.cargo/bin/rustup target add aarch64-unknown-linux-gnu - -RUN rustup default stable - # Install required build dependencies RUN dnf install -y gcc openssl-devel python3-pip util-linux && \ pip3 install setuptools && \ dnf clean all +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" + +RUN /root/.cargo/bin/rustup update \ + && /root/.cargo/bin/rustup target add aarch64-unknown-linux-gnu \ + && /root/.cargo/bin/rustup default stable + ENV CRYPTEIA_BUILD_OS=amzn ENV CRYPTEIA_BUILD_TARGET=aarch64-unknown-linux-gnu From fe5e64f097f5a6c862f6e94595ade0b31cec225c Mon Sep 17 00:00:00 2001 From: Brian Carp Date: Fri, 14 Nov 2025 17:04:45 -0500 Subject: [PATCH 11/11] Set WORKDIR for amzn2023, zip for py27 --- amzn2023/Dockerfile | 2 ++ amzn2023/Dockerfile-arm64 | 2 ++ py27/Dockerfile | 1 + 3 files changed, 5 insertions(+) diff --git a/amzn2023/Dockerfile b/amzn2023/Dockerfile index 432ebcc..b8f8613 100644 --- a/amzn2023/Dockerfile +++ b/amzn2023/Dockerfile @@ -12,6 +12,8 @@ RUN /root/.cargo/bin/rustup update \ && /root/.cargo/bin/rustup target add aarch64-unknown-linux-gnu \ && /root/.cargo/bin/rustup default stable +WORKDIR /var/task + ENV CRYPTEIA_BUILD_OS=amzn ENV CRYPTEIA_BUILD_TARGET=x86_64-unknown-linux-gnu diff --git a/amzn2023/Dockerfile-arm64 b/amzn2023/Dockerfile-arm64 index 6267f07..d512705 100644 --- a/amzn2023/Dockerfile-arm64 +++ b/amzn2023/Dockerfile-arm64 @@ -12,5 +12,7 @@ RUN /root/.cargo/bin/rustup update \ && /root/.cargo/bin/rustup target add aarch64-unknown-linux-gnu \ && /root/.cargo/bin/rustup default stable +WORKDIR /var/task + ENV CRYPTEIA_BUILD_OS=amzn ENV CRYPTEIA_BUILD_TARGET=aarch64-unknown-linux-gnu diff --git a/py27/Dockerfile b/py27/Dockerfile index 29ba187..ad45f5a 100644 --- a/py27/Dockerfile +++ b/py27/Dockerfile @@ -8,6 +8,7 @@ RUN apt update && apt-get install -y \ libssl-dev \ python3-pip \ pkg-config \ + zip \ && rm -rf /var/lib/apt/lists/* RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y