diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 3fda7a4..589ae4e 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,30 +1,49 @@ -FROM mcr.microsoft.com/devcontainers/rust:1-1-bullseye - -RUN sudo apt-get update -y \ - && sudo apt-get upgrade -y - -RUN sudo apt-get install -y --fix-missing zip - -RUN sudo apt-get update -y \ - && sudo apt-get upgrade -y \ - && sudo apt-get install -y zip ltrace - -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y -RUN rustup update \ - && rustup target add aarch64-unknown-linux-gnu - -RUN rustup default stable -# x86_64 to arm64 support. -RUN sudo apt-get install -y \ - qemu \ +FROM mcr.microsoft.com/devcontainers/rust:2-1-bookworm + +RUN apt-get update -y \ + && apt-get upgrade -y \ + && apt-get install -y --fix-missing --no-install-recommends \ + \ + # Zip for packaging + zip \ + \ + # QEMU for multi-architecture support + qemu-system \ binfmt-support \ - qemu-user-static + qemu-user-static \ + \ + # Language runtimes for tests + nodejs \ + ruby \ + php \ + php-common \ + python3-pip \ + \ + # Cross-compilation toolchains + gcc-x86-64-linux-gnu \ + libc6-dev-amd64-cross \ + gcc-aarch64-linux-gnu \ + libc6-dev-arm64-cross \ + \ + # Clean up, enable QEMU, and set Python alternative in a single layer + && rm -rf /var/lib/apt/lists/* \ + && update-binfmts --enable qemu-aarch64 \ + && update-alternatives --install /usr/bin/python python /usr/bin/python3 1 + +# Switch to root to install rust targets and fix permissions +USER root + +# Install rust targets for cross-compilation +RUN rustup update \ + && rustup default stable \ + && rustup target add aarch64-unknown-linux-gnu \ + && rustup target add x86_64-unknown-linux-gnu -# Easy way to install node, ruby, and php -RUN apt-get -y install nodejs ruby php php-common +# Grant vscode user ownership of rustup and cargo directories +RUN chown -R vscode:vscode /usr/local/rustup /usr/local/cargo -# Easy way to install Python. -RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 +# Switch back to vscode user +USER vscode # Multi-platform SAM CLI. https://github.com/aws/aws-sam-cli/issues/3908 -RUN apt-get install -y pip && pip install aws-sam-cli +RUN pip install aws-sam-cli --break-system-packages diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 55620a3..1893e8b 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -5,7 +5,9 @@ }, "features": { "ghcr.io/devcontainers/features/aws-cli:latest": {}, - "ghcr.io/devcontainers/features/docker-in-docker:latest": {}, + "ghcr.io/devcontainers/features/docker-in-docker:latest": { + "install-qemu": false + }, "ghcr.io/customink/codespaces-features/docker-log-level": {}, "ghcr.io/devcontainers/features/sshd:latest": {} }, diff --git a/.devcontainer/postCreate b/.devcontainer/postCreate index 1415bf0..b84146d 100755 --- a/.devcontainer/postCreate +++ b/.devcontainer/postCreate @@ -1,8 +1,14 @@ #!/bin/sh set -e -docker run \ - --rm \ - --privileged \ - multiarch/qemu-user-static \ - --reset -p yes +# Wait for docker to be ready +TIMEOUT=30 +while ! docker info > /dev/null 2>&1; do + echo "Waiting for docker daemon..." + sleep 1 + TIMEOUT=$((TIMEOUT - 1)) + if [ $TIMEOUT -le 0 ]; then + echo "Docker daemon failed to start" + exit 1 + fi +done diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8a8b1f7..a6b963e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,7 +3,7 @@ on: [push, workflow_dispatch] jobs: image: name: Image - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - uses: docker/login-action@v2 @@ -19,7 +19,7 @@ jobs: runCmd: echo DONE! debian-x86-64: name: Debian x86_64 - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: image steps: - name: Checkout @@ -34,11 +34,15 @@ jobs: ./bin/test-local debian-arm64: name: Debian arm64 - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: image steps: - name: Checkout uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - name: Test uses: devcontainers/ci@v0.2 with: @@ -49,7 +53,7 @@ jobs: ./debian/test-arm64 amazon-x86-64: name: AmazonLinux2/x86_64 - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: image steps: - name: Checkout @@ -64,11 +68,15 @@ jobs: ./amzn/test amazon-arm64: name: AmazonLinux2 arm64 - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: image steps: - name: Checkout uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - name: Test uses: devcontainers/ci@v0.2 with: @@ -79,7 +87,7 @@ jobs: ./amzn/test-arm64 ubuntu-py27: name: Ubuntu x86_64 (Python27) - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: image steps: - name: Checkout @@ -92,3 +100,37 @@ jobs: runCmd: | ./py27/setup ./py27/test + amazonlinux2023-x86-64: + name: AmazonLinux2023/x86_64 + runs-on: ubuntu-22.04 + needs: image + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Test + uses: devcontainers/ci@v0.2 + with: + push: never + cacheFrom: ghcr.io/rails-lambda/crypteia-ci + runCmd: | + ./amzn2023/setup + ./amzn2023/test + amazonlinux2023-arm64: + name: AmazonLinux2023 arm64 + runs-on: ubuntu-22.04 + needs: image + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Test + uses: devcontainers/ci@v0.2 + with: + push: never + cacheFrom: ghcr.io/rails-lambda/crypteia-ci + runCmd: | + ./amzn2023/setup-arm64 + ./amzn2023/test-arm64 diff --git a/amzn2023/Dockerfile b/amzn2023/Dockerfile new file mode 100644 index 0000000..b8f8613 --- /dev/null +++ b/amzn2023/Dockerfile @@ -0,0 +1,19 @@ +FROM public.ecr.aws/amazonlinux/amazonlinux:2023 + +# Install required build dependencies +RUN dnf install -y gcc openssl-devel python3-pip util-linux && \ + pip3 install setuptools && \ + dnf clean all + +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" + +RUN /root/.cargo/bin/rustup update \ + && /root/.cargo/bin/rustup target add aarch64-unknown-linux-gnu \ + && /root/.cargo/bin/rustup default stable + +WORKDIR /var/task + +ENV CRYPTEIA_BUILD_OS=amzn +ENV CRYPTEIA_BUILD_TARGET=x86_64-unknown-linux-gnu + diff --git a/amzn2023/Dockerfile-arm64 b/amzn2023/Dockerfile-arm64 new file mode 100644 index 0000000..d512705 --- /dev/null +++ b/amzn2023/Dockerfile-arm64 @@ -0,0 +1,18 @@ +FROM public.ecr.aws/amazonlinux/amazonlinux:2023 + +# Install required build dependencies +RUN dnf install -y gcc openssl-devel python3-pip util-linux && \ + pip3 install setuptools && \ + dnf clean all + +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" + +RUN /root/.cargo/bin/rustup update \ + && /root/.cargo/bin/rustup target add aarch64-unknown-linux-gnu \ + && /root/.cargo/bin/rustup default stable + +WORKDIR /var/task + +ENV CRYPTEIA_BUILD_OS=amzn +ENV CRYPTEIA_BUILD_TARGET=aarch64-unknown-linux-gnu diff --git a/amzn2023/Dockerfile-test b/amzn2023/Dockerfile-test new file mode 100644 index 0000000..9b0e879 --- /dev/null +++ b/amzn2023/Dockerfile-test @@ -0,0 +1,13 @@ +FROM public.ecr.aws/lambda/nodejs:20 + +COPY build/crypteia-amzn /opt/extensions/crypteia +COPY build/libcrypteia-amzn.so /opt/lib/libcrypteia.so + +ENV CRYPTEIA_BUILD_OS=amzn +ENV SKIP_CARGO_TEST=1 + +ENV EXISTING=existingvalue +ENV LD_PRELOAD=/opt/lib/libcrypteia.so + +# For assert.sh support +RUN dnf install -y util-linux && dnf clean all diff --git a/amzn2023/Dockerfile-test-arm64 b/amzn2023/Dockerfile-test-arm64 new file mode 100644 index 0000000..698ef08 --- /dev/null +++ b/amzn2023/Dockerfile-test-arm64 @@ -0,0 +1,14 @@ +FROM public.ecr.aws/lambda/nodejs:20-arm64 + +COPY build/crypteia-amzn-arm64 /opt/extensions/crypteia +COPY build/libcrypteia-amzn-arm64.so /opt/lib/libcrypteia.so + +ENV CRYPTEIA_BUILD_OS=amzn +ENV SKIP_CARGO_TEST=1 + +ENV EXISTING=existingvalue +ENV LD_PRELOAD=/opt/lib/libcrypteia.so + +# For assert.sh support +RUN dnf install -y util-linux && dnf clean all + diff --git a/amzn2023/setup b/amzn2023/setup new file mode 100755 index 0000000..28c6f3d --- /dev/null +++ b/amzn2023/setup @@ -0,0 +1,14 @@ +#!/bin/sh +set -e + +echo "== [amzn2023/Dockerfile] building... ==" +docker build --tag crypteia-lambda-amzn2023 --file amzn2023/Dockerfile . + +echo "== [amzn2023/Dockerfile] bin/setup ==" +docker run \ + --rm \ + --user root \ + --entrypoint "./bin/setup" \ + --volume "${PWD}:/var/task" \ + crypteia-lambda-amzn2023 + diff --git a/amzn2023/setup-arm64 b/amzn2023/setup-arm64 new file mode 100755 index 0000000..0d2c0ac --- /dev/null +++ b/amzn2023/setup-arm64 @@ -0,0 +1,14 @@ +#!/bin/sh +set -e + +echo "== [amzn2023/Dockerfile-arm64] building... ==" +docker build --platform linux/arm64 --tag crypteia-lambda-amzn2023-arm64 --file amzn2023/Dockerfile-arm64 . + +echo "== [amzn2023/Dockerfile-arm64] bin/setup ==" +docker run \ + --platform linux/arm64 \ + --rm \ + --user root \ + --entrypoint "./bin/setup" \ + --volume "${PWD}:/var/task" \ + crypteia-lambda-amzn2023-arm64 diff --git a/amzn2023/test b/amzn2023/test new file mode 100755 index 0000000..6a60c82 --- /dev/null +++ b/amzn2023/test @@ -0,0 +1,24 @@ +#!/bin/sh +set -e + +echo "== [amzn2023/Dockerfile] bin/test ==" +docker run \ + --rm \ + --user root \ + --entrypoint "./test/libcrypteia.sh" \ + --volume "${PWD}:/var/task" \ + --env TEST_LANG=node \ + crypteia-lambda-amzn2023 + +echo "== [amzn2023/Dockerfile-test] lambda/runtime building... ==" +docker build --tag crypteia-lambda-amzn2023-test --file amzn2023/Dockerfile-test . + +echo "== [amzn2023/Dockerfile-test] lambda/runtime bin/test ==" +docker run \ + --rm \ + --user root \ + --entrypoint "./test/libcrypteia.sh" \ + --volume "${PWD}:/var/task" \ + --env TEST_LANG=node \ + crypteia-lambda-amzn2023-test + diff --git a/amzn2023/test-arm64 b/amzn2023/test-arm64 new file mode 100755 index 0000000..3fb26e3 --- /dev/null +++ b/amzn2023/test-arm64 @@ -0,0 +1,24 @@ +#!/bin/sh +set -e + +echo "== [amzn2023/Dockerfile-arm64] bin/test ==" +docker run \ + --rm \ + --user root \ + --entrypoint "./test/libcrypteia.sh" \ + --volume "${PWD}:/var/task" \ + --env TEST_LANG=node \ + --platform=linux/arm64 \ + crypteia-lambda-amzn2023-arm64 + +echo "== [amzn2023/Dockerfile-test-arm64] lambda/runtime building... ==" +docker build --tag crypteia-lambda-amzn2023-test-arm64 --file amzn2023/Dockerfile-test-arm64 . + +echo "== [amzn2023/Dockerfile-test-arm64] lambda/runtime bin/test ==" +docker run \ + --rm \ + --user root \ + --entrypoint "./test/libcrypteia.sh" \ + --volume "${PWD}:/var/task" \ + --env TEST_LANG=node \ + crypteia-lambda-amzn2023-test-arm64 diff --git a/bin/build b/bin/build index ec7323c..3debfd3 100755 --- a/bin/build +++ b/bin/build @@ -3,7 +3,23 @@ set -e # Sync with bin/build, bin/test, & test/libcrypteia.sh. export CRYPTEIA_BUILD_OS="${CRYPTEIA_BUILD_OS:=debian}" -export CRYPTEIA_BUILD_TARGET="${CRYPTEIA_BUILD_TARGET:=x86_64-unknown-linux-gnu}" + +# Auto-detect build target if not already set +if [ -z "${CRYPTEIA_BUILD_TARGET}" ]; then + case "$(uname -m)" in + aarch64) + export CRYPTEIA_BUILD_TARGET="aarch64-unknown-linux-gnu" + ;; + x86_64) + export CRYPTEIA_BUILD_TARGET="x86_64-unknown-linux-gnu" + ;; + *) + echo "Unsupported architecture: $(uname -m)" + exit 1 + ;; + esac +fi + if [ "${CRYPTEIA_BUILD_TARGET}" = "aarch64-unknown-linux-gnu" ]; then export CRYPTEIA_BUILD_SUFFIX="-arm64" fi diff --git a/bin/build-arch b/bin/build-arch index 5c79a61..3b5a868 100755 --- a/bin/build-arch +++ b/bin/build-arch @@ -16,10 +16,28 @@ cargo build \ --target "${CRYPTEIA_BUILD_TARGET}" cp "./target/${CRYPTEIA_BUILD_TARGET}/release/crypteia" "./build/${BIN}" -cp ./target/${CRYPTEIA_BUILD_TARGET}/release/libcrypteia.so "./build/${LIB}" +cp "./target/${CRYPTEIA_BUILD_TARGET}/release/libcrypteia.so" "./build/${LIB}" cd ./build -strip "$BIN" + +# Use appropriate strip command based on target +case "${CRYPTEIA_BUILD_TARGET}" in + aarch64-unknown-linux-gnu) + if [ "$(uname -m)" = "aarch64" ]; then + strip "$BIN" + else + aarch64-linux-gnu-strip "$BIN" 2>/dev/null || strip "$BIN" + fi + ;; + x86_64-unknown-linux-gnu) + if [ "$(uname -m)" = "x86_64" ]; then + strip "$BIN" + else + x86_64-linux-gnu-strip "$BIN" 2>/dev/null || strip "$BIN" + fi + ;; +esac + chmod +x "$BIN" zip -r "${BIN}.zip" "$BIN" zip -r "libcrypteia-${CRYPTEIA_BUILD_OS}${CRYPTEIA_BUILD_SUFFIX}.zip" "$LIB" diff --git a/bin/test b/bin/test index d08ba23..6ede9ab 100755 --- a/bin/test +++ b/bin/test @@ -3,7 +3,23 @@ set -e # Sync with bin/build, bin/test, & test/libcrypteia.sh. export CRYPTEIA_BUILD_OS="${CRYPTEIA_BUILD_OS:=debian}" -export CRYPTEIA_BUILD_TARGET="${CRYPTEIA_BUILD_TARGET:=x86_64-unknown-linux-gnu}" + +# Auto-detect build target if not already set +if [ -z "${CRYPTEIA_BUILD_TARGET}" ]; then + case "$(uname -m)" in + aarch64) + export CRYPTEIA_BUILD_TARGET="aarch64-unknown-linux-gnu" + ;; + x86_64) + export CRYPTEIA_BUILD_TARGET="x86_64-unknown-linux-gnu" + ;; + *) + echo "Unsupported architecture: $(uname -m)" >&2 + exit 1 + ;; + esac +fi + if [ "${CRYPTEIA_BUILD_TARGET}" = "aarch64-unknown-linux-gnu" ]; then export CRYPTEIA_BUILD_SUFFIX="-arm64" fi @@ -12,9 +28,9 @@ if [ ! "${SKIP_CARGO_TEST}" = "1" ]; then cargo test --target "${CRYPTEIA_BUILD_TARGET}" --quiet fi -if [ ! "${CRYPTEIA_BUILD_TARGET}" = "aarch64-unknown-linux-gnu" ]; then - TEST_LANG=node ./test/libcrypteia.sh - TEST_LANG=ruby ./test/libcrypteia.sh - TEST_LANG=php ./test/libcrypteia.sh - TEST_LANG=python ./test/libcrypteia.sh -fi +# Language runtime tests now work on both architectures thanks to proper +# LD_PRELOAD path matching (CRYPTEIA_BUILD_SUFFIX) and installed runtimes +TEST_LANG=node ./test/libcrypteia.sh +TEST_LANG=ruby ./test/libcrypteia.sh +TEST_LANG=php ./test/libcrypteia.sh +TEST_LANG=python ./test/libcrypteia.sh diff --git a/debian/Dockerfile-arm64 b/debian/Dockerfile-arm64 index cce64d5..7bffa13 100644 --- a/debian/Dockerfile-arm64 +++ b/debian/Dockerfile-arm64 @@ -1,6 +1,6 @@ -FROM mcr.microsoft.com/devcontainers/typescript-node:18 +FROM mcr.microsoft.com/devcontainers/typescript-node:22-bookworm -RUN apt update && apt-get install -y python3-pip +RUN apt update && apt-get install -y python3-pip && rm -rf /var/lib/apt/lists/* RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 RUN update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 diff --git a/py27/Dockerfile b/py27/Dockerfile new file mode 100644 index 0000000..ad45f5a --- /dev/null +++ b/py27/Dockerfile @@ -0,0 +1,26 @@ +FROM ubuntu:20.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt update && apt-get install -y \ + curl \ + gcc \ + libssl-dev \ + python3-pip \ + pkg-config \ + zip \ + && rm -rf /var/lib/apt/lists/* + +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" + +RUN /root/.cargo/bin/rustup update \ + && /root/.cargo/bin/rustup target add aarch64-unknown-linux-gnu + +RUN rustup default stable + +WORKDIR /var/task + +ENV CRYPTEIA_BUILD_OS=debian +ENV CRYPTEIA_BUILD_TARGET=x86_64-unknown-linux-gnu + diff --git a/py27/Dockerfile-test b/py27/Dockerfile-test index 779b163..c3628ba 100644 --- a/py27/Dockerfile-test +++ b/py27/Dockerfile-test @@ -13,7 +13,9 @@ RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -COPY ./package/opt /opt +COPY build/crypteia-debian /opt/extensions/crypteia +COPY build/libcrypteia-debian.so /opt/lib/libcrypteia.so +COPY package/opt/crypteia /opt/crypteia ENV CRYPTEIA_BUILD_OS=debian ENV SKIP_CARGO_TEST=1 diff --git a/py27/setup b/py27/setup index 29c823e..5b22893 100755 --- a/py27/setup +++ b/py27/setup @@ -1,7 +1,16 @@ #!/bin/sh set -e -./bin/setup +echo "== [py27/Dockerfile] building... ==" +docker build --tag crypteia-debian-py27 --file py27/Dockerfile . + +echo "== [py27/Dockerfile] bin/setup ==" +docker run \ + --rm \ + --user root \ + --entrypoint "./bin/setup" \ + --volume "${PWD}:/var/task" \ + crypteia-debian-py27 echo "== [py27/Dockerfile-test] building... ==" docker build --tag crypteia-debian-py27-test --file py27/Dockerfile-test .