diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 02a400a7..00000000 --- a/Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -FROM ubuntu:20.04 - -MAINTAINER austrin@kattis.com - -ENV DEBIAN_FRONTEND=noninteractive - -RUN apt-get update && \ - apt-get install -y \ - automake \ - g++ \ - git \ - libboost-all-dev \ - libgmp-dev \ - libgmp10 \ - libgmpxx4ldbl \ - openjdk-8-jdk \ - pandoc \ - python3-minimal \ - python-nh3 \ - python3-pip \ - python3-plastex \ - python3-yaml \ - sudo \ - texlive-fonts-recommended \ - texlive-lang-cyrillic \ - texlive-latex-extra \ - texlive-plain-generic \ - tidy \ - vim - -RUN pip3 install git+https://github.com/kattis/problemtools diff --git a/admin/docker/Dockerfile.build b/admin/docker/Dockerfile.build index e7041fb9..a9f68210 100644 --- a/admin/docker/Dockerfile.build +++ b/admin/docker/Dockerfile.build @@ -1,44 +1,24 @@ -# Package for building the problemtools .deb package -# Ends up in the /usr/local/problemtools_build/deb/ directory +# Docker image with all packages needed to build a problemtools .deb # -# Setting build argument PROBLEMTOOLS_VERSION causes a specific -# version of problemtools to be built (default is latest version of -# develop branch on GitHub) +# Not uploaded anywhere, only used locally during building -FROM ubuntu:22.04 - -LABEL maintainer="austrin@kattis.com" +ARG PROBLEMTOOLS_VERSION=develop +FROM problemtools/runreqs:${PROBLEMTOOLS_VERSION} +LABEL maintainer="contact@kattis.com" ENV DEBIAN_FRONTEND=noninteractive -# Install packages needed for build -RUN apt update && \ - apt install -y \ +# Packages required to build and run problemtools +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + rm -f /etc/apt/apt.conf.d/docker-clean \ + apt-get update && apt-get install -y \ automake \ + build-essential \ debhelper \ - dh-python \ + dh-virtualenv \ dpkg-dev \ g++ \ git \ make \ - libboost-regex-dev \ - libgmp-dev \ - libgmp10 \ - libgmpxx4ldbl \ - pandoc \ - python3 \ - python3-pytest \ - python3-setuptools \ - python3-yaml \ - python3-setuptools - -RUN mkdir -p /usr/local/problemtools_build - -WORKDIR /usr/local/problemtools_build -RUN git clone --recursive https://github.com/kattis/problemtools - -ARG PROBLEMTOOLS_VERSION=develop -RUN cd problemtools && git checkout ${PROBLEMTOOLS_VERSION} && make builddeb - -RUN mkdir -p deb -RUN mv kattis-problemtools*.deb deb/ + libboost-regex-dev diff --git a/admin/docker/Dockerfile.full b/admin/docker/Dockerfile.full index 9fb5196a..7d982b65 100644 --- a/admin/docker/Dockerfile.full +++ b/admin/docker/Dockerfile.full @@ -1,33 +1,23 @@ # Full problemtools docker image, containing problemtools and all # supported programming languages. # +# +# Build requirements: +# - The problemtools .deb package must be available from the host file +# system under a file name matching +# artifacts/deb/kattis-problemtools*.deb +# (Version of that .deb file should match the build argument +# PROBLEMTOOLS_VERSION but this is not checked.) ARG PROBLEMTOOLS_VERSION=develop -FROM problemtools/icpc:${PROBLEMTOOLS_VERSION} - -LABEL maintainer="austrin@kattis.com" +FROM problemtools/runreqs:${PROBLEMTOOLS_VERSION} +LABEL maintainer="contact@kattis.com" ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && \ - apt-get install -y \ - fp-compiler \ - gfortran \ - gnucobol \ - gccgo \ - ghc haskell-platform \ - gnustep-devel gnustep gnustep-make gnustep-common gobjc \ - libgmp3-dev \ - libmozjs-78-dev \ - lua5.4 \ - mono-complete \ - nodejs \ - ocaml-nox \ - pandoc \ - php-cli \ - pypy \ - rustc \ - sbcl \ - scala \ - swi-prolog \ - ; +RUN mkdir -p /usr/local/artifacts +WORKDIR /usr/local/artifacts +COPY artifacts/deb . +RUN dpkg -i kattis-problemtools*.deb + +WORKDIR / diff --git a/admin/docker/Dockerfile.fulllangs b/admin/docker/Dockerfile.fulllangs new file mode 100644 index 00000000..50ca8c36 --- /dev/null +++ b/admin/docker/Dockerfile.fulllangs @@ -0,0 +1,40 @@ +# Docker image with all packages needed to run a problemtools .deb, plus +# language support for all supported languages +# +# Not uploaded anywhere, only used locally during building + +ARG PROBLEMTOOLS_VERSION=develop +FROM problemtools/icpclangs:${PROBLEMTOOLS_VERSION} + +LABEL maintainer="contact@kattis.com" +ENV DEBIAN_FRONTEND=noninteractive + +# All languages, plus curl which we need to fetch pypy2 +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + rm -f /etc/apt/apt.conf.d/docker-clean \ + apt-get update && apt-get install -y \ + curl \ + fp-compiler \ + gfortran \ + gnucobol \ + gccgo \ + ghc \ + gnustep-devel gnustep gnustep-make gnustep-common gobjc \ + lua5.4 \ + mono-complete \ + nodejs \ + ocaml-nox \ + php-cli \ + rustc \ + sbcl \ + scala \ + swi-prolog + +# pypy2 is no longer packaged for Ubuntu, so download tarball (and check a sha256) +RUN curl -LO https://downloads.python.org/pypy/pypy2.7-v7.3.16-linux64.tar.bz2 \ + && echo '04b2fceb712d6f811274825b8a471ee392d3d1b53afc83eb3f42439ce00d8e07 pypy2.7-v7.3.16-linux64.tar.bz2' | sha256sum --check \ + && tar -xf pypy2.7-v7.3.16-linux64.tar.bz2 \ + && mv pypy2.7-v7.3.16-linux64 /opt/pypy \ + && ln -s /opt/pypy/bin/pypy /usr/bin/pypy \ + && rm pypy2.7-v7.3.16-linux64.tar.bz2 diff --git a/admin/docker/Dockerfile.githubci b/admin/docker/Dockerfile.githubci new file mode 100644 index 00000000..024bf773 --- /dev/null +++ b/admin/docker/Dockerfile.githubci @@ -0,0 +1,23 @@ +# Docker image with all deb packages needed for our github actions +# - Building a problemtools deb +# - Running verifyproblem on all examples + +ARG PROBLEMTOOLS_VERSION=develop +FROM problemtools/fulllangs:${PROBLEMTOOLS_VERSION} + +LABEL maintainer="contact@kattis.com" +ENV DEBIAN_FRONTEND=noninteractive + +# Packages required to build and run problemtools +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + rm -f /etc/apt/apt.conf.d/docker-clean \ + apt-get update && apt-get install -y \ + automake \ + build-essential \ + debhelper \ + dh-virtualenv \ + dpkg-dev \ + git \ + make \ + libboost-regex-dev diff --git a/admin/docker/Dockerfile.icpc b/admin/docker/Dockerfile.icpc index 55f2dff6..6366fe01 100644 --- a/admin/docker/Dockerfile.icpc +++ b/admin/docker/Dockerfile.icpc @@ -1,48 +1,22 @@ # Basic problemtools docker image, containing problemtools and the # "ICPC languages" (C, C++, Java, Kotlin, and Python 3) # +# Build requirements: +# - The problemtools .deb package must be available from the host file +# system under a file name matching +# artifacts/deb/kattis-problemtools*.deb +# (Version of that .deb file should match the build argument +# PROBLEMTOOLS_VERSION but this is not checked.) ARG PROBLEMTOOLS_VERSION=develop -FROM problemtools/minimal:${PROBLEMTOOLS_VERSION} - -LABEL maintainer="austrin@kattis.com" +FROM problemtools/icpclangs:${PROBLEMTOOLS_VERSION} +LABEL maintainer="contact@kattis.com" ENV DEBIAN_FRONTEND=noninteractive -# Install C++, Java, and PyPy 3 via their ppa repository (Kotlin is installed below to get a more up-to-date version) -RUN apt-get update && \ - apt-get install -y software-properties-common && \ - add-apt-repository ppa:pypy/ppa && \ - apt-get update && \ - apt-get install -y \ - gcc g++ \ - openjdk-11-jdk openjdk-11-jre \ - pypy3 - -RUN apt-get install -y curl - -ARG KOTLIN_VERSION=1.8.10 -RUN curl "https://github.com/JetBrains/kotlin/releases/download/v$KOTLIN_VERSION/kotlin-compiler-$KOTLIN_VERSION.zip" -L -o kotlin.zip -RUN unzip -q kotlin.zip -RUN rm kotlin.zip - -RUN mv kotlinc/bin/* /usr/bin -RUN mv kotlinc/lib/* /usr/lib - -RUN rm -r kotlinc - -# Reconfigure problemtools: -# - Use PyPy for Python 2 (not available in this image but in the full one) -# - Use PyPy for Python 3 -RUN mkdir -p /etc/kattis/problemtools -RUN echo " \n\ -python2: \n\ - name: 'Python 2 w/PyPy'\n\ - run: '/usr/bin/pypy \"{mainfile}\"'\n\ - \n\ -python3: \n\ - name: 'Python 3 w/PyPy'\n\ - run: '/usr/bin/pypy3 \"{mainfile}\"'\n\ - \n" > /etc/kattis/problemtools/languages.yaml +RUN mkdir -p /usr/local/artifacts +WORKDIR /usr/local/artifacts +COPY artifacts/deb . +RUN dpkg -i kattis-problemtools*.deb WORKDIR / diff --git a/admin/docker/Dockerfile.icpclangs b/admin/docker/Dockerfile.icpclangs new file mode 100644 index 00000000..36fe83ae --- /dev/null +++ b/admin/docker/Dockerfile.icpclangs @@ -0,0 +1,21 @@ +# Docker image with all packages needed to run a problemtools .deb, plus +# language support for the "ICPC languages" (C, C++, Java, Kotlin, and Python 3) +# +# Not uploaded anywhere, only used locally during building + +ARG PROBLEMTOOLS_VERSION=develop +FROM problemtools/runreqs:${PROBLEMTOOLS_VERSION} + +LABEL maintainer="contact@kattis.com" +ENV DEBIAN_FRONTEND=noninteractive + +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + rm -f /etc/apt/apt.conf.d/docker-clean \ + apt-get update && apt-get install -y \ + gcc \ + g++ \ + kotlin \ + openjdk-21-jdk \ + openjdk-21-jre \ + pypy3 diff --git a/admin/docker/Dockerfile.minimal b/admin/docker/Dockerfile.minimal index 886d1a2d..11b7b414 100644 --- a/admin/docker/Dockerfile.minimal +++ b/admin/docker/Dockerfile.minimal @@ -10,27 +10,11 @@ # PROBLEMTOOLS_VERSION but this is not checked.) ARG PROBLEMTOOLS_VERSION=develop -FROM ubuntu:22.04 - -LABEL maintainer="austrin@kattis.com" +FROM problemtools/runreqs:${PROBLEMTOOLS_VERSION} +LABEL maintainer="contact@kattis.com" ENV DEBIAN_FRONTEND=noninteractive -RUN apt update && \ - apt install -y \ - ghostscript \ - libgmpxx4ldbl \ - pandoc \ - python-pkg-resources \ - python3-minimal \ - python3-yaml \ - python3-plastex \ - texlive-fonts-recommended \ - texlive-lang-cyrillic \ - texlive-latex-extra \ - texlive-plain-generic \ - tidy - RUN mkdir -p /usr/local/artifacts WORKDIR /usr/local/artifacts COPY artifacts/deb . diff --git a/admin/docker/Dockerfile.runreqs b/admin/docker/Dockerfile.runreqs new file mode 100644 index 00000000..27451da4 --- /dev/null +++ b/admin/docker/Dockerfile.runreqs @@ -0,0 +1,28 @@ +# Docker image with all packages needed to run a problemtools .deb +# +# Not uploaded anywhere, only used locally during building + +ARG PROBLEMTOOLS_VERSION=develop +FROM ubuntu:24.04 + +LABEL maintainer="contact@kattis.com" +ENV DEBIAN_FRONTEND=noninteractive + +# Packages required to build and run problemtools +# For libgmp, we technically just need libgmpxx4ldbl here, but for readability +# (and we need libgmp-dev in other images), we take libgmp-dev here +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + rm -f /etc/apt/apt.conf.d/docker-clean \ + apt-get update && apt-get install -y \ + dvisvgm \ + ghostscript \ + libgmp-dev \ + pandoc \ + python3 \ + python3-venv \ + texlive-fonts-recommended \ + texlive-lang-cyrillic \ + texlive-latex-extra \ + texlive-plain-generic \ + tidy diff --git a/admin/docker/README.md b/admin/docker/README.md new file mode 100644 index 00000000..5e2c6c40 --- /dev/null +++ b/admin/docker/README.md @@ -0,0 +1,25 @@ +Our docker images. Note that images depend on each other, please use the script +admin/update_docker.sh to build images in the correct order. + +We have 4 images which are only used locally in the build process, and are not +uploaded to a repository. + - `runreqs`: Base image containing just the things needed to run problemtools + - `build`: Base image containing just the things needed to build a deb and run problemtools + - `icpclangs`: Base image containing what is needed to run problemtools, plus the "ICPC languages" + - `fulllangs`: Base image containing what is needed to run problemtools, plus all supported languages + +We have 3 images which are meant for end users: + - `minimal`: Image with problemtools installed, but no languages. + - `icpc`: Image with problemtools plus the "ICPC languages" installed. + - `full`: Image with problemtools and all languages + +We have 1 image which is used in our CI (to speed up things - it takes a few +minutes to apt-get install all languages and runtime requirements): + - `githubci`: Image with all languages and everything needed to build a deb and run problemtools + +Build dependencies: +``` + runreqs -> icpclangs -> fullangs -> githubci + / \ | | + build minimal icpc full +``` diff --git a/admin/update_docker.sh b/admin/update_docker.sh index ce7e64e2..af3d5ae5 100755 --- a/admin/update_docker.sh +++ b/admin/update_docker.sh @@ -8,44 +8,76 @@ if [ "$1" != "" ]; then UPDATE_LATEST=true fi - cd $(dirname $(readlink -f $0))/docker -set -x - -# Make the build image and extract build artifacts -# =============================================== -sudo docker build \ - -f Dockerfile.build \ - -t problemtools/build:${TAG} \ - --no-cache \ - --build-arg PROBLEMTOOLS_VERSION="${TAG}" \ - . + +if [[ -n $(git status -s) ]]; then + echo "Repository is dirty." + git status -s + exit 1 +fi + +if [[ $(git rev-parse --abbrev-ref HEAD) != ${TAG} && $(git describe --exact-match --tags 2>/dev/null) != ${TAG} ]]; then + echo "Repository is currently not on branch/tag ${TAG}." + exit 1 +fi + + +# Make our internal images, and our githubci image. Order is important, images depend on each other +echo "Building intermediate images, plus githubci image" +for IMAGE in runreqs build icpclangs fulllangs githubci; do + docker build \ + -f Dockerfile.${IMAGE} \ + -t problemtools/${IMAGE}:${TAG} \ + --build-arg PROBLEMTOOLS_VERSION="${TAG}" \ + . +done + + +echo "Building deb" mkdir -p artifacts -rm -rf artifacts/deb/* -sudo docker run --rm -v "$(pwd)/artifacts/:/artifacts" problemtools/build:${TAG} cp -r /usr/local/problemtools_build/deb /artifacts +sudo rm -rf artifacts/deb +# Use our build image to build a deb +docker run --rm -v "$(pwd)/../..:/problemtools" -v "$(pwd)/artifacts/deb:/artifacts" problemtools/build:${TAG} \ + /bin/bash -c ' + set -e ; + mkdir /build ; + cd /build ; + git config --global --add safe.directory /problemtools/.git ; + git clone --branch ${TAG} /problemtools ; + cd problemtools ; + make builddeb ; + cp ../*.deb /artifacts' sudo chown -R $USER:$USER artifacts/ -# Build the actual problemtools images -# =============================================== -for IMAGE in minimal icpc full; do - sudo docker build\ - -f Dockerfile.${IMAGE}\ - -t problemtools/${IMAGE}:${TAG}\ - --build-arg PROBLEMTOOLS_VERSION=${TAG}\ - . - if [ "$UPDATE_LATEST" = "true" ]; then - sudo docker tag problemtools/${IMAGE}:${TAG} problemtools/${IMAGE}:latest - fi -done +echo "Testing deb" +if ! docker run --rm -t -v "$(pwd)/../..:/problemtools" -v "$(pwd)/artifacts/deb:/artifacts" problemtools/fulllangs:${TAG} \ + /bin/bash -c ' + set -e ; + shopt -s extglob ; + dpkg -i /artifacts/kattis-problemtools* ; + verifyproblem /problemtools/examples/!(README.md)'; then + echo Running verifyproblem on all examples failed. Please review output above to debug.; + exit 1 +fi +echo Tests pass -# Push to Docker Hub -# =============================================== -sudo docker login +echo "Building complete images with problemtools baked in" for IMAGE in minimal icpc full; do - sudo docker push problemtools/${IMAGE}:${TAG} - if [ "$UPDATE_LATEST" = "true" ]; then - sudo docker push problemtools/${IMAGE}:latest - fi + docker build \ + -f Dockerfile.${IMAGE} \ + -t problemtools/${IMAGE}:${TAG} \ + --build-arg PROBLEMTOOLS_VERSION="${TAG}" \ + . done + + +if [ "${UPDATE_LATEST}" = "true" ]; then + echo "Build complete. If you are happy with the images, run the following:" + for IMAGE in minimal icpc full githubci; do + echo " docker tag problemtools/${IMAGE}:${TAG} problemtools/${IMAGE}:latest" + echo " docker push problemtools/${IMAGE}:${TAG}" + echo " docker push problemtools/${IMAGE}:latest" + done +fi