diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml new file mode 100644 index 00000000..bf8fceb6 --- /dev/null +++ b/.github/workflows/pypi.yml @@ -0,0 +1,88 @@ +name: Publish python distribution to PyPI and TestPyPI + +on: + workflow_dispatch: + inputs: + pushTestPyPi: + description: 'Push package to TestPyPi' + required: true + type: boolean + pushPyPi: + description: 'Push package to PyPi' + required: true + type: boolean + +jobs: + build-pypi: + runs-on: ubuntu-latest + container: + image: quay.io/pypa/manylinux_2_28_x86_64 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # We need full history for setuptools_scm to figure out version + - name: Set safe directory (work around checkout not doing that properly for containers) + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + - name: Install build dependencies for checktestdata + run: yum -y install boost-devel gmp-devel + - name: Build sdist (and broken wheel) + run: /opt/python/cp311-cp311/bin/python -m build + - name: Repair wheel + run: auditwheel repair dist/problemtools-*.whl + - name: Replace broken wheel with repaired wheel + run: | + rm -f dist/*.whl + cp wheelhouse/*.whl dist + - name: Store the distribution packages + uses: actions/upload-artifact@v4 + with: + name: python-package-distributions + path: dist/ + + publish-to-testpypi: + name: Publish Python distribution to TestPyPI + needs: + - build-pypi + runs-on: ubuntu-latest + if: ${{ inputs.pushTestPyPi }} + + environment: + name: testpypi + url: https://test.pypi.org/p/problemtools + + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + + publish-to-pypi: + name: Publish Python distribution to PyPI + needs: + - build-pypi + runs-on: ubuntu-latest + if: ${{ inputs.pushPyPi }} + + environment: + name: pypi + url: https://pypi.org/p/problemtools + + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml new file mode 100644 index 00000000..b3226dfe --- /dev/null +++ b/.github/workflows/python-app.yml @@ -0,0 +1,62 @@ +# This workflow will install Python dependencies, run tests and lint +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python + +name: Python tests + +on: + push: + branches: [ "master", "develop" ] + pull_request: + branches: [ "master", "develop" ] + +permissions: + contents: read + +jobs: + pythontests: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11", "3.12", "3.13"] # 3.11 is the lowest we support, since we want StrEnum + container: + image: problemtools/githubci:latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m venv venv + venv/bin/python --version + venv/bin/pip install mypy ruff pytest + if [ -f requirements.txt ]; then venv/bin/pip install -r requirements.txt; fi + - name: Lint with ruff + run: venv/bin/ruff check --output-format=github + - name: Check ruff formatting + run: venv/bin/ruff format --check --diff + - name: Test with pytest + run: venv/bin/pytest + - name: Run mypy + run: | + venv/bin/mypy --non-interactive --config-file mypy.ini -p problemtools + + packages: # Use a separate job to test debian packaging to speed things up (no need to test this for every python version above) + runs-on: ubuntu-latest + container: + image: problemtools/githubci:latest + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Build debian packages + run: | + make builddeb + - name: Install debian package + run: dpkg -i ../kattis-problemtools_*.deb + - name: Verify examples + run: | + shopt -s extglob + verifyproblem examples/!(README.md) + shell: bash diff --git a/.gitignore b/.gitignore index ddbe0378..18a3eccb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,13 @@ *.pyc *~ +*.swp /.cache/ /problemtools.egg-info/ /support/default_validator/default_validator /support/interactive/interactive +build/ +/problemtools/_version.py + +venv/ +.pytest_cache/ +.mypy_cache/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..889c8316 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,6 @@ +repos: +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.11.8 + hooks: + - id: ruff + - id: ruff-format diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f81f3fcd..00000000 --- a/.travis.yml +++ /dev/null @@ -1,5 +0,0 @@ -language: python -python: - - 3.7 - -script: py.test diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index e9787418..00000000 --- a/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -FROM ubuntu:20.04 - -MAINTAINER austrin@kattis.com - -ENV DEBIAN_FRONTEND=noninteractive - -RUN apt-get update && \ - apt-get install -y \ - automake \ - g++ \ - git \ - libboost-all-dev \ - libgmp-dev \ - libgmp10 \ - libgmpxx4ldbl \ - openjdk-8-jdk \ - python3-minimal \ - python3-pip \ - python3-plastex \ - python3-yaml \ - sudo \ - texlive-fonts-recommended \ - texlive-lang-cyrillic \ - texlive-latex-extra \ - texlive-plain-generic \ - tidy \ - vim - -RUN pip3 install git+https://github.com/kattis/problemtools diff --git a/LICENSE b/LICENSE index 84c39ef9..01cc176a 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2010-2019 Kattis and all respective contributors +Copyright (c) Kattis and all respective contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/MANIFEST.in b/MANIFEST.in index 4d57d2ed..06421ee4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,9 @@ recursive-include problemtools/config * recursive-include problemtools/templates * recursive-include problemtools/tests * +recursive-include examples * recursive-include support * +recursive-include tests * +global-exclude */__pycache__/* +global-exclude *.pyc +recursive-exclude .github * diff --git a/Makefile b/Makefile index 296634e5..bd3337bf 100644 --- a/Makefile +++ b/Makefile @@ -8,3 +8,7 @@ checktestdata: support/checktestdata/bootstrap support/checktestdata/bootstrap: git submodule update --init + +clean: + make -C support clean + rm -rf problemtools.egg-info build diff --git a/README.md b/README.md index 29140990..210a35bb 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ # Kattis Problem Tools -master: -[![Master Build Status](https://travis-ci.org/Kattis/problemtools.svg?branch=master)](https://travis-ci.org/Kattis/problemtools). -develop: -[![Develop Build Status](https://travis-ci.org/Kattis/problemtools.svg?branch=develop)](https://travis-ci.org/Kattis/problemtools) +![Build Status](https://github.com/kattis/problemtools/actions/workflows/python-app.yml/badge.svg?branch=master) +[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)](https://github.com/pre-commit/pre-commit) -These are tools to manage problem packages using the Kattis problem package -format. +These are tools to manage problem packages using the Kattis [problem package +format](https://www.kattis.com/problem-package-format/). The problem package +specification is developed in the [problem package format +repository](https://github.com/Kattis/problem-package-format). ## Programs Provided @@ -17,10 +17,26 @@ The problem tools provide the following three programs: - `problem2pdf`: convert a problem statement to pdf - `problem2html`: convert a problem statement to html -Running any of them with command-line option `-h` gives +Running any of them with the command-line option `-h` gives documentation on what arguments they accept. +## Format versions + +There are currently two versions of the problem package format, +[legacy](https://www.kattis.com/problem-package-format/spec/legacy.html) and +[2023-07-draft](https://www.kattis.com/problem-package-format/spec/2023-07-draft.html). +We have begun work on supporting 2023-07-draft, but there are *many* changes +between legacy and 2023-07-draft which are not yet implemented. Verifyproblem +will do its best to parse and verify problems in 2023-07-draft, but key things +like scoring still behave like legacy. Also, note that 2023-07 is still a draft +standard. + +We advice against packaging production problems in 2023-07-draft. Especially so +if you plan to have problems installed on [Kattis](https://open.kattis.com), where +we currently *only* support installing legacy problems. + + ## Example Problems A few examples of problem packages can be found in [examples](examples). @@ -31,20 +47,21 @@ A few examples of problem packages can be found in [examples](examples). There are four supported ways of installing and running problemtools. (For non-Linux users, "Method 2" below, to use Docker, is probably the least painful.) -### Method 1: Install the Python package +Note that in all methods except for "Method 2", you must manually install +dependencies such as LaTeX and tools for any languages you want to use. See +[Requirements and compatbility](#requirements-and-compatibility) for details. + +### Method 1: Install the Python package using pipx Run ``` -pip3 install git+https://github.com/kattis/problemtools +pipx install problemtools ``` -Or if you don't want a system-wide installation, -``` -pip3 install --user git+https://github.com/kattis/problemtools -``` -With this second option, in order to get the command line scripts, you need -to make sure that the local user bin path used (e.g., on Linux, -`$HOME/.local/bin`) is in your `$PATH`. +In order to get the command line scripts, you need to make sure that the local +user bin path used (e.g., on Linux, `$HOME/.local/bin`) is in your `$PATH`. See +[pipx' installation instructions](https://pipx.pypa.io/stable/installation/) +for information on how to install `pipx` and set up your `$PATH`. In order for problemtools to build and run properly, you also need to have LaTeX and various LaTeX packages installed. See [Requirements and @@ -60,34 +77,45 @@ We maintain three official problemtools Docker images on Docker Hub: - [`problemtools/full`](https://hub.docker.com/r/problemtools/full/): this image contains problemtools along with compilers/interpreters for all supported programming languages. -- [`problemtools/icpc`](https://hub.docker.com/r/problemtools/icpc/): this image contains problemtools along with compilers/interpreters for the programming languages allowed in the International Collegiate Programming Contest (ICPC): C, C++, Java, Python 2+3, and Kotlin. +- [`problemtools/icpc`](https://hub.docker.com/r/problemtools/icpc/): this image contains problemtools along with compilers/interpreters for the programming languages allowed in the International Collegiate Programming Contest (ICPC): C, C++, Java, Kotlin, and Python 3. Note that the compiler/interpreter versions used might not be exactly the same as those used in the current ICPC season. -- [`problemtools/minimal`](https://hub.docker.com/r/problemtools/minimal/): this image only contains problemtools, no additional programming languages. As such as it is not particularly useful on its own, but if you are organizing a contest and want to set up a problemtools environment containing exactly the right set of compilers/interpreters for your contest, this is the recommended starting point. +- [`problemtools/minimal`](https://hub.docker.com/r/problemtools/minimal/): this image only contains problemtools, no additional programming languages. As such, it is not particularly useful on its own, but if you are organizing a contest and want to set up a problemtools environment containing exactly the right set of compilers/interpreters for your contest, this is the recommended starting point. -For example, suppose you want to use the `problemtools/icpc` image. To get started, install the [Docker CLI](https://docs.docker.com/install), and then pull the image: +For example, suppose you want to use the `problemtools/icpc` image. To get started (or update to the latest release), install the [Docker CLI](https://docs.docker.com/install), and then pull the image: docker pull problemtools/icpc -Once the image has finished downloading, you can check that it exists on your system using `docker images`. To launch an interactive container and play around with *verifyproblem*, *problem2pdf*, and *problem2html* run: +The most convenient way to use the container is by creating shell script(s) similar to this and add it to your `$PATH`. If you call the script `verifyproblem.sh`, you could then `verifyproblem.sh examples/hello` to use the icpc docker image to verify examples/hello: +```sh +#!/bin/bash + +if [ $1 -a -d $1 ]; then + docker run --rm -t -v $(dirname $(readlink -f $1)):/work problemtools/icpc verifyproblem /work/$(basename $1) +else + echo No such directory: $1 +fi +``` + +To instead launch an interactive container and play around with *verifyproblem*, *problem2pdf*, and *problem2html* run: docker run --rm -it problemtools/icpc By default, docker containers do _NOT_ persist storage between runs, so any files you create or modify will be lost when the container stops running. Two common ways of dealing with this are: -1) Use a [bind mount](https://docs.docker.com/storage/bind-mounts/) to mount a directory on your machine into the docker container. This can be done as follows (see Docker documentation for further details): +1) Use a [bind mount](https://docs.docker.com/storage/bind-mounts/) to mount a directory on your machine into the docker container. Mounting the current directory to /kattis_work_dir can be done as follows (see Docker documentation for further details): ``` - docker run --rm -it -v ${FULL_PATH_TO_MOUNT}:/kattis_work_dir problemtools/icpc + docker run --rm -it -v $(pwd):/kattis_work_dir problemtools/icpc ``` -2) Persist any changes you want to keep to a remote file system/source control (e.g. a remote Git repository, note however that you would first need to install Git in the image). +2) Persist any changes you want to keep to a remote file system/source control (e.g., a remote Git repository; note, however, that you would first need to install Git in the image). #### Building your own images -If you want a more complete environment in the Docker images (e.g. if +If you want a more complete environment in the Docker images (e.g., if you want to install git or your favorite editor), feel free to extend them in whichever way you like. The `problemtools/{minimal,icpc,full}` images point to the latest -release versions of problemtools. If for some reason you want an +release versions of problemtools. If, for some reason, you want an image containing the latest development version, you have to build it yourself from scratch (while there are `problemtools/{minimal,icpc,full}:develop` Docker images on Docker @@ -95,15 +123,20 @@ Hub, these are only updated sporadically for testing purposes and not kept up to date). -### Method 3: Run directly from the repository. +### Method 3: Run directly from the repository -If you intend to help develop problemtools, or if you just want a -bare-bones way of running them, this is your option. +If you intend to help develop problemtools, or if you just want a bare-bones +way of running them, this is your option. For this method, you need to clone the repository (just downloading a -zip archive of it does not work, because the project has submodules +zip archive of it does not work because the project has submodules that are not included in that zip archive). +Start by setting up your venv, e.g., + + python3 -m venv venv + venv/bin/pip install -r requirements.txt + In order for the tools to work, you first have to compile the various support programs, which can be done by running `make` in the root directory of problemtools. @@ -119,11 +152,11 @@ order for problemtools to work correctly. ### Method 4: Build and install the Debian package -This applies if you are running on Debian or a Debian derivative such +This applies if you are running on Debian or a Debian derivative, such as Ubuntu. As with method 3, you need to clone the repository (just downloading a -zip archive of it does not work, because the project has submodules +zip archive of it does not work because the project has submodules that are not included in that zip archive). Run `make builddeb` in the root of the problemtools repository to @@ -134,7 +167,7 @@ root of the repository). Apart from the build dependencies listed [below](#ubuntu), building the Debian package requires that the following tools are installed: - debhelper dh-python dpkg-dev + debhelper dh-virtualenv dpkg-dev The package can then be installed using (replace `` as appropriate): @@ -178,7 +211,7 @@ problemtools' configuration: are not sure whether you should use it, then you probably shouldn't. This file can be used to specify the system defaults for those problem limits which are not given a fixed default value in the - [problem format specification](http://www.problemarchive.org/wiki/index.php/Problem_Format#limits). + [problem format specification](https://www.kattis.com/problem-package-format/spec/2023-07-draft.html#limits). The system defaults assumed by problemtools can be found in (problemtools/config/problem.yaml). For instance, if you are primarily working against a system with a default memory limit of 2 GiB, @@ -189,35 +222,35 @@ problemtools' configuration: memory: 2048 # (unit is MiB) ``` - (In principle it is possible to override the defaults of other values than the - system-dependent defaults in the problem.yaml metadata files this way, but such - usage is very strongly discouraged.) - ## Requirements and compatibility To build and run the tools, you need Python 3 with the YAML and PlasTeX libraries, -and a LaTeX installation. +and a LaTeX installation. You must also install language tools (e.g., compilers) +for any languages used in problem packages. ### Ubuntu The dependencies needed to *build/install* problemtools can be installed with: - sudo apt install automake g++ make libboost-regex-dev libgmp-dev libgmp10 libgmpxx4ldbl python3 python3-pytest python3-setuptools python3-yaml python3-plastex + sudo apt install python3-venv automake g++ make libboost-regex-dev libgmp-dev python3 git And the dependencies needed to *run* problemtools can be installed with: - sudo apt install ghostscript libgmpxx4ldbl python3-minimal python-pkg-resources python3-plastex python3-yaml texlive-fonts-recommended texlive-lang-cyrillic texlive-latex-extra texlive-plain-generic tidy + sudo apt install ghostscript pandoc python3 texlive-fonts-recommended texlive-lang-cyrillic texlive-latex-extra texlive-plain-generic tidy dvisvgm ### Fedora On Fedora, these dependencies can be installed with: - sudo dnf install boost-regex gcc gmp-devel gmp-c++ python3 python3-pyyaml texlive-latex texlive-collection-fontsrecommended texlive-fancyhdr texlive-subfigure texlive-wrapfig texlive-import texlive-ulem texlive-xifthen texlive-overpic texlive-pbox tidy ghostscript + sudo dnf install boost-regex gcc gmp-devel gmp-c++ pandoc python3 python3-pyyaml texlive-latex texlive-collection-fontsrecommended texlive-fancyhdr texlive-subfigure texlive-wrapfig texlive-import texlive-ulem texlive-xifthen texlive-overpic texlive-pbox tidy ghostscript Followed by: - pip3 install --user plastex + pip3 install --user plastex nh3 + +### Arch +Package is available on the AUR [kattis-problemtools-git](https://aur.archlinux.org/packages/kattis-problemtools-git). Use your favorite AUR helper or follow the installation instructions found [here](https://wiki.archlinux.org/title/Arch_User_Repository#Installing_and_upgrading_packages). ### Other platforms diff --git a/admin/.gitignore b/admin/.gitignore new file mode 100644 index 00000000..a4d089c1 --- /dev/null +++ b/admin/.gitignore @@ -0,0 +1 @@ +pypi_dist/ diff --git a/admin/build_pypi_packages.sh b/admin/build_pypi_packages.sh new file mode 100755 index 00000000..d9567a24 --- /dev/null +++ b/admin/build_pypi_packages.sh @@ -0,0 +1,73 @@ +#!/bin/bash +set -e + +ALLOW_DIRTY=false +TAG=develop + +echo N.B., this script is solely to allow for local testing of whl/src. +echo To actually build and push things to pypi, trigger the pypi flow +echo in github actions, https://github.com/Kattis/problemtools/actions + +while getopts "d" opt; do + case $opt in + d) ALLOW_DIRTY=true ;; + \?) echo "Invalid option: -$opt" ;; + esac +done + +shift $((OPTIND-1)) + +if [ "$1" != "" ]; then + TAG=$1 +fi + +cd $(dirname $(readlink -f $0)) + +if ! ../venv/bin/twine -h > /dev/null 2> /dev/null; then + echo "Did not find twine. Please run ../venv/bin/pip install twine" + exit 1 +fi + +if [[ -n $(git status -s) ]]; then + echo "Repository is dirty." + git status -s + [[ "${ALLOW_DIRTY}" != "true" ]] && exit 1 +fi + +if [[ $(git rev-parse --abbrev-ref HEAD) != ${TAG} && $(git describe --exact-match --tags 2>/dev/null) != ${TAG} ]]; then + echo "Repository is currently not on branch/tag ${TAG}." + [[ "${ALLOW_DIRTY}" != "true" ]] && exit 1 +fi + +echo "Building sdist and manylinux wheel" +sudo rm -rf ./pypi_dist +docker run --rm -v $(pwd)/..:/problemtools -v $(pwd)/pypi_dist:/dist quay.io/pypa/manylinux_2_28_x86_64 /bin/bash -c " + yum -y install boost-devel gmp-devel ; + mkdir /build ; + cd /build ; + git config --global --add safe.directory /problemtools/.git ; + git clone /problemtools ; + cd problemtools ; + git checkout ${TAG} ; + /opt/python/cp311-cp311/bin/python -m build ; + auditwheel repair dist/problemtools-*.whl ; + cp dist/*.tar.gz /dist ; + cp wheelhouse/*.whl /dist" +sudo chown -R $USER:$USER pypi_dist + +../venv/bin/twine check pypi_dist/* + +echo "Running verifyproblem from wheel on all examples" +TEMPDIR=$(mktemp -d) +python3 -m venv "${TEMPDIR}" +"${TEMPDIR}/bin/pip" install pypi_dist/problemtools*manylinux*whl +shopt -s extglob +if ! "${TEMPDIR}/bin/verifyproblem" ../examples/!(README.md); then + echo "Running verifyproblem on all examples failed. Please review output above to debug." + rm -rf "${TEMPDIR}" + exit 1 +fi +rm -rf "${TEMPDIR}" + +echo "Sucessfully built packages. If you're happy with them, upload:" +echo " ../venv/bin/twine upload --verbose pypi_dist/*" diff --git a/admin/docker/Dockerfile.build b/admin/docker/Dockerfile.build index e2f7a3bf..b8080fe1 100644 --- a/admin/docker/Dockerfile.build +++ b/admin/docker/Dockerfile.build @@ -1,43 +1,24 @@ -# Package for building the problemtools .deb package -# Ends up in the /usr/local/problemtools_build/deb/ directory +# Docker image with all packages needed to build a problemtools .deb # -# Setting build argument PROBLEMTOOLS_VERSION causes a specific -# version of problemtools to be built (default is latest version of -# develop branch on GitHub) +# Not uploaded anywhere, only used locally during building -FROM ubuntu:22.04 - -LABEL maintainer="austrin@kattis.com" +ARG PROBLEMTOOLS_VERSION=develop +FROM problemtools/runreqs:${PROBLEMTOOLS_VERSION} +LABEL maintainer="contact@kattis.com" ENV DEBIAN_FRONTEND=noninteractive -# Install packages needed for build -RUN apt update && \ - apt install -y \ +# Packages required to build and run problemtools +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + rm -f /etc/apt/apt.conf.d/docker-clean && \ + apt-get update && apt-get install -y \ automake \ + build-essential \ debhelper \ - dh-python \ + dh-virtualenv \ dpkg-dev \ g++ \ git \ make \ - libboost-regex-dev \ - libgmp-dev \ - libgmp10 \ - libgmpxx4ldbl \ - python3 \ - python3-pytest \ - python3-setuptools \ - python3-yaml \ - python3-setuptools - -RUN mkdir -p /usr/local/problemtools_build - -WORKDIR /usr/local/problemtools_build -RUN git clone --recursive https://github.com/kattis/problemtools - -ARG PROBLEMTOOLS_VERSION=develop -RUN cd problemtools && git checkout ${PROBLEMTOOLS_VERSION} && make builddeb - -RUN mkdir -p deb -RUN mv kattis-problemtools*.deb deb/ + libboost-regex-dev diff --git a/admin/docker/Dockerfile.full b/admin/docker/Dockerfile.full index 40580dd6..75ec8745 100644 --- a/admin/docker/Dockerfile.full +++ b/admin/docker/Dockerfile.full @@ -1,32 +1,23 @@ # Full problemtools docker image, containing problemtools and all # supported programming languages. # +# +# Build requirements: +# - The problemtools .deb package must be available from the host file +# system under a file name matching +# artifacts/deb/kattis-problemtools*.deb +# (Version of that .deb file should match the build argument +# PROBLEMTOOLS_VERSION but this is not checked.) ARG PROBLEMTOOLS_VERSION=develop -FROM problemtools/icpc:${PROBLEMTOOLS_VERSION} - -LABEL maintainer="austrin@kattis.com" +FROM problemtools/fulllangs:${PROBLEMTOOLS_VERSION} +LABEL maintainer="contact@kattis.com" ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && \ - apt-get install -y \ - fp-compiler \ - gfortran \ - gnucobol \ - gccgo \ - ghc haskell-platform \ - gnustep-devel gnustep gnustep-make gnustep-common gobjc \ - libgmp3-dev \ - libmozjs-78-dev \ - lua5.4 \ - mono-complete \ - nodejs \ - ocaml-nox \ - php-cli \ - pypy \ - rustc \ - sbcl \ - scala \ - swi-prolog \ - ; +RUN mkdir -p /usr/local/artifacts +WORKDIR /usr/local/artifacts +COPY artifacts/deb . +RUN dpkg -i kattis-problemtools*.deb + +WORKDIR / diff --git a/admin/docker/Dockerfile.fulllangs b/admin/docker/Dockerfile.fulllangs new file mode 100644 index 00000000..cc0dd3df --- /dev/null +++ b/admin/docker/Dockerfile.fulllangs @@ -0,0 +1,40 @@ +# Docker image with all packages needed to run a problemtools .deb, plus +# language support for all supported languages +# +# Not uploaded anywhere, only used locally during building + +ARG PROBLEMTOOLS_VERSION=develop +FROM problemtools/icpclangs:${PROBLEMTOOLS_VERSION} + +LABEL maintainer="contact@kattis.com" +ENV DEBIAN_FRONTEND=noninteractive + +# All languages, plus curl which we need to fetch pypy2 +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + rm -f /etc/apt/apt.conf.d/docker-clean && \ + apt-get update && apt-get install -y \ + curl \ + fp-compiler \ + gfortran \ + gnucobol \ + gccgo \ + ghc \ + gnustep-devel gnustep gnustep-make gnustep-common gobjc \ + lua5.4 \ + mono-complete \ + nodejs \ + ocaml-nox \ + php-cli \ + rustc \ + sbcl \ + scala \ + swi-prolog + +# pypy2 is no longer packaged for Ubuntu, so download tarball (and check a sha256) +RUN curl -LO https://downloads.python.org/pypy/pypy2.7-v7.3.16-linux64.tar.bz2 \ + && echo '04b2fceb712d6f811274825b8a471ee392d3d1b53afc83eb3f42439ce00d8e07 pypy2.7-v7.3.16-linux64.tar.bz2' | sha256sum --check \ + && tar -xf pypy2.7-v7.3.16-linux64.tar.bz2 \ + && mv pypy2.7-v7.3.16-linux64 /opt/pypy \ + && ln -s /opt/pypy/bin/pypy /usr/bin/pypy \ + && rm pypy2.7-v7.3.16-linux64.tar.bz2 diff --git a/admin/docker/Dockerfile.githubci b/admin/docker/Dockerfile.githubci new file mode 100644 index 00000000..7c4b5217 --- /dev/null +++ b/admin/docker/Dockerfile.githubci @@ -0,0 +1,23 @@ +# Docker image with all deb packages needed for our github actions +# - Building a problemtools deb +# - Running verifyproblem on all examples + +ARG PROBLEMTOOLS_VERSION=develop +FROM problemtools/fulllangs:${PROBLEMTOOLS_VERSION} + +LABEL maintainer="contact@kattis.com" +ENV DEBIAN_FRONTEND=noninteractive + +# Packages required to build and run problemtools +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + rm -f /etc/apt/apt.conf.d/docker-clean && \ + apt-get update && apt-get install -y \ + automake \ + build-essential \ + debhelper \ + dh-virtualenv \ + dpkg-dev \ + git \ + make \ + libboost-regex-dev diff --git a/admin/docker/Dockerfile.icpc b/admin/docker/Dockerfile.icpc index 904529f8..6366fe01 100644 --- a/admin/docker/Dockerfile.icpc +++ b/admin/docker/Dockerfile.icpc @@ -1,37 +1,22 @@ # Basic problemtools docker image, containing problemtools and the # "ICPC languages" (C, C++, Java, Kotlin, and Python 3) # +# Build requirements: +# - The problemtools .deb package must be available from the host file +# system under a file name matching +# artifacts/deb/kattis-problemtools*.deb +# (Version of that .deb file should match the build argument +# PROBLEMTOOLS_VERSION but this is not checked.) ARG PROBLEMTOOLS_VERSION=develop -FROM problemtools/minimal:${PROBLEMTOOLS_VERSION} - -LABEL maintainer="austrin@kattis.com" +FROM problemtools/icpclangs:${PROBLEMTOOLS_VERSION} +LABEL maintainer="contact@kattis.com" ENV DEBIAN_FRONTEND=noninteractive -# Install C++, Java, Kotlin, and PyPy 3 via their ppa repository -RUN apt update && \ - apt install -y software-properties-common && \ - add-apt-repository ppa:pypy/ppa && \ - apt update && \ - apt install -y \ - gcc g++ \ - openjdk-11-jdk openjdk-11-jre \ - kotlin \ - pypy3 - -# Reconfigure problemtools: -# - Use PyPy for Python 2 (not available in this image but in the full one) -# - Use PyPy for Python 3 -RUN mkdir -p /etc/kattis/problemtools -RUN echo " \n\ -python2: \n\ - name: 'Python 2 w/PyPy'\n\ - run: '/usr/bin/pypy \"{mainfile}\"'\n\ - \n\ -python3: \n\ - name: 'Python 3 w/PyPy'\n\ - run: '/usr/bin/pypy3 \"{mainfile}\"'\n\ - \n" > /etc/kattis/problemtools/languages.yaml +RUN mkdir -p /usr/local/artifacts +WORKDIR /usr/local/artifacts +COPY artifacts/deb . +RUN dpkg -i kattis-problemtools*.deb WORKDIR / diff --git a/admin/docker/Dockerfile.icpclangs b/admin/docker/Dockerfile.icpclangs new file mode 100644 index 00000000..c2ce8782 --- /dev/null +++ b/admin/docker/Dockerfile.icpclangs @@ -0,0 +1,21 @@ +# Docker image with all packages needed to run a problemtools .deb, plus +# language support for the "ICPC languages" (C, C++, Java, Kotlin, and Python 3) +# +# Not uploaded anywhere, only used locally during building + +ARG PROBLEMTOOLS_VERSION=develop +FROM problemtools/runreqs:${PROBLEMTOOLS_VERSION} + +LABEL maintainer="contact@kattis.com" +ENV DEBIAN_FRONTEND=noninteractive + +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + rm -f /etc/apt/apt.conf.d/docker-clean && \ + apt-get update && apt-get install -y \ + gcc \ + g++ \ + kotlin \ + openjdk-21-jdk \ + openjdk-21-jre \ + pypy3 diff --git a/admin/docker/Dockerfile.minimal b/admin/docker/Dockerfile.minimal index 534e661f..11b7b414 100644 --- a/admin/docker/Dockerfile.minimal +++ b/admin/docker/Dockerfile.minimal @@ -10,26 +10,11 @@ # PROBLEMTOOLS_VERSION but this is not checked.) ARG PROBLEMTOOLS_VERSION=develop -FROM ubuntu:22.04 - -LABEL maintainer="austrin@kattis.com" +FROM problemtools/runreqs:${PROBLEMTOOLS_VERSION} +LABEL maintainer="contact@kattis.com" ENV DEBIAN_FRONTEND=noninteractive -RUN apt update && \ - apt install -y \ - ghostscript \ - libgmpxx4ldbl \ - python-pkg-resources \ - python3-minimal \ - python3-yaml \ - python3-plastex \ - texlive-fonts-recommended \ - texlive-lang-cyrillic \ - texlive-latex-extra \ - texlive-plain-generic \ - tidy - RUN mkdir -p /usr/local/artifacts WORKDIR /usr/local/artifacts COPY artifacts/deb . diff --git a/admin/docker/Dockerfile.runreqs b/admin/docker/Dockerfile.runreqs new file mode 100644 index 00000000..c27486dd --- /dev/null +++ b/admin/docker/Dockerfile.runreqs @@ -0,0 +1,28 @@ +# Docker image with all packages needed to run a problemtools .deb +# +# Not uploaded anywhere, only used locally during building + +ARG PROBLEMTOOLS_VERSION=develop +FROM ubuntu:24.04 + +LABEL maintainer="contact@kattis.com" +ENV DEBIAN_FRONTEND=noninteractive + +# Packages required to build and run problemtools +# For libgmp, we technically just need libgmpxx4ldbl here, but for readability +# (and we need libgmp-dev in other images), we take libgmp-dev here +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + rm -f /etc/apt/apt.conf.d/docker-clean && \ + apt-get update && apt-get install -y \ + dvisvgm \ + ghostscript \ + libgmp-dev \ + pandoc \ + python3 \ + python3-venv \ + texlive-fonts-recommended \ + texlive-lang-cyrillic \ + texlive-latex-extra \ + texlive-plain-generic \ + tidy diff --git a/admin/docker/README.md b/admin/docker/README.md new file mode 100644 index 00000000..5e2c6c40 --- /dev/null +++ b/admin/docker/README.md @@ -0,0 +1,25 @@ +Our docker images. Note that images depend on each other, please use the script +admin/update_docker.sh to build images in the correct order. + +We have 4 images which are only used locally in the build process, and are not +uploaded to a repository. + - `runreqs`: Base image containing just the things needed to run problemtools + - `build`: Base image containing just the things needed to build a deb and run problemtools + - `icpclangs`: Base image containing what is needed to run problemtools, plus the "ICPC languages" + - `fulllangs`: Base image containing what is needed to run problemtools, plus all supported languages + +We have 3 images which are meant for end users: + - `minimal`: Image with problemtools installed, but no languages. + - `icpc`: Image with problemtools plus the "ICPC languages" installed. + - `full`: Image with problemtools and all languages + +We have 1 image which is used in our CI (to speed up things - it takes a few +minutes to apt-get install all languages and runtime requirements): + - `githubci`: Image with all languages and everything needed to build a deb and run problemtools + +Build dependencies: +``` + runreqs -> icpclangs -> fullangs -> githubci + / \ | | + build minimal icpc full +``` diff --git a/admin/make_release.sh b/admin/make_release.sh index 0343ac27..17ad9416 100755 --- a/admin/make_release.sh +++ b/admin/make_release.sh @@ -1,10 +1,17 @@ #!/usr/bin/env bash # -# Uses git flow and gbp tools (available through Ubuntu packages -# git-flow, git-buildpackage) +# Uses gbp (available through Ubuntu package git-buildpackage) set -e +ALLOW_DIRTY=false +while getopts "d" opt; do + case $opt in + d) ALLOW_DIRTY=true ;; + \?) echo "Invalid option: -$opt" ;; + esac +done + ROOT=$(readlink -f $(dirname $0)/..) VERSION=1.$(date +%Y%m%d) @@ -16,20 +23,50 @@ if [ "$(git tag -l v$VERSION)" != "" ]; then VERSION=$VERSION-rev$REV fi -set -x -git flow release start --showcommands $VERSION +# Steps: +# Pick a version (done by the above loop) +# Update debian/changelog using gbp +# Create and merge pull request with the updated debian/changelog +# Create a github release (using web UI) or gh +# Push to pypi using github action +# Push to docker using admin/update_docker.sh v$VERSION + +CHANGELOG_VERSION=$(dpkg-parsechangelog -l $ROOT/debian/changelog --show-field Version) +if [[ $CHANGELOG_VERSION == $VERSION ]]; then + echo "Debian changelog seems updated" +else + echo "Updating debian changelog (this is surprisingly slow)" + EMAIL=$(git config user.email) gbp dch $ROOT --release --new-version=$VERSION --ignore-branch --git-author --debian-tag='v%(version)s' --debian-branch=release/$VERSION --spawn-editor=never + echo "Please commit the updated changelog, do a pull request, and get it merged, then run this script again on an up-to-date master branch" + exit 0 +fi -# Update _version.py -$ROOT/admin/update_version.py.sh $VERSION +cd $(dirname $(readlink -f $0)) -# Update debian/changelog -gbp dch $ROOT --release --new-version=$VERSION --git-author --debian-tag='v%(version)s' --debian-branch=release/$VERSION --spawn-editor=never +if [[ -n $(git status -s) ]]; then + echo "Repository is dirty." + git status -s + [[ "${ALLOW_DIRTY}" != "true" ]] && exit 1 +fi -git add $ROOT/problemtools/_version.py $ROOT/debian/changelog -git commit -m "Release of version $VERSION: bump version in problemtools/_version.py and debian/changelog" +GITTAG=master +if [[ $(git rev-parse --abbrev-ref HEAD) != ${GITTAG} && $(git describe --exact-match --tags 2>/dev/null) != ${GITTAG} ]]; then + echo "Repository is currently not on branch/tag ${GITTAG}." + [[ "${ALLOW_DIRTY}" != "true" ]] && exit 1 +fi + +THIS_REPO_VERSION=$(git -C $(dirname -- "$0") rev-parse HEAD) +UPSTREAM_VERSION=$(git -C $(dirname -- "$0") ls-remote upstream master | cut -f1) +if [[ $THIS_REPO_VERSION != $UPSTREAM_VERSION ]]; then + echo "Warning: git head of repo does not match upstream. You likely want to update this repo" + [[ "${ALLOW_DIRTY}" != "true" ]] && exit 1 +fi -git flow release finish --showcommands --message "Release $VERSION" $VERSION -echo "After pushing changes to GitHub, please run" -echo " $ROOT/admin/update_docker.sh v$VERSION" +echo "Below is untested, echoing commands instead of running them" +echo "Creating a draft release on github" +echo gh -R Kattis/problemtools release create -d v$VERSION +echo "After finalizing the release on GitHub, please:" +echo " - trigger the pypi release workflow" +echo " - run $ROOT/admin/update_docker.sh v$VERSION" diff --git a/admin/update_docker.sh b/admin/update_docker.sh index ce7e64e2..fe5d2aba 100755 --- a/admin/update_docker.sh +++ b/admin/update_docker.sh @@ -1,51 +1,98 @@ #!/bin/bash set -e -TAG=develop +ALLOW_DIRTY=false +GITTAG=master +DOCKERTAG=develop UPDATE_LATEST=false + +while getopts "d" opt; do + case $opt in + d) ALLOW_DIRTY=true ;; + \?) echo "Invalid option: -$opt" ;; + esac +done + +shift $((OPTIND-1)) + if [ "$1" != "" ]; then - TAG=$1 + GITTAG=$1 + DOCKERTAG=$1 UPDATE_LATEST=true fi - cd $(dirname $(readlink -f $0))/docker -set -x - -# Make the build image and extract build artifacts -# =============================================== -sudo docker build \ - -f Dockerfile.build \ - -t problemtools/build:${TAG} \ - --no-cache \ - --build-arg PROBLEMTOOLS_VERSION="${TAG}" \ - . + +if [[ -n $(git status -s) ]]; then + echo "Repository is dirty." + git status -s + [[ "${ALLOW_DIRTY}" != "true" ]] && exit 1 +fi + +if [[ $(git rev-parse --abbrev-ref HEAD) != ${GITTAG} && $(git describe --exact-match --tags 2>/dev/null) != ${GITTAG} ]]; then + echo "Repository is currently not on branch/tag ${GITTAG}." + [[ "${ALLOW_DIRTY}" != "true" ]] && exit 1 +fi + +echo "Updating Ubuntu base image" +docker pull ubuntu:24.04 + +# Make our internal images, and our githubci image. Order is important, images depend on each other +echo "Building intermediate images, plus githubci image" +for IMAGE in runreqs build icpclangs fulllangs githubci; do + docker build \ + -f Dockerfile.${IMAGE} \ + -t problemtools/${IMAGE}:${DOCKERTAG} \ + --build-arg PROBLEMTOOLS_VERSION="${DOCKERTAG}" \ + . +done + + +echo "Building deb" mkdir -p artifacts -rm -rf artifacts/deb/* -sudo docker run --rm -v "$(pwd)/artifacts/:/artifacts" problemtools/build:${TAG} cp -r /usr/local/problemtools_build/deb /artifacts +sudo rm -rf artifacts/deb +# Use our build image to build a deb +docker run --rm -v "$(pwd)/../..:/problemtools" -v "$(pwd)/artifacts/deb:/artifacts" problemtools/build:${DOCKERTAG} \ + /bin/bash -c " + set -e ; + mkdir /build ; + cd /build ; + git config --global --add safe.directory /problemtools/.git ; + git clone --branch ${GITTAG} /problemtools ; + cd problemtools ; + make builddeb ; + cp ../*.deb /artifacts" sudo chown -R $USER:$USER artifacts/ -# Build the actual problemtools images -# =============================================== -for IMAGE in minimal icpc full; do - sudo docker build\ - -f Dockerfile.${IMAGE}\ - -t problemtools/${IMAGE}:${TAG}\ - --build-arg PROBLEMTOOLS_VERSION=${TAG}\ - . - if [ "$UPDATE_LATEST" = "true" ]; then - sudo docker tag problemtools/${IMAGE}:${TAG} problemtools/${IMAGE}:latest - fi -done +echo "Testing deb" +if ! docker run --rm -t -v "$(pwd)/../..:/problemtools" -v "$(pwd)/artifacts/deb:/artifacts" problemtools/fulllangs:${DOCKERTAG} \ + /bin/bash -c ' + set -e ; + shopt -s extglob ; + dpkg -i /artifacts/kattis-problemtools* ; + verifyproblem /problemtools/examples/!(README.md)'; then + echo Running verifyproblem on all examples failed. Please review output above to debug.; + exit 1 +fi +echo Tests pass -# Push to Docker Hub -# =============================================== -sudo docker login +echo "Building complete images with problemtools baked in" for IMAGE in minimal icpc full; do - sudo docker push problemtools/${IMAGE}:${TAG} - if [ "$UPDATE_LATEST" = "true" ]; then - sudo docker push problemtools/${IMAGE}:latest - fi + docker build \ + -f Dockerfile.${IMAGE} \ + -t problemtools/${IMAGE}:${DOCKERTAG} \ + --build-arg PROBLEMTOOLS_VERSION="${DOCKERTAG}" \ + . done + + +if [ "${UPDATE_LATEST}" = "true" ]; then + echo "Build complete. If you are happy with the images, run the following:" + for IMAGE in minimal icpc full githubci; do + echo " docker tag problemtools/${IMAGE}:${DOCKERTAG} problemtools/${IMAGE}:latest" + echo " docker push problemtools/${IMAGE}:${DOCKERTAG}" + echo " docker push problemtools/${IMAGE}:latest" + done +fi diff --git a/bin/.run_in_venv.sh b/bin/.run_in_venv.sh new file mode 100755 index 00000000..dce93afb --- /dev/null +++ b/bin/.run_in_venv.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# +# Helper script for the other wrapper scripts to check that a venv exists, or +# give a helpful message if it doesn't. + +VENVPATH="$(dirname "$(dirname "$(readlink -f "$0")")")/venv" + +if [ ! -x "$VENVPATH/bin/python" ]; then + echo "I could not find a python venv at $VENVPATH." + echo "To use these wrapper scripts, please set up a venv by:" + echo " cd $(dirname "$VENVPATH")" + echo " python3 -m venv venv" + echo " venv/bin/pip install -r requirements.txt" + exit 1 +fi + +export PYTHONPATH +PYTHONPATH="$(dirname "$(dirname "$(readlink -f "$0")")")${PYTHONPATH:+:}$PYTHONPATH" +exec "$VENVPATH/bin/python" -m "$@" diff --git a/bin/problem2html.sh b/bin/problem2html.sh index eaa9d2d9..4068fbb3 100755 --- a/bin/problem2html.sh +++ b/bin/problem2html.sh @@ -5,6 +5,4 @@ # installing problemtools on the system properly, this script should # not be used. -export PYTHONPATH -PYTHONPATH="$(dirname "$(dirname "$(readlink -f "$0")")")${PYTHONPATH:+:}$PYTHONPATH" -exec python3 -m problemtools.problem2html "$@" +exec "$(dirname "$(readlink -f "$0")")/.run_in_venv.sh" problemtools.problem2html "$@" diff --git a/bin/problem2pdf.sh b/bin/problem2pdf.sh index 949c11e8..8995152d 100755 --- a/bin/problem2pdf.sh +++ b/bin/problem2pdf.sh @@ -5,6 +5,4 @@ # installing problemtools on the system properly, this script should # not be used. -export PYTHONPATH -PYTHONPATH="$(dirname "$(dirname "$(readlink -f "$0")")")${PYTHONPATH:+:}$PYTHONPATH" -exec python3 -m problemtools.problem2pdf "$@" +exec "$(dirname "$(readlink -f "$0")")/.run_in_venv.sh" problemtools.problem2pdf "$@" diff --git a/bin/verifyproblem.sh b/bin/verifyproblem.sh index 364d70c8..48ce5407 100755 --- a/bin/verifyproblem.sh +++ b/bin/verifyproblem.sh @@ -5,6 +5,4 @@ # installing problemtools on the system properly, this script should # not be used. -export PYTHONPATH -PYTHONPATH="$(dirname "$(dirname "$(readlink -f "$0")")")${PYTHONPATH:+:}$PYTHONPATH" -exec python3 -m problemtools.verifyproblem "$@" +exec "$(dirname "$(readlink -f "$0")")/.run_in_venv.sh" problemtools.verifyproblem "$@" diff --git a/debian/changelog b/debian/changelog index 9891d94a..8f0eeec6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,374 @@ +kattis-problemtools (1.20250605) noble; urgency=medium + + [ Tagl ] + * Set working directory for submission + + [ Per Austrin ] + * small fixes to README.md + + [ Tagl ] + * Support overwriting directories within submissions with included directories + + [ JoelNiemela ] + * Simplify problem2html argparser and add type annotations + * Simplify problem2pdf argparser and add type annotations + * Add line break for readability + * Add type annotations to verifyproblem + * Use long name for langparam + * Fix potential error + * Use tuple destructuring syntax + * Don't remove argparser_basic_arguments + * Fix CodeFactor errors + + [ Konráð Elí Sigurgeirsson ] + * Update README.md + + [ Joel Niemelä ] + * Update README.md + + [ JoelNiemela ] + * Update kotlin to version 1.8.10 + + [ Harry Zhang ] + * Fix not showing WA test case verifyproblem.py + + [ Gunnar Kreitz ] + * Fix crash in verifyproblem when in or ans files are not utf-8 + + [ Pehr Söderman ] + * Restructure problem2html + * Restructure restrucure problem2pdf + * Improve debugging output + * Parse args properly. + * Reintroduce the get_subgroup, as it is needed for addproblem. + * Reintroduce get_attachment_paths, as it is needed for addproblem. + + [ Tobias Meggendorfer ] + * Improved logging, fix unicode error, include_dir buildrun + * Guard better agains non-unicode feedback + * Revert delayed import + * Fix missed init, add test case + + [ Maarten Sijm ] + * Explicitly set language versions for Java and Kotlin in languages.yaml + + [ Tobias Meggendorfer ] + * Restore old format + + [ Gunnar Kreitz ] + * Add github action to run pytest and flake8 + * Remove travis config, point badges to github badge + * Check that all symlinks point to something existing within the problem package + * Forbid all absolute symlinks + + [ Simon Lindholm ] + * Type annotation fixes + * Add -j flag for multi-threaded validation + * Compile submissions early, improve cleanup + + [ Pehr Söderman ] + * Adding sanity checks for file sizes. + * Add UUID as an optional field in problem.yaml + * Include examples in the manifest + * Fixing running of tests + * Build debian packages in ci + * Cache and add packages + * Warn if sample is empty + + [ Fredrik Niemelä ] + * Change team to user for default validator + + [ Pehr Söderman ] + * Add a dependency on dvisvgm, which was missing + + [ matistjati ] + * Add markdown support + * Added display math + * Add dependencies for markdown + * Style markdown tables + * Remove temp files + * Statement fix + * Some refactoring + * Added image support in markdown + * Added footnote support + * Code cleanup + * md -> html works + + [ Gunnar Kreitz ] + * Remove non-standard judgeerror.txt from example problems + + [ matistjati ] + * Make md styling more constistent with latex + + [ Pehr Söderman ] + * Bump the language versions for c and c++ + * Bump the language versions for Java + + [ matistjati ] + * md->pdf and Reorganize code + + [ Pehr Söderman ] + * GCC should use gnu17 + + [ matistjati ] + * Better md->pdf tables + * Interactive samples for pdf + * Remove bplusa + * PDF problem name + * Add dependencies + * Add problem names + * Added problem name to test hello package + * Improve security by running pandoc without shell capabilities + * Refactoring + * Even more refactoring + * Remove python3-markdown dependency + * Add problem id to pdf and small fixes + + [ Pehr Söderman ] + * Update languages.yaml + + [ Tagl ] + * Run interactive validation with submission's working directory + + [ Matistjati ] + * Change Rust compilation flags + + [ Hugo Söderbergh ] + * Remove deprecated functionality + * add build to .gitignore + + [ JoelNiemela ] + * Add special case error message when user output file is empty + * Modify error message according to github comment + + [ Hugo Söderbergh ] + * add command-line argument, begin generalizing Problem class + * small fix + * abstract problems further + * catch general exception for detecting problem-format + * Add some documentation + * New abstraction, ProblemPart which makes it easier to implement parts of problems + * Problem is no longer an abstract class + * ProblemStatement now exists for old and new format + * Add TODO for ProblemStatement + * Fix issues with ProblemStatement + * Add some documentation and some final fixes + * Small change + * Allow to give class-type for part in Problem.get + * Whoops small bug crashed code + * Fix bug that crashed multithreading for testcase-validation + * Mark ProblemPart.depends_on() as staticmethod + + [ Matistjati ] + * Disable html + * Change to wikimedia example image + * Sanitize image sources + * Remove SVG dependency + * Better markdown styling + * Better sample styling + * Add \nextsample and \remainingsamples + * Better pdf error handling + * Use {{nextsample}} instead of \nextsample + * Relax image checking (implied by global regex on filenames) + * Add svg dependency + + [ Gunnar Kreitz ] + * Explicitly install build-essential, as deb building blows up on it not being installed + + [ Hugo Söderbergh ] + * fix issues with PR + + [ Gunnar Kreitz ] + * Remove test of verifyproblem.generators (which has been removed) + + [ Hugo Söderbergh ] + * Remove bad break-statement and increase consistency in dictionary access + * more concise regex + * Make Problem constructor default to legacy format + * make tests pass + + [ Matistjati ] + * Add back warning/error logging + + [ Gunnar Kreitz ] + * Add mypy to github workflow + * Change type from list to tuple, helping mypy and being clearer + * Fix name of exception (old one also worked, as parser does import * from Scanner, but felt weird) + * Add type annotations and abstract class markers + * Add getProblemPart for when we need to access problem.classes + * Add python tooling files (and vim swp files) to gitignore + * Fix signatures of run in VIVA and checktestdata to match superclass + * Fix/ignore type errors to let mypy catch errors everywhere but generatedata.py + + [ ElliotRipa ] + * Make cls templates able work with either problem format + * Allow problem statement to use either problem format + * Make template.py detect format version instead + * Provisional updates + * Add formatversion.py + * Minor fixes in imports + * Move version specific functionality to separate file + * Change to flag '-v' for format-version + * Add missing parentheses + * Use dictionary instead of data objects for format data + * Make problem2html.py use -v to specify format version + * Add constants for version names + * Rollback problemset_0.1.cls + * Move initialisation of FORMAT_DATA to setup + * Make formatversion.py use dataobjects instead of dicts + * Fix documentation + * Remove unnecessary initialisation + + [ Matistjati ] + * Start sanitization + apply feedback + * Better sanitization + lots of tests + * problem_statement -> statement + * Better md -> pdf sample rendering + * Another escape + * More careful with images + * Make samplexss more focused + * Experimentally reuse normal LaTeX rendering + * Use problemtools problem2pdf to handle md -> pdf + * Cleanup + * librsvg out of focus for this PR + * Ensure nh3 + * Remove ghostscript sanitization. If it wasn't used before, it probably isn't needed + * Add nh3 to deb build + * Linting + * Add back ghostscript sanitization + * Remove unnecessary test + + [ Gunnar Kreitz ] + * Add make clean to clean up support and the mess left by setuptools + * Change debian packaging to dh_virtualenv + * Update readme, adjusting installation instructions so we can use pip dependencies + * Convert from setup.py to pyproject.toml (and use setuptools-scm for versioning) + * Hook sdist to make python -m build work on a clean checkout + * Update wrapper scripts and README + * Update CI workflow to match readme for build requirements (plus build-essentials) + * Force dh_virtualenv to use builtin venv (debugging CI crash) + * Restructure CI/CD to separate deb building from python unit tests + * Stop exposing __version__, users should use importlib.metadata.version instead + * Hardcode path to python, as dh_virtualenv fails to discover it in CI + * Clean up version parsing. Accept 2023-07-draft and 2023-07 version strings + * Add pydantic models for parsing problem.yaml + * Limit problem.yaml config to only system defaults + * Use new metadata parsing mechanism, and start parsing config for 2023-07. + * Bump python version to 3.11 + * Move tests to outside of the package + * Update manifest to include tests support files in sdist, and remove some clutter + * Remove old hack for plasTeX argument (we require >= 3.0 now) + * Clean up incompletely removed plastex_escape hack. Remove unused variable + * Clean up unused variables, old io import, and multiple commands on lines + * Clean up unused import and comparison with None + * Ruff format + * Clean up imports + * Fix minor things flagged by ruff + * Remove unused variables in tests + * Add ruff configuration + * Apply ruff formatting + * Add ruff pre-commit hook + * Replace flake8 with ruff (both linting and formatting) + * Fix incorrect formatting of pydantic errors + * Move is_interactive and is_scoring to be read from problem metadata directly + * Let Problem read and store problem format information. Warn about incomplete 2023-07 support + * Fix validator discovery for 2023-07. Run through all validation for 2023-07 (even if broken) + + [ Matistjati ] + * Add nh3 as dependency + * Fix test import path + * Apply ruff formatting + * More robust footnote finding + * Don't double-escape HTML in samples + * Ghostscript fixes and tests + + [ Gunnar Kreitz ] + * Fix loading a problem with empty problem.yaml and with no statements + * Add utility method to load problem metadata, including names from statements when needed + * Use load_metadata in verifyproblem. Add temporary fallback conf to fix crashes when failing to load metadata + * Use load_metadata in statement_util + * Add apt-get update in workflow to unbreak CI + + [ Matistjati ] + * Convert some example problems to 2023-07-draft + * Add uuid to guess and oddecho + * Better formatting and error for output_validators + + [ Pehr Söderman ] + * Add missing build requirements to debian build + * Update pyproject + + [ Matistjati ] + * Remove now-duplicated import + + [ Gunnar Kreitz ] + * Remove (AFAICT, broken) support for ancient tex statements (0.1) + * Fix bug where we complained about missing show_test_data_groups for non-legacy + * Default language to en. Remove unused --format-version + * Pass Template a filename to render, and pass that through to the latex template + * Rename problem.md to problem.en.md in tests to follow 2023-07-draft + * Refactor of rendering: unify statement finding code, and use Path more + * Use statement_util to find statements. Add more checks. Try rendering even when there are multiple statements in a language + * Make mypy more picky, also checking PlasTeX usage + * Simplify temporary file usage in markdown -> pdf flow + * Fix bug where problem2html cd:s to bad directory, crashing validation of multiple problems + + [ Pehr Söderman ] + * Update link to kattis controlled domain. + + [ Gunnar Kreitz ] + * Replace formatversion.FormatData with a StrEnum + * Add some documentation in the readme regarding current state of format versions + * Add colorlog to get colors for warnings and errors #312 + * Add Swedish problem names + * Fix the logging plasTeX destroys + * Remove accidental commit + * Fix misleading error when missing problem statemetns + * Improve image handling in markdown statements + * Change URL to one that passes filename suffix filter + * Restructure error counters. Fix errors happening prior to check being ignored in count. + * Fix --bail_on_error and --werror being ignored before check + * Refactor problem loading so we can do fatal errors in setup + * Check file and directory names per standard + * Make missing/compilation failure in grader/output validator fatal + * Restore old API for accessing parts of a problem. Simplify part setup + * Change type of attachments.attachments from list[str] to list[Path] + * Convert Problem.metadata to a property to align better with other naming + * Expose computed timelim + * Add back problemtools.run.get_tool_path to API + * Large restructure of how our docker images are built. + * Remove old Dockerfile in root, unused afaict + * Add marker to let mypy use our type annotations + * Replace authors with Kattis AB (pypi only shows one). Set required python version + * Add script to build packages for pypi + * Fix incorrect error when verifying different. Add helpful hint when directory for wrong version exists + * Use problemtools/githubci image in workflow. Run verifyproblem on all examples. + * Check for incompatible types. Warn for unimplemented types + * Check format of interaction samples. #277 Don't warn about empty sample when it contains interactions. + * Add type methods for all types. Add convenience methods on Problem for easier access + * Improve warning for non-standard output validator languages #258 + * Remove generatedata (never made it into the standard) + * Fix pytest dropping a guess.pdf in working directory. Check PDF magic bytes + * Add -d flag to update_docker to allow easier testing locally + * Fix broken git clone command (`${TAG}` expanded to empty string) + * Fix silly error in docker file, causing apt-get update not to run + * Add -d option to allow building in a dirty rep (to facilitate development of build scripts) + * Workflow that builds and pushes a package to testpypi + * Fix version computation when we build pypi packages + * Fix syntax error in github workflow file + * Fix bug where we crashed if we attempted to load/check twice + * Error if problem name exits in a language without a statement + * Add utility function uses_default_validator for output validation. Warn/error on multiple validators + * Fix new mypy error in mypy 1.16 + * Fix missing support for imgbasedir in md2html + * Fix typo in Dockerfile.full causing it to lack a lot of languages + * Remove year from license - IANAL, but AFAICT it's not needed + * Add warning to pypi package script pointing to the github action now that that's set up + * Change update_docker to default to building from master (but keep :develop tag on docker) + * Add convenient way to run docker. Document need to install languages. + + -- Gunnar Kreitz Thu, 05 Jun 2025 10:59:57 +0200 + kattis-problemtools (1.20231016) jammy; urgency=medium [ Don-Khue Le ] diff --git a/debian/compat b/debian/compat deleted file mode 100644 index ec635144..00000000 --- a/debian/compat +++ /dev/null @@ -1 +0,0 @@ -9 diff --git a/debian/control b/debian/control index 0a635887..a6b27d74 100644 --- a/debian/control +++ b/debian/control @@ -2,13 +2,13 @@ Source: kattis-problemtools Section: devel Priority: optional Maintainer: Per Austrin -Build-Depends: debhelper (>= 8.0.0), g++ (>= 4.8), dh-python, python3, python3-setuptools, python3-pytest, python3-yaml, python3-setuptools, python3-pytest, libboost-regex-dev, libgmp-dev, automake, autoconf +Build-Depends: debhelper-compat (= 13), g++ (>= 4.8), dh-virtualenv, python3, libboost-regex-dev, libgmp-dev, automake, autoconf, git, python3-venv, dpkg-dev Standards-Version: 3.9.4 Homepage: https://github.com/Kattis/problemtools Package: kattis-problemtools Architecture: any -Depends: ${shlibs:Depends}, ${python3:Depends}, ${misc:Depends}, python3-plastex, python3-pkg-resources, texlive-plain-generic, texlive-fonts-recommended, texlive-latex-extra, texlive-lang-cyrillic, tidy, ghostscript +Depends: ${shlibs:Depends}, ${misc:Depends}, pandoc, python3, texlive-plain-generic, texlive-fonts-recommended, texlive-latex-extra, texlive-lang-cyrillic, tidy, ghostscript, dvisvgm Recommends: gcc, g++ Description: Kattis Problem Tools These are tools to manage and verify problem packages in the diff --git a/debian/kattis-problemtools.links b/debian/kattis-problemtools.links new file mode 100644 index 00000000..39354ac1 --- /dev/null +++ b/debian/kattis-problemtools.links @@ -0,0 +1,3 @@ +opt/venvs/kattis-problemtools/bin/verifyproblem usr/bin/verifyproblem +opt/venvs/kattis-problemtools/bin/problem2pdf usr/bin/problem2pdf +opt/venvs/kattis-problemtools/bin/problem2html usr/bin/problem2html diff --git a/debian/rules b/debian/rules index 76a72e36..2c10725f 100755 --- a/debian/rules +++ b/debian/rules @@ -4,12 +4,17 @@ # Uncomment this to turn on verbose mode. #export DH_VERBOSE=1 -# Uncomment this to turn off cleanup. -export PYBUILD_DISABLE=clean +%: + dh $@ --with python-virtualenv -export PYBUILD_AFTER_CLEAN=make -C support distclean -export PYBUILD_TEST_PYTEST=1 -export no_proxy=github.com +override_dh_virtualenv: + dh_virtualenv --builtin-venv --python /usr/bin/python3 -%: - dh $@ --with python3 --buildsystem=pybuild +override_dh_strip: + dh_strip --exclude=/PIL/ --exclude=/pillow.libs/ + +override_dh_shlibdeps: + dh_shlibdeps -X/x86/ -X/PIL/.libs/ -X/pillow.libs/ + +override_dh_dwz: + dh_dwz --exclude=/PIL/ --exclude=/pillow.libs/ diff --git a/examples/README.md b/examples/README.md index 2f6107a3..9d7f9ee5 100644 --- a/examples/README.md +++ b/examples/README.md @@ -24,4 +24,5 @@ more than one language. ## oddecho This is an example of a *scoring* problem where submissions can get -different scores depending on which test groups they solve. It also demonstrates how an input validator might check different constraints for different test groups. +different scores depending on which test groups they solve. It also demonstrates how an input validator might check different constraints for different test groups. The swedish statement showcases how to use images, footnotes +and tables in Markdown. diff --git a/examples/different/output_validators/different_validator/validate.h b/examples/different/output_validators/different_validator/validate.h index 00c896a7..4f653ff1 100644 --- a/examples/different/output_validators/different_validator/validate.h +++ b/examples/different/output_validators/different_validator/validate.h @@ -56,7 +56,6 @@ const int EXITCODE_AC = 42; const int EXITCODE_WA = 43; const std::string FILENAME_AUTHOR_MESSAGE = "teammessage.txt"; const std::string FILENAME_JUDGE_MESSAGE = "judgemessage.txt"; -const std::string FILENAME_JUDGE_ERROR = "judgeerror.txt"; const std::string FILENAME_SCORE = "score.txt"; #define USAGE "%s: judge_in judge_ans feedback_dir < author_out\n" @@ -107,7 +106,7 @@ void wrong_answer(const std::string &msg, ...) { void judge_error(const std::string &msg, ...) { va_list pvar; va_start(pvar, msg); - vreport_feedback(FILENAME_JUDGE_ERROR, msg, pvar); + vreport_feedback(FILENAME_JUDGE_MESSAGE, msg, pvar); assert(0); } diff --git a/examples/different/problem.yaml b/examples/different/problem.yaml index 279a8acb..c67e0aa3 100644 --- a/examples/different/problem.yaml +++ b/examples/different/problem.yaml @@ -5,6 +5,8 @@ ## Author of the problem (default: null) # author: +name: A Different Problem + ## Where the problem was first used (default: null) source: Kattis # source_url: diff --git a/examples/guess/output_validators/guess_validator/validate.cc b/examples/guess/output_validator/guess_validator/validate.cc similarity index 100% rename from examples/guess/output_validators/guess_validator/validate.cc rename to examples/guess/output_validator/guess_validator/validate.cc diff --git a/examples/guess/output_validators/guess_validator/validate.h b/examples/guess/output_validator/guess_validator/validate.h similarity index 97% rename from examples/guess/output_validators/guess_validator/validate.h rename to examples/guess/output_validator/guess_validator/validate.h index 00c896a7..4f653ff1 100644 --- a/examples/guess/output_validators/guess_validator/validate.h +++ b/examples/guess/output_validator/guess_validator/validate.h @@ -56,7 +56,6 @@ const int EXITCODE_AC = 42; const int EXITCODE_WA = 43; const std::string FILENAME_AUTHOR_MESSAGE = "teammessage.txt"; const std::string FILENAME_JUDGE_MESSAGE = "judgemessage.txt"; -const std::string FILENAME_JUDGE_ERROR = "judgeerror.txt"; const std::string FILENAME_SCORE = "score.txt"; #define USAGE "%s: judge_in judge_ans feedback_dir < author_out\n" @@ -107,7 +106,7 @@ void wrong_answer(const std::string &msg, ...) { void judge_error(const std::string &msg, ...) { va_list pvar; va_start(pvar, msg); - vreport_feedback(FILENAME_JUDGE_ERROR, msg, pvar); + vreport_feedback(FILENAME_JUDGE_MESSAGE, msg, pvar); assert(0); } diff --git a/examples/guess/problem.yaml b/examples/guess/problem.yaml index fcb51934..8744f5e7 100644 --- a/examples/guess/problem.yaml +++ b/examples/guess/problem.yaml @@ -1,10 +1,16 @@ +problem_format_version: 2023-07-draft +uuid: 5ca6ba5b-36d5-4eff-8aa7-d967cbc4375e source: Kattis license: cc by-sa -validation: custom interactive +type: interactive +name: + en: Guess the Number + sv: Gissa talet # Override standard limits: say that the TLE solutions provided should # be at least 4 times above the time limit in order for us to be # happy. limits: - time_safety_margin: 4 + time_multipliers: + time_limit_to_tle: 4 diff --git a/examples/guess/problem_statement/problem.en.tex b/examples/guess/statement/problem.en.tex similarity index 100% rename from examples/guess/problem_statement/problem.en.tex rename to examples/guess/statement/problem.en.tex diff --git a/examples/guess/statement/problem.sv.md b/examples/guess/statement/problem.sv.md new file mode 100644 index 00000000..9c49030c --- /dev/null +++ b/examples/guess/statement/problem.sv.md @@ -0,0 +1,20 @@ +Jag tänker på ett hemligt tal mellan $1$ and $100$, kan du gissa vilket? +Givet en gissning kommer jag att berätta om din gissning +var för stor, för liten eller rätt. Du får bara $10$ gissningar, använd +dem klokt! + + +# Interaktion +Ditt program ska skriva ut gissningar om talet. +En gissning är en rad som enbart innehåller ett heltal mellan $1$ och $1000$. +Efter varje gissning måste du flusha standard out. + +Efter varje gissning kan du läs svaret på standard in. +Detta svar är ett av tre ord: + +- `lower` om talet jag tänker på är lägre än din gissning, +- `higher` om talet jag tänker på är högre än din gissning, eller +- `correct` om din gissning är korrekt. + +Efter att ha gissat rätt ska du avsluta ditt program. +Om du gissar fel $10$ gånger får du inga fler chanser och ditt program kommer avbrytas. diff --git a/examples/hello/problem.yaml b/examples/hello/problem.yaml index 194b060f..bc12a981 100644 --- a/examples/hello/problem.yaml +++ b/examples/hello/problem.yaml @@ -1,5 +1,6 @@ source: Kattis license: public domain +name: Hello World! # Fix memory limit at 512 MB. (Note that for most problems, this # should not be done. It is only done in this case because we include diff --git a/examples/hello/submissions/accepted/hello.rs b/examples/hello/submissions/accepted/hello.rs new file mode 100644 index 00000000..47ad8c63 --- /dev/null +++ b/examples/hello/submissions/accepted/hello.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello World!"); +} diff --git a/examples/oddecho/input_format_validators/validator/validator.cpp b/examples/oddecho/input_validators/validator/validator.cpp similarity index 100% rename from examples/oddecho/input_format_validators/validator/validator.cpp rename to examples/oddecho/input_validators/validator/validator.cpp diff --git a/examples/oddecho/input_format_validators/validator/validator.h b/examples/oddecho/input_validators/validator/validator.h similarity index 100% rename from examples/oddecho/input_format_validators/validator/validator.h rename to examples/oddecho/input_validators/validator/validator.h diff --git a/examples/oddecho/problem.yaml b/examples/oddecho/problem.yaml index 1fcd5e21..06b2e7df 100644 --- a/examples/oddecho/problem.yaml +++ b/examples/oddecho/problem.yaml @@ -1,7 +1,9 @@ +problem_format_version: 2023-07-draft +uuid: 025dfeea-eb85-4532-94d1-3108ec03c80f license: cc by-sa -author: Johan Sannemo +credits: Johan Sannemo source: Principles of Algorithmic Problem Solving type: scoring -name: Echo -grading: - show_test_data_groups: true +name: + en: Odd Echo + sv: Udda eko diff --git a/examples/oddecho/statement/cave.jpg b/examples/oddecho/statement/cave.jpg new file mode 100644 index 00000000..670bbeda Binary files /dev/null and b/examples/oddecho/statement/cave.jpg differ diff --git a/examples/oddecho/problem_statement/problem.en.tex b/examples/oddecho/statement/problem.en.tex similarity index 100% rename from examples/oddecho/problem_statement/problem.en.tex rename to examples/oddecho/statement/problem.en.tex diff --git a/examples/oddecho/statement/problem.sv.md b/examples/oddecho/statement/problem.sv.md new file mode 100644 index 00000000..55f9806f --- /dev/null +++ b/examples/oddecho/statement/problem.sv.md @@ -0,0 +1,33 @@ +**EKO! Eko! Ek...** + +![CC-BY-SA 2.0 By William Craig on wikimedia.org](cave.jpg) + +Du älskar att skrika i grottor för att höra dina ord ekade tillbaka till dig. Tyvärr, som en hårt arbetande mjukvaruingenjör, har du +inte tid för att komma ut och skrika i grottor så ofta. Istället skulle du vilja implementera ett program som fungerar som en ersättning för en grotta. + +Ibland vill du mata in några ord i programmet och få dem ekade tillbaka till dig. Men, som det är välkänt, om du skriker för snabbt i en grotta kan ekot störa de nya ord du säger. [^1] Mer specifikt, vartannat ord du säger kommer att störa ekot av ditt tidigare ord. Därför kommer endast det första, tredje, femte och så vidare ordet faktiskt att producera ett eko. + +Din uppgift är att skriva ett program som simulerar detta beteende. + +## Indata + +Den första raden av indata innehåller ett heltal $N$ ($1 \le N \le 10$). + +De följande $N$ raderna innehåller vardera ett ord. Varje ord är högst $100$ bokstäver långt och innehåller endast bokstäverna `a-z`. + +## Utdata + +Skriv ut de ord som har udda index (dvs. första, tredje, femte och så vidare) i inmatningen. + + +## Poängsättning + +Din lösning kommer att testas på en mängd testfallsgrupper. +För att få poäng för en grupp så måste du klara alla testfall i gruppen. + +| Grupp | Poäng | Begränsningar | +|-------|-------|--------------------------| +| 1 | 1 | $N$ är alltid $5$ | +| 2 | 1 | Inga ytterligare begränsningar | + +[^1]: [https://sv.wikipedia.org/wiki/Interferens](https://sv.wikipedia.org/wiki/Interferens) diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 00000000..1a4c5a69 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,6 @@ +[mypy] +ignore_missing_imports = False +follow_untyped_imports = True +install_types = True +check_untyped_defs = True +ignore_errors = False diff --git a/problemtools/ProblemPlasTeX/ProblemsetMacros.py b/problemtools/ProblemPlasTeX/ProblemsetMacros.py index 31794c98..b5bd4f41 100644 --- a/problemtools/ProblemPlasTeX/ProblemsetMacros.py +++ b/problemtools/ProblemPlasTeX/ProblemsetMacros.py @@ -1,16 +1,15 @@ import sys import os import os.path -import io from plasTeX.DOM import Node from plasTeX.Base import Command from plasTeX.Base import DimenCommand from plasTeX.Logging import getLogger -import plasTeX.Packages.graphics as graphics log = getLogger() status = getLogger('status') + # Ugly hack: assume textwidth is 600pt. True for Kattis but not in # general. class textwidth(DimenCommand): @@ -25,7 +24,7 @@ def clean_width(width): nodes = width.childNodes if len(nodes) != 2 or nodes[1].nodeName != 'textwidth': return width - return '%.2f%%' % (100*float(nodes[0])) + return '%.2f%%' % (100 * float(nodes[0])) # \problemheader @@ -33,9 +32,8 @@ class problemheader(Command): args = 'title id:str' def invoke(self, tex): - res = Command.invoke(self, tex) - timelimfile = os.path.join(os.path.dirname(tex.filename), - '..', '.timelimit') + super().invoke(tex) + timelimfile = os.path.join(os.path.dirname(tex.filename), '..', '.timelimit') if os.path.isfile(timelimfile): self.attributes['timelim'] = open(timelimfile, 'r').read() @@ -45,10 +43,10 @@ class sampletable(Command): args = 'header1 file1:str header2 file2:str' def read_sample_file(self, filename): - return io.open(filename, 'r', encoding='utf-8').read() + return open(filename, 'r', encoding='utf-8').read() def invoke(self, tex): - res = Command.invoke(self, tex) + super().invoke(tex) dir = os.path.dirname(tex.filename) file1 = os.path.join(dir, self.attributes['file1']) file2 = os.path.join(dir, self.attributes['file2']) @@ -67,28 +65,32 @@ class sampletableinteractive(Command): args = 'header read write file:str' def read_sample_interaction(self, filename): - data = io.open(filename, 'r', encoding='utf-8').read() + data = open(filename, 'r', encoding='utf-8').read() messages = [] - cur_msg = [] + cur_msg: list[str] = [] cur_mode = None for line in data.split('\n'): - if not line: continue - if line[0] == '<': mode = 'read' - elif line[0] == '>': mode = 'write' - else: continue + if not line: + continue + if line[0] == '<': + mode = 'read' + elif line[0] == '>': + mode = 'write' + else: + continue line = line[1:] if mode != cur_mode: - if cur_mode: messages.append({'mode': cur_mode, - 'data': '\n'.join(cur_msg)}) + if cur_mode: + messages.append({'mode': cur_mode, 'data': '\n'.join(cur_msg)}) cur_msg = [] cur_msg.append(line) cur_mode = mode - if cur_mode: messages.append({'mode': cur_mode, - 'data': '\n'.join(cur_msg)}) + if cur_mode: + messages.append({'mode': cur_mode, 'data': '\n'.join(cur_msg)}) return messages def invoke(self, tex): - res = Command.invoke(self, tex) + super().invoke(tex) dir = os.path.dirname(tex.filename) file = os.path.join(dir, self.attributes['file']) try: @@ -104,21 +106,19 @@ def invoke(self, tex): # \includegraphics implementation) class _graphics_command(Command): def invoke(self, tex): - res = Command.invoke(self, tex) + res = super().invoke(tex) # Overcome plasTeX bug by looking for love in the right place + assert self.ownerDocument is not None # Keep mypy happy basetex = self.ownerDocument.userdata['base_tex_instance'] f = self.attributes['file'] - ext = self.ownerDocument.userdata.getPath( - 'packages/graphicx/extensions', - ['.png', '.jpg', '.jpeg', '.gif', '.pdf']) - paths = self.ownerDocument.userdata.getPath( - 'packages/graphicx/paths', [os.path.dirname(basetex.filename)]) - img = None + ext = self.ownerDocument.userdata.getPath('packages/graphicx/extensions', ['.png', '.jpg', '.jpeg', '.gif', '.pdf']) + paths = self.ownerDocument.userdata.getPath('packages/graphicx/paths', [os.path.dirname(basetex.filename)]) + img: str | None = None # Check for file using graphicspath for p in paths: - for e in ['']+ext: - fname = os.path.join(p, f+e) + for e in [''] + ext: + fname = os.path.join(p, f + e) if os.path.isfile(fname): img = os.path.abspath(fname) break @@ -127,14 +127,14 @@ def invoke(self, tex): # Check for file using kpsewhich if img is None: - for e in ['']+ext: + for e in [''] + ext: try: - img = os.path.abspath(basetex.kpsewhich(f+e)) + img = os.path.abspath(basetex.kpsewhich(f + e)) break except (OSError, IOError): pass - if not os.path.isfile(img): + if img is None or not os.path.isfile(img): log.warning('Could not identify image "%s"' % f) self.imageoverride = img @@ -147,17 +147,20 @@ class illustration(_graphics_command): def invoke(self, tex): res = _graphics_command.invoke(self, tex) - self.style['width'] = '%.2f%%' % (100*self.attributes['width']) + self.style['width'] = '%.2f%%' % (100 * self.attributes['width']) return res + # Dummy for \fontencoding to suppress warnings class fontencoding(Command): args = 'charset:str' + # Dummy for \selectfont to suppress warnings. class selectfont(Command): pass + # Dummy for \ExecuteOptions to suppress warnings. class ExecuteOptions(Command): pass diff --git a/problemtools/ProblemPlasTeX/__init__.py b/problemtools/ProblemPlasTeX/__init__.py index f0a608e7..b136ab7a 100644 --- a/problemtools/ProblemPlasTeX/__init__.py +++ b/problemtools/ProblemPlasTeX/__init__.py @@ -2,7 +2,6 @@ import os import shutil import subprocess -import plasTeX.Renderers from plasTeX.Renderers.PageTemplate import Renderer from plasTeX.Filenames import Filenames from plasTeX.Imagers import Image @@ -10,15 +9,15 @@ log = getLogger() + # Adapted from plasTeX.Imagers.Imager class class ImageConverter(object): fileExtension = '.png' imageAttrs = '' imageUnits = '' - imageTypes = ['.png', '.jpg', '.jpeg', '.gif'] #, '.svg'] - imageConversion = {'.pdf': ['.png', - ['gs', '-dUseCropBox', '-sDEVICE=pngalpha', '-r300', '-o']]} + imageTypes = ['.png', '.jpg', '.jpeg', '.gif'] # , '.svg'] + imageConversion = {'.pdf': ('.png', ['gs', '-dUseCropBox', '-sDEVICE=pngalpha', '-r300', '-o'])} def __init__(self, document): self.config = document.config @@ -28,10 +27,13 @@ def __init__(self, document): self.staticimages = {} # Filename generator - self.newFilename = Filenames(self.config['images'].get('filenames'), - None, - variables={'jobname': document.userdata.get('jobname', '')}, - extension=self.fileExtension, invalid={}) + self.newFilename = Filenames( + self.config['images'].get('filenames'), + None, + variables={'jobname': document.userdata.get('jobname', '')}, + extension=self.fileExtension, + invalid={}, + ) def close(self): return @@ -55,14 +57,14 @@ def getImage(self, node): if oldext in self.imageConversion: # Need to convert image newext = self.imageConversion[oldext][0] - path = os.path.splitext(path)[0]+newext + path = os.path.splitext(path)[0] + newext cmd = self.imageConversion[oldext][1] + [path, name] status = subprocess.call(cmd) if status: log.warning('Failed to convert %s image "%s to %s', oldext, name, newext) else: # Just copy it - path = os.path.splitext(path)[0]+oldext + path = os.path.splitext(path)[0] + oldext shutil.copyfile(name, path) img = Image(path, self.ownerDocument.config['images']) self.staticimages[name] = img @@ -74,25 +76,25 @@ def getImage(self, node): return None - - class ProblemRenderer(Renderer): - """ Renderer for ProblemHTML documents """ + """Renderer for ProblemHTML documents""" fileExtension = '.html' imageTypes = ['.png', '.jpg', '.jpeg', '.gif'] vectorImageTypes = ['.svg'] - def render(self, document): - templatepaths = [os.path.join(os.path.dirname(__file__), '../templates/html'), - os.path.join(os.path.dirname(__file__), '../../templates/html'), - '/usr/lib/problemtools/templates/html'] + def render(self, document, postProcess=None): + templatepaths = [ + os.path.join(os.path.dirname(__file__), '../templates/html'), + os.path.join(os.path.dirname(__file__), '../../templates/html'), + '/usr/lib/problemtools/templates/html', + ] templatepath = None for p in templatepaths: if os.path.isdir(p): templatepath = p break - if templatepath == None: + if templatepath is None: raise Exception('Could not find templates needed for conversion to HTML') # Ugly but unfortunately PlasTeX is quite inflexible when it comes to @@ -113,8 +115,7 @@ def processFileContent(self, document, s): s = Renderer.processFileContent(self, document, s) # Force XHTML syntax on empty tags - s = re.compile(r'(<(?:hr|br|img|link|meta)\b.*?)\s*/?\s*(>)', - re.I|re.S).sub(r'\1 /\2', s) + s = re.compile(r'(<(?:hr|br|img|link|meta)\b.*?)\s*/?\s*(>)', re.I | re.S).sub(r'\1 /\2', s) # Remove empty paragraphs s = re.compile(r'

\s*

', re.I).sub(r'', s) diff --git a/problemtools/ProblemPlasTeX/graphicx.py b/problemtools/ProblemPlasTeX/graphicx.py index 6fa954d4..e9669397 100644 --- a/problemtools/ProblemPlasTeX/graphicx.py +++ b/problemtools/ProblemPlasTeX/graphicx.py @@ -1,9 +1,10 @@ import plasTeX.Packages.graphics as graphics -from ProblemsetMacros import _graphics_command, clean_width +from problemtools.ProblemPlasTeX.ProblemsetMacros import _graphics_command, clean_width # Reimplementation of graphicx package because plasTeX is broken and # annoying. + class includegraphics(_graphics_command): args = '* [ options:dict ] file:str' packageName = 'graphicx' @@ -21,8 +22,10 @@ def invoke(self, tex): self.style['width'] = clean_width(width) return res + class DeclareGraphicsExtensions(graphics.DeclareGraphicsExtensions): packageName = 'graphicx' + class graphicspath(graphics.graphicspath): packageName = 'graphicx' diff --git a/problemtools/ProblemPlasTeX/import.py b/problemtools/ProblemPlasTeX/import.py index 5005936a..e2d4e14f 100644 --- a/problemtools/ProblemPlasTeX/import.py +++ b/problemtools/ProblemPlasTeX/import.py @@ -6,6 +6,7 @@ log = getLogger() status = getLogger('status') + # (Partial) implementation of import.sty because plasTeX does not ship # with an implementation. Only implement \import command which is the # only one we'll use. diff --git a/problemtools/ProblemPlasTeX/listingsutf8.py b/problemtools/ProblemPlasTeX/listingsutf8.py index 2a2d1f56..d022ef03 100644 --- a/problemtools/ProblemPlasTeX/listingsutf8.py +++ b/problemtools/ProblemPlasTeX/listingsutf8.py @@ -4,23 +4,21 @@ import os import io -import ProblemsetMacros - log = getLogger() # Implementation of (parts) of listingsutf8 package since PlasTeX does # not have one + class lstinputlisting(Command): args = '* [ options:dict ] file:str' - def read_file(self, filename): - data = io.open(filename, 'r', encoding='utf-8').read() - data = ProblemsetMacros.plastex_escape(data) - return data + def read_file(self, filename) -> str: + return io.open(filename, 'r', encoding='utf-8').read() - def invoke(self, tex): - res = Command.invoke(self, tex) + def invoke(self, tex) -> None: + super().invoke(tex) + assert self.ownerDocument is not None # Keep mypy happy basetex = self.ownerDocument.userdata['base_tex_instance'] f = self.attributes['file'] # Maybe more paths to look in? diff --git a/problemtools/ProblemPlasTeX/ulem.py b/problemtools/ProblemPlasTeX/ulem.py index c2d9f79f..f891f86b 100644 --- a/problemtools/ProblemPlasTeX/ulem.py +++ b/problemtools/ProblemPlasTeX/ulem.py @@ -1,11 +1,29 @@ from plasTeX.Base.LaTeX.FontSelection import TextCommand -class uline(TextCommand): pass -class uuline(TextCommand): pass -class uwave(TextCommand): pass -class sout(TextCommand): pass -class xout(TextCommand): pass -class dashuline(TextCommand): pass -class dotuline(TextCommand): pass +class uline(TextCommand): + pass + +class uuline(TextCommand): + pass + + +class uwave(TextCommand): + pass + + +class sout(TextCommand): + pass + + +class xout(TextCommand): + pass + + +class dashuline(TextCommand): + pass + + +class dotuline(TextCommand): + pass diff --git a/problemtools/__init__.py b/problemtools/__init__.py index 8dee4bf8..e69de29b 100644 --- a/problemtools/__init__.py +++ b/problemtools/__init__.py @@ -1 +0,0 @@ -from ._version import __version__ diff --git a/problemtools/_version.py b/problemtools/_version.py deleted file mode 100644 index 98cf094a..00000000 --- a/problemtools/_version.py +++ /dev/null @@ -1,2 +0,0 @@ -# Auto-generated from git changelog, do not edit! -__version__ = '1.20231016' diff --git a/problemtools/config.py b/problemtools/config.py index bcc74896..8cd470c9 100644 --- a/problemtools/config.py +++ b/problemtools/config.py @@ -24,12 +24,11 @@ def load_config(configuration_file): try: with open(path, 'r') as config: new_config = yaml.safe_load(config.read()) - except (yaml.parser.ParserError, yaml.parser.ScannerError) as err: + except (yaml.parser.ParserError, yaml.scanner.ScannerError) as err: raise ConfigError('Config file %s: failed to parse: %s' % (path, err)) if res is None: if new_config is None: - raise ConfigError('Base configuration file %s not found in %s' - % (configuration_file, path)) + raise ConfigError('Base configuration file %s not found in %s' % (configuration_file, path)) res = new_config elif new_config is not None: __update_dict(res, new_config) @@ -43,11 +42,11 @@ def __config_file_paths(): priority (i.e., any config in the last path should take precedence over the others). """ - return [os.path.join(os.path.dirname(__file__), 'config'), - os.path.join('/etc', 'kattis', 'problemtools'), - os.path.join(os.environ.get('XDG_CONFIG_HOME', - os.path.join(os.path.expanduser('~'), '.config')), - 'problemtools')] + return [ + os.path.join(os.path.dirname(__file__), 'config'), + os.path.join('/etc', 'kattis', 'problemtools'), + os.path.join(os.environ.get('XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), '.config')), 'problemtools'), + ] def __update_dict(orig, update): @@ -58,10 +57,8 @@ def __update_dict(orig, update): For all other entries (k, v), orig[k] is set to v. """ - for (key, value) in update.items(): - if (key in orig and - isinstance(value, collections.abc.Mapping) and - isinstance(orig[key], collections.abc.Mapping)): + for key, value in update.items(): + if key in orig and isinstance(value, collections.abc.Mapping) and isinstance(orig[key], collections.abc.Mapping): __update_dict(orig[key], value) else: orig[key] = value diff --git a/problemtools/config/languages.yaml b/problemtools/config/languages.yaml index caa6dbbc..de1a6411 100644 --- a/problemtools/config/languages.yaml +++ b/problemtools/config/languages.yaml @@ -98,14 +98,14 @@ c: name: 'C' priority: 950 files: '*.c' - compile: '/usr/bin/gcc -g -O2 -std=gnu99 -static -o {binary} {files} -lm' + compile: '/usr/bin/gcc -g -O2 -std=gnu17 -static -o {binary} {files} -lm' run: '{binary}' cpp: name: 'C++' priority: 1000 files: '*.cc *.C *.cpp *.cxx *.c++' - compile: '/usr/bin/g++ -g -O2 -std=gnu++17 -static -o {binary} {files}' + compile: '/usr/bin/g++ -g -O2 -std=gnu++23 -static -o {binary} {files} -lrt -Wl,--whole-archive -lpthread -Wl,--no-whole-archive' run: '{binary}' csharp: @@ -147,7 +147,7 @@ java: name: 'Java' priority: 800 files: '*.java' - compile: '/usr/bin/javac -encoding UTF-8 -sourcepath {path} -d {path} {files}' + compile: '/usr/bin/javac -source 21 -encoding UTF-8 -sourcepath {path} -d {path} {files}' run: '/usr/bin/java -Dfile.encoding=UTF-8 -XX:+UseSerialGC -Xss64m -Xms{memlim}m -Xmx{memlim}m -cp {path} {mainclass}' javascript: @@ -161,7 +161,7 @@ kotlin: name: 'Kotlin' priority: 250 files: '*.kt' - compile: '/usr/bin/kotlinc -d {path}/ -- {files}' + compile: '/usr/bin/kotlinc -language-version 1.3 -d {path}/ -- {files}' run: '/usr/bin/kotlin -Dfile.encoding=UTF-8 -J-XX:+UseSerialGC -J-Xss64m -J-Xms{memlim}m -J-Xmx{memlim}m -cp {path}/ {Mainclass}Kt' lisp: @@ -209,27 +209,27 @@ prolog: # Python2 with shebang comes before default python3. python2_with_shebang: - name: 'Python 2' + name: 'Python 2 (w/PyPy)' priority: 860 files: '*.py *.py2' shebang: '^#!.*python2\b' - compile: '/usr/bin/python2 -m py_compile {files}' - run: '/usr/bin/python2 "{mainfile}"' + compile: '/usr/bin/pypy -m py_compile {files}' + run: '/usr/bin/pypy "{mainfile}"' python3: - name: 'Python 3' + name: 'Python 3 (w/PyPy3)' priority: 850 files: '*.py *.py3' - compile: '/usr/bin/python3 -m py_compile {files}' - run: '/usr/bin/python3 "{mainfile}"' + compile: '/usr/bin/pypy3 -m py_compile {files}' + run: '/usr/bin/pypy3 "{mainfile}"' # Python2 without shebang comes after python3. python2: - name: 'Python 2' + name: 'Python 2 (w/PyPy)' priority: 840 files: '*.py2' - compile: '/usr/bin/python2 -m py_compile {files}' - run: '/usr/bin/python2 "{mainfile}"' + compile: '/usr/bin/pypy -m py_compile {files}' + run: '/usr/bin/pypy "{mainfile}"' ruby: name: 'Ruby' @@ -245,8 +245,8 @@ rust: name: 'Rust' priority: 575 files: '*.rs' - compile: '/usr/bin/rustc -o{binary} -O --crate-type bin --edition=2018 {files}' - run: '{binary}' + compile: '/usr/bin/rustc -C opt-level=3 -C target-cpu=native --crate-type bin --edition 2021 {mainfile} -o {mainfile}.out' + run: '{mainfile}.out' scala: name: 'Scala' diff --git a/problemtools/config/problem.yaml b/problemtools/config/problem.yaml index 79c9a1d5..c1d7c238 100644 --- a/problemtools/config/problem.yaml +++ b/problemtools/config/problem.yaml @@ -1,31 +1,9 @@ -type: pass-fail -author: '' -source: '' -source_url: '' -license: unknown -rights_owner: '' - -validation: default -validator_flags: '' - limits: - time_multiplier: 5 - time_safety_margin: 2 memory: 1024 output: 8 code: 128 compilation_time: 60 + compilation_memory: 1024 validation_time: 60 validation_memory: 1024 validation_output: 8 - -keywords: '' - -grading: - objective: max - show_test_data_groups: False - -languages: all - -# These are in the spec but currently unsupported -libraries: '' diff --git a/problemtools/formatversion.py b/problemtools/formatversion.py new file mode 100644 index 00000000..757746e2 --- /dev/null +++ b/problemtools/formatversion.py @@ -0,0 +1,47 @@ +import yaml +from enum import StrEnum +from pathlib import Path + + +class FormatVersion(StrEnum): + LEGACY = 'legacy' + V_2023_07 = '2023-07-draft' # When 2023-07 is finalized, replace this and update _missing_ + + @property + def statement_directory(self) -> str: + match self: + case FormatVersion.LEGACY: + return 'problem_statement' + case FormatVersion.V_2023_07: + return 'statement' + + @property + def statement_extensions(self) -> list[str]: + match self: + case FormatVersion.LEGACY: + return ['tex'] + case FormatVersion.V_2023_07: + return ['md', 'tex'] + + @property + def output_validator_directory(self) -> str: + match self: + case FormatVersion.LEGACY: + return 'output_validators' + case FormatVersion.V_2023_07: + return 'output_validator' + + # Support 2023-07 and 2023-07-draft strings. + # This method should be replaced with an alias once we require python 3.13 + @classmethod + def _missing_(cls, value): + if value == '2023-07': + return cls.V_2023_07 + return None + + +def get_format_version(problem_root: Path) -> FormatVersion: + """Loads the version from the problem in problem_root""" + with open(problem_root / 'problem.yaml') as f: + config: dict = yaml.safe_load(f) or {} + return FormatVersion(config.get('problem_format_version', FormatVersion.LEGACY)) diff --git a/problemtools/generatedata.py b/problemtools/generatedata.py deleted file mode 100644 index 3f8abaa5..00000000 --- a/problemtools/generatedata.py +++ /dev/null @@ -1,275 +0,0 @@ -#! /usr/bin/env python3 -# -*- coding: utf-8 -*- -import sys -import os -import glob -import tempfile -import shutil -import yaml -from argparse import ArgumentParser -from multiprocessing import Pool, cpu_count - -from .verifyproblem import Generators, ProblemAspect, Problem, is_RTE, argparser_basic_arguments, initialize_logging - -ALL_EXTENSIONS = ['in', 'ans'] + Generators._VISUALIZER_EXTENSIONS - -def argparser(): - parser = ArgumentParser(description='Generate test data for a problem package in the Kattis problem format.') - parser.add_argument('-g', '--generate', - action='store_true', - help='generate test data') - parser.add_argument('-c', '--clean', - action='store_true', - help='clean up generated files') - parser.add_argument('-C', '--clean_all', - action='store_true', - help='clean up generated and unrecognized files') - parser.add_argument('-n', '--dry_run', - action='store_true', - help='don\'t actually do anything') - parser.add_argument('-j', '--parallelism', - type=int, - default=None, - help='level of parallelism') - argparser_basic_arguments(parser) - parser.add_argument('problemdir', nargs='+') - return parser - - -def clean(prob, args): - ProblemAspect.errors = 0 - ProblemAspect.warnings = 0 - base_path = os.path.join(prob.probdir, 'data') - - testcases = { - case['path']: case - for case in prob.generators._testcases - } - - def walk(name, path): - case_count = 0 - cases = set() - empty = True - for fname in sorted(os.listdir(path)): - curpath = os.path.join(path, fname) - nice_path = os.path.relpath(curpath, base_path) - if '.' in fname: - fname, ext = fname.split('.', 1) - else: - ext = '' - curname = '%s/%s' % (name, fname) - - if os.path.isdir(curpath): - next_empty, next_cases = walk(curname, curpath) - case_count += next_cases - if next_empty: - if not args.dry_run: - os.rmdir(curpath) - else: - empty = False - else: - remove = args.clean_all - is_case = False - if (fname, ext) == ('testdata', 'yaml'): - if curname + '.yaml' in prob.generators._testdata_yaml: - remove = True - elif curname in testcases: - is_case = True - case = testcases[curname] - if ext == 'in': - remove = not case['manual'] - elif ext == 'ans': - remove = case['solution'] is not None - elif ext in Generators._VISUALIZER_EXTENSIONS: - remove = case['visualizer'] is not None - - if remove: - prob.generators.msg('Removing %s' % nice_path) - if not args.dry_run: - os.unlink(curpath) - if is_case and curname not in cases: - cases.add(curname) - case_count += 1 - else: - empty = False - - return empty, case_count - - cases_cleaned = 0 - for directory in prob.generators._data_directories: - path = os.path.join(base_path, directory) - if os.path.isdir(path): - cases_cleaned += walk('data/%s' % directory, path)[1] - return cases_cleaned, ProblemAspect.errors, ProblemAspect.warnings - - -class GenerateState: - prob = None - args = None - -def generate_case(case_idx): - ProblemAspect.errors = 0 - ProblemAspect.warnings = 0 - prob = GenerateState.prob - args = GenerateState.args - case = prob.generators._testcases[case_idx] - - steps = [ - ('input', True, None, '.in'), - ('solution', False, '.in', '.ans'), - ('visualizer', False, '.in', None), - ] - - try: - tmp_dir = tempfile.mkdtemp(prefix='gencase', dir=prob.tmpdir) - staging_dir = os.path.join(tmp_dir, 'staging') - os.mkdir(staging_dir) - out_dir = os.path.join(*([prob.probdir] + case['path'].split('/')[:-1])) - name = case['path'].split('/')[-1] - ok = args.dry_run or os.path.isdir(out_dir) - for (gen_type, mandatory, in_ext, out_ext) in steps: - if not ok: - break - prog = case.get(gen_type) - if prog is None: - ok = not mandatory - continue - prog, pargs = prog - prog = prob.generators._generators.get(prog) - if prog is None: - ok = not mandatory - continue - - if gen_type == 'input': - prob.generators.msg('Generating %s' % case['path'].replace('data/', '', 1)) - - if isinstance(prog, str): - assert gen_type == 'input' - assert prog.endswith('.in') - for ext in ALL_EXTENSIONS: - path = prog[:-2] + ext - if os.path.isfile(path): - shutil.copyfile(path, os.path.join(staging_dir, '%s.%s' % (name, ext))) - else: - errfile = os.path.join(tmp_dir, 'error') - params = {'args': pargs, 'errfile': errfile} - if in_ext is not None: - params['infile'] = os.path.join(staging_dir, name + in_ext) - if out_ext is not None: - outfile = os.path.join(tmp_dir, 'output') - params['outfile'] = outfile - - oldwd = os.getcwd() - os.chdir(staging_dir) - status, _ = prog.run(**params) - os.chdir(oldwd) - if is_RTE(status): - ok = not mandatory - stderr = None - if os.path.isfile(errfile): - with open(errfile, 'r') as f: - stderr = f.read() - prob.generators.error('Generator of type %s crashed with status %s' % (gen_type, status), stderr) - continue - - if out_ext is not None: - dest = os.path.join(staging_dir, name + out_ext) - if not os.path.isfile(dest): - shutil.copyfile(outfile, dest) - if ok: - for fname in os.listdir(staging_dir): - if '.' not in fname: - continue - curname, ext = fname.split('.', 1) - if curname == name and ext in ALL_EXTENSIONS: - fpath = os.path.join(staging_dir, fname) - if os.path.isfile(fpath) and not args.dry_run: - shutil.copyfile(fpath, os.path.join(out_dir, fname)) - return ok, ProblemAspect.errors, ProblemAspect.warnings - finally: - shutil.rmtree(tmp_dir) - - -def generate(prob, args): - - # Create directory structure - created = set() - for case in prob.generators._testcases: - path = os.path.join(*([prob.probdir] + case['path'].split('/')[:-1])) - if path not in created: - created.add(path) - if not os.path.isdir(path) and not args.dry_run: - try: - os.makedirs(path) - except Exception as e: - prob.generators.error('Could not create path %s' % path, e) - - # Populate testdata.yaml files - for path, content in prob.generators._testdata_yaml.items(): - prob.generators.msg('Generating %s' % path.replace('data/', '', 1)) - path = os.path.join(*([prob.probdir] + path.split('/'))) - if not args.dry_run: - try: - with open(path, 'w') as f: - yaml.dump(content, f) - except Exception as e: - prob.generators.error('Could not write %s' % path, e) - - # Generate test cases in parallel - GenerateState.prob = prob - GenerateState.args = args - pool = Pool(args.parallelism) - res = pool.map_async(generate_case, range(len(prob.generators._testcases))) - while not res.ready(): - # Use async polling for better KeyboardInterrupt handling - res.wait(1) - res = res.get() - return [ sum( r[tp] for r in res ) for tp in range(3) ] - - -def main(): - args = argparser().parse_args() - args.parts = ['generators'] - if args.clean_all: - args.clean = True - if not args.clean: - args.generate = True - args.compile_generators = args.generate - if args.parallelism is None: - args.parallelism = cpu_count() - initialize_logging(args) - - total_errors = 0 - for problemdir in args.problemdir: - print('Loading problem %s' % os.path.basename(os.path.realpath(problemdir))) - with Problem(problemdir) as prob: - prob.check(args) - errors = ProblemAspect.errors - warnings = ProblemAspect.warnings - - if prob.shortname is None: - # Skip invalid problem - continue - - def p(x): - return '' if x == 1 else 's' - - status = '' - if args.clean: - cnt, clean_errors, clean_warnings = clean(prob, args) - status += '%d case%s cleaned, ' % (cnt, p(cnt)) - errors += clean_errors - warnings += clean_warnings - if args.generate: - cnt, gen_errors, gen_warnings = generate(prob, args) - status += '%d case%s generated, ' % (cnt, p(cnt)) - errors += gen_errors - warnings += gen_warnings - - print("%s processed: %s%d error%s, %d warning%s" % (prob.shortname, status, errors, p(errors), warnings, p(warnings))) - total_errors += errors - - sys.exit(1 if total_errors > 0 else 0) - -if __name__ == '__main__': - main() diff --git a/problemtools/languages.py b/problemtools/languages.py index 8c0c72c2..1bcb61f6 100644 --- a/problemtools/languages.py +++ b/problemtools/languages.py @@ -2,17 +2,18 @@ This module contains functionality for reading and using configuration of programming languages. """ + import fnmatch import re import string from . import config + class LanguageConfigError(Exception): """Exception class for errors in language configuration.""" - pass - + pass class Language(object): @@ -42,7 +43,6 @@ def __init__(self, lang_id, lang_spec): self.run = None self.update(lang_spec) - def get_source_files(self, file_list): """Given a list of files, determine which ones would be considered source files for the language. @@ -50,12 +50,14 @@ def get_source_files(self, file_list): Args: file_list (list of str): list of file names """ - return [file_name for file_name in file_list - if (any(fnmatch.fnmatch(file_name, glob) - for glob in self.files) - and - self.__matches_shebang(file_name))] - + return [ + file_name + for file_name in file_list + if ( + any(fnmatch.fnmatch(file_name, glob) for glob in self.files) # type: ignore[union-attr] + and self.__matches_shebang(file_name) + ) + ] def update(self, values): """Update a language specification with new values. @@ -66,23 +68,17 @@ def update(self, values): """ # Check that all provided values are known keys - for unknown in set(values)-set(Language.__KEYS): - raise LanguageConfigError( - 'Unknown key "%s" specified for language %s' - % (unknown, self.lang_id)) + for unknown in set(values) - set(Language.__KEYS): + raise LanguageConfigError('Unknown key "%s" specified for language %s' % (unknown, self.lang_id)) - for (key, value) in values.items(): + for key, value in values.items(): # Check type if key == 'priority': if not isinstance(value, int): - raise LanguageConfigError( - 'Language %s: priority must be integer but is %s.' - % (self.lang_id, type(value))) + raise LanguageConfigError('Language %s: priority must be integer but is %s.' % (self.lang_id, type(value))) else: if not isinstance(value, str): - raise LanguageConfigError( - 'Language %s: %s must be string but is %s.' - % (self.lang_id, key, type(value))) + raise LanguageConfigError('Language %s: %s must be string but is %s.' % (self.lang_id, key, type(value))) # Save the value if key == 'shebang': @@ -97,7 +93,6 @@ def update(self, values): self.__check() - def __check(self): """Check that the language specification is valid (all mandatory fields provided, all metavariables used in compile/run @@ -105,46 +100,33 @@ def __check(self): """ # Check that all mandatory fields are provided if self.name is None: - raise LanguageConfigError( - 'Language %s has no name' % self.lang_id) + raise LanguageConfigError(f'Language {self.lang_id} has no name') if self.priority is None: - raise LanguageConfigError( - 'Language %s has no priority' % self.lang_id) + raise LanguageConfigError(f'Language {self.lang_id} has no priority') if self.files is None: - raise LanguageConfigError( - - 'Language %s has no files glob' % self.lang_id) + raise LanguageConfigError(f'Language {self.lang_id} has no files glob') if self.run is None: - raise LanguageConfigError( - 'Language %s has no run command' % self.lang_id) + raise LanguageConfigError(f'Language {self.lang_id} has no run command') # Check that all variables appearing are valid variables = Language.__variables_in_command(self.run) if self.compile is not None: variables = variables | Language.__variables_in_command(self.compile) for unknown in variables - set(Language.__VARIABLES): - raise LanguageConfigError( - 'Unknown variable "{%s}" used for language %s' - % (unknown, self.lang_id)) + raise LanguageConfigError('Unknown variable "{%s}" used for language %s' % (unknown, self.lang_id)) # Check for uniquely defined entry point entry = variables & set(['binary', 'mainfile', 'mainclass', 'Mainclass']) if len(entry) == 0: - raise LanguageConfigError( - 'No entry point variable used for language %s' % self.lang_id) + raise LanguageConfigError('No entry point variable used for language %s' % self.lang_id) if len(entry) > 1: - raise LanguageConfigError( - 'More than one entry point type variable used for language %s' - % self.lang_id) - + raise LanguageConfigError('More than one entry point type variable used for language %s' % self.lang_id) @staticmethod def __variables_in_command(cmd): """List all meta-variables appearing in a string.""" formatter = string.Formatter() - return set(field for _, field, _, _ in formatter.parse(cmd) - if field is not None) - + return set(field for _, field, _, _ in formatter.parse(cmd) if field is not None) def __matches_shebang(self, filename): """Check if a file matches the shebang rule for the language.""" @@ -155,10 +137,6 @@ def __matches_shebang(self, filename): return self.shebang.search(shebang_line) is not None - - - - class Languages(object): """A set of languages.""" @@ -174,7 +152,6 @@ def __init__(self, data=None): if data is not None: self.update(data) - def detect_language(self, file_list): """Auto-detect language for a set of files. @@ -186,7 +163,7 @@ def detect_language(self, file_list): list of files did not match any language in the set. """ result = None - src = [] + src: list[str] = [] prio = 1e99 for lang in self.languages.values(): lang_src = lang.get_source_files(file_list) @@ -198,9 +175,7 @@ def detect_language(self, file_list): def get(self, lang_id): if not isinstance(lang_id, str): - raise LanguageConfigError( - 'Config file error: language IDs must be strings, but %s is %s.' - % (lang_id, type(lang_id))) + raise LanguageConfigError('Config file error: language IDs must be strings, but %s is %s.' % (lang_id, type(lang_id))) return self.languages.get(lang_id, None) def update(self, data): @@ -213,21 +188,19 @@ def update(self, data): for that language will be overridden and updated. """ if not isinstance(data, dict): - raise LanguageConfigError( - 'Config file error: content must be a dictionary, but is %s.' - % (type(data))) + raise LanguageConfigError('Config file error: content must be a dictionary, but is %s.' % (type(data))) - for (lang_id, lang_spec) in data.items(): + for lang_id, lang_spec in data.items(): if not isinstance(lang_id, str): raise LanguageConfigError( - 'Config file error: language IDs must be strings, but %s is %s.' - % (lang_id, type(lang_id))) + 'Config file error: language IDs must be strings, but %s is %s.' % (lang_id, type(lang_id)) + ) if not isinstance(lang_spec, (dict, Language)): raise LanguageConfigError( 'Config file error: language spec must be a dictionary, but spec of language %s is %s.' - % (lang_id, type(lang_spec))) - + % (lang_id, type(lang_spec)) + ) if isinstance(lang_spec, Language): self.languages[lang_id] = lang_spec @@ -236,12 +209,12 @@ def update(self, data): else: self.languages[lang_id].update(lang_spec) - priorities = {} - for (lang_id, lang) in self.languages.items(): + priorities: dict[int, Language] = {} + for lang_id, lang in self.languages.items(): if lang.priority in priorities: raise LanguageConfigError( - 'Languages %s and %s both have priority %d.' - % (lang_id, priorities[lang.priority], lang.priority)) + 'Languages %s and %s both have priority %d.' % (lang_id, priorities[lang.priority], lang.priority) + ) priorities[lang.priority] = lang_id diff --git a/problemtools/md2html.py b/problemtools/md2html.py new file mode 100644 index 00000000..445eca0f --- /dev/null +++ b/problemtools/md2html.py @@ -0,0 +1,151 @@ +#! /usr/bin/env python3 +# -*- coding: utf-8 -*- +import argparse +import hashlib +import html +import os +from pathlib import Path +import re +import shutil +import string +import subprocess + +import nh3 + +from . import statement_util + + +def convert(problem_root: Path, options: argparse.Namespace, statement_file: Path) -> bool: + """Convert a Markdown statement to HTML. Writes output to current working directory. + + Args: + problem: path to problem directory + options: command-line arguments. See problem2html.py + """ + destfile = string.Template(options.destfile).safe_substitute(problem=problem_root.name) + imgbasedir = string.Template(options.imgbasedir).safe_substitute(problem=problem_root.name) + + command = ['pandoc', str(statement_file), '-t', 'html', '--mathjax'] + statement_html = subprocess.run(command, capture_output=True, text=True, shell=False, check=True).stdout + + statement_html = sanitize_html(statement_file.parent, statement_html, imgbasedir) + + templatepaths = [ + os.path.join(os.path.dirname(__file__), 'templates/markdown_html'), + '/usr/lib/problemtools/templates/markdown_html', + ] + templatepath = next( + (p for p in templatepaths if os.path.isdir(p) and os.path.isfile(os.path.join(p, 'default-layout.html'))), None + ) + + if templatepath is None: + raise FileNotFoundError('Could not find directory with markdown templates') + + with open(Path(templatepath) / 'default-layout.html', 'r', encoding='utf-8') as template_file: + template = template_file.read() + + problem_name = statement_util.get_yaml_problem_name(problem_root, options.language) + substitution_params = { + 'statement_html': statement_html, + 'language': options.language, + 'title': html.escape(problem_name) if problem_name else 'Missing problem name', + 'problemid': html.escape(problem_root.name), + } + + statement_html = template % substitution_params + + samples = statement_util.format_samples(problem_root) + # Insert samples at {{nextsample}} and {{remainingsamples}} + statement_html, remaining_samples = statement_util.inject_samples(statement_html, samples) + + # Insert the remaining samples at the bottom + # However, footnotes should be below samples + sample_insertion_position = statement_util.find_footnotes(statement_html) + if sample_insertion_position is None: + # No footnotes, so insert at the end + sample_insertion_position = statement_html.rfind('') + statement_html = ( + statement_html[:sample_insertion_position] + ''.join(remaining_samples) + statement_html[sample_insertion_position:] + ) + + with open(destfile, 'w', encoding='utf-8', errors='xmlcharrefreplace') as output_file: + output_file.write(statement_html) + + if options.css: + shutil.copyfile(os.path.join(templatepath, 'problem.css'), 'problem.css') + + return True + + +def sanitize_html(statement_dir: Path, statement_html: str, imgbasedir: str) -> str: + # Allow footnote ids (the anchor points you jump to) + def is_fn_id(s): + pattern_id_top = r'^fn\d+$' + pattern_id_bottom = r'^fnref\d+$' + return bool(re.fullmatch(pattern_id_top, s)) or bool(re.fullmatch(pattern_id_bottom, s)) + + allowed_classes = ('sample', 'problemheader', 'problembody', 'sampleinteractionwrite', 'sampleinteractionread') + + # Annoying: nh3 will ignore exceptions in attribute_filter + image_fail_reason: list[Exception] = [] + + def attribute_filter(tag, attribute, value): + if attribute == 'class' and value in allowed_classes: + return value + # Never versions of Pandoc will give class="footnotes footnotes-end-of-document" + # We don't want to blindly allow any class with footnotes in it, so only allow footnotes + if attribute == 'class' and 'footnotes' in value: + return 'footnotes' + if tag == 'a' and attribute == 'href': + return value + if tag in ('li', 'a') and attribute == 'id' and is_fn_id(value): + return value + if tag == 'img' and attribute == 'src': + try: + statement_util.assert_image_is_valid(statement_dir, value) + except Exception as e: + nonlocal image_fail_reason + image_fail_reason.append(e) + return None + return copy_image(statement_dir, value, imgbasedir) + return None + + statement_html = nh3.clean( + statement_html, + link_rel='noopener nofollow noreferrer', + attribute_filter=attribute_filter, + tags=nh3.ALLOWED_TAGS | {'img', 'a', 'section'}, + attributes={ + 'table': {'class'}, + 'aside': {'class'}, + 'div': {'class'}, + 'section': {'class'}, + 'img': {'src'}, + 'a': {'href', 'id'}, + 'li': {'id'}, + }, + ) + + if image_fail_reason: + # We don't have a great way to emit multiple errors from here, so just re-raise the first error + raise image_fail_reason[0] + + return statement_html + + +def copy_image(statement_dir: Path, img_src: str, imgbasedir: str) -> str: + """Copy image to working directory (with new filename) and returns the new filename + + Args: + statement_dir: the directory with problem statement files + img_src: the image source as in the Markdown statement + """ + + # We rename to sha256 of contents, and preserve the suffix. This flattens + # the directory structure to a single folders in a simple way. + with open(statement_dir / img_src, 'rb') as f: + filename = hashlib.file_digest(f, 'sha256').hexdigest() + Path(img_src).suffix + + if not os.path.isfile(filename): # check if already copied + shutil.copyfile(statement_dir / img_src, filename) + return imgbasedir + filename diff --git a/problemtools/metadata.py b/problemtools/metadata.py new file mode 100644 index 00000000..25535327 --- /dev/null +++ b/problemtools/metadata.py @@ -0,0 +1,361 @@ +import copy +import datetime +import re +from dataclasses import dataclass, field +from enum import StrEnum +from pathlib import Path +from typing import Any, Literal, Self, Type, Union +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field +import yaml + +from . import config +from . import statement_util +from .formatversion import FormatVersion + + +class ProblemType(StrEnum): + PASS_FAIL = 'pass-fail' + SCORING = 'scoring' + MULTI_PASS = 'multi-pass' + INTERACTIVE = 'interactive' + SUBMIT_ANSWER = 'submit-answer' + + +class License(StrEnum): + UNKNOWN = 'unknown' + PUBLIC_DOMAIN = 'public domain' + CC0 = 'cc0' + CC_BY = 'cc by' + CC_BY_SA = 'cc by-sa' + EDUCATIONAL = 'educational' + PERMISSION = 'permission' + + +@dataclass +class Person: + name: str + email: str | None = None + orcid: str | None = None + kattis: str | None = None + + @classmethod + def from_string(cls: Type[Self], s: str) -> Self: + match = re.match(r'^(.*?)\s+<(.*)>$', s.strip()) + if match: + return cls(name=match.group(1), email=match.group(2)) + return cls(name=s) + + +@dataclass +class Source: + name: str + url: str | None = None + + +@dataclass +class TimeMultipliers: + ac_to_time_limit: float = 2.0 + time_limit_to_tle: float = 1.5 + + +@dataclass +class Limits: + memory: int + output: int + code: int + compilation_time: int + compilation_memory: int + validation_time: int + validation_memory: int + validation_output: int + time_multipliers: TimeMultipliers = field(default_factory=TimeMultipliers) + time_limit: float | None = None + time_resolution: float = 1.0 + validation_passes: int = 2 + + +@dataclass +class Credits: + """ + Credits format where all persons have been converted to Person objects. + For use in our internal representation. + """ + + authors: list[Person] = field(default_factory=list) + contributors: list[Person] = field(default_factory=list) + testers: list[Person] = field(default_factory=list) + translators: dict[str, list[Person]] = field(default_factory=dict) + packagers: list[Person] = field(default_factory=list) + acknowledgements: list[Person] = field(default_factory=list) + + +@dataclass +class InputCredits: + """ + A more permissive dataclass for credits, as the input in 2023-07 looks. + For use when validating input. + """ + + # Type in the input format is messy + PersonOrPersons = Union[str | list[Union[Person, str]]] + + authors: PersonOrPersons = field(default_factory=list) + contributors: PersonOrPersons = field(default_factory=list) + testers: PersonOrPersons = field(default_factory=list) + translators: dict[str, PersonOrPersons] = field(default_factory=dict) + packagers: PersonOrPersons = field(default_factory=list) + acknowledgements: PersonOrPersons = field(default_factory=list) + + +class Metadata2023_07(BaseModel): + """ + The metadata for a problem as input in version 2023-07-draft. + """ + + problem_format_version: str + name: dict[str, str] | str + uuid: UUID | None = None # UUID *is* mandatory, but we deal with that in verifyproblem for better UX + type: list[ProblemType] | ProblemType = ProblemType.PASS_FAIL + version: str | None = None + credits: dict | str | None = None + source: list[Union[str, Source]] | Source | str = [] + license: License = License.UNKNOWN + rights_owner: str | None = None + embargo_until: datetime.datetime | None = None + limits: Limits + keywords: list[str] = [] + languages: list[str] | Literal['all'] = 'all' + allow_file_writing: bool = True + constants: dict[str, int | float | str] = {} + + model_config = ConfigDict(extra='forbid') + + +@dataclass +class LegacyGrading: + objective: Literal['max', 'min'] = 'max' + show_test_data_groups: bool = False + # These 3 fields predate the version called "legacy" + accept_score: float | None = None + reject_score: float | None = None + range: str | None = None + on_reject: Literal['first_error', 'worst_error', 'grade'] | None = None + + +@dataclass +class LegacyLimits: + memory: int + output: int + code: int + compilation_time: int + compilation_memory: int + validation_time: int + validation_memory: int + validation_output: int + time_multiplier: float = 5.0 + time_safety_margin: float = 2.0 + + +class MetadataLegacy(BaseModel): + """ + The metadata for a problem as input in version legacy (plus a few fields + which pre-date the version called legacy). + """ + + problem_format_version: FormatVersion = FormatVersion.LEGACY + type: Literal['pass-fail'] | Literal['scoring'] = 'pass-fail' + name: str | None = None + uuid: UUID | None = None + author: str | None = None + source: str | None = None + source_url: str | None = None + license: License = License.UNKNOWN + rights_owner: str | None = None + limits: LegacyLimits + validation: str = 'default' + validator_flags: str = '' + grading: LegacyGrading = LegacyGrading() + keywords: str = '' + + model_config = ConfigDict(extra='forbid') + + +class Metadata(BaseModel): + """ + The metadata for a problem, as used internally in problemtools. Closely + follows the 2023-07-draft version, but is more fully parsed, and adds + a few legacy fields to represent information not in 2023-07. + + Metadata serializes to a valid 2023-07-draft configuration. + """ + + problem_format_version: FormatVersion + type: list[ProblemType] + name: dict[str, str] + uuid: UUID | None + version: str | None + credits: Credits + source: list[Source] + license: License + rights_owner: str | None + embargo_until: datetime.datetime | None + limits: Limits + keywords: list[str] + languages: list[str] | Literal['all'] + allow_file_writing: bool + constants: dict + legacy_grading: LegacyGrading = Field(default_factory=LegacyGrading, exclude=True) + legacy_validation: str = Field(default='', exclude=True) + legacy_validator_flags: str = Field(default='', exclude=True) + legacy_custom_score: bool = Field(default=False, exclude=True) # True iff legacy_validation is custom and score. + + model_config = ConfigDict(extra='forbid') + + def is_pass_fail(self) -> bool: + return not self.is_scoring() + + def is_scoring(self) -> bool: + return ProblemType.SCORING in self.type + + def is_interactive(self) -> bool: + return ProblemType.INTERACTIVE in self.type + + def is_multi_pass(self) -> bool: + return ProblemType.MULTI_PASS in self.type + + def is_submit_answer(self) -> bool: + return ProblemType.SUBMIT_ANSWER in self.type + + @classmethod + def from_legacy(cls: Type[Self], legacy: MetadataLegacy, names_from_statements: dict[str, str]) -> Self: + metadata = legacy.model_dump() + metadata['type'] = [metadata['type']] + # Support for *ancient* problems where names_from_statements is empty + if names_from_statements: + metadata['name'] = names_from_statements + elif metadata['name']: + metadata['name'] = {'en': metadata['name']} + else: + metadata['name'] = {} + metadata['version'] = None + + def parse_author_field(author: str) -> list[Person]: + authors = re.split(r',\s*|\s+and\s+|\s+&\s+', author) + authors = [x.strip(' \t\r\n') for x in authors] + authors = [x for x in authors if len(x) > 0] + return [Person.from_string(author) for author in authors] + + metadata['credits'] = {} + if metadata['author'] is not None: + metadata['credits']['authors'] = parse_author_field(metadata['author']) + del metadata['author'] + metadata['source'] = [] if metadata['source'] is None else [Source(metadata['source'], metadata['source_url'])] + del metadata['source_url'] + metadata['embargo_until'] = None + metadata['limits']['time_multipliers'] = { + 'ac_to_time_limit': metadata['limits']['time_multiplier'], + 'time_limit_to_tle': metadata['limits']['time_safety_margin'], + } + del metadata['limits']['time_multiplier'] + del metadata['limits']['time_safety_margin'] + metadata['keywords'] = metadata['keywords'].split() + metadata['languages'] = 'all' + metadata['allow_file_writing'] = True + metadata['constants'] = {} + + # The interactive flag from validation now lives in type, copy it over. + validation = metadata['validation'].split() + if validation[0] == 'custom': + if 'interactive' in validation[1:]: + metadata['type'].append('interactive') + if 'score' in validation[1:]: + metadata['legacy_custom_score'] = True + # Copy over the legacy info that does not fit cleanly + for key in 'grading', 'validator_flags', 'validation': + metadata[f'legacy_{key}'] = metadata[key] + del metadata[key] + return cls.model_validate(metadata) + + @classmethod + def from_2023_07(cls: Type[Self], md2023_07: Metadata2023_07) -> Self: + metadata = md2023_07.model_dump() + metadata['type'] = [metadata['type']] if isinstance(metadata['type'], str) else metadata['type'] + metadata['name'] = {'en': metadata['name']} if isinstance(metadata['name'], str) else metadata['name'] + + def parse_source(source: str | Source) -> Source: + return Source(name=source, url=None) if isinstance(source, str) else source + + # Convenience function to deal with the fact that lists of persons/sources are + # either a string, or a list of strings or dicts (if dicts, pydantic + # already parsed those for us). + def parse_list(callback, lst: str | list) -> list: + if isinstance(lst, str): + return [callback(lst)] + return list(map(callback, lst)) + + metadata['source'] = parse_list(parse_source, metadata['source']) + + def parse_person(person: str | Person) -> Person: + return Person.from_string(person) if isinstance(person, str) else person + + if metadata['credits'] is None: + metadata['credits'] = {} + elif isinstance(metadata['credits'], str): + metadata['credits'] = {'authors': [parse_person(metadata['credits'])]} + else: + for key in metadata['credits']: + if key == 'translators': # special case, we nest deeper here + for lang in metadata['credits'][key]: + metadata['credits'][key][lang] = parse_list(parse_person, metadata['credits'][key][lang]) + else: + metadata['credits'][key] = parse_list(parse_person, metadata['credits'][key]) + return cls.model_validate(metadata) + + +def parse_metadata( + version: FormatVersion, + problem_yaml_data: dict[str, Any], + names_from_statements: dict[str, str] | None = None, +) -> Metadata: + """ + Parses a data structure from problem.yaml into a Metadata model + :raises pydantic.ValidationError: We intentionally leak pydantic's exception on errors, as it's well designed + """ + + # We need to mix in the system default config values before doing model validation + data = copy.deepcopy(problem_yaml_data) + # Check if the user has done something silly like making limits a string. If so, we + # don't merge in anything, and let pydantic complain later. + if isinstance(data.get('limits', {}), dict): + system_defaults = config.load_config('problem.yaml') + data['limits'] = system_defaults['limits'] | data.get('limits', {}) + + if version is FormatVersion.LEGACY: + legacy_model = MetadataLegacy.model_validate(data) + return Metadata.from_legacy(legacy_model, names_from_statements or {}) + else: + assert version is FormatVersion.V_2023_07 + model_2023_07 = Metadata2023_07.model_validate(data) + return Metadata.from_2023_07(model_2023_07) + + +def load_metadata(problem_root: Path) -> tuple[Metadata, dict]: + """ + Loads metadata from a problem directory. + + Returns Metadata as well as the raw parsed yaml. The latter is likely only of use to verifyproblem. + Leaks exceptions, which is a bit of a mess. Unclear how to best deal with error handling. + """ + with (problem_root / 'problem.yaml').open() as f: + data = yaml.safe_load(f) + if data is None: # Loading empty yaml returns None + data = {} + + version = FormatVersion(data.get('problem_format_version', FormatVersion.LEGACY)) + if version is FormatVersion.LEGACY: + names_from_statements = statement_util.load_names_from_statements(problem_root, version) + else: + names_from_statements = None + return parse_metadata(version, data, names_from_statements), data diff --git a/problemtools/problem2html.py b/problemtools/problem2html.py index b3f10b1c..671935dd 100644 --- a/problemtools/problem2html.py +++ b/problemtools/problem2html.py @@ -1,70 +1,29 @@ #! /usr/bin/env python3 # -*- coding: utf-8 -*- -import re +import argparse import os.path +import re import string -import argparse -import logging import subprocess +import sys +from pathlib import Path -import plasTeX.TeX -import plasTeX.Logging - -from .ProblemPlasTeX import ProblemRenderer -from .ProblemPlasTeX import ProblemsetMacros -from . import template +from . import tex2html +from . import md2html +from . import statement_util -def convert(problem, options=None): - problem = os.path.realpath(problem) +def convert(options: argparse.Namespace, force_statement_file: Path | None = None) -> None: + problem_root = Path(options.problem).resolve(strict=True) - problembase = os.path.splitext(os.path.basename(problem))[0] - destdir = string.Template(options.destdir).safe_substitute(problem=problembase) - destfile = string.Template(options.destfile).safe_substitute(problem=problembase) - imgbasedir = string.Template(options.imgbasedir).safe_substitute(problem=problembase) - - if options.quiet: - plasTeX.Logging.disableLogging() + if force_statement_file: # Used by verifyproblem to test rendering even if there are multiple statements in a language + statement_file = force_statement_file else: - plasTeX.Logging.getLogger().setLevel(getattr(logging, options.loglevel.upper())) - plasTeX.Logging.getLogger('status').setLevel(getattr(logging, options.loglevel.upper())) - - texfile = problem - # Set up template if necessary - with template.Template(problem, language=options.language) as templ: - texfile = open(templ.get_file_name(), 'r') - - origcwd = os.getcwd() - - # Setup parser and renderer etc - - # plasTeX version 3 changed the name of this argument (and guarding against this - # by checking plasTeX.__version__ fails on plastex v3.0 which failed to update - # __version__) - try: - tex = plasTeX.TeX.TeX(myfile=texfile) - except Exception: - tex = plasTeX.TeX.TeX(file=texfile) + statement_file = statement_util.find_statement(problem_root, options.language) - ProblemsetMacros.init(tex) - - tex.ownerDocument.config['general']['copy-theme-extras'] = options.css - if not options.headers: - tex.ownerDocument.userdata['noheaders'] = True - tex.ownerDocument.config['files']['filename'] = destfile - tex.ownerDocument.config['images']['filenames'] = 'img-$num(4)' - tex.ownerDocument.config['images']['enabled'] = False - tex.ownerDocument.config['images']['imager'] = 'none' - tex.ownerDocument.config['images']['base-url'] = imgbasedir - # tell plasTeX where to search for problemtools' built-in packages - tex.ownerDocument.config['general']['packages-dirs'] = [os.path.join(os.path.dirname(__file__), 'ProblemPlasTeX')] - - renderer = ProblemRenderer() - - if not options.quiet: - print('Parsing TeX source...') - doc = tex.parse() - texfile.close() + destdir = string.Template(options.destdir).safe_substitute(problem=problem_root.name) + destfile = string.Template(options.destfile).safe_substitute(problem=problem_root.name) + origcwd = os.getcwd() # Go to destdir if destdir: @@ -75,12 +34,13 @@ def convert(problem, options=None): try: if not options.quiet: print('Rendering!') - renderer.render(doc) - - # Annoying: I have not figured out any way of stopping the plasTeX - # renderer from generating a .paux file - if os.path.isfile('.paux'): - os.remove('.paux') + match statement_file.suffix: + case '.md': + md2html.convert(problem_root, options, statement_file) + case '.tex': + tex2html.convert(problem_root, options, statement_file) + case _: + raise NotImplementedError('Unsupported file type, expected md or tex: {statement_file.name}') if options.tidy: with open(os.devnull, 'w') as devnull: @@ -92,13 +52,13 @@ def convert(problem, options=None): # identify any large generated files (especially images) if not options.quiet: - for path, dirs, files in os.walk('.'): + for path, _dirs, files in os.walk('.'): for f in files: file_size_kib = os.stat(os.path.join(path, f)).st_size // 1024 if file_size_kib > 1024: - print(f"WARNING: FILE {f} HAS SIZE {file_size_kib} KiB; CONSIDER REDUCING IT") + print(f'WARNING: FILE {f} HAS SIZE {file_size_kib} KiB; CONSIDER REDUCING IT') elif file_size_kib > 300: - print(f"Warning: file {f} has size {file_size_kib} KiB; consider reducing it") + print(f'Warning: file {f} has size {file_size_kib} KiB; consider reducing it') if options.bodyonly: content = open(destfile).read() @@ -109,46 +69,48 @@ def convert(problem, options=None): # restore cwd os.chdir(origcwd) - return True - - -class ConvertOptions: - available = [ - ['bodyonly', 'store_true', '-b', '--body-only', - 'only generate HTML body, no HTML headers', False], - ['css', 'store_false', '-c', '--no-css', - "don't copy CSS file to output directory", True], - ['headers', 'store_false', '-H', '--headers', - "don't generate problem headers (title, problem id, time limit)", True], - ['tidy', 'store_false', '-m', '--messy', - "don't run tidy to postprocess the HTML", True], - ['destdir', 'store', '-d', '--dest-dir', - "output directory", '${problem}_html'], - ['destfile', 'store', '-f', '--dest-file', - "output file name", 'index.html'], - ['language', 'store', '-l', '--language', - 'choose alternate language (2-letter code)', None], - ['loglevel', 'store', '-L', '--log-level', - 'set log level (debug, info, warning, error, critical)', 'warning'], - ['quiet', 'store_true', '-q', '--quiet', - "quiet", False], - ] - - def __init__(self): - for (dest, _, _, _, _, default) in ConvertOptions.available: - setattr(self, dest, default) - self.imgbasedir = '' - - -def main(): - options = ConvertOptions() + +def get_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) - for (dest, action, short, _long, _help, default) in ConvertOptions.available: - parser.add_argument(short, _long, dest=dest, help=_help, action=action, default=default) + + parser.add_argument( + '-b', '--body-only', dest='bodyonly', action='store_true', help='only generate HTML body, no HTML headers', default=False + ) + parser.add_argument( + '-c', '--no-css', dest='css', action='store_false', help="don't copy CSS file to output directory", default=True + ) + parser.add_argument( + '-H', + '--headers', + dest='headers', + action='store_false', + help="don't generate problem headers (title, problem id, time limit)", + default=True, + ) + parser.add_argument( + '-m', '--messy', dest='tidy', action='store_false', help="don't run tidy to postprocess the HTML", default=True + ) + parser.add_argument('-d', '--dest-dir', dest='destdir', help='output directory', default='${problem}_html') + parser.add_argument('-f', '--dest-file', dest='destfile', help='output file name', default='index.html') + parser.add_argument('-l', '--language', dest='language', help='choose language (2-letter code)', default='en') + parser.add_argument( + '-L', '--log-level', dest='loglevel', help='set log level (debug, info, warning, error, critical)', default='warning' + ) + parser.add_argument('-q', '--quiet', dest='quiet', action='store_true', help='quiet', default=False) + parser.add_argument('-i', '--imgbasedir', dest='imgbasedir', default='') parser.add_argument('problem', help='the problem to convert') - options = parser.parse_args(namespace=options) - convert(options.problem, options) + return parser + + +def main() -> None: + parser = get_parser() + options = parser.parse_args() + try: + convert(options) + except Exception as e: + print(e) + sys.exit(1) if __name__ == '__main__': diff --git a/problemtools/problem2pdf.py b/problemtools/problem2pdf.py index 1e039a19..077c18c0 100644 --- a/problemtools/problem2pdf.py +++ b/problemtools/problem2pdf.py @@ -1,24 +1,101 @@ #! /usr/bin/env python3 # -*- coding: utf-8 -*- -import os.path +import argparse +import os +import re import shutil import string -import argparse import subprocess -from . import template - +import sys +import tempfile +from pathlib import Path -def convert(problem, options=None): - if options is None: - options = ConvertOptions() - - problem = os.path.realpath(problem) - problembase = os.path.splitext(os.path.basename(problem))[0] - destfile = string.Template(options.destfile).safe_substitute(problem=problembase) +from . import template +from . import statement_util + + +def convert(options: argparse.Namespace, force_statement_file: Path | None = None) -> bool: + problem_root = Path(options.problem).resolve(strict=True) + + if force_statement_file: # Used by verifyproblem to test rendering even if there are multiple statements in a language + statement_file = force_statement_file + else: + statement_file = statement_util.find_statement(problem_root, options.language) + + match statement_file.suffix: + case '.md': + return md2pdf(options, statement_file) + case '.tex': + return latex2pdf(options, statement_file) + case _: + raise NotImplementedError('Unsupported file type, expected md or tex: {statement_file.name}') + + +def md2pdf(options: argparse.Namespace, statement_file: Path) -> bool: + """Renders a Markdown document to pdf. Uses pandoc md -> tex, then + reuses the normal tex -> pdf pipeline + """ + problem_root = Path(options.problem).resolve(strict=True) + + statement_util.assert_images_are_valid_md(statement_file) + + command = ['pandoc', str(statement_file), '-t', 'latex'] + try: + tex = subprocess.run(command, capture_output=True, text=True, shell=False, check=True).stdout + except subprocess.CalledProcessError as e: + print(f'Error compiling Markdown to pdf: {e.stderr}') + return False + + def format_latex_tables(latex_doc): + # Match table environments produced by pandoc + pattern = r""" + (\\begin\{longtable\}\[\]\{@\{\}) + ([a-z]) + ([a-z]*) + (@\{\}\}) + """ + + def replacer(match): + prefix = match.group(1)[:-3] + first_col = match.group(2) + other_cols = match.group(3) + suffix = match.group(4)[3:] + + # Combine columns with | separators + cols = [first_col] + list(other_cols) + return f'{prefix}|{"|".join(cols)}|{suffix} \\hline' + + return re.sub(pattern, replacer, latex_doc, flags=re.VERBOSE) + + # Add solid outline to tables + tex = format_latex_tables(tex) + tex = tex.replace(r'\toprule', '') + tex = tex.replace(r'\midrule', '') + tex = tex.replace(r'\endhead', '') + tex = tex.replace(r'\bottomrule', '') + tex = tex.replace(r'\tabularnewline', r'\\ \hline') + + # Fix sample inclusions commands + # Currently does not work, as normal problemtools tex -> pdf does not support it + tex = tex.replace(r'\{\{nextsample\}\}', r'\nextsample') + tex = tex.replace(r'\{\{remainingsamples\}\}', r'\remainingsamples') + + problem_name = statement_util.get_yaml_problem_name(problem_root, options.language) + tex = r'\problemname{' + problem_name + '}\n' + tex + with tempfile.NamedTemporaryFile(mode='w', suffix='.tex', dir=statement_file.parent) as temp_tex_file: + temp_tex_file.write(tex) + temp_tex_file.flush() + return latex2pdf(options, Path(temp_tex_file.name)) + + return False + + +def latex2pdf(options: argparse.Namespace, statement_file: Path) -> bool: + problem_root = Path(options.problem).resolve(strict=True) + destfile = string.Template(options.destfile).safe_substitute(problem=problem_root.name) - texfile = problem # Set up template if necessary - with template.Template(problem, language=options.language) as templ: + with template.Template(problem_root, statement_file, options.language) as templ: texfile = templ.get_file_name() origcwd = os.getcwd() @@ -45,35 +122,54 @@ def convert(problem, options=None): if status == 0 and not options.nopdf: shutil.move(os.path.splitext(texfile)[0] + '.pdf', destfile) - return status == 0 - - -class ConvertOptions: - available = [ - ['destfile', 'store', '-o', '--output', - "output file name", '${problem}.pdf'], - ['quiet', 'store_true', '-q', '--quiet', - "quiet", False], - ['language', 'store', '-l', '--language', - 'choose alternate language (2-letter code)', None], - ['nopdf', 'store_true', '-n', '--no-pdf', - 'run pdflatex in -draftmode', False], - ] - - def __init__(self): - for (dest, _, _, _, _, default) in ConvertOptions.available: - setattr(self, dest, default) - - -def main(): + if status: + return False + + # We only sanitize if a PDF was created + if not options.nopdf: + try: + with tempfile.NamedTemporaryFile(suffix='.pdf') as f: + command = [ + 'gs', + '-q', + '-dBATCH', + '-sDEVICE=pdfwrite', + '-dNOPAUSE', + '-dCompatibilityLevel=1.7', + f'-sOutputFile={f.name}', + destfile, + ] + gs_status = subprocess.run(command, capture_output=True, text=True, shell=False, check=True) + if gs_status.returncode != 0: + return False + shutil.copy(f.name, destfile) + except subprocess.CalledProcessError as e: + print(f'Error sanitizing PDF: {e} {e.stderr}') + raise + + return True + + +def get_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) - for (dest, action, short, _long, _help, default) in ConvertOptions.available: - parser.add_argument(short, _long, dest=dest, help=_help, action=action, default=default) + parser.add_argument('-o', '--output', dest='destfile', help='output file name', default='${problem}.pdf') + parser.add_argument('-q', '--quiet', dest='quiet', action='store_true', help='quiet', default=False) + parser.add_argument('-l', '--language', dest='language', help='choose language (2-letter code)', default='en') + parser.add_argument('-n', '--no-pdf', dest='nopdf', action='store_true', help='run pdflatex in -draftmode', default=False) parser.add_argument('problem', help='the problem to convert') + return parser + + +def main() -> None: + parser = get_parser() options = parser.parse_args() - convert(options.problem, options) + try: + convert(options) + except Exception as e: + print(e) + sys.exit(1) if __name__ == '__main__': diff --git a/problemtools/tests/__init__.py b/problemtools/py.typed similarity index 100% rename from problemtools/tests/__init__.py rename to problemtools/py.typed diff --git a/problemtools/run/__init__.py b/problemtools/run/__init__.py index 79de6047..6f5791dc 100644 --- a/problemtools/run/__init__.py +++ b/problemtools/run/__init__.py @@ -1,22 +1,23 @@ """Package for managing execution of external programs in Kattis Problemtools. """ + import re import os from .buildrun import BuildRun from .checktestdata import Checktestdata -from .errors import ProgramError -from .executable import Executable +from .errors import ProgramError as ProgramError from .program import Program from .source import SourceCode from .viva import Viva -from .tools import get_tool_path, get_tool +from .tools import get_tool as get_tool, get_tool_path as get_tool_path from . import rutil -def find_programs(path, pattern='.*', language_config=None, work_dir=None, - include_dir=None, allow_validation_script=False): +def find_programs( + path, pattern='.*', language_config=None, work_dir=None, include_dir=None, allow_validation_script=False +) -> list[Program]: """Find all programs in a directory. Args: @@ -51,18 +52,19 @@ def find_programs(path, pattern='.*', language_config=None, work_dir=None, for name in sorted(os.listdir(path)): if re.match(pattern, name): fullpath = os.path.join(path, name) - run = get_program(fullpath, - language_config=language_config, - work_dir=work_dir, - include_dir=include_dir, - allow_validation_script=allow_validation_script) + run = get_program( + fullpath, + language_config=language_config, + work_dir=work_dir, + include_dir=include_dir, + allow_validation_script=allow_validation_script, + ) if run is not None: ret.append(run) return ret -def get_program(path, language_config=None, work_dir=None, include_dir=None, - allow_validation_script=False): +def get_program(path, language_config=None, work_dir=None, include_dir=None, allow_validation_script=False) -> Program | None: """Get a Program object for a program Args: @@ -102,13 +104,18 @@ def get_program(path, language_config=None, work_dir=None, include_dir=None, files = [path] else: build = os.path.join(path, 'build') - if os.path.isfile(build) and os.access(path, os.X_OK): + if os.path.isfile(build) and os.access(build, os.X_OK): return BuildRun(path, work_dir) files = rutil.list_files_recursive(path) if language_config is not None: lang = language_config.detect_language(files) if lang is not None: - return SourceCode(path, lang, - work_dir=work_dir, include_dir=include_dir) + if include_dir is not None: + lang_dir = os.path.join(include_dir, lang.lang_id) + build = os.path.join(lang_dir, 'build') + if os.path.isfile(build) and os.access(build, os.X_OK): + return BuildRun(path, work_dir=work_dir, include_dir=lang_dir) + + return SourceCode(path, lang, work_dir=work_dir, include_dir=include_dir) return None diff --git a/problemtools/run/buildrun.py b/problemtools/run/buildrun.py index 208527c0..404bcf5e 100644 --- a/problemtools/run/buildrun.py +++ b/problemtools/run/buildrun.py @@ -12,12 +12,13 @@ from .program import Program from . import rutil +log = logging.getLogger(__file__) + class BuildRun(Program): - """Class for build/run-script program. - """ + """Class for build/run-script program.""" - def __init__(self, path, work_dir=None): + def __init__(self, path, work_dir=None, include_dir=None): """Instantiate BuildRun object. Args: @@ -25,15 +26,11 @@ def __init__(self, path, work_dir=None): work_dir (str): name of temp directory in which to run the scripts (if None, will make new temp directory). """ + super().__init__() + if not os.path.isdir(path): raise ProgramError('%s is not a directory' % path) - build = os.path.join(path, 'build') - if not os.path.isfile(build): - raise ProgramError('%s does not have a build script' % path) - if not os.access(build, os.X_OK): - raise ProgramError('%s/build is not executable' % path) - if work_dir is None: work_dir = tempfile.mkdtemp() @@ -47,34 +44,35 @@ def __init__(self, path, work_dir=None): os.makedirs(self.path) rutil.add_files(path, self.path) + if include_dir is not None and os.path.isdir(include_dir): + rutil.add_files(include_dir, self.path) + # Check for existence of build script after copying include_dir, since that could contain the script + build = os.path.join(self.path, 'build') + if not os.path.isfile(build): + raise ProgramError('%s does not have a build script' % path) + if not os.access(build, os.X_OK): + raise ProgramError('%s/build is not executable' % path) - def __str__(self): + def __str__(self) -> str: """String representation""" return '%s/' % (self.path) - - _compile_result = None - def compile(self): + def do_compile(self) -> tuple[bool, str | None]: """Run the build script.""" - if self._compile_result is not None: - return self._compile_result - with open(os.devnull, 'w') as devnull: status = subprocess.call(['./build'], stdout=devnull, stderr=devnull, cwd=self.path) run = os.path.join(self.path, 'run') if status: logging.debug('Build script failed (status %d) when compiling %s\n', status, self.name) - self._compile_result = (False, 'build script failed with exit code %d' % (status)) + return (False, 'build script failed with exit code %d' % (status)) elif not os.path.isfile(run) or not os.access(run, os.X_OK): - self._compile_result = (False, 'build script did not produce an executable called "run"') + return (False, 'build script did not produce an executable called "run"') else: - self._compile_result = (True, None) - return self._compile_result - + return (True, None) - def get_runcmd(self, cwd=None, memlim=None): + def get_runcmd(self, cwd=None, memlim=None) -> list[str]: """Run command for the program. Args: @@ -84,7 +82,6 @@ def get_runcmd(self, cwd=None, memlim=None): path = self.path if cwd is None else os.path.relpath(self.path, cwd) return [os.path.join(path, 'run')] - - def should_skip_memory_rlimit(self): + def should_skip_memory_rlimit(self) -> bool: """Ugly hack (see program.py for details).""" return True diff --git a/problemtools/run/checktestdata.py b/problemtools/run/checktestdata.py index 0fd4a4d7..939c1e30 100644 --- a/problemtools/run/checktestdata.py +++ b/problemtools/run/checktestdata.py @@ -9,8 +9,8 @@ class Checktestdata(Executable): - """Wrapper class for running Checktestdata scripts. - """ + """Wrapper class for running Checktestdata scripts.""" + _CTD_PATH = get_tool_path('checktestdata') def __init__(self, path): @@ -20,33 +20,26 @@ def __init__(self, path): path (str): path to .ctd source file """ if Checktestdata._CTD_PATH is None: - raise ProgramError( - 'Could not locate the Checktestdata program to run %s' % path) - super(Checktestdata, self).__init__(Checktestdata._CTD_PATH, - args=[path]) - + raise ProgramError('Could not locate the Checktestdata program to run %s' % path) + super().__init__(Checktestdata._CTD_PATH, args=[path]) - def __str__(self): + def __str__(self) -> str: """String representation""" return '%s' % (self.args[0]) - - _compile_result = None - def compile(self): + def do_compile(self) -> tuple[bool, str | None]: """Syntax-check the Checktestdata script Returns: (False, None) if the Checktestdata script has syntax errors and (True, None) otherwise """ - if self._compile_result is None: - (status, _) = super(Checktestdata, self).run() - self._compile_result = ((os.WIFEXITED(status) and os.WEXITSTATUS(status) in [0, 1]), None) - return self._compile_result - + (status, _) = super().run() + return ((os.WIFEXITED(status) and os.WEXITSTATUS(status) in [0, 1]), None) - def run(self, infile='/dev/null', outfile='/dev/null', - errfile='/dev/null', args=None, timelim=1000): + def run( + self, infile='/dev/null', outfile='/dev/null', errfile='/dev/null', args=None, timelim=1000, memlim=1024, work_dir=None + ): """Run the Checktestdata script to validate an input file. Args: @@ -66,15 +59,13 @@ def run(self, infile='/dev/null', outfile='/dev/null', runtime (float): runtime of the Checktestdata process in seconds """ - (status, runtime) = super(Checktestdata, self).run(infile=infile, - outfile=outfile, - errfile=errfile, - args=args, - timelim=timelim) + (status, runtime) = super(Checktestdata, self).run( + infile=infile, outfile=outfile, errfile=errfile, args=args, timelim=timelim, memlim=memlim, work_dir=work_dir + ) # This is ugly, switches the accept exit status and our accept # exit status 42. if os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0: - return (42<<8, runtime) + return (42 << 8, runtime) if os.WIFEXITED(status) and os.WEXITSTATUS(status) == 42: return (0, runtime) return (status, runtime) diff --git a/problemtools/run/errors.py b/problemtools/run/errors.py index b71bc1c9..4b8d1db9 100644 --- a/problemtools/run/errors.py +++ b/problemtools/run/errors.py @@ -2,7 +2,8 @@ Error handling. """ + class ProgramError(Exception): - """Base exception class for errors within the run package. - """ + """Base exception class for errors within the run package.""" + pass diff --git a/problemtools/run/executable.py b/problemtools/run/executable.py index b2538e22..8f3a67e7 100644 --- a/problemtools/run/executable.py +++ b/problemtools/run/executable.py @@ -1,13 +1,15 @@ """ Implementation of programs provided by an executable file. """ + import os from .program import Program from .errors import ProgramError + class Executable(Program): - """Class for executable files. - """ + """Class for executable files.""" + def __init__(self, path, args=None): """Instantiate executable object. @@ -17,6 +19,8 @@ def __init__(self, path, args=None): args: list of additional command line arguments that should be passed to the program every time it is executed. """ + super().__init__() + if not os.path.isfile(path) or not os.access(path, os.X_OK): raise ProgramError('%s is not an executable program' % path) self.path = path @@ -26,14 +30,8 @@ def __str__(self): """String representation""" return '%s' % (self.path) - def compile(self): - """Dummy implementation of the compile method -- nothing to check! - """ - return (True, None) - def get_runcmd(self, cwd=None, memlim=None): - """Command to run the program. - """ + """Command to run the program.""" return [self.path] + self.args def should_skip_memory_rlimit(self): diff --git a/problemtools/run/limit.py b/problemtools/run/limit.py index db062dbf..5e56f6cc 100644 --- a/problemtools/run/limit.py +++ b/problemtools/run/limit.py @@ -4,6 +4,7 @@ import resource + def check_limit_capabilities(logger): """Check if the problemtools process is run with appropriate capabilities to set rlimits, and if not, issue warnings. @@ -16,19 +17,21 @@ def check_limit_capabilities(logger): """ (_, cpu_hard) = resource.getrlimit(resource.RLIMIT_CPU) if cpu_hard != resource.RLIM_INFINITY: - logger.warning("Hard CPU rlimit of %d, runs involving higher CPU limits than this may behave incorrectly." - % cpu_hard) + logger.warning('Hard CPU rlimit of %d, runs involving higher CPU limits than this may behave incorrectly.' % cpu_hard) (_, stack_hard) = resource.getrlimit(resource.RLIMIT_STACK) if stack_hard != resource.RLIM_INFINITY: - logger.warning("Hard stack rlimit of %d so I can't set it to unlimited. I will keep it at %d. If you experience unexpected issues (in particular run-time errors) this may be the cause." - % (stack_hard, stack_hard)) + logger.warning( + "Hard stack rlimit of %d so I can't set it to unlimited. I will keep it at %d. If you experience unexpected issues (in particular run-time errors) this may be the cause." + % (stack_hard, stack_hard) + ) (_, mem_hard) = resource.getrlimit(resource.RLIMIT_AS) if mem_hard != resource.RLIM_INFINITY: - logger.warning("Hard memory rlimit of %.0f MB, runs involving a higher memory limit may behave incorrectly. If you experience unexpected issues (in particular run-time errors) this may be the cause." - % (mem_hard/1024.0/1024.0)) - + logger.warning( + 'Hard memory rlimit of %.0f MB, runs involving a higher memory limit may behave incorrectly. If you experience unexpected issues (in particular run-time errors) this may be the cause.' + % (mem_hard / 1024.0 / 1024.0) + ) def try_limit(limit, soft, hard): @@ -49,7 +52,6 @@ def try_limit(limit, soft, hard): resource.setrlimit(limit, (soft, hard)) - def __limit_less(lim1, lim2): """Helper function for comparing two rlimit values, handling "unlimited" correctly. diff --git a/problemtools/run/program.py b/problemtools/run/program.py index b237a507..ae27c377 100644 --- a/problemtools/run/program.py +++ b/problemtools/run/program.py @@ -1,26 +1,40 @@ -"""Abstract base class for programs. -""" +"""Abstract base class for programs.""" + import os from . import limit import resource import signal import logging +import threading from .errors import ProgramError -class Program(object): - """Abstract base class for programs. - """ - runtime = 0 +from abc import ABC, abstractmethod + +log = logging.getLogger(__name__) + + +class Program(ABC): + """Abstract base class for programs.""" + + def __init__(self) -> None: + self.runtime = 0 + self._compile_lock = threading.Lock() + self._compile_result: tuple[bool, str | None] | None = None + + @abstractmethod + def get_runcmd(self, cwd=None, memlim=None) -> list[str]: + pass - def run(self, infile='/dev/null', outfile='/dev/null', errfile='/dev/null', - args=None, timelim=1000, memlim=1024): + def run( + self, infile='/dev/null', outfile='/dev/null', errfile='/dev/null', args=None, timelim=1000, memlim=1024, work_dir=None + ): """Run the program. Args: infile (str): name of file to pass on stdin outfile (str): name of file to send stdout to - errfile (str): name of file to send stderr ro + errfile (str): name of file to send stderr to args (list of str): additional command-line arguments to pass to the program timelim (int): CPU time limit in seconds @@ -39,20 +53,29 @@ def run(self, infile='/dev/null', outfile='/dev/null', errfile='/dev/null', if self.should_skip_memory_rlimit(): memlim = None - status, runtime = self.__run_wait(runcmd + args, - infile, outfile, errfile, - timelim, memlim) + status, runtime = self.__run_wait(runcmd + args, infile, outfile, errfile, timelim, memlim, work_dir) self.runtime = max(self.runtime, runtime) return status, runtime - def code_size(self): + def compile(self) -> tuple[bool, str | None]: + with self._compile_lock: + if self._compile_result is None: + self._compile_result = self.do_compile() + return self._compile_result + + def do_compile(self) -> tuple[bool, str | None]: + """Actually compile the program, if needed. Subclasses should override this method. + Do not call this manually -- use compile() instead.""" + return (True, None) + + def code_size(self) -> int: """Subclasses should override this method with the total size of the source code.""" return 0 - def should_skip_memory_rlimit(self): + def should_skip_memory_rlimit(self) -> bool: """Ugly workaround to accommodate Java -- the JVM will crash and burn if there is a memory rlimit applied and this will probably not change anytime soon [time of writing this: 2017-02-05], see @@ -67,11 +90,9 @@ def should_skip_memory_rlimit(self): """ return False - @staticmethod - def __run_wait(argv, infile, outfile, errfile, timelim, memlim): - logging.debug('run "%s < %s > %s 2> %s"', - ' '.join(argv), infile, outfile, errfile) + def __run_wait(argv, infile, outfile, errfile, timelim, memlim, working_directory=None): + log.debug('run "%s < %s > %s 2> %s"', ' '.join(argv), infile, outfile, errfile) pid = os.fork() if pid == 0: # child try: @@ -84,38 +105,35 @@ def __run_wait(argv, infile, outfile, errfile, timelim, memlim): # # This *shouldn't* cause any verdict changes given the setup for # interactive problems, but reset them anyway, for sanity. - if hasattr(signal, "SIGPIPE"): + if hasattr(signal, 'SIGPIPE'): signal.signal(signal.SIGPIPE, signal.SIG_DFL) - if hasattr(signal, "SIGXFZ"): + if hasattr(signal, 'SIGXFZ'): signal.signal(signal.SIGXFZ, signal.SIG_DFL) - if hasattr(signal, "SIGXFSZ"): + if hasattr(signal, 'SIGXFSZ'): signal.signal(signal.SIGXFSZ, signal.SIG_DFL) if timelim is not None: limit.try_limit(resource.RLIMIT_CPU, timelim, timelim + 1) if memlim is not None: limit.try_limit(resource.RLIMIT_AS, memlim * (1024**2), resource.RLIM_INFINITY) - limit.try_limit(resource.RLIMIT_STACK, - resource.RLIM_INFINITY, resource.RLIM_INFINITY) + limit.try_limit(resource.RLIMIT_STACK, resource.RLIM_INFINITY, resource.RLIM_INFINITY) Program.__setfd(0, infile, os.O_RDONLY) - Program.__setfd(1, outfile, - os.O_WRONLY | os.O_CREAT | os.O_TRUNC) - Program.__setfd(2, errfile, - os.O_WRONLY | os.O_CREAT | os.O_TRUNC) - + Program.__setfd(1, outfile, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) + Program.__setfd(2, errfile, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) + if working_directory is not None: + os.chdir(working_directory) os.execvp(argv[0], argv) except Exception as exc: - print("Oops. Fatal error in child process:") + print('Oops. Fatal error in child process:') print(exc) os.kill(os.getpid(), signal.SIGTERM) # Unreachable - logging.error("Unreachable part of run_wait reached") + log.error('Unreachable part of run_wait reached') os.kill(os.getpid(), signal.SIGTERM) (pid, status, rusage) = os.wait4(pid, 0) return status, rusage.ru_utime + rusage.ru_stime - @staticmethod def __setfd(fd, filename, flag): tmpfd = os.open(filename, flag) diff --git a/problemtools/run/rutil.py b/problemtools/run/rutil.py index 7b889e39..62df8fbe 100644 --- a/problemtools/run/rutil.py +++ b/problemtools/run/rutil.py @@ -1,11 +1,12 @@ -"""Some utility functions for the run module. -""" +"""Some utility functions for the run module.""" + import errno import os import shutil from .errors import ProgramError + def add_files(src, dstdir): """Copy src to dstdir. @@ -30,14 +31,13 @@ def add_files(src, dstdir): srcfile = os.path.join(src, name) destfile = os.path.join(dstdir, name) if os.path.isdir(srcfile): - shutil.copytree(srcfile, destfile) + shutil.copytree(srcfile, destfile, dirs_exist_ok=True) else: shutil.copy(srcfile, destfile) except IOError as exc: # FIXME why is this specific error special-cased if exc.errno == errno.ENOENT: - raise ProgramError( - 'File not found when copying program:\n %s' % exc.filename) + raise ProgramError('File not found when copying program:\n %s' % exc.filename) raise @@ -49,6 +49,6 @@ def list_files_recursive(root): directory and its subdirectories. """ ret = [] - for (path, _, files) in os.walk(root): + for path, _, files in os.walk(root): ret.extend([os.path.join(root, path, filename) for filename in files]) return ret diff --git a/problemtools/run/source.py b/problemtools/run/source.py index a7724bda..514c6cf7 100644 --- a/problemtools/run/source.py +++ b/problemtools/run/source.py @@ -1,6 +1,7 @@ """ Implementation of programs provided by source code. """ + import re import os import shlex @@ -12,9 +13,12 @@ from .program import Program from . import rutil +log = logging.getLogger(__name__) + + class SourceCode(Program): - """Class representing a program provided by source code. - """ + """Class representing a program provided by source code.""" + def __init__(self, path, language, work_dir=None, include_dir=None): """Instantiate SourceCode object @@ -36,6 +40,7 @@ def __init__(self, path, language, work_dir=None, include_dir=None): then the files in include_dir// will be copied into the work_dir along with the source file(s). """ + super().__init__() if path[-1] == '/': path = path[:-1] @@ -58,16 +63,11 @@ def __init__(self, path, language, work_dir=None, include_dir=None): if os.path.isdir(include_dir): rutil.add_files(include_dir, self.path) - self.src = sorted(self.language.get_source_files( - rutil.list_files_recursive(self.path) - )) + self.src = sorted(self.language.get_source_files(rutil.list_files_recursive(self.path))) if len(self.src) == 0: - raise ProgramError('No source files found for language %s in %s' - % (self.language.lang_id, self.name)) + raise ProgramError('No source files found for language %s in %s' % (self.language.lang_id, self.name)) - self.mainfile = next((x for x in self.src - if re.match(r'^main\.', os.path.basename(x), - re.IGNORECASE)), None) + self.mainfile = next((x for x in self.src if re.match(r'^main\.', os.path.basename(x), re.IGNORECASE)), None) if self.mainfile is None: self.mainfile = self.src[0] @@ -76,25 +76,17 @@ def __init__(self, path, language, work_dir=None, include_dir=None): self.binary = os.path.join(self.path, 'run') - - def code_size(self): + def code_size(self) -> int: return sum(os.path.getsize(x) for x in self.src) - - _compile_result = None - - def compile(self): + def do_compile(self) -> tuple[bool, str | None]: """Compile the source code. Returns tuple: (True, None) if compilation succeeded (False, errmsg) otherwise """ - if self._compile_result is not None: - return self._compile_result - if self.language.compile is None: - self._compile_result = (True, None) return (True, None) command = self.get_compilecmd() @@ -103,21 +95,17 @@ def compile(self): if not os.path.isfile(compiler) or not os.access(compiler, os.X_OK): return (False, '%s does not seem to be installed, expected to find compiler at %s' % (self.language.name, compiler)) - logging.debug('compile command: %s', command) + log.debug('compile command: %s', command) try: subprocess.check_output(command, stderr=subprocess.STDOUT) - self._compile_result = (True, None) + return (True, None) except subprocess.CalledProcessError as err: - self._compile_result = (False, err.output.decode('utf8', 'replace')) - - return self._compile_result + return (False, err.output.decode('utf8', 'replace')) - - def get_compilecmd(self): + def get_compilecmd(self) -> list[str]: return shlex.split(self.language.compile.format(**self.__get_substitution())) - def get_runcmd(self, cwd=None, memlim=1024): """Run command for the program. @@ -136,17 +124,14 @@ def get_runcmd(self, cwd=None, memlim=1024): subs['mainfile'] = os.path.relpath(subs['mainfile'], cwd) return shlex.split(self.language.run.format(**subs)) - - def should_skip_memory_rlimit(self): + def should_skip_memory_rlimit(self) -> bool: """Ugly hack (see program.py for details).""" return self.language.name in ['Java', 'Scala', 'Kotlin', 'Common Lisp'] - - def __str__(self): + def __str__(self) -> str: """String representation""" return '%s (%s)' % (self.name, self.language.name) - def __get_substitution(self, memlim=1024): return { 'path': self.path, @@ -155,5 +140,5 @@ def __get_substitution(self, memlim=1024): 'mainfile': self.mainfile, 'mainclass': self.mainclass, 'Mainclass': self.Mainclass, - 'binary': self.binary + 'binary': self.binary, } diff --git a/problemtools/run/tools.py b/problemtools/run/tools.py index 78408cc2..23c30614 100644 --- a/problemtools/run/tools.py +++ b/problemtools/run/tools.py @@ -1,6 +1,7 @@ import os from .executable import Executable + def get_tool_path(name): """Find the path to one of problemtools' external tools. @@ -11,11 +12,12 @@ def get_tool_path(name): Returns: str, path to the tool, or None if the tool was not found. """ - return __locate_executable([os.path.join(os.path.dirname(__file__), - '..', 'support', name), - os.path.join(os.path.dirname(__file__), - '..', '..', 'support', - os.path.splitext(name)[0], name)]) + return __locate_executable( + [ + os.path.join(os.path.dirname(__file__), '..', 'support', name), + os.path.join(os.path.dirname(__file__), '..', '..', 'support', os.path.splitext(name)[0], name), + ] + ) def get_tool(name): @@ -43,5 +45,4 @@ def __locate_executable(candidate_paths): str, first entry of candidate_paths that is an executable file, or None if no such entry. """ - return next((p for p in candidate_paths - if os.path.isfile(p) and os.access(p, os.X_OK)), None) + return next((p for p in candidate_paths if os.path.isfile(p) and os.access(p, os.X_OK)), None) diff --git a/problemtools/run/viva.py b/problemtools/run/viva.py index b6129169..23e35c68 100644 --- a/problemtools/run/viva.py +++ b/problemtools/run/viva.py @@ -9,8 +9,8 @@ class Viva(Executable): - """Wrapper class for running VIVA scripts. - """ + """Wrapper class for running VIVA scripts.""" + _VIVA_PATH = get_tool_path('viva.sh') def __init__(self, path): @@ -20,32 +20,25 @@ def __init__(self, path): path (str): path to .viva source file """ if Viva._VIVA_PATH is None: - raise ProgramError( - 'Could not locate the VIVA program to run %s' % path) - super(Viva, self).__init__(Viva._VIVA_PATH, - args=[path]) - + raise ProgramError('Could not locate the VIVA program to run %s' % path) + super().__init__(Viva._VIVA_PATH, args=[path]) def __str__(self): """String representation""" return '%s' % (self.args[0]) - - _compile_result = None - def compile(self): + def do_compile(self) -> tuple[bool, str | None]: """Syntax-check the VIVA script Returns: (False, None) if the VIVA script has syntax errors and (True, None) otherwise """ - if self._compile_result is None: - (status, _) = super(Viva, self).run() - self._compile_result = ((os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0), None) - return self._compile_result - + (status, _) = super().run() + return ((os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0), None) - def run(self, infile='/dev/null', outfile='/dev/null', - errfile='/dev/null', args=None, timelim=1000): + def run( + self, infile='/dev/null', outfile='/dev/null', errfile='/dev/null', args=None, timelim=1000, memlim=1024, work_dir=None + ): """Run the VIVA script to validate an input file. Args: @@ -69,14 +62,13 @@ def run(self, infile='/dev/null', outfile='/dev/null', if infile != '/dev/null': args = args + [infile] - (status, runtime) = super(Viva, self).run(outfile=outfile, - errfile=errfile, - args=args, - timelim=timelim) + (status, runtime) = super(Viva, self).run( + outfile=outfile, errfile=errfile, args=args, timelim=timelim, memlim=memlim, work_dir=work_dir + ) # This is ugly, switches the accept exit status and our accept # exit status 42. if os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0: - return (42<<8, runtime) + return (42 << 8, runtime) if os.WIFEXITED(status) and os.WEXITSTATUS(status) == 42: return (0, runtime) return (status, runtime) diff --git a/problemtools/statement_util.py b/problemtools/statement_util.py new file mode 100644 index 00000000..5cd7132f --- /dev/null +++ b/problemtools/statement_util.py @@ -0,0 +1,273 @@ +import collections +import html +import json +import os +import re +import subprocess +import tempfile +from pathlib import Path +from typing import Optional, List, Tuple +from urllib.parse import urlparse + +from . import metadata +from .formatversion import FormatVersion, get_format_version + +ALLOWED_IMAGE_EXTENSIONS = ('.png', '.jpg', '.jpeg') # ".svg" +FOOTNOTES_STRINGS = ['
', '