diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 79b82285..64462a66 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -13,8 +13,7 @@ permissions: contents: read jobs: - build: - + pythontests: runs-on: ubuntu-latest strategy: matrix: @@ -25,9 +24,6 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Install apt packages (for debbuild) - run: sudo apt-get install debhelper dh-python python3-pytest libboost-regex-dev build-essential - shell: bash - name: Install dependencies run: | python -m pip install --upgrade pip @@ -40,10 +36,18 @@ jobs: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --exclude examples/ # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude examples/ - - name: Build debian packages - run: make builddeb - name: Test with pytest run: pytest - name: Run mypy run: | mypy --non-interactive --config-file mypy.ini -p problemtools + + packages: # Use a separate job to test debian packaging to speed things up (no need to test this for every python version above) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install apt packages (for debbuild) + run: sudo apt-get install debhelper dh-virtualenv dpkg-dev python3-venv automake g++ make libboost-regex-dev libgmp-dev python3 git build-essential + shell: bash + - name: Build debian packages + run: make builddeb diff --git a/.gitignore b/.gitignore index 0a24bfa5..18a3eccb 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ /support/default_validator/default_validator /support/interactive/interactive build/ +/problemtools/_version.py venv/ .pytest_cache/ diff --git a/Makefile b/Makefile index 296634e5..bd3337bf 100644 --- a/Makefile +++ b/Makefile @@ -8,3 +8,7 @@ checktestdata: support/checktestdata/bootstrap support/checktestdata/bootstrap: git submodule update --init + +clean: + make -C support clean + rm -rf problemtools.egg-info build diff --git a/README.md b/README.md index 601de517..e9cef313 100644 --- a/README.md +++ b/README.md @@ -31,20 +31,17 @@ A few examples of problem packages can be found in [examples](examples). There are four supported ways of installing and running problemtools. (For non-Linux users, "Method 2" below, to use Docker, is probably the least painful.) -### Method 1: Install the Python package +### Method 1: Install the Python package using pipx Run ``` -sudo pip3 install git+https://github.com/kattis/problemtools +pipx install git+https://github.com/kattis/problemtools ``` -Or if you don't want a system-wide installation, -``` -pip3 install --user git+https://github.com/kattis/problemtools -``` -With this second option, in order to get the command line scripts, you need -to make sure that the local user bin path used (e.g., on Linux, -`$HOME/.local/bin`) is in your `$PATH`. +In order to get the command line scripts, you need to make sure that the local +user bin path used (e.g., on Linux, `$HOME/.local/bin`) is in your `$PATH`. See +[pipx' installation instructions](https://pipx.pypa.io/stable/installation/) +for information on how to install `pipx` and set up your `$PATH`. In order for problemtools to build and run properly, you also need to have LaTeX and various LaTeX packages installed. See [Requirements and @@ -95,15 +92,20 @@ Hub, these are only updated sporadically for testing purposes and not kept up to date). -### Method 3: Run directly from the repository. +### Method 3: Run directly from the repository -If you intend to help develop problemtools, or if you just want a -bare-bones way of running them, this is your option. +If you intend to help develop problemtools, or if you just want a bare-bones +way of running them, this is your option. For this method, you need to clone the repository (just downloading a zip archive of it does not work because the project has submodules that are not included in that zip archive). +Start by setting up your venv, e.g., + + python3 -m venv venv + venv/bin/pip install -r requirements.txt + In order for the tools to work, you first have to compile the various support programs, which can be done by running `make` in the root directory of problemtools. @@ -134,7 +136,7 @@ root of the repository). Apart from the build dependencies listed [below](#ubuntu), building the Debian package requires that the following tools are installed: - debhelper dh-python dpkg-dev + debhelper dh-virtualenv dpkg-dev The package can then be installed using (replace `` as appropriate): @@ -203,11 +205,11 @@ and a LaTeX installation. The dependencies needed to *build/install* problemtools can be installed with: - sudo apt install automake g++ make libboost-regex-dev libgmp-dev libgmp10 libgmpxx4ldbl python3 python3-pytest python3-setuptools python3-yaml python3-plastex + sudo apt install python3-venv automake g++ make libboost-regex-dev libgmp-dev python3 git And the dependencies needed to *run* problemtools can be installed with: - sudo apt install ghostscript libgmpxx4ldbl python3-minimal python-pkg-resources python3-plastex python3-yaml texlive-fonts-recommended texlive-lang-cyrillic texlive-latex-extra texlive-plain-generic tidy + sudo apt install ghostscript python3 texlive-fonts-recommended texlive-lang-cyrillic texlive-latex-extra texlive-plain-generic tidy dvisvgm ### Fedora diff --git a/bin/.run_in_venv.sh b/bin/.run_in_venv.sh new file mode 100755 index 00000000..dce93afb --- /dev/null +++ b/bin/.run_in_venv.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# +# Helper script for the other wrapper scripts to check that a venv exists, or +# give a helpful message if it doesn't. + +VENVPATH="$(dirname "$(dirname "$(readlink -f "$0")")")/venv" + +if [ ! -x "$VENVPATH/bin/python" ]; then + echo "I could not find a python venv at $VENVPATH." + echo "To use these wrapper scripts, please set up a venv by:" + echo " cd $(dirname "$VENVPATH")" + echo " python3 -m venv venv" + echo " venv/bin/pip install -r requirements.txt" + exit 1 +fi + +export PYTHONPATH +PYTHONPATH="$(dirname "$(dirname "$(readlink -f "$0")")")${PYTHONPATH:+:}$PYTHONPATH" +exec "$VENVPATH/bin/python" -m "$@" diff --git a/bin/problem2html.sh b/bin/problem2html.sh index eaa9d2d9..4068fbb3 100755 --- a/bin/problem2html.sh +++ b/bin/problem2html.sh @@ -5,6 +5,4 @@ # installing problemtools on the system properly, this script should # not be used. -export PYTHONPATH -PYTHONPATH="$(dirname "$(dirname "$(readlink -f "$0")")")${PYTHONPATH:+:}$PYTHONPATH" -exec python3 -m problemtools.problem2html "$@" +exec "$(dirname "$(readlink -f "$0")")/.run_in_venv.sh" problemtools.problem2html "$@" diff --git a/bin/problem2pdf.sh b/bin/problem2pdf.sh index 949c11e8..8995152d 100755 --- a/bin/problem2pdf.sh +++ b/bin/problem2pdf.sh @@ -5,6 +5,4 @@ # installing problemtools on the system properly, this script should # not be used. -export PYTHONPATH -PYTHONPATH="$(dirname "$(dirname "$(readlink -f "$0")")")${PYTHONPATH:+:}$PYTHONPATH" -exec python3 -m problemtools.problem2pdf "$@" +exec "$(dirname "$(readlink -f "$0")")/.run_in_venv.sh" problemtools.problem2pdf "$@" diff --git a/bin/verifyproblem.sh b/bin/verifyproblem.sh index 364d70c8..48ce5407 100755 --- a/bin/verifyproblem.sh +++ b/bin/verifyproblem.sh @@ -5,6 +5,4 @@ # installing problemtools on the system properly, this script should # not be used. -export PYTHONPATH -PYTHONPATH="$(dirname "$(dirname "$(readlink -f "$0")")")${PYTHONPATH:+:}$PYTHONPATH" -exec python3 -m problemtools.verifyproblem "$@" +exec "$(dirname "$(readlink -f "$0")")/.run_in_venv.sh" problemtools.verifyproblem "$@" diff --git a/debian/compat b/debian/compat deleted file mode 100644 index ec635144..00000000 --- a/debian/compat +++ /dev/null @@ -1 +0,0 @@ -9 diff --git a/debian/control b/debian/control index 42797c8b..4ba8bd76 100644 --- a/debian/control +++ b/debian/control @@ -2,13 +2,13 @@ Source: kattis-problemtools Section: devel Priority: optional Maintainer: Per Austrin -Build-Depends: debhelper (>= 8.0.0), g++ (>= 4.8), dh-python, python3, python3-setuptools, python3-pytest, python3-yaml, python3-setuptools, python3-pytest, libboost-regex-dev, libgmp-dev, automake, autoconf +Build-Depends: debhelper-compat (= 13), g++ (>= 4.8), dh-virtualenv, python3, libboost-regex-dev, libgmp-dev, automake, autoconf, git Standards-Version: 3.9.4 Homepage: https://github.com/Kattis/problemtools Package: kattis-problemtools Architecture: any -Depends: ${shlibs:Depends}, ${python3:Depends}, ${misc:Depends}, python3-plastex, python3-pkg-resources, texlive-plain-generic, texlive-fonts-recommended, texlive-latex-extra, texlive-lang-cyrillic, tidy, ghostscript, dvisvgm +Depends: ${shlibs:Depends}, ${misc:Depends}, python3, texlive-plain-generic, texlive-fonts-recommended, texlive-latex-extra, texlive-lang-cyrillic, tidy, ghostscript, dvisvgm Recommends: gcc, g++ Description: Kattis Problem Tools These are tools to manage and verify problem packages in the diff --git a/debian/kattis-problemtools.links b/debian/kattis-problemtools.links new file mode 100644 index 00000000..39354ac1 --- /dev/null +++ b/debian/kattis-problemtools.links @@ -0,0 +1,3 @@ +opt/venvs/kattis-problemtools/bin/verifyproblem usr/bin/verifyproblem +opt/venvs/kattis-problemtools/bin/problem2pdf usr/bin/problem2pdf +opt/venvs/kattis-problemtools/bin/problem2html usr/bin/problem2html diff --git a/debian/rules b/debian/rules index 76a72e36..2c10725f 100755 --- a/debian/rules +++ b/debian/rules @@ -4,12 +4,17 @@ # Uncomment this to turn on verbose mode. #export DH_VERBOSE=1 -# Uncomment this to turn off cleanup. -export PYBUILD_DISABLE=clean +%: + dh $@ --with python-virtualenv -export PYBUILD_AFTER_CLEAN=make -C support distclean -export PYBUILD_TEST_PYTEST=1 -export no_proxy=github.com +override_dh_virtualenv: + dh_virtualenv --builtin-venv --python /usr/bin/python3 -%: - dh $@ --with python3 --buildsystem=pybuild +override_dh_strip: + dh_strip --exclude=/PIL/ --exclude=/pillow.libs/ + +override_dh_shlibdeps: + dh_shlibdeps -X/x86/ -X/PIL/.libs/ -X/pillow.libs/ + +override_dh_dwz: + dh_dwz --exclude=/PIL/ --exclude=/pillow.libs/ diff --git a/problemtools/__init__.py b/problemtools/__init__.py index 8dee4bf8..e69de29b 100644 --- a/problemtools/__init__.py +++ b/problemtools/__init__.py @@ -1 +0,0 @@ -from ._version import __version__ diff --git a/problemtools/_version.py b/problemtools/_version.py deleted file mode 100644 index 98cf094a..00000000 --- a/problemtools/_version.py +++ /dev/null @@ -1,2 +0,0 @@ -# Auto-generated from git changelog, do not edit! -__version__ = '1.20231016' diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..1152c3d6 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,34 @@ +[build-system] +requires = ["setuptools >= 77.0.0", "setuptools-scm"] +build-backend = "setuptools.build_meta" + +[project] +name = "problemtools" +authors = [ + {name = "Per Austrin", email = "austrin@kattis.com"}, +] +description = "Kattis Problem Tools" +readme = "README.md" +license = "MIT" +dependencies = [ + "PyYAML", + "plasTeX>=3.0", +] +dynamic = [ "version" ] + +[project.scripts] +verifyproblem = "problemtools.verifyproblem:main" +problem2html = "problemtools.problem2html:main" +problem2pdf = "problemtools.problem2pdf:main" + +[project.urls] +Repository = "https://github.com/Kattis/problemtools" + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +include = ["problemtools", "problemtools.*"] + +[tool.setuptools_scm] +version_file = "problemtools/_version.py" diff --git a/requirements.txt b/requirements.txt index ecf975e2..c7dfe198 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ --e . \ No newline at end of file +PyYAML +plasTeX>=3.0 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 68d7b004..00000000 --- a/setup.cfg +++ /dev/null @@ -1,5 +0,0 @@ -[aliases] -# Temporarily disabled until we can make it play well with debhelper -# pybuild (issue is that pytest-runner is not available in -# deb-packaged form from standard repos) -#test=pytest diff --git a/setup.py b/setup.py index a10457e0..410e76b6 100755 --- a/setup.py +++ b/setup.py @@ -1,96 +1,69 @@ #!/usr/bin/env python3 -from setuptools import setup, find_packages -from setuptools.command.bdist_egg import bdist_egg as _bdist_egg -import distutils.cmd -from distutils.command.build import build as _build import os import subprocess +import setuptools +import setuptools.command.build +import setuptools.command.sdist -class BuildSupport(distutils.cmd.Command): + +class BuildSupport(setuptools.Command): """A custom command to build the support programs.""" description = 'build the problemtools support programs' - def initialize_options(self): - pass + build_lib: str | None - def finalize_options(self): - pass + def initialize_options(self) -> None: + self.build_lib = None + + def finalize_options(self) -> None: + self.set_undefined_options('build_py', ('build_lib', 'build_lib')) def run(self): - """Run command.""" - # FIXME this seems very fragile... - dest = os.path.join(os.path.realpath(self.distribution.command_obj['build'].build_lib), - 'problemtools', 'support') + dest = os.path.join(os.path.realpath(self.build_lib), 'problemtools', 'support') command = ['make', '-C', 'support', 'install', 'DESTDIR=%s' % dest] - self.announce('Running command: %s' % ' '.join(command), level=distutils.log.INFO) subprocess.check_call(command) +class CheckoutChecktestdata(setuptools.Command): + """A custom command to build the support programs.""" -class bdist_egg(_bdist_egg): - """Updated bdist_egg command that also builds support.""" + description = 'checkout the git submodule for checktestdata (via make)' - def run(self): - self.run_command('build_support') - _bdist_egg.run(self) + def initialize_options(self) -> None: + pass + def finalize_options(self) -> None: + pass -class build(_build): - """Updated build command that also builds support.""" + def run(self): + command = ['make', 'checktestdata'] + subprocess.check_call(command) +# It's *very* unclear from setuptools' documentation what the best way to do this is. +# +# I think that the ideal way would be to insert BuildSupport as a SubCommand +# (https://setuptools.pypa.io/en/latest/userguide/extension.html), but I cannot find +# any documented way to inject a new subcommand (aside from overwriting one of +# the existing `build_*`, but those are only run conditionally). +class build(setuptools.command.build.build): def run(self): self.run_command('build_support') - _build.run(self) - + super().run() -def get_version(): - base_dir = os.path.dirname(__file__) - - __version__ = None - try: - update_script = os.path.join(base_dir, 'admin', 'update_version.py.sh') - __version__ = subprocess.check_output([update_script]).decode('utf-8').strip() - except: - pass +# To make python -m build work from a fresh checkout, we also need to hook sdist to +# do a git submodule checkout so that the source code for checktestdata is included +# in the sdist (an alternative approach would be to include .git in the sdist (eww). +class sdist(setuptools.command.sdist.sdist): + def run(self): + self.run_command('checkout_checktestdata') + super().run() - if __version__ is None: - version_file = os.path.join(base_dir, 'problemtools', '_version.py') - with open(version_file, 'r') as version_in: - exec(version_in.read()) - - return __version__ - - -setup(name='problemtools', - version=get_version(), - description='Kattis Problem Tools', - maintainer='Per Austrin', - maintainer_email='austrin@kattis.com', - url='https://github.com/Kattis/problemtools', - license='MIT', - packages=find_packages(), - entry_points = { - 'console_scripts': [ - 'verifyproblem=problemtools.verifyproblem:main', - 'problem2html=problemtools.problem2html:main', - 'problem2pdf=problemtools.problem2pdf:main', - 'generatedata=problemtools.generatedata:main', - ] - }, - include_package_data=True, - install_requires=[ - 'PyYAML', - 'plasTeX>=3.0;python_version>="3"' - ], -# Temporarily disabled, see setup.cfg -# For now tests can be run manually with pytest -# setup_requires=['pytest-runner'], -# tests_require=['pytest'], - cmdclass={ +setuptools.setup(cmdclass={ 'build_support': BuildSupport, - 'bdist_egg': bdist_egg, - 'build': build + 'build': build, + 'checkout_checktestdata': CheckoutChecktestdata, + 'sdist': sdist, }, )