From 692c09e057114b2ef2c3e083bad48ffbbc8aa1f1 Mon Sep 17 00:00:00 2001 From: Altan Orhon Date: Mon, 11 Mar 2024 17:41:37 -0700 Subject: [PATCH 1/7] - Move to pyproject.toml - Bump up dependency versions - Pin down the ranges of dependencies to the current major version in order to avoid future breaking changes - Reintroduce `.flake8` because flake8` does not support to `pyproject.toml`-based configuration - Instruct `black` to support current Python versions specified by [Spec 0](https://scientific-python.org/specs/spec-0000/) (3.10-3.12) - Add Python 3.12 to the GitHub workflows --- .flake8 | 5 + .github/workflows/docbuild.yml | 2 +- .github/workflows/publish-to-test-pypi.yml | 6 +- .github/workflows/pythonpackage.yml | 4 +- pyproject.toml | 121 +++++++++++++++++---- setup.cfg | 79 -------------- 6 files changed, 108 insertions(+), 109 deletions(-) create mode 100644 .flake8 delete mode 100644 setup.cfg diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..e59aa93c --- /dev/null +++ b/.flake8 @@ -0,0 +1,5 @@ +[flake8] +max-line-length = 88 +select = C,E,F,W,B,B950 +ignore = E501,N802,N806,W503,E203 +exclude = setup.py,build,dist,doc,examples,cloudknot/data diff --git a/.github/workflows/docbuild.yml b/.github/workflows/docbuild.yml index 1f4aea10..be3e244b 100644 --- a/.github/workflows/docbuild.yml +++ b/.github/workflows/docbuild.yml @@ -8,7 +8,7 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: ["3.10", "3.11"] + python-version: ["3.10", "3.11", "3.12"] steps: - name: Checkout repo diff --git a/.github/workflows/publish-to-test-pypi.yml b/.github/workflows/publish-to-test-pypi.yml index 4d210a82..d3ff5493 100644 --- a/.github/workflows/publish-to-test-pypi.yml +++ b/.github/workflows/publish-to-test-pypi.yml @@ -7,10 +7,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@master - - name: Set up Python 3.10 + - name: Set up Python 3.11 uses: actions/setup-python@v5 with: - python-version: 3.10 + python-version: 3.11 - name: Install pep517 run: >- python -m @@ -36,4 +36,4 @@ jobs: uses: pypa/gh-action-pypi-publish@master with: user: __token__ - password: ${{ secrets.PYPI_PASSWORD }} \ No newline at end of file + password: ${{ secrets.PYPI_PASSWORD }} diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 4893f6a0..4df166e7 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -8,7 +8,7 @@ jobs: strategy: max-parallel: 8 matrix: - python-version: ["3.10", "3.11"] + python-version: ["3.10", "3.11", "3.12"] steps: - name: Checkout repo @@ -39,7 +39,7 @@ jobs: - name: Coveralls Parallel run: | coveralls - if: matrix.python-version == 3.8 + if: matrix.python-version == 3.11 env: COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/pyproject.toml b/pyproject.toml index 9d2a6abb..b8582d40 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,33 +1,106 @@ +[project] +name = "cloudknot" +dynamic = ["version"] +requires-python = ">=3.10" +license = { file = "LICENSE" } +authors = [{ name = "Adam Richie-Halford", email = "richiehalford@gmail.com" }] +maintainers = [ + { name = "Adam Richie-Halford", email = "richiehalford@gmail.com" } +] +description = "Cloudknot: a python library designed to run your existing python code on AWS Batch" +readme = "README.md" +classifiers = [ + "Development Status :: 3 - Alpha", + "Environment :: Console", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Topic :: Scientific/Engineering" +] +dependencies = [ + "awscli>=1.32.60,<2", + "boto3>=1.34.60,<2", + "botocore>=1.34.60,<2", + "cloudpickle>=3.0.0,<4", + "docker>=7.0.0,<8", + "pipreqs>=0.4.13,<0.5", + "tenacity>=8.2.3,<9", +] + +[project.urls] +"Homepage" = "https://nrdg.github.io/cloudknot" +"Source" = "https://github.com/nrdg/cloudknot" + +[project.optional-dependencies] +dev = [ + "black>=23.7.0,<24", + "coverage>=7.4,<8", + "flake8>=7.0.0,<8", + "moto[cloudformation]==4.1.5", + "numpydoc>=1.6.0,<2", + "pre-commit>=3.6.2,<4", + "pydocstyle>=6.3.0,<7", + "pytest-cov>=4.1.0,<5", + "pytest-xdist[psutil]==2.1.0", + "pytest>=8.0.0,<9", + "sphinx>=5.3.0,<6" +] +maint = ["rapidfuzz>=3.6.2,<4"] +examples = ["ipykernel>=6.29.3,<7", "numpy>=1.26.4,<2"] + +[project.scripts] +cloudknot = "cloudknot.cli:main" + [build-system] -requires = ["setuptools>=42", "wheel", "setuptools_scm[toml]>=3.4"] +requires = ["setuptools>=69.1.0", "wheel>=0.43.0", "setuptools_scm>=8.0.3"] build-backend = "setuptools.build_meta" -[tools.setuptools_scm] +[tool.setuptools_scm] + +[tool.setuptools] +package-data = { "cloudknot" = [ + "data/*", + "data/*/*", + "data/*/*/*", + "data/*/*/*/*", + "templates/*" +] } + +[tool.pytest.ini_options] +addopts = "--doctest-modules" +norecursedirs = "cloudknot/data" [tool.black] line-length = 88 -target-version = ['py37'] -exclude = ''' - +target-version = ["py310", "py311", "py312"] +extend-exclude = ''' ( - /( - \.eggs # exclude a few common directories in the - | \.git # root of the project - | \.github - | \.idea - | \.ipynb_checkpoints - | \.hg - | \.pytest_cache - | \.tox - | \.venv - | cloudknot/data - | cloudknot.egg-info - | examples - | doc - | build - | dist - )/ - | _version.py - | cli.py + /( + \.eggs # exclude a few common directories in the + | \.git # root of the project + | \.github + | \.idea + | \.ipynb_checkpoints + | \.hg + | \.pytest_cache + | \.tox + | \.venv + | cloudknot/data + | cloudknot.egg-info + | examples + | doc + | build + | dist + )/ + | _version.py + | cli.py ) ''' + +[tool.pydocstyle] +convention = "numpy" +match = "(?!_version).*\\.py" +match-dir = "cloudknot" + + diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index cc361d29..00000000 --- a/setup.cfg +++ /dev/null @@ -1,79 +0,0 @@ -[aliases] -test = pytest - -[tool:pytest] -addopts = --doctest-modules -norecursedirs = cloudknot/data - -[metadata] -name = cloudknot -url = https://nrdg.github.io/cloudknot -download_url = https://github.com/nrdg/cloudknot -author = Adam Richie-Halford -author_email = richiehalford@gmail.com -maintainer = Adam Richie-Halford -maintainer_email = richiehalford@gmail.com -classifiers = - Development Status :: 3 - Alpha - Environment :: Console - Intended Audience :: Science/Research - License :: OSI Approved :: MIT License - Operating System :: OS Independent - Programming Language :: Python - Topic :: Scientific/Engineering - -license = MIT -description = Cloudknot: a python library designed to run your existing python code on AWS Batch -long_description = file:README.md -long_description_content_type = text/markdown -platforms = OS Independent - -[options] -setup_requires = - setuptools_scm -python_requires = >=3.6 -install_requires = - boto3>=1.34.31 - awscli>=1.32.31 - botocore>=1.34.31 - cloudpickle - docker>=3.0.0 - pipreqs>=0.4.11 - tenacity>=6.3.0 -zip_safe = False -include_package_data = True -packages = find: - -[options.package_data] -cloudknot = data/*, data/*/*, data/*/*/*, data/*/*/*/*, templates/* - -[options.entry_points] -console_scripts = - cloudknot = cloudknot.cli:main - -[options.extras_require] -dev = - black==23.7.0 - coverage==5.3 - flake8==3.8.3 - numpydoc==1.1.0 - moto[cloudformation]==4.1.5 - pre-commit==3.3.3 - pydocstyle==5.1.1 - pytest-cov==2.10.1 - pytest-xdist[psutil]==2.1.0 - pytest==7.4.0 - sphinx==5.0.0 -maint = - rapidfuzz==0.12.2 - -[pydocstyle] -convention = numpy -match = (?!_version).*\.py -match-dir = cloudknot - -[flake8] -max-line-length = 88 -select = C,E,F,W,B,B950 -ignore = E501,N802,N806,W503,E203 -exclude = setup.py,build,dist,doc,examples,cloudknot/data From 25de756c0e250eeb540d6200ce284679fe6dd2b0 Mon Sep 17 00:00:00 2001 From: Altan Orhon Date: Mon, 25 Mar 2024 12:42:31 -0700 Subject: [PATCH 2/7] Updated pyproject.toml structure and formatting --- pyproject.toml | 64 ++++++++++++++++++++++---------------------------- 1 file changed, 28 insertions(+), 36 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b8582d40..49bbcbd3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,27 +5,27 @@ requires-python = ">=3.10" license = { file = "LICENSE" } authors = [{ name = "Adam Richie-Halford", email = "richiehalford@gmail.com" }] maintainers = [ - { name = "Adam Richie-Halford", email = "richiehalford@gmail.com" } + { name = "Adam Richie-Halford", email = "richiehalford@gmail.com" }, ] description = "Cloudknot: a python library designed to run your existing python code on AWS Batch" readme = "README.md" classifiers = [ - "Development Status :: 3 - Alpha", - "Environment :: Console", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Topic :: Scientific/Engineering" + "Development Status :: 3 - Alpha", + "Environment :: Console", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Topic :: Scientific/Engineering", ] dependencies = [ - "awscli>=1.32.60,<2", - "boto3>=1.34.60,<2", - "botocore>=1.34.60,<2", - "cloudpickle>=3.0.0,<4", - "docker>=7.0.0,<8", - "pipreqs>=0.4.13,<0.5", - "tenacity>=8.2.3,<9", + "awscli>=1.32.60,<2", + "boto3>=1.34.66,<2", + "botocore>=1.34.65,<2", + "cloudpickle>=3.0.0,<4", + "docker>=7.0.0,<8", + "pipreqs>=0.4.13,<0.5", + "tenacity>=8.2.3,<9", ] [project.urls] @@ -34,17 +34,17 @@ dependencies = [ [project.optional-dependencies] dev = [ - "black>=23.7.0,<24", - "coverage>=7.4,<8", - "flake8>=7.0.0,<8", - "moto[cloudformation]==4.1.5", - "numpydoc>=1.6.0,<2", - "pre-commit>=3.6.2,<4", - "pydocstyle>=6.3.0,<7", - "pytest-cov>=4.1.0,<5", - "pytest-xdist[psutil]==2.1.0", - "pytest>=8.0.0,<9", - "sphinx>=5.3.0,<6" + "black>=23.7.0,<24", + "coverage>=7.4,<8", + "flake8>=7.0.0,<8", + "moto[cloudformation]==4.1.5", + "numpydoc>=1.6.0,<2", + "pre-commit>=3.6.2,<4", + "pydocstyle>=6.3.0,<7", + "pytest-cov>=4.1.0,<5", + "pytest-xdist[psutil]==2.1.0", + "pytest>=8.0.0,<9", + "sphinx>=5.3.0,<6", ] maint = ["rapidfuzz>=3.6.2,<4"] examples = ["ipykernel>=6.29.3,<7", "numpy>=1.26.4,<2"] @@ -58,14 +58,8 @@ build-backend = "setuptools.build_meta" [tool.setuptools_scm] -[tool.setuptools] -package-data = { "cloudknot" = [ - "data/*", - "data/*/*", - "data/*/*/*", - "data/*/*/*/*", - "templates/*" -] } +[tool.setuptools.package-data] +cloudknot = ["data/**", "templates/*"] [tool.pytest.ini_options] addopts = "--doctest-modules" @@ -102,5 +96,3 @@ extend-exclude = ''' convention = "numpy" match = "(?!_version).*\\.py" match-dir = "cloudknot" - - From 3ec8a35ed1e02d723f415f3a78a306428351c36e Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Tue, 19 Mar 2024 14:54:18 -0700 Subject: [PATCH 3/7] BF: Explicitly build for the linux platform. This resolves situations where you are building this (for example) on a mac M1 machine, which builds images that can't be run in linux. --- cloudknot/templates/Dockerfile.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudknot/templates/Dockerfile.template b/cloudknot/templates/Dockerfile.template index 22b0f863..ee2d9d90 100644 --- a/cloudknot/templates/Dockerfile.template +++ b/cloudknot/templates/Dockerfile.template @@ -4,7 +4,7 @@ ############################################################################### # Use official python base image -FROM ${base_image} +FROM --platform=linux/amd64 ${base_image} # Install python dependencies COPY requirements.txt /tmp/ From dd2cf6457d37270ccf998db80936ce0fa378aa56 Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Tue, 19 Mar 2024 15:04:50 -0700 Subject: [PATCH 4/7] Try to invert the order of things. --- cloudknot/templates/Dockerfile.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudknot/templates/Dockerfile.template b/cloudknot/templates/Dockerfile.template index ee2d9d90..24ad63ce 100644 --- a/cloudknot/templates/Dockerfile.template +++ b/cloudknot/templates/Dockerfile.template @@ -4,7 +4,7 @@ ############################################################################### # Use official python base image -FROM --platform=linux/amd64 ${base_image} +FROM ${base_image} --platform=linux/amd64 # Install python dependencies COPY requirements.txt /tmp/ From 94404181f14ea85ab0cce251c7efcdc9bc82585e Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Tue, 19 Mar 2024 15:45:43 -0700 Subject: [PATCH 5/7] Use the docker API to set the target platform. Instead of using the template for this. --- cloudknot/dockerimage.py | 1 + cloudknot/templates/Dockerfile.template | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cloudknot/dockerimage.py b/cloudknot/dockerimage.py index 3b06b9d3..9d0c3cac 100644 --- a/cloudknot/dockerimage.py +++ b/cloudknot/dockerimage.py @@ -696,6 +696,7 @@ def build(self, tags, image_name=None, nocache=False): rm=True, forcerm=True, nocache=nocache, + platform="linux/amd64", ) # Update the config file images list diff --git a/cloudknot/templates/Dockerfile.template b/cloudknot/templates/Dockerfile.template index 24ad63ce..22b0f863 100644 --- a/cloudknot/templates/Dockerfile.template +++ b/cloudknot/templates/Dockerfile.template @@ -4,7 +4,7 @@ ############################################################################### # Use official python base image -FROM ${base_image} --platform=linux/amd64 +FROM ${base_image} # Install python dependencies COPY requirements.txt /tmp/ From 2be3d0b3c15f8931acd511c2a5d42d63cb6cdbdb Mon Sep 17 00:00:00 2001 From: Altan Orhon Date: Mon, 25 Mar 2024 12:58:21 -0700 Subject: [PATCH 6/7] Update tag pypa/gh-action-pypi-publish as @master is deprecated --- .github/workflows/publish-to-test-pypi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-to-test-pypi.yml b/.github/workflows/publish-to-test-pypi.yml index d3ff5493..1b5e4e5f 100644 --- a/.github/workflows/publish-to-test-pypi.yml +++ b/.github/workflows/publish-to-test-pypi.yml @@ -26,7 +26,7 @@ jobs: --out-dir dist/ . - name: Publish distribution 📦 to Test PyPI - uses: pypa/gh-action-pypi-publish@master + uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ password: ${{ secrets.TEST_PYPI_PASSWORD }} From be816b9c8ff0b2216d4214f73d444248555bc223 Mon Sep 17 00:00:00 2001 From: Altan Orhon Date: Mon, 25 Mar 2024 12:59:51 -0700 Subject: [PATCH 7/7] Update .github/workflows/publish-to-test-pypi.yml Quoted python-version Co-authored-by: Ariel Rokem --- .github/workflows/publish-to-test-pypi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-to-test-pypi.yml b/.github/workflows/publish-to-test-pypi.yml index 1b5e4e5f..8f842f92 100644 --- a/.github/workflows/publish-to-test-pypi.yml +++ b/.github/workflows/publish-to-test-pypi.yml @@ -10,7 +10,7 @@ jobs: - name: Set up Python 3.11 uses: actions/setup-python@v5 with: - python-version: 3.11 + python-version: "3.11" - name: Install pep517 run: >- python -m