From 2b4d478baaddc5bde1315575f80bc6bd79c1d9eb Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 8 Nov 2025 00:01:28 +0500 Subject: [PATCH 1/5] Drop Python 3.9. --- .github/workflows/tests-macos.yml | 1 - .github/workflows/tests-ubuntu.yml | 1 - .github/workflows/tests-windows.yml | 1 - README.rst | 2 +- docs/index.rst | 2 +- pyproject.toml | 5 +++-- tests/test_url.py | 5 ++++- tox.ini | 2 +- w3lib/_types.py | 4 ++-- w3lib/encoding.py | 4 +++- w3lib/http.py | 6 +++--- w3lib/url.py | 4 ++-- 12 files changed, 20 insertions(+), 17 deletions(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 6682858..f4c1dea 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -17,7 +17,6 @@ jobs: fail-fast: false matrix: python-version: - - '3.9' - '3.10' - '3.11' - '3.12' diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 7f05f94..d667e67 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -17,7 +17,6 @@ jobs: fail-fast: false matrix: python-version: - - '3.9' - '3.10' - '3.11' - '3.12' diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index c846e54..fc36ec1 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -17,7 +17,6 @@ jobs: fail-fast: false matrix: python-version: - - '3.9' - '3.10' - '3.11' - '3.12' diff --git a/README.rst b/README.rst index bf95701..ce226f0 100644 --- a/README.rst +++ b/README.rst @@ -27,7 +27,7 @@ This is a Python library of web-related functions, such as: Requirements ============ -Python 3.9+ +Python 3.10+ Install ======= diff --git a/docs/index.rst b/docs/index.rst index c89d2c0..bff851d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -28,7 +28,7 @@ Modules Requirements ============ -Python 3.9+ +Python 3.10+ Install ======= diff --git a/pyproject.toml b/pyproject.toml index 105017d..c3200c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,6 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", @@ -28,7 +27,7 @@ classifiers = [ "Topic :: Internet :: WWW/HTTP", "Topic :: Software Development :: Libraries :: Python Modules", ] -requires-python = ">=3.9" +requires-python = ">=3.10" dynamic = ["version"] [project.urls] @@ -238,6 +237,8 @@ ignore = [ "RUF012", # Use of `assert` detected "S101", + + "UP038", ] [tool.ruff.lint.isort] diff --git a/tests/test_url.py b/tests/test_url.py index c48d400..2b45282 100644 --- a/tests/test_url.py +++ b/tests/test_url.py @@ -4,7 +4,7 @@ import sys from inspect import isclass from pathlib import Path -from typing import Callable +from typing import TYPE_CHECKING from urllib.parse import urlparse import pytest @@ -32,6 +32,9 @@ url_query_parameter, ) +if TYPE_CHECKING: + from collections.abc import Callable + # Test cases for URL-to-safe-URL conversions with a URL and an encoding as # input parameters. # diff --git a/tox.ini b/tox.ini index 9a2de58..58845e2 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py39, py310, py311, py312, py313, py314, pypy3.10, pypy3.11, docs, pylint, typing, pre-commit, twinecheck +envlist = py310, py311, py312, py313, py314, pypy3.10, pypy3.11, docs, pylint, typing, pre-commit, twinecheck [testenv] deps = diff --git a/w3lib/_types.py b/w3lib/_types.py index 93d20e9..a69c2e7 100644 --- a/w3lib/_types.py +++ b/w3lib/_types.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Union +from typing import TypeAlias # the base class UnicodeError doesn't have attributes like start / end -AnyUnicodeError = Union[UnicodeEncodeError, UnicodeDecodeError] +AnyUnicodeError: TypeAlias = UnicodeEncodeError | UnicodeDecodeError diff --git a/w3lib/encoding.py b/w3lib/encoding.py index 90802b2..0bbd39d 100644 --- a/w3lib/encoding.py +++ b/w3lib/encoding.py @@ -8,11 +8,13 @@ import encodings import re from re import Match -from typing import TYPE_CHECKING, Callable, cast +from typing import TYPE_CHECKING, cast import w3lib.util if TYPE_CHECKING: + from collections.abc import Callable + from w3lib._types import AnyUnicodeError _HEADER_ENCODING_RE = re.compile(r"charset=([\w-]+)", re.IGNORECASE) diff --git a/w3lib/http.py b/w3lib/http.py index fe47748..15bd785 100644 --- a/w3lib/http.py +++ b/w3lib/http.py @@ -3,12 +3,12 @@ from base64 import b64encode from collections.abc import Mapping, MutableMapping, Sequence from io import BytesIO -from typing import Any, Union, overload +from typing import Any, TypeAlias, overload from w3lib.util import to_bytes, to_unicode -HeadersDictInput = Mapping[bytes, Union[Any, Sequence[bytes]]] -HeadersDictOutput = MutableMapping[bytes, list[bytes]] +HeadersDictInput: TypeAlias = Mapping[bytes, Any | Sequence[bytes]] +HeadersDictOutput: TypeAlias = MutableMapping[bytes, list[bytes]] @overload diff --git a/w3lib/url.py b/w3lib/url.py index 566d227..bbefbbd 100644 --- a/w3lib/url.py +++ b/w3lib/url.py @@ -12,7 +12,7 @@ import re import string from pathlib import Path -from typing import TYPE_CHECKING, Callable, NamedTuple, cast, overload +from typing import TYPE_CHECKING, NamedTuple, cast, overload from urllib.parse import ( # type: ignore[attr-defined] ParseResult, _coerce_args, @@ -35,7 +35,7 @@ from .util import to_unicode if TYPE_CHECKING: - from collections.abc import Sequence + from collections.abc import Callable, Sequence from ._types import AnyUnicodeError From f4903aa63da768da3d330720418c479fa1d5c6dc Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 8 Nov 2025 00:03:13 +0500 Subject: [PATCH 2/5] Bump pre-commit hooks. --- .pre-commit-config.yaml | 8 ++++---- pyproject.toml | 2 -- tests/test_encoding.py | 6 +++--- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7c59a32..e4b3909 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,19 +1,19 @@ --- repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.12.5 + rev: v0.14.4 hooks: - id: ruff-check args: [--fix] - id: ruff-format - repo: https://github.com/adamchainz/blacken-docs - rev: 1.19.1 + rev: 1.20.0 hooks: - id: blacken-docs additional_dependencies: - - black==25.1.0 + - black==25.9.0 - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - id: end-of-file-fixer - id: trailing-whitespace diff --git a/pyproject.toml b/pyproject.toml index c3200c6..3bc554b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -237,8 +237,6 @@ ignore = [ "RUF012", # Use of `assert` detected "S101", - - "UP038", ] [tool.ruff.lint.isort] diff --git a/tests/test_encoding.py b/tests/test_encoding.py index 3502ed7..cfd2ece 100644 --- a/tests/test_encoding.py +++ b/tests/test_encoding.py @@ -137,7 +137,7 @@ class TestHtmlConversion: def test_unicode_body(self): unicode_string = "\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 \u0442\u0435\u043a\u0441\u0442" original_string = unicode_string.encode("cp1251") - encoding, body_unicode = html_to_unicode(ct("cp1251"), original_string) + _, body_unicode = html_to_unicode(ct("cp1251"), original_string) # check body_as_unicode assert isinstance(body_unicode, str) assert body_unicode == unicode_string @@ -207,7 +207,7 @@ def test_utf8_unexpected_end_of_data_with_valid_utf8_BOM(self): def test_replace_wrong_encoding(self): """Test invalid chars are replaced properly""" - encoding, body_unicode = html_to_unicode(ct("utf-8"), b"PREFIX\xe3\xabSUFFIX") + _, body_unicode = html_to_unicode(ct("utf-8"), b"PREFIX\xe3\xabSUFFIX") # XXX: Policy for replacing invalid chars may suffer minor variations # but it should always contain the unicode replacement char ('\ufffd') assert "\ufffd" in body_unicode, repr(body_unicode) @@ -215,7 +215,7 @@ def test_replace_wrong_encoding(self): assert "SUFFIX" in body_unicode, repr(body_unicode) # Do not destroy html tags due to encoding bugs - encoding, body_unicode = html_to_unicode(ct("utf-8"), b"\xf0value") + _, body_unicode = html_to_unicode(ct("utf-8"), b"\xf0value") assert "value" in body_unicode, repr(body_unicode) def _assert_encoding_detected( From 34f2e9b63e0c86a76fcd4e74444e368b0be820cc Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 8 Nov 2025 00:05:14 +0500 Subject: [PATCH 3/5] Bump GitHub actions. --- .github/workflows/checks.yml | 12 ++++++------ .github/workflows/publish.yml | 8 ++++---- .github/workflows/tests-macos.yml | 6 +++--- .github/workflows/tests-ubuntu.yml | 6 +++--- .github/workflows/tests-windows.yml | 6 +++--- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 4d6b5a9..114dc57 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -21,24 +21,24 @@ jobs: - python-version: '3.13' env: TOXENV: docs - - python-version: '3.13' + - python-version: '3.14' env: TOXENV: pre-commit - - python-version: '3.13' + - python-version: '3.14' env: TOXENV: pylint - - python-version: '3.13' + - python-version: '3.14' env: TOXENV: typing - - python-version: '3.13' + - python-version: '3.14' env: TOXENV: twinecheck steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 5cb1ffb..f8d885b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -11,12 +11,12 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - - name: Set up Python 3.13 - uses: actions/setup-python@v5 + - name: Set up Python 3.14 + uses: actions/setup-python@v6 with: - python-version: '3.13' + python-version: '3.14' - name: Check Tag id: check-release-tag diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index f4c1dea..f971286 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -21,15 +21,15 @@ jobs: - '3.11' - '3.12' - '3.13' - - 3.14.0-rc.1 + - '3.14' - pypy3.10 - pypy3.11 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index d667e67..7207ab4 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -21,15 +21,15 @@ jobs: - '3.11' - '3.12' - '3.13' - - 3.14.0-rc.1 + - '3.14' - pypy3.10 - pypy3.11 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index fc36ec1..cd25b79 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -21,15 +21,15 @@ jobs: - '3.11' - '3.12' - '3.13' - - 3.14.0-rc.1 + - '3.14' - pypy3.10 - pypy3.11 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} From 2816462892ddcf3bf06bf5bac46562d5c366675e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 8 Nov 2025 00:05:48 +0500 Subject: [PATCH 4/5] Drop PyPy 3.10. --- .github/workflows/tests-macos.yml | 1 - .github/workflows/tests-ubuntu.yml | 1 - .github/workflows/tests-windows.yml | 1 - tox.ini | 2 +- 4 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index f971286..7713e3e 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -22,7 +22,6 @@ jobs: - '3.12' - '3.13' - '3.14' - - pypy3.10 - pypy3.11 steps: diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 7207ab4..d289ce4 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -22,7 +22,6 @@ jobs: - '3.12' - '3.13' - '3.14' - - pypy3.10 - pypy3.11 steps: diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index cd25b79..67591dd 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -22,7 +22,6 @@ jobs: - '3.12' - '3.13' - '3.14' - - pypy3.10 - pypy3.11 steps: diff --git a/tox.ini b/tox.ini index 58845e2..60bfd6f 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py310, py311, py312, py313, py314, pypy3.10, pypy3.11, docs, pylint, typing, pre-commit, twinecheck +envlist = py310, py311, py312, py313, py314, pypy3.11, docs, pylint, typing, pre-commit, twinecheck [testenv] deps = From 99461553ee948e0fd809d252aa2379e091b428b2 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 8 Nov 2025 00:09:08 +0500 Subject: [PATCH 5/5] Bump other tools. --- pyproject.toml | 12 +----------- tox.ini | 13 ++++++------- 2 files changed, 7 insertions(+), 18 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3bc554b..4087950 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,18 +65,8 @@ filename = "docs/conf.py" [tool.coverage.run] branch = true -[tool.coverage.report] -exclude_also = [ - "if TYPE_CHECKING:", -] - [tool.mypy] -check_untyped_defs = true - -[[tool.mypy.overrides]] -# All non-tests functions must be typed. -module = "w3lib.*" -allow_untyped_defs = false +strict = true [[tool.mypy.overrides]] # Allow test functions to be untyped diff --git a/tox.ini b/tox.ini index 60bfd6f..a729cc7 100644 --- a/tox.ini +++ b/tox.ini @@ -8,9 +8,8 @@ envlist = py310, py311, py312, py313, py314, pypy3.11, docs, pylint, typing, pre [testenv] deps = - coverage >= 7.2.0 pytest !=3.1.1, !=3.1.2 - pytest-cov + pytest-cov >= 7.0.0 commands = python -m pytest \ --doctest-modules \ @@ -21,14 +20,14 @@ commands = basepython = python3 deps = pytest - mypy==1.17.0 + mypy==1.18.2 commands = - mypy --strict {posargs: w3lib tests} + mypy {posargs: w3lib tests} [testenv:pylint] deps = {[testenv]deps} - pylint==3.3.7 + pylint==4.0.2 commands = pylint docs tests w3lib @@ -46,8 +45,8 @@ skip_install = true [testenv:twinecheck] basepython = python3 deps = - twine==6.1.0 - build==1.2.2.post1 + twine==6.2.0 + build==1.3.0 commands = python -m build --sdist twine check dist/*