diff --git a/.github/workflows/pythontest.yml b/.github/workflows/pythontest.yml index d3a08a3..fae4be8 100644 --- a/.github/workflows/pythontest.yml +++ b/.github/workflows/pythontest.yml @@ -3,13 +3,33 @@ name: Python testing on: [push, pull_request] jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.13" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install ".[dev]" + - name: lint with ruff + run: | + ruff format tdclient --diff --exit-non-zero-on-fix + ruff check tdclient + - name: Run pyright + run: | + pyright tdclient + test: runs-on: ${{ matrix.os }} strategy: max-parallel: 4 matrix: os: [ubuntu-latest, windows-latest] - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v4 @@ -23,9 +43,6 @@ jobs: pip install ".[dev]" pip install -r requirements.txt -r test-requirements.txt pip install -U coveralls pyyaml - - name: Run pyright - run: | - pyright tdclient - name: Run test run: | coverage run --source=tdclient -m pytest tdclient/test diff --git a/README.rst b/README.rst index 50a4389..2259f14 100644 --- a/README.rst +++ b/README.rst @@ -26,7 +26,7 @@ Requirements ``td-client`` supports the following versions of Python. -* Python 3.5+ +* Python 3.10+ * PyPy Install diff --git a/docs/conf.py b/docs/conf.py index 534f521..c060174 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -71,9 +71,7 @@ def linkcode_resolve(domain, info): except Exception: linenum = "" - return "https://github.com/{}/{}/blob/{}/{}/{}#L{}".format( - GH_ORGANIZATION, GH_PROJECT, revision, MODULE, relpath, linenum - ) + return f"https://github.com/{GH_ORGANIZATION}/{GH_PROJECT}/blob/{revision}/{MODULE}/{relpath}#L{linenum}" # -- Project information ----------------------------------------------------- diff --git a/pyproject.toml b/pyproject.toml index 61ea5d2..b439d6e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "td-client" version = "1.5.0" description = "Treasure Data API library for Python" readme = {file = "README.rst", content-type = "text/x-rst; charset=UTF-8"} -requires-python = ">=3.8" +requires-python = ">=3.10" license = {text = "Apache Software License"} authors = [{name = "Treasure Data, Inc.", email = "support@treasure-data.com"}] urls = {homepage = "http://treasuredata.com/"} @@ -18,11 +18,11 @@ classifiers = [ "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", "Topic :: Internet", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", ] @@ -31,7 +31,6 @@ dependencies = [ "python-dateutil", "msgpack>=0.6.2", "urllib3", - "typing-extensions>=4.0.0", ] [project.optional-dependencies] @@ -46,11 +45,16 @@ tdclient = ["py.typed"] [tool.ruff] line-length = 88 +target-version = "py310" [tool.ruff.lint] select = [ "E", + "W", "F", + "I", + "UP", + "B", ] exclude = ["tdclient/test/*"] ignore = ["E203", "E501"] @@ -62,7 +66,7 @@ known-third-party = ["dateutil","msgpack","pkg_resources","pytest","setuptools", include = ["tdclient"] exclude = ["**/__pycache__", "tdclient/test", "docs"] typeCheckingMode = "basic" -pythonVersion = "3.9" +pythonVersion = "3.10" pythonPlatform = "All" reportMissingTypeStubs = false reportUnknownMemberType = false diff --git a/tdclient/__init__.py b/tdclient/__init__.py index f569a41..1765ef1 100644 --- a/tdclient/__init__.py +++ b/tdclient/__init__.py @@ -1,5 +1,3 @@ -from __future__ import annotations - import datetime import time from typing import Any diff --git a/tdclient/api.py b/tdclient/api.py index b513930..8202b44 100644 --- a/tdclient/api.py +++ b/tdclient/api.py @@ -1,7 +1,5 @@ #!/usr/bin/env python -from __future__ import annotations - import contextlib import csv import email.utils @@ -11,7 +9,6 @@ import json import logging import os -import socket import ssl import tempfile import time @@ -34,7 +31,7 @@ from tdclient.schedule_api import ScheduleAPI from tdclient.server_status_api import ServerStatusAPI from tdclient.table_api import TableAPI -from tdclient.types import BytesOrStream +from tdclient.types import BytesOrStream, StreamBody from tdclient.user_api import UserAPI from tdclient.util import ( csv_dict_record_reader, @@ -108,11 +105,11 @@ def __init__( if user_agent is not None: self._user_agent = user_agent else: - self._user_agent = "TD-Client-Python/%s" % (version.__version__) + self._user_agent = f"TD-Client-Python/{version.__version__}" if endpoint is not None: if not urlparse.urlparse(endpoint).scheme: - endpoint = "https://{}".format(endpoint) + endpoint = f"https://{endpoint}" self._endpoint = endpoint elif os.getenv("TD_API_SERVER"): self._endpoint = os.getenv("TD_API_SERVER") @@ -154,7 +151,7 @@ def _init_http( if http_proxy.startswith("http://"): return self._init_http_proxy(http_proxy, **kwargs) else: - return self._init_http_proxy("http://%s" % (http_proxy,), **kwargs) + return self._init_http_proxy(f"http://{http_proxy}", **kwargs) def _init_http_proxy(self, http_proxy: str, **kwargs: Any) -> urllib3.ProxyManager: pool_options = dict(kwargs) @@ -164,7 +161,7 @@ def _init_http_proxy(self, http_proxy: str, **kwargs: Any) -> urllib3.ProxyManag if "@" in netloc: auth, netloc = netloc.split("@", 2) pool_options["proxy_headers"] = urllib3.make_headers(proxy_basic_auth=auth) - return urllib3.ProxyManager("%s://%s" % (scheme, netloc), **pool_options) + return urllib3.ProxyManager(f"{scheme}://{netloc}", **pool_options) def get( self, @@ -214,12 +211,12 @@ def get( self._max_cumul_retry_delay, ) except ( + OSError, urllib3.exceptions.TimeoutStateError, urllib3.exceptions.TimeoutError, urllib3.exceptions.PoolError, http.client.IncompleteRead, TimeoutError, - socket.error, ): pass @@ -235,12 +232,7 @@ def get( retry_delay *= 2 else: raise APIError( - "Retrying stopped after %d seconds. (cumulative: %d/%d)" - % ( - self._max_cumul_retry_delay, - cumul_retry_delay, - self._max_cumul_retry_delay, - ) + f"Retrying stopped after {self._max_cumul_retry_delay} seconds. (cumulative: {cumul_retry_delay}/{self._max_cumul_retry_delay})" ) log.debug( @@ -254,7 +246,7 @@ def get( def post( self, path: str, - params: dict[str, Any] | None = None, + params: dict[str, Any] | bytes | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> contextlib.AbstractContextManager[urllib3.BaseHTTPResponse]: @@ -314,13 +306,15 @@ def post( self._max_cumul_retry_delay, ) except ( + OSError, urllib3.exceptions.TimeoutStateError, urllib3.exceptions.TimeoutError, urllib3.exceptions.PoolError, - socket.error, ): if not self._retry_post_requests: - raise APIError("Retrying stopped by retry_post_requests == False") + raise APIError( + "Retrying stopped by retry_post_requests == False" + ) from None if cumul_retry_delay <= self._max_cumul_retry_delay: log.warning( @@ -334,12 +328,7 @@ def post( retry_delay *= 2 else: raise APIError( - "Retrying stopped after %d seconds. (cumulative: %d/%d)" - % ( - self._max_cumul_retry_delay, - cumul_retry_delay, - self._max_cumul_retry_delay, - ) + f"Retrying stopped after {self._max_cumul_retry_delay} seconds. (cumulative: {cumul_retry_delay}/{self._max_cumul_retry_delay})" ) log.debug( @@ -408,12 +397,12 @@ def put( else: raise APIError("Error %d: %s", response.status, response.data) except ( + OSError, urllib3.exceptions.TimeoutStateError, urllib3.exceptions.TimeoutError, urllib3.exceptions.PoolError, - socket.error, ): - raise APIError("Error: %s" % (repr(response))) + raise APIError(f"Error: {repr(response)}") from None log.debug( "REST PUT response:\n headers: %s\n status: %d\n body: ", @@ -470,10 +459,10 @@ def delete( self._max_cumul_retry_delay, ) except ( + OSError, urllib3.exceptions.TimeoutStateError, urllib3.exceptions.TimeoutError, urllib3.exceptions.PoolError, - socket.error, ): pass @@ -489,12 +478,7 @@ def delete( retry_delay *= 2 else: raise APIError( - "Retrying stopped after %d seconds. (cumulative: %d/%d)" - % ( - self._max_cumul_retry_delay, - cumul_retry_delay, - self._max_cumul_retry_delay, - ) + f"Retrying stopped after {self._max_cumul_retry_delay} seconds. (cumulative: {cumul_retry_delay}/{self._max_cumul_retry_delay})" ) log.debug( @@ -536,7 +520,7 @@ def build_request( # use default headers first _headers = dict(self._headers) # add default headers - _headers["authorization"] = "TD1 %s" % (self._apikey,) + _headers["authorization"] = f"TD1 {self._apikey}" _headers["date"] = email.utils.formatdate(time.time()) _headers["user-agent"] = self._user_agent # override given headers @@ -548,7 +532,7 @@ def send_request( method: str, url: str, fields: dict[str, Any] | None = None, - body: bytes | bytearray | memoryview | array[int] | IO[bytes] | None = None, + body: StreamBody = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> urllib3.BaseHTTPResponse: @@ -571,28 +555,26 @@ def raise_error( status_code = res.status s = body if isinstance(body, str) else body.decode("utf-8") if status_code == 404: - raise errors.NotFoundError("%s: %s" % (msg, s)) + raise errors.NotFoundError(f"{msg}: {s}") elif status_code == 409: - raise errors.AlreadyExistsError("%s: %s" % (msg, s)) + raise errors.AlreadyExistsError(f"{msg}: {s}") elif status_code == 401: - raise errors.AuthError("%s: %s" % (msg, s)) + raise errors.AuthError(f"{msg}: {s}") elif status_code == 403: - raise errors.ForbiddenError("%s: %s" % (msg, s)) + raise errors.ForbiddenError(f"{msg}: {s}") else: - raise errors.APIError("%d: %s: %s" % (status_code, msg, s)) + raise errors.APIError(f"{status_code}: {msg}: {s}") def checked_json(self, body: bytes, required: list[str]) -> dict[str, Any]: js = None try: js = json.loads(body.decode("utf-8")) except ValueError as error: - raise APIError("Unexpected API response: %s: %s" % (error, repr(body))) + raise APIError(f"Unexpected API response: {error}: {repr(body)}") from error js = dict(js) if 0 < [k in js for k in required].count(False): missing = [k for k in required if k not in js] - raise APIError( - "Unexpected API response: %s: %s" % (repr(missing), repr(body)) - ) + raise APIError(f"Unexpected API response: {repr(missing)}: {repr(body)}") return js def close(self) -> None: @@ -619,11 +601,11 @@ def _read_file(self, file_like, fmt, **kwargs): compressed = fmt.endswith(".gz") if compressed: fmt = fmt[0 : len(fmt) - len(".gz")] - reader_name = "_read_%s_file" % (fmt,) + reader_name = f"_read_{fmt}_file" if hasattr(self, reader_name): reader = getattr(self, reader_name) else: - raise TypeError("unknown format: %s" % (fmt,)) + raise TypeError(f"unknown format: {fmt}") if hasattr(file_like, "read"): if compressed: file_like = gzip.GzipFile(fileobj=file_like) diff --git a/tdclient/bulk_import_api.py b/tdclient/bulk_import_api.py index 32d4348..7f51581 100644 --- a/tdclient/bulk_import_api.py +++ b/tdclient/bulk_import_api.py @@ -1,22 +1,16 @@ #!/usr/bin/env python -from __future__ import annotations - import collections import contextlib import gzip import io import os from collections.abc import Iterator -from typing import TYPE_CHECKING, Any +from contextlib import AbstractContextManager +from typing import IO, Any import msgpack - -if TYPE_CHECKING: - from contextlib import AbstractContextManager - from typing import IO - - import urllib3 +import urllib3 from tdclient.types import BulkImportParams, BytesOrStream, DataFormat, FileLike from tdclient.util import create_url @@ -32,14 +26,14 @@ class BulkImportAPI: def get( self, path: str, - params: dict[str, Any] | None = None, + params: dict[str, Any] | bytes | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... def post( self, path: str, - params: dict[str, Any] | None = None, + params: dict[str, Any] | bytes | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... @@ -177,11 +171,11 @@ def validate_part_name(part_name: str) -> None: if 1 < d["."]: raise ValueError( - "part names cannot contain multiple periods: %s" % (repr(part_name)) + f"part names cannot contain multiple periods: {repr(part_name)}" ) if 0 < part_name.find("/"): - raise ValueError("part name must not contain '/': %s" % (repr(part_name))) + raise ValueError(f"part name must not contain '/': {repr(part_name)}") def bulk_import_upload_part( self, name: str, part_name: str, stream: BytesOrStream, size: int @@ -385,5 +379,4 @@ def bulk_import_error_records( decompressor = gzip.GzipFile(fileobj=body) unpacker = msgpack.Unpacker(decompressor, raw=False) - for row in unpacker: - yield row + yield from unpacker diff --git a/tdclient/bulk_import_model.py b/tdclient/bulk_import_model.py index be05e19..8ea061d 100644 --- a/tdclient/bulk_import_model.py +++ b/tdclient/bulk_import_model.py @@ -1,7 +1,5 @@ #!/usr/bin/env python -from __future__ import annotations - import time from collections.abc import Callable, Iterator from typing import TYPE_CHECKING, Any @@ -23,8 +21,8 @@ class BulkImport(Model): STATUS_COMMITTING = "committing" STATUS_COMMITTED = "committed" - def __init__(self, client: Client, **kwargs: Any) -> None: - super(BulkImport, self).__init__(client) + def __init__(self, client: "Client", **kwargs: Any) -> None: + super().__init__(client) self._feed(kwargs) def _feed(self, data: dict[str, Any] | None = None) -> None: @@ -116,7 +114,7 @@ def perform( wait_interval: int = 5, wait_callback: Callable[[], None] | None = None, timeout: float | None = None, - ) -> Job: + ) -> "Job": """Perform bulk import Args: @@ -128,9 +126,7 @@ def perform( """ self.update() if not self.upload_frozen: - raise ( - RuntimeError('bulk import session "%s" is not frozen' % (self.name,)) - ) + raise (RuntimeError(f'bulk import session "{self.name}" is not frozen')) job = self._client.perform_bulk_import(self.name) if wait: job.wait( @@ -164,8 +160,7 @@ def error_record_items(self) -> Iterator[dict[str, Any]]: Yields: Error record """ - for record in self._client.bulk_import_error_records(self.name): - yield record + yield from self._client.bulk_import_error_records(self.name) def upload_part(self, part_name: str, bytes_or_stream: FileLike, size: int) -> bool: """Upload a part to bulk import session diff --git a/tdclient/client.py b/tdclient/client.py index 5ca8e1e..3081102 100644 --- a/tdclient/client.py +++ b/tdclient/client.py @@ -1,12 +1,9 @@ #!/usr/bin/env python -from __future__ import annotations - import datetime import json from collections.abc import Iterator -from typing import Any, cast, Literal - +from typing import Any, Literal, cast from tdclient import api, models from tdclient.types import ( @@ -28,7 +25,7 @@ class Client: def __init__(self, *args: Any, **kwargs: Any) -> None: self._api = api.API(*args, **kwargs) - def __enter__(self) -> Client: + def __enter__(self) -> "Client": return self def __exit__( @@ -103,7 +100,7 @@ def database(self, db_name: str) -> models.Database: for name, kwargs in databases.items(): if name == db_name: return models.Database(self, name, **kwargs) - raise api.NotFoundError("Database '%s' does not exist" % (db_name)) + raise api.NotFoundError(f"Database '{db_name}' does not exist") def create_log_table(self, db_name: str, table_name: str) -> bool: """ @@ -212,7 +209,7 @@ def table(self, db_name: str, table_name: str) -> models.Table: for table in tables: if table.table_name == table_name: return table - raise api.NotFoundError("Table '%s.%s' does not exist" % (db_name, table_name)) + raise api.NotFoundError(f"Table '{db_name}.{table_name}' does not exist") def tail( self, @@ -281,7 +278,7 @@ def query( """ # for compatibility, assume type is hive unless specifically specified if type not in ["hive", "pig", "impala", "presto", "trino"]: - raise ValueError("The specified query type is not supported: %s" % (type)) + raise ValueError(f"The specified query type is not supported: {type}") # Cast type to expected literal since we've validated it query_type = cast(Literal["hive", "presto", "trino", "bulkload"], type) job_id = self.api.query( @@ -359,8 +356,7 @@ def job_result_each(self, job_id: str | int) -> Iterator[dict[str, Any]]: Returns: an iterator of result set """ - for row in self.api.job_result_each(str(job_id)): - yield row + yield from self.api.job_result_each(str(job_id)) def job_result_format( self, job_id: str | int, format: ResultFormat, header: bool = False @@ -397,14 +393,13 @@ def job_result_format_each( Returns: an iterator of rows in result set """ - for row in self.api.job_result_format_each( + yield from self.api.job_result_format_each( str(job_id), format, header=header, store_tmpfile=store_tmpfile, num_threads=num_threads, - ): - yield row + ) def download_job_result( self, job_id: str | int, path: str, num_threads: int = 4 @@ -561,8 +556,7 @@ def bulk_import_error_records(self, name: str) -> Iterator[dict[str, Any]]: Returns: an iterator of error records """ - for record in self.api.bulk_import_error_records(name): - yield record + yield from self.api.bulk_import_error_records(name) def bulk_import(self, name: str) -> models.BulkImport: """Get a bulk import session diff --git a/tdclient/connection.py b/tdclient/connection.py index 7639668..3ff539a 100644 --- a/tdclient/connection.py +++ b/tdclient/connection.py @@ -1,15 +1,13 @@ #!/usr/bin/env python -from __future__ import annotations - -from typing import TYPE_CHECKING, Any, Callable +from collections.abc import Callable +from types import TracebackType +from typing import TYPE_CHECKING, Any from tdclient import api, cursor, errors from tdclient.types import Priority if TYPE_CHECKING: - from types import TracebackType - from tdclient.cursor import Cursor @@ -22,7 +20,7 @@ def __init__( priority: Priority | None = None, retry_limit: int | None = None, wait_interval: int | None = None, - wait_callback: Callable[[Cursor], None] | None = None, + wait_callback: Callable[["Cursor"], None] | None = None, **kwargs: Any, ) -> None: cursor_kwargs = dict() @@ -43,7 +41,7 @@ def __init__( self._api = api.API(**kwargs) self._cursor_kwargs = cursor_kwargs - def __enter__(self) -> Connection: + def __enter__(self) -> "Connection": return self def __exit__( @@ -67,5 +65,5 @@ def commit(self) -> None: def rollback(self) -> None: raise errors.NotSupportedError - def cursor(self) -> Cursor: + def cursor(self) -> "Cursor": return cursor.Cursor(self._api, **self._cursor_kwargs) diff --git a/tdclient/connector_api.py b/tdclient/connector_api.py index 77f7271..93da78b 100644 --- a/tdclient/connector_api.py +++ b/tdclient/connector_api.py @@ -1,14 +1,10 @@ #!/usr/bin/env python -from __future__ import annotations - import json -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from contextlib import AbstractContextManager +from contextlib import AbstractContextManager +from typing import Any - import urllib3 +import urllib3 from tdclient.types import BytesOrStream from tdclient.util import create_url, normalize_connector_config @@ -29,7 +25,11 @@ def get( **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... def post( - self, url: str, params: Any, headers: dict[str, str] | None = None + self, + path: str, + params: dict[str, Any] | bytes | None = None, + headers: dict[str, str] | None = None, + **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... def put( self, @@ -226,7 +226,7 @@ def connector_create(self, name, database, table, job, params=None): code, body = res.status, res.read() if code != 200: self.raise_error( - "DataConnectorSession: %s created failed" % (name,), res, body + f"DataConnectorSession: {name} created failed", res, body ) return self.checked_json(body, []) @@ -243,7 +243,7 @@ def connector_show(self, name): code, body = res.status, res.read() if code != 200: self.raise_error( - "DataConnectorSession: %s retrieve failed" % (name,), res, body + f"DataConnectorSession: {name} retrieve failed", res, body ) return self.checked_json(body, []) @@ -269,7 +269,7 @@ def connector_update(self, name, job): code, body = res.status, res.read() if code != 200: self.raise_error( - "DataConnectorSession: %s update failed" % (name,), res, body + f"DataConnectorSession: {name} update failed", res, body ) return self.checked_json(body, []) @@ -286,7 +286,7 @@ def connector_delete(self, name): code, body = res.status, res.read() if code != 200: self.raise_error( - "DataConnectorSession: %s delete failed" % (name,), res, body + f"DataConnectorSession: {name} delete failed", res, body ) return self.checked_json(body, []) @@ -303,7 +303,7 @@ def connector_history(self, name): code, body = res.status, res.read() if code != 200: self.raise_error( - "history of DataConnectorSession: %s retrieve failed" % (name,), + f"history of DataConnectorSession: {name} retrieve failed", res, body, ) @@ -335,6 +335,6 @@ def connector_run(self, name, **kwargs): code, body = res.status, res.read() if code != 200: self.raise_error( - "DataConnectorSession: %s job create failed" % (name,), res, body + f"DataConnectorSession: {name} job create failed", res, body ) return self.checked_json(body, []) diff --git a/tdclient/cursor.py b/tdclient/cursor.py index 72b2030..d334f6b 100644 --- a/tdclient/cursor.py +++ b/tdclient/cursor.py @@ -1,9 +1,8 @@ #!/usr/bin/env python -from __future__ import annotations - import time -from typing import TYPE_CHECKING, Any, Callable +from collections.abc import Callable +from typing import TYPE_CHECKING, Any from tdclient import errors @@ -14,9 +13,9 @@ class Cursor: def __init__( self, - api: API, + api: "API", wait_interval: int = 5, - wait_callback: Callable[[Cursor], None] | None = None, + wait_callback: Callable[["Cursor"], None] | None = None, **kwargs: Any, ) -> None: self._api = api @@ -30,7 +29,7 @@ def __init__( self.wait_callback = wait_callback @property - def api(self) -> API: + def api(self) -> "API": return self._api @property @@ -87,9 +86,7 @@ def _do_execute(self) -> None: ) else: if status in ["error", "killed"]: - raise errors.InternalError( - "job error: %s: %s" % (self._executed, status) - ) + raise errors.InternalError(f"job error: {self._executed}: {status}") else: time.sleep(self.wait_interval) if callable(self.wait_callback): @@ -134,8 +131,7 @@ def fetchmany(self, size: int | None = None) -> list[Any]: return rows else: raise errors.InternalError( - "index out of bound (%d out of %d)" - % (self._rownumber, self._rowcount) + f"index out of bound ({self._rownumber} out of {self._rowcount})" ) def fetchall(self) -> list[Any]: diff --git a/tdclient/database_api.py b/tdclient/database_api.py index e8208ac..3e1964e 100644 --- a/tdclient/database_api.py +++ b/tdclient/database_api.py @@ -1,13 +1,9 @@ #!/usr/bin/env python -from __future__ import annotations +from contextlib import AbstractContextManager +from typing import Any -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from contextlib import AbstractContextManager - - import urllib3 +import urllib3 from tdclient.util import create_url, get_or_else, parse_date @@ -22,14 +18,14 @@ class DatabaseAPI: def get( self, path: str, - params: dict[str, Any] | None = None, + params: dict[str, Any] | bytes | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... def post( self, path: str, - params: dict[str, Any] | None = None, + params: dict[str, Any] | bytes | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... diff --git a/tdclient/database_model.py b/tdclient/database_model.py index 7652d63..64b3ac1 100644 --- a/tdclient/database_model.py +++ b/tdclient/database_model.py @@ -1,7 +1,5 @@ #!/usr/bin/env python -from __future__ import annotations - import datetime from typing import TYPE_CHECKING, Any @@ -19,8 +17,8 @@ class Database(Model): PERMISSIONS = ["administrator", "full_access", "import_only", "query_only"] PERMISSION_LIST_TABLES = ["administrator", "full_access"] - def __init__(self, client: Client, db_name: str, **kwargs: Any) -> None: - super(Database, self).__init__(client) + def __init__(self, client: "Client", db_name: str, **kwargs: Any) -> None: + super().__init__(client) self._db_name = db_name self._tables: list[Table] | None = kwargs.get("tables") self._count: int | None = kwargs.get("count") @@ -57,7 +55,7 @@ def name(self) -> str: """ return self._db_name - def tables(self) -> list[Table]: + def tables(self) -> list["Table"]: """ Returns: a list of :class:`tdclient.model.Table` @@ -67,7 +65,7 @@ def tables(self) -> list[Table]: assert self._tables is not None return self._tables - def create_log_table(self, name: str) -> Table: + def create_log_table(self, name: str) -> "Table": """ Args: name (str): name of new log table @@ -77,7 +75,7 @@ def create_log_table(self, name: str) -> Table: """ return self._client.create_log_table(self._db_name, name) - def table(self, table_name: str) -> Table: + def table(self, table_name: str) -> "Table": """ Args: table_name (str): name of a table @@ -95,7 +93,7 @@ def delete(self) -> bool: """ return self._client.delete_database(self._db_name) - def query(self, q: str, **kwargs: Any) -> Job: + def query(self, q: str, **kwargs: Any) -> "Job": """Run a query on the database Args: diff --git a/tdclient/export_api.py b/tdclient/export_api.py index c7a57f7..558f84c 100644 --- a/tdclient/export_api.py +++ b/tdclient/export_api.py @@ -1,16 +1,12 @@ #!/usr/bin/env python -from __future__ import annotations +from contextlib import AbstractContextManager +from typing import Any -from typing import TYPE_CHECKING, Any +import urllib3 -if TYPE_CHECKING: - from contextlib import AbstractContextManager - - import urllib3 - -from tdclient.util import create_url from tdclient.types import ExportParams +from tdclient.util import create_url class ExportAPI: @@ -23,7 +19,7 @@ class ExportAPI: def post( self, path: str, - params: dict[str, Any] | None = None, + params: dict[str, Any] | bytes | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... diff --git a/tdclient/import_api.py b/tdclient/import_api.py index a7af59d..cf97b57 100644 --- a/tdclient/import_api.py +++ b/tdclient/import_api.py @@ -1,16 +1,11 @@ #!/usr/bin/env python -from __future__ import annotations - import contextlib import os -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from contextlib import AbstractContextManager - from typing import IO +from contextlib import AbstractContextManager +from typing import IO, Any - import urllib3 +import urllib3 from tdclient.types import BytesOrStream, DataFormat, FileLike from tdclient.util import create_url diff --git a/tdclient/job_api.py b/tdclient/job_api.py index 475821d..1f1643c 100644 --- a/tdclient/job_api.py +++ b/tdclient/job_api.py @@ -1,7 +1,5 @@ #!/usr/bin/env python -from __future__ import annotations - import codecs import gzip import json @@ -10,19 +8,15 @@ import tempfile from collections.abc import Iterator from concurrent.futures import ThreadPoolExecutor -from typing import TYPE_CHECKING, Any, Literal +from contextlib import AbstractContextManager +from typing import Any, Literal import msgpack - -if TYPE_CHECKING: - from contextlib import AbstractContextManager - - import urllib3 +import urllib3 from tdclient.types import Priority from tdclient.util import create_url, get_or_else, parse_date - log = logging.getLogger(__name__) @@ -36,13 +30,13 @@ class JobAPI: def get( self, url: str, - params: dict[str, Any] | None = None, + params: dict[str, Any] | bytes | None = None, headers: dict[str, str] | None = None, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... def post( self, path: str, - params: dict[str, Any] | None = None, + params: dict[str, Any] | bytes | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... @@ -241,8 +235,7 @@ def job_result_each(self, job_id: str) -> Iterator[dict[str, Any]]: Yields: Row in a result """ - for row in self.job_result_format_each(job_id, "msgpack"): - yield row + yield from self.job_result_format_each(job_id, "msgpack") def job_result_format( self, job_id: str, format: str, header: bool = False @@ -454,7 +447,7 @@ def query( if priority_name in self.JOB_PRIORITY: priority_value = self.JOB_PRIORITY[priority_name] else: - raise ValueError("unknown job priority: %s" % (priority_name,)) + raise ValueError(f"unknown job priority: {priority_name}") else: priority_value = priority params["priority"] = priority_value diff --git a/tdclient/job_model.py b/tdclient/job_model.py index 8e2e35c..8c21042 100644 --- a/tdclient/job_model.py +++ b/tdclient/job_model.py @@ -1,7 +1,5 @@ #!/usr/bin/env python -from __future__ import annotations - import time import warnings from collections.abc import Callable, Iterator @@ -35,12 +33,12 @@ def type(self) -> str: """ return self._type - def __init__(self, fields: list[Schema.Field] | None = None) -> None: + def __init__(self, fields: list["Schema.Field"] | None = None) -> None: fields = [] if fields is None else fields self._fields = fields @property - def fields(self) -> list[Schema.Field]: + def fields(self) -> list["Schema.Field"]: """ TODO: add docstring """ @@ -67,9 +65,9 @@ class Job(Model): JOB_PRIORITY = {-2: "VERY LOW", -1: "LOW", 0: "NORMAL", 1: "HIGH", 2: "VERY HIGH"} def __init__( - self, client: Client, job_id: str, type: str, query: str | None, **kwargs: Any + self, client: "Client", job_id: str, type: str, query: str | None, **kwargs: Any ) -> None: - super(Job, self).__init__(client) + super().__init__(client) self._job_id = job_id self._type = type self._query = query @@ -111,7 +109,8 @@ def update(self) -> None: def _update_status(self) -> None: warnings.warn( - "_update_status() will be removed from future release. Please use update() instaed.", + "_update_status() will be removed from future release. Please use update() instead.", + stacklevel=2, category=DeprecationWarning, ) self.update() @@ -204,7 +203,7 @@ def wait( self, timeout: float | None = None, wait_interval: int = 5, - wait_callback: Callable[[Job], None] | None = None, + wait_callback: Callable[["Job"], None] | None = None, ) -> None: """Sleep until the job has been finished diff --git a/tdclient/result_api.py b/tdclient/result_api.py index 8a1f979..51614af 100644 --- a/tdclient/result_api.py +++ b/tdclient/result_api.py @@ -1,16 +1,12 @@ #!/usr/bin/env python -from __future__ import annotations +from contextlib import AbstractContextManager +from typing import Any -from typing import TYPE_CHECKING, Any +import urllib3 -if TYPE_CHECKING: - from contextlib import AbstractContextManager - - import urllib3 - -from tdclient.util import create_url from tdclient.types import ResultParams +from tdclient.util import create_url class ResultAPI: @@ -23,14 +19,14 @@ class ResultAPI: def get( self, path: str, - params: dict[str, Any] | None = None, + params: dict[str, Any] | bytes | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... def post( self, path: str, - params: dict[str, Any] | None = None, + params: dict[str, Any] | bytes | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... diff --git a/tdclient/result_model.py b/tdclient/result_model.py index 78f6e3c..34f2c50 100644 --- a/tdclient/result_model.py +++ b/tdclient/result_model.py @@ -1,7 +1,5 @@ #!/usr/bin/env python -from __future__ import annotations - from typing import TYPE_CHECKING from tdclient.model import Model @@ -13,8 +11,8 @@ class Result(Model): """Result on Treasure Data Service""" - def __init__(self, client: Client, name: str, url: str, org_name: str) -> None: - super(Result, self).__init__(client) + def __init__(self, client: "Client", name: str, url: str, org_name: str) -> None: + super().__init__(client) self._name = name self._url = url self._org_name = org_name diff --git a/tdclient/schedule_api.py b/tdclient/schedule_api.py index 1410d77..adc76c7 100644 --- a/tdclient/schedule_api.py +++ b/tdclient/schedule_api.py @@ -1,18 +1,14 @@ #!/usr/bin/env python -from __future__ import annotations - import datetime -from typing import TYPE_CHECKING, Any +from contextlib import AbstractContextManager +from typing import Any + +import urllib3 from tdclient.types import ScheduleParams from tdclient.util import create_url, get_or_else, parse_date -if TYPE_CHECKING: - from contextlib import AbstractContextManager - - import urllib3 - class ScheduleAPI: """Access to Schedule API @@ -24,14 +20,14 @@ class ScheduleAPI: def get( self, path: str, - params: dict[str, Any] | None = None, + params: dict[str, Any] | bytes | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... def post( self, path: str, - params: dict[str, Any] | None = None, + params: dict[str, Any] | bytes | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... diff --git a/tdclient/schedule_model.py b/tdclient/schedule_model.py index 161a661..4b54ed3 100644 --- a/tdclient/schedule_model.py +++ b/tdclient/schedule_model.py @@ -1,7 +1,5 @@ #!/usr/bin/env python -from __future__ import annotations - import datetime from typing import TYPE_CHECKING, Any @@ -17,14 +15,14 @@ class ScheduledJob(Job): def __init__( self, - client: Client, + client: "Client", scheduled_at: datetime.datetime, job_id: str, type: str, query: str | None, **kwargs: Any, ) -> None: - super(ScheduledJob, self).__init__(client, job_id, type, query, **kwargs) + super().__init__(client, job_id, type, query, **kwargs) self._scheduled_at = scheduled_at @property @@ -36,8 +34,8 @@ def scheduled_at(self) -> datetime.datetime: class Schedule(Model): """Schedule on Treasure Data Service""" - def __init__(self, client: Client, *args: Any, **kwargs: Any) -> None: - super(Schedule, self).__init__(client) + def __init__(self, client: "Client", *args: Any, **kwargs: Any) -> None: + super().__init__(client) if 0 < len(args): self._name: str | None = args[0] self._cron: str | None = args[1] diff --git a/tdclient/server_status_api.py b/tdclient/server_status_api.py index 499cbe5..d0f95c7 100644 --- a/tdclient/server_status_api.py +++ b/tdclient/server_status_api.py @@ -1,13 +1,9 @@ #!/usr/bin/env python -from __future__ import annotations +from contextlib import AbstractContextManager +from typing import Any -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from contextlib import AbstractContextManager - - import urllib3 +import urllib3 class ServerStatusAPI: @@ -35,7 +31,7 @@ def server_status(self) -> str: with self.get("/v3/system/server_status") as res: code, body = res.status, res.read() if code != 200: - return "Server is down (%d)" % (code,) + return f"Server is down ({code})" js = self.checked_json(body, ["status"]) status = js["status"] return status diff --git a/tdclient/table_api.py b/tdclient/table_api.py index 62b11aa..ab362f3 100644 --- a/tdclient/table_api.py +++ b/tdclient/table_api.py @@ -1,16 +1,11 @@ #!/usr/bin/env python -from __future__ import annotations - import json -from typing import TYPE_CHECKING, Any +from contextlib import AbstractContextManager +from typing import Any import msgpack - -if TYPE_CHECKING: - from contextlib import AbstractContextManager - - import urllib3 +import urllib3 from tdclient.util import create_url, get_or_else, parse_date @@ -25,14 +20,14 @@ class TableAPI: def get( self, path: str, - params: dict[str, Any] | None = None, + params: dict[str, Any] | bytes | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... def post( self, path: str, - params: dict[str, Any] | None = None, + params: dict[str, Any] | bytes | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... @@ -122,7 +117,7 @@ def _create_table( ) as res: code, body = res.status, res.read() if code != 200: - self.raise_error("Create %s table failed" % (type), res, body) + self.raise_error(f"Create {type} table failed", res, body) return True def swap_table(self, db: str, table1: str, table2: str) -> bool: diff --git a/tdclient/table_model.py b/tdclient/table_model.py index e632937..2a5248f 100644 --- a/tdclient/table_model.py +++ b/tdclient/table_model.py @@ -1,7 +1,5 @@ #!/usr/bin/env python -from __future__ import annotations - import datetime from typing import TYPE_CHECKING, Any @@ -17,7 +15,7 @@ class Table(Model): """Database table on Treasure Data Service""" def __init__(self, *args: Any, **kwargs: Any) -> None: - super(Table, self).__init__(args[0]) + super().__init__(args[0]) self.database: Database | None = None self._db_name: str = args[1] @@ -141,7 +139,7 @@ def permission(self) -> str | None: @property def identifier(self) -> str: """a string identifier of the table""" - return "%s.%s" % (self._db_name, self._table_name) + return f"{self._db_name}.{self._table_name}" def delete(self) -> str: """a string represents the type of deleted table""" @@ -208,7 +206,7 @@ def import_file( self._db_name, self._table_name, format, file, unique_id=unique_id ) - def export_data(self, storage_type: str, **kwargs: Any) -> Job: + def export_data(self, storage_type: str, **kwargs: Any) -> "Job": """Export data from Treasure Data Service Args: @@ -260,8 +258,8 @@ def estimated_storage_size_string(self) -> str: float(self._estimated_storage_size) / (1024 * 1024 * 1024) ) else: - return "%d GB" % int( - float(self._estimated_storage_size) / (1024 * 1024 * 1024) + return ( + f"{int(float(self._estimated_storage_size) / (1024 * 1024 * 1024))} GB" ) def _update_database(self) -> None: diff --git a/tdclient/types.py b/tdclient/types.py index 1ee0d49..2d035ab 100644 --- a/tdclient/types.py +++ b/tdclient/types.py @@ -1,56 +1,51 @@ """Type definitions for td-client-python.""" -from __future__ import annotations - from array import array -from typing import IO, TYPE_CHECKING - -from typing_extensions import Literal, TypeAlias, TypedDict - -if TYPE_CHECKING: - from collections.abc import Callable - from typing import Any +from collections.abc import Callable +from typing import IO, Any, Literal, TypeAlias, TypedDict # File-like types -FileLike: TypeAlias = "str | bytes | IO[bytes]" +FileLike: TypeAlias = str | bytes | IO[bytes] """Type for file inputs: file path, bytes, or file-like object.""" -BytesOrStream: TypeAlias = "bytes | bytearray | IO[bytes]" +BytesOrStream: TypeAlias = bytes | bytearray | IO[bytes] """Type for byte data or streams (excluding file paths).""" StreamBody: TypeAlias = "bytes | bytearray | memoryview | array[int] | IO[bytes] | None" """Type for HTTP request body.""" # Query engine types -QueryEngineType: TypeAlias = 'Literal["presto", "hive"]' +QueryEngineType: TypeAlias = Literal["presto", "hive"] """Type for query engine selection.""" -EngineVersion: TypeAlias = 'Literal["stable", "experimental"]' +EngineVersion: TypeAlias = Literal["stable", "experimental"] """Type for engine version selection.""" -Priority: TypeAlias = ( - 'Literal[-2, -1, 0, 1, 2, "VERY LOW", "LOW", "NORMAL", "HIGH", "VERY HIGH"]' -) +Priority: TypeAlias = Literal[ + -2, -1, 0, 1, 2, "VERY LOW", "LOW", "NORMAL", "HIGH", "VERY HIGH" +] """Type for job priority levels (numeric or string).""" # Data format types -ExportFileFormat: TypeAlias = 'Literal["jsonl.gz", "tsv.gz", "json.gz"]' +ExportFileFormat: TypeAlias = Literal["jsonl.gz", "tsv.gz", "json.gz"] """Type for export file formats.""" -DataFormat: TypeAlias = 'Literal["msgpack", "msgpack.gz", "json", "json.gz", "csv", "csv.gz", "tsv", "tsv.gz"]' +DataFormat: TypeAlias = Literal[ + "msgpack", "msgpack.gz", "json", "json.gz", "csv", "csv.gz", "tsv", "tsv.gz" +] """Type for data import/export formats.""" -ResultFormat: TypeAlias = 'Literal["msgpack", "json", "csv", "tsv"]' +ResultFormat: TypeAlias = Literal["msgpack", "json", "csv", "tsv"] """Type for query result formats.""" # Utility types for CSV parsing and data processing -CSVValue: TypeAlias = "int | float | str | bool | None" +CSVValue: TypeAlias = int | float | str | bool | None """Type for values parsed from CSV files.""" -Converter: TypeAlias = "Callable[[str], Any]" +Converter: TypeAlias = Callable[[str], Any] """Type for converter functions that parse string values.""" -Record: TypeAlias = "dict[str, Any]" +Record: TypeAlias = dict[str, Any] """Type for data records (dictionaries with string keys and any values).""" diff --git a/tdclient/user_api.py b/tdclient/user_api.py index 6212155..e420a0f 100644 --- a/tdclient/user_api.py +++ b/tdclient/user_api.py @@ -1,13 +1,9 @@ #!/usr/bin/env python -from __future__ import annotations +from contextlib import AbstractContextManager +from typing import Any -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from contextlib import AbstractContextManager - - import urllib3 +import urllib3 from tdclient.util import create_url @@ -17,14 +13,14 @@ class UserAPI: def get( self, path: str, - params: dict[str, Any] | None = None, + params: dict[str, Any] | bytes | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... def post( self, path: str, - params: dict[str, Any] | None = None, + params: dict[str, Any] | bytes | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... diff --git a/tdclient/user_model.py b/tdclient/user_model.py index 4e1045a..b7011f8 100644 --- a/tdclient/user_model.py +++ b/tdclient/user_model.py @@ -1,7 +1,5 @@ #!/usr/bin/env python -from __future__ import annotations - from typing import TYPE_CHECKING, Any from tdclient.model import Model @@ -15,14 +13,14 @@ class User(Model): def __init__( self, - client: Client, + client: "Client", name: str, org_name: str, role_names: list[str], email: str, **kwargs: Any, ) -> None: - super(User, self).__init__(client) + super().__init__(client) self._name = name self._org_name = org_name self._role_names = role_names diff --git a/tdclient/util.py b/tdclient/util.py index 55bd63e..2504312 100644 --- a/tdclient/util.py +++ b/tdclient/util.py @@ -1,5 +1,3 @@ -from __future__ import annotations - import csv import io import logging @@ -12,7 +10,7 @@ import dateutil.parser import msgpack -from tdclient.types import CSVValue, Converter, Record +from tdclient.types import Converter, CSVValue, Record log = logging.getLogger(__name__) @@ -43,6 +41,7 @@ def validate_record(record: Record) -> bool: if not any(k in record for k in ("time", b"time")): warnings.warn( 'records should have "time" column to import records properly.', + stacklevel=2, category=RuntimeWarning, ) return True @@ -129,9 +128,10 @@ def merge_dtypes_and_converters( our_converters[column_name] = DTYPE_TO_CALLABLE[dtype] except KeyError: raise ValueError( - "Unrecognized dtype %r, must be one of %s" - % (dtype, ", ".join(repr(k) for k in sorted(DTYPE_TO_CALLABLE))) - ) + "Unrecognized dtype {!r}, must be one of {}".format( + dtype, ", ".join(repr(k) for k in sorted(DTYPE_TO_CALLABLE)) + ) + ) from None if converters is not None: for column_name, parse_fn in converters.items(): our_converters[column_name] = parse_fn @@ -201,8 +201,7 @@ def csv_dict_record_reader( data) and whose values are the column values. """ reader = csv.DictReader(io.TextIOWrapper(file_like, encoding), dialect=dialect) - for row in reader: - yield row + yield from reader def csv_text_record_reader( @@ -232,7 +231,7 @@ def csv_text_record_reader( """ reader = csv.reader(io.TextIOWrapper(file_like, encoding), dialect=dialect) for row in reader: - yield dict(zip(columns, row)) + yield dict(zip(columns, row, strict=False)) def read_csv_records( @@ -299,7 +298,7 @@ def normalized_msgpack(value: Any) -> Any: Returns: Normalized value """ - if isinstance(value, (list, tuple)): + if isinstance(value, list | tuple): return [normalized_msgpack(v) for v in value] elif isinstance(value, dict): return dict( diff --git a/test-requirements.txt b/test-requirements.txt index 009e61f..6703b69 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,4 +1,3 @@ coveralls>=1.1,<1.2 -mock>=1.3,<1.4 -pytest>=4.0,<=7.2 +pytest>=8.3 tox>=3.0,<4.0