From f4b3293e902cc1f3fff0495bd8d8d797c02d33e6 Mon Sep 17 00:00:00 2001 From: Aki Ariga Date: Thu, 30 Oct 2025 18:39:42 -0700 Subject: [PATCH 01/14] Apply I rule --- pyproject.toml | 4 ++++ tdclient/client.py | 3 +-- tdclient/export_api.py | 2 +- tdclient/job_api.py | 1 - tdclient/result_api.py | 2 +- tdclient/util.py | 2 +- 6 files changed, 8 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 61ea5d2..ea6d640 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,11 @@ line-length = 88 [tool.ruff.lint] select = [ "E", + # "W", "F", + "I", + # "UP", + # "B", ] exclude = ["tdclient/test/*"] ignore = ["E203", "E501"] diff --git a/tdclient/client.py b/tdclient/client.py index 5ca8e1e..62a8e3f 100644 --- a/tdclient/client.py +++ b/tdclient/client.py @@ -5,8 +5,7 @@ import datetime import json from collections.abc import Iterator -from typing import Any, cast, Literal - +from typing import Any, Literal, cast from tdclient import api, models from tdclient.types import ( diff --git a/tdclient/export_api.py b/tdclient/export_api.py index 73ba1b7..a8f18e6 100644 --- a/tdclient/export_api.py +++ b/tdclient/export_api.py @@ -9,8 +9,8 @@ import urllib3 -from tdclient.util import create_url from tdclient.types import ExportParams +from tdclient.util import create_url class ExportAPI: diff --git a/tdclient/job_api.py b/tdclient/job_api.py index ca1bf49..422f717 100644 --- a/tdclient/job_api.py +++ b/tdclient/job_api.py @@ -22,7 +22,6 @@ from tdclient.types import Priority from tdclient.util import create_url, get_or_else, parse_date - log = logging.getLogger(__name__) diff --git a/tdclient/result_api.py b/tdclient/result_api.py index 2eb0e18..4c06e7e 100644 --- a/tdclient/result_api.py +++ b/tdclient/result_api.py @@ -9,8 +9,8 @@ import urllib3 -from tdclient.util import create_url from tdclient.types import ResultParams +from tdclient.util import create_url class ResultAPI: diff --git a/tdclient/util.py b/tdclient/util.py index 55bd63e..9df3b43 100644 --- a/tdclient/util.py +++ b/tdclient/util.py @@ -12,7 +12,7 @@ import dateutil.parser import msgpack -from tdclient.types import CSVValue, Converter, Record +from tdclient.types import Converter, CSVValue, Record log = logging.getLogger(__name__) From e676a58e4f5142f5a744d584f5558e924b96119a Mon Sep 17 00:00:00 2001 From: Aki Ariga Date: Thu, 30 Oct 2025 18:46:03 -0700 Subject: [PATCH 02/14] Apply UP rule --- docs/conf.py | 4 +-- pyproject.toml | 4 +-- tdclient/api.py | 62 +++++++++++++---------------------- tdclient/bulk_import_api.py | 7 ++-- tdclient/bulk_import_model.py | 9 ++--- tdclient/client.py | 17 ++++------ tdclient/connector_api.py | 12 +++---- tdclient/cursor.py | 7 ++-- tdclient/database_model.py | 2 +- tdclient/job_api.py | 5 ++- tdclient/job_model.py | 2 +- tdclient/result_model.py | 2 +- tdclient/schedule_model.py | 4 +-- tdclient/server_status_api.py | 2 +- tdclient/table_api.py | 2 +- tdclient/table_model.py | 8 ++--- tdclient/user_model.py | 2 +- tdclient/util.py | 8 ++--- 18 files changed, 64 insertions(+), 95 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 534f521..c060174 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -71,9 +71,7 @@ def linkcode_resolve(domain, info): except Exception: linenum = "" - return "https://github.com/{}/{}/blob/{}/{}/{}#L{}".format( - GH_ORGANIZATION, GH_PROJECT, revision, MODULE, relpath, linenum - ) + return f"https://github.com/{GH_ORGANIZATION}/{GH_PROJECT}/blob/{revision}/{MODULE}/{relpath}#L{linenum}" # -- Project information ----------------------------------------------------- diff --git a/pyproject.toml b/pyproject.toml index ea6d640..2305a27 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,10 +50,10 @@ line-length = 88 [tool.ruff.lint] select = [ "E", - # "W", + "W", "F", "I", - # "UP", + "UP", # "B", ] exclude = ["tdclient/test/*"] diff --git a/tdclient/api.py b/tdclient/api.py index b513930..1eee188 100644 --- a/tdclient/api.py +++ b/tdclient/api.py @@ -11,7 +11,6 @@ import json import logging import os -import socket import ssl import tempfile import time @@ -108,11 +107,11 @@ def __init__( if user_agent is not None: self._user_agent = user_agent else: - self._user_agent = "TD-Client-Python/%s" % (version.__version__) + self._user_agent = f"TD-Client-Python/{version.__version__}" if endpoint is not None: if not urlparse.urlparse(endpoint).scheme: - endpoint = "https://{}".format(endpoint) + endpoint = f"https://{endpoint}" self._endpoint = endpoint elif os.getenv("TD_API_SERVER"): self._endpoint = os.getenv("TD_API_SERVER") @@ -154,7 +153,7 @@ def _init_http( if http_proxy.startswith("http://"): return self._init_http_proxy(http_proxy, **kwargs) else: - return self._init_http_proxy("http://%s" % (http_proxy,), **kwargs) + return self._init_http_proxy(f"http://{http_proxy}", **kwargs) def _init_http_proxy(self, http_proxy: str, **kwargs: Any) -> urllib3.ProxyManager: pool_options = dict(kwargs) @@ -164,7 +163,7 @@ def _init_http_proxy(self, http_proxy: str, **kwargs: Any) -> urllib3.ProxyManag if "@" in netloc: auth, netloc = netloc.split("@", 2) pool_options["proxy_headers"] = urllib3.make_headers(proxy_basic_auth=auth) - return urllib3.ProxyManager("%s://%s" % (scheme, netloc), **pool_options) + return urllib3.ProxyManager(f"{scheme}://{netloc}", **pool_options) def get( self, @@ -214,12 +213,12 @@ def get( self._max_cumul_retry_delay, ) except ( + OSError, urllib3.exceptions.TimeoutStateError, urllib3.exceptions.TimeoutError, urllib3.exceptions.PoolError, http.client.IncompleteRead, TimeoutError, - socket.error, ): pass @@ -235,12 +234,7 @@ def get( retry_delay *= 2 else: raise APIError( - "Retrying stopped after %d seconds. (cumulative: %d/%d)" - % ( - self._max_cumul_retry_delay, - cumul_retry_delay, - self._max_cumul_retry_delay, - ) + f"Retrying stopped after {self._max_cumul_retry_delay} seconds. (cumulative: {cumul_retry_delay}/{self._max_cumul_retry_delay})" ) log.debug( @@ -314,10 +308,10 @@ def post( self._max_cumul_retry_delay, ) except ( + OSError, urllib3.exceptions.TimeoutStateError, urllib3.exceptions.TimeoutError, urllib3.exceptions.PoolError, - socket.error, ): if not self._retry_post_requests: raise APIError("Retrying stopped by retry_post_requests == False") @@ -334,12 +328,7 @@ def post( retry_delay *= 2 else: raise APIError( - "Retrying stopped after %d seconds. (cumulative: %d/%d)" - % ( - self._max_cumul_retry_delay, - cumul_retry_delay, - self._max_cumul_retry_delay, - ) + f"Retrying stopped after {self._max_cumul_retry_delay} seconds. (cumulative: {cumul_retry_delay}/{self._max_cumul_retry_delay})" ) log.debug( @@ -408,12 +397,12 @@ def put( else: raise APIError("Error %d: %s", response.status, response.data) except ( + OSError, urllib3.exceptions.TimeoutStateError, urllib3.exceptions.TimeoutError, urllib3.exceptions.PoolError, - socket.error, ): - raise APIError("Error: %s" % (repr(response))) + raise APIError(f"Error: {repr(response)}") log.debug( "REST PUT response:\n headers: %s\n status: %d\n body: ", @@ -470,10 +459,10 @@ def delete( self._max_cumul_retry_delay, ) except ( + OSError, urllib3.exceptions.TimeoutStateError, urllib3.exceptions.TimeoutError, urllib3.exceptions.PoolError, - socket.error, ): pass @@ -489,12 +478,7 @@ def delete( retry_delay *= 2 else: raise APIError( - "Retrying stopped after %d seconds. (cumulative: %d/%d)" - % ( - self._max_cumul_retry_delay, - cumul_retry_delay, - self._max_cumul_retry_delay, - ) + f"Retrying stopped after {self._max_cumul_retry_delay} seconds. (cumulative: {cumul_retry_delay}/{self._max_cumul_retry_delay})" ) log.debug( @@ -536,7 +520,7 @@ def build_request( # use default headers first _headers = dict(self._headers) # add default headers - _headers["authorization"] = "TD1 %s" % (self._apikey,) + _headers["authorization"] = f"TD1 {self._apikey}" _headers["date"] = email.utils.formatdate(time.time()) _headers["user-agent"] = self._user_agent # override given headers @@ -571,28 +555,26 @@ def raise_error( status_code = res.status s = body if isinstance(body, str) else body.decode("utf-8") if status_code == 404: - raise errors.NotFoundError("%s: %s" % (msg, s)) + raise errors.NotFoundError(f"{msg}: {s}") elif status_code == 409: - raise errors.AlreadyExistsError("%s: %s" % (msg, s)) + raise errors.AlreadyExistsError(f"{msg}: {s}") elif status_code == 401: - raise errors.AuthError("%s: %s" % (msg, s)) + raise errors.AuthError(f"{msg}: {s}") elif status_code == 403: - raise errors.ForbiddenError("%s: %s" % (msg, s)) + raise errors.ForbiddenError(f"{msg}: {s}") else: - raise errors.APIError("%d: %s: %s" % (status_code, msg, s)) + raise errors.APIError(f"{status_code}: {msg}: {s}") def checked_json(self, body: bytes, required: list[str]) -> dict[str, Any]: js = None try: js = json.loads(body.decode("utf-8")) except ValueError as error: - raise APIError("Unexpected API response: %s: %s" % (error, repr(body))) + raise APIError(f"Unexpected API response: {error}: {repr(body)}") js = dict(js) if 0 < [k in js for k in required].count(False): missing = [k for k in required if k not in js] - raise APIError( - "Unexpected API response: %s: %s" % (repr(missing), repr(body)) - ) + raise APIError(f"Unexpected API response: {repr(missing)}: {repr(body)}") return js def close(self) -> None: @@ -619,11 +601,11 @@ def _read_file(self, file_like, fmt, **kwargs): compressed = fmt.endswith(".gz") if compressed: fmt = fmt[0 : len(fmt) - len(".gz")] - reader_name = "_read_%s_file" % (fmt,) + reader_name = f"_read_{fmt}_file" if hasattr(self, reader_name): reader = getattr(self, reader_name) else: - raise TypeError("unknown format: %s" % (fmt,)) + raise TypeError(f"unknown format: {fmt}") if hasattr(file_like, "read"): if compressed: file_like = gzip.GzipFile(fileobj=file_like) diff --git a/tdclient/bulk_import_api.py b/tdclient/bulk_import_api.py index 9b257c6..12c3c27 100644 --- a/tdclient/bulk_import_api.py +++ b/tdclient/bulk_import_api.py @@ -164,11 +164,11 @@ def validate_part_name(part_name: str) -> None: if 1 < d["."]: raise ValueError( - "part names cannot contain multiple periods: %s" % (repr(part_name)) + f"part names cannot contain multiple periods: {repr(part_name)}" ) if 0 < part_name.find("/"): - raise ValueError("part name must not contain '/': %s" % (repr(part_name))) + raise ValueError(f"part name must not contain '/': {repr(part_name)}") def bulk_import_upload_part( self, name: str, part_name: str, stream: BytesOrStream, size: int @@ -372,5 +372,4 @@ def bulk_import_error_records( decompressor = gzip.GzipFile(fileobj=body) unpacker = msgpack.Unpacker(decompressor, raw=False) - for row in unpacker: - yield row + yield from unpacker diff --git a/tdclient/bulk_import_model.py b/tdclient/bulk_import_model.py index be05e19..b6092c1 100644 --- a/tdclient/bulk_import_model.py +++ b/tdclient/bulk_import_model.py @@ -24,7 +24,7 @@ class BulkImport(Model): STATUS_COMMITTED = "committed" def __init__(self, client: Client, **kwargs: Any) -> None: - super(BulkImport, self).__init__(client) + super().__init__(client) self._feed(kwargs) def _feed(self, data: dict[str, Any] | None = None) -> None: @@ -128,9 +128,7 @@ def perform( """ self.update() if not self.upload_frozen: - raise ( - RuntimeError('bulk import session "%s" is not frozen' % (self.name,)) - ) + raise (RuntimeError(f'bulk import session "{self.name}" is not frozen')) job = self._client.perform_bulk_import(self.name) if wait: job.wait( @@ -164,8 +162,7 @@ def error_record_items(self) -> Iterator[dict[str, Any]]: Yields: Error record """ - for record in self._client.bulk_import_error_records(self.name): - yield record + yield from self._client.bulk_import_error_records(self.name) def upload_part(self, part_name: str, bytes_or_stream: FileLike, size: int) -> bool: """Upload a part to bulk import session diff --git a/tdclient/client.py b/tdclient/client.py index 62a8e3f..e34fe0e 100644 --- a/tdclient/client.py +++ b/tdclient/client.py @@ -102,7 +102,7 @@ def database(self, db_name: str) -> models.Database: for name, kwargs in databases.items(): if name == db_name: return models.Database(self, name, **kwargs) - raise api.NotFoundError("Database '%s' does not exist" % (db_name)) + raise api.NotFoundError(f"Database '{db_name}' does not exist") def create_log_table(self, db_name: str, table_name: str) -> bool: """ @@ -211,7 +211,7 @@ def table(self, db_name: str, table_name: str) -> models.Table: for table in tables: if table.table_name == table_name: return table - raise api.NotFoundError("Table '%s.%s' does not exist" % (db_name, table_name)) + raise api.NotFoundError(f"Table '{db_name}.{table_name}' does not exist") def tail( self, @@ -280,7 +280,7 @@ def query( """ # for compatibility, assume type is hive unless specifically specified if type not in ["hive", "pig", "impala", "presto", "trino"]: - raise ValueError("The specified query type is not supported: %s" % (type)) + raise ValueError(f"The specified query type is not supported: {type}") # Cast type to expected literal since we've validated it query_type = cast(Literal["hive", "presto", "trino", "bulkload"], type) job_id = self.api.query( @@ -358,8 +358,7 @@ def job_result_each(self, job_id: str | int) -> Iterator[dict[str, Any]]: Returns: an iterator of result set """ - for row in self.api.job_result_each(str(job_id)): - yield row + yield from self.api.job_result_each(str(job_id)) def job_result_format( self, job_id: str | int, format: ResultFormat, header: bool = False @@ -396,14 +395,13 @@ def job_result_format_each( Returns: an iterator of rows in result set """ - for row in self.api.job_result_format_each( + yield from self.api.job_result_format_each( str(job_id), format, header=header, store_tmpfile=store_tmpfile, num_threads=num_threads, - ): - yield row + ) def download_job_result( self, job_id: str | int, path: str, num_threads: int = 4 @@ -560,8 +558,7 @@ def bulk_import_error_records(self, name: str) -> Iterator[dict[str, Any]]: Returns: an iterator of error records """ - for record in self.api.bulk_import_error_records(name): - yield record + yield from self.api.bulk_import_error_records(name) def bulk_import(self, name: str) -> models.BulkImport: """Get a bulk import session diff --git a/tdclient/connector_api.py b/tdclient/connector_api.py index 75549d7..71b93af 100644 --- a/tdclient/connector_api.py +++ b/tdclient/connector_api.py @@ -210,7 +210,7 @@ def connector_create(self, name, database, table, job, params=None): code, body = res.status, res.read() if code != 200: self.raise_error( - "DataConnectorSession: %s created failed" % (name,), res, body + f"DataConnectorSession: {name} created failed", res, body ) return self.checked_json(body, []) @@ -227,7 +227,7 @@ def connector_show(self, name): code, body = res.status, res.read() if code != 200: self.raise_error( - "DataConnectorSession: %s retrieve failed" % (name,), res, body + f"DataConnectorSession: {name} retrieve failed", res, body ) return self.checked_json(body, []) @@ -253,7 +253,7 @@ def connector_update(self, name, job): code, body = res.status, res.read() if code != 200: self.raise_error( - "DataConnectorSession: %s update failed" % (name,), res, body + f"DataConnectorSession: {name} update failed", res, body ) return self.checked_json(body, []) @@ -270,7 +270,7 @@ def connector_delete(self, name): code, body = res.status, res.read() if code != 200: self.raise_error( - "DataConnectorSession: %s delete failed" % (name,), res, body + f"DataConnectorSession: {name} delete failed", res, body ) return self.checked_json(body, []) @@ -287,7 +287,7 @@ def connector_history(self, name): code, body = res.status, res.read() if code != 200: self.raise_error( - "history of DataConnectorSession: %s retrieve failed" % (name,), + f"history of DataConnectorSession: {name} retrieve failed", res, body, ) @@ -319,6 +319,6 @@ def connector_run(self, name, **kwargs): code, body = res.status, res.read() if code != 200: self.raise_error( - "DataConnectorSession: %s job create failed" % (name,), res, body + f"DataConnectorSession: {name} job create failed", res, body ) return self.checked_json(body, []) diff --git a/tdclient/cursor.py b/tdclient/cursor.py index 72b2030..260d750 100644 --- a/tdclient/cursor.py +++ b/tdclient/cursor.py @@ -87,9 +87,7 @@ def _do_execute(self) -> None: ) else: if status in ["error", "killed"]: - raise errors.InternalError( - "job error: %s: %s" % (self._executed, status) - ) + raise errors.InternalError(f"job error: {self._executed}: {status}") else: time.sleep(self.wait_interval) if callable(self.wait_callback): @@ -134,8 +132,7 @@ def fetchmany(self, size: int | None = None) -> list[Any]: return rows else: raise errors.InternalError( - "index out of bound (%d out of %d)" - % (self._rownumber, self._rowcount) + f"index out of bound ({self._rownumber} out of {self._rowcount})" ) def fetchall(self) -> list[Any]: diff --git a/tdclient/database_model.py b/tdclient/database_model.py index 7652d63..91f4d8d 100644 --- a/tdclient/database_model.py +++ b/tdclient/database_model.py @@ -20,7 +20,7 @@ class Database(Model): PERMISSION_LIST_TABLES = ["administrator", "full_access"] def __init__(self, client: Client, db_name: str, **kwargs: Any) -> None: - super(Database, self).__init__(client) + super().__init__(client) self._db_name = db_name self._tables: list[Table] | None = kwargs.get("tables") self._count: int | None = kwargs.get("count") diff --git a/tdclient/job_api.py b/tdclient/job_api.py index 422f717..337d767 100644 --- a/tdclient/job_api.py +++ b/tdclient/job_api.py @@ -236,8 +236,7 @@ def job_result_each(self, job_id: str) -> Iterator[dict[str, Any]]: Yields: Row in a result """ - for row in self.job_result_format_each(job_id, "msgpack"): - yield row + yield from self.job_result_format_each(job_id, "msgpack") def job_result_format( self, job_id: str, format: str, header: bool = False @@ -449,7 +448,7 @@ def query( if priority_name in self.JOB_PRIORITY: priority_value = self.JOB_PRIORITY[priority_name] else: - raise ValueError("unknown job priority: %s" % (priority_name,)) + raise ValueError(f"unknown job priority: {priority_name}") else: priority_value = priority params["priority"] = priority_value diff --git a/tdclient/job_model.py b/tdclient/job_model.py index 8e2e35c..ff78d0c 100644 --- a/tdclient/job_model.py +++ b/tdclient/job_model.py @@ -69,7 +69,7 @@ class Job(Model): def __init__( self, client: Client, job_id: str, type: str, query: str | None, **kwargs: Any ) -> None: - super(Job, self).__init__(client) + super().__init__(client) self._job_id = job_id self._type = type self._query = query diff --git a/tdclient/result_model.py b/tdclient/result_model.py index 78f6e3c..4fe2c11 100644 --- a/tdclient/result_model.py +++ b/tdclient/result_model.py @@ -14,7 +14,7 @@ class Result(Model): """Result on Treasure Data Service""" def __init__(self, client: Client, name: str, url: str, org_name: str) -> None: - super(Result, self).__init__(client) + super().__init__(client) self._name = name self._url = url self._org_name = org_name diff --git a/tdclient/schedule_model.py b/tdclient/schedule_model.py index 161a661..a85f82b 100644 --- a/tdclient/schedule_model.py +++ b/tdclient/schedule_model.py @@ -24,7 +24,7 @@ def __init__( query: str | None, **kwargs: Any, ) -> None: - super(ScheduledJob, self).__init__(client, job_id, type, query, **kwargs) + super().__init__(client, job_id, type, query, **kwargs) self._scheduled_at = scheduled_at @property @@ -37,7 +37,7 @@ class Schedule(Model): """Schedule on Treasure Data Service""" def __init__(self, client: Client, *args: Any, **kwargs: Any) -> None: - super(Schedule, self).__init__(client) + super().__init__(client) if 0 < len(args): self._name: str | None = args[0] self._cron: str | None = args[1] diff --git a/tdclient/server_status_api.py b/tdclient/server_status_api.py index 2cae65a..6d975a2 100644 --- a/tdclient/server_status_api.py +++ b/tdclient/server_status_api.py @@ -31,7 +31,7 @@ def server_status(self) -> str: with self.get("/v3/system/server_status") as res: code, body = res.status, res.read() if code != 200: - return "Server is down (%d)" % (code,) + return f"Server is down ({code})" js = self.checked_json(body, ["status"]) status = js["status"] return status diff --git a/tdclient/table_api.py b/tdclient/table_api.py index 0f17b44..2dcd45e 100644 --- a/tdclient/table_api.py +++ b/tdclient/table_api.py @@ -114,7 +114,7 @@ def _create_table( ) as res: code, body = res.status, res.read() if code != 200: - self.raise_error("Create %s table failed" % (type), res, body) + self.raise_error(f"Create {type} table failed", res, body) return True def swap_table(self, db: str, table1: str, table2: str) -> bool: diff --git a/tdclient/table_model.py b/tdclient/table_model.py index e632937..4de608c 100644 --- a/tdclient/table_model.py +++ b/tdclient/table_model.py @@ -17,7 +17,7 @@ class Table(Model): """Database table on Treasure Data Service""" def __init__(self, *args: Any, **kwargs: Any) -> None: - super(Table, self).__init__(args[0]) + super().__init__(args[0]) self.database: Database | None = None self._db_name: str = args[1] @@ -141,7 +141,7 @@ def permission(self) -> str | None: @property def identifier(self) -> str: """a string identifier of the table""" - return "%s.%s" % (self._db_name, self._table_name) + return f"{self._db_name}.{self._table_name}" def delete(self) -> str: """a string represents the type of deleted table""" @@ -260,8 +260,8 @@ def estimated_storage_size_string(self) -> str: float(self._estimated_storage_size) / (1024 * 1024 * 1024) ) else: - return "%d GB" % int( - float(self._estimated_storage_size) / (1024 * 1024 * 1024) + return ( + f"{int(float(self._estimated_storage_size) / (1024 * 1024 * 1024))} GB" ) def _update_database(self) -> None: diff --git a/tdclient/user_model.py b/tdclient/user_model.py index 4e1045a..c1ac80e 100644 --- a/tdclient/user_model.py +++ b/tdclient/user_model.py @@ -22,7 +22,7 @@ def __init__( email: str, **kwargs: Any, ) -> None: - super(User, self).__init__(client) + super().__init__(client) self._name = name self._org_name = org_name self._role_names = role_names diff --git a/tdclient/util.py b/tdclient/util.py index 9df3b43..b85a2af 100644 --- a/tdclient/util.py +++ b/tdclient/util.py @@ -129,8 +129,9 @@ def merge_dtypes_and_converters( our_converters[column_name] = DTYPE_TO_CALLABLE[dtype] except KeyError: raise ValueError( - "Unrecognized dtype %r, must be one of %s" - % (dtype, ", ".join(repr(k) for k in sorted(DTYPE_TO_CALLABLE))) + "Unrecognized dtype {!r}, must be one of {}".format( + dtype, ", ".join(repr(k) for k in sorted(DTYPE_TO_CALLABLE)) + ) ) if converters is not None: for column_name, parse_fn in converters.items(): @@ -201,8 +202,7 @@ def csv_dict_record_reader( data) and whose values are the column values. """ reader = csv.DictReader(io.TextIOWrapper(file_like, encoding), dialect=dialect) - for row in reader: - yield row + yield from reader def csv_text_record_reader( From 372ff2700d08e8a9bf7efd66e9464f07c9aaea3a Mon Sep 17 00:00:00 2001 From: Aki Ariga Date: Thu, 30 Oct 2025 18:48:40 -0700 Subject: [PATCH 03/14] Apply B rule --- pyproject.toml | 2 +- tdclient/api.py | 8 +++++--- tdclient/job_model.py | 1 + tdclient/util.py | 3 ++- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2305a27..e151c67 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ select = [ "F", "I", "UP", - # "B", + "B", ] exclude = ["tdclient/test/*"] ignore = ["E203", "E501"] diff --git a/tdclient/api.py b/tdclient/api.py index 1eee188..a0d1011 100644 --- a/tdclient/api.py +++ b/tdclient/api.py @@ -314,7 +314,9 @@ def post( urllib3.exceptions.PoolError, ): if not self._retry_post_requests: - raise APIError("Retrying stopped by retry_post_requests == False") + raise APIError( + "Retrying stopped by retry_post_requests == False" + ) from None if cumul_retry_delay <= self._max_cumul_retry_delay: log.warning( @@ -402,7 +404,7 @@ def put( urllib3.exceptions.TimeoutError, urllib3.exceptions.PoolError, ): - raise APIError(f"Error: {repr(response)}") + raise APIError(f"Error: {repr(response)}") from None log.debug( "REST PUT response:\n headers: %s\n status: %d\n body: ", @@ -570,7 +572,7 @@ def checked_json(self, body: bytes, required: list[str]) -> dict[str, Any]: try: js = json.loads(body.decode("utf-8")) except ValueError as error: - raise APIError(f"Unexpected API response: {error}: {repr(body)}") + raise APIError(f"Unexpected API response: {error}: {repr(body)}") from error js = dict(js) if 0 < [k in js for k in required].count(False): missing = [k for k in required if k not in js] diff --git a/tdclient/job_model.py b/tdclient/job_model.py index ff78d0c..c55fb53 100644 --- a/tdclient/job_model.py +++ b/tdclient/job_model.py @@ -112,6 +112,7 @@ def update(self) -> None: def _update_status(self) -> None: warnings.warn( "_update_status() will be removed from future release. Please use update() instaed.", + stacklevel=2, category=DeprecationWarning, ) self.update() diff --git a/tdclient/util.py b/tdclient/util.py index b85a2af..8d5e646 100644 --- a/tdclient/util.py +++ b/tdclient/util.py @@ -43,6 +43,7 @@ def validate_record(record: Record) -> bool: if not any(k in record for k in ("time", b"time")): warnings.warn( 'records should have "time" column to import records properly.', + stacklevel=2, category=RuntimeWarning, ) return True @@ -132,7 +133,7 @@ def merge_dtypes_and_converters( "Unrecognized dtype {!r}, must be one of {}".format( dtype, ", ".join(repr(k) for k in sorted(DTYPE_TO_CALLABLE)) ) - ) + ) from None if converters is not None: for column_name, parse_fn in converters.items(): our_converters[column_name] = parse_fn From e2e988386809c8781d21b37cf2fcf2cb8a597665 Mon Sep 17 00:00:00 2001 From: Aki Ariga Date: Thu, 30 Oct 2025 18:52:01 -0700 Subject: [PATCH 04/14] Lint check on CI --- .github/workflows/pythontest.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pythontest.yml b/.github/workflows/pythontest.yml index d3a08a3..374c1ac 100644 --- a/.github/workflows/pythontest.yml +++ b/.github/workflows/pythontest.yml @@ -23,6 +23,10 @@ jobs: pip install ".[dev]" pip install -r requirements.txt -r test-requirements.txt pip install -U coveralls pyyaml + - name: lint with ruff + run: | + ruff format tdclient --diff --exit-non-zero-on-fix + ruff check tdclient tests - name: Run pyright run: | pyright tdclient From f2177b6ace78f82f8a91368a851b4a33ba464832 Mon Sep 17 00:00:00 2001 From: Aki Ariga Date: Thu, 30 Oct 2025 18:53:10 -0700 Subject: [PATCH 05/14] Separate lint on CI --- .github/workflows/pythontest.yml | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pythontest.yml b/.github/workflows/pythontest.yml index 374c1ac..02d917a 100644 --- a/.github/workflows/pythontest.yml +++ b/.github/workflows/pythontest.yml @@ -3,6 +3,26 @@ name: Python testing on: [push, pull_request] jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install ".[dev]" + - name: lint with ruff + run: | + ruff format tdclient --diff --exit-non-zero-on-fix + ruff check tdclient + - name: Run pyright + run: | + pyright tdclient + test: runs-on: ${{ matrix.os }} strategy: @@ -23,13 +43,6 @@ jobs: pip install ".[dev]" pip install -r requirements.txt -r test-requirements.txt pip install -U coveralls pyyaml - - name: lint with ruff - run: | - ruff format tdclient --diff --exit-non-zero-on-fix - ruff check tdclient tests - - name: Run pyright - run: | - pyright tdclient - name: Run test run: | coverage run --source=tdclient -m pytest tdclient/test From 7b45202ff8d82a9477fa4c038dc0f2c8b7bbe118 Mon Sep 17 00:00:00 2001 From: Aki Ariga Date: Thu, 30 Oct 2025 18:57:16 -0700 Subject: [PATCH 06/14] Support Python 3.13 and 3.14, Drop 3.8 and 3.9 --- .github/workflows/pythontest.yml | 4 ++-- pyproject.toml | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pythontest.yml b/.github/workflows/pythontest.yml index 02d917a..fae4be8 100644 --- a/.github/workflows/pythontest.yml +++ b/.github/workflows/pythontest.yml @@ -10,7 +10,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 with: - python-version: "3.12" + python-version: "3.13" - name: Install dependencies run: | python -m pip install --upgrade pip @@ -29,7 +29,7 @@ jobs: max-parallel: 4 matrix: os: [ubuntu-latest, windows-latest] - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v4 diff --git a/pyproject.toml b/pyproject.toml index e151c67..8d70228 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "td-client" version = "1.5.0" description = "Treasure Data API library for Python" readme = {file = "README.rst", content-type = "text/x-rst; charset=UTF-8"} -requires-python = ">=3.8" +requires-python = ">=3.10" license = {text = "Apache Software License"} authors = [{name = "Treasure Data, Inc.", email = "support@treasure-data.com"}] urls = {homepage = "http://treasuredata.com/"} @@ -18,11 +18,11 @@ classifiers = [ "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", "Topic :: Internet", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", ] @@ -66,7 +66,7 @@ known-third-party = ["dateutil","msgpack","pkg_resources","pytest","setuptools", include = ["tdclient"] exclude = ["**/__pycache__", "tdclient/test", "docs"] typeCheckingMode = "basic" -pythonVersion = "3.9" +pythonVersion = "3.10" pythonPlatform = "All" reportMissingTypeStubs = false reportUnknownMemberType = false From 3a11770c2e3feb58fd40ac0477665a27e424ee87 Mon Sep 17 00:00:00 2001 From: Aki Ariga Date: Thu, 30 Oct 2025 18:59:03 -0700 Subject: [PATCH 07/14] ruff check --fix --unsafe-fixes --- tdclient/connection.py | 3 ++- tdclient/cursor.py | 3 ++- tdclient/types.py | 4 ++-- tdclient/util.py | 4 ++-- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/tdclient/connection.py b/tdclient/connection.py index 7639668..430bbc6 100644 --- a/tdclient/connection.py +++ b/tdclient/connection.py @@ -2,7 +2,8 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Callable +from collections.abc import Callable +from typing import TYPE_CHECKING, Any from tdclient import api, cursor, errors from tdclient.types import Priority diff --git a/tdclient/cursor.py b/tdclient/cursor.py index 260d750..acabbb4 100644 --- a/tdclient/cursor.py +++ b/tdclient/cursor.py @@ -3,7 +3,8 @@ from __future__ import annotations import time -from typing import TYPE_CHECKING, Any, Callable +from collections.abc import Callable +from typing import TYPE_CHECKING, Any from tdclient import errors diff --git a/tdclient/types.py b/tdclient/types.py index 1ee0d49..62e0994 100644 --- a/tdclient/types.py +++ b/tdclient/types.py @@ -3,9 +3,9 @@ from __future__ import annotations from array import array -from typing import IO, TYPE_CHECKING +from typing import IO, TYPE_CHECKING, Literal, TypeAlias -from typing_extensions import Literal, TypeAlias, TypedDict +from typing_extensions import TypedDict if TYPE_CHECKING: from collections.abc import Callable diff --git a/tdclient/util.py b/tdclient/util.py index 8d5e646..37d5e83 100644 --- a/tdclient/util.py +++ b/tdclient/util.py @@ -233,7 +233,7 @@ def csv_text_record_reader( """ reader = csv.reader(io.TextIOWrapper(file_like, encoding), dialect=dialect) for row in reader: - yield dict(zip(columns, row)) + yield dict(zip(columns, row, strict=False)) def read_csv_records( @@ -300,7 +300,7 @@ def normalized_msgpack(value: Any) -> Any: Returns: Normalized value """ - if isinstance(value, (list, tuple)): + if isinstance(value, list | tuple): return [normalized_msgpack(v) for v in value] elif isinstance(value, dict): return dict( From 6787fe549be2e23ab0ed6c2a4adb6a1e380b599b Mon Sep 17 00:00:00 2001 From: Aki Ariga Date: Thu, 30 Oct 2025 19:05:24 -0700 Subject: [PATCH 08/14] Adopt new version Python typing syntax --- pyproject.toml | 2 +- tdclient/__init__.py | 2 -- tdclient/api.py | 6 ++---- tdclient/bulk_import_api.py | 12 +++-------- tdclient/bulk_import_model.py | 6 ++---- tdclient/client.py | 4 +--- tdclient/connection.py | 11 ++++------ tdclient/connector_api.py | 10 +++------ tdclient/cursor.py | 8 +++---- tdclient/database_api.py | 10 +++------ tdclient/database_model.py | 12 +++++------ tdclient/export_api.py | 10 +++------ tdclient/import_api.py | 11 +++------- tdclient/job_api.py | 11 +++------- tdclient/job_model.py | 10 ++++----- tdclient/result_api.py | 10 +++------ tdclient/result_model.py | 4 +--- tdclient/schedule_api.py | 12 ++++------- tdclient/schedule_model.py | 6 ++---- tdclient/server_status_api.py | 10 +++------ tdclient/table_api.py | 11 +++------- tdclient/table_model.py | 4 +--- tdclient/types.py | 39 +++++++++++++++-------------------- tdclient/user_api.py | 10 +++------ tdclient/user_model.py | 4 +--- tdclient/util.py | 2 -- 26 files changed, 78 insertions(+), 159 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8d70228..b439d6e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,6 @@ dependencies = [ "python-dateutil", "msgpack>=0.6.2", "urllib3", - "typing-extensions>=4.0.0", ] [project.optional-dependencies] @@ -46,6 +45,7 @@ tdclient = ["py.typed"] [tool.ruff] line-length = 88 +target-version = "py310" [tool.ruff.lint] select = [ diff --git a/tdclient/__init__.py b/tdclient/__init__.py index f569a41..1765ef1 100644 --- a/tdclient/__init__.py +++ b/tdclient/__init__.py @@ -1,5 +1,3 @@ -from __future__ import annotations - import datetime import time from typing import Any diff --git a/tdclient/api.py b/tdclient/api.py index a0d1011..952eba8 100644 --- a/tdclient/api.py +++ b/tdclient/api.py @@ -1,7 +1,5 @@ #!/usr/bin/env python -from __future__ import annotations - import contextlib import csv import email.utils @@ -33,7 +31,7 @@ from tdclient.schedule_api import ScheduleAPI from tdclient.server_status_api import ServerStatusAPI from tdclient.table_api import TableAPI -from tdclient.types import BytesOrStream +from tdclient.types import BytesOrStream, StreamBody from tdclient.user_api import UserAPI from tdclient.util import ( csv_dict_record_reader, @@ -534,7 +532,7 @@ def send_request( method: str, url: str, fields: dict[str, Any] | None = None, - body: bytes | bytearray | memoryview | array[int] | IO[bytes] | None = None, + body: StreamBody = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> urllib3.BaseHTTPResponse: diff --git a/tdclient/bulk_import_api.py b/tdclient/bulk_import_api.py index 12c3c27..1b7413d 100644 --- a/tdclient/bulk_import_api.py +++ b/tdclient/bulk_import_api.py @@ -1,22 +1,16 @@ #!/usr/bin/env python -from __future__ import annotations - import collections import contextlib import gzip import io import os from collections.abc import Iterator -from typing import TYPE_CHECKING, Any +from contextlib import AbstractContextManager +from typing import IO, Any import msgpack - -if TYPE_CHECKING: - from contextlib import AbstractContextManager - from typing import IO - - import urllib3 +import urllib3 from tdclient.types import BulkImportParams, BytesOrStream, DataFormat, FileLike from tdclient.util import create_url diff --git a/tdclient/bulk_import_model.py b/tdclient/bulk_import_model.py index b6092c1..8ea061d 100644 --- a/tdclient/bulk_import_model.py +++ b/tdclient/bulk_import_model.py @@ -1,7 +1,5 @@ #!/usr/bin/env python -from __future__ import annotations - import time from collections.abc import Callable, Iterator from typing import TYPE_CHECKING, Any @@ -23,7 +21,7 @@ class BulkImport(Model): STATUS_COMMITTING = "committing" STATUS_COMMITTED = "committed" - def __init__(self, client: Client, **kwargs: Any) -> None: + def __init__(self, client: "Client", **kwargs: Any) -> None: super().__init__(client) self._feed(kwargs) @@ -116,7 +114,7 @@ def perform( wait_interval: int = 5, wait_callback: Callable[[], None] | None = None, timeout: float | None = None, - ) -> Job: + ) -> "Job": """Perform bulk import Args: diff --git a/tdclient/client.py b/tdclient/client.py index e34fe0e..3081102 100644 --- a/tdclient/client.py +++ b/tdclient/client.py @@ -1,7 +1,5 @@ #!/usr/bin/env python -from __future__ import annotations - import datetime import json from collections.abc import Iterator @@ -27,7 +25,7 @@ class Client: def __init__(self, *args: Any, **kwargs: Any) -> None: self._api = api.API(*args, **kwargs) - def __enter__(self) -> Client: + def __enter__(self) -> "Client": return self def __exit__( diff --git a/tdclient/connection.py b/tdclient/connection.py index 430bbc6..3ff539a 100644 --- a/tdclient/connection.py +++ b/tdclient/connection.py @@ -1,16 +1,13 @@ #!/usr/bin/env python -from __future__ import annotations - from collections.abc import Callable +from types import TracebackType from typing import TYPE_CHECKING, Any from tdclient import api, cursor, errors from tdclient.types import Priority if TYPE_CHECKING: - from types import TracebackType - from tdclient.cursor import Cursor @@ -23,7 +20,7 @@ def __init__( priority: Priority | None = None, retry_limit: int | None = None, wait_interval: int | None = None, - wait_callback: Callable[[Cursor], None] | None = None, + wait_callback: Callable[["Cursor"], None] | None = None, **kwargs: Any, ) -> None: cursor_kwargs = dict() @@ -44,7 +41,7 @@ def __init__( self._api = api.API(**kwargs) self._cursor_kwargs = cursor_kwargs - def __enter__(self) -> Connection: + def __enter__(self) -> "Connection": return self def __exit__( @@ -68,5 +65,5 @@ def commit(self) -> None: def rollback(self) -> None: raise errors.NotSupportedError - def cursor(self) -> Cursor: + def cursor(self) -> "Cursor": return cursor.Cursor(self._api, **self._cursor_kwargs) diff --git a/tdclient/connector_api.py b/tdclient/connector_api.py index 71b93af..6b1dd25 100644 --- a/tdclient/connector_api.py +++ b/tdclient/connector_api.py @@ -1,14 +1,10 @@ #!/usr/bin/env python -from __future__ import annotations - import json -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from contextlib import AbstractContextManager +from contextlib import AbstractContextManager +from typing import Any - import urllib3 +import urllib3 from tdclient.util import create_url, normalize_connector_config diff --git a/tdclient/cursor.py b/tdclient/cursor.py index acabbb4..d334f6b 100644 --- a/tdclient/cursor.py +++ b/tdclient/cursor.py @@ -1,7 +1,5 @@ #!/usr/bin/env python -from __future__ import annotations - import time from collections.abc import Callable from typing import TYPE_CHECKING, Any @@ -15,9 +13,9 @@ class Cursor: def __init__( self, - api: API, + api: "API", wait_interval: int = 5, - wait_callback: Callable[[Cursor], None] | None = None, + wait_callback: Callable[["Cursor"], None] | None = None, **kwargs: Any, ) -> None: self._api = api @@ -31,7 +29,7 @@ def __init__( self.wait_callback = wait_callback @property - def api(self) -> API: + def api(self) -> "API": return self._api @property diff --git a/tdclient/database_api.py b/tdclient/database_api.py index 3ebd9be..91f98b2 100644 --- a/tdclient/database_api.py +++ b/tdclient/database_api.py @@ -1,13 +1,9 @@ #!/usr/bin/env python -from __future__ import annotations +from contextlib import AbstractContextManager +from typing import Any -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from contextlib import AbstractContextManager - - import urllib3 +import urllib3 from tdclient.util import create_url, get_or_else, parse_date diff --git a/tdclient/database_model.py b/tdclient/database_model.py index 91f4d8d..64b3ac1 100644 --- a/tdclient/database_model.py +++ b/tdclient/database_model.py @@ -1,7 +1,5 @@ #!/usr/bin/env python -from __future__ import annotations - import datetime from typing import TYPE_CHECKING, Any @@ -19,7 +17,7 @@ class Database(Model): PERMISSIONS = ["administrator", "full_access", "import_only", "query_only"] PERMISSION_LIST_TABLES = ["administrator", "full_access"] - def __init__(self, client: Client, db_name: str, **kwargs: Any) -> None: + def __init__(self, client: "Client", db_name: str, **kwargs: Any) -> None: super().__init__(client) self._db_name = db_name self._tables: list[Table] | None = kwargs.get("tables") @@ -57,7 +55,7 @@ def name(self) -> str: """ return self._db_name - def tables(self) -> list[Table]: + def tables(self) -> list["Table"]: """ Returns: a list of :class:`tdclient.model.Table` @@ -67,7 +65,7 @@ def tables(self) -> list[Table]: assert self._tables is not None return self._tables - def create_log_table(self, name: str) -> Table: + def create_log_table(self, name: str) -> "Table": """ Args: name (str): name of new log table @@ -77,7 +75,7 @@ def create_log_table(self, name: str) -> Table: """ return self._client.create_log_table(self._db_name, name) - def table(self, table_name: str) -> Table: + def table(self, table_name: str) -> "Table": """ Args: table_name (str): name of a table @@ -95,7 +93,7 @@ def delete(self) -> bool: """ return self._client.delete_database(self._db_name) - def query(self, q: str, **kwargs: Any) -> Job: + def query(self, q: str, **kwargs: Any) -> "Job": """Run a query on the database Args: diff --git a/tdclient/export_api.py b/tdclient/export_api.py index a8f18e6..000d75e 100644 --- a/tdclient/export_api.py +++ b/tdclient/export_api.py @@ -1,13 +1,9 @@ #!/usr/bin/env python -from __future__ import annotations +from contextlib import AbstractContextManager +from typing import Any -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from contextlib import AbstractContextManager - - import urllib3 +import urllib3 from tdclient.types import ExportParams from tdclient.util import create_url diff --git a/tdclient/import_api.py b/tdclient/import_api.py index 146b959..cad23ef 100644 --- a/tdclient/import_api.py +++ b/tdclient/import_api.py @@ -1,16 +1,11 @@ #!/usr/bin/env python -from __future__ import annotations - import contextlib import os -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from contextlib import AbstractContextManager - from typing import IO +from contextlib import AbstractContextManager +from typing import IO, Any - import urllib3 +import urllib3 from tdclient.types import BytesOrStream, DataFormat, FileLike from tdclient.util import create_url diff --git a/tdclient/job_api.py b/tdclient/job_api.py index 337d767..c91c024 100644 --- a/tdclient/job_api.py +++ b/tdclient/job_api.py @@ -1,7 +1,5 @@ #!/usr/bin/env python -from __future__ import annotations - import codecs import gzip import json @@ -10,14 +8,11 @@ import tempfile from collections.abc import Iterator from concurrent.futures import ThreadPoolExecutor -from typing import TYPE_CHECKING, Any, Literal +from contextlib import AbstractContextManager +from typing import Any, Literal import msgpack - -if TYPE_CHECKING: - from contextlib import AbstractContextManager - - import urllib3 +import urllib3 from tdclient.types import Priority from tdclient.util import create_url, get_or_else, parse_date diff --git a/tdclient/job_model.py b/tdclient/job_model.py index c55fb53..fcf27c7 100644 --- a/tdclient/job_model.py +++ b/tdclient/job_model.py @@ -1,7 +1,5 @@ #!/usr/bin/env python -from __future__ import annotations - import time import warnings from collections.abc import Callable, Iterator @@ -35,12 +33,12 @@ def type(self) -> str: """ return self._type - def __init__(self, fields: list[Schema.Field] | None = None) -> None: + def __init__(self, fields: list["Schema.Field"] | None = None) -> None: fields = [] if fields is None else fields self._fields = fields @property - def fields(self) -> list[Schema.Field]: + def fields(self) -> list["Schema.Field"]: """ TODO: add docstring """ @@ -67,7 +65,7 @@ class Job(Model): JOB_PRIORITY = {-2: "VERY LOW", -1: "LOW", 0: "NORMAL", 1: "HIGH", 2: "VERY HIGH"} def __init__( - self, client: Client, job_id: str, type: str, query: str | None, **kwargs: Any + self, client: "Client", job_id: str, type: str, query: str | None, **kwargs: Any ) -> None: super().__init__(client) self._job_id = job_id @@ -205,7 +203,7 @@ def wait( self, timeout: float | None = None, wait_interval: int = 5, - wait_callback: Callable[[Job], None] | None = None, + wait_callback: Callable[["Job"], None] | None = None, ) -> None: """Sleep until the job has been finished diff --git a/tdclient/result_api.py b/tdclient/result_api.py index 4c06e7e..e10558b 100644 --- a/tdclient/result_api.py +++ b/tdclient/result_api.py @@ -1,13 +1,9 @@ #!/usr/bin/env python -from __future__ import annotations +from contextlib import AbstractContextManager +from typing import Any -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from contextlib import AbstractContextManager - - import urllib3 +import urllib3 from tdclient.types import ResultParams from tdclient.util import create_url diff --git a/tdclient/result_model.py b/tdclient/result_model.py index 4fe2c11..34f2c50 100644 --- a/tdclient/result_model.py +++ b/tdclient/result_model.py @@ -1,7 +1,5 @@ #!/usr/bin/env python -from __future__ import annotations - from typing import TYPE_CHECKING from tdclient.model import Model @@ -13,7 +11,7 @@ class Result(Model): """Result on Treasure Data Service""" - def __init__(self, client: Client, name: str, url: str, org_name: str) -> None: + def __init__(self, client: "Client", name: str, url: str, org_name: str) -> None: super().__init__(client) self._name = name self._url = url diff --git a/tdclient/schedule_api.py b/tdclient/schedule_api.py index 0764e1a..5ab4615 100644 --- a/tdclient/schedule_api.py +++ b/tdclient/schedule_api.py @@ -1,18 +1,14 @@ #!/usr/bin/env python -from __future__ import annotations - import datetime -from typing import TYPE_CHECKING, Any +from contextlib import AbstractContextManager +from typing import Any + +import urllib3 from tdclient.types import ScheduleParams from tdclient.util import create_url, get_or_else, parse_date -if TYPE_CHECKING: - from contextlib import AbstractContextManager - - import urllib3 - class ScheduleAPI: """Access to Schedule API diff --git a/tdclient/schedule_model.py b/tdclient/schedule_model.py index a85f82b..4b54ed3 100644 --- a/tdclient/schedule_model.py +++ b/tdclient/schedule_model.py @@ -1,7 +1,5 @@ #!/usr/bin/env python -from __future__ import annotations - import datetime from typing import TYPE_CHECKING, Any @@ -17,7 +15,7 @@ class ScheduledJob(Job): def __init__( self, - client: Client, + client: "Client", scheduled_at: datetime.datetime, job_id: str, type: str, @@ -36,7 +34,7 @@ def scheduled_at(self) -> datetime.datetime: class Schedule(Model): """Schedule on Treasure Data Service""" - def __init__(self, client: Client, *args: Any, **kwargs: Any) -> None: + def __init__(self, client: "Client", *args: Any, **kwargs: Any) -> None: super().__init__(client) if 0 < len(args): self._name: str | None = args[0] diff --git a/tdclient/server_status_api.py b/tdclient/server_status_api.py index 6d975a2..7ce326f 100644 --- a/tdclient/server_status_api.py +++ b/tdclient/server_status_api.py @@ -1,13 +1,9 @@ #!/usr/bin/env python -from __future__ import annotations +from contextlib import AbstractContextManager +from typing import Any -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from contextlib import AbstractContextManager - - import urllib3 +import urllib3 class ServerStatusAPI: diff --git a/tdclient/table_api.py b/tdclient/table_api.py index 2dcd45e..1670f0e 100644 --- a/tdclient/table_api.py +++ b/tdclient/table_api.py @@ -1,16 +1,11 @@ #!/usr/bin/env python -from __future__ import annotations - import json -from typing import TYPE_CHECKING, Any +from contextlib import AbstractContextManager +from typing import Any import msgpack - -if TYPE_CHECKING: - from contextlib import AbstractContextManager - - import urllib3 +import urllib3 from tdclient.util import create_url, get_or_else, parse_date diff --git a/tdclient/table_model.py b/tdclient/table_model.py index 4de608c..2a5248f 100644 --- a/tdclient/table_model.py +++ b/tdclient/table_model.py @@ -1,7 +1,5 @@ #!/usr/bin/env python -from __future__ import annotations - import datetime from typing import TYPE_CHECKING, Any @@ -208,7 +206,7 @@ def import_file( self._db_name, self._table_name, format, file, unique_id=unique_id ) - def export_data(self, storage_type: str, **kwargs: Any) -> Job: + def export_data(self, storage_type: str, **kwargs: Any) -> "Job": """Export data from Treasure Data Service Args: diff --git a/tdclient/types.py b/tdclient/types.py index 62e0994..2d035ab 100644 --- a/tdclient/types.py +++ b/tdclient/types.py @@ -1,56 +1,51 @@ """Type definitions for td-client-python.""" -from __future__ import annotations - from array import array -from typing import IO, TYPE_CHECKING, Literal, TypeAlias - -from typing_extensions import TypedDict - -if TYPE_CHECKING: - from collections.abc import Callable - from typing import Any +from collections.abc import Callable +from typing import IO, Any, Literal, TypeAlias, TypedDict # File-like types -FileLike: TypeAlias = "str | bytes | IO[bytes]" +FileLike: TypeAlias = str | bytes | IO[bytes] """Type for file inputs: file path, bytes, or file-like object.""" -BytesOrStream: TypeAlias = "bytes | bytearray | IO[bytes]" +BytesOrStream: TypeAlias = bytes | bytearray | IO[bytes] """Type for byte data or streams (excluding file paths).""" StreamBody: TypeAlias = "bytes | bytearray | memoryview | array[int] | IO[bytes] | None" """Type for HTTP request body.""" # Query engine types -QueryEngineType: TypeAlias = 'Literal["presto", "hive"]' +QueryEngineType: TypeAlias = Literal["presto", "hive"] """Type for query engine selection.""" -EngineVersion: TypeAlias = 'Literal["stable", "experimental"]' +EngineVersion: TypeAlias = Literal["stable", "experimental"] """Type for engine version selection.""" -Priority: TypeAlias = ( - 'Literal[-2, -1, 0, 1, 2, "VERY LOW", "LOW", "NORMAL", "HIGH", "VERY HIGH"]' -) +Priority: TypeAlias = Literal[ + -2, -1, 0, 1, 2, "VERY LOW", "LOW", "NORMAL", "HIGH", "VERY HIGH" +] """Type for job priority levels (numeric or string).""" # Data format types -ExportFileFormat: TypeAlias = 'Literal["jsonl.gz", "tsv.gz", "json.gz"]' +ExportFileFormat: TypeAlias = Literal["jsonl.gz", "tsv.gz", "json.gz"] """Type for export file formats.""" -DataFormat: TypeAlias = 'Literal["msgpack", "msgpack.gz", "json", "json.gz", "csv", "csv.gz", "tsv", "tsv.gz"]' +DataFormat: TypeAlias = Literal[ + "msgpack", "msgpack.gz", "json", "json.gz", "csv", "csv.gz", "tsv", "tsv.gz" +] """Type for data import/export formats.""" -ResultFormat: TypeAlias = 'Literal["msgpack", "json", "csv", "tsv"]' +ResultFormat: TypeAlias = Literal["msgpack", "json", "csv", "tsv"] """Type for query result formats.""" # Utility types for CSV parsing and data processing -CSVValue: TypeAlias = "int | float | str | bool | None" +CSVValue: TypeAlias = int | float | str | bool | None """Type for values parsed from CSV files.""" -Converter: TypeAlias = "Callable[[str], Any]" +Converter: TypeAlias = Callable[[str], Any] """Type for converter functions that parse string values.""" -Record: TypeAlias = "dict[str, Any]" +Record: TypeAlias = dict[str, Any] """Type for data records (dictionaries with string keys and any values).""" diff --git a/tdclient/user_api.py b/tdclient/user_api.py index 2a6a892..c40b238 100644 --- a/tdclient/user_api.py +++ b/tdclient/user_api.py @@ -1,13 +1,9 @@ #!/usr/bin/env python -from __future__ import annotations +from contextlib import AbstractContextManager +from typing import Any -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from contextlib import AbstractContextManager - - import urllib3 +import urllib3 from tdclient.util import create_url diff --git a/tdclient/user_model.py b/tdclient/user_model.py index c1ac80e..b7011f8 100644 --- a/tdclient/user_model.py +++ b/tdclient/user_model.py @@ -1,7 +1,5 @@ #!/usr/bin/env python -from __future__ import annotations - from typing import TYPE_CHECKING, Any from tdclient.model import Model @@ -15,7 +13,7 @@ class User(Model): def __init__( self, - client: Client, + client: "Client", name: str, org_name: str, role_names: list[str], diff --git a/tdclient/util.py b/tdclient/util.py index 37d5e83..2504312 100644 --- a/tdclient/util.py +++ b/tdclient/util.py @@ -1,5 +1,3 @@ -from __future__ import annotations - import csv import io import logging From 84fd4f21ac4d8eee7a0ae68aecdef3e4d4f91c47 Mon Sep 17 00:00:00 2001 From: Aki Ariga Date: Thu, 30 Oct 2025 19:30:27 -0700 Subject: [PATCH 09/14] Bump pytest --- test-requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 009e61f..6703b69 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,4 +1,3 @@ coveralls>=1.1,<1.2 -mock>=1.3,<1.4 -pytest>=4.0,<=7.2 +pytest>=8.3 tox>=3.0,<4.0 From db560640724f89da1fd8e63ebc611abc8418dca2 Mon Sep 17 00:00:00 2001 From: Aki Ariga Date: Thu, 30 Oct 2025 20:11:01 -0700 Subject: [PATCH 10/14] Remove redundant kw argument Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- tdclient/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdclient/util.py b/tdclient/util.py index 2504312..ea3cdde 100644 --- a/tdclient/util.py +++ b/tdclient/util.py @@ -231,7 +231,7 @@ def csv_text_record_reader( """ reader = csv.reader(io.TextIOWrapper(file_like, encoding), dialect=dialect) for row in reader: - yield dict(zip(columns, row, strict=False)) + yield dict(zip(columns, row)) def read_csv_records( From ecba831c119ce3d0dccf8134b753a3a486303c84 Mon Sep 17 00:00:00 2001 From: Aki Ariga Date: Thu, 30 Oct 2025 20:25:24 -0700 Subject: [PATCH 11/14] Revert "Remove redundant kw argument" This reverts commit db560640724f89da1fd8e63ebc611abc8418dca2. --- tdclient/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdclient/util.py b/tdclient/util.py index ea3cdde..2504312 100644 --- a/tdclient/util.py +++ b/tdclient/util.py @@ -231,7 +231,7 @@ def csv_text_record_reader( """ reader = csv.reader(io.TextIOWrapper(file_like, encoding), dialect=dialect) for row in reader: - yield dict(zip(columns, row)) + yield dict(zip(columns, row, strict=False)) def read_csv_records( From 05f56ad50c43f8442387aa8295b8c5bb83404c03 Mon Sep 17 00:00:00 2001 From: Aki Ariga Date: Thu, 30 Oct 2025 22:15:52 -0700 Subject: [PATCH 12/14] Fix typo Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- tdclient/job_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdclient/job_model.py b/tdclient/job_model.py index c55fb53..5a66f2d 100644 --- a/tdclient/job_model.py +++ b/tdclient/job_model.py @@ -111,7 +111,7 @@ def update(self) -> None: def _update_status(self) -> None: warnings.warn( - "_update_status() will be removed from future release. Please use update() instaed.", + "_update_status() will be removed from future release. Please use update() instead.", stacklevel=2, category=DeprecationWarning, ) From c4ab7367b7aa91573ad2fb87ac6c3575094ccd76 Mon Sep 17 00:00:00 2001 From: Aki Ariga Date: Fri, 31 Oct 2025 00:34:58 -0700 Subject: [PATCH 13/14] Update Python version support in README --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 50a4389..2259f14 100644 --- a/README.rst +++ b/README.rst @@ -26,7 +26,7 @@ Requirements ``td-client`` supports the following versions of Python. -* Python 3.5+ +* Python 3.10+ * PyPy Install From 947d13623f6806b087a90ae6efe40fb18f84c65a Mon Sep 17 00:00:00 2001 From: Aki Ariga Date: Thu, 30 Oct 2025 21:57:08 -0700 Subject: [PATCH 14/14] Fix override signatures --- tdclient/api.py | 2 +- tdclient/bulk_import_api.py | 4 ++-- tdclient/connector_api.py | 6 +++++- tdclient/database_api.py | 4 ++-- tdclient/export_api.py | 2 +- tdclient/job_api.py | 4 ++-- tdclient/result_api.py | 4 ++-- tdclient/schedule_api.py | 4 ++-- tdclient/table_api.py | 4 ++-- tdclient/user_api.py | 4 ++-- 10 files changed, 21 insertions(+), 17 deletions(-) diff --git a/tdclient/api.py b/tdclient/api.py index a0d1011..5e96006 100644 --- a/tdclient/api.py +++ b/tdclient/api.py @@ -248,7 +248,7 @@ def get( def post( self, path: str, - params: dict[str, Any] | None = None, + params: dict[str, Any] | bytes | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> contextlib.AbstractContextManager[urllib3.BaseHTTPResponse]: diff --git a/tdclient/bulk_import_api.py b/tdclient/bulk_import_api.py index 3027528..474b6be 100644 --- a/tdclient/bulk_import_api.py +++ b/tdclient/bulk_import_api.py @@ -32,14 +32,14 @@ class BulkImportAPI: def get( self, path: str, - params: dict[str, Any] | None = None, + params: dict[str, Any] | bytes | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... def post( self, path: str, - params: dict[str, Any] | None = None, + params: dict[str, Any] | bytes | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... diff --git a/tdclient/connector_api.py b/tdclient/connector_api.py index 669a4ea..91bef02 100644 --- a/tdclient/connector_api.py +++ b/tdclient/connector_api.py @@ -29,7 +29,11 @@ def get( **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... def post( - self, url: str, params: Any, headers: dict[str, str] | None = None + self, + path: str, + params: dict[str, Any] | bytes | None = None, + headers: dict[str, str] | None = None, + **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... def put( self, diff --git a/tdclient/database_api.py b/tdclient/database_api.py index e8208ac..2ac2319 100644 --- a/tdclient/database_api.py +++ b/tdclient/database_api.py @@ -22,14 +22,14 @@ class DatabaseAPI: def get( self, path: str, - params: dict[str, Any] | None = None, + params: dict[str, Any] | bytes | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... def post( self, path: str, - params: dict[str, Any] | None = None, + params: dict[str, Any] | bytes | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... diff --git a/tdclient/export_api.py b/tdclient/export_api.py index 9069cae..9c7cb23 100644 --- a/tdclient/export_api.py +++ b/tdclient/export_api.py @@ -23,7 +23,7 @@ class ExportAPI: def post( self, path: str, - params: dict[str, Any] | None = None, + params: dict[str, Any] | bytes | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... diff --git a/tdclient/job_api.py b/tdclient/job_api.py index 07cb4cd..1b4bf26 100644 --- a/tdclient/job_api.py +++ b/tdclient/job_api.py @@ -35,13 +35,13 @@ class JobAPI: def get( self, url: str, - params: dict[str, Any] | None = None, + params: dict[str, Any] | bytes | None = None, headers: dict[str, str] | None = None, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... def post( self, path: str, - params: dict[str, Any] | None = None, + params: dict[str, Any] | bytes | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... diff --git a/tdclient/result_api.py b/tdclient/result_api.py index 6204eb3..86e72eb 100644 --- a/tdclient/result_api.py +++ b/tdclient/result_api.py @@ -23,14 +23,14 @@ class ResultAPI: def get( self, path: str, - params: dict[str, Any] | None = None, + params: dict[str, Any] | bytes | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... def post( self, path: str, - params: dict[str, Any] | None = None, + params: dict[str, Any] | bytes | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... diff --git a/tdclient/schedule_api.py b/tdclient/schedule_api.py index 1410d77..127e841 100644 --- a/tdclient/schedule_api.py +++ b/tdclient/schedule_api.py @@ -24,14 +24,14 @@ class ScheduleAPI: def get( self, path: str, - params: dict[str, Any] | None = None, + params: dict[str, Any] | bytes | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... def post( self, path: str, - params: dict[str, Any] | None = None, + params: dict[str, Any] | bytes | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... diff --git a/tdclient/table_api.py b/tdclient/table_api.py index 7e91304..d84406f 100644 --- a/tdclient/table_api.py +++ b/tdclient/table_api.py @@ -25,14 +25,14 @@ class TableAPI: def get( self, path: str, - params: dict[str, Any] | None = None, + params: dict[str, Any] | bytes | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... def post( self, path: str, - params: dict[str, Any] | None = None, + params: dict[str, Any] | bytes | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... diff --git a/tdclient/user_api.py b/tdclient/user_api.py index 6212155..a1dcb25 100644 --- a/tdclient/user_api.py +++ b/tdclient/user_api.py @@ -17,14 +17,14 @@ class UserAPI: def get( self, path: str, - params: dict[str, Any] | None = None, + params: dict[str, Any] | bytes | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... def post( self, path: str, - params: dict[str, Any] | None = None, + params: dict[str, Any] | bytes | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ...