From 632aeae46978eafb5136811cbeb318d9bdc976d9 Mon Sep 17 00:00:00 2001 From: "guangli.bao" Date: Thu, 30 Oct 2025 16:56:42 +0800 Subject: [PATCH 1/4] fix test_output xfail ut case Signed-off-by: guangli.bao --- src/guidellm/benchmark/output.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/guidellm/benchmark/output.py b/src/guidellm/benchmark/output.py index 6e17de5b..4523ffae 100644 --- a/src/guidellm/benchmark/output.py +++ b/src/guidellm/benchmark/output.py @@ -690,7 +690,7 @@ def _get_benchmark_extras_headers_and_values( values: list[str] = [ benchmark.benchmarker.profile.model_dump_json(), json.dumps(benchmark.benchmarker.backend), - json.dumps(benchmark.benchmarker.requests["data"]), + json.dumps(benchmark.benchmarker.requests["attributes"]["data"]), ] if len(headers) != len(values): From 3fed53a7d81fed5c1a9d080a0750e8ddc2da969d Mon Sep 17 00:00:00 2001 From: "guangli.bao" Date: Thu, 30 Oct 2025 17:04:51 +0800 Subject: [PATCH 2/4] fix some code quality check error Signed-off-by: guangli.bao --- src/guidellm/benchmark/benchmarker.py | 2 +- .../data/deserializers/deserializer.py | 3 ++- src/guidellm/data/loaders.py | 1 - src/guidellm/data/preprocessors/formatters.py | 8 ++------ src/guidellm/data/preprocessors/mappers.py | 4 +--- .../data/preprocessors/preprocessor.py | 3 +-- src/guidellm/preprocess/dataset.py | 2 +- src/guidellm/scheduler/strategies.py | 18 ++++++++++++------ src/guidellm/scheduler/worker.py | 6 +----- src/guidellm/utils/cli.py | 1 + 10 files changed, 22 insertions(+), 26 deletions(-) diff --git a/src/guidellm/benchmark/benchmarker.py b/src/guidellm/benchmark/benchmarker.py index 4b17d23d..8a46d44e 100644 --- a/src/guidellm/benchmark/benchmarker.py +++ b/src/guidellm/benchmark/benchmarker.py @@ -13,7 +13,7 @@ import uuid from abc import ABC from collections.abc import AsyncIterator, Iterable -from typing import Generic +from typing import Any, Generic from guidellm.benchmark.profile import Profile from guidellm.benchmark.progress import BenchmarkerProgress diff --git a/src/guidellm/data/deserializers/deserializer.py b/src/guidellm/data/deserializers/deserializer.py index f1041f20..cddd4876 100644 --- a/src/guidellm/data/deserializers/deserializer.py +++ b/src/guidellm/data/deserializers/deserializer.py @@ -107,8 +107,10 @@ def _deserialize_with_registered_deserializers( if len(errors) > 0: err_msgs = "" + def sort_key(item): return (isinstance(item[1], DataNotSupportedError), item[0]) + for key, err in sorted(errors.items(), key=sort_key): err_msgs += f"\n - Deserializer '{key}': ({type(err).__name__}) {err}" raise ValueError( @@ -141,4 +143,3 @@ def _deserialize_with_specified_deserializer( random_seed=random_seed, **data_kwargs, ) - diff --git a/src/guidellm/data/loaders.py b/src/guidellm/data/loaders.py index e260eef5..b4ee38da 100644 --- a/src/guidellm/data/loaders.py +++ b/src/guidellm/data/loaders.py @@ -17,7 +17,6 @@ __all__ = ["DataLoader", "DatasetsIterator"] - class DatasetsIterator(TorchIterableDataset): def __init__( self, diff --git a/src/guidellm/data/preprocessors/formatters.py b/src/guidellm/data/preprocessors/formatters.py index 272cf604..5a869403 100644 --- a/src/guidellm/data/preprocessors/formatters.py +++ b/src/guidellm/data/preprocessors/formatters.py @@ -56,9 +56,7 @@ def __init__( self.stream: bool = stream self.max_tokens: int | None = max_tokens or max_completion_tokens - def __call__( - self, columns: dict[str, list[Any]] - ) -> GenerationRequest: + def __call__(self, columns: dict[str, list[Any]]) -> GenerationRequest: """ :param columns: A dict of GenerativeDatasetColumnType to Any """ @@ -396,9 +394,7 @@ def __call__( # noqa: C901 class GenerativeAudioTranslationRequestFormatter( GenerativeAudioTranscriptionRequestFormatter ): - def __call__( - self, columns: dict[str, list[Any]] - ) -> GenerationRequest: + def __call__(self, columns: dict[str, list[Any]]) -> GenerationRequest: result = super().__call__(columns) result.request_type = "audio_translations" return result diff --git a/src/guidellm/data/preprocessors/mappers.py b/src/guidellm/data/preprocessors/mappers.py index 1eced9fe..e5196f73 100644 --- a/src/guidellm/data/preprocessors/mappers.py +++ b/src/guidellm/data/preprocessors/mappers.py @@ -167,9 +167,7 @@ def __init__( dict[GenerativeDatasetColumnType, list[tuple[int, str]]] | None ) - def __call__( - self, row: dict[str, Any] - ) -> dict[str, list[Any]]: + def __call__(self, row: dict[str, Any]) -> dict[str, list[Any]]: if self.datasets_column_mappings is None: raise ValueError("DefaultGenerativeColumnMapper not setup with data.") diff --git a/src/guidellm/data/preprocessors/preprocessor.py b/src/guidellm/data/preprocessors/preprocessor.py index 0b4bc49a..e95ad75d 100644 --- a/src/guidellm/data/preprocessors/preprocessor.py +++ b/src/guidellm/data/preprocessors/preprocessor.py @@ -12,8 +12,7 @@ @runtime_checkable class DatasetPreprocessor(Protocol): - def __call__(self, item: dict[str, Any]) -> ( - GenerationRequest | dict[str, Any]): ... + def __call__(self, item: dict[str, Any]) -> GenerationRequest | dict[str, Any]: ... @runtime_checkable diff --git a/src/guidellm/preprocess/dataset.py b/src/guidellm/preprocess/dataset.py index cacce3f5..49ce7b09 100644 --- a/src/guidellm/preprocess/dataset.py +++ b/src/guidellm/preprocess/dataset.py @@ -238,7 +238,7 @@ def process_dataset( prompt_tokens: str | Path, output_tokens: str | Path, processor_args: dict[str, Any] | None = None, - data_args: dict[str, Any] | None = None, + data_args: dict[str, Any] | None = None, # noqa: ARG001 short_prompt_strategy: ShortPromptStrategy = ShortPromptStrategy.IGNORE, pad_char: str | None = None, concat_delimiter: str | None = None, diff --git a/src/guidellm/scheduler/strategies.py b/src/guidellm/scheduler/strategies.py index 448266cf..e1473b93 100644 --- a/src/guidellm/scheduler/strategies.py +++ b/src/guidellm/scheduler/strategies.py @@ -506,8 +506,10 @@ def init_processes_start(self, start_time: float): if self._processes_lock is None: raise RuntimeError("_processes_lock is None in init_processes_start") if self._offset is None: - raise RuntimeError("_offset is None in init_processes_start; was " - "init_processes_timings not called?") + raise RuntimeError( + "_offset is None in init_processes_start; was " + "init_processes_timings not called?" + ) with self._processes_lock: self._offset.value = start_time @@ -527,11 +529,15 @@ async def next_request_time(self, offset: int) -> float: next_delay = self._random.expovariate(self.rate) if self._processes_lock is None: - raise RuntimeError("_processes_lock is None in next_request_time; was " - "init_processes_timings not called?") + raise RuntimeError( + "_processes_lock is None in next_request_time; was " + "init_processes_timings not called?" + ) if self._offset is None: - raise RuntimeError("_offset is None in next_request_time; was " - "init_processes_timings not called?") + raise RuntimeError( + "_offset is None in next_request_time; was " + "init_processes_timings not called?" + ) with self._processes_lock: self._offset.value += next_delay diff --git a/src/guidellm/scheduler/worker.py b/src/guidellm/scheduler/worker.py index 45b4042b..977635fa 100644 --- a/src/guidellm/scheduler/worker.py +++ b/src/guidellm/scheduler/worker.py @@ -363,7 +363,6 @@ async def _process_next_request(self, target_start: float): async for resp, info in self.backend.resolve( # type: ignore[attr-defined] request, request_info, None ): - response = resp request_info = info if request_info is None: @@ -407,10 +406,7 @@ async def _dequeue_next_request( return request, request_info async def _schedule_request( - self, - request: RequestT, - request_info: RequestInfo, - target_start: float + self, request: RequestT, request_info: RequestInfo, target_start: float ): current_time = time.time() request_info.timings.scheduled_at = current_time diff --git a/src/guidellm/utils/cli.py b/src/guidellm/utils/cli.py index c4783f65..9af6841a 100644 --- a/src/guidellm/utils/cli.py +++ b/src/guidellm/utils/cli.py @@ -31,6 +31,7 @@ def parse_list_floats(ctx, param, value): # noqa: ARG001 f"of floats/ints. Error: {e}" ) from e + def parse_json(ctx, param, value): # noqa: ARG001 if value is None or value == [None]: return None From 37e0c979f0a8138ce2fba8ddc7540af03ff26b70 Mon Sep 17 00:00:00 2001 From: "guangli.bao" Date: Thu, 30 Oct 2025 17:31:10 +0800 Subject: [PATCH 3/4] remove marker xfail Signed-off-by: guangli.bao --- tests/unit/benchmark/test_output.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/benchmark/test_output.py b/tests/unit/benchmark/test_output.py index 3425fa1d..416a9b2b 100644 --- a/tests/unit/benchmark/test_output.py +++ b/tests/unit/benchmark/test_output.py @@ -91,7 +91,6 @@ def test_file_yaml(): mock_path.unlink() -@pytest.mark.xfail(reason="old and broken", run=False) @pytest.mark.asyncio async def test_file_csv(): args = BenchmarkGenerativeTextArgs(target="http://localhost:8000", data=["test"]) From 62c4a11cb86c491633a9d8e6103c13794d399db0 Mon Sep 17 00:00:00 2001 From: "guangli.bao" Date: Mon, 3 Nov 2025 09:23:59 +0800 Subject: [PATCH 4/4] update benchmark mocker requests Signed-off-by: guangli.bao --- src/guidellm/benchmark/output.py | 2 +- tests/unit/mock_benchmark.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/guidellm/benchmark/output.py b/src/guidellm/benchmark/output.py index 4523ffae..6e17de5b 100644 --- a/src/guidellm/benchmark/output.py +++ b/src/guidellm/benchmark/output.py @@ -690,7 +690,7 @@ def _get_benchmark_extras_headers_and_values( values: list[str] = [ benchmark.benchmarker.profile.model_dump_json(), json.dumps(benchmark.benchmarker.backend), - json.dumps(benchmark.benchmarker.requests["attributes"]["data"]), + json.dumps(benchmark.benchmarker.requests["data"]), ] if len(headers) != len(values): diff --git a/tests/unit/mock_benchmark.py b/tests/unit/mock_benchmark.py index e06ffed8..0546d28f 100644 --- a/tests/unit/mock_benchmark.py +++ b/tests/unit/mock_benchmark.py @@ -105,9 +105,7 @@ def mock_generative_benchmark() -> GenerativeBenchmark: benchmarker=BenchmarkerDict( profile=SynchronousProfile.create("synchronous", rate=None), requests={ - "attributes": { - "data": "prompt_tokens=256,output_tokens=128", - }, + "data": "prompt_tokens=256,output_tokens=128", }, backend={}, environment={},