Skip to content

Commit a5cd7c9

Browse files
LiqinruiGliqinrui
andauthored
[BugFix] rollback max_tokens and min_tokens when continue to infer (#5082)
* [BugFix] rollback max_tokens and min_tokens when continue to infer * [BugFix] rollback max_tokens and min_tokens when continue to infer * [fix] add more logger info: max_tokens --------- Co-authored-by: liqinrui <liqinrui@baidu.com>
1 parent 43f0c75 commit a5cd7c9

File tree

2 files changed

+8
-5
lines changed

2 files changed

+8
-5
lines changed

fastdeploy/entrypoints/engine_client.py

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -298,15 +298,20 @@ def valid_parameters(self, data):
298298

299299
if data.get("max_tokens") is not None:
300300
if data["max_tokens"] < 1 or data["max_tokens"] >= self.max_model_len:
301-
raise ParameterError("max_tokens", f"max_tokens can be defined [1, {self.max_model_len}).")
301+
api_server_logger.error(
302+
f"req_id:{data['request_id']}, max_tokens must be defined [1, {self.max_model_len}), but now it's {data['max_tokens']}."
303+
)
304+
raise ValueError(
305+
f"max_tokens can be defined [1, {self.max_model_len}), but now it's {data['max_tokens']}."
306+
)
302307

303308
if data.get("reasoning_max_tokens") is not None:
304309
if data["reasoning_max_tokens"] < 1:
305310
raise ParameterError("reasoning_max_tokens", "reasoning_max_tokens must be greater than 1")
306311
if data["reasoning_max_tokens"] > data["max_tokens"]:
307312
data["reasoning_max_tokens"] = data["max_tokens"]
308313
api_server_logger.warning(
309-
f"req_id: {data['request_id']}, reasoning_max_tokens exceeds max_tokens, the value of reasoning_max_tokens will be adjusted to match that of max_tokens"
314+
f"req_id: {data['request_id']}, reasoning_max_tokens exceeds max_tokens, the value of reasoning_max_tokens will be adjusted to {data['max_tokens']}"
310315
)
311316
if data.get("temperature") is not None and abs(data["temperature"]) < 1e-6:
312317
data["temperature"] = 1e-6

fastdeploy/input/ernie4_5_vl_processor/ernie4_5_vl_processor.py

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -264,14 +264,12 @@ def process_request_dict(self, request, max_model_len=None):
264264
if max_model_len is not None and len(request["prompt_token_ids"]) > max_model_len:
265265
request["prompt_token_ids"] = request["prompt_token_ids"][: max_model_len - 1]
266266

267-
tmp_max_tokens = 0
268267
if request.get("max_tokens") is None:
269268
request["max_tokens"] = max(1, max_model_len - len(request["prompt_token_ids"]))
270-
tmp_max_tokens = request["max_tokens"]
271269
else:
272270
request["max_tokens"] = min(max_model_len - len(request["prompt_token_ids"]), request["max_tokens"])
273271
if request.get("reasoning_max_tokens") is None:
274-
request["reasoning_max_tokens"] = max(int(tmp_max_tokens * 0.8), 1)
272+
request["reasoning_max_tokens"] = max(int(request["max_tokens"] * 0.8), 1)
275273
data_processor_logger.info(f"Processed request {request}")
276274

277275
if request.get("top_p") is not None and request.get("top_p") < _SAMPLING_EPS:

0 commit comments

Comments
 (0)