From 83e8b5d03a78408fa7ebf078abfbd2f7928f6c5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Varga?= <54496628+gaborvar@users.noreply.github.com> Date: Sat, 26 Oct 2024 05:14:20 +0200 Subject: [PATCH 1/3] Update message_builder.py to preserve tool call parameters tool_calls and tool_call_id are needed to correctly inform the model about historic tool calls. Also fixed a related error message. --- src/openai_messages_token_helper/message_builder.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/openai_messages_token_helper/message_builder.py b/src/openai_messages_token_helper/message_builder.py index 86203de..790b1cd 100644 --- a/src/openai_messages_token_helper/message_builder.py +++ b/src/openai_messages_token_helper/message_builder.py @@ -147,8 +147,12 @@ def build_messages( logging.info("Reached max tokens of %d, history will be truncated", max_tokens) break - if message["role"] is None or message["content"] is None: - raise ValueError("Few-shot messages must have both role and content") - message_builder.insert_message(message["role"], message["content"], index=append_index) # type: ignore[arg-type] + if message["role"] is None or (message["content"] is None and message.get("role") != "assistant"): + raise ValueError("Past messages must have both role and content, unless it is a tools message from assistant.") + + message_builder.insert_message(message["role"], message["content"], + tool_calls=message.get("tool_calls",None), + tool_call_id=message.get("tool_call_id",None), + index=append_index) # type: ignore[arg-type] total_token_count += potential_message_count return message_builder.all_messages From fd2e5d6a4a50b98d7d851f951aff7ad38ea47889 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Varga?= <54496628+gaborvar@users.noreply.github.com> Date: Sat, 26 Oct 2024 05:45:54 +0200 Subject: [PATCH 2/3] Update model_helper.py to handle tool calls For tool call requests by the model, accept their typical parameters rather than throwing an error. The goal of this fix is limited to preventing undue errors: impact of tool calls on token counting is only a guess which needs to revised. --- src/openai_messages_token_helper/model_helper.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/openai_messages_token_helper/model_helper.py b/src/openai_messages_token_helper/model_helper.py index 1d131a1..01280f7 100644 --- a/src/openai_messages_token_helper/model_helper.py +++ b/src/openai_messages_token_helper/model_helper.py @@ -116,8 +116,14 @@ def count_tokens_for_message(model: str, message: ChatCompletionMessageParam, de num_tokens += len(encoding.encode(item["text"])) elif item["type"] == "image_url": num_tokens += count_tokens_for_image(item["image_url"]["url"], item["image_url"]["detail"], model) + elif item["type"] == "function": + num_tokens += 1 # A guess, to be revised. What is the impact of a tool/function call on tokens? elif isinstance(value, str): num_tokens += len(encoding.encode(value)) + elif (key in ["content", "refusal", "function_call"] and value is None + and message.get("role")=="assistant" + and message.get("tool_calls") is not None): # If the model calls a tool/function then content can be None + num_tokens += 0 # a guess at this stage, to be revised else: raise ValueError(f"Could not encode unsupported message value type: {type(value)}") if key == "name": From e0b1265e9205f4cda0bc8a183aecbace926bf093 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Varga?= <54496628+gaborvar@users.noreply.github.com> Date: Sat, 26 Oct 2024 06:54:08 +0200 Subject: [PATCH 3/3] Update count_tokens_for_message() docstring with a type --- src/openai_messages_token_helper/model_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/openai_messages_token_helper/model_helper.py b/src/openai_messages_token_helper/model_helper.py index 01280f7..ff96612 100644 --- a/src/openai_messages_token_helper/model_helper.py +++ b/src/openai_messages_token_helper/model_helper.py @@ -91,7 +91,7 @@ def count_tokens_for_message(model: str, message: ChatCompletionMessageParam, de Args: model (str): The name of the model to use for encoding. - message (Mapping): The message to encode, in a dictionary-like object. + message (dict): The message to encode, in a dictionary-like object. default_to_cl100k (bool): Whether to default to the CL100k encoding if the model is not found. Returns: int: The total number of tokens required to encode the message.