Upgrade openai to 4.91.1 and fix typing issues

james-deee · james-deee · commit d82c949b2467 · 2025-04-05T11:28:19.000-05:00
diff --git a/.changeset/eleven-terms-clap.md b/.changeset/eleven-terms-clap.md
@@ -0,0 +1,5 @@
+---
+'token.js': minor
+---
+
+Upgrade the openai client library to 4.91.1
diff --git a/package.json b/package.json
@@ -40,7 +40,7 @@
     "cohere-ai": "7.10.6",
     "mime-types": "^2.1.35",
     "nanoid": "^5.0.7",
-    "openai": "4.52.2"
+    "openai": "4.91.1"
   },
   "devDependencies": {
     "@babel/eslint-parser": "^7.18.2",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
diff --git a/src/handlers/ai21.ts b/src/handlers/ai21.ts
@@ -10,7 +10,7 @@ import {
 } from '../userTypes/index.js'
 import { BaseHandler } from './base.js'
 import { InputError } from './types.js'
-import { getTimestamp } from './utils.js'
+import { convertMessageContentToString, getTimestamp } from './utils.js'
 
 type AI21ChatCompletionParams = {
   model: string
@@ -73,7 +73,7 @@ const convertMessages = (
     if (i === 0 && message.role === 'system') {
       output.push({
         role: 'system',
-        content: message.content,
+        content: convertMessageContentToString(message.content),
       })
     } else if (
       message.role === 'user' ||
@@ -268,6 +268,10 @@ export class AI21Handler extends BaseHandler<AI21Model> {
       const convertedChoices = data.choices.map((choice) => {
         return {
           ...choice,
+          message: {
+            ...choice.message,
+            refusal: null,
+          },
           logprobs: null,
         }
       })
diff --git a/src/handlers/anthropic.ts b/src/handlers/anthropic.ts
@@ -28,6 +28,7 @@ import { BaseHandler } from './base.js'
 import { InputError, InvariantError } from './types.js'
 import {
   consoleWarn,
+  convertMessageContentToString,
   fetchThenParseImage,
   getTimestamp,
   isEmptyObject,
@@ -247,12 +248,14 @@ const toChatCompletionChoiceMessage = (
     const messageContent = content.every(isToolUseBlock) ? null : ''
     return {
       role,
+      refusal: null,
       content: messageContent,
       tool_calls: toolCalls,
     }
   } else {
     return {
       role,
+      refusal: null,
       content: textBlocks.map((textBlock) => textBlock.text).join('\n'),
       tool_calls: toolCalls,
     }
@@ -361,7 +364,7 @@ export const convertMessages = async (
   // unchanged.
   let systemMessage: string | undefined
   if (clonedMessages.length > 0 && clonedMessages[0].role === 'system') {
-    systemMessage = clonedMessages[0].content
+    systemMessage = convertMessageContentToString(clonedMessages[0].content)
     clonedMessages.shift()
   }
 
@@ -448,7 +451,7 @@ export const convertMessages = async (
               return {
                 type: 'text',
                 text,
-              }
+              } as TextBlockParam
             } else {
               const parsedImage = await fetchThenParseImage(e.image_url.url)
               return {
@@ -458,7 +461,7 @@ export const convertMessages = async (
                   media_type: parsedImage.mimeType,
                   type: 'base64',
                 },
-              }
+              } as ImageBlockParam
             }
           })
         )
diff --git a/src/handlers/bedrock.ts b/src/handlers/bedrock.ts
@@ -29,6 +29,7 @@ import { BaseHandler } from './base.js'
 import { InputError, InvariantError, MIMEType } from './types.js'
 import {
   consoleWarn,
+  convertMessageContentToString,
   fetchThenParseImage,
   getTimestamp,
   normalizeTemperature,
@@ -84,6 +85,7 @@ const toChatCompletionChoiceMessage = (
 ): CompletionResponse['choices'][0]['message'] => {
   if (output?.message?.content === undefined) {
     return {
+      refusal: null,
       content: '',
       role: 'assistant',
     }
@@ -146,13 +148,15 @@ const toChatCompletionChoiceMessage = (
       : ''
     return {
       role,
+      refusal: null,
       content: messageContent,
       tool_calls: toolCalls,
     }
   } else {
     const content = textBlocks.map((textBlock) => textBlock.text).join('\n')
     return {
       role,
+      refusal: null,
       content,
       tool_calls: toolCalls,
     }
@@ -187,7 +191,10 @@ export const convertMessages = async (
   const systemMessages: Array<SystemContentBlock> = []
   if (supportsSystemMessages(model)) {
     while (clonedMessages.length > 0 && clonedMessages[0].role === 'system') {
-      systemMessages.push({ text: clonedMessages[0].content })
+      const messageContent = convertMessageContentToString(
+        clonedMessages[0].content
+      )
+      systemMessages.push({ text: messageContent })
       clonedMessages.shift()
     }
   }
@@ -247,7 +254,7 @@ export const convertMessages = async (
           toolUseId: message.tool_call_id,
           content: [
             {
-              text: message.content,
+              text: convertMessageContentToString(message.content),
             },
           ],
         },
diff --git a/src/handlers/cohere.ts b/src/handlers/cohere.ts
@@ -28,7 +28,11 @@ import {
 } from '../userTypes/index.js'
 import { BaseHandler } from './base.js'
 import { InputError, InvariantError, MessageRole } from './types.js'
-import { consoleWarn, getTimestamp } from './utils.js'
+import {
+  consoleWarn,
+  convertMessageContentToString,
+  getTimestamp,
+} from './utils.js'
 
 type CohereMessageRole = 'CHATBOT' | 'SYSTEM' | 'USER' | 'TOOL'
 
@@ -41,6 +45,8 @@ const convertRole = (role: MessageRole): CohereMessageRole => {
     return 'TOOL'
   } else if (role === 'user') {
     return 'USER'
+  } else if (role === 'developer') {
+    return 'SYSTEM'
   } else {
     throw new InputError(`Unknown role: ${role}`)
   }
@@ -259,12 +265,13 @@ const toToolResult = (
     )
   }
 
+  const tollCallContentStr = convertMessageContentToString(toolMessage.content)
   const toolResult: ToolResult = {
     call: {
       name: toolCall.function.name,
       parameters: JSON.parse(toolCall.function.arguments),
     },
-    outputs: [JSON.parse(toolMessage.content)],
+    outputs: [JSON.parse(tollCallContentStr)],
   }
   return toolResult
 }
@@ -318,9 +325,10 @@ export const convertMessages = (
         })
       }
     } else if (message.role === 'assistant') {
+      const messageContentStr = convertMessageContentToString(message.content)
       chatHistory.push({
         role: convertRole(message.role),
-        message: message.content ?? '',
+        message: messageContentStr,
         toolCalls: message.tool_calls?.map((toolCall) => {
           return {
             name: toolCall.function.name,
@@ -567,6 +575,7 @@ export class CohereHandler extends BaseHandler<CohereModel> {
             logprobs: null,
             message: {
               role: 'assistant',
+              refusal: null, // openai requires this field, fill in if Cohere ever supports
               content: response.text,
               tool_calls: toolCalls,
             },
diff --git a/src/handlers/gemini.ts b/src/handlers/gemini.ts
@@ -31,7 +31,12 @@ import {
 } from '../userTypes/index.js'
 import { BaseHandler } from './base.js'
 import { InputError } from './types.js'
-import { consoleWarn, fetchThenParseImage, getTimestamp } from './utils.js'
+import {
+  consoleWarn,
+  convertMessageContentToString,
+  fetchThenParseImage,
+  getTimestamp,
+} from './utils.js'
 
 // Google's `GenerateContentCandidate.content` field should be optional, but it's a required field
 // in Google's types. This field can be undefined if a content filter is triggered when the user
@@ -164,7 +169,9 @@ export const convertMessageToContent = async (
           {
             functionResponse: {
               name: message.tool_call_id,
-              response: JSON.parse(message.content),
+              response: JSON.parse(
+                convertMessageContentToString(message.content)
+              ),
             },
           },
         ],
@@ -304,6 +311,7 @@ export const convertResponseMessage = (
     content: candidate.content?.parts.map((part) => part.text).join('') ?? null,
     role: 'assistant',
     tool_calls: convertToolCalls(candidate),
+    refusal: null,
   }
 }
 
diff --git a/src/handlers/mistral.ts b/src/handlers/mistral.ts
@@ -26,6 +26,7 @@ import {
 } from '../userTypes/index.js'
 import { BaseHandler } from './base.js'
 import { InputError } from './types.js'
+import { convertMessageContentToString } from './utils.js'
 
 export const findLinkedToolCallName = (
   messages: ChatCompletionMessage[],
@@ -46,6 +47,7 @@ export const convertMessages = (
   messages: (ChatCompletionMessageParam | ChatCompletionMessage)[]
 ): Array<Message | ChatCompletionResponseChoice['message']> => {
   return messages.map((message) => {
+    const messageContent = convertMessageContentToString(message.content)
     if (message.role === 'tool') {
       const name = findLinkedToolCallName(
         messages as ChatCompletionMessage[],
@@ -55,20 +57,20 @@ export const convertMessages = (
       return {
         name,
         role: 'tool',
-        content: message.content,
+        content: messageContent,
         tool_call_id: message.tool_call_id,
       }
     }
 
     if (message.role === 'system') {
       return {
         role: message.role,
-        content: message.content ?? '',
+        content: messageContent,
       }
     } else if (message.role === 'assistant') {
       return {
         role: message.role,
-        content: message.content ?? '',
+        content: messageContent,
         tool_calls: message.tool_calls ?? null,
       }
     } else if (message.role === 'user') {
@@ -239,6 +241,7 @@ const toCompletionResponse = (
         index: choice.index,
         message: {
           role: 'assistant',
+          refusal: null,
           content: choice.message.content,
           tool_calls: convertToolCalls(choice.message.tool_calls),
         },
diff --git a/src/handlers/types.ts b/src/handlers/types.ts
@@ -1,4 +1,10 @@
-export type MessageRole = 'system' | 'user' | 'assistant' | 'tool' | 'function'
+export type MessageRole =
+  | 'system'
+  | 'user'
+  | 'assistant'
+  | 'tool'
+  | 'function'
+  | 'developer'
 export type MIMEType = 'image/jpeg' | 'image/png' | 'image/gif' | 'image/webp'
 
 export class InputError extends Error {
diff --git a/src/handlers/utils.ts b/src/handlers/utils.ts
@@ -1,5 +1,6 @@
 import chalk from 'chalk'
 import { lookup } from 'mime-types'
+import OpenAI from 'openai'
 
 import { LLMChatModel, LLMProvider } from '../chat/index.js'
 import { models } from '../models.js'
@@ -270,3 +271,21 @@ export const isObject = (variable: any): boolean => {
     variable && typeof variable === 'object' && variable.constructor === Object
   )
 }
+
+export const convertMessageContentToString = (
+  messageContent: OpenAI.Chat.Completions.ChatCompletionMessageParam['content']
+): string => {
+  if (!messageContent) {
+    return ''
+  }
+
+  return (
+    (typeof messageContent === 'string'
+      ? messageContent
+      : messageContent
+          .map(
+            (m: OpenAI.Chat.Completions.ChatCompletionContentPartText) => m.text
+          )
+          .join('\n')) ?? ''
+  )
+}
diff --git a/test/automated/handlers/cohere.test.ts b/test/automated/handlers/cohere.test.ts
@@ -299,6 +299,13 @@ describe('convertMessages', () => {
         role: 'tool',
         content: '{"temperature":"22","unit":"fahrenheit"}',
       },
+      {
+        tool_call_id: 'call_lhMKBlOSnwwq5BZDCGo5SVTJ',
+        role: 'tool',
+        content: [
+          { text: '{"temperature":"100","unit":"fahrenheit"}', type: 'text' },
+        ],
+      },
     ]
 
     const { messages, lastUserMessage, toolResults } =
@@ -379,6 +386,20 @@ describe('convertMessages', () => {
           },
         ],
       },
+      {
+        call: {
+          name: 'get_current_weather',
+          parameters: {
+            location: 'Paris, France',
+          },
+        },
+        outputs: [
+          {
+            temperature: '100',
+            unit: 'fahrenheit',
+          },
+        ],
+      },
     ])
   })
 })
diff --git a/test/automated/handlers/gemini.test.ts b/test/automated/handlers/gemini.test.ts
diff --git a/test/automated/handlers/mistral.test.ts b/test/automated/handlers/mistral.test.ts

-Original file line number
+Diff line change
@@ @@ -0,0 +1,5 @@ @@
 +---
 +'token.js': minor
 +---
++
 +Upgrade the openai client library to 4.91.1