Migrate AI SDK integration from v5 to v6

.changeset/0000-ai-sdk-v6.md

@@ -0,0 +1,5 @@
+---
+"evalite": minor
+---
+
+Migrated AI SDK integration from v5 to v6. The `ai` peer dependency is now `^6` and `@ai-sdk/provider` is `^3`. Users on AI SDK v5 should stay on the previous Evalite version.

apps/evalite-docs/src/content/docs/api/ai-sdk.mdx

@@ -35,12 +35,12 @@ evalite("My Eval", {
 
 ```typescript
 wrapAISDKModel(
-  model: LanguageModelV2,
+  model: LanguageModel,
   options?: {
     tracing?: boolean;
     caching?: boolean;
   }
-): LanguageModelV2
+): LanguageModel
 ```
 
 **Parameters:**
@@ -137,32 +137,34 @@ const result = await streamText({
 const text = await result.text;
 ```
 
-**Generate Object:**
+**Structured Output:**
 
 ```typescript
-import { generateObject } from "ai";
+import { generateText, Output } from "ai";
 import { z } from "zod";
 
-const result = await generateObject({
+const result = await generateText({
   model: wrapAISDKModel(openai("gpt-4")),
-  schema: z.object({ name: z.string() }),
+  output: Output.object({ schema: z.object({ name: z.string() }) }),
   prompt: "Generate a person",
 });
+
+const person = result.output; // { name: string }
 ```
 
-**Stream Object:**
+**Streaming Structured Output:**
 
 ```typescript
-import { streamObject } from "ai";
+import { streamText, Output } from "ai";
 import { z } from "zod";
 
-const result = await streamObject({
+const result = streamText({
   model: wrapAISDKModel(openai("gpt-4")),
-  schema: z.object({ name: z.string() }),
+  output: Output.object({ schema: z.object({ name: z.string() }) }),
   prompt: "Generate a person",
 });
 
-const object = await result.object;
+const person = await result.output; // { name: string }
 ```
 
 ## Behavior in Production

apps/evalite-ui/package.json

@@ -26,7 +26,7 @@
     "@tanstack/react-router": "^1.117.0",
     "@tanstack/router-devtools": "^1.117.0",
     "@tanstack/zod-adapter": "^1.117.0",
-    "ai": "^5.0.89",
+    "ai": "^6",
     "class-variance-authority": "^0.7.1",
     "clsx": "^2.1.1",
     "date-fns": "^4.1.0",

packages/evalite-tests/package.json

@@ -12,7 +12,7 @@
     "evalite": "workspace:*",
     "evalite-ui": "workspace:*",
     "strip-ansi": "^7.1.2",
-    "@ai-sdk/openai": "^2.0.42",
-    "ai": "^5.0.59"
+    "@ai-sdk/openai": "^3.0.0",
+    "ai": "^6"
   }
 }

packages/evalite-tests/tests/fixtures/ai-sdk-caching-config-disabled/caching.eval.ts

@@ -1,32 +1,40 @@
 import { generateText } from "ai";
-import { MockLanguageModelV2 } from "ai/test";
+import { MockLanguageModelV3 } from "ai/test";
 import { wrapAISDKModel } from "evalite/ai-sdk";
 import { evalite } from "evalite";
 
-const model = new MockLanguageModelV2({
-  doGenerate: async (options) => ({
-    rawCall: { rawPrompt: null, rawSettings: {} },
-    finishReason: "stop",
-    usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30 },
+const model = new MockLanguageModelV3({
+  doGenerate: {
+    finishReason: { unified: "stop", raw: undefined },
+    usage: {
+      inputTokens: {
+        total: 10,
+        noCache: undefined,
+        cacheRead: undefined,
+        cacheWrite: undefined,
+      },
+      outputTokens: { total: 20, text: undefined, reasoning: undefined },
+    },
     content: [{ type: "text", text: `Response for task` }],
     warnings: [],
-    providerMetadata: undefined,
-    request: undefined,
-    response: undefined,
-  }),
+  },
 });
 
-const scorerModel = new MockLanguageModelV2({
-  doGenerate: async (options) => ({
-    rawCall: { rawPrompt: null, rawSettings: {} },
-    finishReason: "stop",
-    usage: { inputTokens: 5, outputTokens: 10, totalTokens: 15 },
+const scorerModel = new MockLanguageModelV3({
+  doGenerate: {
+    finishReason: { unified: "stop", raw: undefined },
+    usage: {
+      inputTokens: {
+        total: 5,
+        noCache: undefined,
+        cacheRead: undefined,
+        cacheWrite: undefined,
+      },
+      outputTokens: { total: 10, text: undefined, reasoning: undefined },
+    },
     content: [{ type: "text", text: `1` }],
     warnings: [],
-    providerMetadata: undefined,
-    request: undefined,
-    response: undefined,
-  }),
+  },
 });
 
 const tracedModel = wrapAISDKModel(model);

packages/evalite-tests/tests/fixtures/ai-sdk-caching-config-precedence/caching.eval.ts

@@ -1,32 +1,40 @@
 import { generateText } from "ai";
-import { MockLanguageModelV2 } from "ai/test";
+import { MockLanguageModelV3 } from "ai/test";
 import { wrapAISDKModel } from "evalite/ai-sdk";
 import { evalite } from "evalite";
 
-const model = new MockLanguageModelV2({
-  doGenerate: async (options) => ({
-    rawCall: { rawPrompt: null, rawSettings: {} },
-    finishReason: "stop",
-    usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30 },
+const model = new MockLanguageModelV3({
+  doGenerate: {
+    finishReason: { unified: "stop", raw: undefined },
+    usage: {
+      inputTokens: {
+        total: 10,
+        noCache: undefined,
+        cacheRead: undefined,
+        cacheWrite: undefined,
+      },
+      outputTokens: { total: 20, text: undefined, reasoning: undefined },
+    },
     content: [{ type: "text", text: `Response for task` }],
     warnings: [],
-    providerMetadata: undefined,
-    request: undefined,
-    response: undefined,
-  }),
+  },
 });
 
-const scorerModel = new MockLanguageModelV2({
-  doGenerate: async (options) => ({
-    rawCall: { rawPrompt: null, rawSettings: {} },
-    finishReason: "stop",
-    usage: { inputTokens: 5, outputTokens: 10, totalTokens: 15 },
+const scorerModel = new MockLanguageModelV3({
+  doGenerate: {
+    finishReason: { unified: "stop", raw: undefined },
+    usage: {
+      inputTokens: {
+        total: 5,
+        noCache: undefined,
+        cacheRead: undefined,
+        cacheWrite: undefined,
+      },
+      outputTokens: { total: 10, text: undefined, reasoning: undefined },
+    },
     content: [{ type: "text", text: `1` }],
     warnings: [],
-    providerMetadata: undefined,
-    request: undefined,
-    response: undefined,
-  }),
+  },
 });
 
 const tracedModel = wrapAISDKModel(model);

packages/evalite-tests/tests/fixtures/ai-sdk-caching/caching.eval.ts

@@ -1,32 +1,40 @@
 import { generateText } from "ai";
-import { MockLanguageModelV2 } from "ai/test";
+import { MockLanguageModelV3 } from "ai/test";
 import { wrapAISDKModel } from "evalite/ai-sdk";
 import { evalite } from "evalite";
 
-const model = new MockLanguageModelV2({
-  doGenerate: async (options) => ({
-    rawCall: { rawPrompt: null, rawSettings: {} },
-    finishReason: "stop",
-    usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30 },
+const model = new MockLanguageModelV3({
+  doGenerate: {
+    finishReason: { unified: "stop", raw: undefined },
+    usage: {
+      inputTokens: {
+        total: 10,
+        noCache: undefined,
+        cacheRead: undefined,
+        cacheWrite: undefined,
+      },
+      outputTokens: { total: 20, text: undefined, reasoning: undefined },
+    },
     content: [{ type: "text", text: `Response for task` }],
     warnings: [],
-    providerMetadata: undefined,
-    request: undefined,
-    response: undefined,
-  }),
+  },
 });
 
-const scorerModel = new MockLanguageModelV2({
-  doGenerate: async (options) => ({
-    rawCall: { rawPrompt: null, rawSettings: {} },
-    finishReason: "stop",
-    usage: { inputTokens: 5, outputTokens: 10, totalTokens: 15 },
+const scorerModel = new MockLanguageModelV3({
+  doGenerate: {
+    finishReason: { unified: "stop", raw: undefined },
+    usage: {
+      inputTokens: {
+        total: 5,
+        noCache: undefined,
+        cacheRead: undefined,
+        cacheWrite: undefined,
+      },
+      outputTokens: { total: 10, text: undefined, reasoning: undefined },
+    },
     content: [{ type: "text", text: `1` }],
     warnings: [],
-    providerMetadata: undefined,
-    request: undefined,
-    response: undefined,
-  }),
+  },
 });
 
 const tracedModel = wrapAISDKModel(model);

packages/evalite-tests/tests/fixtures/ai-sdk-traces/traces.eval.ts

@@ -1,13 +1,20 @@
 import { generateText } from "ai";
-import { MockLanguageModelV2 } from "ai/test";
+import { MockLanguageModelV3 } from "ai/test";
 import { wrapAISDKModel } from "evalite/ai-sdk";
 import { evalite } from "evalite";
 
-const model = new MockLanguageModelV2({
-  doGenerate: async (options) => ({
-    rawCall: { rawPrompt: null, rawSettings: {} },
-    finishReason: "stop",
-    usage: { inputTokens: 10, outputTokens: 20, totalTokens: 35 },
+const model = new MockLanguageModelV3({
+  doGenerate: {
+    finishReason: { unified: "stop", raw: undefined },
+    usage: {
+      inputTokens: {
+        total: 10,
+        noCache: undefined,
+        cacheRead: undefined,
+        cacheWrite: undefined,
+      },
+      outputTokens: { total: 20, text: undefined, reasoning: undefined },
+    },
     content: [
       { type: "text", text: `Hello, world!` },
       {
@@ -18,10 +25,7 @@ const model = new MockLanguageModelV2({
       },
     ],
     warnings: [],
-    providerMetadata: undefined,
-    request: undefined,
-    response: undefined,
-  }),
+  },
 });
 
 const tracedModel = wrapAISDKModel(model);

packages/evalite/package.json

@@ -71,7 +71,7 @@
   },
   "peerDependencies": {
     "better-sqlite3": "^11.6.0",
-    "ai": "^5"
+    "ai": "^6"
   },
   "peerDependenciesMeta": {
     "better-sqlite3": {
@@ -82,11 +82,11 @@
     }
   },
   "devDependencies": {
-    "@ai-sdk/provider": "^2.0.0",
+    "@ai-sdk/provider": "^3.0.0",
     "@types/better-sqlite3": "^7.6.13",
     "@types/js-levenshtein": "^1.1.3",
     "@types/ws": "^8.18.1",
-    "ai": "^5.0.59",
+    "ai": "^6",
     "better-sqlite3": "^11.6.0",
     "unstorage": "^1.17.1"
   }