From 29816570d8257456ceb0812325636a390507ff5e Mon Sep 17 00:00:00 2001 From: Ken Jiang Date: Tue, 18 Nov 2025 14:55:39 -0500 Subject: [PATCH 1/4] add braintrust evalite --- .changeset/0000-braintrust-storage.md | 27 + package.json | 3 +- packages/braintrust/README.md | 96 ++ packages/braintrust/examples/.env.example | 3 + packages/braintrust/examples/README.md | 92 ++ .../braintrust/examples/evalite.config.ts | 28 + packages/braintrust/examples/simple.eval.ts | 25 + packages/braintrust/package.json | 33 + packages/braintrust/src/index.ts | 578 ++++++++++++ packages/braintrust/tsconfig.json | 8 + packages/evalite/src/run-evalite.ts | 13 + packages/example/braintrust-test.eval.ts | 24 + packages/example/evalite.config.braintrust.ts | 38 + .../example/evalite.config.sqlite-backup.ts | 5 + packages/example/evalite.config.ts | 8 +- packages/example/package.json | 1 + pnpm-lock.yaml | 876 ++++++++++++++++++ 17 files changed, 1856 insertions(+), 2 deletions(-) create mode 100644 .changeset/0000-braintrust-storage.md create mode 100644 packages/braintrust/README.md create mode 100644 packages/braintrust/examples/.env.example create mode 100644 packages/braintrust/examples/README.md create mode 100644 packages/braintrust/examples/evalite.config.ts create mode 100644 packages/braintrust/examples/simple.eval.ts create mode 100644 packages/braintrust/package.json create mode 100644 packages/braintrust/src/index.ts create mode 100644 packages/braintrust/tsconfig.json create mode 100644 packages/example/braintrust-test.eval.ts create mode 100644 packages/example/evalite.config.braintrust.ts create mode 100644 packages/example/evalite.config.sqlite-backup.ts diff --git a/.changeset/0000-braintrust-storage.md b/.changeset/0000-braintrust-storage.md new file mode 100644 index 00000000..87b8374b --- /dev/null +++ b/.changeset/0000-braintrust-storage.md @@ -0,0 +1,27 @@ +--- +"evalite": minor +"@evalite/braintrust": major +--- + +**Breaking Change:** Braintrust storage has been moved to a separate package `@evalite/braintrust`. This reduces the bundle size of the main `evalite` package for users who don't need Braintrust integration. + +**What changed:** + +- Braintrust storage is now in a separate package `@evalite/braintrust` +- The main `evalite` package no longer includes the `braintrust` dependency +- The `evalite/braintrust-storage` export has been removed + +**Migration:** +If you were using Braintrust storage: + +1. Install the new package: `pnpm add @evalite/braintrust` (or `npm install @evalite/braintrust`) +2. Update your imports in `evalite.config.ts`: + - **Before:** `import { createBraintrustStorage } from "evalite/braintrust-storage";` + - **After:** `import { createBraintrustStorage } from "@evalite/braintrust";` +3. The API remains unchanged - only the import path has changed + +**Benefits:** + +- Main `evalite` package is lighter and faster to install +- Braintrust dependency only installed when needed +- Better separation of concerns diff --git a/package.json b/package.json index 0bb653e1..61522d8e 100644 --- a/package.json +++ b/package.json @@ -11,8 +11,9 @@ "scripts": { "beta:release": "pnpm run ci && changeset version && git add . && git commit -m 'Bump version' && git push && pnpm changeset publish", "build:evalite": "pnpm --filter evalite build", + "build:braintrust": "pnpm --filter @evalite/braintrust build", "build:evalite-ui": "pnpm --filter evalite-ui build && pnpm --filter evalite-ui after-build", - "build": "pnpm run build:evalite && pnpm run build:evalite-ui", + "build": "pnpm run build:evalite && pnpm run build:braintrust && pnpm run build:evalite-ui", "dev": "pnpm build:evalite && pnpm build:evalite-ui && pnpm --filter evalite --filter evalite-tests --parallel run dev", "dev:evalite": "pnpm --filter evalite dev", "dev:evalite-ui": "pnpm --filter evalite-ui dev", diff --git a/packages/braintrust/README.md b/packages/braintrust/README.md new file mode 100644 index 00000000..7964af02 --- /dev/null +++ b/packages/braintrust/README.md @@ -0,0 +1,96 @@ +# @evalite/braintrust + +Braintrust storage backend for Evalite. Store your evaluation results in [Braintrust](https://www.braintrust.dev) and view them in the Braintrust web UI. + +## Installation + +```bash +npm install @evalite/braintrust evalite +# or +pnpm add @evalite/braintrust evalite +# or +yarn add @evalite/braintrust evalite +``` + +## Usage + +Configure Evalite to use Braintrust storage in your `evalite.config.ts`: + +```typescript +import { defineConfig } from "evalite/config"; +import { createBraintrustStorage } from "@evalite/braintrust"; + +export default defineConfig({ + storage: async () => { + return await createBraintrustStorage({ + // Required: Your Braintrust project name + projectName: "My Evalite Project", + + // Optional: Custom experiment name (defaults to timestamp) + experimentName: "my-experiment", + + // Optional: API key (defaults to BRAINTRUST_API_KEY env var) + apiKey: process.env.BRAINTRUST_API_KEY, + + // Optional: Custom Braintrust app URL + appUrl: "https://www.braintrust.dev", + }); + }, +}); +``` + +## Environment Variables + +Set your Braintrust API key: + +```bash +export BRAINTRUST_API_KEY="your-api-key-here" +``` + +## Features + +- **Automatic URL generation**: After running evals, you'll see a URL to view results in Braintrust +- **Nested traces**: LLM calls and traces are logged as nested spans in Braintrust +- **Scores and metadata**: All scorers and custom columns are preserved +- **Experiment organization**: Results are organized by project and experiment name + +## Example + +See the [examples directory](./examples) for a complete working example. + +To run the example: + +```bash +cd examples +export BRAINTRUST_API_KEY="your-api-key" +pnpm evalite run simple.eval.ts +``` + +## API Reference + +### `createBraintrustStorage(options)` + +Creates a new Braintrust storage backend. + +**Options:** + +- `projectName` (string, required): The Braintrust project name +- `experimentName` (string, optional): Custom experiment name. Defaults to `evalite-{timestamp}` +- `apiKey` (string, optional): Braintrust API key. Defaults to `BRAINTRUST_API_KEY` env var +- `appUrl` (string, optional): Custom Braintrust app URL. Defaults to `https://www.braintrust.dev` + +**Returns:** `Promise` + +## How It Works + +When you run evals with Braintrust storage: + +1. Each eval suite creates a Braintrust experiment +2. Each test case becomes a top-level span (row) in the experiment +3. Scores are logged to their corresponding spans +4. LLM traces are logged as nested spans under their test case +5. After the run completes, you get a URL to view results in Braintrust + +## License + +MIT diff --git a/packages/braintrust/examples/.env.example b/packages/braintrust/examples/.env.example new file mode 100644 index 00000000..df63c366 --- /dev/null +++ b/packages/braintrust/examples/.env.example @@ -0,0 +1,3 @@ +# Braintrust API Key +# Get your API key from https://www.braintrust.dev +BRAINTRUST_API_KEY=your-api-key-here diff --git a/packages/braintrust/examples/README.md b/packages/braintrust/examples/README.md new file mode 100644 index 00000000..cade4d7d --- /dev/null +++ b/packages/braintrust/examples/README.md @@ -0,0 +1,92 @@ +# Braintrust Storage Example + +This example demonstrates how to use `@evalite/braintrust` to store evaluation results in Braintrust. + +## Prerequisites + +1. Get a Braintrust API key from [braintrust.dev](https://www.braintrust.dev) +2. Set your API key as an environment variable: + ```bash + export BRAINTRUST_API_KEY="your-api-key-here" + ``` + +## Setup + +1. Build the required packages (from the monorepo root): + + ```bash + cd /path/to/evalite + pnpm build + ``` + +2. Set your Braintrust API key: + ```bash + export BRAINTRUST_API_KEY="your-api-key-here" + ``` + +## Running the Example + +From the monorepo root: + +```bash +# Navigate to the examples directory +cd packages/braintrust/examples + +# Run the example once +../../../packages/evalite/dist/bin.js run simple.eval.ts + +# Or use the evalite command if you have it installed globally +evalite run simple.eval.ts +``` + +Or from anywhere in the monorepo with pnpm: + +```bash +pnpm --filter evalite exec evalite run packages/braintrust/examples/simple.eval.ts +``` + +## What to Expect + +When you run the example **with a valid API key**, you should see: + +1. The eval running and completing (3 test cases) +2. All tests passing with 100% score +3. A message with a Braintrust URL: + ``` + View results in Braintrust: https://www.braintrust.dev/app/... + ``` +4. Click the URL to view your results in the Braintrust web UI! + +**Without an API key**, the eval will still run locally and show results, but: + +- You'll see errors about missing `BRAINTRUST_API_KEY` in stderr +- Results won't be uploaded to Braintrust +- No URL will be displayed (or it may 404) + +## What's Happening + +The example: + +- Uses `createBraintrustStorage()` in `evalite.config.ts` to configure Braintrust storage +- Runs a simple eval that tests string concatenation +- Sends results (inputs, outputs, scores, traces) to Braintrust +- Generates a URL where you can view and analyze the results + +## Files + +- `simple.eval.ts` - A simple evaluation to test the storage integration +- `evalite.config.ts` - Configuration that enables Braintrust storage +- `package.json` - Dependencies for running the example + +## Customization + +You can customize the Braintrust configuration in `evalite.config.ts`: + +```typescript +await createBraintrustStorage({ + projectName: "My Project Name", // Required + experimentName: "my-experiment", // Optional (defaults to timestamp) + apiKey: "your-api-key", // Optional (defaults to env var) + appUrl: "https://www.braintrust.dev", // Optional (custom instance) +}); +``` diff --git a/packages/braintrust/examples/evalite.config.ts b/packages/braintrust/examples/evalite.config.ts new file mode 100644 index 00000000..7683ac9e --- /dev/null +++ b/packages/braintrust/examples/evalite.config.ts @@ -0,0 +1,28 @@ +import { defineConfig } from "evalite/config"; +import { createBraintrustStorage } from "../dist/index.js"; + +/** + * Example Evalite configuration using Braintrust storage. + * + * To use this: + * 1. Set BRAINTRUST_API_KEY environment variable + * 2. Run: evalite run simple.eval.ts + * 3. Click the Braintrust URL in the output to view results + */ +export default defineConfig({ + storage: async () => { + return await createBraintrustStorage({ + // Required: Your Braintrust project name + projectName: "@evalite/braintrust Example", + + // Optional: Custom experiment name (defaults to timestamp) + experimentName: `example-run-${new Date().toISOString().split("T")[0]}`, + + // Optional: API key (defaults to BRAINTRUST_API_KEY env var) + // apiKey: process.env.BRAINTRUST_API_KEY, + + // Optional: Custom Braintrust app URL + // appUrl: "https://www.braintrust.dev", + }); + }, +}); diff --git a/packages/braintrust/examples/simple.eval.ts b/packages/braintrust/examples/simple.eval.ts new file mode 100644 index 00000000..293753c0 --- /dev/null +++ b/packages/braintrust/examples/simple.eval.ts @@ -0,0 +1,25 @@ +import { evalite } from "evalite"; + +/** + * Simple example demonstrating Braintrust storage integration. + * This eval tests a basic string concatenation task. + */ +evalite("Braintrust Storage Example", { + data: () => [ + { input: "Hello", expected: "Hello, World!" }, + { input: "Goodbye", expected: "Goodbye, World!" }, + { input: "Testing", expected: "Testing, World!" }, + ], + task: async (input) => { + // Simulate a simple LLM task + return `${input}, World!`; + }, + scorers: [ + (input, output, expected) => { + return { + name: "exact_match", + score: output === expected ? 1 : 0, + }; + }, + ], +}); diff --git a/packages/braintrust/package.json b/packages/braintrust/package.json new file mode 100644 index 00000000..8598d302 --- /dev/null +++ b/packages/braintrust/package.json @@ -0,0 +1,33 @@ +{ + "name": "@evalite/braintrust", + "version": "1.0.0-beta.14", + "type": "module", + "description": "Braintrust storage backend for Evalite", + "homepage": "https://evalite.dev", + "license": "MIT", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "./dist/*" + ], + "exports": { + ".": "./dist/index.js" + }, + "scripts": { + "dev": "tsc -w", + "build": "tsc", + "test": "vitest run", + "typecheck": "tsc" + }, + "dependencies": { + "braintrust": "^0.4.9" + }, + "peerDependencies": { + "evalite": "^1.0.0-beta.14" + }, + "devDependencies": { + "evalite": "workspace:*", + "typescript": "5.6.2", + "vitest": "^2.1.8" + } +} diff --git a/packages/braintrust/src/index.ts b/packages/braintrust/src/index.ts new file mode 100644 index 00000000..b4c65a71 --- /dev/null +++ b/packages/braintrust/src/index.ts @@ -0,0 +1,578 @@ +import type { Evalite } from "evalite"; +import { + init, + currentSpan, + _internalGetGlobalState, + type Experiment, + type Span, +} from "braintrust"; + +interface BraintrustStorageOptions { + /** + * The Braintrust project name (required) + */ + projectName: string; + /** + * Optional experiment name. If not provided, a timestamp will be used. + */ + experimentName?: string; + /** + * Optional API key. If not provided, will use BRAINTRUST_API_KEY env var. + */ + apiKey?: string; + /** + * Optional app URL for Braintrust (defaults to https://www.braintrust.dev) + */ + appUrl?: string; +} + +/** + * In-memory entity storage for local queries. + * Braintrust doesn't provide direct query APIs during logging, + * so we maintain a local cache of entities. + */ +interface EntityStore { + runs: Map; + suites: Map; + evals: Map; + scores: Map; + traces: Map; + cache: Map; +} + +/** + * Map Evalite entity IDs to Braintrust Span objects + */ +interface SpanMap { + evals: Map; // eval ID -> Braintrust span + traces: Map; // trace ID -> Braintrust span +} + +export class BraintrustStorage implements Evalite.Storage { + private experiment: Experiment | null = null; + private experimentUrl: string | null = null; + private options: BraintrustStorageOptions; + private entityStore: EntityStore; + private spanMap: SpanMap; + private nextId = { + run: 1, + suite: 1, + eval: 1, + score: 1, + trace: 1, + }; + + private constructor(options: BraintrustStorageOptions) { + this.options = options; + this.entityStore = { + runs: new Map(), + suites: new Map(), + evals: new Map(), + scores: new Map(), + traces: new Map(), + cache: new Map(), + }; + this.spanMap = { + evals: new Map(), + traces: new Map(), + }; + } + + static create(options: BraintrustStorageOptions): BraintrustStorage { + return new BraintrustStorage(options); + } + + /** + * Get the Braintrust experiment URL for viewing results + */ + getExperimentUrl(): string | null { + return this.experimentUrl; + } + + /** + * Initialize the Braintrust experiment (lazy initialization on first run) + */ + private async initExperiment(): Promise { + if (!this.experiment) { + const experimentName = + this.options.experimentName || `evalite-${new Date().toISOString()}`; + + this.experiment = init(this.options.projectName, { + experiment: experimentName, + apiKey: this.options.apiKey, + appUrl: this.options.appUrl, + }); + + // Get the experiment URL from Braintrust's summarize() method + // This ensures we use the correct URL format that Braintrust expects + try { + const summary = await this.experiment.summarize({ + summarizeScores: false, // We only need the URL, not scores + }); + this.experimentUrl = summary.experimentUrl || null; + } catch (e) { + // If summarize fails (e.g., no API key), don't set URL + console.warn("Could not get experiment URL from Braintrust:", e); + this.experimentUrl = null; + } + } + + return this.experiment; + } + + runs = { + create: async ( + opts: Evalite.Storage.Runs.CreateOpts + ): Promise => { + // Initialize Braintrust experiment when first run is created + await this.initExperiment(); + + const run: Evalite.Storage.Entities.Run = { + id: this.nextId.run++, + runType: opts.runType, + created_at: new Date().toISOString(), + }; + + this.entityStore.runs.set(run.id, run); + return run; + }, + + getMany: async ( + opts?: Evalite.Storage.Runs.GetManyOpts + ): Promise => { + let runs = Array.from(this.entityStore.runs.values()); + + // Apply filters + if (opts?.ids && opts.ids.length > 0) { + runs = runs.filter((r) => opts.ids!.includes(r.id)); + } + + if (opts?.runType) { + runs = runs.filter((r) => r.runType === opts.runType); + } + + if (opts?.createdAt) { + runs = runs.filter((r) => r.created_at === opts.createdAt); + } + + if (opts?.createdAfter) { + runs = runs.filter((r) => r.created_at > opts.createdAfter!); + } + + if (opts?.createdBefore) { + runs = runs.filter((r) => r.created_at < opts.createdBefore!); + } + + // Apply sorting + const orderBy = opts?.orderBy ?? "created_at"; + const orderDirection = opts?.orderDirection ?? "desc"; + runs.sort((a, b) => { + const aVal = a[orderBy]; + const bVal = b[orderBy]; + const comparison = aVal > bVal ? 1 : aVal < bVal ? -1 : 0; + return orderDirection === "asc" ? comparison : -comparison; + }); + + // Apply limit + if (opts?.limit) { + runs = runs.slice(0, opts.limit); + } + + return runs; + }, + }; + + suites = { + create: async ( + opts: Evalite.Storage.Suites.CreateOpts + ): Promise => { + // Initialize experiment but don't create a span for the suite + // Suites are just organizational units in Evalite - each eval will be a row + await this.initExperiment(); + + const suite: Evalite.Storage.Entities.Suite = { + id: this.nextId.suite++, + run_id: opts.runId, + name: opts.name, + status: "running", + filepath: opts.filepath, + duration: 0, + created_at: new Date().toISOString(), + variant_name: opts.variantName, + variant_group: opts.variantGroup, + }; + + this.entityStore.suites.set(suite.id, suite); + return suite; + }, + + update: async ( + opts: Evalite.Storage.Suites.UpdateOpts + ): Promise => { + const suite = this.entityStore.suites.get(opts.id); + if (!suite) { + throw new Error(`Suite with id ${opts.id} not found`); + } + + suite.status = opts.status; + this.entityStore.suites.set(opts.id, suite); + + return suite; + }, + + getMany: async ( + opts?: Evalite.Storage.Suites.GetManyOpts + ): Promise => { + let suites = Array.from(this.entityStore.suites.values()); + + // Apply filters + if (opts?.ids && opts.ids.length > 0) { + suites = suites.filter((s) => opts.ids!.includes(s.id)); + } + + if (opts?.runIds && opts.runIds.length > 0) { + suites = suites.filter((s) => opts.runIds!.includes(s.run_id)); + } + + if (opts?.name) { + suites = suites.filter((s) => s.name === opts.name); + } + + if (opts?.statuses && opts.statuses.length > 0) { + suites = suites.filter((s) => opts.statuses!.includes(s.status)); + } + + if (opts?.createdAt) { + suites = suites.filter((s) => s.created_at === opts.createdAt); + } + + if (opts?.createdAfter) { + suites = suites.filter((s) => s.created_at > opts.createdAfter!); + } + + if (opts?.createdBefore) { + suites = suites.filter((s) => s.created_at < opts.createdBefore!); + } + + // Apply sorting + const orderBy = opts?.orderBy ?? "created_at"; + const orderDirection = opts?.orderDirection ?? "desc"; + suites.sort((a, b) => { + const aVal = a[orderBy]; + const bVal = b[orderBy]; + const comparison = aVal > bVal ? 1 : aVal < bVal ? -1 : 0; + return orderDirection === "asc" ? comparison : -comparison; + }); + + // Apply limit + if (opts?.limit) { + suites = suites.slice(0, opts.limit); + } + + return suites; + }, + }; + + evals = { + create: async ( + opts: Evalite.Storage.Evals.CreateOpts + ): Promise => { + const experiment = await this.initExperiment(); + + const eval_: Evalite.Storage.Entities.Eval = { + id: this.nextId.eval++, + suite_id: opts.suiteId, + duration: opts.duration, + input: opts.input, + output: opts.output, + expected: opts.expected, + created_at: new Date().toISOString(), + col_order: opts.order, + status: opts.status, + rendered_columns: opts.renderedColumns, + trial_index: opts.trialIndex, + }; + + // Get the suite to include its metadata + const suite = this.entityStore.suites.get(opts.suiteId); + + // Create a top-level span for each eval (test case) + // Each eval becomes a row in the Braintrust experiment + const span = experiment.startSpan({ + name: suite?.name + ? `${suite.name} - Test ${opts.order}` + : `Test ${opts.order}`, + type: "eval", + spanAttributes: { + // Include suite metadata so we can filter/group by suite in Braintrust + suite_name: suite?.name, + filepath: suite?.filepath, + variant_name: suite?.variant_name, + variant_group: suite?.variant_group, + trial_index: opts.trialIndex, + test_index: opts.order, + }, + }); + + // Store the span object + this.spanMap.evals.set(eval_.id, span); + + this.entityStore.evals.set(eval_.id, eval_); + return eval_; + }, + + update: async ( + opts: Evalite.Storage.Evals.UpdateOpts + ): Promise => { + const experiment = await this.initExperiment(); + const eval_ = this.entityStore.evals.get(opts.id); + if (!eval_) { + throw new Error(`Eval with id ${opts.id} not found`); + } + + // Update the eval entity + eval_.output = opts.output; + eval_.duration = opts.duration; + eval_.input = opts.input; + eval_.expected = opts.expected; + eval_.status = opts.status; + eval_.rendered_columns = opts.renderedColumns; + eval_.trial_index = opts.trialIndex; + + // Get the existing span and log the updated data + const span = this.spanMap.evals.get(opts.id); + if (span) { + span.log({ + input: opts.input, + output: opts.output, + expected: opts.expected, + metadata: { + duration: opts.duration, + status: opts.status, + rendered_columns: opts.renderedColumns, + }, + }); + + // End the span only if the eval is finished + if (opts.status !== "running") { + span.end(); + } + } + + this.entityStore.evals.set(opts.id, eval_); + return eval_; + }, + + getMany: async ( + opts?: Evalite.Storage.Evals.GetManyOpts + ): Promise => { + let evals = Array.from(this.entityStore.evals.values()); + + // Apply filters + if (opts?.ids && opts.ids.length > 0) { + evals = evals.filter((e) => opts.ids!.includes(e.id)); + } + + if (opts?.suiteIds && opts.suiteIds.length > 0) { + evals = evals.filter((e) => opts.suiteIds!.includes(e.suite_id)); + } + + if (opts?.order !== undefined) { + evals = evals.filter((e) => e.col_order === opts.order); + } + + if (opts?.statuses && opts.statuses.length > 0) { + evals = evals.filter((e) => opts.statuses!.includes(e.status)); + } + + // Sort by col_order + evals.sort((a, b) => a.col_order - b.col_order); + + return evals; + }, + }; + + scores = { + create: async ( + opts: Evalite.Storage.Scores.CreateOpts + ): Promise => { + const experiment = await this.initExperiment(); + + const score: Evalite.Storage.Entities.Score = { + id: this.nextId.score++, + eval_id: opts.evalId, + name: opts.name, + score: opts.score, + description: opts.description, + metadata: opts.metadata, + created_at: new Date().toISOString(), + }; + + // Get the eval's span and log the score to it + const span = this.spanMap.evals.get(opts.evalId); + if (span) { + // Add scores to the span + span.log({ + scores: { + [opts.name]: opts.score, + }, + }); + } + + this.entityStore.scores.set(score.id, score); + return score; + }, + + getMany: async ( + opts?: Evalite.Storage.Scores.GetManyOpts + ): Promise => { + let scores = Array.from(this.entityStore.scores.values()); + + // Apply filters + if (opts?.ids && opts.ids.length > 0) { + scores = scores.filter((s) => opts.ids!.includes(s.id)); + } + + if (opts?.evalIds && opts.evalIds.length > 0) { + scores = scores.filter((s) => opts.evalIds!.includes(s.eval_id)); + } + + return scores; + }, + }; + + traces = { + create: async ( + opts: Evalite.Storage.Traces.CreateOpts + ): Promise => { + const experiment = await this.initExperiment(); + + const trace: Evalite.Storage.Entities.Trace = { + id: this.nextId.trace++, + eval_id: opts.evalId, + input: opts.input, + output: opts.output, + start_time: opts.start, + end_time: opts.end, + input_tokens: opts.inputTokens, + output_tokens: opts.outputTokens, + total_tokens: opts.totalTokens, + col_order: opts.order, + }; + + // Get the eval's span as parent + const parentSpan = this.spanMap.evals.get(opts.evalId); + + // Create a nested span for the trace + const parent = parentSpan ? await parentSpan.export() : undefined; + const span = experiment.startSpan({ + name: "LLM Call", + type: "llm", + parent, + // Convert performance.now() timestamp to Unix epoch seconds + startTime: (performance.timeOrigin + opts.start) / 1000, + }); + + span.log({ + input: opts.input, + output: opts.output, + metrics: { + tokens: opts.totalTokens, + prompt_tokens: opts.inputTokens, + completion_tokens: opts.outputTokens, + }, + }); + + this.spanMap.traces.set(trace.id, span); + // Convert performance.now() timestamp to Unix epoch seconds + span.end({ endTime: (performance.timeOrigin + opts.end) / 1000 }); + + this.entityStore.traces.set(trace.id, trace); + return trace; + }, + + getMany: async ( + opts?: Evalite.Storage.Traces.GetManyOpts + ): Promise => { + let traces = Array.from(this.entityStore.traces.values()); + + // Apply filters + if (opts?.ids && opts.ids.length > 0) { + traces = traces.filter((t) => opts.ids!.includes(t.id)); + } + + if (opts?.evalIds && opts.evalIds.length > 0) { + traces = traces.filter((t) => opts.evalIds!.includes(t.eval_id)); + } + + // Sort by col_order + traces.sort((a, b) => a.col_order - b.col_order); + + return traces; + }, + }; + + cache = { + get: async ( + keyHash: string + ): Promise<{ value: unknown; duration: number } | null> => { + const entry = this.entityStore.cache.get(keyHash); + if (!entry) { + return null; + } + + // Check if cache is expired (older than 24 hours) + const oneDayAgo = Date.now() - 24 * 60 * 60 * 1000; + if (entry.created_at < oneDayAgo) { + this.entityStore.cache.delete(keyHash); + return null; + } + + return { + value: entry.value, + duration: entry.duration, + }; + }, + + set: async ( + keyHash: string, + data: { value: unknown; duration: number } + ): Promise => { + this.entityStore.cache.set(keyHash, { + value: data.value, + duration: data.duration, + created_at: Date.now(), + }); + }, + + delete: async (keyHash: string): Promise => { + this.entityStore.cache.delete(keyHash); + }, + + clear: async (): Promise => { + this.entityStore.cache.clear(); + }, + }; + + async close(): Promise { + if (this.experiment) { + // Flush any pending logs to Braintrust + await this.experiment.flush(); + } + } + + async [Symbol.asyncDispose](): Promise { + await this.close(); + } +} + +/** + * Create a new Braintrust storage backend + * @param options - Configuration options for Braintrust storage + * @returns A new BraintrustStorage instance + */ +export const createBraintrustStorage = async ( + options: BraintrustStorageOptions +): Promise => { + return BraintrustStorage.create(options); +}; diff --git a/packages/braintrust/tsconfig.json b/packages/braintrust/tsconfig.json new file mode 100644 index 00000000..0b60d3ac --- /dev/null +++ b/packages/braintrust/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "@total-typescript/tsconfig/tsc/no-dom/library", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src"] +} diff --git a/packages/evalite/src/run-evalite.ts b/packages/evalite/src/run-evalite.ts index a6b28709..9bb7dce9 100644 --- a/packages/evalite/src/run-evalite.ts +++ b/packages/evalite/src/run-evalite.ts @@ -389,6 +389,19 @@ export const runEvalite = async (opts: { }); } + // If using Braintrust storage, output the experiment URL + // Use duck typing to check for getExperimentUrl method + if ( + storage && + typeof storage === "object" && + "getExperimentUrl" in storage + ) { + const experimentUrl = (storage as any).getExperimentUrl(); + if (experimentUrl) { + console.log(`\n View results in Braintrust: ${experimentUrl}\n`); + } + } + if (typeof exitCode === "number" && opts.mode === "run-once-and-exit") { process.exit(exitCode); } diff --git a/packages/example/braintrust-test.eval.ts b/packages/example/braintrust-test.eval.ts new file mode 100644 index 00000000..cf4cd29e --- /dev/null +++ b/packages/example/braintrust-test.eval.ts @@ -0,0 +1,24 @@ +import { evalite } from "evalite"; + +/** + * Simple test eval to validate Braintrust storage integration + */ +evalite("Braintrust Storage Test", { + data: () => [ + { input: "Hello", expected: "Hello, World!" }, + { input: "Goodbye", expected: "Goodbye, World!" }, + { input: "Test", expected: "Test, World!" }, + ], + task: async (input) => { + // Simple task that adds ", World!" to the input + return `${input}, World!`; + }, + scorers: [ + (input, output, expected) => { + return { + name: "exact_match", + score: output === expected ? 1 : 0, + }; + }, + ], +}); diff --git a/packages/example/evalite.config.braintrust.ts b/packages/example/evalite.config.braintrust.ts new file mode 100644 index 00000000..ed53b540 --- /dev/null +++ b/packages/example/evalite.config.braintrust.ts @@ -0,0 +1,38 @@ +import { defineConfig } from "evalite/config"; +import { createBraintrustStorage } from "@evalite/braintrust"; + +/** + * Example configuration for using Braintrust as the storage backend. + * + * To use this configuration: + * 1. Install the Braintrust package: `pnpm add @evalite/braintrust` + * 2. Rename this file to `evalite.config.ts` (or use --config flag) + * 3. Set the BRAINTRUST_API_KEY environment variable + * 4. Run evalite as normal + * + * Your evaluation results will be stored in Braintrust and you'll + * receive a URL to view them in the Braintrust web UI. + */ +export default defineConfig({ + storage: async () => { + return await createBraintrustStorage({ + // Required: Your Braintrust project name + projectName: "Evalite Example Project", + + // Optional: Specific experiment name + // If not provided, a timestamp will be used + experimentName: `evalite-run-${new Date().toISOString().split("T")[0]}`, + + // Optional: API key (defaults to BRAINTRUST_API_KEY env var) + // apiKey: process.env.BRAINTRUST_API_KEY, + + // Optional: Custom Braintrust API URL + // baseUrl: "https://api.braintrust.dev", + }); + }, + + // Other Evalite configuration options work as normal + scoreThreshold: 80, + testTimeout: 30000, + maxConcurrency: 5, +}); diff --git a/packages/example/evalite.config.sqlite-backup.ts b/packages/example/evalite.config.sqlite-backup.ts new file mode 100644 index 00000000..27fce45d --- /dev/null +++ b/packages/example/evalite.config.sqlite-backup.ts @@ -0,0 +1,5 @@ +import { defineConfig } from "evalite/config"; + +export default defineConfig({ + // .env files are now loaded automatically! +}); diff --git a/packages/example/evalite.config.ts b/packages/example/evalite.config.ts index 27fce45d..a38a819c 100644 --- a/packages/example/evalite.config.ts +++ b/packages/example/evalite.config.ts @@ -1,5 +1,11 @@ import { defineConfig } from "evalite/config"; +import { createBraintrustStorage } from "@evalite/braintrust"; export default defineConfig({ - // .env files are now loaded automatically! + storage: async () => { + return await createBraintrustStorage({ + projectName: "Evalite Test Project", + experimentName: `test-run-${Date.now()}`, + }); + }, }); diff --git a/packages/example/package.json b/packages/example/package.json index 3e05f516..7a743541 100644 --- a/packages/example/package.json +++ b/packages/example/package.json @@ -12,6 +12,7 @@ "dependencies": { "unstorage": "^1.17.1", "evalite": "workspace:*", + "@evalite/braintrust": "workspace:*", "ai": "^5.0.59", "@ai-sdk/openai": "^2.0.42", "@ai-sdk/provider": "^2.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cf550788..5cbfc3d5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -200,6 +200,22 @@ importers: specifier: ^4.2.1 version: 4.3.2(typescript@5.6.2)(vite@7.1.12(@types/node@22.7.7)(jiti@2.6.1)(lightningcss@1.30.1)(terser@5.31.5)(tsx@4.19.3)(yaml@2.8.1)) + packages/braintrust: + dependencies: + braintrust: + specifier: ^0.4.9 + version: 0.4.9(zod@4.1.12) + devDependencies: + evalite: + specifier: workspace:* + version: link:../evalite + typescript: + specifier: 5.6.2 + version: 5.6.2 + vitest: + specifier: ^2.1.8 + version: 2.1.9(@types/node@22.7.7)(lightningcss@1.30.1)(msw@2.11.3(@types/node@22.7.7)(typescript@5.6.2))(terser@5.31.5) + packages/evalite: dependencies: '@fastify/static': @@ -293,6 +309,9 @@ importers: '@ai-sdk/provider': specifier: ^2.0.0 version: 2.0.0 + '@evalite/braintrust': + specifier: workspace:* + version: link:../braintrust ai: specifier: ^5.0.59 version: 5.0.59(zod@3.25.76) @@ -344,6 +363,10 @@ packages: peerDependencies: zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/provider@1.1.3': + resolution: {integrity: sha512-qZMxYJ0qqX/RfnuIaab+zp8UAeJn/ygXXAffR5I4N0n1IrvA6qBsjc8hXLmBiMV2zoXlifkacF7sEFnYnjBcqg==} + engines: {node: '>=18'} + '@ai-sdk/provider@2.0.0': resolution: {integrity: sha512-6o7Y2SeO9vFKB8lArHXehNuusnpddKPk7xqL7T2/b+OvXMRIXUO1rR4wcv1hAFUAT9avGZshty3Wlua/XA7TvA==} engines: {node: '>=18'} @@ -1549,6 +1572,12 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + '@kwsites/file-exists@1.1.1': + resolution: {integrity: sha512-m9/5YGR18lIwxSFDwfE3oA7bWuq9kdau6ugN4H2rJeyhFQZcG9AgSHkQtSD15a8WvTgfz9aikZMrKPHvbpqFiw==} + + '@kwsites/promise-deferred@1.1.1': + resolution: {integrity: sha512-GaHYm+c0O9MjZRu0ongGBRbinu8gVAMd2UZjji6jVmqKtZluZnptXGWhz1E8j8D2HJ3f/yMxKAUC0b+57wncIw==} + '@lukeed/ms@2.0.2': resolution: {integrity: sha512-9I2Zn6+NJLfaGoz9jN3lpwDgAYvfGeNYdbAIjJOqzs4Tpc+VU3Jqq4IofSUBKajiDS8k9fZIg18/z13mpk1bsA==} engines: {node: '>=8'} @@ -1569,6 +1598,9 @@ packages: resolution: {integrity: sha512-2+BzZbjRO7Ct61k8fMNHEtoKjeWI9pIlHFTqBwZ5icHpqszIgEZbjb1MW5Z0+bITTCTl3gk4PDBxs9tA/csXvA==} engines: {node: '>=18'} + '@next/env@14.2.33': + resolution: {integrity: sha512-CgVHNZ1fRIlxkLhIX22flAZI/HmpDaZ8vwyJ/B0SDPTBuLZ1PJ+DWMjCHhqnExfmSQzA/PbZi8OAc7PAq2w9IA==} + '@nodelib/fs.scandir@2.1.5': resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} engines: {node: '>= 8'} @@ -2865,6 +2897,15 @@ packages: '@ungap/structured-clone@1.2.0': resolution: {integrity: sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==} + '@vercel/functions@1.6.0': + resolution: {integrity: sha512-R6FKQrYT5MZs5IE1SqeCJWxMuBdHawFcCZboKKw8p7s+6/mcd55Gx6tWmyKnQTyrSEA04NH73Tc9CbqpEle8RA==} + engines: {node: '>= 16'} + peerDependencies: + '@aws-sdk/credential-provider-web-identity': '*' + peerDependenciesMeta: + '@aws-sdk/credential-provider-web-identity': + optional: true + '@vercel/oidc@3.0.3': resolution: {integrity: sha512-yNEQvPcVrK9sIe637+I0jD6leluPxzwJKx/Haw6F4H77CdDsszUn5V3o96LPziXkSNE2B83+Z3mjqGKBK/R6Gg==} engines: {node: '>= 20'} @@ -2875,9 +2916,23 @@ packages: peerDependencies: vite: ^4.2.0 || ^5.0.0 || ^6.0.0 + '@vitest/expect@2.1.9': + resolution: {integrity: sha512-UJCIkTBenHeKT1TTlKMJWy1laZewsRIzYighyYiJKZreqtdxSos/S1t+ktRMQWu2CKqaarrkeszJx1cgC5tGZw==} + '@vitest/expect@4.0.1': resolution: {integrity: sha512-KtvGLN/IWoZfg68JF2q/zbDEo+UJTWnc7suYJ8RF+ZTBeBcBz4NIOJDxO4Q3bEY9GsOYhgy5cOevcVPFh4+V7g==} + '@vitest/mocker@2.1.9': + resolution: {integrity: sha512-tVL6uJgoUdi6icpxmdrn5YNo3g3Dxv+IHJBr0GXHaEdTcw3F+cPKnsXFhli6nO+f/6SDKPHEK1UN+k+TQv0Ehg==} + peerDependencies: + msw: ^2.4.9 + vite: ^5.0.0 + peerDependenciesMeta: + msw: + optional: true + vite: + optional: true + '@vitest/mocker@4.0.1': resolution: {integrity: sha512-fwmvg8YvwSAE41Hyhul7dL4UzPhG+k2VaZCcL+aHagLx4qlNQgKYTw7coF4YdjAxSBBt0b408gQFYMX1Qeqweg==} peerDependencies: @@ -2889,24 +2944,43 @@ packages: vite: optional: true + '@vitest/pretty-format@2.1.9': + resolution: {integrity: sha512-KhRIdGV2U9HOUzxfiHmY8IFHTdqtOhIzCpd8WRdJiE7D/HUcZVD0EgQCVjm+Q9gkUXWgBvMmTtZgIG48wq7sOQ==} + '@vitest/pretty-format@4.0.1': resolution: {integrity: sha512-6nq3JY/zQ91+oX1vd4fajiVNyA/HMhaF9cOw5P9cQi6ML7PRi7ilVaQ77PulF+4kvUKr9bcLm9GoAtwlVFbGzw==} + '@vitest/runner@2.1.9': + resolution: {integrity: sha512-ZXSSqTFIrzduD63btIfEyOmNcBmQvgOVsPNPe0jYtESiXkhd8u2erDLnMxmGrDCwHCCHE7hxwRDCT3pt0esT4g==} + '@vitest/runner@4.0.1': resolution: {integrity: sha512-nxUoWmw7ZX2OiSNwolJeSOOzrrR/o79wRTwP7HhiW/lDFwQHtWMj9snMhrdvccFqanvI8897E81eXjgDbrRvqA==} + '@vitest/snapshot@2.1.9': + resolution: {integrity: sha512-oBO82rEjsxLNJincVhLhaxxZdEtV0EFHMK5Kmx5sJ6H9L183dHECjiefOAdnqpIgT5eZwT04PoggUnW88vOBNQ==} + '@vitest/snapshot@4.0.1': resolution: {integrity: sha512-CvfsEWutEIN/Z9ScXYup7YwlPeK9JICrV7FN9p3pVytsyh+aCHAH0PUi//YlTiQ7T8qYxJYpUrAwZL9XqmZ5ZA==} + '@vitest/spy@2.1.9': + resolution: {integrity: sha512-E1B35FwzXXTs9FHNK6bDszs7mtydNi5MIfUWpceJ8Xbfb1gBMscAnwLbEu+B44ed6W3XjL9/ehLPHR1fkf1KLQ==} + '@vitest/spy@4.0.1': resolution: {integrity: sha512-Hj0/TBQ2EN72wDpfKiUf63mRCkE0ZiSGXGeDDvW9T3LBKVVApItd0GyQLDBIe03kWbyK9gOTEbJVVWthcLFzCg==} + '@vitest/utils@2.1.9': + resolution: {integrity: sha512-v0psaMSkNJ3A2NMrUEHFRzJtDPFn+/VWZ5WxImB21T9fjucJRmS7xCS3ppEnARb9y11OAzaD+P2Ps+b+BGX5iQ==} + '@vitest/utils@4.0.1': resolution: {integrity: sha512-uRrACgpIz5sxuT87ml7xhh7EdKtW8k0N9oSFVBPl8gHB/JfLObLe9dXO6ZrsNN55FzciGIRqIEILgTQvg1eNHw==} abstract-logging@2.0.1: resolution: {integrity: sha512-2BjRTZxTPvheOvGbBslFSYOUkr+SjPtOnrLP33f+VIWLzezQpZcqVg7ja3L4dBXmzzgwT+a029jRx5PCi3JuiA==} + accepts@1.3.8: + resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==} + engines: {node: '>= 0.6'} + acorn-jsx@5.3.2: resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==} peerDependencies: @@ -3000,6 +3074,9 @@ packages: resolution: {integrity: sha512-COROpnaoap1E2F000S62r6A60uHZnmlvomhfyT2DlTcrY1OrBKn2UhH7qn5wTC9zMvD0AY7csdPSNwKP+7WiQw==} engines: {node: '>= 0.4'} + array-flatten@1.1.1: + resolution: {integrity: sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==} + array-iterate@2.0.1: resolution: {integrity: sha512-I1jXZMjAgCMmxT4qxXfPXa6SthSoE8h6gkSI9BGGNv8mP8G/v0blc+qFnZu6K42vTOiuME596QaLO0TP3Lk0xg==} @@ -3007,6 +3084,10 @@ packages: resolution: {integrity: sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==} engines: {node: '>=8'} + assertion-error@2.0.1: + resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} + engines: {node: '>=12'} + astral-regex@2.0.0: resolution: {integrity: sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==} engines: {node: '>=8'} @@ -3099,6 +3180,10 @@ packages: bl@4.1.0: resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} + body-parser@1.20.3: + resolution: {integrity: sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==} + engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} + boolbase@1.0.0: resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==} @@ -3116,6 +3201,12 @@ packages: resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} engines: {node: '>=8'} + braintrust@0.4.9: + resolution: {integrity: sha512-Gyd2Zp/KoWZ26DV/rl+j2TzVR3TCPMFwrUOwTqTt+GN+oqxO1v/SR9pUFovc7TuJdsLzPp0pw0svm7NN10XnDw==} + hasBin: true + peerDependencies: + zod: ^3.25.34 + browserslist@4.24.2: resolution: {integrity: sha512-ZIc+Q62revdMcqC6aChtW4jz3My3klmCO1fEmINZY/8J3EpBg5/A/D0AKmBveUh6pgoeycoMkVMko84tuYS+Gg==} engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} @@ -3132,6 +3223,22 @@ packages: buffer@5.7.1: resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} + bytes@3.1.2: + resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} + engines: {node: '>= 0.8'} + + cac@6.7.14: + resolution: {integrity: sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==} + engines: {node: '>=8'} + + call-bind-apply-helpers@1.0.2: + resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} + engines: {node: '>= 0.4'} + + call-bound@1.0.4: + resolution: {integrity: sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==} + engines: {node: '>= 0.4'} + callsites@3.1.0: resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} engines: {node: '>=6'} @@ -3149,6 +3256,10 @@ packages: ccount@2.0.1: resolution: {integrity: sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==} + chai@5.3.3: + resolution: {integrity: sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==} + engines: {node: '>=18'} + chai@6.2.0: resolution: {integrity: sha512-aUTnJc/JipRzJrNADXVvpVqi6CO0dn3nx4EVPxijri+fj3LUUDyZQOgVeW54Ob3Y1Xh9Iz8f+CgaCl8v0mn9bA==} engines: {node: '>=18'} @@ -3176,6 +3287,10 @@ packages: chardet@2.1.1: resolution: {integrity: sha512-PsezH1rqdV9VvyNhxxOW32/d75r01NY7TQCmOqomRo15ZSOKbpTFVsfjghxo6JloQUCGnH4k1LGu0R4yCLlWQQ==} + check-error@2.1.1: + resolution: {integrity: sha512-OAlb+T7V4Op9OwdkjmguYRqncdlx5JiofwOAUkmTF+jNdHwzTaTs4sRAGpzLF3oOz5xAyDGrPgeIDFQmDOTiJw==} + engines: {node: '>= 16'} + chevrotain-allstar@0.3.1: resolution: {integrity: sha512-b7g+y9A0v4mxCW1qUhf3BSVPg+/NvGErk/dOkrDaHA0nQIQGAtrOjlX//9OQtRlSCy+x9rfB5N8yC71lH1nvMw==} peerDependencies: @@ -3218,6 +3333,10 @@ packages: resolution: {integrity: sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==} engines: {node: '>=18'} + cli-progress@3.12.0: + resolution: {integrity: sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A==} + engines: {node: '>=4'} + cli-spinners@2.9.2: resolution: {integrity: sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==} engines: {node: '>=6'} @@ -3292,12 +3411,23 @@ packages: resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==} engines: {node: '>= 0.6'} + content-type@1.0.5: + resolution: {integrity: sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==} + engines: {node: '>= 0.6'} + convert-source-map@2.0.0: resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} cookie-es@1.2.2: resolution: {integrity: sha512-+W7VmiVINB+ywl1HGXJXmrqkOhpKrIiVZV6tQuV54ZyQC7MMuBt81Vc336GMLoHBq5hV/F9eXgt5Mnx0Rha5Fg==} + cookie-signature@1.0.6: + resolution: {integrity: sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==} + + cookie@0.7.1: + resolution: {integrity: sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==} + engines: {node: '>= 0.6'} + cookie@0.7.2: resolution: {integrity: sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==} engines: {node: '>= 0.6'} @@ -3306,6 +3436,10 @@ packages: resolution: {integrity: sha512-Xd8lFX4LM9QEEwxQpF9J9NTUh8pmdJO0cyRJhFiDoLTk2eH8FXlRv2IFGYVadZpqI3j8fhNrSdKCeYPxiAhLXw==} engines: {node: '>=18'} + cors@2.8.5: + resolution: {integrity: sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==} + engines: {node: '>= 0.10'} + cose-base@1.0.3: resolution: {integrity: sha512-s9whTXInMSgAp/NVXVNuVxVKzGH2qck3aQlVHxDCdAEPgtMKwc4Wq6/QKhgdEdgbLSi9rBTAcPoRa6JpiG4ksg==} @@ -3492,6 +3626,14 @@ packages: dayjs@1.11.18: resolution: {integrity: sha512-zFBQ7WFRvVRhKcWoUh+ZA1g2HVgUbsZm9sbddh8EC5iv93sui8DVVz1Npvz+r6meo9VKfa8NyLWBsQK1VvIKPA==} + debug@2.6.9: + resolution: {integrity: sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + debug@4.3.6: resolution: {integrity: sha512-O/09Bd4Z1fBrU4VzkhFqVgpPzaGbw6Sm9FEkBT1A/YBXQFGuuSxa1dN2nxgxS34JmKXqYx8CZAwEVoJFImUXIg==} engines: {node: '>=6.0'} @@ -3529,6 +3671,10 @@ packages: resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} engines: {node: '>=10'} + deep-eql@5.0.2: + resolution: {integrity: sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==} + engines: {node: '>=6'} + deep-extend@0.6.0: resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} engines: {node: '>=4.0.0'} @@ -3553,6 +3699,10 @@ packages: destr@2.0.5: resolution: {integrity: sha512-ugFTXCtDZunbzasqBxrK93Ik/DRYsO6S/fedkWEMKqt04xZ4csmnmwGDBAb07QWNaGMAmnTIemsYZCksjATwsA==} + destroy@1.2.0: + resolution: {integrity: sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==} + engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} + detect-indent@6.1.0: resolution: {integrity: sha512-reYkTUJAZb9gUuZ2RvVCNhVHdg62RHnJ7WJl8ftMi4diZ6NWlciOzQN88pUhSELEwflJht4oQDv0F0BMlwaYtA==} engines: {node: '>=8'} @@ -3619,12 +3769,19 @@ packages: resolution: {integrity: sha512-2QF/g9/zTaPDc3BjNcVTGoBbXBgYfMTTceLaYcFJ/W9kggFUkhxD/hMEeuLKbugyef9SqAx8cpgwlIP/jinUTA==} engines: {node: '>=4'} + dunder-proto@1.0.1: + resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} + engines: {node: '>= 0.4'} + duplexify@4.1.3: resolution: {integrity: sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==} eastasianwidth@0.2.0: resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} + ee-first@1.1.1: + resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} + electron-to-chromium@1.5.141: resolution: {integrity: sha512-qS+qH9oqVYc1ooubTiB9l904WVyM6qNYxtOEEGReoZXw3xlqeYdFr5GclNzbkAufWgwWLEPoDi3d9MoRwwIjGw==} @@ -3643,6 +3800,14 @@ packages: emoji-regex@9.2.2: resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==} + encodeurl@1.0.2: + resolution: {integrity: sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==} + engines: {node: '>= 0.8'} + + encodeurl@2.0.0: + resolution: {integrity: sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==} + engines: {node: '>= 0.8'} + end-of-stream@1.4.4: resolution: {integrity: sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==} @@ -3662,12 +3827,24 @@ packages: resolution: {integrity: sha512-xUtoPkMggbz0MPyPiIWr1Kp4aeWJjDZ6SMvURhimjdZgsRuDplF5/s9hcgGhyXMhs+6vpnuoiZ2kFiu3FMnS8Q==} engines: {node: '>=18'} + es-define-property@1.0.1: + resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==} + engines: {node: '>= 0.4'} + + es-errors@1.3.0: + resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} + engines: {node: '>= 0.4'} + es-module-lexer@1.5.4: resolution: {integrity: sha512-MVNK56NiMrOwitFB7cqDwq0CQutbw+0BvLshJSse0MUNU+y1FC3bUS/AQg7oUng+/wKrrki7JfmwtVHkVfPLlw==} es-module-lexer@1.7.0: resolution: {integrity: sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==} + es-object-atoms@1.1.1: + resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==} + engines: {node: '>= 0.4'} + esast-util-from-estree@2.0.0: resolution: {integrity: sha512-4CyanoAudUSBAn5K13H4JhsMH6L9ZP7XbLVe/dKybkxMO7eDyLsT8UHl9TRNrU2Gr9nz+FovfSIjuXWJ81uVwQ==} @@ -3791,12 +3968,20 @@ packages: resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==} engines: {node: '>=0.10.0'} + etag@1.8.1: + resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} + engines: {node: '>= 0.6'} + eventemitter3@4.0.7: resolution: {integrity: sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==} eventemitter3@5.0.1: resolution: {integrity: sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==} + eventsource-parser@1.1.2: + resolution: {integrity: sha512-v0eOBUbiaFojBu2s2NPBfYUoRR9GjcDNvCXVaqEf5vVfpIAh9f8RCo4vXTP8c63QRKCFwoLpMpTdPwwhEKVgzA==} + engines: {node: '>=14.18'} + eventsource-parser@3.0.6: resolution: {integrity: sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==} engines: {node: '>=18.0.0'} @@ -3809,6 +3994,10 @@ packages: resolution: {integrity: sha512-JhFGDVJ7tmDJItKhYgJCGLOWjuK9vPxiXoUFLwLDc99NlmklilbiQJwoctZtt13+xMw91MCk/REan6MWHqDjyA==} engines: {node: '>=12.0.0'} + express@4.21.2: + resolution: {integrity: sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==} + engines: {node: '>= 0.10.0'} + expressive-code@0.38.3: resolution: {integrity: sha512-COM04AiUotHCKJgWdn7NtW2lqu8OW8owAidMpkXt1qxrZ9Q2iC7+tok/1qIn2ocGnczvr9paIySgGnEwFeEQ8Q==} @@ -3897,6 +4086,10 @@ packages: resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} engines: {node: '>=8'} + finalhandler@1.3.1: + resolution: {integrity: sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==} + engines: {node: '>= 0.8'} + find-my-way@9.1.0: resolution: {integrity: sha512-Y5jIsuYR4BwWDYYQ2A/RWWE6gD8a0FMgtU+HOq1WKku+Cwdz8M1v8wcAmRXXM1/iqtoqg06v+LjAxMYbCjViMw==} engines: {node: '>=14'} @@ -3931,9 +4124,17 @@ packages: resolution: {integrity: sha512-Ld2g8rrAyMYFXBhEqMz8ZAHBi4J4uS1i/CxGMDnjyFWddMXLVcDp051DZfu+t7+ab7Wv6SMqpWmyFIj5UbfFvg==} engines: {node: '>=14'} + forwarded@0.2.0: + resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} + engines: {node: '>= 0.6'} + fraction.js@4.3.7: resolution: {integrity: sha512-ZsDfxO51wGAXREY55a7la9LScWpwv9RxIrYABrlvOFBlH/ShPnrtsXeuUIfXKKOVicNxQ+o8JTbJvjS4M89yew==} + fresh@0.5.2: + resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==} + engines: {node: '>= 0.6'} + fs-constants@1.0.0: resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} @@ -3950,6 +4151,9 @@ packages: engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] + function-bind@1.1.2: + resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + gensync@1.0.0-beta.2: resolution: {integrity: sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==} engines: {node: '>=6.9.0'} @@ -3966,6 +4170,10 @@ packages: resolution: {integrity: sha512-QZjmEOC+IT1uk6Rx0sX22V6uHWVwbdbxf1faPqJ1QhLdGgsRGCZoyaQBm/piRdJy/D2um6hM1UP7ZEeQ4EkP+Q==} engines: {node: '>=18'} + get-intrinsic@1.3.0: + resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} + engines: {node: '>= 0.4'} + get-nonce@1.0.1: resolution: {integrity: sha512-FJhYRoDaiatfEkUK8HKlicmu/3SGFD51q3itKDGoSTysQJBnfOcxU5GxnhE1E6soB76MbT0MBtnKJuXyAx+96Q==} engines: {node: '>=6'} @@ -3974,6 +4182,10 @@ packages: resolution: {integrity: sha512-QB9NKEeDg3xxVwCCwJQ9+xycaz6pBB6iQ76wiWMl1927n0Kir6alPiP+yuiICLLU4jpMe08dXfpebuQppFA2zw==} engines: {node: '>=16'} + get-proto@1.0.1: + resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} + engines: {node: '>= 0.4'} + get-stream@9.0.1: resolution: {integrity: sha512-kVCxPF3vQM/N0B1PmoqVUqgHP+EeVjmZSQn+1oCRPxd2P21P2F19lIgbR3HBosbB1PUhOAoctJnfEn2GbN2eZA==} engines: {node: '>=18'} @@ -4031,6 +4243,10 @@ packages: peerDependencies: csstype: ^3.0.10 + gopd@1.2.0: + resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} + engines: {node: '>= 0.4'} + graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} @@ -4055,6 +4271,14 @@ packages: resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} engines: {node: '>=8'} + has-symbols@1.1.0: + resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} + engines: {node: '>= 0.4'} + + hasown@2.0.2: + resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} + engines: {node: '>= 0.4'} + hast-util-embedded@3.0.0: resolution: {integrity: sha512-naH8sld4Pe2ep03qqULEtvYr7EjrLK2QHY8KJR6RJkTUjPGObe1vnx585uzem2hGra+s1q08DZZpfgDVYRbaXA==} @@ -4164,6 +4388,10 @@ packages: i18next@23.16.8: resolution: {integrity: sha512-06r/TitrM88Mg5FdUXAKL96dJMzgqLE5dv3ryBAra4KCwD9mJ4ndOTS95ZuymIGoE+2hzfdaMak2X11/es7ZWg==} + iconv-lite@0.4.24: + resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==} + engines: {node: '>=0.10.0'} + iconv-lite@0.6.3: resolution: {integrity: sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==} engines: {node: '>=0.10.0'} @@ -4216,6 +4444,10 @@ packages: invariant@2.2.4: resolution: {integrity: sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==} + ipaddr.js@1.9.1: + resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} + engines: {node: '>= 0.10'} + ipaddr.js@2.2.0: resolution: {integrity: sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA==} engines: {node: '>= 10'} @@ -4538,6 +4770,9 @@ packages: resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==} hasBin: true + loupe@3.2.1: + resolution: {integrity: sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==} + lru-cache@10.4.3: resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} @@ -4579,6 +4814,10 @@ packages: engines: {node: '>= 20'} hasBin: true + math-intrinsics@1.1.0: + resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} + engines: {node: '>= 0.4'} + mdast-util-definitions@6.0.0: resolution: {integrity: sha512-scTllyX6pnYNZH/AIp/0ePz6s4cZtARxImwoPJ7kS42n+MnVsI4XbnG6d4ibehRIldYMWM2LD7ImQblVhUejVQ==} @@ -4636,6 +4875,13 @@ packages: mdast-util-to-string@4.0.0: resolution: {integrity: sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg==} + media-typer@0.3.0: + resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==} + engines: {node: '>= 0.6'} + + merge-descriptors@1.0.3: + resolution: {integrity: sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==} + merge2@1.4.1: resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} engines: {node: '>= 8'} @@ -4643,6 +4889,10 @@ packages: mermaid@11.12.0: resolution: {integrity: sha512-ZudVx73BwrMJfCFmSSJT84y6u5brEoV8DOItdHomNLz32uBjNrelm7mg95X7g+C6UoQH/W6mBLGDEDv73JdxBg==} + methods@1.1.2: + resolution: {integrity: sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==} + engines: {node: '>= 0.6'} + micromark-core-commonmark@2.0.2: resolution: {integrity: sha512-FKjQKbxd1cibWMM1P9N+H8TwlgGgSkWZMmfuVucLCHaYqeSvJ0hFeHsIa65pA2nYbes0f8LDHPMrd9X7Ujxg9w==} @@ -4758,6 +5008,19 @@ packages: resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==} engines: {node: '>=8.6'} + mime-db@1.52.0: + resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} + engines: {node: '>= 0.6'} + + mime-types@2.1.35: + resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} + engines: {node: '>= 0.6'} + + mime@1.6.0: + resolution: {integrity: sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==} + engines: {node: '>=4'} + hasBin: true + mime@3.0.0: resolution: {integrity: sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==} engines: {node: '>=10.0.0'} @@ -4807,6 +5070,9 @@ packages: resolution: {integrity: sha512-eu38+hdgojoyq63s+yTpN4XMBdt5l8HhMhc4VKLO9KM5caLIBvUm4thi7fFaxyTmCKeNnXZ5pAlBwCUnhA09uw==} engines: {node: '>=10'} + ms@2.0.0: + resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} + ms@2.1.2: resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==} @@ -4823,6 +5089,10 @@ packages: typescript: optional: true + mustache@4.2.0: + resolution: {integrity: sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==} + hasBin: true + mute-stream@2.0.0: resolution: {integrity: sha512-WWdIxpyjEn+FhQJQQv9aQAYlHoNVdzIzUySNV1gHUPDSdZJ3yZn7pAAbQcV7B56Mvu881q9FZV+0Vx2xC44VWA==} engines: {node: ^18.17.0 || >=20.5.0} @@ -4847,6 +5117,10 @@ packages: natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} + negotiator@0.6.3: + resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==} + engines: {node: '>= 0.6'} + neotraverse@0.6.18: resolution: {integrity: sha512-Z4SmBUweYa09+o6pG+eASabEpP6QkQ70yHj351pQoEXIs8uHbaU2DWVmzBANKgflPa47A50PtB2+NgRpQvr7vA==} engines: {node: '>= 10'} @@ -4888,6 +5162,10 @@ packages: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} + object-inspect@1.13.4: + resolution: {integrity: sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==} + engines: {node: '>= 0.4'} + ofetch@1.4.1: resolution: {integrity: sha512-QZj2DfGplQAr2oj9KzceK9Hwz6Whxazmn85yYeVuS3u9XTMOGMRx0kO95MQ+vLsj/S/NwBDMMLU5hpxvI6Tklw==} @@ -4895,6 +5173,10 @@ packages: resolution: {integrity: sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==} engines: {node: '>=14.0.0'} + on-finished@2.4.1: + resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==} + engines: {node: '>= 0.8'} + once@1.4.0: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} @@ -4991,6 +5273,10 @@ packages: parse5@7.2.1: resolution: {integrity: sha512-BuBYQYlv1ckiPdQi/ohiivi9Sagc9JG+Ozs0r7b/0iK3sKmrb0b9FdWdBbOdx6hBCM/F9Ir82ofnBhtZOjCRPQ==} + parseurl@1.3.3: + resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} + engines: {node: '>= 0.8'} + path-data-parser@0.1.0: resolution: {integrity: sha512-NOnmBpt5Y2RWbuv0LMzsayp3lVylAHLPUTut412ZA3l+C4uw4ZVkQbjShYCQ8TCpUMdPapr4YjUqLYD6v68j+w==} @@ -5006,6 +5292,9 @@ packages: resolution: {integrity: sha512-ypGJsmGtdXUOeM5u93TyeIEfEhM6s+ljAhrk5vAvSx8uyY/02OvrZnA0YNGUrPXfpJMgI1ODd3nwz8Npx4O4cg==} engines: {node: 20 || >=22} + path-to-regexp@0.1.12: + resolution: {integrity: sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==} + path-to-regexp@6.3.0: resolution: {integrity: sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==} @@ -5013,9 +5302,16 @@ packages: resolution: {integrity: sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==} engines: {node: '>=8'} + pathe@1.1.2: + resolution: {integrity: sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ==} + pathe@2.0.3: resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} + pathval@2.0.1: + resolution: {integrity: sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==} + engines: {node: '>= 14.16'} + peek-readable@5.3.1: resolution: {integrity: sha512-GVlENSDW6KHaXcd9zkZltB7tCLosKB/4Hg0fqBJkAoBgYG2Tn1xtMgXtSUuMU9AK/gCm/tTdT8mgAeF4YNeeqw==} engines: {node: '>=14.16'} @@ -5064,6 +5360,10 @@ packages: pkg-types@2.3.0: resolution: {integrity: sha512-SIqCzDRg0s9npO5XQ3tNZioRY1uK06lA41ynBC1YmFTmnY6FjUjVt6s4LoADmwoig1qqD0oK8h1p/8mlMx8Oig==} + pluralize@8.0.0: + resolution: {integrity: sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==} + engines: {node: '>=4'} + points-on-curve@0.2.0: resolution: {integrity: sha512-0mYKnYYe9ZcqMCWhUjItv/oHjvgEsfKvnUTg8sAtnHr3GVy7rGkXCb6d5cSyqrWqL4k81b9CPg3urd+T7aop3A==} @@ -5146,6 +5446,10 @@ packages: property-information@7.1.0: resolution: {integrity: sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ==} + proxy-addr@2.0.7: + resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} + engines: {node: '>= 0.10'} + pump@3.0.2: resolution: {integrity: sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==} @@ -5153,6 +5457,10 @@ packages: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'} + qs@6.13.0: + resolution: {integrity: sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==} + engines: {node: '>=0.6'} + quansync@0.2.11: resolution: {integrity: sha512-AifT7QEbW9Nri4tAwR5M/uzpBuqfZf+zwaEM/QkzEjj7NBuFD2rBuy0K3dE+8wltbezDV7JMA0WfnCPYRSYbXA==} @@ -5168,6 +5476,14 @@ packages: radix3@1.1.2: resolution: {integrity: sha512-b484I/7b8rDEdSDKckSSBA8knMpcdsXudlE/LNL639wFoHKwLbEkQFZHWEYwDC0wa0FKUcCY+GAF73Z7wxNVFA==} + range-parser@1.2.1: + resolution: {integrity: sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==} + engines: {node: '>= 0.6'} + + raw-body@2.5.2: + resolution: {integrity: sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==} + engines: {node: '>= 0.8'} + rc@1.2.8: resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} hasBin: true @@ -5472,6 +5788,10 @@ packages: engines: {node: '>=10'} hasBin: true + send@0.19.0: + resolution: {integrity: sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==} + engines: {node: '>= 0.8.0'} + seroval-plugins@1.2.1: resolution: {integrity: sha512-H5vs53+39+x4Udwp4J5rNZfgFuA+Lt+uU+09w1gYBVWomtAl98B+E9w7yC05Xc81/HgLvJdlyqJbU0fJCKCmdw==} engines: {node: '>=10'} @@ -5482,6 +5802,10 @@ packages: resolution: {integrity: sha512-yBxFFs3zmkvKNmR0pFSU//rIsYjuX418TnlDmc2weaq5XFDqDIV/NOMPBoLrbxjLH42p4UzRuXHryXh9dYcKcw==} engines: {node: '>=10'} + serve-static@1.16.2: + resolution: {integrity: sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==} + engines: {node: '>= 0.8.0'} + set-cookie-parser@2.7.1: resolution: {integrity: sha512-IOc8uWeOZgnb3ptbCURJWNjWUPcO3ZnTTdzsurqERrP6nPyv+paC55vJM0LpOlT2ne+Ix+9+CRG1MNLlyZ4GjQ==} @@ -5510,6 +5834,22 @@ packages: shiki@3.13.0: resolution: {integrity: sha512-aZW4l8Og16CokuCLf8CF8kq+KK2yOygapU5m3+hoGw0Mdosc6fPitjM+ujYarppj5ZIKGyPDPP1vqmQhr+5/0g==} + side-channel-list@1.0.0: + resolution: {integrity: sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==} + engines: {node: '>= 0.4'} + + side-channel-map@1.0.1: + resolution: {integrity: sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==} + engines: {node: '>= 0.4'} + + side-channel-weakmap@1.0.2: + resolution: {integrity: sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==} + engines: {node: '>= 0.4'} + + side-channel@1.1.0: + resolution: {integrity: sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==} + engines: {node: '>= 0.4'} + siginfo@2.0.0: resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} @@ -5523,6 +5863,9 @@ packages: simple-get@4.0.1: resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==} + simple-git@3.30.0: + resolution: {integrity: sha512-q6lxyDsCmEal/MEGhP1aVyQ3oxnagGlBDOVSIB4XUVLl1iZh0Pah6ebC9V4xBap/RfgP2WlI8EKs0WS0rMEJHg==} + simple-swizzle@0.2.2: resolution: {integrity: sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==} @@ -5547,6 +5890,10 @@ packages: resolution: {integrity: sha512-iOBWFgUX7caIZiuutICxVgX1SdxwAVFFKwt1EvMYYec/NWO5meOJ6K5uQxhrYBdQJne4KxiqZc+KptFOWFSI9w==} engines: {node: '>=18'} + slugify@1.6.6: + resolution: {integrity: sha512-h+z7HKHYXj6wJU+AnS/+IH8Uh9fdcX1Lrhg1/VMdf9PwoBQXFcXiAdsy2tSK0P6gKwJLXp02r90ahUCqHk9rrw==} + engines: {node: '>=8.0.0'} + solid-js@1.9.5: resolution: {integrity: sha512-ogI3DaFcyn6UhYhrgcyRAMbu/buBJitYQASZz5WzfQVPP10RD2AbCoRZ517psnezrasyCbWzIxZ6kVqet768xw==} @@ -5751,10 +6098,22 @@ packages: resolution: {integrity: sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==} engines: {node: '>=12.0.0'} + tinypool@1.1.1: + resolution: {integrity: sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==} + engines: {node: ^18.0.0 || >=20.0.0} + + tinyrainbow@1.2.0: + resolution: {integrity: sha512-weEDEq7Z5eTHPDh4xjX789+fHfF+P8boiFB+0vbWzpbnbsEr/GRaohi/uMKxg8RZMXnl1ItAi/IUHWMsjDV7kQ==} + engines: {node: '>=14.0.0'} + tinyrainbow@3.0.3: resolution: {integrity: sha512-PSkbLUoxOFRzJYjjxHJt9xro7D+iilgMX/C9lawzVuYiIdcihh9DXmVibBe8lmcFrRi/VzlPjBxbN7rH24q8/Q==} engines: {node: '>=14.0.0'} + tinyspy@3.0.2: + resolution: {integrity: sha512-n1cw8k1k0x4pgA2+9XrOkFydTerNcJ1zWCO5Nn9scWHTD+5tp8dghT2x1uduQePZTZgd3Tupf+x9BxJjeJi77Q==} + engines: {node: '>=14.0.0'} + tldts-core@7.0.17: resolution: {integrity: sha512-DieYoGrP78PWKsrXr8MZwtQ7GLCUeLxihtjC1jZsW1DnvSMdKPitJSe8OSYDM2u5H6g3kWJZpePqkp43TfLh0g==} @@ -5839,6 +6198,10 @@ packages: resolution: {integrity: sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==} engines: {node: '>=16'} + type-is@1.6.18: + resolution: {integrity: sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==} + engines: {node: '>= 0.6'} + typescript-eslint@8.46.2: resolution: {integrity: sha512-vbw8bOmiuYNdzzV3lsiWv6sRwjyuKJMQqWulBOU7M0RrxedXledX8G8kBbQeiOYDnTfiXz0Y4081E1QMNB6iQg==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -5906,6 +6269,10 @@ packages: resolution: {integrity: sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==} engines: {node: '>= 4.0.0'} + unpipe@1.0.0: + resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} + engines: {node: '>= 0.8'} + unplugin@2.1.2: resolution: {integrity: sha512-Q3LU0e4zxKfRko1wMV2HmP8lB9KWislY7hxXpxd+lGx0PRInE4vhMBVEZwpdVYHvtqzhSrzuIfErsob6bQfCzw==} engines: {node: '>=18.12.0'} @@ -6018,10 +6385,22 @@ packages: util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + utils-merge@1.0.1: + resolution: {integrity: sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==} + engines: {node: '>= 0.4.0'} + uuid@11.1.0: resolution: {integrity: sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==} hasBin: true + uuid@9.0.1: + resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==} + hasBin: true + + vary@1.1.2: + resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} + engines: {node: '>= 0.8'} + vfile-location@5.0.3: resolution: {integrity: sha512-5yXvWDEgqeiYiBe1lbxYF7UMAIm/IcopxMHrMQDq3nvKcjPKIhZklUKL+AE7J7uApI4kwe2snsK+eI6UTj9EHg==} @@ -6034,6 +6413,11 @@ packages: victory-vendor@36.9.2: resolution: {integrity: sha512-PnpQQMuxlwYdocC8fIJqVXvkeViHYzotI+NJrCuav0ZYFoq912ZHBk3mCeuj+5/VpodOjPe1z0Fk2ihgzlXqjQ==} + vite-node@2.1.9: + resolution: {integrity: sha512-AM9aQ/IPrW/6ENLQg3AGY4K1N2TGZdR5e4gu/MmmR2xR3Ll1+dib+nook92g4TV3PXVyeyxdWwtaCAiUL0hMxA==} + engines: {node: ^18.0.0 || >=20.0.0} + hasBin: true + vite-tsconfig-paths@4.3.2: resolution: {integrity: sha512-0Vd/a6po6Q+86rPlntHye7F31zA2URZMbH8M3saAZ/xR9QoGN/L21bxEGfXdWmFdNkqPpRdxFT7nmNe12e9/uA==} peerDependencies: @@ -6161,6 +6545,31 @@ packages: vite: optional: true + vitest@2.1.9: + resolution: {integrity: sha512-MSmPM9REYqDGBI8439mA4mWhV5sKmDlBKWIYbA3lRb2PTHACE0mgKwA8yQ2xq9vxDTuk4iPrECBAEW2aoFXY0Q==} + engines: {node: ^18.0.0 || >=20.0.0} + hasBin: true + peerDependencies: + '@edge-runtime/vm': '*' + '@types/node': ^18.0.0 || >=20.0.0 + '@vitest/browser': 2.1.9 + '@vitest/ui': 2.1.9 + happy-dom: '*' + jsdom: '*' + peerDependenciesMeta: + '@edge-runtime/vm': + optional: true + '@types/node': + optional: true + '@vitest/browser': + optional: true + '@vitest/ui': + optional: true + happy-dom: + optional: true + jsdom: + optional: true + vitest@4.0.1: resolution: {integrity: sha512-4rwTfUNF0MExMZBiNirkzZpeyUZGOs3JD76N2qHNP9i6w6/bff7MRv2I9yFJKd1ICxzn2igpra+E4t9o2EfQhw==} engines: {node: ^20.0.0 || ^22.0.0 || >=24.0.0} @@ -6400,6 +6809,10 @@ snapshots: eventsource-parser: 3.0.6 zod: 3.24.1 + '@ai-sdk/provider@1.1.3': + dependencies: + json-schema: 0.4.0 + '@ai-sdk/provider@2.0.0': dependencies: json-schema: 0.4.0 @@ -7623,6 +8036,14 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 + '@kwsites/file-exists@1.1.1': + dependencies: + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + + '@kwsites/promise-deferred@1.1.1': {} + '@lukeed/ms@2.0.2': {} '@manypkg/find-root@1.1.0': @@ -7685,6 +8106,8 @@ snapshots: strict-event-emitter: 0.5.1 optional: true + '@next/env@14.2.33': {} + '@nodelib/fs.scandir@2.1.5': dependencies: '@nodelib/fs.stat': 2.0.5 @@ -8934,6 +9357,8 @@ snapshots: '@ungap/structured-clone@1.2.0': {} + '@vercel/functions@1.6.0': {} + '@vercel/oidc@3.0.3': {} '@vitejs/plugin-react@4.3.4(vite@7.1.12(@types/node@22.7.7)(jiti@2.6.1)(lightningcss@1.30.1)(terser@5.31.5)(tsx@4.19.3)(yaml@2.8.1))': @@ -8947,6 +9372,13 @@ snapshots: transitivePeerDependencies: - supports-color + '@vitest/expect@2.1.9': + dependencies: + '@vitest/spy': 2.1.9 + '@vitest/utils': 2.1.9 + chai: 5.3.3 + tinyrainbow: 1.2.0 + '@vitest/expect@4.0.1': dependencies: '@standard-schema/spec': 1.0.0 @@ -8956,6 +9388,15 @@ snapshots: chai: 6.2.0 tinyrainbow: 3.0.3 + '@vitest/mocker@2.1.9(msw@2.11.3(@types/node@22.7.7)(typescript@5.6.2))(vite@5.4.11(@types/node@22.7.7)(lightningcss@1.30.1)(terser@5.31.5))': + dependencies: + '@vitest/spy': 2.1.9 + estree-walker: 3.0.3 + magic-string: 0.30.19 + optionalDependencies: + msw: 2.11.3(@types/node@22.7.7)(typescript@5.6.2) + vite: 5.4.11(@types/node@22.7.7)(lightningcss@1.30.1)(terser@5.31.5) + '@vitest/mocker@4.0.1(msw@2.11.3(@types/node@22.7.7)(typescript@5.6.2))(vite@7.1.10(@types/node@22.7.7)(jiti@2.6.1)(lightningcss@1.30.1)(terser@5.31.5)(tsx@4.19.0)(yaml@2.8.1))': dependencies: '@vitest/spy': 4.0.1 @@ -8965,23 +9406,48 @@ snapshots: msw: 2.11.3(@types/node@22.7.7)(typescript@5.6.2) vite: 7.1.10(@types/node@22.7.7)(jiti@2.6.1)(lightningcss@1.30.1)(terser@5.31.5)(tsx@4.19.0)(yaml@2.8.1) + '@vitest/pretty-format@2.1.9': + dependencies: + tinyrainbow: 1.2.0 + '@vitest/pretty-format@4.0.1': dependencies: tinyrainbow: 3.0.3 + '@vitest/runner@2.1.9': + dependencies: + '@vitest/utils': 2.1.9 + pathe: 1.1.2 + '@vitest/runner@4.0.1': dependencies: '@vitest/utils': 4.0.1 pathe: 2.0.3 + '@vitest/snapshot@2.1.9': + dependencies: + '@vitest/pretty-format': 2.1.9 + magic-string: 0.30.19 + pathe: 1.1.2 + '@vitest/snapshot@4.0.1': dependencies: '@vitest/pretty-format': 4.0.1 magic-string: 0.30.19 pathe: 2.0.3 + '@vitest/spy@2.1.9': + dependencies: + tinyspy: 3.0.2 + '@vitest/spy@4.0.1': {} + '@vitest/utils@2.1.9': + dependencies: + '@vitest/pretty-format': 2.1.9 + loupe: 3.2.1 + tinyrainbow: 1.2.0 + '@vitest/utils@4.0.1': dependencies: '@vitest/pretty-format': 4.0.1 @@ -8989,6 +9455,11 @@ snapshots: abstract-logging@2.0.1: {} + accepts@1.3.8: + dependencies: + mime-types: 2.1.35 + negotiator: 0.6.3 + acorn-jsx@5.3.2(acorn@8.14.0): dependencies: acorn: 8.14.0 @@ -9084,10 +9555,14 @@ snapshots: aria-query@5.3.2: {} + array-flatten@1.1.1: {} + array-iterate@2.0.1: {} array-union@2.1.0: {} + assertion-error@2.0.1: {} + astral-regex@2.0.0: {} astring@1.9.0: {} @@ -9266,6 +9741,23 @@ snapshots: inherits: 2.0.4 readable-stream: 3.6.2 + body-parser@1.20.3: + dependencies: + bytes: 3.1.2 + content-type: 1.0.5 + debug: 2.6.9 + depd: 2.0.0 + destroy: 1.2.0 + http-errors: 2.0.0 + iconv-lite: 0.4.24 + on-finished: 2.4.1 + qs: 6.13.0 + raw-body: 2.5.2 + type-is: 1.6.18 + unpipe: 1.0.0 + transitivePeerDependencies: + - supports-color + boolbase@1.0.0: {} boxen@8.0.1: @@ -9292,6 +9784,34 @@ snapshots: dependencies: fill-range: 7.1.1 + braintrust@0.4.9(zod@4.1.12): + dependencies: + '@ai-sdk/provider': 1.1.3 + '@next/env': 14.2.33 + '@vercel/functions': 1.6.0 + argparse: 2.0.1 + chalk: 4.1.2 + cli-progress: 3.12.0 + cors: 2.8.5 + dotenv: 16.6.1 + esbuild: 0.25.11 + eventsource-parser: 1.1.2 + express: 4.21.2 + graceful-fs: 4.2.11 + http-errors: 2.0.0 + minimatch: 9.0.5 + mustache: 4.2.0 + pluralize: 8.0.0 + simple-git: 3.30.0 + slugify: 1.6.6 + source-map: 0.7.4 + uuid: 9.0.1 + zod: 4.1.12 + zod-to-json-schema: 3.23.5(zod@4.1.12) + transitivePeerDependencies: + - '@aws-sdk/credential-provider-web-identity' + - supports-color + browserslist@4.24.2: dependencies: caniuse-lite: 1.0.30001680 @@ -9314,6 +9834,20 @@ snapshots: base64-js: 1.5.1 ieee754: 1.2.1 + bytes@3.1.2: {} + + cac@6.7.14: {} + + call-bind-apply-helpers@1.0.2: + dependencies: + es-errors: 1.3.0 + function-bind: 1.1.2 + + call-bound@1.0.4: + dependencies: + call-bind-apply-helpers: 1.0.2 + get-intrinsic: 1.3.0 + callsites@3.1.0: {} camelcase@8.0.0: {} @@ -9324,6 +9858,14 @@ snapshots: ccount@2.0.1: {} + chai@5.3.3: + dependencies: + assertion-error: 2.0.1 + check-error: 2.1.1 + deep-eql: 5.0.2 + loupe: 3.2.1 + pathval: 2.0.1 + chai@6.2.0: {} chalk@4.1.2: @@ -9343,6 +9885,8 @@ snapshots: chardet@2.1.1: {} + check-error@2.1.1: {} + chevrotain-allstar@0.3.1(chevrotain@11.0.3): dependencies: chevrotain: 11.0.3 @@ -9391,6 +9935,10 @@ snapshots: dependencies: restore-cursor: 5.1.0 + cli-progress@3.12.0: + dependencies: + string-width: 4.2.3 + cli-spinners@2.9.2: {} cli-truncate@5.1.0: @@ -9452,14 +10000,25 @@ snapshots: dependencies: safe-buffer: 5.2.1 + content-type@1.0.5: {} + convert-source-map@2.0.0: {} cookie-es@1.2.2: {} + cookie-signature@1.0.6: {} + + cookie@0.7.1: {} + cookie@0.7.2: {} cookie@1.0.1: {} + cors@2.8.5: + dependencies: + object-assign: 4.1.1 + vary: 1.1.2 + cose-base@1.0.3: dependencies: layout-base: 1.0.2 @@ -9672,6 +10231,10 @@ snapshots: dayjs@1.11.18: {} + debug@2.6.9: + dependencies: + ms: 2.0.0 + debug@4.3.6: dependencies: ms: 2.1.2 @@ -9694,6 +10257,8 @@ snapshots: dependencies: mimic-response: 3.1.0 + deep-eql@5.0.2: {} + deep-extend@0.6.0: {} deep-is@0.1.4: {} @@ -9710,6 +10275,8 @@ snapshots: destr@2.0.5: {} + destroy@1.2.0: {} + detect-indent@6.1.0: {} detect-libc@2.0.2: {} @@ -9757,6 +10324,12 @@ snapshots: dset@3.1.4: {} + dunder-proto@1.0.1: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-errors: 1.3.0 + gopd: 1.2.0 + duplexify@4.1.3: dependencies: end-of-stream: 1.4.4 @@ -9766,6 +10339,8 @@ snapshots: eastasianwidth@0.2.0: {} + ee-first@1.1.1: {} + electron-to-chromium@1.5.141: {} electron-to-chromium@1.5.58: {} @@ -9778,6 +10353,10 @@ snapshots: emoji-regex@9.2.2: {} + encodeurl@1.0.2: {} + + encodeurl@2.0.0: {} + end-of-stream@1.4.4: dependencies: once: 1.4.0 @@ -9796,10 +10375,18 @@ snapshots: environment@1.1.0: {} + es-define-property@1.0.1: {} + + es-errors@1.3.0: {} + es-module-lexer@1.5.4: {} es-module-lexer@1.7.0: {} + es-object-atoms@1.1.1: + dependencies: + es-errors: 1.3.0 + esast-util-from-estree@2.0.0: dependencies: '@types/estree-jsx': 1.0.5 @@ -10052,16 +10639,56 @@ snapshots: esutils@2.0.3: {} + etag@1.8.1: {} + eventemitter3@4.0.7: {} eventemitter3@5.0.1: {} + eventsource-parser@1.1.2: {} + eventsource-parser@3.0.6: {} expand-template@2.0.3: {} expect-type@1.2.2: {} + express@4.21.2: + dependencies: + accepts: 1.3.8 + array-flatten: 1.1.1 + body-parser: 1.20.3 + content-disposition: 0.5.4 + content-type: 1.0.5 + cookie: 0.7.1 + cookie-signature: 1.0.6 + debug: 2.6.9 + depd: 2.0.0 + encodeurl: 2.0.0 + escape-html: 1.0.3 + etag: 1.8.1 + finalhandler: 1.3.1 + fresh: 0.5.2 + http-errors: 2.0.0 + merge-descriptors: 1.0.3 + methods: 1.1.2 + on-finished: 2.4.1 + parseurl: 1.3.3 + path-to-regexp: 0.1.12 + proxy-addr: 2.0.7 + qs: 6.13.0 + range-parser: 1.2.1 + safe-buffer: 5.2.1 + send: 0.19.0 + serve-static: 1.16.2 + setprototypeof: 1.2.0 + statuses: 2.0.1 + type-is: 1.6.18 + utils-merge: 1.0.1 + vary: 1.1.2 + transitivePeerDependencies: + - supports-color + expressive-code@0.38.3: dependencies: '@expressive-code/core': 0.38.3 @@ -10164,6 +10791,18 @@ snapshots: dependencies: to-regex-range: 5.0.1 + finalhandler@1.3.1: + dependencies: + debug: 2.6.9 + encodeurl: 2.0.0 + escape-html: 1.0.3 + on-finished: 2.4.1 + parseurl: 1.3.3 + statuses: 2.0.1 + unpipe: 1.0.0 + transitivePeerDependencies: + - supports-color + find-my-way@9.1.0: dependencies: fast-deep-equal: 3.1.3 @@ -10201,8 +10840,12 @@ snapshots: cross-spawn: 7.0.6 signal-exit: 4.1.0 + forwarded@0.2.0: {} + fraction.js@4.3.7: {} + fresh@0.5.2: {} + fs-constants@1.0.0: {} fs-extra@7.0.1: @@ -10220,6 +10863,8 @@ snapshots: fsevents@2.3.3: optional: true + function-bind@1.1.2: {} + gensync@1.0.0-beta.2: {} get-caller-file@2.0.5: {} @@ -10228,10 +10873,28 @@ snapshots: get-east-asian-width@1.4.0: {} + get-intrinsic@1.3.0: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-define-property: 1.0.1 + es-errors: 1.3.0 + es-object-atoms: 1.1.1 + function-bind: 1.1.2 + get-proto: 1.0.1 + gopd: 1.2.0 + has-symbols: 1.1.0 + hasown: 2.0.2 + math-intrinsics: 1.1.0 + get-nonce@1.0.1: {} get-port@7.1.0: {} + get-proto@1.0.1: + dependencies: + dunder-proto: 1.0.1 + es-object-atoms: 1.1.1 + get-stream@9.0.1: dependencies: '@sec-ant/readable-stream': 0.4.1 @@ -10289,6 +10952,8 @@ snapshots: dependencies: csstype: 3.1.3 + gopd@1.2.0: {} + graceful-fs@4.2.11: {} graphemer@1.4.0: {} @@ -10319,6 +10984,12 @@ snapshots: has-flag@4.0.0: {} + has-symbols@1.1.0: {} + + hasown@2.0.2: + dependencies: + function-bind: 1.1.2 + hast-util-embedded@3.0.0: dependencies: '@types/hast': 3.0.4 @@ -10570,6 +11241,10 @@ snapshots: dependencies: '@babel/runtime': 7.25.4 + iconv-lite@0.4.24: + dependencies: + safer-buffer: 2.1.2 + iconv-lite@0.6.3: dependencies: safer-buffer: 2.1.2 @@ -10609,6 +11284,8 @@ snapshots: dependencies: loose-envify: 1.4.0 + ipaddr.js@1.9.1: {} + ipaddr.js@2.2.0: {} iron-webcrypto@1.2.1: {} @@ -10877,6 +11554,8 @@ snapshots: dependencies: js-tokens: 4.0.0 + loupe@3.2.1: {} + lru-cache@10.4.3: {} lru-cache@11.0.2: {} @@ -10913,6 +11592,8 @@ snapshots: marked@16.4.1: {} + math-intrinsics@1.1.0: {} + mdast-util-definitions@6.0.0: dependencies: '@types/mdast': 4.0.4 @@ -11107,6 +11788,10 @@ snapshots: dependencies: '@types/mdast': 4.0.4 + media-typer@0.3.0: {} + + merge-descriptors@1.0.3: {} + merge2@1.4.1: {} mermaid@11.12.0: @@ -11134,6 +11819,8 @@ snapshots: transitivePeerDependencies: - supports-color + methods@1.1.2: {} + micromark-core-commonmark@2.0.2: dependencies: decode-named-character-reference: 1.0.2 @@ -11425,6 +12112,14 @@ snapshots: braces: 3.0.3 picomatch: 2.3.1 + mime-db@1.52.0: {} + + mime-types@2.1.35: + dependencies: + mime-db: 1.52.0 + + mime@1.6.0: {} + mime@3.0.0: {} mimic-function@5.0.1: {} @@ -11464,6 +12159,8 @@ snapshots: mrmime@2.0.0: {} + ms@2.0.0: {} + ms@2.1.2: {} ms@2.1.3: {} @@ -11495,6 +12192,8 @@ snapshots: - '@types/node' optional: true + mustache@4.2.0: {} + mute-stream@2.0.0: optional: true @@ -11508,6 +12207,8 @@ snapshots: natural-compare@1.4.0: {} + negotiator@0.6.3: {} + neotraverse@0.6.18: {} nlcst-to-string@4.0.0: @@ -11538,6 +12239,8 @@ snapshots: object-assign@4.1.1: {} + object-inspect@1.13.4: {} + ofetch@1.4.1: dependencies: destr: 2.0.5 @@ -11546,6 +12249,10 @@ snapshots: on-exit-leak-free@2.1.2: {} + on-finished@2.4.1: + dependencies: + ee-first: 1.1.1 + once@1.4.0: dependencies: wrappy: 1.0.2 @@ -11671,6 +12378,8 @@ snapshots: dependencies: entities: 4.5.0 + parseurl@1.3.3: {} + path-data-parser@0.1.0: {} path-exists@4.0.0: {} @@ -11682,13 +12391,19 @@ snapshots: lru-cache: 11.0.2 minipass: 7.1.2 + path-to-regexp@0.1.12: {} + path-to-regexp@6.3.0: optional: true path-type@4.0.0: {} + pathe@1.1.2: {} + pathe@2.0.3: {} + pathval@2.0.1: {} + peek-readable@5.3.1: {} picocolors@1.1.1: {} @@ -11739,6 +12454,8 @@ snapshots: exsolve: 1.0.7 pathe: 2.0.3 + pluralize@8.0.0: {} + points-on-curve@0.2.0: {} points-on-path@0.2.1: @@ -11825,6 +12542,11 @@ snapshots: property-information@7.1.0: {} + proxy-addr@2.0.7: + dependencies: + forwarded: 0.2.0 + ipaddr.js: 1.9.1 + pump@3.0.2: dependencies: end-of-stream: 1.4.4 @@ -11832,6 +12554,10 @@ snapshots: punycode@2.3.1: {} + qs@6.13.0: + dependencies: + side-channel: 1.1.0 + quansync@0.2.11: {} queue-microtask@1.2.3: {} @@ -11842,6 +12568,15 @@ snapshots: radix3@1.1.2: {} + range-parser@1.2.1: {} + + raw-body@2.5.2: + dependencies: + bytes: 3.1.2 + http-errors: 2.0.0 + iconv-lite: 0.4.24 + unpipe: 1.0.0 + rc@1.2.8: dependencies: deep-extend: 0.6.0 @@ -12291,12 +13026,39 @@ snapshots: semver@7.6.3: {} + send@0.19.0: + dependencies: + debug: 2.6.9 + depd: 2.0.0 + destroy: 1.2.0 + encodeurl: 1.0.2 + escape-html: 1.0.3 + etag: 1.8.1 + fresh: 0.5.2 + http-errors: 2.0.0 + mime: 1.6.0 + ms: 2.1.3 + on-finished: 2.4.1 + range-parser: 1.2.1 + statuses: 2.0.1 + transitivePeerDependencies: + - supports-color + seroval-plugins@1.2.1(seroval@1.2.1): dependencies: seroval: 1.2.1 seroval@1.2.1: {} + serve-static@1.16.2: + dependencies: + encodeurl: 2.0.0 + escape-html: 1.0.3 + parseurl: 1.3.3 + send: 0.19.0 + transitivePeerDependencies: + - supports-color + set-cookie-parser@2.7.1: {} setprototypeof@1.2.0: {} @@ -12365,6 +13127,34 @@ snapshots: '@shikijs/vscode-textmate': 10.0.2 '@types/hast': 3.0.4 + side-channel-list@1.0.0: + dependencies: + es-errors: 1.3.0 + object-inspect: 1.13.4 + + side-channel-map@1.0.1: + dependencies: + call-bound: 1.0.4 + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + object-inspect: 1.13.4 + + side-channel-weakmap@1.0.2: + dependencies: + call-bound: 1.0.4 + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + object-inspect: 1.13.4 + side-channel-map: 1.0.1 + + side-channel@1.1.0: + dependencies: + es-errors: 1.3.0 + object-inspect: 1.13.4 + side-channel-list: 1.0.0 + side-channel-map: 1.0.1 + side-channel-weakmap: 1.0.2 + siginfo@2.0.0: {} signal-exit@4.1.0: {} @@ -12377,6 +13167,14 @@ snapshots: once: 1.4.0 simple-concat: 1.0.1 + simple-git@3.30.0: + dependencies: + '@kwsites/file-exists': 1.1.1 + '@kwsites/promise-deferred': 1.1.1 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + simple-swizzle@0.2.2: dependencies: is-arrayish: 0.3.2 @@ -12403,6 +13201,8 @@ snapshots: ansi-styles: 6.2.1 is-fullwidth-code-point: 5.1.0 + slugify@1.6.6: {} + solid-js@1.9.5: dependencies: csstype: 3.1.3 @@ -12638,8 +13438,14 @@ snapshots: fdir: 6.5.0(picomatch@4.0.3) picomatch: 4.0.3 + tinypool@1.1.1: {} + + tinyrainbow@1.2.0: {} + tinyrainbow@3.0.3: {} + tinyspy@3.0.2: {} + tldts-core@7.0.17: optional: true @@ -12715,6 +13521,11 @@ snapshots: type-fest@4.41.0: optional: true + type-is@1.6.18: + dependencies: + media-typer: 0.3.0 + mime-types: 2.1.35 + typescript-eslint@8.46.2(eslint@9.38.0(jiti@2.6.1))(typescript@5.6.2): dependencies: '@typescript-eslint/eslint-plugin': 8.46.2(@typescript-eslint/parser@8.46.2(eslint@9.38.0(jiti@2.6.1))(typescript@5.6.2))(eslint@9.38.0(jiti@2.6.1))(typescript@5.6.2) @@ -12796,6 +13607,8 @@ snapshots: universalify@0.1.2: {} + unpipe@1.0.0: {} + unplugin@2.1.2: dependencies: acorn: 8.14.0 @@ -12852,8 +13665,14 @@ snapshots: util-deprecate@1.0.2: {} + utils-merge@1.0.1: {} + uuid@11.1.0: {} + uuid@9.0.1: {} + + vary@1.1.2: {} + vfile-location@5.0.3: dependencies: '@types/unist': 3.0.3 @@ -12886,6 +13705,24 @@ snapshots: d3-time: 3.1.0 d3-timer: 3.0.1 + vite-node@2.1.9(@types/node@22.7.7)(lightningcss@1.30.1)(terser@5.31.5): + dependencies: + cac: 6.7.14 + debug: 4.4.3 + es-module-lexer: 1.7.0 + pathe: 1.1.2 + vite: 5.4.11(@types/node@22.7.7)(lightningcss@1.30.1)(terser@5.31.5) + transitivePeerDependencies: + - '@types/node' + - less + - lightningcss + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + vite-tsconfig-paths@4.3.2(typescript@5.6.2)(vite@7.1.12(@types/node@22.7.7)(jiti@2.6.1)(lightningcss@1.30.1)(terser@5.31.5)(tsx@4.19.3)(yaml@2.8.1)): dependencies: debug: 4.3.6 @@ -12963,6 +13800,41 @@ snapshots: optionalDependencies: vite: 5.4.11(@types/node@22.7.7)(lightningcss@1.30.1)(terser@5.31.5) + vitest@2.1.9(@types/node@22.7.7)(lightningcss@1.30.1)(msw@2.11.3(@types/node@22.7.7)(typescript@5.6.2))(terser@5.31.5): + dependencies: + '@vitest/expect': 2.1.9 + '@vitest/mocker': 2.1.9(msw@2.11.3(@types/node@22.7.7)(typescript@5.6.2))(vite@5.4.11(@types/node@22.7.7)(lightningcss@1.30.1)(terser@5.31.5)) + '@vitest/pretty-format': 2.1.9 + '@vitest/runner': 2.1.9 + '@vitest/snapshot': 2.1.9 + '@vitest/spy': 2.1.9 + '@vitest/utils': 2.1.9 + chai: 5.3.3 + debug: 4.4.3 + expect-type: 1.2.2 + magic-string: 0.30.19 + pathe: 1.1.2 + std-env: 3.10.0 + tinybench: 2.9.0 + tinyexec: 0.3.2 + tinypool: 1.1.1 + tinyrainbow: 1.2.0 + vite: 5.4.11(@types/node@22.7.7)(lightningcss@1.30.1)(terser@5.31.5) + vite-node: 2.1.9(@types/node@22.7.7)(lightningcss@1.30.1)(terser@5.31.5) + why-is-node-running: 2.3.0 + optionalDependencies: + '@types/node': 22.7.7 + transitivePeerDependencies: + - less + - lightningcss + - msw + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + vitest@4.0.1(@types/debug@4.1.12)(@types/node@22.7.7)(jiti@2.6.1)(lightningcss@1.30.1)(msw@2.11.3(@types/node@22.7.7)(typescript@5.6.2))(terser@5.31.5)(tsx@4.19.0)(yaml@2.8.1): dependencies: '@vitest/expect': 4.0.1 @@ -13145,6 +14017,10 @@ snapshots: dependencies: zod: 3.25.76 + zod-to-json-schema@3.23.5(zod@4.1.12): + dependencies: + zod: 4.1.12 + zod-to-ts@1.2.0(typescript@5.9.3)(zod@3.25.76): dependencies: typescript: 5.9.3 From 81a57accb3f6a2aa79cbd0f5bc3a5f0bb49a7f99 Mon Sep 17 00:00:00 2001 From: Ken Jiang Date: Thu, 20 Nov 2025 13:15:53 -0500 Subject: [PATCH 2/4] rm dead code --- .changeset/0000-braintrust-consolidation.md | 27 ++ .changeset/0000-braintrust-storage.md | 27 -- package.json | 3 +- packages/braintrust/README.md | 96 ------ packages/braintrust/examples/.env.example | 3 - packages/braintrust/examples/README.md | 92 ------ .../braintrust/examples/evalite.config.ts | 28 -- packages/braintrust/examples/simple.eval.ts | 25 -- packages/braintrust/package.json | 33 --- packages/braintrust/tsconfig.json | 8 - packages/evalite/package.json | 8 +- .../index.ts => evalite/src/braintrust.ts} | 15 +- packages/evalite/src/run-evalite.ts | 3 + .../{ => braintrust}/braintrust-test.eval.ts | 0 .../evalite.config.ts} | 4 +- packages/example/evalite.config.ts | 2 +- packages/example/package.json | 2 +- pnpm-lock.yaml | 276 +++--------------- 18 files changed, 81 insertions(+), 571 deletions(-) create mode 100644 .changeset/0000-braintrust-consolidation.md delete mode 100644 .changeset/0000-braintrust-storage.md delete mode 100644 packages/braintrust/README.md delete mode 100644 packages/braintrust/examples/.env.example delete mode 100644 packages/braintrust/examples/README.md delete mode 100644 packages/braintrust/examples/evalite.config.ts delete mode 100644 packages/braintrust/examples/simple.eval.ts delete mode 100644 packages/braintrust/package.json delete mode 100644 packages/braintrust/tsconfig.json rename packages/{braintrust/src/index.ts => evalite/src/braintrust.ts} (98%) rename packages/example/{ => braintrust}/braintrust-test.eval.ts (100%) rename packages/example/{evalite.config.braintrust.ts => braintrust/evalite.config.ts} (89%) diff --git a/.changeset/0000-braintrust-consolidation.md b/.changeset/0000-braintrust-consolidation.md new file mode 100644 index 00000000..c9acb189 --- /dev/null +++ b/.changeset/0000-braintrust-consolidation.md @@ -0,0 +1,27 @@ +--- +"evalite": minor +--- + +Added Braintrust storage integration to the main `evalite` package. + +**What's new:** + +- Braintrust storage is now available as a subpath export: `evalite/braintrust` +- `braintrust` is an optional peer dependency (similar to the AI SDK integration pattern) +- Use `createBraintrustStorage()` to store eval results in Braintrust for tracking and visualization + +**Usage:** + +1. Install `braintrust` as a peer dependency: `pnpm add braintrust` +2. Import in your `evalite.config.ts`: + + ```ts + import { createBraintrustStorage } from "evalite/braintrust"; + + export default { + storage: createBraintrustStorage({ + projectName: "my-project", + apiKey: process.env.BRAINTRUST_API_KEY, + }), + }; + ``` diff --git a/.changeset/0000-braintrust-storage.md b/.changeset/0000-braintrust-storage.md deleted file mode 100644 index 87b8374b..00000000 --- a/.changeset/0000-braintrust-storage.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -"evalite": minor -"@evalite/braintrust": major ---- - -**Breaking Change:** Braintrust storage has been moved to a separate package `@evalite/braintrust`. This reduces the bundle size of the main `evalite` package for users who don't need Braintrust integration. - -**What changed:** - -- Braintrust storage is now in a separate package `@evalite/braintrust` -- The main `evalite` package no longer includes the `braintrust` dependency -- The `evalite/braintrust-storage` export has been removed - -**Migration:** -If you were using Braintrust storage: - -1. Install the new package: `pnpm add @evalite/braintrust` (or `npm install @evalite/braintrust`) -2. Update your imports in `evalite.config.ts`: - - **Before:** `import { createBraintrustStorage } from "evalite/braintrust-storage";` - - **After:** `import { createBraintrustStorage } from "@evalite/braintrust";` -3. The API remains unchanged - only the import path has changed - -**Benefits:** - -- Main `evalite` package is lighter and faster to install -- Braintrust dependency only installed when needed -- Better separation of concerns diff --git a/package.json b/package.json index 61522d8e..0bb653e1 100644 --- a/package.json +++ b/package.json @@ -11,9 +11,8 @@ "scripts": { "beta:release": "pnpm run ci && changeset version && git add . && git commit -m 'Bump version' && git push && pnpm changeset publish", "build:evalite": "pnpm --filter evalite build", - "build:braintrust": "pnpm --filter @evalite/braintrust build", "build:evalite-ui": "pnpm --filter evalite-ui build && pnpm --filter evalite-ui after-build", - "build": "pnpm run build:evalite && pnpm run build:braintrust && pnpm run build:evalite-ui", + "build": "pnpm run build:evalite && pnpm run build:evalite-ui", "dev": "pnpm build:evalite && pnpm build:evalite-ui && pnpm --filter evalite --filter evalite-tests --parallel run dev", "dev:evalite": "pnpm --filter evalite dev", "dev:evalite-ui": "pnpm --filter evalite-ui dev", diff --git a/packages/braintrust/README.md b/packages/braintrust/README.md deleted file mode 100644 index 7964af02..00000000 --- a/packages/braintrust/README.md +++ /dev/null @@ -1,96 +0,0 @@ -# @evalite/braintrust - -Braintrust storage backend for Evalite. Store your evaluation results in [Braintrust](https://www.braintrust.dev) and view them in the Braintrust web UI. - -## Installation - -```bash -npm install @evalite/braintrust evalite -# or -pnpm add @evalite/braintrust evalite -# or -yarn add @evalite/braintrust evalite -``` - -## Usage - -Configure Evalite to use Braintrust storage in your `evalite.config.ts`: - -```typescript -import { defineConfig } from "evalite/config"; -import { createBraintrustStorage } from "@evalite/braintrust"; - -export default defineConfig({ - storage: async () => { - return await createBraintrustStorage({ - // Required: Your Braintrust project name - projectName: "My Evalite Project", - - // Optional: Custom experiment name (defaults to timestamp) - experimentName: "my-experiment", - - // Optional: API key (defaults to BRAINTRUST_API_KEY env var) - apiKey: process.env.BRAINTRUST_API_KEY, - - // Optional: Custom Braintrust app URL - appUrl: "https://www.braintrust.dev", - }); - }, -}); -``` - -## Environment Variables - -Set your Braintrust API key: - -```bash -export BRAINTRUST_API_KEY="your-api-key-here" -``` - -## Features - -- **Automatic URL generation**: After running evals, you'll see a URL to view results in Braintrust -- **Nested traces**: LLM calls and traces are logged as nested spans in Braintrust -- **Scores and metadata**: All scorers and custom columns are preserved -- **Experiment organization**: Results are organized by project and experiment name - -## Example - -See the [examples directory](./examples) for a complete working example. - -To run the example: - -```bash -cd examples -export BRAINTRUST_API_KEY="your-api-key" -pnpm evalite run simple.eval.ts -``` - -## API Reference - -### `createBraintrustStorage(options)` - -Creates a new Braintrust storage backend. - -**Options:** - -- `projectName` (string, required): The Braintrust project name -- `experimentName` (string, optional): Custom experiment name. Defaults to `evalite-{timestamp}` -- `apiKey` (string, optional): Braintrust API key. Defaults to `BRAINTRUST_API_KEY` env var -- `appUrl` (string, optional): Custom Braintrust app URL. Defaults to `https://www.braintrust.dev` - -**Returns:** `Promise` - -## How It Works - -When you run evals with Braintrust storage: - -1. Each eval suite creates a Braintrust experiment -2. Each test case becomes a top-level span (row) in the experiment -3. Scores are logged to their corresponding spans -4. LLM traces are logged as nested spans under their test case -5. After the run completes, you get a URL to view results in Braintrust - -## License - -MIT diff --git a/packages/braintrust/examples/.env.example b/packages/braintrust/examples/.env.example deleted file mode 100644 index df63c366..00000000 --- a/packages/braintrust/examples/.env.example +++ /dev/null @@ -1,3 +0,0 @@ -# Braintrust API Key -# Get your API key from https://www.braintrust.dev -BRAINTRUST_API_KEY=your-api-key-here diff --git a/packages/braintrust/examples/README.md b/packages/braintrust/examples/README.md deleted file mode 100644 index cade4d7d..00000000 --- a/packages/braintrust/examples/README.md +++ /dev/null @@ -1,92 +0,0 @@ -# Braintrust Storage Example - -This example demonstrates how to use `@evalite/braintrust` to store evaluation results in Braintrust. - -## Prerequisites - -1. Get a Braintrust API key from [braintrust.dev](https://www.braintrust.dev) -2. Set your API key as an environment variable: - ```bash - export BRAINTRUST_API_KEY="your-api-key-here" - ``` - -## Setup - -1. Build the required packages (from the monorepo root): - - ```bash - cd /path/to/evalite - pnpm build - ``` - -2. Set your Braintrust API key: - ```bash - export BRAINTRUST_API_KEY="your-api-key-here" - ``` - -## Running the Example - -From the monorepo root: - -```bash -# Navigate to the examples directory -cd packages/braintrust/examples - -# Run the example once -../../../packages/evalite/dist/bin.js run simple.eval.ts - -# Or use the evalite command if you have it installed globally -evalite run simple.eval.ts -``` - -Or from anywhere in the monorepo with pnpm: - -```bash -pnpm --filter evalite exec evalite run packages/braintrust/examples/simple.eval.ts -``` - -## What to Expect - -When you run the example **with a valid API key**, you should see: - -1. The eval running and completing (3 test cases) -2. All tests passing with 100% score -3. A message with a Braintrust URL: - ``` - View results in Braintrust: https://www.braintrust.dev/app/... - ``` -4. Click the URL to view your results in the Braintrust web UI! - -**Without an API key**, the eval will still run locally and show results, but: - -- You'll see errors about missing `BRAINTRUST_API_KEY` in stderr -- Results won't be uploaded to Braintrust -- No URL will be displayed (or it may 404) - -## What's Happening - -The example: - -- Uses `createBraintrustStorage()` in `evalite.config.ts` to configure Braintrust storage -- Runs a simple eval that tests string concatenation -- Sends results (inputs, outputs, scores, traces) to Braintrust -- Generates a URL where you can view and analyze the results - -## Files - -- `simple.eval.ts` - A simple evaluation to test the storage integration -- `evalite.config.ts` - Configuration that enables Braintrust storage -- `package.json` - Dependencies for running the example - -## Customization - -You can customize the Braintrust configuration in `evalite.config.ts`: - -```typescript -await createBraintrustStorage({ - projectName: "My Project Name", // Required - experimentName: "my-experiment", // Optional (defaults to timestamp) - apiKey: "your-api-key", // Optional (defaults to env var) - appUrl: "https://www.braintrust.dev", // Optional (custom instance) -}); -``` diff --git a/packages/braintrust/examples/evalite.config.ts b/packages/braintrust/examples/evalite.config.ts deleted file mode 100644 index 7683ac9e..00000000 --- a/packages/braintrust/examples/evalite.config.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { defineConfig } from "evalite/config"; -import { createBraintrustStorage } from "../dist/index.js"; - -/** - * Example Evalite configuration using Braintrust storage. - * - * To use this: - * 1. Set BRAINTRUST_API_KEY environment variable - * 2. Run: evalite run simple.eval.ts - * 3. Click the Braintrust URL in the output to view results - */ -export default defineConfig({ - storage: async () => { - return await createBraintrustStorage({ - // Required: Your Braintrust project name - projectName: "@evalite/braintrust Example", - - // Optional: Custom experiment name (defaults to timestamp) - experimentName: `example-run-${new Date().toISOString().split("T")[0]}`, - - // Optional: API key (defaults to BRAINTRUST_API_KEY env var) - // apiKey: process.env.BRAINTRUST_API_KEY, - - // Optional: Custom Braintrust app URL - // appUrl: "https://www.braintrust.dev", - }); - }, -}); diff --git a/packages/braintrust/examples/simple.eval.ts b/packages/braintrust/examples/simple.eval.ts deleted file mode 100644 index 293753c0..00000000 --- a/packages/braintrust/examples/simple.eval.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { evalite } from "evalite"; - -/** - * Simple example demonstrating Braintrust storage integration. - * This eval tests a basic string concatenation task. - */ -evalite("Braintrust Storage Example", { - data: () => [ - { input: "Hello", expected: "Hello, World!" }, - { input: "Goodbye", expected: "Goodbye, World!" }, - { input: "Testing", expected: "Testing, World!" }, - ], - task: async (input) => { - // Simulate a simple LLM task - return `${input}, World!`; - }, - scorers: [ - (input, output, expected) => { - return { - name: "exact_match", - score: output === expected ? 1 : 0, - }; - }, - ], -}); diff --git a/packages/braintrust/package.json b/packages/braintrust/package.json deleted file mode 100644 index 8598d302..00000000 --- a/packages/braintrust/package.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "name": "@evalite/braintrust", - "version": "1.0.0-beta.14", - "type": "module", - "description": "Braintrust storage backend for Evalite", - "homepage": "https://evalite.dev", - "license": "MIT", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "./dist/*" - ], - "exports": { - ".": "./dist/index.js" - }, - "scripts": { - "dev": "tsc -w", - "build": "tsc", - "test": "vitest run", - "typecheck": "tsc" - }, - "dependencies": { - "braintrust": "^0.4.9" - }, - "peerDependencies": { - "evalite": "^1.0.0-beta.14" - }, - "devDependencies": { - "evalite": "workspace:*", - "typescript": "5.6.2", - "vitest": "^2.1.8" - } -} diff --git a/packages/braintrust/tsconfig.json b/packages/braintrust/tsconfig.json deleted file mode 100644 index 0b60d3ac..00000000 --- a/packages/braintrust/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "@total-typescript/tsconfig/tsc/no-dom/library", - "compilerOptions": { - "outDir": "dist", - "rootDir": "src" - }, - "include": ["src"] -} diff --git a/packages/evalite/package.json b/packages/evalite/package.json index 744fe0e6..a17b0446 100644 --- a/packages/evalite/package.json +++ b/packages/evalite/package.json @@ -44,6 +44,7 @@ "./reporter": "./dist/reporter.js", "./traces": "./dist/traces.js", "./ai-sdk": "./dist/ai-sdk.js", + "./braintrust": "./dist/braintrust.js", "./runner": "./dist/run-evalite.js", "./export-static": "./dist/export-static.js", "./sqlite-storage": "./dist/storage/sqlite.js", @@ -71,7 +72,8 @@ }, "peerDependencies": { "better-sqlite3": "^11.6.0", - "ai": "^5" + "ai": "^5", + "braintrust": "^0.4.9" }, "peerDependenciesMeta": { "better-sqlite3": { @@ -79,6 +81,9 @@ }, "ai": { "optional": true + }, + "braintrust": { + "optional": true } }, "devDependencies": { @@ -88,6 +93,7 @@ "@types/ws": "^8.18.1", "ai": "^5.0.59", "better-sqlite3": "^11.6.0", + "braintrust": "^0.4.9", "unstorage": "^1.17.1" } } diff --git a/packages/braintrust/src/index.ts b/packages/evalite/src/braintrust.ts similarity index 98% rename from packages/braintrust/src/index.ts rename to packages/evalite/src/braintrust.ts index b4c65a71..3f22cfac 100644 --- a/packages/braintrust/src/index.ts +++ b/packages/evalite/src/braintrust.ts @@ -1,11 +1,6 @@ -import type { Evalite } from "evalite"; -import { - init, - currentSpan, - _internalGetGlobalState, - type Experiment, - type Span, -} from "braintrust"; +import type { Evalite } from "./types.js"; +import type { Experiment, Span } from "braintrust"; +import { init, _internalGetGlobalState } from "braintrust"; interface BraintrustStorageOptions { /** @@ -324,7 +319,7 @@ export class BraintrustStorage implements Evalite.Storage { update: async ( opts: Evalite.Storage.Evals.UpdateOpts ): Promise => { - const experiment = await this.initExperiment(); + await this.initExperiment(); const eval_ = this.entityStore.evals.get(opts.id); if (!eval_) { throw new Error(`Eval with id ${opts.id} not found`); @@ -396,7 +391,7 @@ export class BraintrustStorage implements Evalite.Storage { create: async ( opts: Evalite.Storage.Scores.CreateOpts ): Promise => { - const experiment = await this.initExperiment(); + await this.initExperiment(); const score: Evalite.Storage.Entities.Score = { id: this.nextId.score++, diff --git a/packages/evalite/src/run-evalite.ts b/packages/evalite/src/run-evalite.ts index 9bb7dce9..5e8255df 100644 --- a/packages/evalite/src/run-evalite.ts +++ b/packages/evalite/src/run-evalite.ts @@ -402,6 +402,9 @@ export const runEvalite = async (opts: { } } + // Close storage to flush any pending data (e.g., to Braintrust) + await storage.close(); + if (typeof exitCode === "number" && opts.mode === "run-once-and-exit") { process.exit(exitCode); } diff --git a/packages/example/braintrust-test.eval.ts b/packages/example/braintrust/braintrust-test.eval.ts similarity index 100% rename from packages/example/braintrust-test.eval.ts rename to packages/example/braintrust/braintrust-test.eval.ts diff --git a/packages/example/evalite.config.braintrust.ts b/packages/example/braintrust/evalite.config.ts similarity index 89% rename from packages/example/evalite.config.braintrust.ts rename to packages/example/braintrust/evalite.config.ts index ed53b540..c9a2d583 100644 --- a/packages/example/evalite.config.braintrust.ts +++ b/packages/example/braintrust/evalite.config.ts @@ -1,11 +1,11 @@ import { defineConfig } from "evalite/config"; -import { createBraintrustStorage } from "@evalite/braintrust"; +import { createBraintrustStorage } from "evalite/braintrust"; /** * Example configuration for using Braintrust as the storage backend. * * To use this configuration: - * 1. Install the Braintrust package: `pnpm add @evalite/braintrust` + * 1. Install the Braintrust package: `pnpm add braintrust` * 2. Rename this file to `evalite.config.ts` (or use --config flag) * 3. Set the BRAINTRUST_API_KEY environment variable * 4. Run evalite as normal diff --git a/packages/example/evalite.config.ts b/packages/example/evalite.config.ts index a38a819c..90f14678 100644 --- a/packages/example/evalite.config.ts +++ b/packages/example/evalite.config.ts @@ -1,5 +1,5 @@ import { defineConfig } from "evalite/config"; -import { createBraintrustStorage } from "@evalite/braintrust"; +import { createBraintrustStorage } from "evalite/braintrust"; export default defineConfig({ storage: async () => { diff --git a/packages/example/package.json b/packages/example/package.json index 7a743541..30d2c371 100644 --- a/packages/example/package.json +++ b/packages/example/package.json @@ -12,7 +12,7 @@ "dependencies": { "unstorage": "^1.17.1", "evalite": "workspace:*", - "@evalite/braintrust": "workspace:*", + "braintrust": "^0.4.9", "ai": "^5.0.59", "@ai-sdk/openai": "^2.0.42", "@ai-sdk/provider": "^2.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5cbfc3d5..c87a4fb6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -200,22 +200,6 @@ importers: specifier: ^4.2.1 version: 4.3.2(typescript@5.6.2)(vite@7.1.12(@types/node@22.7.7)(jiti@2.6.1)(lightningcss@1.30.1)(terser@5.31.5)(tsx@4.19.3)(yaml@2.8.1)) - packages/braintrust: - dependencies: - braintrust: - specifier: ^0.4.9 - version: 0.4.9(zod@4.1.12) - devDependencies: - evalite: - specifier: workspace:* - version: link:../evalite - typescript: - specifier: 5.6.2 - version: 5.6.2 - vitest: - specifier: ^2.1.8 - version: 2.1.9(@types/node@22.7.7)(lightningcss@1.30.1)(msw@2.11.3(@types/node@22.7.7)(typescript@5.6.2))(terser@5.31.5) - packages/evalite: dependencies: '@fastify/static': @@ -279,6 +263,9 @@ importers: better-sqlite3: specifier: ^11.6.0 version: 11.6.0 + braintrust: + specifier: ^0.4.9 + version: 0.4.9(zod@4.1.12) unstorage: specifier: ^1.17.1 version: 1.17.1 @@ -309,12 +296,12 @@ importers: '@ai-sdk/provider': specifier: ^2.0.0 version: 2.0.0 - '@evalite/braintrust': - specifier: workspace:* - version: link:../braintrust ai: specifier: ^5.0.59 version: 5.0.59(zod@3.25.76) + braintrust: + specifier: ^0.4.9 + version: 0.4.9(zod@3.25.76) dotenv: specifier: ^16.4.5 version: 16.4.5 @@ -2916,23 +2903,9 @@ packages: peerDependencies: vite: ^4.2.0 || ^5.0.0 || ^6.0.0 - '@vitest/expect@2.1.9': - resolution: {integrity: sha512-UJCIkTBenHeKT1TTlKMJWy1laZewsRIzYighyYiJKZreqtdxSos/S1t+ktRMQWu2CKqaarrkeszJx1cgC5tGZw==} - '@vitest/expect@4.0.1': resolution: {integrity: sha512-KtvGLN/IWoZfg68JF2q/zbDEo+UJTWnc7suYJ8RF+ZTBeBcBz4NIOJDxO4Q3bEY9GsOYhgy5cOevcVPFh4+V7g==} - '@vitest/mocker@2.1.9': - resolution: {integrity: sha512-tVL6uJgoUdi6icpxmdrn5YNo3g3Dxv+IHJBr0GXHaEdTcw3F+cPKnsXFhli6nO+f/6SDKPHEK1UN+k+TQv0Ehg==} - peerDependencies: - msw: ^2.4.9 - vite: ^5.0.0 - peerDependenciesMeta: - msw: - optional: true - vite: - optional: true - '@vitest/mocker@4.0.1': resolution: {integrity: sha512-fwmvg8YvwSAE41Hyhul7dL4UzPhG+k2VaZCcL+aHagLx4qlNQgKYTw7coF4YdjAxSBBt0b408gQFYMX1Qeqweg==} peerDependencies: @@ -2944,33 +2917,18 @@ packages: vite: optional: true - '@vitest/pretty-format@2.1.9': - resolution: {integrity: sha512-KhRIdGV2U9HOUzxfiHmY8IFHTdqtOhIzCpd8WRdJiE7D/HUcZVD0EgQCVjm+Q9gkUXWgBvMmTtZgIG48wq7sOQ==} - '@vitest/pretty-format@4.0.1': resolution: {integrity: sha512-6nq3JY/zQ91+oX1vd4fajiVNyA/HMhaF9cOw5P9cQi6ML7PRi7ilVaQ77PulF+4kvUKr9bcLm9GoAtwlVFbGzw==} - '@vitest/runner@2.1.9': - resolution: {integrity: sha512-ZXSSqTFIrzduD63btIfEyOmNcBmQvgOVsPNPe0jYtESiXkhd8u2erDLnMxmGrDCwHCCHE7hxwRDCT3pt0esT4g==} - '@vitest/runner@4.0.1': resolution: {integrity: sha512-nxUoWmw7ZX2OiSNwolJeSOOzrrR/o79wRTwP7HhiW/lDFwQHtWMj9snMhrdvccFqanvI8897E81eXjgDbrRvqA==} - '@vitest/snapshot@2.1.9': - resolution: {integrity: sha512-oBO82rEjsxLNJincVhLhaxxZdEtV0EFHMK5Kmx5sJ6H9L183dHECjiefOAdnqpIgT5eZwT04PoggUnW88vOBNQ==} - '@vitest/snapshot@4.0.1': resolution: {integrity: sha512-CvfsEWutEIN/Z9ScXYup7YwlPeK9JICrV7FN9p3pVytsyh+aCHAH0PUi//YlTiQ7T8qYxJYpUrAwZL9XqmZ5ZA==} - '@vitest/spy@2.1.9': - resolution: {integrity: sha512-E1B35FwzXXTs9FHNK6bDszs7mtydNi5MIfUWpceJ8Xbfb1gBMscAnwLbEu+B44ed6W3XjL9/ehLPHR1fkf1KLQ==} - '@vitest/spy@4.0.1': resolution: {integrity: sha512-Hj0/TBQ2EN72wDpfKiUf63mRCkE0ZiSGXGeDDvW9T3LBKVVApItd0GyQLDBIe03kWbyK9gOTEbJVVWthcLFzCg==} - '@vitest/utils@2.1.9': - resolution: {integrity: sha512-v0psaMSkNJ3A2NMrUEHFRzJtDPFn+/VWZ5WxImB21T9fjucJRmS7xCS3ppEnARb9y11OAzaD+P2Ps+b+BGX5iQ==} - '@vitest/utils@4.0.1': resolution: {integrity: sha512-uRrACgpIz5sxuT87ml7xhh7EdKtW8k0N9oSFVBPl8gHB/JfLObLe9dXO6ZrsNN55FzciGIRqIEILgTQvg1eNHw==} @@ -3084,10 +3042,6 @@ packages: resolution: {integrity: sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==} engines: {node: '>=8'} - assertion-error@2.0.1: - resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} - engines: {node: '>=12'} - astral-regex@2.0.0: resolution: {integrity: sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==} engines: {node: '>=8'} @@ -3227,10 +3181,6 @@ packages: resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} engines: {node: '>= 0.8'} - cac@6.7.14: - resolution: {integrity: sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==} - engines: {node: '>=8'} - call-bind-apply-helpers@1.0.2: resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} engines: {node: '>= 0.4'} @@ -3256,10 +3206,6 @@ packages: ccount@2.0.1: resolution: {integrity: sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==} - chai@5.3.3: - resolution: {integrity: sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==} - engines: {node: '>=18'} - chai@6.2.0: resolution: {integrity: sha512-aUTnJc/JipRzJrNADXVvpVqi6CO0dn3nx4EVPxijri+fj3LUUDyZQOgVeW54Ob3Y1Xh9Iz8f+CgaCl8v0mn9bA==} engines: {node: '>=18'} @@ -3287,10 +3233,6 @@ packages: chardet@2.1.1: resolution: {integrity: sha512-PsezH1rqdV9VvyNhxxOW32/d75r01NY7TQCmOqomRo15ZSOKbpTFVsfjghxo6JloQUCGnH4k1LGu0R4yCLlWQQ==} - check-error@2.1.1: - resolution: {integrity: sha512-OAlb+T7V4Op9OwdkjmguYRqncdlx5JiofwOAUkmTF+jNdHwzTaTs4sRAGpzLF3oOz5xAyDGrPgeIDFQmDOTiJw==} - engines: {node: '>= 16'} - chevrotain-allstar@0.3.1: resolution: {integrity: sha512-b7g+y9A0v4mxCW1qUhf3BSVPg+/NvGErk/dOkrDaHA0nQIQGAtrOjlX//9OQtRlSCy+x9rfB5N8yC71lH1nvMw==} peerDependencies: @@ -3671,10 +3613,6 @@ packages: resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} engines: {node: '>=10'} - deep-eql@5.0.2: - resolution: {integrity: sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==} - engines: {node: '>=6'} - deep-extend@0.6.0: resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} engines: {node: '>=4.0.0'} @@ -4770,9 +4708,6 @@ packages: resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==} hasBin: true - loupe@3.2.1: - resolution: {integrity: sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==} - lru-cache@10.4.3: resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} @@ -5302,16 +5237,9 @@ packages: resolution: {integrity: sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==} engines: {node: '>=8'} - pathe@1.1.2: - resolution: {integrity: sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ==} - pathe@2.0.3: resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} - pathval@2.0.1: - resolution: {integrity: sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==} - engines: {node: '>= 14.16'} - peek-readable@5.3.1: resolution: {integrity: sha512-GVlENSDW6KHaXcd9zkZltB7tCLosKB/4Hg0fqBJkAoBgYG2Tn1xtMgXtSUuMU9AK/gCm/tTdT8mgAeF4YNeeqw==} engines: {node: '>=14.16'} @@ -6098,22 +6026,10 @@ packages: resolution: {integrity: sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==} engines: {node: '>=12.0.0'} - tinypool@1.1.1: - resolution: {integrity: sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==} - engines: {node: ^18.0.0 || >=20.0.0} - - tinyrainbow@1.2.0: - resolution: {integrity: sha512-weEDEq7Z5eTHPDh4xjX789+fHfF+P8boiFB+0vbWzpbnbsEr/GRaohi/uMKxg8RZMXnl1ItAi/IUHWMsjDV7kQ==} - engines: {node: '>=14.0.0'} - tinyrainbow@3.0.3: resolution: {integrity: sha512-PSkbLUoxOFRzJYjjxHJt9xro7D+iilgMX/C9lawzVuYiIdcihh9DXmVibBe8lmcFrRi/VzlPjBxbN7rH24q8/Q==} engines: {node: '>=14.0.0'} - tinyspy@3.0.2: - resolution: {integrity: sha512-n1cw8k1k0x4pgA2+9XrOkFydTerNcJ1zWCO5Nn9scWHTD+5tp8dghT2x1uduQePZTZgd3Tupf+x9BxJjeJi77Q==} - engines: {node: '>=14.0.0'} - tldts-core@7.0.17: resolution: {integrity: sha512-DieYoGrP78PWKsrXr8MZwtQ7GLCUeLxihtjC1jZsW1DnvSMdKPitJSe8OSYDM2u5H6g3kWJZpePqkp43TfLh0g==} @@ -6413,11 +6329,6 @@ packages: victory-vendor@36.9.2: resolution: {integrity: sha512-PnpQQMuxlwYdocC8fIJqVXvkeViHYzotI+NJrCuav0ZYFoq912ZHBk3mCeuj+5/VpodOjPe1z0Fk2ihgzlXqjQ==} - vite-node@2.1.9: - resolution: {integrity: sha512-AM9aQ/IPrW/6ENLQg3AGY4K1N2TGZdR5e4gu/MmmR2xR3Ll1+dib+nook92g4TV3PXVyeyxdWwtaCAiUL0hMxA==} - engines: {node: ^18.0.0 || >=20.0.0} - hasBin: true - vite-tsconfig-paths@4.3.2: resolution: {integrity: sha512-0Vd/a6po6Q+86rPlntHye7F31zA2URZMbH8M3saAZ/xR9QoGN/L21bxEGfXdWmFdNkqPpRdxFT7nmNe12e9/uA==} peerDependencies: @@ -6545,31 +6456,6 @@ packages: vite: optional: true - vitest@2.1.9: - resolution: {integrity: sha512-MSmPM9REYqDGBI8439mA4mWhV5sKmDlBKWIYbA3lRb2PTHACE0mgKwA8yQ2xq9vxDTuk4iPrECBAEW2aoFXY0Q==} - engines: {node: ^18.0.0 || >=20.0.0} - hasBin: true - peerDependencies: - '@edge-runtime/vm': '*' - '@types/node': ^18.0.0 || >=20.0.0 - '@vitest/browser': 2.1.9 - '@vitest/ui': 2.1.9 - happy-dom: '*' - jsdom: '*' - peerDependenciesMeta: - '@edge-runtime/vm': - optional: true - '@types/node': - optional: true - '@vitest/browser': - optional: true - '@vitest/ui': - optional: true - happy-dom: - optional: true - jsdom: - optional: true - vitest@4.0.1: resolution: {integrity: sha512-4rwTfUNF0MExMZBiNirkzZpeyUZGOs3JD76N2qHNP9i6w6/bff7MRv2I9yFJKd1ICxzn2igpra+E4t9o2EfQhw==} engines: {node: ^20.0.0 || ^22.0.0 || >=24.0.0} @@ -9372,13 +9258,6 @@ snapshots: transitivePeerDependencies: - supports-color - '@vitest/expect@2.1.9': - dependencies: - '@vitest/spy': 2.1.9 - '@vitest/utils': 2.1.9 - chai: 5.3.3 - tinyrainbow: 1.2.0 - '@vitest/expect@4.0.1': dependencies: '@standard-schema/spec': 1.0.0 @@ -9388,15 +9267,6 @@ snapshots: chai: 6.2.0 tinyrainbow: 3.0.3 - '@vitest/mocker@2.1.9(msw@2.11.3(@types/node@22.7.7)(typescript@5.6.2))(vite@5.4.11(@types/node@22.7.7)(lightningcss@1.30.1)(terser@5.31.5))': - dependencies: - '@vitest/spy': 2.1.9 - estree-walker: 3.0.3 - magic-string: 0.30.19 - optionalDependencies: - msw: 2.11.3(@types/node@22.7.7)(typescript@5.6.2) - vite: 5.4.11(@types/node@22.7.7)(lightningcss@1.30.1)(terser@5.31.5) - '@vitest/mocker@4.0.1(msw@2.11.3(@types/node@22.7.7)(typescript@5.6.2))(vite@7.1.10(@types/node@22.7.7)(jiti@2.6.1)(lightningcss@1.30.1)(terser@5.31.5)(tsx@4.19.0)(yaml@2.8.1))': dependencies: '@vitest/spy': 4.0.1 @@ -9406,48 +9276,23 @@ snapshots: msw: 2.11.3(@types/node@22.7.7)(typescript@5.6.2) vite: 7.1.10(@types/node@22.7.7)(jiti@2.6.1)(lightningcss@1.30.1)(terser@5.31.5)(tsx@4.19.0)(yaml@2.8.1) - '@vitest/pretty-format@2.1.9': - dependencies: - tinyrainbow: 1.2.0 - '@vitest/pretty-format@4.0.1': dependencies: tinyrainbow: 3.0.3 - '@vitest/runner@2.1.9': - dependencies: - '@vitest/utils': 2.1.9 - pathe: 1.1.2 - '@vitest/runner@4.0.1': dependencies: '@vitest/utils': 4.0.1 pathe: 2.0.3 - '@vitest/snapshot@2.1.9': - dependencies: - '@vitest/pretty-format': 2.1.9 - magic-string: 0.30.19 - pathe: 1.1.2 - '@vitest/snapshot@4.0.1': dependencies: '@vitest/pretty-format': 4.0.1 magic-string: 0.30.19 pathe: 2.0.3 - '@vitest/spy@2.1.9': - dependencies: - tinyspy: 3.0.2 - '@vitest/spy@4.0.1': {} - '@vitest/utils@2.1.9': - dependencies: - '@vitest/pretty-format': 2.1.9 - loupe: 3.2.1 - tinyrainbow: 1.2.0 - '@vitest/utils@4.0.1': dependencies: '@vitest/pretty-format': 4.0.1 @@ -9561,8 +9406,6 @@ snapshots: array-union@2.1.0: {} - assertion-error@2.0.1: {} - astral-regex@2.0.0: {} astring@1.9.0: {} @@ -9784,6 +9627,34 @@ snapshots: dependencies: fill-range: 7.1.1 + braintrust@0.4.9(zod@3.25.76): + dependencies: + '@ai-sdk/provider': 1.1.3 + '@next/env': 14.2.33 + '@vercel/functions': 1.6.0 + argparse: 2.0.1 + chalk: 4.1.2 + cli-progress: 3.12.0 + cors: 2.8.5 + dotenv: 16.6.1 + esbuild: 0.25.11 + eventsource-parser: 1.1.2 + express: 4.21.2 + graceful-fs: 4.2.11 + http-errors: 2.0.0 + minimatch: 9.0.5 + mustache: 4.2.0 + pluralize: 8.0.0 + simple-git: 3.30.0 + slugify: 1.6.6 + source-map: 0.7.4 + uuid: 9.0.1 + zod: 3.25.76 + zod-to-json-schema: 3.23.5(zod@3.25.76) + transitivePeerDependencies: + - '@aws-sdk/credential-provider-web-identity' + - supports-color + braintrust@0.4.9(zod@4.1.12): dependencies: '@ai-sdk/provider': 1.1.3 @@ -9836,8 +9707,6 @@ snapshots: bytes@3.1.2: {} - cac@6.7.14: {} - call-bind-apply-helpers@1.0.2: dependencies: es-errors: 1.3.0 @@ -9858,14 +9727,6 @@ snapshots: ccount@2.0.1: {} - chai@5.3.3: - dependencies: - assertion-error: 2.0.1 - check-error: 2.1.1 - deep-eql: 5.0.2 - loupe: 3.2.1 - pathval: 2.0.1 - chai@6.2.0: {} chalk@4.1.2: @@ -9885,8 +9746,6 @@ snapshots: chardet@2.1.1: {} - check-error@2.1.1: {} - chevrotain-allstar@0.3.1(chevrotain@11.0.3): dependencies: chevrotain: 11.0.3 @@ -10257,8 +10116,6 @@ snapshots: dependencies: mimic-response: 3.1.0 - deep-eql@5.0.2: {} - deep-extend@0.6.0: {} deep-is@0.1.4: {} @@ -11554,8 +11411,6 @@ snapshots: dependencies: js-tokens: 4.0.0 - loupe@3.2.1: {} - lru-cache@10.4.3: {} lru-cache@11.0.2: {} @@ -12398,12 +12253,8 @@ snapshots: path-type@4.0.0: {} - pathe@1.1.2: {} - pathe@2.0.3: {} - pathval@2.0.1: {} - peek-readable@5.3.1: {} picocolors@1.1.1: {} @@ -13438,14 +13289,8 @@ snapshots: fdir: 6.5.0(picomatch@4.0.3) picomatch: 4.0.3 - tinypool@1.1.1: {} - - tinyrainbow@1.2.0: {} - tinyrainbow@3.0.3: {} - tinyspy@3.0.2: {} - tldts-core@7.0.17: optional: true @@ -13705,24 +13550,6 @@ snapshots: d3-time: 3.1.0 d3-timer: 3.0.1 - vite-node@2.1.9(@types/node@22.7.7)(lightningcss@1.30.1)(terser@5.31.5): - dependencies: - cac: 6.7.14 - debug: 4.4.3 - es-module-lexer: 1.7.0 - pathe: 1.1.2 - vite: 5.4.11(@types/node@22.7.7)(lightningcss@1.30.1)(terser@5.31.5) - transitivePeerDependencies: - - '@types/node' - - less - - lightningcss - - sass - - sass-embedded - - stylus - - sugarss - - supports-color - - terser - vite-tsconfig-paths@4.3.2(typescript@5.6.2)(vite@7.1.12(@types/node@22.7.7)(jiti@2.6.1)(lightningcss@1.30.1)(terser@5.31.5)(tsx@4.19.3)(yaml@2.8.1)): dependencies: debug: 4.3.6 @@ -13800,41 +13627,6 @@ snapshots: optionalDependencies: vite: 5.4.11(@types/node@22.7.7)(lightningcss@1.30.1)(terser@5.31.5) - vitest@2.1.9(@types/node@22.7.7)(lightningcss@1.30.1)(msw@2.11.3(@types/node@22.7.7)(typescript@5.6.2))(terser@5.31.5): - dependencies: - '@vitest/expect': 2.1.9 - '@vitest/mocker': 2.1.9(msw@2.11.3(@types/node@22.7.7)(typescript@5.6.2))(vite@5.4.11(@types/node@22.7.7)(lightningcss@1.30.1)(terser@5.31.5)) - '@vitest/pretty-format': 2.1.9 - '@vitest/runner': 2.1.9 - '@vitest/snapshot': 2.1.9 - '@vitest/spy': 2.1.9 - '@vitest/utils': 2.1.9 - chai: 5.3.3 - debug: 4.4.3 - expect-type: 1.2.2 - magic-string: 0.30.19 - pathe: 1.1.2 - std-env: 3.10.0 - tinybench: 2.9.0 - tinyexec: 0.3.2 - tinypool: 1.1.1 - tinyrainbow: 1.2.0 - vite: 5.4.11(@types/node@22.7.7)(lightningcss@1.30.1)(terser@5.31.5) - vite-node: 2.1.9(@types/node@22.7.7)(lightningcss@1.30.1)(terser@5.31.5) - why-is-node-running: 2.3.0 - optionalDependencies: - '@types/node': 22.7.7 - transitivePeerDependencies: - - less - - lightningcss - - msw - - sass - - sass-embedded - - stylus - - sugarss - - supports-color - - terser - vitest@4.0.1(@types/debug@4.1.12)(@types/node@22.7.7)(jiti@2.6.1)(lightningcss@1.30.1)(msw@2.11.3(@types/node@22.7.7)(typescript@5.6.2))(terser@5.31.5)(tsx@4.19.0)(yaml@2.8.1): dependencies: '@vitest/expect': 4.0.1 From 5d5ce3403522f104ac5dbbd82a66fa1e24a9e11e Mon Sep 17 00:00:00 2001 From: Ken Jiang Date: Mon, 15 Dec 2025 17:28:24 -0500 Subject: [PATCH 3/4] dedup code --- packages/evalite/src/braintrust.ts | 341 +++--------------- .../braintrust/braintrust-test.eval.ts | 57 ++- 2 files changed, 96 insertions(+), 302 deletions(-) diff --git a/packages/evalite/src/braintrust.ts b/packages/evalite/src/braintrust.ts index 3f22cfac..926d411b 100644 --- a/packages/evalite/src/braintrust.ts +++ b/packages/evalite/src/braintrust.ts @@ -1,6 +1,7 @@ import type { Evalite } from "./types.js"; import type { Experiment, Span } from "braintrust"; -import { init, _internalGetGlobalState } from "braintrust"; +import { init } from "braintrust"; +import { InMemoryStorage } from "./storage/in-memory.js"; interface BraintrustStorageOptions { /** @@ -21,20 +22,6 @@ interface BraintrustStorageOptions { appUrl?: string; } -/** - * In-memory entity storage for local queries. - * Braintrust doesn't provide direct query APIs during logging, - * so we maintain a local cache of entities. - */ -interface EntityStore { - runs: Map; - suites: Map; - evals: Map; - scores: Map; - traces: Map; - cache: Map; -} - /** * Map Evalite entity IDs to Braintrust Span objects */ @@ -47,26 +34,12 @@ export class BraintrustStorage implements Evalite.Storage { private experiment: Experiment | null = null; private experimentUrl: string | null = null; private options: BraintrustStorageOptions; - private entityStore: EntityStore; + private localStorage: InMemoryStorage; private spanMap: SpanMap; - private nextId = { - run: 1, - suite: 1, - eval: 1, - score: 1, - trace: 1, - }; private constructor(options: BraintrustStorageOptions) { this.options = options; - this.entityStore = { - runs: new Map(), - suites: new Map(), - evals: new Map(), - scores: new Map(), - traces: new Map(), - cache: new Map(), - }; + this.localStorage = InMemoryStorage.create(); this.spanMap = { evals: new Map(), traces: new Map(), @@ -121,59 +94,13 @@ export class BraintrustStorage implements Evalite.Storage { ): Promise => { // Initialize Braintrust experiment when first run is created await this.initExperiment(); - - const run: Evalite.Storage.Entities.Run = { - id: this.nextId.run++, - runType: opts.runType, - created_at: new Date().toISOString(), - }; - - this.entityStore.runs.set(run.id, run); - return run; + return this.localStorage.runs.create(opts); }, - getMany: async ( + getMany: ( opts?: Evalite.Storage.Runs.GetManyOpts ): Promise => { - let runs = Array.from(this.entityStore.runs.values()); - - // Apply filters - if (opts?.ids && opts.ids.length > 0) { - runs = runs.filter((r) => opts.ids!.includes(r.id)); - } - - if (opts?.runType) { - runs = runs.filter((r) => r.runType === opts.runType); - } - - if (opts?.createdAt) { - runs = runs.filter((r) => r.created_at === opts.createdAt); - } - - if (opts?.createdAfter) { - runs = runs.filter((r) => r.created_at > opts.createdAfter!); - } - - if (opts?.createdBefore) { - runs = runs.filter((r) => r.created_at < opts.createdBefore!); - } - - // Apply sorting - const orderBy = opts?.orderBy ?? "created_at"; - const orderDirection = opts?.orderDirection ?? "desc"; - runs.sort((a, b) => { - const aVal = a[orderBy]; - const bVal = b[orderBy]; - const comparison = aVal > bVal ? 1 : aVal < bVal ? -1 : 0; - return orderDirection === "asc" ? comparison : -comparison; - }); - - // Apply limit - if (opts?.limit) { - runs = runs.slice(0, opts.limit); - } - - return runs; + return this.localStorage.runs.getMany(opts); }, }; @@ -184,87 +111,19 @@ export class BraintrustStorage implements Evalite.Storage { // Initialize experiment but don't create a span for the suite // Suites are just organizational units in Evalite - each eval will be a row await this.initExperiment(); - - const suite: Evalite.Storage.Entities.Suite = { - id: this.nextId.suite++, - run_id: opts.runId, - name: opts.name, - status: "running", - filepath: opts.filepath, - duration: 0, - created_at: new Date().toISOString(), - variant_name: opts.variantName, - variant_group: opts.variantGroup, - }; - - this.entityStore.suites.set(suite.id, suite); - return suite; + return this.localStorage.suites.create(opts); }, - update: async ( + update: ( opts: Evalite.Storage.Suites.UpdateOpts ): Promise => { - const suite = this.entityStore.suites.get(opts.id); - if (!suite) { - throw new Error(`Suite with id ${opts.id} not found`); - } - - suite.status = opts.status; - this.entityStore.suites.set(opts.id, suite); - - return suite; + return this.localStorage.suites.update(opts); }, - getMany: async ( + getMany: ( opts?: Evalite.Storage.Suites.GetManyOpts ): Promise => { - let suites = Array.from(this.entityStore.suites.values()); - - // Apply filters - if (opts?.ids && opts.ids.length > 0) { - suites = suites.filter((s) => opts.ids!.includes(s.id)); - } - - if (opts?.runIds && opts.runIds.length > 0) { - suites = suites.filter((s) => opts.runIds!.includes(s.run_id)); - } - - if (opts?.name) { - suites = suites.filter((s) => s.name === opts.name); - } - - if (opts?.statuses && opts.statuses.length > 0) { - suites = suites.filter((s) => opts.statuses!.includes(s.status)); - } - - if (opts?.createdAt) { - suites = suites.filter((s) => s.created_at === opts.createdAt); - } - - if (opts?.createdAfter) { - suites = suites.filter((s) => s.created_at > opts.createdAfter!); - } - - if (opts?.createdBefore) { - suites = suites.filter((s) => s.created_at < opts.createdBefore!); - } - - // Apply sorting - const orderBy = opts?.orderBy ?? "created_at"; - const orderDirection = opts?.orderDirection ?? "desc"; - suites.sort((a, b) => { - const aVal = a[orderBy]; - const bVal = b[orderBy]; - const comparison = aVal > bVal ? 1 : aVal < bVal ? -1 : 0; - return orderDirection === "asc" ? comparison : -comparison; - }); - - // Apply limit - if (opts?.limit) { - suites = suites.slice(0, opts.limit); - } - - return suites; + return this.localStorage.suites.getMany(opts); }, }; @@ -274,22 +133,14 @@ export class BraintrustStorage implements Evalite.Storage { ): Promise => { const experiment = await this.initExperiment(); - const eval_: Evalite.Storage.Entities.Eval = { - id: this.nextId.eval++, - suite_id: opts.suiteId, - duration: opts.duration, - input: opts.input, - output: opts.output, - expected: opts.expected, - created_at: new Date().toISOString(), - col_order: opts.order, - status: opts.status, - rendered_columns: opts.renderedColumns, - trial_index: opts.trialIndex, - }; + // Store locally first to get the ID + const eval_ = await this.localStorage.evals.create(opts); // Get the suite to include its metadata - const suite = this.entityStore.suites.get(opts.suiteId); + const suites = await this.localStorage.suites.getMany({ + ids: [opts.suiteId], + }); + const suite = suites[0]; // Create a top-level span for each eval (test case) // Each eval becomes a row in the Braintrust experiment @@ -309,10 +160,9 @@ export class BraintrustStorage implements Evalite.Storage { }, }); - // Store the span object - this.spanMap.evals.set(eval_.id, span); + // Store the span object using the local ID + this.spanMap.evals.set(eval_.id as number, span); - this.entityStore.evals.set(eval_.id, eval_); return eval_; }, @@ -320,22 +170,12 @@ export class BraintrustStorage implements Evalite.Storage { opts: Evalite.Storage.Evals.UpdateOpts ): Promise => { await this.initExperiment(); - const eval_ = this.entityStore.evals.get(opts.id); - if (!eval_) { - throw new Error(`Eval with id ${opts.id} not found`); - } - // Update the eval entity - eval_.output = opts.output; - eval_.duration = opts.duration; - eval_.input = opts.input; - eval_.expected = opts.expected; - eval_.status = opts.status; - eval_.rendered_columns = opts.renderedColumns; - eval_.trial_index = opts.trialIndex; + // Update locally + const eval_ = await this.localStorage.evals.update(opts); // Get the existing span and log the updated data - const span = this.spanMap.evals.get(opts.id); + const span = this.spanMap.evals.get(opts.id as number); if (span) { span.log({ input: opts.input, @@ -354,36 +194,13 @@ export class BraintrustStorage implements Evalite.Storage { } } - this.entityStore.evals.set(opts.id, eval_); return eval_; }, - getMany: async ( + getMany: ( opts?: Evalite.Storage.Evals.GetManyOpts ): Promise => { - let evals = Array.from(this.entityStore.evals.values()); - - // Apply filters - if (opts?.ids && opts.ids.length > 0) { - evals = evals.filter((e) => opts.ids!.includes(e.id)); - } - - if (opts?.suiteIds && opts.suiteIds.length > 0) { - evals = evals.filter((e) => opts.suiteIds!.includes(e.suite_id)); - } - - if (opts?.order !== undefined) { - evals = evals.filter((e) => e.col_order === opts.order); - } - - if (opts?.statuses && opts.statuses.length > 0) { - evals = evals.filter((e) => opts.statuses!.includes(e.status)); - } - - // Sort by col_order - evals.sort((a, b) => a.col_order - b.col_order); - - return evals; + return this.localStorage.evals.getMany(opts); }, }; @@ -393,46 +210,31 @@ export class BraintrustStorage implements Evalite.Storage { ): Promise => { await this.initExperiment(); - const score: Evalite.Storage.Entities.Score = { - id: this.nextId.score++, - eval_id: opts.evalId, - name: opts.name, - score: opts.score, - description: opts.description, - metadata: opts.metadata, - created_at: new Date().toISOString(), - }; + // Store locally + const score = await this.localStorage.scores.create(opts); // Get the eval's span and log the score to it - const span = this.spanMap.evals.get(opts.evalId); + const span = this.spanMap.evals.get(opts.evalId as number); if (span) { + // Braintrust requires scores to be between 0 and 1 + // Clamp the score to this range, treating null as 0 + const normalizedScore = Math.max(0, Math.min(1, opts.score ?? 0)); + // Add scores to the span span.log({ scores: { - [opts.name]: opts.score, + [opts.name]: normalizedScore, }, }); } - this.entityStore.scores.set(score.id, score); return score; }, - getMany: async ( + getMany: ( opts?: Evalite.Storage.Scores.GetManyOpts ): Promise => { - let scores = Array.from(this.entityStore.scores.values()); - - // Apply filters - if (opts?.ids && opts.ids.length > 0) { - scores = scores.filter((s) => opts.ids!.includes(s.id)); - } - - if (opts?.evalIds && opts.evalIds.length > 0) { - scores = scores.filter((s) => opts.evalIds!.includes(s.eval_id)); - } - - return scores; + return this.localStorage.scores.getMany(opts); }, }; @@ -442,21 +244,11 @@ export class BraintrustStorage implements Evalite.Storage { ): Promise => { const experiment = await this.initExperiment(); - const trace: Evalite.Storage.Entities.Trace = { - id: this.nextId.trace++, - eval_id: opts.evalId, - input: opts.input, - output: opts.output, - start_time: opts.start, - end_time: opts.end, - input_tokens: opts.inputTokens, - output_tokens: opts.outputTokens, - total_tokens: opts.totalTokens, - col_order: opts.order, - }; + // Store locally first + const trace = await this.localStorage.traces.create(opts); // Get the eval's span as parent - const parentSpan = this.spanMap.evals.get(opts.evalId); + const parentSpan = this.spanMap.evals.get(opts.evalId as number); // Create a nested span for the trace const parent = parentSpan ? await parentSpan.export() : undefined; @@ -478,74 +270,40 @@ export class BraintrustStorage implements Evalite.Storage { }, }); - this.spanMap.traces.set(trace.id, span); + this.spanMap.traces.set(trace.id as number, span); // Convert performance.now() timestamp to Unix epoch seconds span.end({ endTime: (performance.timeOrigin + opts.end) / 1000 }); - this.entityStore.traces.set(trace.id, trace); return trace; }, - getMany: async ( + getMany: ( opts?: Evalite.Storage.Traces.GetManyOpts ): Promise => { - let traces = Array.from(this.entityStore.traces.values()); - - // Apply filters - if (opts?.ids && opts.ids.length > 0) { - traces = traces.filter((t) => opts.ids!.includes(t.id)); - } - - if (opts?.evalIds && opts.evalIds.length > 0) { - traces = traces.filter((t) => opts.evalIds!.includes(t.eval_id)); - } - - // Sort by col_order - traces.sort((a, b) => a.col_order - b.col_order); - - return traces; + return this.localStorage.traces.getMany(opts); }, }; cache = { - get: async ( + get: ( keyHash: string ): Promise<{ value: unknown; duration: number } | null> => { - const entry = this.entityStore.cache.get(keyHash); - if (!entry) { - return null; - } - - // Check if cache is expired (older than 24 hours) - const oneDayAgo = Date.now() - 24 * 60 * 60 * 1000; - if (entry.created_at < oneDayAgo) { - this.entityStore.cache.delete(keyHash); - return null; - } - - return { - value: entry.value, - duration: entry.duration, - }; + return this.localStorage.cache.get(keyHash); }, - set: async ( + set: ( keyHash: string, data: { value: unknown; duration: number } ): Promise => { - this.entityStore.cache.set(keyHash, { - value: data.value, - duration: data.duration, - created_at: Date.now(), - }); + return this.localStorage.cache.set(keyHash, data); }, - delete: async (keyHash: string): Promise => { - this.entityStore.cache.delete(keyHash); + delete: (keyHash: string): Promise => { + return this.localStorage.cache.delete(keyHash); }, - clear: async (): Promise => { - this.entityStore.cache.clear(); + clear: (): Promise => { + return this.localStorage.cache.clear(); }, }; @@ -554,6 +312,7 @@ export class BraintrustStorage implements Evalite.Storage { // Flush any pending logs to Braintrust await this.experiment.flush(); } + await this.localStorage.close(); } async [Symbol.asyncDispose](): Promise { diff --git a/packages/example/braintrust/braintrust-test.eval.ts b/packages/example/braintrust/braintrust-test.eval.ts index cf4cd29e..b02dcaae 100644 --- a/packages/example/braintrust/braintrust-test.eval.ts +++ b/packages/example/braintrust/braintrust-test.eval.ts @@ -1,24 +1,59 @@ +import { openai } from "@ai-sdk/openai"; +import { generateText } from "ai"; import { evalite } from "evalite"; +import { wrapAISDKModel } from "evalite/ai-sdk"; +import { answerRelevancy } from "evalite/scorers"; + +const model = wrapAISDKModel(openai("gpt-4.1-mini")); /** - * Simple test eval to validate Braintrust storage integration + * Test eval to validate Braintrust storage integration with LLM calls + * and LLM-as-a-judge scoring */ evalite("Braintrust Storage Test", { data: () => [ - { input: "Hello", expected: "Hello, World!" }, - { input: "Goodbye", expected: "Goodbye, World!" }, - { input: "Test", expected: "Test, World!" }, + { + input: "What is the capital of France?", + expected: "Paris", + }, + { + input: "What is the capital of Japan?", + expected: "Tokyo", + }, + { + input: "What is the capital of Germany?", + expected: "Berlin", + }, ], task: async (input) => { - // Simple task that adds ", World!" to the input - return `${input}, World!`; + const result = await generateText({ + model, + prompt: input, + system: + "You are a geography assistant. Answer with just the city name, nothing else.", + }); + return result.text; }, scorers: [ - (input, output, expected) => { - return { - name: "exact_match", - score: output === expected ? 1 : 0, - }; + { + name: "exact_match", + scorer: ({ output, expected }) => { + const normalizedOutput = output.trim().toLowerCase(); + const normalizedExpected = expected.trim().toLowerCase(); + return normalizedOutput === normalizedExpected ? 1 : 0; + }, + }, + { + name: "Answer Relevancy", + description: + "LLM-as-a-judge: evaluates if the answer is relevant to the question", + scorer: ({ input, output }) => + answerRelevancy({ + question: input, + answer: output, + model, + embeddingModel: openai.embedding("text-embedding-3-small"), + }), }, ], }); From 43f83323bddf8dfe07c50e3c3c9e0a7352b19119 Mon Sep 17 00:00:00 2001 From: Ken Jiang Date: Mon, 15 Dec 2025 17:31:16 -0500 Subject: [PATCH 4/4] keep close() inside condtional --- packages/evalite/src/run-evalite.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/evalite/src/run-evalite.ts b/packages/evalite/src/run-evalite.ts index 5e8255df..727c7c4a 100644 --- a/packages/evalite/src/run-evalite.ts +++ b/packages/evalite/src/run-evalite.ts @@ -400,10 +400,10 @@ export const runEvalite = async (opts: { if (experimentUrl) { console.log(`\n View results in Braintrust: ${experimentUrl}\n`); } - } - // Close storage to flush any pending data (e.g., to Braintrust) - await storage.close(); + // Close storage to flush any pending data to Braintrust + await storage.close(); + } if (typeof exitCode === "number" && opts.mode === "run-once-and-exit") { process.exit(exitCode);