Add headless evals flags example

examples/example-evals-flags/.env.example

@@ -0,0 +1,6 @@
+AXIOM_URL="https://api.axiom.co"
+AXIOM_TOKEN="xaat-******"
+AXIOM_DATASET="my_dataset"
+AXIOM_EVAL_DATASET="my_eval_dataset"
+
+OPENAI_API_KEY="sk-proj-******"

examples/example-evals-flags/.gitignore

@@ -0,0 +1,41 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.*
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/versions
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# local env files
+.env
+.env*.local
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts

examples/example-evals-flags/README.md

@@ -0,0 +1,40 @@
+# Example: Evals - Writing Agent
+
+A headless example demonstrating trace capture with Axiom's AI SDK for a marketplace listing writer.
+
+## What This Shows
+
+- **Trace capture** with `withSpan` and `wrapAISDKModel` 
+- **Tool calling** with `wrapTools` for content moderation
+- **Headless architecture** - runs via CLI, no web framework
+
+## The Use Case
+
+A writing agent for "Acme" marketplace that transforms raw seller descriptions into polished product listings. Includes a `checkProhibitedItems` tool to demonstrate tool tracing.
+
+**Example:**
+- **Input**: `"Used flip flops. Blue color. Size 9. Decent condition."`
+- **Output**: `"Comfortable blue flip flops in size 9, perfect for casual summer wear..."`
+
+## Quick Start
+
+```bash
+# Install dependencies (from repo root)
+pnpm install
+
+# Set environment variables
+export AXIOM_URL="your-axiom-url"
+export AXIOM_TOKEN="your-axiom-token"  
+export AXIOM_DATASET="your-dataset"
+export OPENAI_API_KEY="your-openai-key"
+
+# Generate a single listing (with trace)
+npm run generate
+```
+
+## Key Files
+
+- `src/lib/service.ts` - Writing agent with tool calling
+- `src/lib/tools/check-prohibited-items.ts` - Content moderation tool
+- `src/lib/scripts/generate.ts` - CLI script for generations
+

examples/example-evals-flags/axiom.config.ts

@@ -0,0 +1,17 @@
+import { defineConfig } from 'axiom/ai/config';
+import { setupAppInstrumentation } from './src/instrumentation.node';
+
+export default defineConfig({
+  eval: {
+    url: process.env.AXIOM_URL,
+    token: process.env.AXIOM_TOKEN,
+    dataset: process.env.AXIOM_EVAL_DATASET,
+
+    include: ['**/*.eval.{ts,js}'],
+    exclude: [],
+
+    instrumentation: ({ url, token, dataset }) => setupAppInstrumentation({ url, token, dataset }),
+
+    timeoutMs: 60_000,
+  },
+});

examples/example-evals-flags/env.ts

@@ -0,0 +1,21 @@
+import { z } from 'zod';
+
+const envSchema = z.object({
+  AXIOM_URL: z.string(),
+  AXIOM_TOKEN: z.string(),
+  AXIOM_DATASET: z.string(),
+  AXIOM_EVAL_DATASET: z.string(),
+  OPENAI_API_KEY: z.string(),
+});
+
+try {
+  envSchema.parse(process.env);
+} catch (error) {
+  console.error(error);
+}
+
+declare global {
+  namespace NodeJS {
+    interface ProcessEnv extends z.infer<typeof envSchema> {}
+  }
+}

examples/example-evals-flags/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "example-evals-flags",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "eval": "node ../../packages/ai/dist/bin.js eval .",
+    "generate": "tsx src/lib/scripts/generate.ts",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@ai-sdk/openai": "2.0.0-beta.11",
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.202.0",
+    "@opentelemetry/resources": "^2.1.0",
+    "@opentelemetry/sdk-trace-node": "^2.0.1",
+    "@opentelemetry/semantic-conventions": "^1.34.0",
+    "ai": "5.0.0-beta.24",
+    "axiom": "workspace:*",
+    "zod": "catalog:"
+  },
+  "devDependencies": {
+    "@types/node": "^22.17.2",
+    "autoevals": "^0.0.130",
+    "chalk": "^5.3.0",
+    "dotenv": "^16.4.5",
+    "tsx": "^4.19.2",
+    "typescript": "catalog:"
+  }
+}

examples/example-evals-flags/src/instrumentation.node.ts

@@ -0,0 +1,51 @@
+import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
+import { resourceFromAttributes } from '@opentelemetry/resources';
+import { BatchSpanProcessor, NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
+import { ATTR_SERVICE_NAME } from '@opentelemetry/semantic-conventions';
+import { initAxiomAI, RedactionPolicy } from 'axiom/ai';
+import type { AxiomEvalInstrumentationHook } from 'axiom/ai/config';
+import { tracer } from './tracer';
+
+let provider: NodeTracerProvider | undefined;
+
+export const setupAppInstrumentation: AxiomEvalInstrumentationHook = async (options) => {
+  if (provider) {
+    return { provider };
+  }
+
+  const dataset = options.dataset;
+  const url = options.url;
+  const token = options.token;
+
+  if (!dataset) {
+    throw new Error('Dataset is required to initialize tracing');
+  }
+
+  if (!url) {
+    throw new Error('URL is required to initialize tracing');
+  }
+
+  if (!token) {
+    throw new Error('Token is required to initialize tracing');
+  }
+
+  const exporter = new OTLPTraceExporter({
+    url: `${url}/v1/traces`,
+    headers: {
+      Authorization: `Bearer ${token}`,
+      'X-Axiom-Dataset': dataset,
+    },
+  });
+
+  provider = new NodeTracerProvider({
+    resource: resourceFromAttributes({
+      [ATTR_SERVICE_NAME]: 'example-evals-flags',
+    }),
+    spanProcessors: [new BatchSpanProcessor(exporter)],
+  });
+
+  provider.register();
+  initAxiomAI({ tracer, redactionPolicy: RedactionPolicy.AxiomDefault });
+
+  return { provider };
+};

examples/example-evals-flags/src/instrumentation.ts

@@ -0,0 +1,10 @@
+export async function register() {
+  if (process.env.NEXT_RUNTIME === 'nodejs') {
+    const { setupAppInstrumentation } = await import('./instrumentation.node');
+    await setupAppInstrumentation({
+      url: process.env.AXIOM_URL,
+      token: process.env.AXIOM_TOKEN,
+      dataset: process.env.AXIOM_DATASET,
+    });
+  }
+}