diff --git a/index.ts b/index.ts index 99c68b0..b925708 100644 --- a/index.ts +++ b/index.ts @@ -19,159 +19,90 @@ import { runTestVerification, } from "./lib/output-test-runner.ts"; import { resultWriteTool, testComponentTool } from "./lib/tools/index.ts"; +import { getModelPricingDisplay, formatCost, formatMTokCost } from "./lib/pricing.ts"; import { - buildPricingMap, - lookupPricingFromMap, - getModelPricingDisplay, - formatCost, - formatMTokCost, -} from "./lib/pricing.ts"; -import type { LanguageModel } from "ai"; + gateway, + getGatewayModelsAndPricing, + selectModelsFromGateway, + type PricingMap, + type PricingLookup, + type PricingResult, +} from "./lib/providers/ai-gateway.ts"; import { - intro, - multiselect, - isCancel, - cancel, - text, - select, - confirm, - note, -} from "@clack/prompts"; -import { gateway } from "ai"; - -async function validateAndConfirmPricing( - models: string[], - pricingMap: ReturnType, -) { - const lookups = new Map>(); - - for (const modelId of models) { - const lookup = lookupPricingFromMap(modelId, pricingMap); - lookups.set(modelId, lookup); - } - - const modelsWithPricing = models.filter((m) => lookups.get(m) !== null); - const modelsWithoutPricing = models.filter((m) => lookups.get(m) === null); - - if (modelsWithoutPricing.length === 0) { - const pricingLines = models.map((modelId) => { - const lookup = lookups.get(modelId)!; - const display = getModelPricingDisplay(lookup.pricing); - const cacheReadText = - display.cacheReadCostPerMTok !== undefined - ? `, ${formatMTokCost(display.cacheReadCostPerMTok)}/MTok cache read` - : ""; - const cacheWriteText = - display.cacheCreationCostPerMTok !== undefined - ? `, ${formatMTokCost(display.cacheCreationCostPerMTok)}/MTok cache write` - : ""; - return `${modelId}\n → ${formatMTokCost(display.inputCostPerMTok)}/MTok in, ${formatMTokCost(display.outputCostPerMTok)}/MTok out${cacheReadText}${cacheWriteText}`; - }); - - note(pricingLines.join("\n\n"), "šŸ’° Pricing Found"); - - const usePricing = await confirm({ - message: "Enable cost calculation?", - initialValue: true, - }); - - if (isCancel(usePricing)) { - cancel("Operation cancelled."); - process.exit(0); - } - - return { enabled: usePricing, lookups }; - } else { - const lines: string[] = []; - - if (modelsWithoutPricing.length > 0) { - lines.push("No pricing found for:"); - for (const modelId of modelsWithoutPricing) { - lines.push(` āœ— ${modelId}`); - } - } - - if (modelsWithPricing.length > 0) { - lines.push(""); - lines.push("Pricing available for:"); - for (const modelId of modelsWithPricing) { - const lookup = lookups.get(modelId)!; - const display = getModelPricingDisplay(lookup.pricing); - const cacheReadText = - display.cacheReadCostPerMTok !== undefined - ? `, ${formatMTokCost(display.cacheReadCostPerMTok)}/MTok cache read` - : ""; - const cacheWriteText = - display.cacheCreationCostPerMTok !== undefined - ? `, ${formatMTokCost(display.cacheCreationCostPerMTok)}/MTok cache write` - : ""; - lines.push( - ` āœ“ ${modelId} (${formatMTokCost(display.inputCostPerMTok)}/MTok in, ${formatMTokCost(display.outputCostPerMTok)}/MTok out${cacheReadText}${cacheWriteText})`, - ); - } - } + configureLMStudio, + selectModelsFromLMStudio, + getLMStudioModel, + isLMStudioModel, + type LMStudioConfig, +} from "./lib/providers/lmstudio.ts"; +import type { LanguageModel } from "ai"; +import { intro, isCancel, cancel, select, confirm, text } from "@clack/prompts"; +import { buildPricingMap } from "./lib/pricing.ts"; - lines.push(""); - lines.push("Cost calculation will be disabled."); +type ProviderType = "gateway" | "lmstudio"; - note(lines.join("\n"), "āš ļø Pricing Incomplete"); +interface ProviderConfig { + type: ProviderType; + lmstudio?: LMStudioConfig; +} - const proceed = await confirm({ - message: "Continue without pricing?", - initialValue: true, - }); +async function selectProvider(): Promise { + const provider = await select({ + message: "Select model provider", + options: [ + { + value: "gateway", + label: "Vercel AI Gateway", + hint: "Cloud-hosted models via Vercel", + }, + { + value: "lmstudio", + label: "LM Studio", + hint: "Local models via LM Studio", + }, + ], + initialValue: "gateway", + }); - if (isCancel(proceed) || !proceed) { - cancel("Operation cancelled."); - process.exit(0); - } + if (isCancel(provider)) { + cancel("Operation cancelled."); + process.exit(0); + } - return { enabled: false, lookups }; + if (provider === "lmstudio") { + const lmstudioConfig = await configureLMStudio(); + return { type: "lmstudio", lmstudio: lmstudioConfig }; } + + return { type: "gateway" }; } async function selectOptions() { intro("šŸš€ Svelte AI Bench"); - const available_models = await gateway.getAvailableModels(); - - const pricingMap = buildPricingMap(available_models.models); + const providerConfig = await selectProvider(); - const models = await multiselect({ - message: "Select model(s) to benchmark", - options: [{ value: "custom", label: "Custom" }].concat( - available_models.models.reduce>( - (arr, model) => { - if (model.modelType === "language") { - arr.push({ value: model.id, label: model.name }); - } - return arr; - }, - [], - ), - ), - }); - - if (isCancel(models)) { - cancel("Operation cancelled."); - process.exit(0); - } + let pricingMap: PricingMap; + let selectedModels: string[]; + let pricing: PricingResult; - if (models.includes("custom")) { - const custom_model = await text({ - message: "Enter custom model id", - }); - if (isCancel(custom_model)) { - cancel("Operation cancelled."); - process.exit(0); - } - models.push(custom_model); + if (providerConfig.type === "gateway") { + const gatewayData = await getGatewayModelsAndPricing(); + pricingMap = gatewayData.pricingMap; + const result = await selectModelsFromGateway(pricingMap); + selectedModels = result.selectedModels; + pricing = result.pricing; + } else { + pricingMap = buildPricingMap([]); + selectedModels = await selectModelsFromLMStudio( + providerConfig.lmstudio!.baseURL, + ); + pricing = { + enabled: false, + lookups: new Map(), + }; } - const selectedModels = models.filter((model) => model !== "custom"); - - const pricing = await validateAndConfirmPricing(selectedModels, pricingMap); - const mcp_integration = await select({ message: "Which MCP integration to use?", options: [ @@ -233,6 +164,7 @@ async function selectOptions() { mcp, testingTool, pricing, + providerConfig, }; } @@ -246,6 +178,17 @@ function parseCommandString(commandString: string): { return { command, args }; } +function getModelForId( + modelId: string, + providerConfig: ProviderConfig, +): LanguageModel { + if (isLMStudioModel(modelId)) { + return getLMStudioModel(modelId, providerConfig.lmstudio?.baseURL); + } + + return gateway.languageModel(modelId); +} + async function runSingleTest( test: TestDefinition, model: LanguageModel, @@ -375,7 +318,8 @@ async function runSingleTest( } async function main() { - const { models, mcp, testingTool, pricing } = await selectOptions(); + const { models, mcp, testingTool, pricing, providerConfig } = + await selectOptions(); const mcpServerUrl = mcp; const mcpEnabled = !!mcp; @@ -389,6 +333,13 @@ async function main() { console.log("ā•‘ SvelteBench 2.0 - Multi-Test ā•‘"); console.log("ā•šā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•"); + console.log( + `\nšŸ”Œ Provider: ${providerConfig.type === "gateway" ? "Vercel AI Gateway" : "LM Studio"}`, + ); + if (providerConfig.type === "lmstudio" && providerConfig.lmstudio) { + console.log(` URL: ${providerConfig.lmstudio.baseURL}`); + } + console.log("\nšŸ“‹ Models:"); for (const modelId of models) { const lookup = pricing.lookups.get(modelId); @@ -408,6 +359,9 @@ async function main() { ); } else { console.log(` ${modelId}`); + if (isLMStudioModel(modelId)) { + console.log(` šŸ–„ļø Local model (free)`); + } } } @@ -486,7 +440,7 @@ async function main() { ); } - const model = gateway.languageModel(modelId); + const model = getModelForId(modelId, providerConfig); const testResults = []; const startTime = Date.now(); @@ -566,7 +520,6 @@ async function main() { } console.log(`Total cost: ${formatCost(totalCost.totalCost)}`); - // Simulate cache savings cacheSimulation = simulateCacheSavings( testResults, pricingLookup.pricing, @@ -620,10 +573,17 @@ async function main() { mcpTransportType: mcpEnabled ? mcpTransportType : null, timestamp: new Date().toISOString(), model: modelId, + provider: providerConfig.type, pricingKey: pricingLookup?.matchedKey ?? null, pricing: pricingInfo, totalCost, cacheSimulation, + lmstudio: + providerConfig.type === "lmstudio" && providerConfig.lmstudio + ? { + baseURL: providerConfig.lmstudio.baseURL, + } + : null, }, }; diff --git a/lib/providers/ai-gateway.ts b/lib/providers/ai-gateway.ts new file mode 100644 index 0000000..4d7eac5 --- /dev/null +++ b/lib/providers/ai-gateway.ts @@ -0,0 +1,155 @@ +import { gateway } from "ai"; +import { multiselect, isCancel, cancel, text, confirm, note } from "@clack/prompts"; +import { + buildPricingMap, + lookupPricingFromMap, + getModelPricingDisplay, + formatMTokCost, +} from "../pricing.ts"; + +export { gateway }; + +export type PricingMap = ReturnType; +export type PricingLookup = ReturnType; + +export interface PricingResult { + enabled: boolean; + lookups: Map; +} + +export async function getGatewayModelsAndPricing() { + const availableModels = await gateway.getAvailableModels(); + const pricingMap = buildPricingMap(availableModels.models); + return { models: availableModels.models, pricingMap }; +} + +export async function validateAndConfirmPricing( + models: string[], + pricingMap: PricingMap, +): Promise { + const lookups = new Map(); + + for (const modelId of models) { + const lookup = lookupPricingFromMap(modelId, pricingMap); + lookups.set(modelId, lookup); + } + + const modelsWithPricing = models.filter((m) => lookups.get(m) !== null); + const modelsWithoutPricing = models.filter((m) => lookups.get(m) === null); + + if (modelsWithoutPricing.length === 0) { + const pricingLines = models.map((modelId) => { + const lookup = lookups.get(modelId)!; + const display = getModelPricingDisplay(lookup.pricing); + const cacheReadText = + display.cacheReadCostPerMTok !== undefined + ? `, ${formatMTokCost(display.cacheReadCostPerMTok)}/MTok cache read` + : ""; + const cacheWriteText = + display.cacheCreationCostPerMTok !== undefined + ? `, ${formatMTokCost(display.cacheCreationCostPerMTok)}/MTok cache write` + : ""; + return `${modelId}\n → ${formatMTokCost(display.inputCostPerMTok)}/MTok in, ${formatMTokCost(display.outputCostPerMTok)}/MTok out${cacheReadText}${cacheWriteText}`; + }); + + note(pricingLines.join("\n\n"), "šŸ’° Pricing Found"); + + const usePricing = await confirm({ + message: "Enable cost calculation?", + initialValue: true, + }); + + if (isCancel(usePricing)) { + cancel("Operation cancelled."); + process.exit(0); + } + + return { enabled: usePricing, lookups }; + } else { + const lines: string[] = []; + + if (modelsWithoutPricing.length > 0) { + lines.push("No pricing found for:"); + for (const modelId of modelsWithoutPricing) { + lines.push(` āœ— ${modelId}`); + } + } + + if (modelsWithPricing.length > 0) { + lines.push(""); + lines.push("Pricing available for:"); + for (const modelId of modelsWithPricing) { + const lookup = lookups.get(modelId)!; + const display = getModelPricingDisplay(lookup.pricing); + const cacheReadText = + display.cacheReadCostPerMTok !== undefined + ? `, ${formatMTokCost(display.cacheReadCostPerMTok)}/MTok cache read` + : ""; + const cacheWriteText = + display.cacheCreationCostPerMTok !== undefined + ? `, ${formatMTokCost(display.cacheCreationCostPerMTok)}/MTok cache write` + : ""; + lines.push( + ` āœ“ ${modelId} (${formatMTokCost(display.inputCostPerMTok)}/MTok in, ${formatMTokCost(display.outputCostPerMTok)}/MTok out${cacheReadText}${cacheWriteText})`, + ); + } + } + + lines.push(""); + lines.push("Cost calculation will be disabled."); + + note(lines.join("\n"), "āš ļø Pricing Incomplete"); + + const proceed = await confirm({ + message: "Continue without pricing?", + initialValue: true, + }); + + if (isCancel(proceed) || !proceed) { + cancel("Operation cancelled."); + process.exit(0); + } + + return { enabled: false, lookups }; + } +} + +export async function selectModelsFromGateway(pricingMap: PricingMap) { + const availableModels = await gateway.getAvailableModels(); + + const models = await multiselect({ + message: "Select model(s) to benchmark", + options: [{ value: "custom", label: "Custom" }].concat( + availableModels.models.reduce>( + (arr, model) => { + if (model.modelType === "language") { + arr.push({ value: model.id, label: model.name }); + } + return arr; + }, + [], + ), + ), + }); + + if (isCancel(models)) { + cancel("Operation cancelled."); + process.exit(0); + } + + if (models.includes("custom")) { + const customModel = await text({ + message: "Enter custom model id", + }); + if (isCancel(customModel)) { + cancel("Operation cancelled."); + process.exit(0); + } + models.push(customModel); + } + + const selectedModels = models.filter((model) => model !== "custom"); + const pricing = await validateAndConfirmPricing(selectedModels, pricingMap); + + return { selectedModels, pricing }; +} diff --git a/lib/providers/lmstudio.ts b/lib/providers/lmstudio.ts new file mode 100644 index 0000000..f763d5b --- /dev/null +++ b/lib/providers/lmstudio.ts @@ -0,0 +1,154 @@ +import { createOpenAICompatible } from "@ai-sdk/openai-compatible"; +import { + multiselect, + isCancel, + cancel, + confirm, + text, + spinner, + note, +} from "@clack/prompts"; +import type { LanguageModel } from "ai"; + +export function createLMStudioProvider( + baseURL: string = "http://localhost:1234/v1", +) { + return createOpenAICompatible({ + name: "lmstudio", + baseURL, + }); +} + +export const lmstudio = createLMStudioProvider(); + +export interface LMStudioModel { + id: string; + object: string; + owned_by: string; +} + +interface LMStudioModelsResponse { + object: string; + data: LMStudioModel[]; +} + +export interface LMStudioConfig { + baseURL: string; +} + +export async function fetchLMStudioModels( + baseURL: string = "http://localhost:1234/v1", +): Promise { + try { + const response = await fetch(`${baseURL}/models`, { + method: "GET", + headers: { + "Content-Type": "application/json", + }, + }); + + if (!response.ok) { + console.error( + `Failed to fetch LM Studio models: ${response.status} ${response.statusText}`, + ); + return null; + } + + const data = (await response.json()) as LMStudioModelsResponse; + return data.data || []; + } catch { + return null; + } +} + +export async function configureLMStudio(): Promise { + const customUrl = await confirm({ + message: "Use custom LM Studio URL? (default: http://localhost:1234/v1)", + initialValue: false, + }); + + if (isCancel(customUrl)) { + cancel("Operation cancelled."); + process.exit(0); + } + + let baseURL = "http://localhost:1234/v1"; + + if (customUrl) { + const urlInput = await text({ + message: "Enter LM Studio server URL", + placeholder: "http://localhost:1234/v1", + }); + + if (isCancel(urlInput)) { + cancel("Operation cancelled."); + process.exit(0); + } + + baseURL = urlInput || "http://localhost:1234/v1"; + } + + return { baseURL }; +} + +export async function selectModelsFromLMStudio( + baseURL: string, +): Promise { + const s = spinner(); + s.start("Connecting to LM Studio..."); + + const lmstudioModels = await fetchLMStudioModels(baseURL); + + if (lmstudioModels === null) { + s.stop("Failed to connect to LM Studio"); + note( + `Could not connect to LM Studio at ${baseURL}\n\nMake sure:\n1. LM Studio is running\n2. A model is loaded\n3. The local server is started (Local Server tab → Start Server)`, + "āŒ Connection Failed", + ); + cancel("Cannot proceed without LM Studio connection."); + process.exit(1); + } + + if (lmstudioModels.length === 0) { + s.stop("No models found"); + note( + `LM Studio is running but no models are loaded.\n\nPlease load a model in LM Studio and try again.`, + "āš ļø No Models Available", + ); + cancel("Cannot proceed without loaded models."); + process.exit(1); + } + + s.stop(`Found ${lmstudioModels.length} model(s)`); + + const models = await multiselect({ + message: "Select model(s) to benchmark", + options: lmstudioModels.map((model) => ({ + value: model.id, + label: model.id, + hint: model.owned_by !== "unknown" ? `by ${model.owned_by}` : undefined, + })), + }); + + if (isCancel(models)) { + cancel("Operation cancelled."); + process.exit(0); + } + + return models.map((m) => `lmstudio/${m}`); +} + +export function getLMStudioModel( + modelId: string, + baseURL?: string, +): LanguageModel { + const actualModelId = modelId.startsWith("lmstudio/") + ? modelId.replace("lmstudio/", "") + : modelId; + const provider = createLMStudioProvider(baseURL); + return provider(actualModelId); +} + +export function isLMStudioModel(modelId: string): boolean { + return modelId.startsWith("lmstudio/"); +} diff --git a/lib/report.ts b/lib/report.ts index a2ff3c2..d2a671d 100644 --- a/lib/report.ts +++ b/lib/report.ts @@ -82,16 +82,22 @@ export interface TotalCostInfo { cachedInputTokens: number; } +interface LMStudioMetadata { + baseURL: string; +} + interface Metadata { mcpEnabled: boolean; mcpServerUrl: string | null; mcpTransportType?: string | null; timestamp: string; model: string; + provider?: "gateway" | "lmstudio"; pricingKey?: string | null; pricing?: PricingInfo | null; totalCost?: TotalCostInfo | null; cacheSimulation?: ReturnType | null; + lmstudio?: LMStudioMetadata | null; } export interface SingleTestResult {