From 018b7d006db69d43e2597b8337eaf4c4a610c340 Mon Sep 17 00:00:00 2001 From: Aditya Puranik Date: Fri, 16 Jan 2026 23:13:27 +0000 Subject: [PATCH 1/4] feat(providers): add LiteLLM provider for Agent blocks --- .../app/api/providers/litellm/models/route.ts | 69 ++ .../providers/provider-models-loader.tsx | 4 + apps/sim/components/icons.tsx | 16 + apps/sim/hooks/queries/providers.ts | 1 + apps/sim/lib/core/config/env.ts | 2 + apps/sim/providers/litellm/index.ts | 657 ++++++++++++++++++ apps/sim/providers/litellm/utils.ts | 14 + apps/sim/providers/models.ts | 26 + apps/sim/providers/registry.ts | 2 + apps/sim/providers/types.ts | 1 + apps/sim/providers/utils.ts | 18 +- apps/sim/stores/providers/store.ts | 1 + apps/sim/stores/providers/types.ts | 2 +- 13 files changed, 811 insertions(+), 2 deletions(-) create mode 100644 apps/sim/app/api/providers/litellm/models/route.ts create mode 100644 apps/sim/providers/litellm/index.ts create mode 100644 apps/sim/providers/litellm/utils.ts diff --git a/apps/sim/app/api/providers/litellm/models/route.ts b/apps/sim/app/api/providers/litellm/models/route.ts new file mode 100644 index 0000000000..ed1376385a --- /dev/null +++ b/apps/sim/app/api/providers/litellm/models/route.ts @@ -0,0 +1,69 @@ +import { createLogger } from '@sim/logger' +import { type NextRequest, NextResponse } from 'next/server' +import { env } from '@/lib/core/config/env' +import { filterBlacklistedModels, isProviderBlacklisted } from '@/providers/utils' + +const logger = createLogger('LiteLLMModelsAPI') + +/** + * Get available LiteLLM models + */ +export async function GET(_request: NextRequest) { + if (isProviderBlacklisted('litellm')) { + logger.info('LiteLLM provider is blacklisted, returning empty models') + return NextResponse.json({ models: [] }) + } + + const baseUrl = (env.LITELLM_BASE_URL || '').replace(/\/$/, '') + + if (!baseUrl) { + logger.info('LITELLM_BASE_URL not configured') + return NextResponse.json({ models: [] }) + } + + try { + logger.info('Fetching LiteLLM models', { + baseUrl, + }) + + const headers: Record = { + 'Content-Type': 'application/json', + } + + if (env.LITELLM_API_KEY) { + headers.Authorization = `Bearer ${env.LITELLM_API_KEY}` + } + + const response = await fetch(`${baseUrl}/v1/models`, { + headers, + next: { revalidate: 60 }, + }) + + if (!response.ok) { + logger.warn('LiteLLM service is not available', { + status: response.status, + statusText: response.statusText, + }) + return NextResponse.json({ models: [] }) + } + + const data = (await response.json()) as { data: Array<{ id: string }> } + const allModels = data.data.map((model) => `litellm/${model.id}`) + const models = filterBlacklistedModels(allModels) + + logger.info('Successfully fetched LiteLLM models', { + count: models.length, + filtered: allModels.length - models.length, + models, + }) + + return NextResponse.json({ models }) + } catch (error) { + logger.error('Failed to fetch LiteLLM models', { + error: error instanceof Error ? error.message : 'Unknown error', + baseUrl, + }) + + return NextResponse.json({ models: [] }) + } +} diff --git a/apps/sim/app/workspace/[workspaceId]/providers/provider-models-loader.tsx b/apps/sim/app/workspace/[workspaceId]/providers/provider-models-loader.tsx index 06344ae759..628e8434ed 100644 --- a/apps/sim/app/workspace/[workspaceId]/providers/provider-models-loader.tsx +++ b/apps/sim/app/workspace/[workspaceId]/providers/provider-models-loader.tsx @@ -4,6 +4,7 @@ import { useEffect } from 'react' import { createLogger } from '@sim/logger' import { useProviderModels } from '@/hooks/queries/providers' import { + updateLiteLLMProviderModels, updateOllamaProviderModels, updateOpenRouterProviderModels, updateVLLMProviderModels, @@ -30,6 +31,8 @@ function useSyncProvider(provider: ProviderName) { updateOllamaProviderModels(data.models) } else if (provider === 'vllm') { updateVLLMProviderModels(data.models) + } else if (provider === 'litellm') { + updateLiteLLMProviderModels(data.models) } else if (provider === 'openrouter') { void updateOpenRouterProviderModels(data.models) if (data.modelInfo) { @@ -54,6 +57,7 @@ export function ProviderModelsLoader() { useSyncProvider('base') useSyncProvider('ollama') useSyncProvider('vllm') + useSyncProvider('litellm') useSyncProvider('openrouter') return null } diff --git a/apps/sim/components/icons.tsx b/apps/sim/components/icons.tsx index 2c1bcb62bf..f7847bb03f 100644 --- a/apps/sim/components/icons.tsx +++ b/apps/sim/components/icons.tsx @@ -3679,6 +3679,22 @@ export function VllmIcon(props: SVGProps) { ) } +export function LiteLLMIcon(props: SVGProps) { + return ( + + LiteLLM + + + ) +} + export function PosthogIcon(props: SVGProps) { return ( = { base: '/api/providers/base/models', ollama: '/api/providers/ollama/models', vllm: '/api/providers/vllm/models', + litellm: '/api/providers/litellm/models', openrouter: '/api/providers/openrouter/models', } diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index ae0e5cbe01..874397bc2b 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -85,6 +85,8 @@ export const env = createEnv({ OLLAMA_URL: z.string().url().optional(), // Ollama local LLM server URL VLLM_BASE_URL: z.string().url().optional(), // vLLM self-hosted base URL (OpenAI-compatible) VLLM_API_KEY: z.string().optional(), // Optional bearer token for vLLM + LITELLM_BASE_URL: z.string().url().optional(), // LiteLLM proxy base URL (OpenAI-compatible) + LITELLM_API_KEY: z.string().optional(), // Optional bearer token for LiteLLM ELEVENLABS_API_KEY: z.string().min(1).optional(), // ElevenLabs API key for text-to-speech in deployed chat SERPER_API_KEY: z.string().min(1).optional(), // Serper API key for online search EXA_API_KEY: z.string().min(1).optional(), // Exa AI API key for enhanced online search diff --git a/apps/sim/providers/litellm/index.ts b/apps/sim/providers/litellm/index.ts new file mode 100644 index 0000000000..3707259e1b --- /dev/null +++ b/apps/sim/providers/litellm/index.ts @@ -0,0 +1,657 @@ +import { createLogger } from '@sim/logger' +import OpenAI from 'openai' +import type { ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/completions' +import { env } from '@/lib/core/config/env' +import type { StreamingExecution } from '@/executor/types' +import { MAX_TOOL_ITERATIONS } from '@/providers' +import { createReadableStreamFromLiteLLMStream } from '@/providers/litellm/utils' +import { getProviderDefaultModel, getProviderModels } from '@/providers/models' +import type { + ProviderConfig, + ProviderRequest, + ProviderResponse, + TimeSegment, +} from '@/providers/types' +import { + calculateCost, + prepareToolExecution, + prepareToolsWithUsageControl, + trackForcedToolUsage, +} from '@/providers/utils' +import { useProvidersStore } from '@/stores/providers' +import { executeTool } from '@/tools' + +const logger = createLogger('LiteLLMProvider') +const LITELLM_VERSION = '1.0.0' + +export const litellmProvider: ProviderConfig = { + id: 'litellm', + name: 'LiteLLM', + description: 'LiteLLM proxy for 100+ LLM providers', + version: LITELLM_VERSION, + models: getProviderModels('litellm'), + defaultModel: getProviderDefaultModel('litellm'), + + async initialize() { + if (typeof window !== 'undefined') { + logger.info('Skipping LiteLLM initialization on client side to avoid CORS issues') + return + } + + const baseUrl = (env.LITELLM_BASE_URL || '').replace(/\/$/, '') + if (!baseUrl) { + logger.info('LITELLM_BASE_URL not configured, skipping initialization') + return + } + + try { + const headers: Record = { + 'Content-Type': 'application/json', + } + + if (env.LITELLM_API_KEY) { + headers.Authorization = `Bearer ${env.LITELLM_API_KEY}` + } + + const response = await fetch(`${baseUrl}/v1/models`, { headers }) + if (!response.ok) { + useProvidersStore.getState().setProviderModels('litellm', []) + logger.warn('LiteLLM service is not available. The provider will be disabled.') + return + } + + const data = (await response.json()) as { data: Array<{ id: string }> } + const models = data.data.map((model) => `litellm/${model.id}`) + + this.models = models + useProvidersStore.getState().setProviderModels('litellm', models) + + logger.info(`Discovered ${models.length} LiteLLM model(s):`, { models }) + } catch (error) { + logger.warn('LiteLLM model instantiation failed. The provider will be disabled.', { + error: error instanceof Error ? error.message : 'Unknown error', + }) + } + }, + + executeRequest: async ( + request: ProviderRequest + ): Promise => { + logger.info('Preparing LiteLLM request', { + model: request.model, + hasSystemPrompt: !!request.systemPrompt, + hasMessages: !!request.messages?.length, + hasTools: !!request.tools?.length, + toolCount: request.tools?.length || 0, + hasResponseFormat: !!request.responseFormat, + stream: !!request.stream, + }) + + const baseUrl = (request.azureEndpoint || env.LITELLM_BASE_URL || '').replace(/\/$/, '') + if (!baseUrl) { + throw new Error('LITELLM_BASE_URL is required for LiteLLM provider') + } + + const apiKey = request.apiKey || env.LITELLM_API_KEY || 'empty' + const litellm = new OpenAI({ + apiKey, + baseURL: `${baseUrl}/v1`, + }) + + const allMessages = [] as any[] + + if (request.systemPrompt) { + allMessages.push({ + role: 'system', + content: request.systemPrompt, + }) + } + + if (request.context) { + allMessages.push({ + role: 'user', + content: request.context, + }) + } + + if (request.messages) { + allMessages.push(...request.messages) + } + + const tools = request.tools?.length + ? request.tools.map((tool) => ({ + type: 'function', + function: { + name: tool.id, + description: tool.description, + parameters: tool.parameters, + }, + })) + : undefined + + const payload: any = { + model: request.model.replace(/^litellm\//, ''), + messages: allMessages, + } + + if (request.temperature !== undefined) payload.temperature = request.temperature + if (request.maxTokens !== undefined) payload.max_tokens = request.maxTokens + + if (request.responseFormat) { + payload.response_format = { + type: 'json_schema', + json_schema: { + name: request.responseFormat.name || 'response_schema', + schema: request.responseFormat.schema || request.responseFormat, + strict: request.responseFormat.strict !== false, + }, + } + + logger.info('Added JSON schema response format to LiteLLM request') + } + + let preparedTools: ReturnType | null = null + let hasActiveTools = false + + if (tools?.length) { + preparedTools = prepareToolsWithUsageControl(tools, request.tools, logger, 'litellm') + const { tools: filteredTools, toolChoice } = preparedTools + + if (filteredTools?.length && toolChoice) { + payload.tools = filteredTools + payload.tool_choice = toolChoice + hasActiveTools = true + + logger.info('LiteLLM request configuration:', { + toolCount: filteredTools.length, + toolChoice: + typeof toolChoice === 'string' + ? toolChoice + : toolChoice.type === 'function' + ? `force:${toolChoice.function.name}` + : 'unknown', + model: payload.model, + }) + } + } + + const providerStartTime = Date.now() + const providerStartTimeISO = new Date(providerStartTime).toISOString() + + try { + if (request.stream && (!tools || tools.length === 0 || !hasActiveTools)) { + logger.info('Using streaming response for LiteLLM request') + + const streamingParams: ChatCompletionCreateParamsStreaming = { + ...payload, + stream: true, + stream_options: { include_usage: true }, + } + const streamResponse = await litellm.chat.completions.create(streamingParams) + + const streamingResult = { + stream: createReadableStreamFromLiteLLMStream(streamResponse, (content, usage) => { + let cleanContent = content + if (cleanContent && request.responseFormat) { + cleanContent = cleanContent.replace(/```json\n?|\n?```/g, '').trim() + } + + streamingResult.execution.output.content = cleanContent + streamingResult.execution.output.tokens = { + input: usage.prompt_tokens, + output: usage.completion_tokens, + total: usage.total_tokens, + } + + const costResult = calculateCost( + request.model, + usage.prompt_tokens, + usage.completion_tokens + ) + streamingResult.execution.output.cost = { + input: costResult.input, + output: costResult.output, + total: costResult.total, + } + + const streamEndTime = Date.now() + const streamEndTimeISO = new Date(streamEndTime).toISOString() + + if (streamingResult.execution.output.providerTiming) { + streamingResult.execution.output.providerTiming.endTime = streamEndTimeISO + streamingResult.execution.output.providerTiming.duration = + streamEndTime - providerStartTime + + if (streamingResult.execution.output.providerTiming.timeSegments?.[0]) { + streamingResult.execution.output.providerTiming.timeSegments[0].endTime = + streamEndTime + streamingResult.execution.output.providerTiming.timeSegments[0].duration = + streamEndTime - providerStartTime + } + } + }), + execution: { + success: true, + output: { + content: '', + model: request.model, + tokens: { input: 0, output: 0, total: 0 }, + toolCalls: undefined, + providerTiming: { + startTime: providerStartTimeISO, + endTime: new Date().toISOString(), + duration: Date.now() - providerStartTime, + timeSegments: [ + { + type: 'model', + name: 'Streaming response', + startTime: providerStartTime, + endTime: Date.now(), + duration: Date.now() - providerStartTime, + }, + ], + }, + cost: { input: 0, output: 0, total: 0 }, + }, + logs: [], + metadata: { + startTime: providerStartTimeISO, + endTime: new Date().toISOString(), + duration: Date.now() - providerStartTime, + }, + }, + } as StreamingExecution + + return streamingResult as StreamingExecution + } + + const initialCallTime = Date.now() + + const originalToolChoice = payload.tool_choice + + const forcedTools = preparedTools?.forcedTools || [] + let usedForcedTools: string[] = [] + + const checkForForcedToolUsage = ( + response: any, + toolChoice: string | { type: string; function?: { name: string }; name?: string; any?: any } + ) => { + if (typeof toolChoice === 'object' && response.choices[0]?.message?.tool_calls) { + const toolCallsResponse = response.choices[0].message.tool_calls + const result = trackForcedToolUsage( + toolCallsResponse, + toolChoice, + logger, + 'litellm', + forcedTools, + usedForcedTools + ) + hasUsedForcedTool = result.hasUsedForcedTool + usedForcedTools = result.usedForcedTools + } + } + + let currentResponse = await litellm.chat.completions.create(payload) + const firstResponseTime = Date.now() - initialCallTime + + let content = currentResponse.choices[0]?.message?.content || '' + + if (content && request.responseFormat) { + content = content.replace(/```json\n?|\n?```/g, '').trim() + } + + const tokens = { + input: currentResponse.usage?.prompt_tokens || 0, + output: currentResponse.usage?.completion_tokens || 0, + total: currentResponse.usage?.total_tokens || 0, + } + const toolCalls = [] + const toolResults = [] + const currentMessages = [...allMessages] + let iterationCount = 0 + + let modelTime = firstResponseTime + let toolsTime = 0 + + let hasUsedForcedTool = false + + const timeSegments: TimeSegment[] = [ + { + type: 'model', + name: 'Initial response', + startTime: initialCallTime, + endTime: initialCallTime + firstResponseTime, + duration: firstResponseTime, + }, + ] + + checkForForcedToolUsage(currentResponse, originalToolChoice) + + while (iterationCount < MAX_TOOL_ITERATIONS) { + if (currentResponse.choices[0]?.message?.content) { + content = currentResponse.choices[0].message.content + if (request.responseFormat) { + content = content.replace(/```json\n?|\n?```/g, '').trim() + } + } + + const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls + if (!toolCallsInResponse || toolCallsInResponse.length === 0) { + break + } + + logger.info( + `Processing ${toolCallsInResponse.length} tool calls (iteration ${iterationCount + 1}/${MAX_TOOL_ITERATIONS})` + ) + + const toolsStartTime = Date.now() + + const toolExecutionPromises = toolCallsInResponse.map(async (toolCall) => { + const toolCallStartTime = Date.now() + const toolName = toolCall.function.name + + try { + const toolArgs = JSON.parse(toolCall.function.arguments) + const tool = request.tools?.find((t) => t.id === toolName) + + if (!tool) return null + + const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) + const result = await executeTool(toolName, executionParams, true) + const toolCallEndTime = Date.now() + + return { + toolCall, + toolName, + toolParams, + result, + startTime: toolCallStartTime, + endTime: toolCallEndTime, + duration: toolCallEndTime - toolCallStartTime, + } + } catch (error) { + const toolCallEndTime = Date.now() + logger.error('Error processing tool call:', { error, toolName }) + + return { + toolCall, + toolName, + toolParams: {}, + result: { + success: false, + output: undefined, + error: error instanceof Error ? error.message : 'Tool execution failed', + }, + startTime: toolCallStartTime, + endTime: toolCallEndTime, + duration: toolCallEndTime - toolCallStartTime, + } + } + }) + + const executionResults = await Promise.allSettled(toolExecutionPromises) + + currentMessages.push({ + role: 'assistant', + content: null, + tool_calls: toolCallsInResponse.map((tc) => ({ + id: tc.id, + type: 'function', + function: { + name: tc.function.name, + arguments: tc.function.arguments, + }, + })), + }) + + for (const settledResult of executionResults) { + if (settledResult.status === 'rejected' || !settledResult.value) continue + + const { toolCall, toolName, toolParams, result, startTime, endTime, duration } = + settledResult.value + + timeSegments.push({ + type: 'tool', + name: toolName, + startTime: startTime, + endTime: endTime, + duration: duration, + }) + + let resultContent: any + if (result.success) { + toolResults.push(result.output) + resultContent = result.output + } else { + resultContent = { + error: true, + message: result.error || 'Tool execution failed', + tool: toolName, + } + } + + toolCalls.push({ + name: toolName, + arguments: toolParams, + startTime: new Date(startTime).toISOString(), + endTime: new Date(endTime).toISOString(), + duration: duration, + result: resultContent, + success: result.success, + }) + + currentMessages.push({ + role: 'tool', + tool_call_id: toolCall.id, + content: JSON.stringify(resultContent), + }) + } + + const thisToolsTime = Date.now() - toolsStartTime + toolsTime += thisToolsTime + + const nextPayload = { + ...payload, + messages: currentMessages, + } + + if (typeof originalToolChoice === 'object' && hasUsedForcedTool && forcedTools.length > 0) { + const remainingTools = forcedTools.filter((tool) => !usedForcedTools.includes(tool)) + + if (remainingTools.length > 0) { + nextPayload.tool_choice = { + type: 'function', + function: { name: remainingTools[0] }, + } + logger.info(`Forcing next tool: ${remainingTools[0]}`) + } else { + nextPayload.tool_choice = 'auto' + logger.info('All forced tools have been used, switching to auto tool_choice') + } + } + + const nextModelStartTime = Date.now() + + currentResponse = await litellm.chat.completions.create(nextPayload) + + checkForForcedToolUsage(currentResponse, nextPayload.tool_choice) + + const nextModelEndTime = Date.now() + const thisModelTime = nextModelEndTime - nextModelStartTime + + timeSegments.push({ + type: 'model', + name: `Model response (iteration ${iterationCount + 1})`, + startTime: nextModelStartTime, + endTime: nextModelEndTime, + duration: thisModelTime, + }) + + modelTime += thisModelTime + + if (currentResponse.choices[0]?.message?.content) { + content = currentResponse.choices[0].message.content + if (request.responseFormat) { + content = content.replace(/```json\n?|\n?```/g, '').trim() + } + } + + if (currentResponse.usage) { + tokens.input += currentResponse.usage.prompt_tokens || 0 + tokens.output += currentResponse.usage.completion_tokens || 0 + tokens.total += currentResponse.usage.total_tokens || 0 + } + + iterationCount++ + } + + if (request.stream) { + logger.info('Using streaming for final response after tool processing') + + const accumulatedCost = calculateCost(request.model, tokens.input, tokens.output) + + const streamingParams: ChatCompletionCreateParamsStreaming = { + ...payload, + messages: currentMessages, + tool_choice: 'auto', + stream: true, + stream_options: { include_usage: true }, + } + const streamResponse = await litellm.chat.completions.create(streamingParams) + + const streamingResult = { + stream: createReadableStreamFromLiteLLMStream(streamResponse, (content, usage) => { + let cleanContent = content + if (cleanContent && request.responseFormat) { + cleanContent = cleanContent.replace(/```json\n?|\n?```/g, '').trim() + } + + streamingResult.execution.output.content = cleanContent + streamingResult.execution.output.tokens = { + input: tokens.input + usage.prompt_tokens, + output: tokens.output + usage.completion_tokens, + total: tokens.total + usage.total_tokens, + } + + const streamCost = calculateCost( + request.model, + usage.prompt_tokens, + usage.completion_tokens + ) + streamingResult.execution.output.cost = { + input: accumulatedCost.input + streamCost.input, + output: accumulatedCost.output + streamCost.output, + total: accumulatedCost.total + streamCost.total, + } + }), + execution: { + success: true, + output: { + content: '', + model: request.model, + tokens: { + input: tokens.input, + output: tokens.output, + total: tokens.total, + }, + toolCalls: + toolCalls.length > 0 + ? { + list: toolCalls, + count: toolCalls.length, + } + : undefined, + providerTiming: { + startTime: providerStartTimeISO, + endTime: new Date().toISOString(), + duration: Date.now() - providerStartTime, + modelTime: modelTime, + toolsTime: toolsTime, + firstResponseTime: firstResponseTime, + iterations: iterationCount + 1, + timeSegments: timeSegments, + }, + cost: { + input: accumulatedCost.input, + output: accumulatedCost.output, + total: accumulatedCost.total, + }, + }, + logs: [], + metadata: { + startTime: providerStartTimeISO, + endTime: new Date().toISOString(), + duration: Date.now() - providerStartTime, + }, + }, + } as StreamingExecution + + return streamingResult as StreamingExecution + } + + const providerEndTime = Date.now() + const providerEndTimeISO = new Date(providerEndTime).toISOString() + const totalDuration = providerEndTime - providerStartTime + + return { + content, + model: request.model, + tokens, + toolCalls: toolCalls.length > 0 ? toolCalls : undefined, + toolResults: toolResults.length > 0 ? toolResults : undefined, + timing: { + startTime: providerStartTimeISO, + endTime: providerEndTimeISO, + duration: totalDuration, + modelTime: modelTime, + toolsTime: toolsTime, + firstResponseTime: firstResponseTime, + iterations: iterationCount + 1, + timeSegments: timeSegments, + }, + } + } catch (error) { + const providerEndTime = Date.now() + const providerEndTimeISO = new Date(providerEndTime).toISOString() + const totalDuration = providerEndTime - providerStartTime + + let errorMessage = error instanceof Error ? error.message : String(error) + let errorType: string | undefined + let errorCode: number | undefined + + if (error && typeof error === 'object' && 'error' in error) { + const litellmError = error.error as any + if (litellmError && typeof litellmError === 'object') { + errorMessage = litellmError.message || errorMessage + errorType = litellmError.type + errorCode = litellmError.code + } + } + + logger.error('Error in LiteLLM request:', { + error: errorMessage, + errorType, + errorCode, + duration: totalDuration, + }) + + const enhancedError = new Error(errorMessage) + // @ts-ignore + enhancedError.timing = { + startTime: providerStartTimeISO, + endTime: providerEndTimeISO, + duration: totalDuration, + } + if (errorType) { + // @ts-ignore + enhancedError.litellmErrorType = errorType + } + if (errorCode) { + // @ts-ignore + enhancedError.litellmErrorCode = errorCode + } + + throw enhancedError + } + }, +} diff --git a/apps/sim/providers/litellm/utils.ts b/apps/sim/providers/litellm/utils.ts new file mode 100644 index 0000000000..f779f95c70 --- /dev/null +++ b/apps/sim/providers/litellm/utils.ts @@ -0,0 +1,14 @@ +import type { ChatCompletionChunk } from 'openai/resources/chat/completions' +import type { CompletionUsage } from 'openai/resources/completions' +import { createOpenAICompatibleStream } from '@/providers/utils' + +/** + * Creates a ReadableStream from a LiteLLM streaming response. + * Uses the shared OpenAI-compatible streaming utility. + */ +export function createReadableStreamFromLiteLLMStream( + litellmStream: AsyncIterable, + onComplete?: (content: string, usage: CompletionUsage) => void +): ReadableStream { + return createOpenAICompatibleStream(litellmStream, 'LiteLLM', onComplete) +} diff --git a/apps/sim/providers/models.ts b/apps/sim/providers/models.ts index eb655e66fb..f22897de93 100644 --- a/apps/sim/providers/models.ts +++ b/apps/sim/providers/models.ts @@ -16,6 +16,7 @@ import { DeepseekIcon, GeminiIcon, GroqIcon, + LiteLLMIcon, MistralIcon, OllamaIcon, OpenAIIcon, @@ -93,6 +94,19 @@ export const PROVIDER_DEFINITIONS: Record = { }, models: [], }, + litellm: { + id: 'litellm', + name: 'LiteLLM', + icon: LiteLLMIcon, + description: 'LiteLLM proxy for 100+ LLM providers', + defaultModel: 'litellm/gpt-3.5-turbo', + modelPatterns: [/^litellm\//], + capabilities: { + temperature: { min: 0, max: 2 }, + toolUsageControl: true, + }, + models: [], + }, openai: { id: 'openai', name: 'OpenAI', @@ -2181,6 +2195,18 @@ export function updateVLLMModels(models: string[]): void { })) } +export function updateLiteLLMModels(models: string[]): void { + PROVIDER_DEFINITIONS.litellm.models = models.map((modelId) => ({ + id: modelId, + pricing: { + input: 0, + output: 0, + updatedAt: new Date().toISOString().split('T')[0], + }, + capabilities: {}, + })) +} + export function updateOpenRouterModels(models: string[]): void { PROVIDER_DEFINITIONS.openrouter.models = models.map((modelId) => ({ id: modelId, diff --git a/apps/sim/providers/registry.ts b/apps/sim/providers/registry.ts index 1b12656b91..1dae6c0bab 100644 --- a/apps/sim/providers/registry.ts +++ b/apps/sim/providers/registry.ts @@ -6,6 +6,7 @@ import { cerebrasProvider } from '@/providers/cerebras' import { deepseekProvider } from '@/providers/deepseek' import { googleProvider } from '@/providers/google' import { groqProvider } from '@/providers/groq' +import { litellmProvider } from '@/providers/litellm' import { mistralProvider } from '@/providers/mistral' import { ollamaProvider } from '@/providers/ollama' import { openaiProvider } from '@/providers/openai' @@ -27,6 +28,7 @@ const providerRegistry: Record = { cerebras: cerebrasProvider, groq: groqProvider, vllm: vllmProvider, + litellm: litellmProvider, mistral: mistralProvider, 'azure-openai': azureOpenAIProvider, openrouter: openRouterProvider, diff --git a/apps/sim/providers/types.ts b/apps/sim/providers/types.ts index 3522a6f026..1d4f312882 100644 --- a/apps/sim/providers/types.ts +++ b/apps/sim/providers/types.ts @@ -14,6 +14,7 @@ export type ProviderId = | 'ollama' | 'openrouter' | 'vllm' + | 'litellm' | 'bedrock' export interface ModelPricing { diff --git a/apps/sim/providers/utils.ts b/apps/sim/providers/utils.ts index 73d08735d1..b67ba6cce8 100644 --- a/apps/sim/providers/utils.ts +++ b/apps/sim/providers/utils.ts @@ -85,6 +85,7 @@ export const providers: Record = { cerebras: buildProviderMetadata('cerebras'), groq: buildProviderMetadata('groq'), vllm: buildProviderMetadata('vllm'), + litellm: buildProviderMetadata('litellm'), mistral: buildProviderMetadata('mistral'), 'azure-openai': buildProviderMetadata('azure-openai'), openrouter: buildProviderMetadata('openrouter'), @@ -103,6 +104,12 @@ export function updateVLLMProviderModels(models: string[]): void { providers.vllm.models = getProviderModelsFromDefinitions('vllm') } +export function updateLiteLLMProviderModels(models: string[]): void { + const { updateLiteLLMModels } = require('@/providers/models') + updateLiteLLMModels(models) + providers.litellm.models = getProviderModelsFromDefinitions('litellm') +} + export async function updateOpenRouterProviderModels(models: string[]): Promise { const { updateOpenRouterModels } = await import('@/providers/models') updateOpenRouterModels(models) @@ -113,7 +120,10 @@ export function getBaseModelProviders(): Record { const allProviders = Object.entries(providers) .filter( ([providerId]) => - providerId !== 'ollama' && providerId !== 'vllm' && providerId !== 'openrouter' + providerId !== 'ollama' && + providerId !== 'vllm' && + providerId !== 'litellm' && + providerId !== 'openrouter' ) .reduce( (map, [providerId, config]) => { @@ -624,6 +634,12 @@ export function getApiKey(provider: string, model: string, userProvidedKey?: str return userProvidedKey || 'empty' } + const isLiteLLMModel = + provider === 'litellm' || useProvidersStore.getState().providers.litellm.models.includes(model) + if (isLiteLLMModel) { + return userProvidedKey || 'empty' + } + // Bedrock uses its own credentials (bedrockAccessKeyId/bedrockSecretKey), not apiKey const isBedrockModel = provider === 'bedrock' || model.startsWith('bedrock/') if (isBedrockModel) { diff --git a/apps/sim/stores/providers/store.ts b/apps/sim/stores/providers/store.ts index 72b3523a44..f7c2bc52c7 100644 --- a/apps/sim/stores/providers/store.ts +++ b/apps/sim/stores/providers/store.ts @@ -9,6 +9,7 @@ export const useProvidersStore = create((set, get) => ({ base: { models: [], isLoading: false }, ollama: { models: [], isLoading: false }, vllm: { models: [], isLoading: false }, + litellm: { models: [], isLoading: false }, openrouter: { models: [], isLoading: false }, }, openRouterModelInfo: {}, diff --git a/apps/sim/stores/providers/types.ts b/apps/sim/stores/providers/types.ts index e267d1c3ae..d89ce68d3b 100644 --- a/apps/sim/stores/providers/types.ts +++ b/apps/sim/stores/providers/types.ts @@ -1,4 +1,4 @@ -export type ProviderName = 'ollama' | 'vllm' | 'openrouter' | 'base' +export type ProviderName = 'ollama' | 'vllm' | 'litellm' | 'openrouter' | 'base' export interface OpenRouterModelInfo { id: string From ae55966a4cf63bd83d43258eeaf7c98863aa67cb Mon Sep 17 00:00:00 2001 From: Aditya Puranik Date: Fri, 16 Jan 2026 23:14:42 +0000 Subject: [PATCH 2/4] docs: add guide for remaining LiteLLM work --- LITELLM_REMAINING_WORK.md | 160 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 LITELLM_REMAINING_WORK.md diff --git a/LITELLM_REMAINING_WORK.md b/LITELLM_REMAINING_WORK.md new file mode 100644 index 0000000000..aa9c682071 --- /dev/null +++ b/LITELLM_REMAINING_WORK.md @@ -0,0 +1,160 @@ +# LiteLLM Integration - Remaining Work + +## Current Status + +**Completed**: LiteLLM provider works for **Agent blocks** in workflows. + +**Pending**: LiteLLM integration with **Copilot** (sim.ai's AI assistant). + +--- + +## What's Done + +- LiteLLM provider implementation (`providers/litellm/`) +- API route for model discovery (`/api/providers/litellm/models`) +- Environment variables (`LITELLM_BASE_URL`, `LITELLM_API_KEY`) +- Full tool execution and streaming support +- Provider registered in store and registry + +--- + +## Remaining: Copilot Integration + +The Copilot has a hardcoded model list separate from the provider system. To enable LiteLLM models in the Copilot, modify these files: + +### 1. Add LiteLLM to valid provider IDs + +**File**: `apps/sim/lib/copilot/config.ts` + +Add `'litellm'` to `VALID_PROVIDER_IDS`: + +```typescript +const VALID_PROVIDER_IDS = [ + 'openai', + 'azure-openai', + 'anthropic', + 'google', + 'deepseek', + 'xai', + 'cerebras', + 'mistral', + 'groq', + 'ollama', + 'litellm', // ADD THIS +] as const +``` + +### 2. Update Copilot model options (Frontend) + +**File**: `apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/copilot/components/user-input/constants.ts` + +Option A - Add static LiteLLM entry: +```typescript +export const MODEL_OPTIONS = [ + // ... existing options + { value: 'litellm/default', label: 'LiteLLM', provider: 'litellm' }, +] +``` + +Option B - Make model list dynamic by fetching from provider store. + +### 3. Update API validation schema + +**File**: `apps/sim/app/api/copilot/chat/route.ts` + +Update `ChatMessageSchema` to accept LiteLLM models. Find the `model` field validation and either: + +- Add a regex pattern for `litellm/*` models +- Or dynamically validate against available provider models + +### 4. Update Copilot state types + +**File**: `apps/sim/stores/panel/copilot/types.ts` + +Update the `selectedModel` type in `CopilotState` to include LiteLLM model pattern: + +```typescript +selectedModel: 'claude-4.5-opus' | 'claude-4.5-sonnet' | /* ... */ | `litellm/${string}` +``` + +--- + +## Testing Checklist + +After implementing Copilot integration: + +- [ ] LiteLLM models appear in Copilot model selector +- [ ] Can select and use LiteLLM model in Copilot chat +- [ ] Streaming works in Copilot with LiteLLM +- [ ] Run `bun run lint && bun run type-check` + +--- + +## Submitting the PR + +### 1. Push to your fork + +```bash +git push origin feat/litellm-provider +``` + +### 2. Create Pull Request + +Go to: https://github.com/simstudioai/sim/compare + +- Click "compare across forks" +- Base repository: `simstudioai/sim` +- Base branch: `staging` (NOT `main`) +- Head repository: `adityapuranik99/sim` +- Compare branch: `feat/litellm-provider` + +### 3. PR Template + +**Title**: `feat(providers): add LiteLLM provider integration` + +**Body**: +```markdown +## Summary +- Add LiteLLM as a new provider for Sim +- Enables connecting LiteLLM proxy to access 100+ LLM providers (including GitHub Copilot) +- Uses OpenAI-compatible API pattern + +## Changes +- New provider: `apps/sim/providers/litellm/` +- New API route: `apps/sim/app/api/providers/litellm/models/` +- Environment variables: `LITELLM_BASE_URL`, `LITELLM_API_KEY` + +## Test plan +- [ ] Set `LITELLM_BASE_URL` in .env +- [ ] Verify models appear with `litellm/` prefix in Agent block +- [ ] Test chat completion through Agent block +- [ ] Verify streaming works +- [ ] Run `bun run lint && bun run type-check` + +## Note +This PR enables LiteLLM for Agent blocks. Copilot integration can be added in a follow-up PR. +``` + +--- + +## Environment Setup + +To test LiteLLM locally, add to `apps/sim/.env`: + +```bash +LITELLM_BASE_URL=http://localhost:4000 +LITELLM_API_KEY=sk-your-key # optional +``` + +Start LiteLLM proxy: +```bash +pip install 'litellm[proxy]' +litellm --model gpt-4o --port 4000 +``` + +--- + +## Questions? + +- Discord: https://discord.gg/Hr4UWYEcTT +- GitHub Issues: https://github.com/simstudioai/sim/issues From 0dab40fc94456b8605e5cefab49d87781ce6a395 Mon Sep 17 00:00:00 2001 From: Aditya Puranik Date: Fri, 16 Jan 2026 23:46:24 +0000 Subject: [PATCH 3/4] feat(copilot): add LiteLLM model support --- apps/sim/app/api/copilot/chat/route.ts | 2 +- .../copilot/components/user-input/constants.ts | 1 + apps/sim/components/icons.tsx | 11 +++-------- apps/sim/lib/copilot/api.ts | 2 +- apps/sim/lib/copilot/config.ts | 1 + apps/sim/lib/copilot/models.ts | 1 + apps/sim/providers/utils.ts | 5 ++++- 7 files changed, 12 insertions(+), 11 deletions(-) diff --git a/apps/sim/app/api/copilot/chat/route.ts b/apps/sim/app/api/copilot/chat/route.ts index 9d31bf5c36..0d864560f0 100644 --- a/apps/sim/app/api/copilot/chat/route.ts +++ b/apps/sim/app/api/copilot/chat/route.ts @@ -41,7 +41,7 @@ const ChatMessageSchema = z.object({ userMessageId: z.string().optional(), // ID from frontend for the user message chatId: z.string().optional(), workflowId: z.string().min(1, 'Workflow ID is required'), - model: z.enum(COPILOT_MODEL_IDS).optional().default('claude-4.5-opus'), +model: z.enum(COPILOT_MODEL_IDS).optional().default('claude-4.5-opus'), mode: z.enum(COPILOT_REQUEST_MODES).optional().default('agent'), prefetch: z.boolean().optional(), createNewChat: z.boolean().optional().default(false), diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/copilot/components/user-input/constants.ts b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/copilot/components/user-input/constants.ts index 2ad930bad0..060c40b27b 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/copilot/components/user-input/constants.ts +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/copilot/components/user-input/constants.ts @@ -241,6 +241,7 @@ export const MODEL_OPTIONS = [ { value: 'gpt-5.2-codex', label: 'GPT 5.2 Codex' }, { value: 'gpt-5.2-pro', label: 'GPT 5.2 Pro' }, { value: 'gemini-3-pro', label: 'Gemini 3 Pro' }, + { value: 'litellm', label: 'LiteLLM' }, ] as const /** diff --git a/apps/sim/components/icons.tsx b/apps/sim/components/icons.tsx index f7847bb03f..f8710d458c 100644 --- a/apps/sim/components/icons.tsx +++ b/apps/sim/components/icons.tsx @@ -3683,14 +3683,9 @@ export function LiteLLMIcon(props: SVGProps) { return ( LiteLLM - + + 🚅 + ) } diff --git a/apps/sim/lib/copilot/api.ts b/apps/sim/lib/copilot/api.ts index c680f9751c..fe6a0e65aa 100644 --- a/apps/sim/lib/copilot/api.ts +++ b/apps/sim/lib/copilot/api.ts @@ -66,7 +66,7 @@ export interface SendMessageRequest { userMessageId?: string // ID from frontend for the user message chatId?: string workflowId?: string - mode?: CopilotMode | CopilotTransportMode +mode?: CopilotMode | CopilotTransportMode model?: CopilotModelId prefetch?: boolean createNewChat?: boolean diff --git a/apps/sim/lib/copilot/config.ts b/apps/sim/lib/copilot/config.ts index 4b9c89274c..f479c24aa6 100644 --- a/apps/sim/lib/copilot/config.ts +++ b/apps/sim/lib/copilot/config.ts @@ -19,6 +19,7 @@ const VALID_PROVIDER_IDS: readonly ProviderId[] = [ 'mistral', 'groq', 'ollama', + 'litellm', ] as const /** diff --git a/apps/sim/lib/copilot/models.ts b/apps/sim/lib/copilot/models.ts index 83a90169be..b6c3db05aa 100644 --- a/apps/sim/lib/copilot/models.ts +++ b/apps/sim/lib/copilot/models.ts @@ -21,6 +21,7 @@ export const COPILOT_MODEL_IDS = [ 'claude-4.5-opus', 'claude-4.1-opus', 'gemini-3-pro', + 'litellm', ] as const export type CopilotModelId = (typeof COPILOT_MODEL_IDS)[number] diff --git a/apps/sim/providers/utils.ts b/apps/sim/providers/utils.ts index b67ba6cce8..a7d4ab2c8b 100644 --- a/apps/sim/providers/utils.ts +++ b/apps/sim/providers/utils.ts @@ -170,7 +170,10 @@ export function getProviderFromModel(model: string): ProviderId { let providerId: ProviderId | null = null - if (normalizedModel in getAllModelProviders()) { + // Check if the model is actually a provider ID (e.g., "litellm" from copilot selector) + if (normalizedModel in providers) { + providerId = normalizedModel as ProviderId + } else if (normalizedModel in getAllModelProviders()) { providerId = getAllModelProviders()[normalizedModel] } else { for (const [id, config] of Object.entries(providers)) { From 9c71f8a8b930190e5f766f8774804f4de273cc26 Mon Sep 17 00:00:00 2001 From: Aditya Puranik Date: Fri, 16 Jan 2026 23:46:31 +0000 Subject: [PATCH 4/4] chore: remove completed guide --- LITELLM_REMAINING_WORK.md | 160 -------------------------------------- 1 file changed, 160 deletions(-) delete mode 100644 LITELLM_REMAINING_WORK.md diff --git a/LITELLM_REMAINING_WORK.md b/LITELLM_REMAINING_WORK.md deleted file mode 100644 index aa9c682071..0000000000 --- a/LITELLM_REMAINING_WORK.md +++ /dev/null @@ -1,160 +0,0 @@ -# LiteLLM Integration - Remaining Work - -## Current Status - -**Completed**: LiteLLM provider works for **Agent blocks** in workflows. - -**Pending**: LiteLLM integration with **Copilot** (sim.ai's AI assistant). - ---- - -## What's Done - -- LiteLLM provider implementation (`providers/litellm/`) -- API route for model discovery (`/api/providers/litellm/models`) -- Environment variables (`LITELLM_BASE_URL`, `LITELLM_API_KEY`) -- Full tool execution and streaming support -- Provider registered in store and registry - ---- - -## Remaining: Copilot Integration - -The Copilot has a hardcoded model list separate from the provider system. To enable LiteLLM models in the Copilot, modify these files: - -### 1. Add LiteLLM to valid provider IDs - -**File**: `apps/sim/lib/copilot/config.ts` - -Add `'litellm'` to `VALID_PROVIDER_IDS`: - -```typescript -const VALID_PROVIDER_IDS = [ - 'openai', - 'azure-openai', - 'anthropic', - 'google', - 'deepseek', - 'xai', - 'cerebras', - 'mistral', - 'groq', - 'ollama', - 'litellm', // ADD THIS -] as const -``` - -### 2. Update Copilot model options (Frontend) - -**File**: `apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/copilot/components/user-input/constants.ts` - -Option A - Add static LiteLLM entry: -```typescript -export const MODEL_OPTIONS = [ - // ... existing options - { value: 'litellm/default', label: 'LiteLLM', provider: 'litellm' }, -] -``` - -Option B - Make model list dynamic by fetching from provider store. - -### 3. Update API validation schema - -**File**: `apps/sim/app/api/copilot/chat/route.ts` - -Update `ChatMessageSchema` to accept LiteLLM models. Find the `model` field validation and either: - -- Add a regex pattern for `litellm/*` models -- Or dynamically validate against available provider models - -### 4. Update Copilot state types - -**File**: `apps/sim/stores/panel/copilot/types.ts` - -Update the `selectedModel` type in `CopilotState` to include LiteLLM model pattern: - -```typescript -selectedModel: 'claude-4.5-opus' | 'claude-4.5-sonnet' | /* ... */ | `litellm/${string}` -``` - ---- - -## Testing Checklist - -After implementing Copilot integration: - -- [ ] LiteLLM models appear in Copilot model selector -- [ ] Can select and use LiteLLM model in Copilot chat -- [ ] Streaming works in Copilot with LiteLLM -- [ ] Run `bun run lint && bun run type-check` - ---- - -## Submitting the PR - -### 1. Push to your fork - -```bash -git push origin feat/litellm-provider -``` - -### 2. Create Pull Request - -Go to: https://github.com/simstudioai/sim/compare - -- Click "compare across forks" -- Base repository: `simstudioai/sim` -- Base branch: `staging` (NOT `main`) -- Head repository: `adityapuranik99/sim` -- Compare branch: `feat/litellm-provider` - -### 3. PR Template - -**Title**: `feat(providers): add LiteLLM provider integration` - -**Body**: -```markdown -## Summary -- Add LiteLLM as a new provider for Sim -- Enables connecting LiteLLM proxy to access 100+ LLM providers (including GitHub Copilot) -- Uses OpenAI-compatible API pattern - -## Changes -- New provider: `apps/sim/providers/litellm/` -- New API route: `apps/sim/app/api/providers/litellm/models/` -- Environment variables: `LITELLM_BASE_URL`, `LITELLM_API_KEY` - -## Test plan -- [ ] Set `LITELLM_BASE_URL` in .env -- [ ] Verify models appear with `litellm/` prefix in Agent block -- [ ] Test chat completion through Agent block -- [ ] Verify streaming works -- [ ] Run `bun run lint && bun run type-check` - -## Note -This PR enables LiteLLM for Agent blocks. Copilot integration can be added in a follow-up PR. -``` - ---- - -## Environment Setup - -To test LiteLLM locally, add to `apps/sim/.env`: - -```bash -LITELLM_BASE_URL=http://localhost:4000 -LITELLM_API_KEY=sk-your-key # optional -``` - -Start LiteLLM proxy: -```bash -pip install 'litellm[proxy]' -litellm --model gpt-4o --port 4000 -``` - ---- - -## Questions? - -- Discord: https://discord.gg/Hr4UWYEcTT -- GitHub Issues: https://github.com/simstudioai/sim/issues