Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions internal-packages/llm-model-catalog/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,11 @@
"@trigger.dev/core": "workspace:*",
"@trigger.dev/database": "workspace:*"
},
"devDependencies": {
"vitest": "3.1.4"
},
"scripts": {
"test": "vitest --sequence.concurrent=false --no-file-parallelism",
"typecheck": "tsc --noEmit",
"generate": "node scripts/generate.mjs",
"sync-prices": "bash scripts/sync-model-prices.sh && node scripts/generate.mjs",
Expand Down
121 changes: 121 additions & 0 deletions internal-packages/llm-model-catalog/src/sync.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
import { describe, it, expect, vi } from "vitest";
import { syncLlmCatalog } from "./sync.js";
import { defaultModelPrices } from "./defaultPrices.js";

const gpt4oDef = defaultModelPrices.find((m) => m.modelName === "gpt-4o");
if (!gpt4oDef) {
throw new Error("expected gpt-4o in defaultModelPrices");
}

describe("syncLlmCatalog", () => {
it("rebuilds pricing tiers and prices for existing default-source models", async () => {
const existingId = "existing-gpt4o";

const llmModelUpdate = vi.fn();
const llmPricingTierDeleteMany = vi.fn();
const llmPricingTierCreate = vi.fn();

const prisma = {
llmModel: {
findFirst: vi.fn(async (args: { where: { modelName: string } }) => {
if (args.where.modelName === "gpt-4o") {
return {
id: existingId,
source: "default",
provider: "openai",
description: "stale description",
contextWindow: 999,
maxOutputTokens: 888,
capabilities: ["legacy"],
isHidden: true,
baseModelName: "legacy-base",
};
}
return null;
}),
},
$transaction: vi.fn(async (fn: (tx: unknown) => Promise<void>) => {
await fn({
llmModel: { update: llmModelUpdate },
llmPricingTier: {
deleteMany: llmPricingTierDeleteMany,
create: llmPricingTierCreate,
},
});
}),
};

await syncLlmCatalog(prisma as never);

expect(prisma.$transaction).toHaveBeenCalledTimes(1);

expect(llmModelUpdate).toHaveBeenCalledWith({
where: { id: existingId },
data: expect.objectContaining({
matchPattern: gpt4oDef.matchPattern,
startDate: gpt4oDef.startDate ? new Date(gpt4oDef.startDate) : null,
}),
});

expect(llmPricingTierDeleteMany).toHaveBeenCalledWith({
where: { modelId: existingId },
});

expect(llmPricingTierCreate).toHaveBeenCalledTimes(gpt4oDef.pricingTiers.length);

const firstTier = gpt4oDef.pricingTiers[0];
expect(llmPricingTierCreate).toHaveBeenCalledWith({
data: {
modelId: existingId,
name: firstTier.name,
isDefault: firstTier.isDefault,
priority: firstTier.priority,
conditions: firstTier.conditions,
prices: {
create: expect.arrayContaining(
Object.entries(firstTier.prices).map(([usageType, price]) => ({
modelId: existingId,
usageType,
price,
}))
),
},
},
});

const createCall = llmPricingTierCreate.mock.calls[0][0] as {
data: { prices: { create: { usageType: string; price: number; modelId: string }[] } };
};
expect(createCall.data.prices.create).toHaveLength(Object.keys(firstTier.prices).length);
});

it("does not rebuild pricing for non-default source models", async () => {
const prisma = {
llmModel: {
findFirst: vi.fn(async (args: { where: { modelName: string } }) => {
if (args.where.modelName === "gpt-4o") {
return {
id: "admin-edited",
source: "admin",
provider: null,
description: null,
contextWindow: null,
maxOutputTokens: null,
capabilities: [],
isHidden: false,
baseModelName: null,
};
}
return null;
}),
},
$transaction: vi.fn(),
};

const result = await syncLlmCatalog(prisma as never);

expect(prisma.$transaction).not.toHaveBeenCalled();
expect(result.modelsUpdated).toBe(0);
expect(result.modelsSkipped).toBeGreaterThan(0);
});
});
67 changes: 51 additions & 16 deletions internal-packages/llm-model-catalog/src/sync.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,27 @@
import type { PrismaClient } from "@trigger.dev/database";
import type { Prisma, PrismaClient } from "@trigger.dev/database";
import { defaultModelPrices } from "./defaultPrices.js";
import { modelCatalog } from "./modelCatalog.js";
import type { DefaultModelDefinition } from "./types.js";

function pricingTierCreateData(
modelId: string,
tier: DefaultModelDefinition["pricingTiers"][number]
): Prisma.LlmPricingTierUncheckedCreateInput {
return {
modelId,
name: tier.name,
isDefault: tier.isDefault,
priority: tier.priority,
conditions: tier.conditions,
prices: {
create: Object.entries(tier.prices).map(([usageType, price]) => ({
modelId,
usageType,
price,
})),
},
};
}

export async function syncLlmCatalog(prisma: PrismaClient): Promise<{
modelsUpdated: number;
Expand Down Expand Up @@ -31,21 +52,35 @@ export async function syncLlmCatalog(prisma: PrismaClient): Promise<{

const catalog = modelCatalog[modelDef.modelName];

await prisma.llmModel.update({
where: { id: existing.id },
data: {
// Update match pattern and start date from Langfuse (may have changed)
matchPattern: modelDef.matchPattern,
startDate: modelDef.startDate ? new Date(modelDef.startDate) : null,
// Update catalog metadata
provider: catalog?.provider ?? existing.provider,
description: catalog?.description ?? existing.description,
contextWindow: catalog?.contextWindow ?? existing.contextWindow,
maxOutputTokens: catalog?.maxOutputTokens ?? existing.maxOutputTokens,
capabilities: catalog?.capabilities ?? existing.capabilities,
isHidden: catalog?.isHidden ?? existing.isHidden,
baseModelName: catalog?.baseModelName ?? existing.baseModelName,
},
await prisma.$transaction(async (tx) => {
await tx.llmModel.update({
where: { id: existing.id },
data: {
// Update match pattern and start date from Langfuse (may have changed)
matchPattern: modelDef.matchPattern,
startDate: modelDef.startDate ? new Date(modelDef.startDate) : null,
// Update catalog metadata
provider: catalog?.provider ?? existing.provider,
description: catalog?.description ?? existing.description,
contextWindow:
catalog?.contextWindow === undefined ? existing.contextWindow : catalog.contextWindow,
maxOutputTokens:
catalog?.maxOutputTokens === undefined
? existing.maxOutputTokens
: catalog.maxOutputTokens,
capabilities: catalog?.capabilities ?? existing.capabilities,
isHidden: catalog?.isHidden ?? existing.isHidden,
baseModelName: catalog?.baseModelName ?? existing.baseModelName,
},
});

await tx.llmPricingTier.deleteMany({ where: { modelId: existing.id } });

for (const tier of modelDef.pricingTiers) {
await tx.llmPricingTier.create({
data: pricingTierCreateData(existing.id, tier),
});
}
});

modelsUpdated++;
Expand Down
15 changes: 15 additions & 0 deletions internal-packages/llm-model-catalog/vitest.config.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
import { defineConfig } from "vitest/config";

export default defineConfig({
test: {
include: ["**/*.test.ts"],
globals: true,
isolate: true,
fileParallelism: false,
poolOptions: {
threads: {
singleThread: true,
},
},
},
});
6 changes: 5 additions & 1 deletion pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading