Skip to content

Commit b075678

Browse files
ericallamericallam
authored
fix(llm-catalog): refresh default model pricing on sync (#3281)
- Rebuild llm_pricing_tiers and llm_prices in syncLlmCatalog for source=default - Add vitest config, sync regression tests, and pin vitest 3.1.4 - Update pnpm-lock.yaml for the new devDependency
1 parent 5447cf4 commit b075678

File tree

6 files changed

+296
-19
lines changed

6 files changed

+296
-19
lines changed

‎internal-packages/llm-model-catalog/package.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,12 @@
99
"@trigger.dev/core": "workspace:*",
1010
"@trigger.dev/database": "workspace:*"
1111
},
12+
"devDependencies": {
13+
"@internal/testcontainers": "workspace:*",
14+
"vitest": "3.1.4"
15+
},
1216
"scripts": {
17+
"test": "vitest --sequence.concurrent=false --no-file-parallelism",
1318
"typecheck": "tsc --noEmit",
1419
"generate": "node scripts/generate.mjs",
1520
"sync-prices": "bash scripts/sync-model-prices.sh && node scripts/generate.mjs",

‎internal-packages/llm-model-catalog/src/sync.test.ts‎

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
import type { PrismaClient } from "@trigger.dev/database";
2+
import { postgresTest } from "@internal/testcontainers";
3+
import { generateFriendlyId } from "@trigger.dev/core/v3/isomorphic";
4+
import { describe, expect } from "vitest";
5+
import { defaultModelPrices } from "./defaultPrices.js";
6+
import { modelCatalog } from "./modelCatalog.js";
7+
import { syncLlmCatalog } from "./sync.js";
8+
9+
function getGpt4oDefinition() {
10+
const def = defaultModelPrices.find((m) => m.modelName === "gpt-4o");
11+
if (def === undefined) {
12+
throw new Error("expected gpt-4o in defaultModelPrices");
13+
}
14+
return def;
15+
}
16+
17+
const gpt4oDef = getGpt4oDefinition();
18+
19+
function getGeminiProDefinition() {
20+
const def = defaultModelPrices.find((m) => m.modelName === "gemini-pro");
21+
if (def === undefined) {
22+
throw new Error("expected gemini-pro in defaultModelPrices");
23+
}
24+
return def;
25+
}
26+
27+
const geminiProDef = getGeminiProDefinition();
28+
29+
/** If sync used `catalog?.baseModelName ?? existing.baseModelName`, sync would keep this string instead of clearing to null. */
30+
const STALE_BASE_MODEL_NAME = "wrong-base-model-sentinel";
31+
32+
const STALE_INPUT_PRICE = 0.099;
33+
const STALE_OUTPUT_PRICE = 0.088;
34+
35+
async function createGpt4oWithStalePricing(
36+
prisma: PrismaClient,
37+
source: "default" | "admin"
38+
) {
39+
const model = await prisma.llmModel.create({
40+
data: {
41+
friendlyId: generateFriendlyId("llm_model"),
42+
projectId: null,
43+
modelName: gpt4oDef.modelName,
44+
matchPattern: "^stale-pattern$",
45+
startDate: gpt4oDef.startDate ? new Date(gpt4oDef.startDate) : null,
46+
source,
47+
provider: "stale-provider",
48+
description: "stale description",
49+
contextWindow: 111,
50+
maxOutputTokens: 222,
51+
capabilities: ["stale-cap"],
52+
isHidden: true,
53+
baseModelName: "stale-base",
54+
},
55+
});
56+
57+
await prisma.llmPricingTier.create({
58+
data: {
59+
modelId: model.id,
60+
name: "Standard",
61+
isDefault: true,
62+
priority: 0,
63+
conditions: [],
64+
prices: {
65+
create: [
66+
{ modelId: model.id, usageType: "input", price: STALE_INPUT_PRICE },
67+
{ modelId: model.id, usageType: "output", price: STALE_OUTPUT_PRICE },
68+
],
69+
},
70+
},
71+
});
72+
73+
return model;
74+
}
75+
76+
async function createGeminiProWithStaleBaseModelName(prisma: PrismaClient) {
77+
const catalogEntry = modelCatalog[geminiProDef.modelName];
78+
expect(catalogEntry).toBeDefined();
79+
expect(catalogEntry.baseModelName).toBeNull();
80+
81+
const model = await prisma.llmModel.create({
82+
data: {
83+
friendlyId: generateFriendlyId("llm_model"),
84+
projectId: null,
85+
modelName: geminiProDef.modelName,
86+
matchPattern: "^stale-gemini-pattern$",
87+
startDate: geminiProDef.startDate ? new Date(geminiProDef.startDate) : null,
88+
source: "default",
89+
provider: "stale-provider",
90+
description: "stale description",
91+
contextWindow: 111,
92+
maxOutputTokens: 222,
93+
capabilities: ["stale-cap"],
94+
isHidden: true,
95+
baseModelName: STALE_BASE_MODEL_NAME,
96+
},
97+
});
98+
99+
const tier = geminiProDef.pricingTiers[0];
100+
await prisma.llmPricingTier.create({
101+
data: {
102+
modelId: model.id,
103+
name: tier.name,
104+
isDefault: tier.isDefault,
105+
priority: tier.priority,
106+
conditions: tier.conditions,
107+
prices: {
108+
create: Object.entries(tier.prices).map(([usageType, price]) => ({
109+
modelId: model.id,
110+
usageType,
111+
price,
112+
})),
113+
},
114+
},
115+
});
116+
117+
return model;
118+
}
119+
120+
async function loadGpt4oWithTiers(prisma: PrismaClient) {
121+
return prisma.llmModel.findFirst({
122+
where: { projectId: null, modelName: gpt4oDef.modelName },
123+
include: {
124+
pricingTiers: {
125+
include: { prices: true },
126+
orderBy: { priority: "asc" },
127+
},
128+
},
129+
});
130+
}
131+
132+
function expectBundledGpt4oPricing(model: NonNullable<Awaited<ReturnType<typeof loadGpt4oWithTiers>>>) {
133+
expect(model.matchPattern).toBe(gpt4oDef.matchPattern);
134+
expect(model.pricingTiers).toHaveLength(gpt4oDef.pricingTiers.length);
135+
136+
const dbTier = model.pricingTiers[0];
137+
const defTier = gpt4oDef.pricingTiers[0];
138+
expect(dbTier.name).toBe(defTier.name);
139+
expect(dbTier.isDefault).toBe(defTier.isDefault);
140+
expect(dbTier.priority).toBe(defTier.priority);
141+
142+
const priceByType = new Map(dbTier.prices.map((p) => [p.usageType, Number(p.price)]));
143+
for (const [usageType, expected] of Object.entries(defTier.prices)) {
144+
expect(priceByType.get(usageType)).toBeCloseTo(expected, 12);
145+
}
146+
expect(priceByType.size).toBe(Object.keys(defTier.prices).length);
147+
}
148+
149+
describe("syncLlmCatalog", () => {
150+
postgresTest(
151+
"rebuilds gpt-4o pricing tiers from bundled defaults when source is default",
152+
async ({ prisma }) => {
153+
await createGpt4oWithStalePricing(prisma, "default");
154+
155+
const result = await syncLlmCatalog(prisma);
156+
157+
expect(result.modelsUpdated).toBe(1);
158+
expect(result.modelsSkipped).toBe(defaultModelPrices.length - 1);
159+
160+
const after = await loadGpt4oWithTiers(prisma);
161+
expect(after).not.toBeNull();
162+
expectBundledGpt4oPricing(after!);
163+
}
164+
);
165+
166+
postgresTest(
167+
"does not replace pricing tiers when model source is not default",
168+
async ({ prisma }) => {
169+
await createGpt4oWithStalePricing(prisma, "admin");
170+
171+
const result = await syncLlmCatalog(prisma);
172+
173+
expect(result.modelsUpdated).toBe(0);
174+
expect(result.modelsSkipped).toBeGreaterThanOrEqual(1);
175+
176+
const after = await loadGpt4oWithTiers(prisma);
177+
expect(after).not.toBeNull();
178+
expect(after!.matchPattern).toBe("^stale-pattern$");
179+
expect(after!.pricingTiers).toHaveLength(1);
180+
const prices = after!.pricingTiers[0].prices;
181+
const input = prices.find((p) => p.usageType === "input");
182+
const output = prices.find((p) => p.usageType === "output");
183+
expect(Number(input?.price)).toBeCloseTo(STALE_INPUT_PRICE, 12);
184+
expect(Number(output?.price)).toBeCloseTo(STALE_OUTPUT_PRICE, 12);
185+
expect(prices).toHaveLength(2);
186+
}
187+
);
188+
189+
postgresTest(
190+
"clears baseModelName when bundled catalog has null (regression for nullish-coalescing merge)",
191+
async ({ prisma }) => {
192+
await createGeminiProWithStaleBaseModelName(prisma);
193+
194+
await syncLlmCatalog(prisma);
195+
196+
const after = await prisma.llmModel.findFirst({
197+
where: { projectId: null, modelName: geminiProDef.modelName },
198+
});
199+
expect(after).not.toBeNull();
200+
expect(after!.baseModelName).toBeNull();
201+
}
202+
);
203+
});

‎internal-packages/llm-model-catalog/src/sync.ts‎

Lines changed: 63 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,27 @@
1-
import type { PrismaClient } from "@trigger.dev/database";
1+
import type { Prisma, PrismaClient } from "@trigger.dev/database";
22
import { defaultModelPrices } from "./defaultPrices.js";
33
import { modelCatalog } from "./modelCatalog.js";
4+
import type { DefaultModelDefinition } from "./types.js";
5+
6+
function pricingTierCreateData(
7+
modelId: string,
8+
tier: DefaultModelDefinition["pricingTiers"][number]
9+
): Prisma.LlmPricingTierUncheckedCreateInput {
10+
return {
11+
modelId,
12+
name: tier.name,
13+
isDefault: tier.isDefault,
14+
priority: tier.priority,
15+
conditions: tier.conditions,
16+
prices: {
17+
create: Object.entries(tier.prices).map(([usageType, price]) => ({
18+
modelId,
19+
usageType,
20+
price,
21+
})),
22+
},
23+
};
24+
}
425

526
export async function syncLlmCatalog(prisma: PrismaClient): Promise<{
627
modelsUpdated: number;
@@ -31,24 +52,49 @@ export async function syncLlmCatalog(prisma: PrismaClient): Promise<{
3152

3253
const catalog = modelCatalog[modelDef.modelName];
3354

34-
await prisma.llmModel.update({
35-
where: { id: existing.id },
36-
data: {
37-
// Update match pattern and start date from Langfuse (may have changed)
38-
matchPattern: modelDef.matchPattern,
39-
startDate: modelDef.startDate ? new Date(modelDef.startDate) : null,
40-
// Update catalog metadata
41-
provider: catalog?.provider ?? existing.provider,
42-
description: catalog?.description ?? existing.description,
43-
contextWindow: catalog?.contextWindow ?? existing.contextWindow,
44-
maxOutputTokens: catalog?.maxOutputTokens ?? existing.maxOutputTokens,
45-
capabilities: catalog?.capabilities ?? existing.capabilities,
46-
isHidden: catalog?.isHidden ?? existing.isHidden,
47-
baseModelName: catalog?.baseModelName ?? existing.baseModelName,
48-
},
55+
const applied = await prisma.$transaction(async (tx) => {
56+
const updateResult = await tx.llmModel.updateMany({
57+
where: { id: existing.id, source: "default" },
58+
data: {
59+
// Update match pattern and start date from Langfuse (may have changed)
60+
matchPattern: modelDef.matchPattern,
61+
startDate: modelDef.startDate ? new Date(modelDef.startDate) : null,
62+
// Update catalog metadata
63+
provider: catalog?.provider ?? existing.provider,
64+
description: catalog?.description ?? existing.description,
65+
contextWindow:
66+
catalog?.contextWindow === undefined ? existing.contextWindow : catalog.contextWindow,
67+
maxOutputTokens:
68+
catalog?.maxOutputTokens === undefined
69+
? existing.maxOutputTokens
70+
: catalog.maxOutputTokens,
71+
capabilities: catalog?.capabilities ?? existing.capabilities,
72+
isHidden: catalog?.isHidden ?? existing.isHidden,
73+
baseModelName:
74+
catalog?.baseModelName === undefined
75+
? existing.baseModelName
76+
: catalog.baseModelName,
77+
},
78+
});
79+
80+
if (updateResult.count !== 1) {
81+
return false;
82+
}
83+
84+
await tx.llmPricingTier.deleteMany({ where: { modelId: existing.id } });
85+
86+
for (const tier of modelDef.pricingTiers) {
87+
await tx.llmPricingTier.create({
88+
data: pricingTierCreateData(existing.id, tier),
89+
});
90+
}
91+
92+
return true;
4993
});
5094

51-
modelsUpdated++;
95+
if (applied) {
96+
modelsUpdated++;
97+
}
5298
}
5399

54100
return { modelsUpdated, modelsSkipped };

‎internal-packages/llm-model-catalog/tsconfig.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,5 +15,5 @@
1515
"strict": true,
1616
"resolveJsonModule": true
1717
},
18-
"exclude": ["node_modules"]
18+
"exclude": ["node_modules", "**/*.test.ts"]
1919
}

‎internal-packages/llm-model-catalog/vitest.config.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
import { defineConfig } from "vitest/config";
2+
3+
export default defineConfig({
4+
test: {
5+
include: ["**/*.test.ts"],
6+
globals: true,
7+
isolate: true,
8+
fileParallelism: false,
9+
poolOptions: {
10+
threads: {
11+
singleThread: true,
12+
},
13+
},
14+
testTimeout: 120_000,
15+
},
16+
});

‎pnpm-lock.yaml‎

Lines changed: 8 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)