|
1 | 1 | import { existsSync } from "node:fs" |
2 | 2 | import { readdir, readFile } from "node:fs/promises" |
3 | 3 | import path from "node:path" |
4 | | -import { type Checkpoint, checkpointSchema, type RunEvent, type RunSetting } from "@perstack/core" |
5 | | -import { getRunDir } from "@perstack/runtime" |
| 4 | +import { |
| 5 | + type Checkpoint, |
| 6 | + checkpointSchema, |
| 7 | + type Job, |
| 8 | + jobSchema, |
| 9 | + type RunEvent, |
| 10 | + type RunSetting, |
| 11 | +} from "@perstack/core" |
| 12 | +import { getCheckpointDir, getCheckpointPath, getRunDir } from "@perstack/runtime" |
| 13 | + |
| 14 | +export async function getAllJobs(): Promise<Job[]> { |
| 15 | + const dataDir = path.resolve(process.cwd(), "perstack") |
| 16 | + if (!existsSync(dataDir)) { |
| 17 | + return [] |
| 18 | + } |
| 19 | + const jobsDir = path.resolve(dataDir, "jobs") |
| 20 | + if (!existsSync(jobsDir)) { |
| 21 | + return [] |
| 22 | + } |
| 23 | + const jobDirs = await readdir(jobsDir, { withFileTypes: true }).then((dirs) => |
| 24 | + dirs.filter((dir) => dir.isDirectory()).map((dir) => dir.name), |
| 25 | + ) |
| 26 | + if (jobDirs.length === 0) { |
| 27 | + return [] |
| 28 | + } |
| 29 | + const jobs: Job[] = [] |
| 30 | + for (const jobDir of jobDirs) { |
| 31 | + const jobPath = path.resolve(jobsDir, jobDir, "job.json") |
| 32 | + try { |
| 33 | + const jobContent = await readFile(jobPath, "utf-8") |
| 34 | + jobs.push(jobSchema.parse(JSON.parse(jobContent))) |
| 35 | + } catch { |
| 36 | + // Ignore invalid jobs |
| 37 | + } |
| 38 | + } |
| 39 | + return jobs.sort((a, b) => b.startedAt - a.startedAt) |
| 40 | +} |
6 | 41 |
|
7 | 42 | export async function getAllRuns(): Promise<RunSetting[]> { |
8 | 43 | const dataDir = path.resolve(process.cwd(), "perstack") |
@@ -67,63 +102,30 @@ export async function getMostRecentRunInJob(jobId: string): Promise<RunSetting> |
67 | 102 | return runs[0] |
68 | 103 | } |
69 | 104 |
|
70 | | -export async function getCheckpoints( |
71 | | - jobId: string, |
72 | | - runId: string, |
73 | | -): Promise<{ timestamp: string; stepNumber: string; id: string }[]> { |
74 | | - const runDir = getRunDir(jobId, runId) |
75 | | - if (!existsSync(runDir)) { |
| 105 | +export async function getCheckpointsByJobId(jobId: string): Promise<Checkpoint[]> { |
| 106 | + const checkpointDir = getCheckpointDir(jobId) |
| 107 | + if (!existsSync(checkpointDir)) { |
76 | 108 | return [] |
77 | 109 | } |
78 | | - return await readdir(runDir).then((files) => |
79 | | - files |
80 | | - .filter((file) => file.startsWith("checkpoint-")) |
81 | | - .map((file) => { |
82 | | - const [_, timestamp, stepNumber, id] = file.split(".")[0].split("-") |
83 | | - return { |
84 | | - timestamp, |
85 | | - stepNumber, |
86 | | - id, |
87 | | - } |
88 | | - }) |
89 | | - .sort((a, b) => Number(a.stepNumber) - Number(b.stepNumber)), |
90 | | - ) |
91 | | -} |
92 | | - |
93 | | -export async function getCheckpoint(checkpointId: string): Promise<Checkpoint> { |
94 | | - const run = await getMostRecentRun() |
95 | | - const runDir = getRunDir(run.jobId, run.runId) |
96 | | - const checkpointPath = path.resolve(runDir, `checkpoint-${checkpointId}.json`) |
97 | | - const checkpoint = await readFile(checkpointPath, "utf-8") |
98 | | - return checkpointSchema.parse(JSON.parse(checkpoint)) |
99 | | -} |
100 | | - |
101 | | -export async function getMostRecentCheckpoint(jobId?: string, runId?: string): Promise<Checkpoint> { |
102 | | - let runJobId: string |
103 | | - let runIdForCheckpoint: string |
104 | | - if (jobId && runId) { |
105 | | - runJobId = jobId |
106 | | - runIdForCheckpoint = runId |
107 | | - } else { |
108 | | - const run = await getMostRecentRun() |
109 | | - runJobId = run.jobId |
110 | | - runIdForCheckpoint = run.runId |
111 | | - } |
112 | | - const runDir = getRunDir(runJobId, runIdForCheckpoint) |
113 | | - const checkpointFiles = await readdir(runDir, { withFileTypes: true }).then((files) => |
114 | | - files.filter((file) => file.isFile() && file.name.startsWith("checkpoint-")), |
115 | | - ) |
116 | | - if (checkpointFiles.length === 0) { |
117 | | - throw new Error(`No checkpoints found for run ${runIdForCheckpoint}`) |
118 | | - } |
| 110 | + const files = await readdir(checkpointDir) |
| 111 | + const checkpointFiles = files.filter((file) => file.endsWith(".json")) |
119 | 112 | const checkpoints = await Promise.all( |
120 | 113 | checkpointFiles.map(async (file) => { |
121 | | - const checkpointPath = path.resolve(runDir, file.name) |
122 | | - const checkpoint = await readFile(checkpointPath, "utf-8") |
123 | | - return checkpointSchema.parse(JSON.parse(checkpoint)) |
| 114 | + const checkpointPath = path.resolve(checkpointDir, file) |
| 115 | + const content = await readFile(checkpointPath, "utf-8") |
| 116 | + return checkpointSchema.parse(JSON.parse(content)) |
124 | 117 | }), |
125 | 118 | ) |
126 | | - return checkpoints.sort((a, b) => b.stepNumber - a.stepNumber)[0] |
| 119 | + return checkpoints.sort((a, b) => a.stepNumber - b.stepNumber) |
| 120 | +} |
| 121 | + |
| 122 | +export async function getMostRecentCheckpoint(jobId?: string): Promise<Checkpoint> { |
| 123 | + const targetJobId = jobId ?? (await getMostRecentRun()).jobId |
| 124 | + const checkpoints = await getCheckpointsByJobId(targetJobId) |
| 125 | + if (checkpoints.length === 0) { |
| 126 | + throw new Error(`No checkpoints found for job ${targetJobId}`) |
| 127 | + } |
| 128 | + return checkpoints[checkpoints.length - 1] |
127 | 129 | } |
128 | 130 |
|
129 | 131 | export async function getRecentExperts( |
@@ -168,51 +170,26 @@ export async function getEvents( |
168 | 170 | .sort((a, b) => Number(a.stepNumber) - Number(b.stepNumber)), |
169 | 171 | ) |
170 | 172 | } |
171 | | -export async function getCheckpointById( |
172 | | - jobId: string, |
173 | | - runId: string, |
174 | | - checkpointId: string, |
175 | | -): Promise<Checkpoint> { |
176 | | - const runDir = getRunDir(jobId, runId) |
177 | | - const files = await readdir(runDir) |
178 | | - const checkpointFile = files.find( |
179 | | - (file) => file.startsWith("checkpoint-") && file.includes(`-${checkpointId}.`), |
180 | | - ) |
181 | | - if (!checkpointFile) { |
182 | | - throw new Error(`Checkpoint ${checkpointId} not found in run ${runId}`) |
| 173 | +export async function getCheckpointById(jobId: string, checkpointId: string): Promise<Checkpoint> { |
| 174 | + const checkpointPath = getCheckpointPath(jobId, checkpointId) |
| 175 | + if (!existsSync(checkpointPath)) { |
| 176 | + throw new Error(`Checkpoint ${checkpointId} not found in job ${jobId}`) |
183 | 177 | } |
184 | | - const checkpointPath = path.resolve(runDir, checkpointFile) |
185 | 178 | const checkpoint = await readFile(checkpointPath, "utf-8") |
186 | 179 | return checkpointSchema.parse(JSON.parse(checkpoint)) |
187 | 180 | } |
188 | 181 | export async function getCheckpointsWithDetails( |
189 | 182 | jobId: string, |
190 | | - runId: string, |
191 | | -): Promise< |
192 | | - { id: string; runId: string; stepNumber: number; timestamp: number; contextWindowUsage: number }[] |
193 | | -> { |
194 | | - const runDir = getRunDir(jobId, runId) |
195 | | - if (!existsSync(runDir)) { |
196 | | - return [] |
197 | | - } |
198 | | - const files = await readdir(runDir) |
199 | | - const checkpointFiles = files.filter((file) => file.startsWith("checkpoint-")) |
200 | | - const checkpoints = await Promise.all( |
201 | | - checkpointFiles.map(async (file) => { |
202 | | - const [_, timestamp, stepNumber, id] = file.split(".")[0].split("-") |
203 | | - const checkpointPath = path.resolve(runDir, file) |
204 | | - const content = await readFile(checkpointPath, "utf-8") |
205 | | - const checkpoint = checkpointSchema.parse(JSON.parse(content)) |
206 | | - return { |
207 | | - id, |
208 | | - runId, |
209 | | - stepNumber: Number(stepNumber), |
210 | | - timestamp: Number(timestamp), |
211 | | - contextWindowUsage: checkpoint.contextWindowUsage ?? 0, |
212 | | - } |
213 | | - }), |
214 | | - ) |
215 | | - return checkpoints.sort((a, b) => b.stepNumber - a.stepNumber) |
| 183 | +): Promise<{ id: string; runId: string; stepNumber: number; contextWindowUsage: number }[]> { |
| 184 | + const checkpoints = await getCheckpointsByJobId(jobId) |
| 185 | + return checkpoints |
| 186 | + .map((cp) => ({ |
| 187 | + id: cp.id, |
| 188 | + runId: cp.runId, |
| 189 | + stepNumber: cp.stepNumber, |
| 190 | + contextWindowUsage: cp.contextWindowUsage ?? 0, |
| 191 | + })) |
| 192 | + .sort((a, b) => b.stepNumber - a.stepNumber) |
216 | 193 | } |
217 | 194 | export async function getEventsWithDetails( |
218 | 195 | jobId: string, |
|
0 commit comments