From 8558b34455274226c81ea3a802974f979bba5354 Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Wed, 4 Mar 2026 12:17:03 +0100 Subject: [PATCH 01/10] feat: parallelize describe queries --- .../src/type-generator/query-registry.ts | 202 ++++++++++++------ packages/appkit/src/type-generator/spinner.ts | 4 + .../tests/generate-queries.test.ts | 23 +- 3 files changed, 161 insertions(+), 68 deletions(-) diff --git a/packages/appkit/src/type-generator/query-registry.ts b/packages/appkit/src/type-generator/query-registry.ts index 2ba27584..956aca9a 100644 --- a/packages/appkit/src/type-generator/query-registry.ts +++ b/packages/appkit/src/type-generator/query-registry.ts @@ -116,19 +116,6 @@ function generateUnknownResultQuery(sql: string, queryName: string): string { }`; } -function cacheFailedQuery( - cache: ReturnType, - querySchemas: QuerySchema[], - sql: string, - queryName: string, - sqlHash: string, -): void { - const type = generateUnknownResultQuery(sql, queryName); - querySchemas.push({ name: queryName, type }); - cache.queries[queryName] = { hash: sqlHash, type, retry: true }; - saveCache(cache); -} - export function extractParameterTypes(sql: string): Record { const paramTypes: Record = {}; const regex = @@ -169,73 +156,168 @@ export async function generateQueriesFromDescribe( const cache = noCache ? { version: CACHE_VERSION, queries: {} } : loadCache(); const client = new WorkspaceClient({}); - const querySchemas: QuerySchema[] = []; const spinner = new Spinner(); - // process each query file + // Phase 1: Read files, check cache, separate cached vs uncached + const cachedResults: Array<{ index: number; schema: QuerySchema }> = []; + const uncachedQueries: Array<{ + index: number; + queryName: string; + sql: string; + sqlHash: string; + cleanedSql: string; + }> = []; + for (let i = 0; i < queryFiles.length; i++) { const file = queryFiles[i]; const rawName = path.basename(file, ".sql"); const queryName = normalizeQueryName(rawName); - // read query file content const sql = fs.readFileSync(path.join(queryFolder, file), "utf8"); const sqlHash = hashSQL(sql); - // check cache (skip if marked for retry after a failed DESCRIBE) const cached = cache.queries[queryName]; if (cached && cached.hash === sqlHash && !cached.retry) { - querySchemas.push({ name: queryName, type: cached.type }); + cachedResults.push({ + index: i, + schema: { name: queryName, type: cached.type }, + }); spinner.start(`Processing ${queryName} (${i + 1}/${queryFiles.length})`); spinner.stop(`✓ ${queryName} (cached)`); - continue; + } else { + const sqlWithDefaults = sql.replace(/:([a-zA-Z_]\w*)/g, "''"); + const cleanedSql = sqlWithDefaults.trim().replace(/;\s*$/, ""); + uncachedQueries.push({ index: i, queryName, sql, sqlHash, cleanedSql }); } + } - spinner.start(`Processing ${queryName} (${i + 1}/${queryFiles.length})`); - - const sqlWithDefaults = sql.replace(/:([a-zA-Z_]\w*)/g, "''"); - - // strip trailing semicolon for DESCRIBE QUERY - const cleanedSql = sqlWithDefaults.trim().replace(/;\s*$/, ""); - - // execute DESCRIBE QUERY to get schema without running the actual query - try { - const result = (await client.statementExecution.executeStatement({ - statement: `DESCRIBE QUERY ${cleanedSql}`, - warehouse_id: warehouseId, - })) as DatabricksStatementExecutionResponse; - - if (result.status.state === "FAILED") { - const sqlError = - result.status.error?.message || "Query execution failed"; - cacheFailedQuery(cache, querySchemas, sql, queryName, sqlHash); - spinner.stop(`✗ ${queryName} - failed`); - spinner.printDetail(`SQL Error: ${sqlError}`); - spinner.printDetail(`Query: ${cleanedSql.slice(0, 200)}`); - continue; + // Phase 2: Execute all uncached DESCRIBE calls in parallel + type DescribeResult = + | { + status: "ok"; + index: number; + schema: QuerySchema; + cacheEntry: { hash: string; type: string; retry: boolean }; + } + | { + status: "fail"; + index: number; + schema: QuerySchema; + cacheEntry: { hash: string; type: string; retry: boolean }; + errorLines: string[]; + }; + + const freshResults: Array<{ index: number; schema: QuerySchema }> = []; + const startTime = performance.now(); + + if (uncachedQueries.length > 0) { + let completed = 0; + const total = uncachedQueries.length; + spinner.start( + `Describing ${total} ${total === 1 ? "query" : "queries"} (0/${total})`, + ); + + const settled = await Promise.allSettled( + uncachedQueries.map( + async ({ + index, + queryName, + sql, + sqlHash, + cleanedSql, + }): Promise => { + const result = (await client.statementExecution.executeStatement({ + statement: `DESCRIBE QUERY ${cleanedSql}`, + warehouse_id: warehouseId, + })) as DatabricksStatementExecutionResponse; + + completed++; + spinner.update( + `Describing ${total} ${total === 1 ? "query" : "queries"} (${completed}/${total})`, + ); + + if (result.status.state === "FAILED") { + const sqlError = + result.status.error?.message || "Query execution failed"; + const type = generateUnknownResultQuery(sql, queryName); + return { + status: "fail", + index, + schema: { name: queryName, type }, + cacheEntry: { hash: sqlHash, type, retry: true }, + errorLines: [ + `SQL Error: ${sqlError}`, + `Query: ${cleanedSql.slice(0, 200)}`, + ], + }; + } + + const { type, hasResults } = convertToQueryType( + result, + sql, + queryName, + ); + return { + status: "ok", + index, + schema: { name: queryName, type }, + cacheEntry: { hash: sqlHash, type, retry: !hasResults }, + }; + }, + ), + ); + + spinner.stop(`✓ Described ${total} ${total === 1 ? "query" : "queries"}`); + + // Print per-query results + for (let i = 0; i < settled.length; i++) { + const entry = settled[i]; + const { queryName } = uncachedQueries[i]; + + if (entry.status === "fulfilled") { + const res = entry.value; + freshResults.push({ index: res.index, schema: res.schema }); + cache.queries[queryName] = res.cacheEntry; + + if (res.status === "ok") { + spinner.printDetail(`✓ ${queryName}`); + } else { + spinner.printDetail(`✗ ${queryName} - failed`); + for (const line of res.errorLines) { + spinner.printDetail(` ${line}`); + } + } + } else { + const errorMessage = + entry.reason instanceof Error + ? entry.reason.message + : "Unknown error"; + const { sql, sqlHash, index } = uncachedQueries[i]; + const type = generateUnknownResultQuery(sql, queryName); + freshResults.push({ index, schema: { name: queryName, type } }); + cache.queries[queryName] = { hash: sqlHash, type, retry: true }; + + spinner.printDetail(`✗ ${queryName}`); + spinner.printDetail(` ${errorMessage}`); } - - // convert result to query schema - const { type, hasResults } = convertToQueryType(result, sql, queryName); - querySchemas.push({ name: queryName, type }); - - // update cache immediately so successful results survive partial failures - // retry if DESCRIBE returned no columns (result: unknown) - const retry = !hasResults; - cache.queries[queryName] = { hash: sqlHash, type, retry }; - saveCache(cache); - - spinner.stop(`✓ ${queryName}`); - } catch (error) { - const errorMessage = - error instanceof Error ? error.message : "Unknown error"; - spinner.stop(`✗ ${queryName}`); - spinner.printDetail(errorMessage); - cacheFailedQuery(cache, querySchemas, sql, queryName, sqlHash); } + + // Save cache once after all queries are processed + saveCache(cache); } - return querySchemas; + const elapsed = ((performance.now() - startTime) / 1000).toFixed(2); + logger.debug( + "%d new queries, %d cached. Typegen: %ss", + uncachedQueries.length, + cachedResults.length, + elapsed, + ); + + // Merge and sort by original file index for deterministic output + return [...cachedResults, ...freshResults] + .sort((a, b) => a.index - b.index) + .map((r) => r.schema); } /** diff --git a/packages/appkit/src/type-generator/spinner.ts b/packages/appkit/src/type-generator/spinner.ts index 5a6e04b6..add1a629 100644 --- a/packages/appkit/src/type-generator/spinner.ts +++ b/packages/appkit/src/type-generator/spinner.ts @@ -17,6 +17,10 @@ export class Spinner { }, 300); } + update(text: string) { + this.text = text; + } + stop(finalText?: string) { if (this.interval) { clearInterval(this.interval); diff --git a/packages/appkit/src/type-generator/tests/generate-queries.test.ts b/packages/appkit/src/type-generator/tests/generate-queries.test.ts index f4948e3c..8264e102 100644 --- a/packages/appkit/src/type-generator/tests/generate-queries.test.ts +++ b/packages/appkit/src/type-generator/tests/generate-queries.test.ts @@ -26,6 +26,7 @@ vi.mock("@databricks/sdk-experimental", () => ({ vi.mock("../spinner", () => ({ Spinner: vi.fn(() => ({ start: vi.fn(), + update: vi.fn(), stop: mocks.spinnerStop, printDetail: mocks.spinnerPrintDetail, })), @@ -69,7 +70,8 @@ describe("generateQueriesFromDescribe", () => { expect(schemas[0].name).toBe("users"); expect(schemas[0].type).toContain("id: number"); expect(schemas[0].type).toContain("name: string"); - expect(mocks.spinnerStop).toHaveBeenCalledWith("✓ users"); + expect(mocks.spinnerStop).toHaveBeenCalledWith("✓ Described 1 query"); + expect(mocks.spinnerPrintDetail).toHaveBeenCalledWith("✓ users"); expect(mocks.saveCache).toHaveBeenCalledTimes(1); }); @@ -89,9 +91,12 @@ describe("generateQueriesFromDescribe", () => { expect(schemas).toHaveLength(1); expect(schemas[0].name).toBe("bad_table"); expect(schemas[0].type).toContain("result: unknown"); - expect(mocks.spinnerStop).toHaveBeenCalledWith("✗ bad_table - failed"); + expect(mocks.spinnerStop).toHaveBeenCalledWith("✓ Described 1 query"); expect(mocks.spinnerPrintDetail).toHaveBeenCalledWith( - "SQL Error: Table or view not found: bad_table", + "✗ bad_table - failed", + ); + expect(mocks.spinnerPrintDetail).toHaveBeenCalledWith( + expect.stringContaining("SQL Error: Table or view not found: bad_table"), ); expect(mocks.spinnerPrintDetail).toHaveBeenCalledWith( expect.stringContaining("Query:"), @@ -112,9 +117,10 @@ describe("generateQueriesFromDescribe", () => { expect(schemas).toHaveLength(1); expect(schemas[0].name).toBe("query"); expect(schemas[0].type).toContain("result: unknown"); - expect(mocks.spinnerStop).toHaveBeenCalledWith("✗ query - failed"); + expect(mocks.spinnerStop).toHaveBeenCalledWith("✓ Described 1 query"); + expect(mocks.spinnerPrintDetail).toHaveBeenCalledWith("✗ query - failed"); expect(mocks.spinnerPrintDetail).toHaveBeenCalledWith( - "SQL Error: Query execution failed", + expect.stringContaining("SQL Error: Query execution failed"), ); expect(mocks.saveCache).toHaveBeenCalledTimes(1); }); @@ -147,8 +153,8 @@ describe("generateQueriesFromDescribe", () => { expect(schemas[1].name).toBe("bad"); expect(schemas[1].type).toContain("result: unknown"); - // saveCache called for both queries (success + failure with retry: true) - expect(mocks.saveCache).toHaveBeenCalledTimes(2); + // saveCache called once after all parallel queries complete + expect(mocks.saveCache).toHaveBeenCalledTimes(1); }); test("all queries fail — caches with retry flag, all unknown result types", async () => { @@ -172,7 +178,8 @@ describe("generateQueriesFromDescribe", () => { expect(schemas[1].name).toBe("b"); expect(schemas[1].type).toContain("result: unknown"); - expect(mocks.saveCache).toHaveBeenCalledTimes(2); + // saveCache called once after all parallel queries complete + expect(mocks.saveCache).toHaveBeenCalledTimes(1); }); test("unknown result type includes parameters from SQL", async () => { From b6fcd68d749f8abccfeed8bb1ff253968dc25330 Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Wed, 4 Mar 2026 13:04:55 +0100 Subject: [PATCH 02/10] feat: batch queries and concurrent i/o --- packages/appkit/src/type-generator/cache.ts | 28 ++- packages/appkit/src/type-generator/index.ts | 4 +- .../src/type-generator/query-registry.ts | 201 ++++++++++-------- .../tests/generate-queries.test.ts | 68 ++++-- .../appkit/src/type-generator/vite-plugin.ts | 4 +- 5 files changed, 173 insertions(+), 132 deletions(-) diff --git a/packages/appkit/src/type-generator/cache.ts b/packages/appkit/src/type-generator/cache.ts index 3135e2d0..8b22ca96 100644 --- a/packages/appkit/src/type-generator/cache.ts +++ b/packages/appkit/src/type-generator/cache.ts @@ -1,5 +1,5 @@ import crypto from "node:crypto"; -import fs from "node:fs"; +import fs from "node:fs/promises"; import path from "node:path"; import { createLogger } from "../logging/logger"; @@ -50,31 +50,29 @@ export function hashSQL(sql: string): string { * If the cache is not found, run the query explain * @returns - the cache */ -export function loadCache(): Cache { +export async function loadCache(): Promise { const cachePath = path.join(CACHE_DIR, CACHE_FILE); try { - if (!fs.existsSync(CACHE_DIR)) { - fs.mkdirSync(CACHE_DIR, { recursive: true }); - } + await fs.mkdir(CACHE_DIR, { recursive: true }); - if (fs.existsSync(cachePath)) { - const cache = JSON.parse(fs.readFileSync(cachePath, "utf8")) as Cache; - if (cache.version === CACHE_VERSION) { - return cache; - } + const raw = await fs.readFile(cachePath, "utf8"); + const cache = JSON.parse(raw) as Cache; + if (cache.version === CACHE_VERSION) { + return cache; + } + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== "ENOENT") { + logger.warn("Cache file is corrupted, flushing cache completely."); } - } catch { - logger.warn("Cache file is corrupted, flushing cache completely."); } return { version: CACHE_VERSION, queries: {} }; } /** * Save the cache to the file system - * The cache is saved as a JSON file, it is used to avoid running the query explain multiple times * @param cache - cache object to save */ -export function saveCache(cache: Cache): void { +export async function saveCache(cache: Cache): Promise { const cachePath = path.join(CACHE_DIR, CACHE_FILE); - fs.writeFileSync(cachePath, JSON.stringify(cache, null, 2), "utf8"); + await fs.writeFile(cachePath, JSON.stringify(cache, null, 2), "utf8"); } diff --git a/packages/appkit/src/type-generator/index.ts b/packages/appkit/src/type-generator/index.ts index c41bc577..5cdb65c3 100644 --- a/packages/appkit/src/type-generator/index.ts +++ b/packages/appkit/src/type-generator/index.ts @@ -1,4 +1,4 @@ -import fs from "node:fs"; +import fs from "node:fs/promises"; import dotenv from "dotenv"; import { createLogger } from "../logging/logger"; import { generateQueriesFromDescribe } from "./query-registry"; @@ -67,7 +67,7 @@ export async function generateFromEntryPoint(options: { const typeDeclarations = generateTypeDeclarations(queryRegistry); - fs.writeFileSync(outFile, typeDeclarations, "utf-8"); + await fs.writeFile(outFile, typeDeclarations, "utf-8"); logger.debug("Type generation complete!"); } diff --git a/packages/appkit/src/type-generator/query-registry.ts b/packages/appkit/src/type-generator/query-registry.ts index 956aca9a..11f1bea7 100644 --- a/packages/appkit/src/type-generator/query-registry.ts +++ b/packages/appkit/src/type-generator/query-registry.ts @@ -1,4 +1,4 @@ -import fs from "node:fs"; +import fs from "node:fs/promises"; import path from "node:path"; import { WorkspaceClient } from "@databricks/sdk-experimental"; import { createLogger } from "../logging/logger"; @@ -141,24 +141,32 @@ export function extractParameterTypes(sql: string): Record { export async function generateQueriesFromDescribe( queryFolder: string, warehouseId: string, - options: { noCache?: boolean } = {}, + options: { noCache?: boolean; concurrency?: number } = {}, ): Promise { - const { noCache = false } = options; - - // read all query files in the folder - const queryFiles = fs - .readdirSync(queryFolder) - .filter((file) => file.endsWith(".sql")); - + const { noCache = false, concurrency = 10 } = options; + + // read all query files and cache in parallel + const [allFiles, cache] = await Promise.all([ + fs.readdir(queryFolder), + noCache + ? ({ version: CACHE_VERSION, queries: {} } as Awaited< + ReturnType + >) + : loadCache(), + ]); + + const queryFiles = allFiles.filter((file) => file.endsWith(".sql")); logger.debug("Found %d SQL queries", queryFiles.length); - // load cache - const cache = noCache ? { version: CACHE_VERSION, queries: {} } : loadCache(); - const client = new WorkspaceClient({}); const spinner = new Spinner(); - // Phase 1: Read files, check cache, separate cached vs uncached + // Read all SQL files in parallel + const sqlContents = await Promise.all( + queryFiles.map((file) => fs.readFile(path.join(queryFolder, file), "utf8")), + ); + + // Phase 1: Check cache, separate cached vs uncached const cachedResults: Array<{ index: number; schema: QuerySchema }> = []; const uncachedQueries: Array<{ index: number; @@ -173,7 +181,7 @@ export async function generateQueriesFromDescribe( const rawName = path.basename(file, ".sql"); const queryName = normalizeQueryName(rawName); - const sql = fs.readFileSync(path.join(queryFolder, file), "utf8"); + const sql = sqlContents[i]; const sqlHash = hashSQL(sql); const cached = cache.queries[queryName]; @@ -217,93 +225,102 @@ export async function generateQueriesFromDescribe( `Describing ${total} ${total === 1 ? "query" : "queries"} (0/${total})`, ); - const settled = await Promise.allSettled( - uncachedQueries.map( - async ({ + const describeOne = async ({ + index, + queryName, + sql, + sqlHash, + cleanedSql, + }: (typeof uncachedQueries)[number]): Promise => { + const result = (await client.statementExecution.executeStatement({ + statement: `DESCRIBE QUERY ${cleanedSql}`, + warehouse_id: warehouseId, + })) as DatabricksStatementExecutionResponse; + + completed++; + spinner.update( + `Describing ${total} ${total === 1 ? "query" : "queries"} (${completed}/${total})`, + ); + + if (result.status.state === "FAILED") { + const sqlError = + result.status.error?.message || "Query execution failed"; + const type = generateUnknownResultQuery(sql, queryName); + return { + status: "fail", index, - queryName, - sql, - sqlHash, - cleanedSql, - }): Promise => { - const result = (await client.statementExecution.executeStatement({ - statement: `DESCRIBE QUERY ${cleanedSql}`, - warehouse_id: warehouseId, - })) as DatabricksStatementExecutionResponse; - - completed++; - spinner.update( - `Describing ${total} ${total === 1 ? "query" : "queries"} (${completed}/${total})`, - ); - - if (result.status.state === "FAILED") { - const sqlError = - result.status.error?.message || "Query execution failed"; - const type = generateUnknownResultQuery(sql, queryName); - return { - status: "fail", - index, - schema: { name: queryName, type }, - cacheEntry: { hash: sqlHash, type, retry: true }, - errorLines: [ - `SQL Error: ${sqlError}`, - `Query: ${cleanedSql.slice(0, 200)}`, - ], - }; - } - - const { type, hasResults } = convertToQueryType( - result, - sql, - queryName, - ); - return { - status: "ok", - index, - schema: { name: queryName, type }, - cacheEntry: { hash: sqlHash, type, retry: !hasResults }, - }; - }, - ), - ); - - spinner.stop(`✓ Described ${total} ${total === 1 ? "query" : "queries"}`); - - // Print per-query results - for (let i = 0; i < settled.length; i++) { - const entry = settled[i]; - const { queryName } = uncachedQueries[i]; + schema: { name: queryName, type }, + cacheEntry: { hash: sqlHash, type, retry: true }, + errorLines: [ + `SQL Error: ${sqlError}`, + `Query: ${cleanedSql.slice(0, 200)}`, + ], + }; + } - if (entry.status === "fulfilled") { - const res = entry.value; - freshResults.push({ index: res.index, schema: res.schema }); - cache.queries[queryName] = res.cacheEntry; + const { type, hasResults } = convertToQueryType(result, sql, queryName); + return { + status: "ok", + index, + schema: { name: queryName, type }, + cacheEntry: { hash: sqlHash, type, retry: !hasResults }, + }; + }; - if (res.status === "ok") { - spinner.printDetail(`✓ ${queryName}`); - } else { - spinner.printDetail(`✗ ${queryName} - failed`); - for (const line of res.errorLines) { - spinner.printDetail(` ${line}`); + // Process in chunks, saving cache after each chunk + const processBatchResults = ( + settled: PromiseSettledResult[], + batchOffset: number, + ) => { + for (let i = 0; i < settled.length; i++) { + const entry = settled[i]; + const { queryName } = uncachedQueries[batchOffset + i]; + + if (entry.status === "fulfilled") { + const res = entry.value; + freshResults.push({ index: res.index, schema: res.schema }); + cache.queries[queryName] = res.cacheEntry; + + if (res.status === "ok") { + spinner.printDetail(`✓ ${queryName}`); + } else { + spinner.printDetail(`✗ ${queryName} - failed`); + for (const line of res.errorLines) { + spinner.printDetail(` ${line}`); + } } + } else { + const errorMessage = + entry.reason instanceof Error + ? entry.reason.message + : "Unknown error"; + const { sql, sqlHash, index } = uncachedQueries[batchOffset + i]; + const type = generateUnknownResultQuery(sql, queryName); + freshResults.push({ index, schema: { name: queryName, type } }); + cache.queries[queryName] = { hash: sqlHash, type, retry: true }; + + spinner.printDetail(`✗ ${queryName}`); + spinner.printDetail(` ${errorMessage}`); } - } else { - const errorMessage = - entry.reason instanceof Error - ? entry.reason.message - : "Unknown error"; - const { sql, sqlHash, index } = uncachedQueries[i]; - const type = generateUnknownResultQuery(sql, queryName); - freshResults.push({ index, schema: { name: queryName, type } }); - cache.queries[queryName] = { hash: sqlHash, type, retry: true }; + } + }; - spinner.printDetail(`✗ ${queryName}`); - spinner.printDetail(` ${errorMessage}`); + if (uncachedQueries.length > concurrency) { + for (let b = 0; b < uncachedQueries.length; b += concurrency) { + const batch = uncachedQueries.slice(b, b + concurrency); + const batchResults = await Promise.allSettled(batch.map(describeOne)); + processBatchResults(batchResults, b); + await saveCache(cache); } + } else { + const settled = await Promise.allSettled( + uncachedQueries.map(describeOne), + ); + processBatchResults(settled, 0); + await saveCache(cache); } - // Save cache once after all queries are processed - saveCache(cache); + spinner.stop(`✓ Described ${total} ${total === 1 ? "query" : "queries"}`); } const elapsed = ((performance.now() - startTime) / 1000).toFixed(2); diff --git a/packages/appkit/src/type-generator/tests/generate-queries.test.ts b/packages/appkit/src/type-generator/tests/generate-queries.test.ts index 8264e102..367bc1ea 100644 --- a/packages/appkit/src/type-generator/tests/generate-queries.test.ts +++ b/packages/appkit/src/type-generator/tests/generate-queries.test.ts @@ -1,8 +1,8 @@ import { beforeEach, describe, expect, test, vi } from "vitest"; const mocks = vi.hoisted(() => ({ - readdirSync: vi.fn(), - readFileSync: vi.fn(), + readdir: vi.fn(), + readFile: vi.fn(), executeStatement: vi.fn(), spinnerStop: vi.fn(), spinnerPrintDetail: vi.fn(), @@ -10,10 +10,10 @@ const mocks = vi.hoisted(() => ({ saveCache: vi.fn(), })); -vi.mock("node:fs", () => ({ +vi.mock("node:fs/promises", () => ({ default: { - readdirSync: mocks.readdirSync, - readFileSync: mocks.readFileSync, + readdir: mocks.readdir, + readFile: mocks.readFile, }, })); @@ -53,8 +53,8 @@ describe("generateQueriesFromDescribe", () => { }); test("success path — returns query schema", async () => { - mocks.readdirSync.mockReturnValue(["users.sql"]); - mocks.readFileSync.mockReturnValue( + mocks.readdir.mockResolvedValue(["users.sql"]); + mocks.readFile.mockResolvedValue( "SELECT id, name FROM users WHERE status = :status", ); mocks.executeStatement.mockResolvedValue( @@ -76,8 +76,8 @@ describe("generateQueriesFromDescribe", () => { }); test("FAILED status with error message — reports SQL error and produces unknown result type", async () => { - mocks.readdirSync.mockReturnValue(["bad_table.sql"]); - mocks.readFileSync.mockReturnValue("SELECT * FROM bad_table"); + mocks.readdir.mockResolvedValue(["bad_table.sql"]); + mocks.readFile.mockResolvedValue("SELECT * FROM bad_table"); mocks.executeStatement.mockResolvedValue({ statement_id: "stmt-2", status: { @@ -105,8 +105,8 @@ describe("generateQueriesFromDescribe", () => { }); test("FAILED status without error message — uses fallback message and produces unknown result type", async () => { - mocks.readdirSync.mockReturnValue(["query.sql"]); - mocks.readFileSync.mockReturnValue("SELECT 1"); + mocks.readdir.mockResolvedValue(["query.sql"]); + mocks.readFile.mockResolvedValue("SELECT 1"); mocks.executeStatement.mockResolvedValue({ statement_id: "stmt-3", status: { state: "FAILED" }, @@ -126,10 +126,10 @@ describe("generateQueriesFromDescribe", () => { }); test("partial failure — caches success, unknown result for failure, output includes both", async () => { - mocks.readdirSync.mockReturnValue(["good.sql", "bad.sql"]); - mocks.readFileSync - .mockReturnValueOnce("SELECT id FROM good_table WHERE status = :status") - .mockReturnValueOnce("SELECT * FROM missing_table"); + mocks.readdir.mockResolvedValue(["good.sql", "bad.sql"]); + mocks.readFile + .mockResolvedValueOnce("SELECT id FROM good_table WHERE status = :status") + .mockResolvedValueOnce("SELECT * FROM missing_table"); mocks.executeStatement .mockResolvedValueOnce(succeededResult([["id", "INT", null]])) @@ -158,10 +158,10 @@ describe("generateQueriesFromDescribe", () => { }); test("all queries fail — caches with retry flag, all unknown result types", async () => { - mocks.readdirSync.mockReturnValue(["a.sql", "b.sql"]); - mocks.readFileSync - .mockReturnValueOnce("SELECT * FROM table_a") - .mockReturnValueOnce("SELECT * FROM table_b"); + mocks.readdir.mockResolvedValue(["a.sql", "b.sql"]); + mocks.readFile + .mockResolvedValueOnce("SELECT * FROM table_a") + .mockResolvedValueOnce("SELECT * FROM table_b"); mocks.executeStatement .mockRejectedValueOnce(new Error("Connection refused")) @@ -182,9 +182,35 @@ describe("generateQueriesFromDescribe", () => { expect(mocks.saveCache).toHaveBeenCalledTimes(1); }); + test("concurrency batching — saves cache after each batch", async () => { + // 3 queries with concurrency=2 → 2 batches (2 + 1), saveCache called twice + mocks.readdir.mockResolvedValue(["q1.sql", "q2.sql", "q3.sql"]); + mocks.readFile + .mockResolvedValueOnce("SELECT id FROM t1") + .mockResolvedValueOnce("SELECT id FROM t2") + .mockResolvedValueOnce("SELECT id FROM t3"); + + mocks.executeStatement + .mockResolvedValueOnce(succeededResult([["id", "INT", null]])) + .mockResolvedValueOnce(succeededResult([["id", "INT", null]])) + .mockResolvedValueOnce(succeededResult([["id", "INT", null]])); + + const schemas = await generateQueriesFromDescribe("/queries", "wh-123", { + concurrency: 2, + }); + + expect(schemas).toHaveLength(3); + expect(schemas[0].name).toBe("q1"); + expect(schemas[1].name).toBe("q2"); + expect(schemas[2].name).toBe("q3"); + + // 2 batches → 2 saveCache calls + expect(mocks.saveCache).toHaveBeenCalledTimes(2); + }); + test("unknown result type includes parameters from SQL", async () => { - mocks.readdirSync.mockReturnValue(["parameterized.sql"]); - mocks.readFileSync.mockReturnValue( + mocks.readdir.mockResolvedValue(["parameterized.sql"]); + mocks.readFile.mockResolvedValue( "-- @param status STRING\nSELECT * FROM t WHERE status = :status AND org = :org", ); mocks.executeStatement.mockRejectedValueOnce(new Error("timeout")); diff --git a/packages/appkit/src/type-generator/vite-plugin.ts b/packages/appkit/src/type-generator/vite-plugin.ts index 998daf0a..741308f2 100644 --- a/packages/appkit/src/type-generator/vite-plugin.ts +++ b/packages/appkit/src/type-generator/vite-plugin.ts @@ -1,4 +1,4 @@ -import fs from "node:fs"; +import { existsSync } from "node:fs"; import path from "node:path"; import type { Plugin } from "vite"; import { createLogger } from "../logging/logger"; @@ -62,7 +62,7 @@ export function appKitTypesPlugin(options?: AppKitTypesPluginOptions): Plugin { return false; } - if (!fs.existsSync(path.join(process.cwd(), "config", "queries"))) { + if (!existsSync(path.join(process.cwd(), "config", "queries"))) { return false; } From 6f9724c1a52cba148a485a3ccc4fb3eea60bdcb5 Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Wed, 4 Mar 2026 13:16:09 +0100 Subject: [PATCH 03/10] fail-safe concurrency Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- packages/appkit/src/type-generator/query-registry.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/appkit/src/type-generator/query-registry.ts b/packages/appkit/src/type-generator/query-registry.ts index 11f1bea7..45ce208f 100644 --- a/packages/appkit/src/type-generator/query-registry.ts +++ b/packages/appkit/src/type-generator/query-registry.ts @@ -143,7 +143,11 @@ export async function generateQueriesFromDescribe( warehouseId: string, options: { noCache?: boolean; concurrency?: number } = {}, ): Promise { - const { noCache = false, concurrency = 10 } = options; + const { noCache = false, concurrency: rawConcurrency = 10 } = options; + const concurrency = + typeof rawConcurrency === "number" && Number.isFinite(rawConcurrency) + ? Math.max(1, Math.floor(rawConcurrency)) + : 10; // read all query files and cache in parallel const [allFiles, cache] = await Promise.all([ From 030d3cc255e56c86a51b919ae8e4642b7b5f927a Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Wed, 4 Mar 2026 13:31:52 +0100 Subject: [PATCH 04/10] chore: improve error message when cache cannot be read --- packages/appkit/src/type-generator/cache.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/appkit/src/type-generator/cache.ts b/packages/appkit/src/type-generator/cache.ts index 8b22ca96..0d45afc8 100644 --- a/packages/appkit/src/type-generator/cache.ts +++ b/packages/appkit/src/type-generator/cache.ts @@ -61,8 +61,11 @@ export async function loadCache(): Promise { return cache; } } catch (err) { - if ((err as NodeJS.ErrnoException).code !== "ENOENT") { - logger.warn("Cache file is corrupted, flushing cache completely."); + const error = err as NodeJS.ErrnoException; + if (error.code !== "ENOENT") { + logger.warn( + `Failed to load cache file at ${cachePath} (code: ${error.code ?? "UNKNOWN"}). Assuming empty cache.`, + ); } } return { version: CACHE_VERSION, queries: {} }; From 96a413670ac1ca2c8caefe675422168814740b19 Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Thu, 5 Mar 2026 15:29:48 +0100 Subject: [PATCH 05/10] stop batching terminal stdout This reverts commit 856007eef1a703105906a9460971e499a69afc7a. --- packages/appkit/src/type-generator/cache.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/appkit/src/type-generator/cache.ts b/packages/appkit/src/type-generator/cache.ts index 0d45afc8..8b22ca96 100644 --- a/packages/appkit/src/type-generator/cache.ts +++ b/packages/appkit/src/type-generator/cache.ts @@ -61,11 +61,8 @@ export async function loadCache(): Promise { return cache; } } catch (err) { - const error = err as NodeJS.ErrnoException; - if (error.code !== "ENOENT") { - logger.warn( - `Failed to load cache file at ${cachePath} (code: ${error.code ?? "UNKNOWN"}). Assuming empty cache.`, - ); + if ((err as NodeJS.ErrnoException).code !== "ENOENT") { + logger.warn("Cache file is corrupted, flushing cache completely."); } } return { version: CACHE_VERSION, queries: {} }; From 3ffb55ae816fe4b982fdd78cc437f8bc44d9b91a Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Thu, 5 Mar 2026 15:42:02 +0100 Subject: [PATCH 06/10] chore: fix progress logs --- packages/appkit/src/type-generator/query-registry.ts | 6 +++--- packages/appkit/src/type-generator/spinner.ts | 6 +++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/appkit/src/type-generator/query-registry.ts b/packages/appkit/src/type-generator/query-registry.ts index 45ce208f..1e26221c 100644 --- a/packages/appkit/src/type-generator/query-registry.ts +++ b/packages/appkit/src/type-generator/query-registry.ts @@ -195,7 +195,7 @@ export async function generateQueriesFromDescribe( schema: { name: queryName, type: cached.type }, }); spinner.start(`Processing ${queryName} (${i + 1}/${queryFiles.length})`); - spinner.stop(`✓ ${queryName} (cached)`); + spinner.stop(`✓ ${queryName} (cache HIT)`); } else { const sqlWithDefaults = sql.replace(/:([a-zA-Z_]\w*)/g, "''"); const cleanedSql = sqlWithDefaults.trim().replace(/;\s*$/, ""); @@ -286,9 +286,9 @@ export async function generateQueriesFromDescribe( cache.queries[queryName] = res.cacheEntry; if (res.status === "ok") { - spinner.printDetail(`✓ ${queryName}`); + spinner.printDetail(`✓ ${queryName} (cache MISS)`); } else { - spinner.printDetail(`✗ ${queryName} - failed`); + spinner.printDetail(`✗ ${queryName} - failed (cache MISS)`); for (const line of res.errorLines) { spinner.printDetail(` ${line}`); } diff --git a/packages/appkit/src/type-generator/spinner.ts b/packages/appkit/src/type-generator/spinner.ts index add1a629..d5607f55 100644 --- a/packages/appkit/src/type-generator/spinner.ts +++ b/packages/appkit/src/type-generator/spinner.ts @@ -31,6 +31,10 @@ export class Spinner { } printDetail(text: string) { - process.stdout.write(`\x1b[2m ${text}\x1b[0m\n`); + // Clear spinner line, print detail, then redraw spinner + process.stdout.write(`\x1b[2K\r\x1b[2m ${text}\x1b[0m\n`); + if (this.interval) { + process.stdout.write(` ${this.text}${this.frames[this.current]}`); + } } } From 3e53787af6f17eb3488dd926d28e584ab54a37bc Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Thu, 5 Mar 2026 16:00:27 +0100 Subject: [PATCH 07/10] chore: upload logs for consistency --- .../src/type-generator/query-registry.ts | 67 ++++++++++++------- 1 file changed, 41 insertions(+), 26 deletions(-) diff --git a/packages/appkit/src/type-generator/query-registry.ts b/packages/appkit/src/type-generator/query-registry.ts index 1e26221c..58d5713d 100644 --- a/packages/appkit/src/type-generator/query-registry.ts +++ b/packages/appkit/src/type-generator/query-registry.ts @@ -1,6 +1,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { WorkspaceClient } from "@databricks/sdk-experimental"; +import pc from "picocolors"; import { createLogger } from "../logging/logger"; import { CACHE_VERSION, hashSQL, loadCache, saveCache } from "./cache"; import { Spinner } from "./spinner"; @@ -170,6 +171,8 @@ export async function generateQueriesFromDescribe( queryFiles.map((file) => fs.readFile(path.join(queryFolder, file), "utf8")), ); + const startTime = performance.now(); + // Phase 1: Check cache, separate cached vs uncached const cachedResults: Array<{ index: number; schema: QuerySchema }> = []; const uncachedQueries: Array<{ @@ -179,6 +182,11 @@ export async function generateQueriesFromDescribe( sqlHash: string; cleanedSql: string; }> = []; + const logEntries: Array<{ + queryName: string; + status: "HIT" | "MISS"; + failed?: boolean; + }> = []; for (let i = 0; i < queryFiles.length; i++) { const file = queryFiles[i]; @@ -194,8 +202,7 @@ export async function generateQueriesFromDescribe( index: i, schema: { name: queryName, type: cached.type }, }); - spinner.start(`Processing ${queryName} (${i + 1}/${queryFiles.length})`); - spinner.stop(`✓ ${queryName} (cache HIT)`); + logEntries.push({ queryName, status: "HIT" }); } else { const sqlWithDefaults = sql.replace(/:([a-zA-Z_]\w*)/g, "''"); const cleanedSql = sqlWithDefaults.trim().replace(/;\s*$/, ""); @@ -220,7 +227,6 @@ export async function generateQueriesFromDescribe( }; const freshResults: Array<{ index: number; schema: QuerySchema }> = []; - const startTime = performance.now(); if (uncachedQueries.length > 0) { let completed = 0; @@ -284,27 +290,17 @@ export async function generateQueriesFromDescribe( const res = entry.value; freshResults.push({ index: res.index, schema: res.schema }); cache.queries[queryName] = res.cacheEntry; - - if (res.status === "ok") { - spinner.printDetail(`✓ ${queryName} (cache MISS)`); - } else { - spinner.printDetail(`✗ ${queryName} - failed (cache MISS)`); - for (const line of res.errorLines) { - spinner.printDetail(` ${line}`); - } - } + logEntries.push({ + queryName, + status: "MISS", + failed: res.status === "fail", + }); } else { - const errorMessage = - entry.reason instanceof Error - ? entry.reason.message - : "Unknown error"; const { sql, sqlHash, index } = uncachedQueries[batchOffset + i]; const type = generateUnknownResultQuery(sql, queryName); freshResults.push({ index, schema: { name: queryName, type } }); cache.queries[queryName] = { hash: sqlHash, type, retry: true }; - - spinner.printDetail(`✗ ${queryName}`); - spinner.printDetail(` ${errorMessage}`); + logEntries.push({ queryName, status: "MISS", failed: true }); } } }; @@ -324,16 +320,35 @@ export async function generateQueriesFromDescribe( await saveCache(cache); } - spinner.stop(`✓ Described ${total} ${total === 1 ? "query" : "queries"}`); + spinner.stop(""); } const elapsed = ((performance.now() - startTime) / 1000).toFixed(2); - logger.debug( - "%d new queries, %d cached. Typegen: %ss", - uncachedQueries.length, - cachedResults.length, - elapsed, - ); + + // Print formatted table + if (logEntries.length > 0) { + const separator = pc.dim("─".repeat(50)); + console.log(""); + console.log( + ` ${pc.bold("Typegen Queries")} ${pc.dim(`(${logEntries.length})`)}`, + ); + console.log(` ${separator}`); + for (const entry of logEntries) { + const tag = + entry.status === "HIT" + ? `cache ${pc.green("HIT ")}` + : `cache ${pc.red("MISS")}`; + const name = entry.failed + ? pc.dim(pc.strikethrough(entry.queryName)) + : entry.queryName; + console.log(` ${tag} ${name}`); + } + console.log(` ${separator}`); + console.log( + ` ${uncachedQueries.length} new, ${cachedResults.length} from cache. ${pc.dim(`${elapsed}s`)}`, + ); + console.log(""); + } // Merge and sort by original file index for deterministic output return [...cachedResults, ...freshResults] From e9ac4caf31d0490c36fa787894bcf439c8e9c12e Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Thu, 5 Mar 2026 16:12:15 +0100 Subject: [PATCH 08/10] chore: update tests --- .../src/type-generator/query-registry.ts | 4 ++-- .../tests/generate-queries.test.ts | 20 +++---------------- 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/packages/appkit/src/type-generator/query-registry.ts b/packages/appkit/src/type-generator/query-registry.ts index 58d5713d..8cf34d2b 100644 --- a/packages/appkit/src/type-generator/query-registry.ts +++ b/packages/appkit/src/type-generator/query-registry.ts @@ -336,8 +336,8 @@ export async function generateQueriesFromDescribe( for (const entry of logEntries) { const tag = entry.status === "HIT" - ? `cache ${pc.green("HIT ")}` - : `cache ${pc.red("MISS")}`; + ? `cache ${pc.bold(pc.green("HIT "))}` + : `cache ${pc.bold(pc.red("MISS"))}`; const name = entry.failed ? pc.dim(pc.strikethrough(entry.queryName)) : entry.queryName; diff --git a/packages/appkit/src/type-generator/tests/generate-queries.test.ts b/packages/appkit/src/type-generator/tests/generate-queries.test.ts index 367bc1ea..ac43ef9e 100644 --- a/packages/appkit/src/type-generator/tests/generate-queries.test.ts +++ b/packages/appkit/src/type-generator/tests/generate-queries.test.ts @@ -70,8 +70,7 @@ describe("generateQueriesFromDescribe", () => { expect(schemas[0].name).toBe("users"); expect(schemas[0].type).toContain("id: number"); expect(schemas[0].type).toContain("name: string"); - expect(mocks.spinnerStop).toHaveBeenCalledWith("✓ Described 1 query"); - expect(mocks.spinnerPrintDetail).toHaveBeenCalledWith("✓ users"); + expect(mocks.spinnerStop).toHaveBeenCalledWith(""); expect(mocks.saveCache).toHaveBeenCalledTimes(1); }); @@ -91,16 +90,7 @@ describe("generateQueriesFromDescribe", () => { expect(schemas).toHaveLength(1); expect(schemas[0].name).toBe("bad_table"); expect(schemas[0].type).toContain("result: unknown"); - expect(mocks.spinnerStop).toHaveBeenCalledWith("✓ Described 1 query"); - expect(mocks.spinnerPrintDetail).toHaveBeenCalledWith( - "✗ bad_table - failed", - ); - expect(mocks.spinnerPrintDetail).toHaveBeenCalledWith( - expect.stringContaining("SQL Error: Table or view not found: bad_table"), - ); - expect(mocks.spinnerPrintDetail).toHaveBeenCalledWith( - expect.stringContaining("Query:"), - ); + expect(mocks.spinnerStop).toHaveBeenCalledWith(""); expect(mocks.saveCache).toHaveBeenCalledTimes(1); }); @@ -117,11 +107,7 @@ describe("generateQueriesFromDescribe", () => { expect(schemas).toHaveLength(1); expect(schemas[0].name).toBe("query"); expect(schemas[0].type).toContain("result: unknown"); - expect(mocks.spinnerStop).toHaveBeenCalledWith("✓ Described 1 query"); - expect(mocks.spinnerPrintDetail).toHaveBeenCalledWith("✗ query - failed"); - expect(mocks.spinnerPrintDetail).toHaveBeenCalledWith( - expect.stringContaining("SQL Error: Query execution failed"), - ); + expect(mocks.spinnerStop).toHaveBeenCalledWith(""); expect(mocks.saveCache).toHaveBeenCalledTimes(1); }); From 3194aceb022d4d57f4d5d2f5c5b1c690a1af5ccb Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Thu, 5 Mar 2026 17:36:27 +0100 Subject: [PATCH 09/10] chore: error reports per query --- .../client/src/appKitTypes.d.ts | 89 ++----------------- .../src/type-generator/query-registry.ts | 75 ++++++++++++---- 2 files changed, 67 insertions(+), 97 deletions(-) diff --git a/apps/dev-playground/client/src/appKitTypes.d.ts b/apps/dev-playground/client/src/appKitTypes.d.ts index 0e0ae0b0..535a2862 100644 --- a/apps/dev-playground/client/src/appKitTypes.d.ts +++ b/apps/dev-playground/client/src/appKitTypes.d.ts @@ -13,48 +13,22 @@ declare module "@databricks/appkit-ui/react" { /** DATE - use sql.date() */ endDate: SQLDateMarker; }; - result: Array<{ - /** @sqlType STRING */ - app_name: string; - /** @sqlType STRING */ - day_of_week: string; - /** @sqlType DECIMAL(35,2) */ - spend: number; - }>; + result: unknown; }; apps_list: { name: "apps_list"; parameters: Record; - result: Array<{ - /** @sqlType STRING */ - id: string; - /** @sqlType STRING */ - name: string; - /** @sqlType STRING */ - creator: string; - /** @sqlType STRING */ - tags: string; - /** @sqlType DECIMAL(38,6) */ - totalSpend: number; - /** @sqlType DATE */ - createdAt: string; - }>; + result: unknown; }; cost_recommendations: { name: "cost_recommendations"; parameters: Record; - result: Array<{ - /** @sqlType INT */ - dummy: number; - }>; + result: unknown; }; example: { name: "example"; parameters: Record; - result: Array<{ - /** @sqlType BOOLEAN */ - "(1 = 1)": boolean; - }>; + result: unknown; }; spend_data: { name: "spend_data"; @@ -72,14 +46,7 @@ declare module "@databricks/appkit-ui/react" { /** STRING - use sql.string() */ creator: SQLStringMarker; }; - result: Array<{ - /** @sqlType STRING */ - group_key: string; - /** @sqlType TIMESTAMP */ - aggregation_period: string; - /** @sqlType DECIMAL(38,6) */ - cost_usd: number; - }>; + result: unknown; }; spend_summary: { name: "spend_summary"; @@ -91,14 +58,7 @@ declare module "@databricks/appkit-ui/react" { /** DATE - use sql.date() */ startDate: SQLDateMarker; }; - result: Array<{ - /** @sqlType DECIMAL(33,0) */ - total: number; - /** @sqlType DECIMAL(33,0) */ - average: number; - /** @sqlType DECIMAL(33,0) */ - forecasted: number; - }>; + result: unknown; }; sql_helpers_test: { name: "sql_helpers_test"; @@ -116,24 +76,7 @@ declare module "@databricks/appkit-ui/react" { /** STRING - use sql.string() */ binaryParam: SQLStringMarker; }; - result: Array<{ - /** @sqlType STRING */ - string_value: string; - /** @sqlType STRING */ - number_value: string; - /** @sqlType STRING */ - boolean_value: string; - /** @sqlType STRING */ - date_value: string; - /** @sqlType STRING */ - timestamp_value: string; - /** @sqlType BINARY */ - binary_value: string; - /** @sqlType STRING */ - binary_hex: string; - /** @sqlType INT */ - binary_length: number; - }>; + result: unknown; }; top_contributors: { name: "top_contributors"; @@ -145,12 +88,7 @@ declare module "@databricks/appkit-ui/react" { /** DATE - use sql.date() */ endDate: SQLDateMarker; }; - result: Array<{ - /** @sqlType STRING */ - app_name: string; - /** @sqlType DECIMAL(38,6) */ - total_cost_usd: number; - }>; + result: unknown; }; untagged_apps: { name: "untagged_apps"; @@ -162,16 +100,7 @@ declare module "@databricks/appkit-ui/react" { /** DATE - use sql.date() */ endDate: SQLDateMarker; }; - result: Array<{ - /** @sqlType STRING */ - app_name: string; - /** @sqlType STRING */ - creator: string; - /** @sqlType DECIMAL(38,6) */ - total_cost_usd: number; - /** @sqlType DECIMAL(38,10) */ - avg_period_cost_usd: number; - }>; + result: unknown; }; } } diff --git a/packages/appkit/src/type-generator/query-registry.ts b/packages/appkit/src/type-generator/query-registry.ts index 8cf34d2b..57e341a0 100644 --- a/packages/appkit/src/type-generator/query-registry.ts +++ b/packages/appkit/src/type-generator/query-registry.ts @@ -14,6 +14,29 @@ import { const logger = createLogger("type-generator:query-registry"); +/** + * Parse a raw API/SDK error into a structured code + message. + * Handles Databricks-style JSON bodies embedded in the message string, + * e.g. `Response from server (Bad Request) {"error_code":"...","message":"..."}`. + */ +function parseError(raw: string): { code?: string; message: string } { + const jsonMatch = raw.match(/\{[\s\S]*\}/); + if (jsonMatch) { + try { + const parsed = JSON.parse(jsonMatch[0]); + if (parsed.error_code || parsed.message) { + return { + code: parsed.error_code, + message: parsed.message || raw, + }; + } + } catch { + // not valid JSON, fall through + } + } + return { message: raw }; +} + /** * Extract parameters from a SQL query * @param sql - the SQL query to extract parameters from @@ -186,6 +209,7 @@ export async function generateQueriesFromDescribe( queryName: string; status: "HIT" | "MISS"; failed?: boolean; + error?: { code?: string; message: string }; }> = []; for (let i = 0; i < queryFiles.length; i++) { @@ -223,7 +247,7 @@ export async function generateQueriesFromDescribe( index: number; schema: QuerySchema; cacheEntry: { hash: string; type: string; retry: boolean }; - errorLines: string[]; + error: { code?: string; message: string }; }; const freshResults: Array<{ index: number; schema: QuerySchema }> = []; @@ -261,10 +285,7 @@ export async function generateQueriesFromDescribe( index, schema: { name: queryName, type }, cacheEntry: { hash: sqlHash, type, retry: true }, - errorLines: [ - `SQL Error: ${sqlError}`, - `Query: ${cleanedSql.slice(0, 200)}`, - ], + error: parseError(sqlError), }; } @@ -294,13 +315,23 @@ export async function generateQueriesFromDescribe( queryName, status: "MISS", failed: res.status === "fail", + error: res.status === "fail" ? res.error : undefined, }); } else { const { sql, sqlHash, index } = uncachedQueries[batchOffset + i]; const type = generateUnknownResultQuery(sql, queryName); freshResults.push({ index, schema: { name: queryName, type } }); cache.queries[queryName] = { hash: sqlHash, type, retry: true }; - logEntries.push({ queryName, status: "MISS", failed: true }); + logEntries.push({ + queryName, + status: "MISS", + failed: true, + error: parseError( + entry.reason instanceof Error + ? entry.reason.message + : String(entry.reason), + ), + }); } } }; @@ -327,6 +358,7 @@ export async function generateQueriesFromDescribe( // Print formatted table if (logEntries.length > 0) { + const maxNameLen = Math.max(...logEntries.map((e) => e.queryName.length)); const separator = pc.dim("─".repeat(50)); console.log(""); console.log( @@ -334,19 +366,28 @@ export async function generateQueriesFromDescribe( ); console.log(` ${separator}`); for (const entry of logEntries) { - const tag = - entry.status === "HIT" - ? `cache ${pc.bold(pc.green("HIT "))}` - : `cache ${pc.bold(pc.red("MISS"))}`; - const name = entry.failed - ? pc.dim(pc.strikethrough(entry.queryName)) - : entry.queryName; - console.log(` ${tag} ${name}`); + const tag = entry.failed + ? pc.bold(pc.red("ERROR")) + : entry.status === "HIT" + ? `cache ${pc.bold(pc.green("HIT "))}` + : `cache ${pc.bold(pc.yellow("MISS "))}`; + const rawName = entry.queryName.padEnd(maxNameLen); + const name = entry.failed ? pc.dim(pc.strikethrough(rawName)) : rawName; + const reason = entry.error ? ` ${entry.error.message}` : ""; + console.log(` ${tag} ${name}${reason}`); } + const newCount = logEntries.filter( + (e) => e.status === "MISS" && !e.failed, + ).length; + const cacheCount = logEntries.filter( + (e) => e.status === "HIT" && !e.failed, + ).length; + const errorCount = logEntries.filter((e) => e.failed).length; console.log(` ${separator}`); - console.log( - ` ${uncachedQueries.length} new, ${cachedResults.length} from cache. ${pc.dim(`${elapsed}s`)}`, - ); + const parts = [`${newCount} new`, `${cacheCount} from cache`]; + if (errorCount > 0) + parts.push(`${errorCount} ${errorCount === 1 ? "error" : "errors"}`); + console.log(` ${parts.join(", ")}. ${pc.dim(`${elapsed}s`)}`); console.log(""); } From 8e2ebe67bd388060aa6232c10c65d0d6bd1e2eeb Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Thu, 5 Mar 2026 18:11:03 +0100 Subject: [PATCH 10/10] chore: reformat error messages --- .../client/src/appKitTypes.d.ts | 89 +++++++++++++++++-- .../src/type-generator/query-registry.ts | 22 +++-- 2 files changed, 96 insertions(+), 15 deletions(-) diff --git a/apps/dev-playground/client/src/appKitTypes.d.ts b/apps/dev-playground/client/src/appKitTypes.d.ts index 535a2862..0e0ae0b0 100644 --- a/apps/dev-playground/client/src/appKitTypes.d.ts +++ b/apps/dev-playground/client/src/appKitTypes.d.ts @@ -13,22 +13,48 @@ declare module "@databricks/appkit-ui/react" { /** DATE - use sql.date() */ endDate: SQLDateMarker; }; - result: unknown; + result: Array<{ + /** @sqlType STRING */ + app_name: string; + /** @sqlType STRING */ + day_of_week: string; + /** @sqlType DECIMAL(35,2) */ + spend: number; + }>; }; apps_list: { name: "apps_list"; parameters: Record; - result: unknown; + result: Array<{ + /** @sqlType STRING */ + id: string; + /** @sqlType STRING */ + name: string; + /** @sqlType STRING */ + creator: string; + /** @sqlType STRING */ + tags: string; + /** @sqlType DECIMAL(38,6) */ + totalSpend: number; + /** @sqlType DATE */ + createdAt: string; + }>; }; cost_recommendations: { name: "cost_recommendations"; parameters: Record; - result: unknown; + result: Array<{ + /** @sqlType INT */ + dummy: number; + }>; }; example: { name: "example"; parameters: Record; - result: unknown; + result: Array<{ + /** @sqlType BOOLEAN */ + "(1 = 1)": boolean; + }>; }; spend_data: { name: "spend_data"; @@ -46,7 +72,14 @@ declare module "@databricks/appkit-ui/react" { /** STRING - use sql.string() */ creator: SQLStringMarker; }; - result: unknown; + result: Array<{ + /** @sqlType STRING */ + group_key: string; + /** @sqlType TIMESTAMP */ + aggregation_period: string; + /** @sqlType DECIMAL(38,6) */ + cost_usd: number; + }>; }; spend_summary: { name: "spend_summary"; @@ -58,7 +91,14 @@ declare module "@databricks/appkit-ui/react" { /** DATE - use sql.date() */ startDate: SQLDateMarker; }; - result: unknown; + result: Array<{ + /** @sqlType DECIMAL(33,0) */ + total: number; + /** @sqlType DECIMAL(33,0) */ + average: number; + /** @sqlType DECIMAL(33,0) */ + forecasted: number; + }>; }; sql_helpers_test: { name: "sql_helpers_test"; @@ -76,7 +116,24 @@ declare module "@databricks/appkit-ui/react" { /** STRING - use sql.string() */ binaryParam: SQLStringMarker; }; - result: unknown; + result: Array<{ + /** @sqlType STRING */ + string_value: string; + /** @sqlType STRING */ + number_value: string; + /** @sqlType STRING */ + boolean_value: string; + /** @sqlType STRING */ + date_value: string; + /** @sqlType STRING */ + timestamp_value: string; + /** @sqlType BINARY */ + binary_value: string; + /** @sqlType STRING */ + binary_hex: string; + /** @sqlType INT */ + binary_length: number; + }>; }; top_contributors: { name: "top_contributors"; @@ -88,7 +145,12 @@ declare module "@databricks/appkit-ui/react" { /** DATE - use sql.date() */ endDate: SQLDateMarker; }; - result: unknown; + result: Array<{ + /** @sqlType STRING */ + app_name: string; + /** @sqlType DECIMAL(38,6) */ + total_cost_usd: number; + }>; }; untagged_apps: { name: "untagged_apps"; @@ -100,7 +162,16 @@ declare module "@databricks/appkit-ui/react" { /** DATE - use sql.date() */ endDate: SQLDateMarker; }; - result: unknown; + result: Array<{ + /** @sqlType STRING */ + app_name: string; + /** @sqlType STRING */ + creator: string; + /** @sqlType DECIMAL(38,6) */ + total_cost_usd: number; + /** @sqlType DECIMAL(38,10) */ + avg_period_cost_usd: number; + }>; }; } } diff --git a/packages/appkit/src/type-generator/query-registry.ts b/packages/appkit/src/type-generator/query-registry.ts index 57e341a0..3ef12abf 100644 --- a/packages/appkit/src/type-generator/query-registry.ts +++ b/packages/appkit/src/type-generator/query-registry.ts @@ -276,9 +276,17 @@ export async function generateQueriesFromDescribe( `Describing ${total} ${total === 1 ? "query" : "queries"} (${completed}/${total})`, ); + logger.debug( + "DESCRIBE result for %s: state=%s, rows=%d", + queryName, + result.status.state, + result.result?.data_array?.length ?? 0, + ); + if (result.status.state === "FAILED") { const sqlError = result.status.error?.message || "Query execution failed"; + logger.warn("DESCRIBE failed for %s: %s", queryName, sqlError); const type = generateUnknownResultQuery(sql, queryName); return { status: "fail", @@ -319,6 +327,11 @@ export async function generateQueriesFromDescribe( }); } else { const { sql, sqlHash, index } = uncachedQueries[batchOffset + i]; + const reason = + entry.reason instanceof Error + ? entry.reason.message + : String(entry.reason); + logger.warn("DESCRIBE rejected for %s: %s", queryName, reason); const type = generateUnknownResultQuery(sql, queryName); freshResults.push({ index, schema: { name: queryName, type } }); cache.queries[queryName] = { hash: sqlHash, type, retry: true }; @@ -326,11 +339,7 @@ export async function generateQueriesFromDescribe( queryName, status: "MISS", failed: true, - error: parseError( - entry.reason instanceof Error - ? entry.reason.message - : String(entry.reason), - ), + error: parseError(reason), }); } } @@ -373,7 +382,8 @@ export async function generateQueriesFromDescribe( : `cache ${pc.bold(pc.yellow("MISS "))}`; const rawName = entry.queryName.padEnd(maxNameLen); const name = entry.failed ? pc.dim(pc.strikethrough(rawName)) : rawName; - const reason = entry.error ? ` ${entry.error.message}` : ""; + const errorCode = entry.error?.message.match(/\[([^\]]+)\]/)?.[1]; + const reason = errorCode ? ` ${pc.dim(errorCode)}` : ""; console.log(` ${tag} ${name}${reason}`); } const newCount = logEntries.filter(