Skip to content

feat: show all flag validation failures #178

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat: show all flag validation failures #178

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 63 additions & 16 deletions packages/ai/src/cli/utils/parse-flag-overrides.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { type ZodObject, type z } from 'zod';
import { type ZodError, type ZodObject, type z } from 'zod';
import { formatZodErrors, generateFlagExamples } from './format-zod-errors.js';
import { readFileSync } from 'node:fs';
import { resolve } from 'node:path';
Expand Down Expand Up @@ -70,47 +70,94 @@ export function extractAndValidateFlagOverrides<S extends z.ZodObject<any>>(
return { cleanedArgv, overrides: overrides as any };
}

export type FlagValidationError =
| { type: 'invalid_path'; path: string }
| { type: 'invalid_value'; zodError: ZodError };

export interface FlagValidationResult {
success: boolean;
errors: FlagValidationError[];
}

/**
* Validate already-parsed flag overrides against a Zod schema.
* Use this when you have flag overrides in dot-notation form (e.g., { 'model.temperature': 0.7 })
* and want to validate them against a schema before running evals.
* Returns validation result without side effects (no console output, no process.exit).
*
* @param overrides - Flag overrides in dot-notation form
* @param overrides - Flag overrides in dot-notation form (e.g., { 'model.temperature': 0.7 })
* @param flagSchema - Zod schema to validate against
* @returns Validation result with any errors found
*/
export function validateFlagOverrides(overrides: FlagOverrides, flagSchema?: unknown): void {
export function collectFlagValidationErrors(
overrides: FlagOverrides,
flagSchema?: unknown,
): FlagValidationResult {
// No schema provided = no validation, any flags allowed
if (!flagSchema || Object.keys(overrides).length === 0) {
return;
return { success: true, errors: [] };
}

const schema = flagSchema as any;
const errors: FlagValidationError[] = [];

// First pass: check all paths exist in schema
for (const dotPath of Object.keys(overrides)) {
const segments = parsePath(dotPath);
if (!isValidPath(schema, segments)) {
console.error('❌ Invalid CLI flags:');
console.error(` • flag '${dotPath}': Invalid flag path`);
process.exit(1);
errors.push({ type: 'invalid_path', path: dotPath });
}
}

// If there are invalid paths, don't proceed to value validation
if (errors.length > 0) {
return { success: false, errors };
}

// Second pass: validate values using nested object approach
const nestedObject = dotNotationToNested(overrides);
const result = schema.strict().partial().safeParse(nestedObject);

if (!result.success) {
console.error('❌ Invalid CLI flags:');
console.error(formatZodErrors(result.error));
errors.push({ type: 'invalid_value', zodError: result.error });
}

return { success: errors.length === 0, errors };
}

/**
* Print flag validation errors to console and exit.
*/
export function printFlagValidationErrorsAndExit(errors: FlagValidationError[]): never {
console.error('❌ Invalid CLI flags:');

for (const error of errors) {
if (error.type === 'invalid_path') {
console.error(` • flag '${error.path}': Invalid flag path`);
} else {
console.error(formatZodErrors(error.zodError));

const examples = generateFlagExamples(result.error);
if (examples.length > 0) {
console.error('\n💡 Valid examples:');
examples.forEach((example) => console.error(` ${example}`));
const examples = generateFlagExamples(error.zodError);
if (examples.length > 0) {
console.error('\n💡 Valid examples:');
examples.forEach((example) => console.error(` ${example}`));
}
}
}

process.exit(1);
process.exit(1);
}

/**
* Validate already-parsed flag overrides against a Zod schema.
* Use this when you have flag overrides in dot-notation form (e.g., { 'model.temperature': 0.7 })
* and want to validate them against a schema before running evals.
*
* @param overrides - Flag overrides in dot-notation form
* @param flagSchema - Zod schema to validate against
*/
export function validateFlagOverrides(overrides: FlagOverrides, flagSchema?: unknown): void {
const result = collectFlagValidationErrors(overrides, flagSchema);
if (!result.success) {
printFlagValidationErrorsAndExit(result.errors);
}
}

Expand Down
132 changes: 95 additions & 37 deletions packages/ai/test/cli/utils/parse-flag-overrides.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
import {
collectFlagValidationErrors,
extractOverrides,
validateFlagOverrides,
} from '../../../src/cli/utils/parse-flag-overrides';
Expand Down Expand Up @@ -364,20 +365,7 @@ describe('extractOverrides', () => {
});
});

describe('validateFlagOverrides', () => {
let mockConsoleError: any;
let mockProcessExit: any;

beforeEach(() => {
mockConsoleError = vi.spyOn(console, 'error').mockImplementation(() => {});
mockProcessExit = vi.spyOn(process, 'exit').mockImplementation((() => {}) as any);
});

afterEach(() => {
mockConsoleError.mockRestore();
mockProcessExit.mockRestore();
});

describe('collectFlagValidationErrors', () => {
const testSchema = z.object({
model: z.object({
temperature: z.number().min(0).max(2).default(0.7),
Expand All @@ -386,66 +374,136 @@ describe('validateFlagOverrides', () => {
debug: z.boolean().default(false),
});

it('passes validation for valid flags', () => {
it('returns success for valid flags', () => {
const overrides = {
'model.temperature': 0.9,
'model.name': 'gpt-4',
debug: true,
};

validateFlagOverrides(overrides, testSchema);
const result = collectFlagValidationErrors(overrides, testSchema);

expect(mockProcessExit).not.toHaveBeenCalled();
expect(mockConsoleError).not.toHaveBeenCalled();
expect(result.success).toBe(true);
expect(result.errors).toEqual([]);
});

it('passes for empty overrides', () => {
validateFlagOverrides({}, testSchema);
it('returns success for empty overrides', () => {
const result = collectFlagValidationErrors({}, testSchema);

expect(mockProcessExit).not.toHaveBeenCalled();
expect(result.success).toBe(true);
expect(result.errors).toEqual([]);
});

it('returns success when no schema provided', () => {
const result = collectFlagValidationErrors({ 'any.path': 'value' });

expect(result.success).toBe(true);
expect(result.errors).toEqual([]);
});

it('errors on invalid flag path', () => {
it('returns error for invalid flag path', () => {
const overrides = {
'model.unknown': 'value',
};

validateFlagOverrides(overrides, testSchema);
const result = collectFlagValidationErrors(overrides, testSchema);

expect(mockConsoleError).toHaveBeenCalledWith('❌ Invalid CLI flags:');
expect(mockConsoleError).toHaveBeenCalledWith(
expect.stringContaining("flag 'model.unknown': Invalid flag path"),
);
expect(mockProcessExit).toHaveBeenCalledWith(1);
expect(result.success).toBe(false);
expect(result.errors).toEqual([{ type: 'invalid_path', path: 'model.unknown' }]);
});

it('errors on completely unknown namespace', () => {
it('returns error for completely unknown namespace', () => {
const overrides = {
'unknown.flag': 'value',
};

validateFlagOverrides(overrides, testSchema);
const result = collectFlagValidationErrors(overrides, testSchema);

expect(mockConsoleError).toHaveBeenCalledWith('❌ Invalid CLI flags:');
expect(mockProcessExit).toHaveBeenCalledWith(1);
expect(result.success).toBe(false);
expect(result.errors).toEqual([{ type: 'invalid_path', path: 'unknown.flag' }]);
});

it('returns all invalid path errors, not just the first', () => {
const overrides = {
'model.unknown': 'value',
'another.invalid': 123,
'third.bad.path': true,
};

const result = collectFlagValidationErrors(overrides, testSchema);

expect(result.success).toBe(false);
expect(result.errors).toHaveLength(3);
expect(result.errors).toContainEqual({ type: 'invalid_path', path: 'model.unknown' });
expect(result.errors).toContainEqual({ type: 'invalid_path', path: 'another.invalid' });
expect(result.errors).toContainEqual({ type: 'invalid_path', path: 'third.bad.path' });
});

it('errors on invalid value type', () => {
it('returns error for invalid value type', () => {
const overrides = {
'model.temperature': 'not-a-number',
};

validateFlagOverrides(overrides, testSchema);
const result = collectFlagValidationErrors(overrides, testSchema);

expect(mockConsoleError).toHaveBeenCalledWith('❌ Invalid CLI flags:');
expect(mockProcessExit).toHaveBeenCalledWith(1);
expect(result.success).toBe(false);
expect(result.errors).toHaveLength(1);
expect(result.errors[0].type).toBe('invalid_value');
});

it('errors on value out of range', () => {
it('returns error for value out of range', () => {
const overrides = {
'model.temperature': 5, // max is 2
};

const result = collectFlagValidationErrors(overrides, testSchema);

expect(result.success).toBe(false);
expect(result.errors).toHaveLength(1);
expect(result.errors[0].type).toBe('invalid_value');
});
});

describe('validateFlagOverrides', () => {
let mockConsoleError: any;
let mockProcessExit: any;

beforeEach(() => {
mockConsoleError = vi.spyOn(console, 'error').mockImplementation(() => {});
mockProcessExit = vi.spyOn(process, 'exit').mockImplementation((() => {}) as any);
});

afterEach(() => {
mockConsoleError.mockRestore();
mockProcessExit.mockRestore();
});

const testSchema = z.object({
model: z.object({
temperature: z.number().min(0).max(2).default(0.7),
name: z.string().default('gpt-4o'),
}),
debug: z.boolean().default(false),
});

it('does not exit for valid flags', () => {
const overrides = {
'model.temperature': 0.9,
'model.name': 'gpt-4',
debug: true,
};

validateFlagOverrides(overrides, testSchema);

expect(mockProcessExit).not.toHaveBeenCalled();
expect(mockConsoleError).not.toHaveBeenCalled();
});

it('prints errors and exits on invalid flags', () => {
const overrides = {
'model.unknown': 'value',
};

validateFlagOverrides(overrides, testSchema);

expect(mockConsoleError).toHaveBeenCalledWith('❌ Invalid CLI flags:');
Expand Down
Loading