diff --git a/packages/ai/src/cli/utils/parse-flag-overrides.ts b/packages/ai/src/cli/utils/parse-flag-overrides.ts
index e6617dd..17b267d 100644
--- a/packages/ai/src/cli/utils/parse-flag-overrides.ts
+++ b/packages/ai/src/cli/utils/parse-flag-overrides.ts
@@ -1,4 +1,4 @@
-import { type ZodObject, type z } from 'zod';
+import { type ZodError, type ZodObject, type z } from 'zod';
 import { formatZodErrors, generateFlagExamples } from './format-zod-errors.js';
 import { readFileSync } from 'node:fs';
 import { resolve } from 'node:path';
@@ -70,47 +70,94 @@ export function extractAndValidateFlagOverrides<S extends z.ZodObject<any>>(
   return { cleanedArgv, overrides: overrides as any };
 }
 
+export type FlagValidationError =
+  | { type: 'invalid_path'; path: string }
+  | { type: 'invalid_value'; zodError: ZodError };
+
+export interface FlagValidationResult {
+  success: boolean;
+  errors: FlagValidationError[];
+}
+
 /**
  * Validate already-parsed flag overrides against a Zod schema.
- * Use this when you have flag overrides in dot-notation form (e.g., { 'model.temperature': 0.7 })
- * and want to validate them against a schema before running evals.
+ * Returns validation result without side effects (no console output, no process.exit).
  *
- * @param overrides - Flag overrides in dot-notation form
+ * @param overrides - Flag overrides in dot-notation form (e.g., { 'model.temperature': 0.7 })
  * @param flagSchema - Zod schema to validate against
+ * @returns Validation result with any errors found
  */
-export function validateFlagOverrides(overrides: FlagOverrides, flagSchema?: unknown): void {
+export function collectFlagValidationErrors(
+  overrides: FlagOverrides,
+  flagSchema?: unknown,
+): FlagValidationResult {
   // No schema provided = no validation, any flags allowed
   if (!flagSchema || Object.keys(overrides).length === 0) {
-    return;
+    return { success: true, errors: [] };
   }
 
   const schema = flagSchema as any;
+  const errors: FlagValidationError[] = [];
 
   // First pass: check all paths exist in schema
   for (const dotPath of Object.keys(overrides)) {
     const segments = parsePath(dotPath);
     if (!isValidPath(schema, segments)) {
-      console.error('❌ Invalid CLI flags:');
-      console.error(`  • flag '${dotPath}': Invalid flag path`);
-      process.exit(1);
+      errors.push({ type: 'invalid_path', path: dotPath });
     }
   }
 
+  // If there are invalid paths, don't proceed to value validation
+  if (errors.length > 0) {
+    return { success: false, errors };
+  }
+
   // Second pass: validate values using nested object approach
   const nestedObject = dotNotationToNested(overrides);
   const result = schema.strict().partial().safeParse(nestedObject);
 
   if (!result.success) {
-    console.error('❌ Invalid CLI flags:');
-    console.error(formatZodErrors(result.error));
+    errors.push({ type: 'invalid_value', zodError: result.error });
+  }
+
+  return { success: errors.length === 0, errors };
+}
+
+/**
+ * Print flag validation errors to console and exit.
+ */
+export function printFlagValidationErrorsAndExit(errors: FlagValidationError[]): never {
+  console.error('❌ Invalid CLI flags:');
+
+  for (const error of errors) {
+    if (error.type === 'invalid_path') {
+      console.error(`  • flag '${error.path}': Invalid flag path`);
+    } else {
+      console.error(formatZodErrors(error.zodError));
 
-    const examples = generateFlagExamples(result.error);
-    if (examples.length > 0) {
-      console.error('\n💡 Valid examples:');
-      examples.forEach((example) => console.error(`  ${example}`));
+      const examples = generateFlagExamples(error.zodError);
+      if (examples.length > 0) {
+        console.error('\n💡 Valid examples:');
+        examples.forEach((example) => console.error(`  ${example}`));
+      }
     }
+  }
 
-    process.exit(1);
+  process.exit(1);
+}
+
+/**
+ * Validate already-parsed flag overrides against a Zod schema.
+ * Use this when you have flag overrides in dot-notation form (e.g., { 'model.temperature': 0.7 })
+ * and want to validate them against a schema before running evals.
+ *
+ * @param overrides - Flag overrides in dot-notation form
+ * @param flagSchema - Zod schema to validate against
+ */
+export function validateFlagOverrides(overrides: FlagOverrides, flagSchema?: unknown): void {
+  const result = collectFlagValidationErrors(overrides, flagSchema);
+  if (!result.success) {
+    printFlagValidationErrorsAndExit(result.errors);
   }
 }
 
diff --git a/packages/ai/test/cli/utils/parse-flag-overrides.test.ts b/packages/ai/test/cli/utils/parse-flag-overrides.test.ts
index af75552..50b0f3b 100644
--- a/packages/ai/test/cli/utils/parse-flag-overrides.test.ts
+++ b/packages/ai/test/cli/utils/parse-flag-overrides.test.ts
@@ -1,5 +1,6 @@
 import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
 import {
+  collectFlagValidationErrors,
   extractOverrides,
   validateFlagOverrides,
 } from '../../../src/cli/utils/parse-flag-overrides';
@@ -364,20 +365,7 @@ describe('extractOverrides', () => {
   });
 });
 
-describe('validateFlagOverrides', () => {
-  let mockConsoleError: any;
-  let mockProcessExit: any;
-
-  beforeEach(() => {
-    mockConsoleError = vi.spyOn(console, 'error').mockImplementation(() => {});
-    mockProcessExit = vi.spyOn(process, 'exit').mockImplementation((() => {}) as any);
-  });
-
-  afterEach(() => {
-    mockConsoleError.mockRestore();
-    mockProcessExit.mockRestore();
-  });
-
+describe('collectFlagValidationErrors', () => {
   const testSchema = z.object({
     model: z.object({
       temperature: z.number().min(0).max(2).default(0.7),
@@ -386,66 +374,136 @@ describe('validateFlagOverrides', () => {
     debug: z.boolean().default(false),
   });
 
-  it('passes validation for valid flags', () => {
+  it('returns success for valid flags', () => {
     const overrides = {
       'model.temperature': 0.9,
       'model.name': 'gpt-4',
       debug: true,
     };
 
-    validateFlagOverrides(overrides, testSchema);
+    const result = collectFlagValidationErrors(overrides, testSchema);
 
-    expect(mockProcessExit).not.toHaveBeenCalled();
-    expect(mockConsoleError).not.toHaveBeenCalled();
+    expect(result.success).toBe(true);
+    expect(result.errors).toEqual([]);
   });
 
-  it('passes for empty overrides', () => {
-    validateFlagOverrides({}, testSchema);
+  it('returns success for empty overrides', () => {
+    const result = collectFlagValidationErrors({}, testSchema);
 
-    expect(mockProcessExit).not.toHaveBeenCalled();
+    expect(result.success).toBe(true);
+    expect(result.errors).toEqual([]);
+  });
+
+  it('returns success when no schema provided', () => {
+    const result = collectFlagValidationErrors({ 'any.path': 'value' });
+
+    expect(result.success).toBe(true);
+    expect(result.errors).toEqual([]);
   });
 
-  it('errors on invalid flag path', () => {
+  it('returns error for invalid flag path', () => {
     const overrides = {
       'model.unknown': 'value',
     };
 
-    validateFlagOverrides(overrides, testSchema);
+    const result = collectFlagValidationErrors(overrides, testSchema);
 
-    expect(mockConsoleError).toHaveBeenCalledWith('❌ Invalid CLI flags:');
-    expect(mockConsoleError).toHaveBeenCalledWith(
-      expect.stringContaining("flag 'model.unknown': Invalid flag path"),
-    );
-    expect(mockProcessExit).toHaveBeenCalledWith(1);
+    expect(result.success).toBe(false);
+    expect(result.errors).toEqual([{ type: 'invalid_path', path: 'model.unknown' }]);
   });
 
-  it('errors on completely unknown namespace', () => {
+  it('returns error for completely unknown namespace', () => {
     const overrides = {
       'unknown.flag': 'value',
     };
 
-    validateFlagOverrides(overrides, testSchema);
+    const result = collectFlagValidationErrors(overrides, testSchema);
 
-    expect(mockConsoleError).toHaveBeenCalledWith('❌ Invalid CLI flags:');
-    expect(mockProcessExit).toHaveBeenCalledWith(1);
+    expect(result.success).toBe(false);
+    expect(result.errors).toEqual([{ type: 'invalid_path', path: 'unknown.flag' }]);
+  });
+
+  it('returns all invalid path errors, not just the first', () => {
+    const overrides = {
+      'model.unknown': 'value',
+      'another.invalid': 123,
+      'third.bad.path': true,
+    };
+
+    const result = collectFlagValidationErrors(overrides, testSchema);
+
+    expect(result.success).toBe(false);
+    expect(result.errors).toHaveLength(3);
+    expect(result.errors).toContainEqual({ type: 'invalid_path', path: 'model.unknown' });
+    expect(result.errors).toContainEqual({ type: 'invalid_path', path: 'another.invalid' });
+    expect(result.errors).toContainEqual({ type: 'invalid_path', path: 'third.bad.path' });
   });
 
-  it('errors on invalid value type', () => {
+  it('returns error for invalid value type', () => {
     const overrides = {
       'model.temperature': 'not-a-number',
     };
 
-    validateFlagOverrides(overrides, testSchema);
+    const result = collectFlagValidationErrors(overrides, testSchema);
 
-    expect(mockConsoleError).toHaveBeenCalledWith('❌ Invalid CLI flags:');
-    expect(mockProcessExit).toHaveBeenCalledWith(1);
+    expect(result.success).toBe(false);
+    expect(result.errors).toHaveLength(1);
+    expect(result.errors[0].type).toBe('invalid_value');
   });
 
-  it('errors on value out of range', () => {
+  it('returns error for value out of range', () => {
     const overrides = {
       'model.temperature': 5, // max is 2
     };
 
+    const result = collectFlagValidationErrors(overrides, testSchema);
+
+    expect(result.success).toBe(false);
+    expect(result.errors).toHaveLength(1);
+    expect(result.errors[0].type).toBe('invalid_value');
+  });
+});
+
+describe('validateFlagOverrides', () => {
+  let mockConsoleError: any;
+  let mockProcessExit: any;
+
+  beforeEach(() => {
+    mockConsoleError = vi.spyOn(console, 'error').mockImplementation(() => {});
+    mockProcessExit = vi.spyOn(process, 'exit').mockImplementation((() => {}) as any);
+  });
+
+  afterEach(() => {
+    mockConsoleError.mockRestore();
+    mockProcessExit.mockRestore();
+  });
+
+  const testSchema = z.object({
+    model: z.object({
+      temperature: z.number().min(0).max(2).default(0.7),
+      name: z.string().default('gpt-4o'),
+    }),
+    debug: z.boolean().default(false),
+  });
+
+  it('does not exit for valid flags', () => {
+    const overrides = {
+      'model.temperature': 0.9,
+      'model.name': 'gpt-4',
+      debug: true,
+    };
+
+    validateFlagOverrides(overrides, testSchema);
+
+    expect(mockProcessExit).not.toHaveBeenCalled();
+    expect(mockConsoleError).not.toHaveBeenCalled();
+  });
+
+  it('prints errors and exits on invalid flags', () => {
+    const overrides = {
+      'model.unknown': 'value',
+    };
+
     validateFlagOverrides(overrides, testSchema);
 
     expect(mockConsoleError).toHaveBeenCalledWith('❌ Invalid CLI flags:');