|
| 1 | +import * as fs from 'fs'; |
| 2 | +import * as path from 'path'; |
| 3 | +import { join } from 'path'; |
| 4 | + |
| 5 | +export interface TestFailure { |
| 6 | + testName: string; |
| 7 | + type: 'empty-scip-index' | 'missing-output' | 'content-mismatch' | 'orphaned-output'; |
| 8 | + message: string; |
| 9 | +} |
| 10 | + |
| 11 | +export interface ValidationResults { |
| 12 | + passed: string[]; |
| 13 | + failed: TestFailure[]; |
| 14 | + skipped: string[]; |
| 15 | +} |
| 16 | + |
| 17 | +export interface TestRunnerOptions { |
| 18 | + snapshotRoot: string; |
| 19 | + filterTests?: string; |
| 20 | + failFast: boolean; |
| 21 | + quiet: boolean; |
| 22 | + mode: 'check' | 'update'; |
| 23 | +} |
| 24 | + |
| 25 | +export interface SingleTestOptions { |
| 26 | + check: boolean; |
| 27 | + quiet: boolean; |
| 28 | +} |
| 29 | + |
| 30 | +function validateFilterTestNames(inputDirectory: string, filterTestNames: string[]): void { |
| 31 | + const availableTests = fs.readdirSync(inputDirectory); |
| 32 | + const missingTests = filterTestNames.filter(name => !availableTests.includes(name)); |
| 33 | + |
| 34 | + if (missingTests.length > 0) { |
| 35 | + console.error(`ERROR: The following test names were not found: ${missingTests.join(', ')}. Available tests: ${availableTests.join(', ')}`); |
| 36 | + process.exit(1); |
| 37 | + } |
| 38 | +} |
| 39 | + |
| 40 | +function handleOrphanedOutputs(inputDirectory: string, outputDirectory: string, mode: 'check' | 'update'): TestFailure[] { |
| 41 | + if (!fs.existsSync(outputDirectory)) { |
| 42 | + return []; |
| 43 | + } |
| 44 | + |
| 45 | + const inputTests = new Set(fs.readdirSync(inputDirectory)); |
| 46 | + const outputTests = fs.readdirSync(outputDirectory); |
| 47 | + const orphanedOutputs: TestFailure[] = []; |
| 48 | + |
| 49 | + for (const outputTest of outputTests) { |
| 50 | + if (!inputTests.has(outputTest)) { |
| 51 | + if (mode === 'update') { |
| 52 | + // Delete orphaned output directory in update mode |
| 53 | + const orphanedPath = path.join(outputDirectory, outputTest); |
| 54 | + fs.rmSync(orphanedPath, { recursive: true, force: true }); |
| 55 | + console.log(`Delete output folder with no corresponding input folder: ${outputTest}`); |
| 56 | + } else { |
| 57 | + // Report as failure in check mode |
| 58 | + orphanedOutputs.push({ |
| 59 | + testName: outputTest, |
| 60 | + type: 'orphaned-output', |
| 61 | + message: `Output folder exists but no corresponding input folder found` |
| 62 | + }); |
| 63 | + } |
| 64 | + } |
| 65 | + } |
| 66 | + |
| 67 | + return orphanedOutputs; |
| 68 | +} |
| 69 | + |
| 70 | +function reportResults(results: ValidationResults): void { |
| 71 | + const totalTests = results.passed.length + results.failed.length + results.skipped.length; |
| 72 | + console.assert(totalTests > 0, 'No tests found'); |
| 73 | + |
| 74 | + for (const failure of results.failed) { |
| 75 | + console.error(`FAIL [${failure.testName}]: ${failure.message}`); |
| 76 | + } |
| 77 | + |
| 78 | + let summaryStr = `\n${results.passed.length}/${totalTests} tests passed, ${results.failed.length} failed`; |
| 79 | + if (results.skipped.length > 0) { |
| 80 | + summaryStr += `, ${results.skipped.length} skipped`; |
| 81 | + } |
| 82 | + console.log(summaryStr); |
| 83 | + |
| 84 | + if (results.failed.length > 0) { |
| 85 | + process.exit(1); |
| 86 | + } |
| 87 | +} |
| 88 | + |
| 89 | +export class TestRunner { |
| 90 | + constructor(private options: TestRunnerOptions) {} |
| 91 | + |
| 92 | + runTests( |
| 93 | + runSingleTest: (testName: string, inputDir: string, outputDir: string) => ValidationResults |
| 94 | + ): void { |
| 95 | + const inputDirectory = path.resolve(join(this.options.snapshotRoot, 'input')); |
| 96 | + const outputDirectory = path.resolve(join(this.options.snapshotRoot, 'output')); |
| 97 | + |
| 98 | + const results: ValidationResults = { |
| 99 | + passed: [], |
| 100 | + failed: [], |
| 101 | + skipped: [] |
| 102 | + }; |
| 103 | + |
| 104 | + // Pre-execution validation: determine test directories to process |
| 105 | + let snapshotDirectories = fs.readdirSync(inputDirectory); |
| 106 | + let isFilterMode = false; |
| 107 | + |
| 108 | + if (this.options.filterTests) { |
| 109 | + // Filter to specific tests |
| 110 | + const filterTestNames = this.options.filterTests.split(',').map(name => name.trim()); |
| 111 | + isFilterMode = true; |
| 112 | + |
| 113 | + // Validate filter test names exist |
| 114 | + validateFilterTestNames(inputDirectory, filterTestNames); |
| 115 | + |
| 116 | + snapshotDirectories = snapshotDirectories.filter(dir => filterTestNames.includes(dir)); |
| 117 | + } |
| 118 | + |
| 119 | + // In non-filtering mode, handle orphaned outputs |
| 120 | + if (!isFilterMode) { |
| 121 | + const orphanedOutputs = handleOrphanedOutputs(inputDirectory, outputDirectory, this.options.mode); |
| 122 | + |
| 123 | + // In check mode, orphaned outputs are reported as failures |
| 124 | + if (orphanedOutputs.length > 0) { |
| 125 | + results.failed.push(...orphanedOutputs); |
| 126 | + |
| 127 | + if (this.options.failFast) { |
| 128 | + reportResults(results); |
| 129 | + return; |
| 130 | + } |
| 131 | + } |
| 132 | + } |
| 133 | + |
| 134 | + for (let i = 0; i < snapshotDirectories.length; i++) { |
| 135 | + const testName = snapshotDirectories[i]; |
| 136 | + if (!this.options.quiet) { |
| 137 | + console.log(`Processing test: ${testName}`); |
| 138 | + } |
| 139 | + |
| 140 | + try { |
| 141 | + const testResults = runSingleTest( |
| 142 | + testName, |
| 143 | + inputDirectory, |
| 144 | + outputDirectory, |
| 145 | + ); |
| 146 | + |
| 147 | + // Merge results |
| 148 | + results.passed.push(...testResults.passed); |
| 149 | + results.failed.push(...testResults.failed); |
| 150 | + |
| 151 | + // Check for fail-fast condition |
| 152 | + if (this.options.failFast && testResults.failed.length > 0) { |
| 153 | + // Track remaining tests as skipped |
| 154 | + for (let j = i + 1; j < snapshotDirectories.length; j++) { |
| 155 | + results.skipped.push(snapshotDirectories[j]); |
| 156 | + } |
| 157 | + reportResults(results); |
| 158 | + return; |
| 159 | + } |
| 160 | + } catch (error) { |
| 161 | + results.failed.push({ |
| 162 | + testName, |
| 163 | + type: 'empty-scip-index', |
| 164 | + message: `Test runner failed: ${error}` |
| 165 | + }); |
| 166 | + |
| 167 | + if (this.options.failFast) { |
| 168 | + for (let j = i + 1; j < snapshotDirectories.length; j++) { |
| 169 | + results.skipped.push(snapshotDirectories[j]); |
| 170 | + } |
| 171 | + reportResults(results); |
| 172 | + return; |
| 173 | + } |
| 174 | + } |
| 175 | + } |
| 176 | + |
| 177 | + reportResults(results); |
| 178 | + } |
| 179 | +} |
0 commit comments