Skip to content

Commit 5523524

Browse files
fix: Make tests more robust & fix path bug
1 parent a29134b commit 5523524

File tree

10 files changed

+482
-56
lines changed

10 files changed

+482
-56
lines changed

.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@ yarn-debug.log*
66
yarn-error.log*
77
lerna-debug.log*
88

9+
# Editor files
10+
.idea/
11+
912
# Diagnostic reports (https://nodejs.org/api/report.html)
1013
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
1114

packages/pyright-internal/src/analyzer/program.ts

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -331,7 +331,15 @@ export class Program {
331331

332332
addTrackedFile(filePath: string, isThirdPartyImport = false, isInPyTypedPackage = false): SourceFile {
333333
let sourceFileInfo = this.getSourceFileInfo(filePath);
334-
const importName = this._getImportNameForFile(filePath);
334+
let importName = this._getImportNameForFile(filePath);
335+
// HACK(scip-python): When adding tracked files for imports, we end up passing
336+
// normalized paths as the argument. However, _getImportNameForFile seemingly
337+
// needs a non-normalized path, which cannot be recovered directly from a
338+
// normalized path. However, in practice, the non-normalized path seems to
339+
// be stored on the sourceFileInfo, so attempt to use that instead.
340+
if (importName === '' && sourceFileInfo) {
341+
importName = this._getImportNameForFile(sourceFileInfo.sourceFile.getFilePath());
342+
}
335343

336344
if (sourceFileInfo) {
337345
// The module name may have changed based on updates to the

packages/pyright-scip/CONTRIBUTING.md

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,9 +73,32 @@ node ./index.js <other args>
7373
npm run check-snapshots
7474
```
7575

76+
#### Filter specific snapshot tests
77+
78+
Use the `--filter-tests` flag to run only specific snapshot tests:
79+
```bash
80+
# Using npm scripts (note the -- to pass arguments)
81+
npm run check-snapshots -- --filter-tests test1,test2,test3
82+
```
83+
84+
Available snapshot tests can be found in `snapshots/input/`.
85+
7686
Using a different Python version other than the one specified
7787
in `.tool-versions` may also lead to errors.
7888

89+
## Making changes to Pyright internals
90+
91+
When modifying code in the `pyright-internal` package:
92+
93+
1. Keep changes minimal: Every change introduces a risk of
94+
merge conflicts. Adding doc comments is fine, but avoid
95+
changing functionality if possible. Instead of changing
96+
access modifiers, prefer copying small functions into
97+
scip-pyright logic.
98+
2. Use a `NOTE(scip-python):` prefix when adding comments to
99+
make it clearer which comments were added by upstream
100+
maintainers vs us.
101+
79102
## Publishing releases
80103

81104
1. Change the version in `packages/pyright-scip/package.json`

packages/pyright-scip/src/config.ts

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import {
1717
} from 'pyright-internal/common/pathUtils';
1818
import { PyrightFileSystem } from 'pyright-internal/pyrightFileSystem';
1919
import { ScipConfig } from './lib';
20+
import { sendStatus } from './status';
2021

2122
const configFileNames = ['scip-pyrightconfig.json', 'pyrightconfig.json'];
2223
const pyprojectTomlName = 'pyproject.toml';
@@ -100,7 +101,7 @@ export class ScipPyrightConfig {
100101
if (configFilePath) {
101102
projectRoot = getDirectoryPath(configFilePath);
102103
} else {
103-
this._console.log(`No configuration file found.`);
104+
sendStatus(`No configuration file found.`);
104105
configFilePath = undefined;
105106
}
106107
}
@@ -115,9 +116,9 @@ export class ScipPyrightConfig {
115116

116117
if (pyprojectFilePath) {
117118
projectRoot = getDirectoryPath(pyprojectFilePath);
118-
this._console.log(`pyproject.toml file found at ${projectRoot}.`);
119+
sendStatus(`pyproject.toml file found at ${projectRoot}.`);
119120
} else {
120-
this._console.log(`No pyproject.toml file found.`);
121+
sendStatus(`No pyproject.toml file found.`);
121122
}
122123
}
123124

@@ -180,7 +181,7 @@ export class ScipPyrightConfig {
180181
this._console.info(`Loading configuration file at ${configFilePath}`);
181182
configJsonObj = this._parseJsonConfigFile(configFilePath);
182183
} else if (pyprojectFilePath) {
183-
this._console.info(`Loading pyproject.toml file at ${pyprojectFilePath}`);
184+
sendStatus(`Loading pyproject.toml file at ${pyprojectFilePath}`);
184185
configJsonObj = this._parsePyprojectTomlFile(pyprojectFilePath);
185186
}
186187

packages/pyright-scip/src/indexer.ts

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -117,15 +117,15 @@ export class Indexer {
117117
this.projectFiles = targetFiles;
118118
}
119119

120-
console.log('Total Project Files', this.projectFiles.size);
120+
sendStatus(`Total Project Files ${this.projectFiles.size}`);
121121

122122
const host = new FullAccessHost(fs);
123123
this.importResolver = new ImportResolver(fs, this.pyrightConfig, host);
124124

125125
this.program = new Program(this.importResolver, this.pyrightConfig);
126-
// Normalize paths to ensure consistency with other code paths.
127-
const normalizedProjectFiles = [...this.projectFiles].map((path: string) => normalizePathCase(fs, path));
128-
this.program.setTrackedFiles(normalizedProjectFiles);
126+
// setTrackedFiles internally handles path normalization, so we don't normalize
127+
// paths here.
128+
this.program.setTrackedFiles([...this.projectFiles]);
129129

130130
if (scipConfig.projectNamespace) {
131131
setProjectNamespace(scipConfig.projectName, this.scipConfig.projectNamespace!);
@@ -194,7 +194,9 @@ export class Indexer {
194194
let projectSourceFiles: SourceFile[] = [];
195195
withStatus('Index workspace and track project files', () => {
196196
this.program.indexWorkspace((filepath: string) => {
197-
// Filter out filepaths not part of this project
197+
// Do not index files outside the project because SCIP doesn't support it.
198+
//
199+
// Both filepath and this.scipConfig.projectRoot are NOT normalized.
198200
if (filepath.indexOf(this.scipConfig.projectRoot) != 0) {
199201
return;
200202
}
@@ -204,6 +206,7 @@ export class Indexer {
204206

205207
let requestsImport = sourceFile.getImports();
206208
requestsImport.forEach((entry) =>
209+
// entry.resolvedPaths are all normalized.
207210
entry.resolvedPaths.forEach((value) => {
208211
this.program.addTrackedFile(value, true, false);
209212
})

packages/pyright-scip/src/lib.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -310,10 +310,10 @@ export function writeSnapshot(outputPath: string, obtained: string): void {
310310
fs.writeFileSync(outputPath, obtained, { flag: 'w' });
311311
}
312312

313-
export function diffSnapshot(outputPath: string, obtained: string): void {
313+
export function diffSnapshot(outputPath: string, obtained: string): 'equal' | 'different' {
314314
let existing = fs.readFileSync(outputPath, { encoding: 'utf8' });
315315
if (obtained === existing) {
316-
return;
316+
return 'equal';
317317
}
318318

319319
console.error(
@@ -326,7 +326,7 @@ export function diffSnapshot(outputPath: string, obtained: string): void {
326326
'(what the current code produces). Run the command "npm run update-snapshots" to accept the new behavior.'
327327
)
328328
);
329-
exit(1);
329+
return 'different';
330330
}
331331

332332
function occurrencesByLine(a: scip.Occurrence, b: scip.Occurrence): number {

packages/pyright-scip/src/main-impl.ts

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,8 @@ import { sendStatus, setQuiet, setShowProgressRateLimit } from './status';
1010
import { Indexer } from './indexer';
1111
import { exit } from 'process';
1212

13-
function indexAction(options: IndexOptions): void {
13+
14+
export function indexAction(options: IndexOptions): void {
1415
setQuiet(options.quiet);
1516
if (options.showProgressRateLimit !== undefined) {
1617
setShowProgressRateLimit(options.showProgressRateLimit);
@@ -91,6 +92,8 @@ function snapshotAction(snapshotRoot: string, options: SnapshotOptions): void {
9192

9293
const scipIndexPath = path.join(projectRoot, options.output);
9394
const scipIndex = scip.Index.deserializeBinary(fs.readFileSync(scipIndexPath));
95+
96+
let hasDiff = false;
9497
for (const doc of scipIndex.documents) {
9598
if (doc.relative_path.startsWith('..')) {
9699
continue;
@@ -103,11 +106,15 @@ function snapshotAction(snapshotRoot: string, options: SnapshotOptions): void {
103106
const outputPath = path.resolve(outputDirectory, snapshotDir, relativeToInputDirectory);
104107

105108
if (options.check) {
106-
diffSnapshot(outputPath, obtained);
109+
const diffResult = diffSnapshot(outputPath, obtained);
110+
hasDiff = hasDiff || diffResult === 'different';
107111
} else {
108112
writeSnapshot(outputPath, obtained);
109113
}
110114
}
115+
if (hasDiff) {
116+
exit(1);
117+
}
111118
}
112119
}
113120

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
import * as fs from 'fs';
2+
import * as path from 'path';
3+
import { join } from 'path';
4+
5+
export interface TestFailure {
6+
testName: string;
7+
type: 'empty-scip-index' | 'missing-output' | 'content-mismatch' | 'orphaned-output' | 'caught-exception';
8+
message: string;
9+
}
10+
11+
export interface ValidationResults {
12+
passed: string[];
13+
failed: TestFailure[];
14+
skipped: string[];
15+
}
16+
17+
export interface TestRunnerOptions {
18+
snapshotRoot: string;
19+
filterTests?: string;
20+
failFast: boolean;
21+
quiet: boolean;
22+
mode: 'check' | 'update';
23+
}
24+
25+
export interface SingleTestOptions {
26+
check: boolean;
27+
quiet: boolean;
28+
}
29+
30+
function validateFilterTestNames(inputDirectory: string, filterTestNames: string[]): void {
31+
const availableTests = fs.readdirSync(inputDirectory);
32+
const missingTests = filterTestNames.filter(name => !availableTests.includes(name));
33+
34+
if (missingTests.length > 0) {
35+
console.error(`ERROR: The following test names were not found: ${missingTests.join(', ')}. Available tests: ${availableTests.join(', ')}`);
36+
process.exit(1);
37+
}
38+
}
39+
40+
function handleOrphanedOutputs(inputTests: Set<string>, outputDirectory: string, mode: 'check' | 'update'): TestFailure[] {
41+
if (!fs.existsSync(outputDirectory)) {
42+
return [];
43+
}
44+
45+
const outputTests = fs.readdirSync(outputDirectory);
46+
const orphanedOutputs: TestFailure[] = [];
47+
48+
for (const outputTest of outputTests) {
49+
if (inputTests.has(outputTest)) {
50+
continue;
51+
}
52+
if (mode === 'update') {
53+
const orphanedPath = path.join(outputDirectory, outputTest);
54+
fs.rmSync(orphanedPath, { recursive: true, force: true });
55+
console.log(`Delete output folder with no corresponding input folder: ${outputTest}`);
56+
continue;
57+
}
58+
orphanedOutputs.push({
59+
testName: outputTest,
60+
type: 'orphaned-output',
61+
message: `Output folder exists but no corresponding input folder found`
62+
});
63+
}
64+
65+
return orphanedOutputs;
66+
}
67+
68+
function reportResults(results: ValidationResults): void {
69+
const totalTests = results.passed.length + results.failed.length + results.skipped.length;
70+
console.assert(totalTests > 0, 'No tests found');
71+
72+
for (const failure of results.failed) {
73+
console.error(`FAIL [${failure.testName}]: ${failure.message}`);
74+
}
75+
76+
let summaryStr = `\n${results.passed.length}/${totalTests} tests passed, ${results.failed.length} failed`;
77+
if (results.skipped.length > 0) {
78+
summaryStr += `, ${results.skipped.length} skipped`;
79+
}
80+
console.log(summaryStr);
81+
82+
if (results.failed.length > 0) {
83+
process.exit(1);
84+
}
85+
}
86+
87+
export class TestRunner {
88+
constructor(private options: TestRunnerOptions) {}
89+
90+
runTests(
91+
runSingleTest: (testName: string, inputDir: string, outputDir: string) => ValidationResults
92+
): void {
93+
const inputDirectory = path.resolve(join(this.options.snapshotRoot, 'input'));
94+
const outputDirectory = path.resolve(join(this.options.snapshotRoot, 'output'));
95+
96+
const results: ValidationResults = {
97+
passed: [],
98+
failed: [],
99+
skipped: []
100+
};
101+
102+
let snapshotDirectories = fs.readdirSync(inputDirectory);
103+
104+
const orphanedOutputs = handleOrphanedOutputs(new Set(snapshotDirectories), outputDirectory, this.options.mode);
105+
if (orphanedOutputs.length > 0) {
106+
results.failed.push(...orphanedOutputs);
107+
if (this.options.failFast) {
108+
reportResults(results);
109+
return;
110+
}
111+
}
112+
113+
if (this.options.filterTests) {
114+
const filterTestNames = this.options.filterTests.split(',').map(name => name.trim());
115+
validateFilterTestNames(inputDirectory, filterTestNames);
116+
snapshotDirectories = snapshotDirectories.filter(dir => filterTestNames.includes(dir));
117+
if (snapshotDirectories.length === 0) {
118+
console.error(`No tests found matching filter: ${this.options.filterTests}`);
119+
process.exit(1);
120+
}
121+
}
122+
123+
for (let i = 0; i < snapshotDirectories.length; i++) {
124+
const testName = snapshotDirectories[i];
125+
if (!this.options.quiet) {
126+
console.log(`--- Running snapshot test: ${testName} ---`);
127+
}
128+
129+
let testResults: ValidationResults;
130+
try {
131+
testResults = runSingleTest(
132+
testName,
133+
inputDirectory,
134+
outputDirectory,
135+
);
136+
} catch (error) {
137+
testResults = {
138+
passed: [],
139+
failed: [{
140+
testName,
141+
type: 'caught-exception',
142+
message: `Test runner failed: ${error}`
143+
}],
144+
skipped: []
145+
};
146+
}
147+
148+
results.passed.push(...testResults.passed);
149+
results.failed.push(...testResults.failed);
150+
151+
if (this.options.failFast && testResults.failed.length > 0) {
152+
for (let j = i + 1; j < snapshotDirectories.length; j++) {
153+
results.skipped.push(snapshotDirectories[j]);
154+
}
155+
reportResults(results);
156+
return;
157+
}
158+
}
159+
160+
reportResults(results);
161+
}
162+
}

0 commit comments

Comments
 (0)