From 4b33986a972799eaa84f0f74682e95debd6c36fe Mon Sep 17 00:00:00 2001 From: Yannick Daveluy Date: Tue, 18 Mar 2025 20:44:59 +0100 Subject: [PATCH 1/3] add an option to generate optional properties in ast --- packages/langium-cli/langium-config-schema.json | 4 ++++ packages/langium-cli/src/generator/ast-generator.ts | 3 ++- packages/langium-cli/src/package-types.ts | 2 ++ .../src/grammar/type-system/type-collector/types.ts | 9 +++++---- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/packages/langium-cli/langium-config-schema.json b/packages/langium-cli/langium-config-schema.json index d61962d4b..0efb83751 100644 --- a/packages/langium-cli/langium-config-schema.json +++ b/packages/langium-cli/langium-config-schema.json @@ -171,6 +171,10 @@ "langiumInternal": { "description": "A flag to determine whether langium uses itself to bootstrap", "type": "boolean" + }, + "optionalProperties": { + "description": "Use optional properties in generated ast (except for boolean and arrays properties)", + "type": "boolean" } }, "required": [ diff --git a/packages/langium-cli/src/generator/ast-generator.ts b/packages/langium-cli/src/generator/ast-generator.ts index 32b3251ec..c893da6d1 100644 --- a/packages/langium-cli/src/generator/ast-generator.ts +++ b/packages/langium-cli/src/generator/ast-generator.ts @@ -15,6 +15,7 @@ import { collectKeywords, collectTerminalRegexps } from './langium-util.js'; export function generateAst(services: LangiumCoreServices, grammars: Grammar[], config: LangiumConfig): string { const astTypes = collectAst(grammars, services.shared.workspace.LangiumDocuments); const importFrom = config.langiumInternal ? `../../syntax-tree${config.importExtension}` : 'langium'; + /* eslint-disable @typescript-eslint/indent */ const fileNode = expandToNode` ${generatedHeader} @@ -25,7 +26,7 @@ export function generateAst(services: LangiumCoreServices, grammars: Grammar[], ${generateTerminalConstants(grammars, config)} ${joinToNode(astTypes.unions, union => union.toAstTypesString(isAstType(union.type)), { appendNewLineIfNotEmpty: true })} - ${joinToNode(astTypes.interfaces, iFace => iFace.toAstTypesString(true), { appendNewLineIfNotEmpty: true })} + ${joinToNode(astTypes.interfaces, iFace => iFace.toAstTypesString(true, config.optionalProperties), { appendNewLineIfNotEmpty: true })} ${ astTypes.unions = astTypes.unions.filter(e => isAstType(e.type)), generateAstReflection(config, astTypes) diff --git a/packages/langium-cli/src/package-types.ts b/packages/langium-cli/src/package-types.ts index c0cc53f1b..59f5c3d81 100644 --- a/packages/langium-cli/src/package-types.ts +++ b/packages/langium-cli/src/package-types.ts @@ -30,6 +30,8 @@ export interface LangiumConfig { chevrotainParserConfig?: IParserConfig, /** The following option is meant to be used only by Langium itself */ langiumInternal?: boolean + /** Use optional properties in generated ast (except for boolean and arrays properties) */ + optionalProperties?: boolean } export interface LangiumLanguageConfig { diff --git a/packages/langium/src/grammar/type-system/type-collector/types.ts b/packages/langium/src/grammar/type-system/type-collector/types.ts index 273154be7..8c16a6058 100644 --- a/packages/langium/src/grammar/type-system/type-collector/types.ts +++ b/packages/langium/src/grammar/type-system/type-collector/types.ts @@ -218,7 +218,7 @@ export class InterfaceType { this.abstract = abstract; } - toAstTypesString(reflectionInfo: boolean): string { + toAstTypesString(reflectionInfo: boolean, optionalProperties?: boolean): string { const interfaceSuperTypes = this.interfaceSuperTypes.map(e => e.name); const superTypes = interfaceSuperTypes.length > 0 ? distinctAndSorted([...interfaceSuperTypes]) : ['langium.AstNode']; const interfaceNode = expandToNode` @@ -233,7 +233,7 @@ export class InterfaceType { body.append(`readonly $type: ${distinctAndSorted([...this.typeNames]).map(e => `'${e}'`).join(' | ')};`).appendNewLine(); } body.append( - pushProperties(this.properties, 'AstType') + pushProperties(this.properties, 'AstType', optionalProperties) ); }); interfaceNode.append('}').appendNewLine(); @@ -253,7 +253,7 @@ export class InterfaceType { return toString( expandToNode` interface ${name}${superTypes.length > 0 ? ` extends ${superTypes}` : ''} { - ${pushProperties(this.properties, 'DeclaredType', reservedWords)} + ${pushProperties(this.properties, 'DeclaredType', undefined, reservedWords)} } `.appendNewLine() ); @@ -408,12 +408,13 @@ function typeParenthesis(type: PropertyType, name: string): string { function pushProperties( properties: Property[], mode: 'AstType' | 'DeclaredType', + optionalProperties?: boolean, reserved = new Set() ): Generated { function propertyToString(property: Property): string { const name = mode === 'AstType' ? property.name : escapeReservedWords(property.name, reserved); - const optional = property.optional && !isMandatoryPropertyType(property.type); + const optional = !isMandatoryPropertyType(property.type) && (property.optional || optionalProperties); const propType = propertyTypeToString(property.type, mode); return `${name}${optional ? '?' : ''}: ${propType};`; } From 5e942c8cdc455db8ef58acc80cb5cc686bb8a733 Mon Sep 17 00:00:00 2001 From: Yannick Daveluy Date: Wed, 19 Mar 2025 18:15:32 +0100 Subject: [PATCH 2/3] add test case --- .../test/generator/ast-generator.test.ts | 54 +++++++++++++++++-- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/packages/langium-cli/test/generator/ast-generator.test.ts b/packages/langium-cli/test/generator/ast-generator.test.ts index 09769651f..cf20534e2 100644 --- a/packages/langium-cli/test/generator/ast-generator.test.ts +++ b/packages/langium-cli/test/generator/ast-generator.test.ts @@ -186,6 +186,51 @@ describe('Ast generator', () => { } `); + testGeneratedInterface('optionalProperties option shall generate properties as optional in interfaces', ` + grammar TestGrammar + + interface A { + str: string + strOpt?: string + strArray: string[] + bool: boolean + boolOpt?: boolean + boolArray: bool[] + ref: @A + refOpt?: @A + refArray: @A[] + ctn: A + ctnOpt?: A + ctnArray: A[] + } + + hidden terminal WS: /\\s+/; + terminal ID: /[_a-zA-Z][\\w_]*/; + `, expandToString` + export interface A extends langium.AstNode { + readonly $container: A; + readonly $type: 'A'; + bool: boolean; + boolArray: Array; + boolOpt: boolean; + ctn?: A; + ctnArray: Array; + ctnOpt?: A; + ref?: langium.Reference; + refArray: Array>; + refOpt?: langium.Reference; + str?: string; + strArray: Array; + strOpt?: string; + } + + export const A = 'A'; + + export function isA(item: unknown): item is A { + return reflection.isInstance(item, A); + } + `, true); + testGeneratedAst('should generate checker functions for datatype rules of type number', ` grammar TestGrammar @@ -504,8 +549,8 @@ async function testTerminalConstants(grammar: string, expected: string) { expect(relevantPart).toEqual(expectedPart); } -function testGeneratedInterface(name: string, grammar: string, expected: string): void { - testGenerated(name, grammar, expected, 'export interface', 'export type testAstType'); +function testGeneratedInterface(name: string, grammar: string, expected: string, optionalProperties = false): void { + testGenerated(name, grammar, expected, 'export interface', 'export type testAstType', 0, optionalProperties); } function testGeneratedAst(name: string, grammar: string, expected: string): void { @@ -519,13 +564,14 @@ function testTypeMetaData(name: string, grammar: string, expected: string): void function testReferenceType(name: string, grammar: string, expected: string): void { testGenerated(name, grammar, expected, 'getReferenceType', 'getTypeMetaData'); } -function testGenerated(name: string, grammar: string, expected: string, start: string, end: string, startCount = 0): void { +function testGenerated(name: string, grammar: string, expected: string, start: string, end: string, startCount = 0, optionalProperties = false): void { test(name, async () => { const result = (await parse(grammar)).parseResult; const config: LangiumConfig = { [RelativePath]: './', projectName: 'test', - languages: [] + languages: [], + optionalProperties: optionalProperties }; const expectedPart = normalizeEOL(expected).trim(); const typesFileContent = generateAst(services.grammar, [result.value], config); From 76b576e2e0e994578367e83315b96f561ec56a06 Mon Sep 17 00:00:00 2001 From: Yannick Daveluy Date: Thu, 27 Mar 2025 18:01:34 +0100 Subject: [PATCH 3/3] generate a partial ast --- .../langium-cli/langium-config-schema.json | 4 +- packages/langium-cli/src/generate.ts | 9 ++-- .../src/generator/ast-generator.ts | 42 ++++++++++++++- packages/langium-cli/src/package-types.ts | 4 +- .../test/generator/ast-generator.test.ts | 54 ++----------------- .../type-system/type-collector/types.ts | 29 ++++++---- 6 files changed, 72 insertions(+), 70 deletions(-) diff --git a/packages/langium-cli/langium-config-schema.json b/packages/langium-cli/langium-config-schema.json index 0efb83751..083a6dd49 100644 --- a/packages/langium-cli/langium-config-schema.json +++ b/packages/langium-cli/langium-config-schema.json @@ -172,8 +172,8 @@ "description": "A flag to determine whether langium uses itself to bootstrap", "type": "boolean" }, - "optionalProperties": { - "description": "Use optional properties in generated ast (except for boolean and arrays properties)", + "generatePartialAst": { + "description": "Generate a partial AST with optional properties (usefull to access the parsed AST with an incomplete document)", "type": "boolean" } }, diff --git a/packages/langium-cli/src/generate.ts b/packages/langium-cli/src/generate.ts index be6888006..329c885f8 100644 --- a/packages/langium-cli/src/generate.ts +++ b/packages/langium-cli/src/generate.ts @@ -11,7 +11,7 @@ import { loadConfig } from './package.js'; import { AstUtils, GrammarAST } from 'langium'; import { createLangiumGrammarServices, resolveImport, resolveImportUri, resolveTransitiveImports } from 'langium/grammar'; import { NodeFileSystem } from 'langium/node'; -import { generateAst } from './generator/ast-generator.js'; +import { generateAst, generateAstPartial } from './generator/ast-generator.js'; import { serializeGrammar } from './generator/grammar-serializer.js'; import { generateModule } from './generator/module-generator.js'; import { generateBnf } from './generator/bnf-generator.js'; @@ -327,7 +327,7 @@ export async function runGenerator(config: LangiumConfig, options: GenerateOptio const output = path.resolve(relPath, config.out ?? 'src/generated'); log('log', options, `Writing generated files to ${chalk.white.bold(output)}`); - if (await rmdirWithFail(output, ['ast.ts', 'grammar.ts', 'module.ts'], options)) { + if (await rmdirWithFail(output, ['ast.ts', 'ast-partial.ts', 'grammar.ts', 'module.ts'], options)) { return buildResult(false); } if (await mkdirWithFail(output, options)) { @@ -336,7 +336,10 @@ export async function runGenerator(config: LangiumConfig, options: GenerateOptio const genAst = generateAst(grammarServices, embeddedGrammars, config); await writeWithFail(path.resolve(updateLangiumInternalAstPath(output, config), 'ast.ts'), genAst, options); - + if(config.generatePartialAst) { + const genAstPartial = generateAstPartial(grammarServices, embeddedGrammars, config); + await writeWithFail(path.resolve(updateLangiumInternalAstPath(output, config), 'ast-partial.ts'), genAstPartial, options); + } const serializedGrammar = serializeGrammar(grammarServices, embeddedGrammars, config); await writeWithFail(path.resolve(output, 'grammar.ts'), serializedGrammar, options); diff --git a/packages/langium-cli/src/generator/ast-generator.ts b/packages/langium-cli/src/generator/ast-generator.ts index c893da6d1..6628c804b 100644 --- a/packages/langium-cli/src/generator/ast-generator.ts +++ b/packages/langium-cli/src/generator/ast-generator.ts @@ -12,7 +12,7 @@ import { collectAst, collectTypeHierarchy, findReferenceTypes, isAstType, mergeT import { generatedHeader } from './node-util.js'; import { collectKeywords, collectTerminalRegexps } from './langium-util.js'; -export function generateAst(services: LangiumCoreServices, grammars: Grammar[], config: LangiumConfig): string { +export function generateAst(services: LangiumCoreServices, grammars: Grammar[], config: LangiumConfig,): string { const astTypes = collectAst(grammars, services.shared.workspace.LangiumDocuments); const importFrom = config.langiumInternal ? `../../syntax-tree${config.importExtension}` : 'langium'; @@ -26,7 +26,7 @@ export function generateAst(services: LangiumCoreServices, grammars: Grammar[], ${generateTerminalConstants(grammars, config)} ${joinToNode(astTypes.unions, union => union.toAstTypesString(isAstType(union.type)), { appendNewLineIfNotEmpty: true })} - ${joinToNode(astTypes.interfaces, iFace => iFace.toAstTypesString(true, config.optionalProperties), { appendNewLineIfNotEmpty: true })} + ${joinToNode(astTypes.interfaces, iFace => iFace.toAstTypesString(true, false), { appendNewLineIfNotEmpty: true })} ${ astTypes.unions = astTypes.unions.filter(e => isAstType(e.type)), generateAstReflection(config, astTypes) @@ -35,6 +35,30 @@ export function generateAst(services: LangiumCoreServices, grammars: Grammar[], return toString(fileNode); /* eslint-enable @typescript-eslint/indent */ } +export function generateAstPartial(services: LangiumCoreServices, grammars: Grammar[], config: LangiumConfig,): string { + const astTypes = collectAst(grammars, services.shared.workspace.LangiumDocuments); + const importFrom = config.langiumInternal ? `../../syntax-tree${config.importExtension}` : 'langium'; + + /* eslint-disable @typescript-eslint/indent */ + const fileNode = expandToNode` + ${generatedHeader} + + /* eslint-disable */ + import * as langium from '${importFrom}'; + import * as ast from './ast.js'; + + ${generateTerminalConstantsPartial(grammars, config)} + + ${joinToNode(astTypes.unions, union => union.toAstTypesString(isAstType(union.type), true), { appendNewLineIfNotEmpty: true })} + ${joinToNode(astTypes.interfaces, iFace => iFace.toAstTypesString(true, true), { appendNewLineIfNotEmpty: true })} + ${ + astTypes.unions = astTypes.unions.filter(e => isAstType(e.type)), + generateAstReflectionPartial(config, astTypes) + } + `; + return toString(fileNode); + /* eslint-enable @typescript-eslint/indent */ +} function generateAstReflection(config: LangiumConfig, astTypes: AstTypes): Generated { const typeNames: string[] = astTypes.interfaces.map(t => t.name) @@ -69,6 +93,14 @@ function generateAstReflection(config: LangiumConfig, astTypes: AstTypes): Gener `.appendNewLine(); } +function generateAstReflectionPartial(config: LangiumConfig, _astTypes: AstTypes): Generated { + + return expandToNode` + + export type { ${config.projectName}AstType, ${config.projectName}AstReflection } from './ast.js'; + export const reflection = ast.reflection; + `.appendNewLine(); +} function buildTypeMetaDataMethod(astTypes: AstTypes): Generated { /* eslint-disable @typescript-eslint/indent */ return expandToNode` @@ -253,3 +285,9 @@ function generateTerminalConstants(grammars: Grammar[], config: LangiumConfig): export type ${config.projectName}TokenNames = ${config.projectName}TerminalNames | ${config.projectName}KeywordNames; `.appendNewLine(); } + +function generateTerminalConstantsPartial(grammars: Grammar[], config: LangiumConfig): Generated { + return expandToNode` + export { ${config.projectName}Terminals, type ${config.projectName}TerminalNames, type ${config.projectName}KeywordNames, type ${config.projectName}TokenNames } from './ast.js'; + `.appendNewLine(); +} diff --git a/packages/langium-cli/src/package-types.ts b/packages/langium-cli/src/package-types.ts index 59f5c3d81..402b7a98a 100644 --- a/packages/langium-cli/src/package-types.ts +++ b/packages/langium-cli/src/package-types.ts @@ -30,8 +30,8 @@ export interface LangiumConfig { chevrotainParserConfig?: IParserConfig, /** The following option is meant to be used only by Langium itself */ langiumInternal?: boolean - /** Use optional properties in generated ast (except for boolean and arrays properties) */ - optionalProperties?: boolean + /** Generate a partial AST with optional properties (usefull to access the parsed AST with an incomplete document) */ + generatePartialAst?: boolean } export interface LangiumLanguageConfig { diff --git a/packages/langium-cli/test/generator/ast-generator.test.ts b/packages/langium-cli/test/generator/ast-generator.test.ts index cf20534e2..09769651f 100644 --- a/packages/langium-cli/test/generator/ast-generator.test.ts +++ b/packages/langium-cli/test/generator/ast-generator.test.ts @@ -186,51 +186,6 @@ describe('Ast generator', () => { } `); - testGeneratedInterface('optionalProperties option shall generate properties as optional in interfaces', ` - grammar TestGrammar - - interface A { - str: string - strOpt?: string - strArray: string[] - bool: boolean - boolOpt?: boolean - boolArray: bool[] - ref: @A - refOpt?: @A - refArray: @A[] - ctn: A - ctnOpt?: A - ctnArray: A[] - } - - hidden terminal WS: /\\s+/; - terminal ID: /[_a-zA-Z][\\w_]*/; - `, expandToString` - export interface A extends langium.AstNode { - readonly $container: A; - readonly $type: 'A'; - bool: boolean; - boolArray: Array; - boolOpt: boolean; - ctn?: A; - ctnArray: Array; - ctnOpt?: A; - ref?: langium.Reference; - refArray: Array>; - refOpt?: langium.Reference; - str?: string; - strArray: Array; - strOpt?: string; - } - - export const A = 'A'; - - export function isA(item: unknown): item is A { - return reflection.isInstance(item, A); - } - `, true); - testGeneratedAst('should generate checker functions for datatype rules of type number', ` grammar TestGrammar @@ -549,8 +504,8 @@ async function testTerminalConstants(grammar: string, expected: string) { expect(relevantPart).toEqual(expectedPart); } -function testGeneratedInterface(name: string, grammar: string, expected: string, optionalProperties = false): void { - testGenerated(name, grammar, expected, 'export interface', 'export type testAstType', 0, optionalProperties); +function testGeneratedInterface(name: string, grammar: string, expected: string): void { + testGenerated(name, grammar, expected, 'export interface', 'export type testAstType'); } function testGeneratedAst(name: string, grammar: string, expected: string): void { @@ -564,14 +519,13 @@ function testTypeMetaData(name: string, grammar: string, expected: string): void function testReferenceType(name: string, grammar: string, expected: string): void { testGenerated(name, grammar, expected, 'getReferenceType', 'getTypeMetaData'); } -function testGenerated(name: string, grammar: string, expected: string, start: string, end: string, startCount = 0, optionalProperties = false): void { +function testGenerated(name: string, grammar: string, expected: string, start: string, end: string, startCount = 0): void { test(name, async () => { const result = (await parse(grammar)).parseResult; const config: LangiumConfig = { [RelativePath]: './', projectName: 'test', - languages: [], - optionalProperties: optionalProperties + languages: [] }; const expectedPart = normalizeEOL(expected).trim(); const typesFileContent = generateAst(services.grammar, [result.value], config); diff --git a/packages/langium/src/grammar/type-system/type-collector/types.ts b/packages/langium/src/grammar/type-system/type-collector/types.ts index 8c16a6058..f75ea3485 100644 --- a/packages/langium/src/grammar/type-system/type-collector/types.ts +++ b/packages/langium/src/grammar/type-system/type-collector/types.ts @@ -121,14 +121,14 @@ export class UnionType { this.dataType = options?.dataType; } - toAstTypesString(reflectionInfo: boolean): string { + toAstTypesString(reflectionInfo: boolean, isPartial?: boolean): string { const unionNode = expandToNode` export type ${this.name} = ${propertyTypeToString(this.type, 'AstType')}; `.appendNewLine(); if (reflectionInfo) { unionNode.appendNewLine() - .append(addReflectionInfo(this.name)); + .append(addReflectionInfo(this.name, isPartial)); } if (this.dataType) { @@ -218,7 +218,7 @@ export class InterfaceType { this.abstract = abstract; } - toAstTypesString(reflectionInfo: boolean, optionalProperties?: boolean): string { + toAstTypesString(reflectionInfo: boolean, isPartial?: boolean): string { const interfaceSuperTypes = this.interfaceSuperTypes.map(e => e.name); const superTypes = interfaceSuperTypes.length > 0 ? distinctAndSorted([...interfaceSuperTypes]) : ['langium.AstNode']; const interfaceNode = expandToNode` @@ -233,7 +233,7 @@ export class InterfaceType { body.append(`readonly $type: ${distinctAndSorted([...this.typeNames]).map(e => `'${e}'`).join(' | ')};`).appendNewLine(); } body.append( - pushProperties(this.properties, 'AstType', optionalProperties) + pushProperties(this.properties, 'AstType', isPartial) ); }); interfaceNode.append('}').appendNewLine(); @@ -241,7 +241,7 @@ export class InterfaceType { if (reflectionInfo) { interfaceNode .appendNewLine() - .append(addReflectionInfo(this.name)); + .append(addReflectionInfo(this.name, isPartial)); } return toString(interfaceNode); @@ -253,7 +253,7 @@ export class InterfaceType { return toString( expandToNode` interface ${name}${superTypes.length > 0 ? ` extends ${superTypes}` : ''} { - ${pushProperties(this.properties, 'DeclaredType', undefined, reservedWords)} + ${pushProperties(this.properties, 'DeclaredType', false, reservedWords)} } `.appendNewLine() ); @@ -408,13 +408,13 @@ function typeParenthesis(type: PropertyType, name: string): string { function pushProperties( properties: Property[], mode: 'AstType' | 'DeclaredType', - optionalProperties?: boolean, + isPartial?: boolean, reserved = new Set() ): Generated { function propertyToString(property: Property): string { const name = mode === 'AstType' ? property.name : escapeReservedWords(property.name, reserved); - const optional = !isMandatoryPropertyType(property.type) && (property.optional || optionalProperties); + const optional = !isMandatoryPropertyType(property.type) && property.defaultValue === undefined && (property.optional || isPartial); const propType = propertyTypeToString(property.type, mode); return `${name}${optional ? '?' : ''}: ${propType};`; } @@ -441,14 +441,21 @@ export function isMandatoryPropertyType(propertyType: PropertyType): boolean { } } -function addReflectionInfo(name: string): Generated { - return expandToNode` +function addReflectionInfo(name: string, isPartial?: boolean): Generated { + return (isPartial ? + expandToNode` + export const ${name} = ast.${name}; + + export function is${name}(item: unknown): item is ${name} { + return reflection.isInstance(item, ${name}); + } + `: expandToNode` export const ${name} = '${name}'; export function is${name}(item: unknown): item is ${name} { return reflection.isInstance(item, ${name}); } - `.appendNewLine(); + `).appendNewLine(); } function addDataTypeReflectionInfo(union: UnionType): Generated {