Skip to content

Commit e866a2d

Browse files
JordanBoltonMNJordanBoltonMN
authored
Move several methods from TextUtils to IdentifierUtils (#390)
* initial commit * updating package * npm audit fix * npm audit fix --force * initial commit * initial commit of identifierUtils
1 parent 6e72f15 commit e866a2d

File tree

11 files changed

+251
-228
lines changed

11 files changed

+251
-228
lines changed

‎package-lock.json‎

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@microsoft/powerquery-parser",
3-
"version": "0.15.11",
3+
"version": "0.16.0",
44
"description": "A parser for the Power Query/M formula language.",
55
"author": "Microsoft",
66
"license": "MIT",

‎src/powerquery-parser/language/identifierUtils.ts‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT license.
3+
4+
import { Assert, Pattern, StringUtils } from "../common";
5+
6+
export enum IdentifierKind {
7+
Generalized = "Generalized",
8+
Invalid = "Invalid",
9+
Quote = "Quote",
10+
QuoteRequired = "QuoteRequired",
11+
Regular = "Regular",
12+
}
13+
14+
// Assuming the text is a quoted identifier, finds the quotes that enclose the identifier.
15+
// Otherwise returns undefined.
16+
export function findQuotedIdentifierQuotes(text: string, index: number): StringUtils.FoundQuotes | undefined {
17+
if (text[index] !== "#") {
18+
return undefined;
19+
}
20+
21+
return StringUtils.findQuotes(text, index + 1);
22+
}
23+
24+
// Determines what kind of identifier the text is.
25+
// It's possible that the text is a partially completed identifier,
26+
// which is why we have the `allowTrailingPeriod` parameter.
27+
export function getIdentifierKind(text: string, allowTrailingPeriod: boolean): IdentifierKind {
28+
if (isRegularIdentifier(text, allowTrailingPeriod)) {
29+
return IdentifierKind.Regular;
30+
} else if (isQuotedIdentifier(text)) {
31+
return isRegularIdentifier(text.slice(2, -1), false) ? IdentifierKind.Quote : IdentifierKind.QuoteRequired;
32+
} else if (isGeneralizedIdentifier(text)) {
33+
return IdentifierKind.Generalized;
34+
} else {
35+
return IdentifierKind.Invalid;
36+
}
37+
}
38+
39+
// Assuming the text is an identifier, returns the length of the identifier.
40+
export function getIdentifierLength(text: string, index: number, allowTrailingPeriod: boolean): number | undefined {
41+
const startingIndex: number = index;
42+
const textLength: number = text.length;
43+
44+
let state: IdentifierRegexpState = IdentifierRegexpState.Start;
45+
let matchLength: number | undefined;
46+
47+
while (state !== IdentifierRegexpState.Done) {
48+
if (index === textLength) {
49+
return index - startingIndex;
50+
}
51+
52+
switch (state) {
53+
case IdentifierRegexpState.Start:
54+
matchLength = StringUtils.regexMatchLength(Pattern.IdentifierStartCharacter, text, index);
55+
56+
if (matchLength === undefined) {
57+
state = IdentifierRegexpState.Done;
58+
} else {
59+
state = IdentifierRegexpState.RegularIdentifier;
60+
index += matchLength;
61+
}
62+
63+
break;
64+
65+
case IdentifierRegexpState.RegularIdentifier:
66+
// Don't consider `..` or `...` part of an identifier.
67+
if (allowTrailingPeriod && text[index] === "." && text[index + 1] !== ".") {
68+
index += 1;
69+
}
70+
71+
matchLength = StringUtils.regexMatchLength(Pattern.IdentifierPartCharacters, text, index);
72+
73+
if (matchLength === undefined) {
74+
state = IdentifierRegexpState.Done;
75+
} else {
76+
index += matchLength;
77+
78+
// Don't consider `..` or `...` part of an identifier.
79+
if (allowTrailingPeriod && text[index] === "." && text[index + 1] !== ".") {
80+
index += 1;
81+
}
82+
}
83+
84+
break;
85+
86+
default:
87+
throw Assert.isNever(state);
88+
}
89+
}
90+
91+
return index !== startingIndex ? index - startingIndex : undefined;
92+
}
93+
94+
// Assuming the text is a generalized identifier, returns the length of the identifier.
95+
export function getGeneralizedIdentifierLength(text: string, index: number): number | undefined {
96+
const startingIndex: number = index;
97+
const textLength: number = text.length;
98+
99+
let continueMatching: boolean = true;
100+
101+
while (continueMatching) {
102+
const currentChr: string = text[index];
103+
104+
if (currentChr === " ") {
105+
index += 1;
106+
} else if (currentChr === ".") {
107+
if (text[index - 1] === ".") {
108+
continueMatching = false;
109+
break;
110+
}
111+
112+
index += 1;
113+
} else {
114+
const matchLength: number | undefined = StringUtils.regexMatchLength(
115+
Pattern.IdentifierPartCharacters,
116+
text,
117+
index,
118+
);
119+
120+
if (matchLength === undefined) {
121+
continueMatching = false;
122+
break;
123+
}
124+
125+
index += matchLength;
126+
}
127+
128+
if (index >= textLength) {
129+
continueMatching = false;
130+
}
131+
}
132+
133+
return index !== startingIndex ? index - startingIndex : undefined;
134+
}
135+
136+
export function isGeneralizedIdentifier(text: string): boolean {
137+
return getGeneralizedIdentifierLength(text, 0) === text.length;
138+
}
139+
140+
export function isRegularIdentifier(text: string, allowTrailingPeriod: boolean): boolean {
141+
return getIdentifierLength(text, 0, allowTrailingPeriod) === text.length;
142+
}
143+
144+
export function isQuotedIdentifier(text: string): boolean {
145+
return findQuotedIdentifierQuotes(text, 0) !== undefined;
146+
}
147+
148+
// Removes the quotes from a quoted identifier if possible.
149+
export function normalizeIdentifier(text: string): string {
150+
if (isQuotedIdentifier(text)) {
151+
const stripped: string = text.slice(2, -1);
152+
153+
return isRegularIdentifier(stripped, false) ? stripped : text;
154+
} else {
155+
return text;
156+
}
157+
}
158+
159+
const enum IdentifierRegexpState {
160+
Done = "Done",
161+
RegularIdentifier = "RegularIdentifier",
162+
Start = "Start",
163+
}

‎src/powerquery-parser/language/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
// Licensed under the MIT license.
33

44
import * as Comment from "./comment";
5+
export * as IdentifierUtils from "./identifierUtils";
56
export * as TextUtils from "./textUtils";
67
import * as Token from "./token";
78

‎src/powerquery-parser/language/textUtils.ts‎

Lines changed: 0 additions & 153 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,6 @@
11
// Copyright (c) Microsoft Corporation.
22
// Licensed under the MIT license.
33

4-
import { Assert, Pattern, StringUtils } from "../common";
5-
6-
export enum IdentifierKind {
7-
Generalized = "Generalized",
8-
Invalid = "Invalid",
9-
Quote = "Quote",
10-
QuoteRequired = "QuoteRequired",
11-
Regular = "Regular",
12-
}
13-
144
export function escape(text: string): string {
155
let result: string = text;
166

@@ -21,143 +11,6 @@ export function escape(text: string): string {
2111
return result;
2212
}
2313

24-
export function identifierKind(text: string, allowTrailingPeriod: boolean): IdentifierKind {
25-
if (isRegularIdentifier(text, allowTrailingPeriod)) {
26-
return IdentifierKind.Regular;
27-
} else if (isQuotedIdentifier(text)) {
28-
return isRegularIdentifier(text.slice(2, -1), false) ? IdentifierKind.Quote : IdentifierKind.QuoteRequired;
29-
} else if (isGeneralizedIdentifier(text)) {
30-
return IdentifierKind.Generalized;
31-
} else {
32-
return IdentifierKind.Invalid;
33-
}
34-
}
35-
36-
export function isGeneralizedIdentifier(text: string): boolean {
37-
return generalizedIdentifierLength(text, 0) === text.length;
38-
}
39-
40-
export function isRegularIdentifier(text: string, allowTrailingPeriod: boolean): boolean {
41-
return identifierLength(text, 0, allowTrailingPeriod) === text.length;
42-
}
43-
44-
export function isQuotedIdentifier(text: string): boolean {
45-
return quotedIdentifier(text, 0) !== undefined;
46-
}
47-
48-
export function identifierLength(text: string, index: number, allowTrailingPeriod: boolean): number | undefined {
49-
const startingIndex: number = index;
50-
const textLength: number = text.length;
51-
52-
let state: IdentifierRegexpState = IdentifierRegexpState.Start;
53-
let matchLength: number | undefined;
54-
55-
while (state !== IdentifierRegexpState.Done) {
56-
if (index === textLength) {
57-
return index - startingIndex;
58-
}
59-
60-
switch (state) {
61-
case IdentifierRegexpState.Start:
62-
matchLength = StringUtils.regexMatchLength(Pattern.IdentifierStartCharacter, text, index);
63-
64-
if (matchLength === undefined) {
65-
state = IdentifierRegexpState.Done;
66-
} else {
67-
state = IdentifierRegexpState.RegularIdentifier;
68-
index += matchLength;
69-
}
70-
71-
break;
72-
73-
case IdentifierRegexpState.RegularIdentifier:
74-
// Don't consider `..` or `...` part of an identifier.
75-
if (allowTrailingPeriod && text[index] === "." && text[index + 1] !== ".") {
76-
index += 1;
77-
}
78-
79-
matchLength = StringUtils.regexMatchLength(Pattern.IdentifierPartCharacters, text, index);
80-
81-
if (matchLength === undefined) {
82-
state = IdentifierRegexpState.Done;
83-
} else {
84-
index += matchLength;
85-
86-
// Don't consider `..` or `...` part of an identifier.
87-
if (allowTrailingPeriod && text[index] === "." && text[index + 1] !== ".") {
88-
index += 1;
89-
}
90-
}
91-
92-
break;
93-
94-
default:
95-
throw Assert.isNever(state);
96-
}
97-
}
98-
99-
return index !== startingIndex ? index - startingIndex : undefined;
100-
}
101-
102-
export function generalizedIdentifierLength(text: string, index: number): number | undefined {
103-
const startingIndex: number = index;
104-
const textLength: number = text.length;
105-
106-
let continueMatching: boolean = true;
107-
108-
while (continueMatching) {
109-
const currentChr: string = text[index];
110-
111-
if (currentChr === " ") {
112-
index += 1;
113-
} else if (currentChr === ".") {
114-
if (text[index - 1] === ".") {
115-
continueMatching = false;
116-
break;
117-
}
118-
119-
index += 1;
120-
} else {
121-
const matchLength: number | undefined = StringUtils.regexMatchLength(
122-
Pattern.IdentifierPartCharacters,
123-
text,
124-
index,
125-
);
126-
127-
if (matchLength === undefined) {
128-
continueMatching = false;
129-
break;
130-
}
131-
132-
index += matchLength;
133-
}
134-
135-
if (index >= textLength) {
136-
continueMatching = false;
137-
}
138-
}
139-
140-
return index !== startingIndex ? index - startingIndex : undefined;
141-
}
142-
143-
export function quotedIdentifier(text: string, index: number): StringUtils.FoundQuotes | undefined {
144-
if (text[index] !== "#") {
145-
return undefined;
146-
}
147-
148-
return StringUtils.findQuotes(text, index + 1);
149-
}
150-
151-
export function normalizeIdentifier(text: string): string {
152-
if (isQuotedIdentifier(text)) {
153-
const stripped: string = text.slice(2, -1);
154-
155-
return isRegularIdentifier(stripped, false) ? stripped : text;
156-
} else {
157-
return text;
158-
}
159-
}
160-
16114
export function unescape(text: string): string {
16215
let result: string = text;
16316

@@ -168,12 +21,6 @@ export function unescape(text: string): string {
16821
return result;
16922
}
17023

171-
const enum IdentifierRegexpState {
172-
Done = "Done",
173-
RegularIdentifier = "RegularIdentifier",
174-
Start = "Start",
175-
}
176-
17724
const EscapedWhitespaceRegexp: ReadonlyArray<[RegExp, string]> = [
17825
[/#\(cr,lf\)/gm, "\r\n"],
17926
[/#\(cr\)/gm, "\r"],

‎src/powerquery-parser/lexer/lexer.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ import {
1414
ResultUtils,
1515
StringUtils,
1616
} from "../common";
17-
import { Keyword, TextUtils, Token } from "../language";
17+
import { IdentifierUtils, Keyword, Token } from "../language";
1818
import { LexError } from ".";
1919
import { LexSettings } from "./lexSettings";
2020

@@ -1137,7 +1137,7 @@ function indexOfRegexEnd(pattern: RegExp, text: string, positionStart: number):
11371137
}
11381138

11391139
function indexOfIdentifierEnd(text: string, positionStart: number): number | undefined {
1140-
const length: number | undefined = TextUtils.identifierLength(text, positionStart, true);
1140+
const length: number | undefined = IdentifierUtils.getIdentifierLength(text, positionStart, true);
11411141

11421142
return length !== undefined ? positionStart + length : undefined;
11431143
}

0 commit comments

Comments
 (0)