From 8a2e7ddc356d59f9277eb05895827d0e18f8223f Mon Sep 17 00:00:00 2001 From: Nathan Northcutt Date: Fri, 9 Aug 2024 18:15:23 +0200 Subject: [PATCH 01/21] merging in some regex tooling and bug fixes --- packages/type-utils/math.ts | 542 ++++++++++---------- packages/type-utils/regex.ts | 890 +++++++++++++++++++++++++++++++++ packages/type-utils/strings.ts | 247 ++++++++- 3 files changed, 1395 insertions(+), 284 deletions(-) create mode 100644 packages/type-utils/regex.ts diff --git a/packages/type-utils/math.ts b/packages/type-utils/math.ts index a6f838a..71ef999 100644 --- a/packages/type-utils/math.ts +++ b/packages/type-utils/math.ts @@ -14,14 +14,22 @@ export type Abs = /** * Increment a number */ -export type Increment = - Add extends infer I extends number ? I : never +export type Increment = Add< + N, + 1 +> extends infer I extends number + ? I + : never /** * Decrement a number */ -export type Decrement = - Subtract extends infer D extends number ? D : never +export type Decrement = Subtract< + N, + 1 +> extends infer D extends number + ? D + : never /** * Check if L > R @@ -29,14 +37,14 @@ export type Decrement = export type GT = [L] extends [R] ? false : IsNegative extends IsNegative - ? IsNegative extends true - ? SplitDecimal<`${Abs}`, `${Abs}`> extends true // if both negative, reverse GT - ? false - : true - : SplitDecimal<`${L}`, `${R}`> - : IsNegative extends true + ? IsNegative extends true + ? SplitDecimal<`${Abs}`, `${Abs}`> extends true // if both negative, reverse GT ? false : true + : SplitDecimal<`${L}`, `${R}`> + : IsNegative extends true + ? false + : true /** * Check if L >= R @@ -44,42 +52,46 @@ export type GT = [L] extends [R] export type GTE = [L] extends [R] ? true : IsNegative extends IsNegative - ? IsNegative extends true - ? SplitDecimal<`${Abs}`, `${Abs}`, true> extends true // if both negative, reverse GTE - ? false - : true - : SplitDecimal<`${L}`, `${R}`, true> - : IsNegative extends true + ? IsNegative extends true + ? SplitDecimal<`${Abs}`, `${Abs}`, true> extends true // if both negative, reverse GTE ? false : true + : SplitDecimal<`${L}`, `${R}`, true> + : IsNegative extends true + ? false + : true /** * Check if L <= R */ -export type LTE = - GT extends true ? false : true +export type LTE = GT extends true + ? false + : true /** * Check if L < R */ -export type LT = - GTE extends true ? false : true +export type LT = GTE extends true + ? false + : true /** * Perform "subtraction" of the two numbers */ -export type Subtract = - IsNegative extends IsNegative - ? IsNegative extends true - ? Subtract> // -l - (-r) = r - l - : _Subtract<`${L}`, `${R}`> extends infer N extends number - ? N - : never // l - r - : IsNegative extends true - ? Add, R> extends infer N extends number - ? _Negate // -l - (r) = - (l + r) - : never - : Add> // l - (- r) = l + r +export type Subtract< + L extends number, + R extends number +> = IsNegative extends IsNegative + ? IsNegative extends true + ? Subtract> // -l - (-r) = r - l + : _Subtract<`${L}`, `${R}`> extends infer N extends number + ? N + : never // l - r + : IsNegative extends true + ? Add, R> extends infer N extends number + ? _Negate // -l - (r) = - (l + r) + : never + : Add> // l - (- r) = l + r /** * Perform "addition" of the two numbers @@ -87,18 +99,16 @@ export type Subtract = export type Add = L extends 0 ? R : IsNegative extends IsNegative - ? IsNegative extends true - ? Add, Abs> extends infer N extends number - ? _Negate - : never // -l + -r = - (l + r) - : _Add<`${L}`, `${R}`> extends infer N extends number - ? N - : never // l + r - : IsNegative extends true - ? Subtract> // -l + r = r - l - : Subtract> // l + (-r) = l - r - -// 77 + 4 = 4 + 7 (11) => 1 carry, 7 + carry = 8 = 81 + ? IsNegative extends true + ? Add, Abs> extends infer N extends number + ? _Negate + : never // -l + -r = - (l + r) + : _Add<`${L}`, `${R}`> extends infer N extends number + ? N + : never // l + r + : IsNegative extends true + ? Subtract> // -l + r = r - l + : Subtract> // l + (-r) = l - r //////////////////////////////// // Utility Methods @@ -129,7 +139,7 @@ type Add_Digits_Arr = [ ["6", "7", "8", "9", "0", "1", "2", "3", "4", "5"], ["7", "8", "9", "0", "1", "2", "3", "4", "5", "6"], ["8", "9", "0", "1", "2", "3", "4", "5", "6", "7"], - ["9", "0", "1", "2", "3", "4", "5", "6", "7", "8"], + ["9", "0", "1", "2", "3", "4", "5", "6", "7", "8"] ] /** @@ -145,7 +155,7 @@ type Sub_Digits_Arr = [ ["6", "5", "4", "3", "2", "1", "0", "9", "8", "7"], ["7", "6", "5", "4", "3", "2", "1", "0", "9", "8"], ["8", "7", "6", "5", "4", "3", "2", "1", "0", "9"], - ["9", "8", "7", "6", "5", "4", "3", "2", "1", "0"], + ["9", "8", "7", "6", "5", "4", "3", "2", "1", "0"] ] /** @@ -161,7 +171,7 @@ type Add_CarryDigits = [ [0, 0, 0, 0, 1, 1, 1, 1, 1, 1], [0, 0, 0, 1, 1, 1, 1, 1, 1, 1], [0, 0, 1, 1, 1, 1, 1, 1, 1, 1], - [0, 1, 1, 1, 1, 1, 1, 1, 1, 1], + [0, 1, 1, 1, 1, 1, 1, 1, 1, 1] ] /** @@ -177,7 +187,7 @@ type Sub_Borrow_Digits = [ [0, 0, 0, 0, 0, 0, 0, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 1], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] ] /** @@ -188,12 +198,12 @@ type CheckLength = L extends "" ? 0 : -1 : R extends "" - ? 1 - : L extends `${SingleDigits}${infer LR extends string}` - ? R extends `${SingleDigits}${infer RR extends string}` - ? CheckLength - : 1 - : 0 + ? 1 + : L extends `${infer _}${infer LR extends string}` + ? R extends `${infer _}${infer RR extends string}` + ? CheckLength + : 1 + : 0 /** * Wrapper to check for decimal places and split into LHS, RHS comparisons @@ -201,7 +211,7 @@ type CheckLength = L extends "" type SplitDecimal< L extends string, R extends string, - GTE extends boolean = false, + GTE extends boolean = false > = L extends `${infer LN extends string}.${infer LD extends string}` ? R extends `${infer RN extends string}.${infer RD extends string}` ? CheckLHS extends true // both decimals check LHS then RHS @@ -210,33 +220,38 @@ type SplitDecimal< : CheckRHS : "false" : LN extends R // IF LHS === R, then decimal makes it bigger - ? true - : CheckLHS // IF LHS is gt then it's bigger + ? true + : CheckLHS // IF LHS is gt then it's bigger : CheckLHS // Just compare left hand side /** * Compare the LHS checking size first */ -type CheckLHS = - CheckLength extends infer V extends number - ? V extends 1 - ? true - : V extends -1 - ? false - : L extends `${infer LS extends SingleDigits}` - ? R extends `${infer RS extends SingleDigits}` - ? GTE extends true - ? _GTE - : _GT - : never - : L extends `${infer LS extends SingleDigits}${infer LR extends string}` - ? R extends `${infer RS extends SingleDigits}${infer RR extends string}` - ? _GTE extends true - ? CheckLHS - : false - : never - : never +type CheckLHS< + L extends string, + R extends string, + GTE extends boolean = false +> = CheckLength extends infer V extends number + ? V extends 1 + ? true + : V extends -1 + ? false + : L extends `${infer LS extends SingleDigits}` + ? R extends `${infer RS extends SingleDigits}` + ? GTE extends true + ? _GTE + : _GT + : never + : L extends `${infer LS extends SingleDigits}${infer LR extends string}` + ? R extends `${infer RS extends SingleDigits}${infer RR extends string}` + ? _GT extends true + ? true + : LS extends RS + ? CheckLHS + : false + : never : never + : never /** * Just keep checking the next values until one runs out or the left is smaller @@ -245,22 +260,22 @@ type CheckLHS = type CheckRHS< L extends string, R extends string, - GTE extends boolean = false, + GTE extends boolean = false > = L extends "" ? false : R extends "" - ? true - : L extends `${infer LS extends SingleDigits}${infer LR extends string}` - ? R extends `${infer RS extends SingleDigits}${infer RR extends string}` - ? _GTE extends true - ? LR extends RR - ? LR extends "" - ? true - : GTE - : CheckRHS - : false - : never - : never + ? true + : L extends `${infer LS extends SingleDigits}${infer LR extends string}` + ? R extends `${infer RS extends SingleDigits}${infer RR extends string}` + ? _GTE extends true + ? LR extends RR + ? LR extends "" + ? true + : GTE + : CheckRHS + : false + : never + : never /** * Single digit compare for L > R @@ -268,79 +283,81 @@ type CheckRHS< type _GT = L extends "0" ? false : L extends "1" - ? R extends "0" - ? true - : false - : L extends "2" - ? R extends "1" | "0" - ? true - : false - : L extends "3" - ? R extends "2" | "1" | "0" - ? true - : false - : L extends "4" - ? R extends "3" | "2" | "1" | "0" - ? true - : false - : L extends "5" - ? R extends "5" | "6" | "7" | "8" | "9" - ? false - : true - : L extends "6" - ? R extends "6" | "7" | "8" | "9" - ? false - : true - : L extends "7" - ? R extends "7" | "8" | "9" - ? false - : true - : L extends "8" - ? R extends "8" | "9" - ? false - : true - : R extends "9" - ? false - : true + ? R extends "0" + ? true + : false + : L extends "2" + ? R extends "1" | "0" + ? true + : false + : L extends "3" + ? R extends "2" | "1" | "0" + ? true + : false + : L extends "4" + ? R extends "3" | "2" | "1" | "0" + ? true + : false + : L extends "5" + ? R extends "5" | "6" | "7" | "8" | "9" + ? false + : true + : L extends "6" + ? R extends "6" | "7" | "8" | "9" + ? false + : true + : L extends "7" + ? R extends "7" | "8" | "9" + ? false + : true + : L extends "8" + ? R extends "8" | "9" + ? false + : true + : R extends "9" + ? false + : true /** * Single digit compare for L >= R */ type _GTE = L extends "0" - ? true + ? R extends "0" + ? true + : false : L extends "1" - ? R extends "0" | "1" - ? true - : false - : L extends "2" - ? R extends "0" | "1" | "2" - ? true - : false - : L extends "3" - ? R extends "0" | "1" | "2" | "3" - ? true - : false - : L extends "4" - ? R extends "0" | "1" | "2" | "3" | "4" - ? true - : false - : L extends "5" - ? R extends "6" | "7" | "8" | "9" - ? false - : true - : L extends "6" - ? R extends "7" | "8" | "9" - ? false - : true - : L extends "7" - ? R extends "8" | "9" - ? false - : true - : L extends "8" - ? R extends "9" - ? false - : true - : true + ? R extends "0" | "1" + ? true + : false + : L extends "2" + ? R extends "0" | "1" | "2" + ? true + : false + : L extends "3" + ? R extends "0" | "1" | "2" | "3" + ? true + : false + : L extends "4" + ? R extends "0" | "1" | "2" | "3" | "4" + ? true + : false + : L extends "5" + ? R extends "6" | "7" | "8" | "9" + ? false + : true + : L extends "6" + ? R extends "7" | "8" | "9" + ? false + : true + : L extends "7" + ? R extends "8" | "9" + ? false + : true + : L extends "8" + ? R extends "9" + ? false + : true + : true /** * Convert a string representation to the corresponding numeric value @@ -358,14 +375,13 @@ type TrimZero = N extends `0${infer _}` ? TrimZero<_> : N * * NOTE: This DOES NOT handle decimals yet */ -type _Add = - CheckLength extends -1 - ? LongFormAddition extends infer Res extends string - ? StringAsNumber - : never - : LongFormAddition extends infer Res extends string - ? StringAsNumber - : never +type _Add = CheckLength extends -1 + ? LongFormAddition extends infer Res extends string + ? StringAsNumber + : never + : LongFormAddition extends infer Res extends string + ? StringAsNumber + : never /** * Wrapper to ensure we call long form with the "largest" value on the left @@ -375,12 +391,12 @@ type _Add = type _Subtract = L extends R ? 0 : CheckLHS extends false - ? LongFormSubtraction extends infer Res extends string - ? _Negate> - : never - : LongFormSubtraction extends infer Res extends string - ? StringAsNumber - : never + ? LongFormSubtraction extends infer Res extends string + ? _Negate> + : never + : LongFormSubtraction extends infer Res extends string + ? StringAsNumber + : never /** * Perform long form subtraction with the left being the larger number @@ -388,30 +404,30 @@ type _Subtract = L extends R type LongFormSubtraction< L extends string, R extends string, - B extends boolean = false, + B extends boolean = false > = L extends "" ? "" : R extends "" - ? B extends true - ? SplitRightDigit extends [ - infer LD extends SingleDigits, - infer LS extends string, - ] - ? LS extends "" - ? SubtractDigit - : `${LS}${SubtractDigit}` - : never - : L - : LFSNextState extends [ - infer ND extends SingleDigits, - infer LS extends string, - infer RS extends string, - infer Chk extends boolean, - ] - ? LongFormSubtraction extends infer Arr extends string - ? `${Arr}${ND}` - : never + ? B extends true + ? SplitRightDigit extends [ + infer LD extends SingleDigits, + infer LS extends string + ] + ? LS extends "" + ? SubtractDigit + : `${LS}${SubtractDigit}` : never + : L + : LFSNextState extends [ + infer ND extends SingleDigits, + infer LS extends string, + infer RS extends string, + infer Chk extends boolean + ] + ? LongFormSubtraction extends infer Arr extends string + ? `${Arr}${ND}` + : never + : never /** * Perform addition one digit at a time, carrying results @@ -419,70 +435,76 @@ type LongFormSubtraction< type LongFormAddition< L extends string, R extends string, - C extends boolean = false, + C extends boolean = false > = L extends "" ? C extends true ? "1" : "" : R extends "" - ? C extends true - ? SplitRightDigit extends [ - infer LD extends SingleDigits, - infer LS extends string, - ] - ? GetCarry extends true - ? LS extends "" - ? `1${AddDigit}` - : AddDigit + ? C extends true + ? SplitRightDigit extends [ + infer LD extends SingleDigits, + infer LS extends string + ] + ? GetCarry extends true + ? LS extends "" + ? `1${AddDigit}` : AddDigit - : never - : L - : LFANextState extends [ - infer ND extends SingleDigits, - infer LS extends string, - infer RS extends string, - infer Chk extends boolean, - ] - ? LongFormAddition extends infer Arr extends string - ? `${Arr}${ND}` - : never + : AddDigit : never + : L + : LFANextState extends [ + infer ND extends SingleDigits, + infer LS extends string, + infer RS extends string, + infer Chk extends boolean + ] + ? LongFormAddition extends infer Arr extends string + ? `${Arr}${ND}` + : never + : never /** * Get the next state for the LongFormAddition */ -type LFANextState = - SplitRightDigit extends [ - infer LD extends SingleDigits, - infer LS extends string, - ] - ? SplitRightDigit extends [ - infer RD extends SingleDigits, - infer RS extends string, - ] - ? C extends true - ? [AddDigitWithCarry, LS, RS, Carry] - : [AddDigitWithCarry, LS, RS, Carry] - : never +type LFANextState< + L extends string, + R extends string, + C extends boolean +> = SplitRightDigit extends [ + infer LD extends SingleDigits, + infer LS extends string +] + ? SplitRightDigit extends [ + infer RD extends SingleDigits, + infer RS extends string + ] + ? C extends true + ? [AddDigitWithCarry, LS, RS, Carry] + : [AddDigitWithCarry, LS, RS, Carry] : never + : never /** * Calculate the next state for LongFormSubtraction */ -type LFSNextState = - SplitRightDigit extends [ - infer LD extends SingleDigits, - infer LS extends string, - ] - ? SplitRightDigit extends [ - infer RD extends SingleDigits, - infer RS extends string, - ] - ? B extends true - ? [SubtractDigitWithBorrow, LS, RS, Borrow] - : [SubtractDigitWithBorrow, LS, RS, Borrow] - : never +type LFSNextState< + L extends string, + R extends string, + B extends boolean +> = SplitRightDigit extends [ + infer LD extends SingleDigits, + infer LS extends string +] + ? SplitRightDigit extends [ + infer RD extends SingleDigits, + infer RS extends string + ] + ? B extends true + ? [SubtractDigitWithBorrow, LS, RS, Borrow] + : [SubtractDigitWithBorrow, LS, RS, Borrow] : never + : never /** * Extract the right digit for adding @@ -495,15 +517,15 @@ type SplitRightDigit = : never : never : N extends `${infer SD extends SingleDigits}` - ? [SD, ""] - : never + ? [SD, ""] + : never /** * Subtract a digit */ type SubtractDigit< L extends SingleDigits, - R extends SingleDigits, + R extends SingleDigits > = L extends keyof Sub_Digits_Arr ? R extends keyof Sub_Digits_Arr[L] ? Sub_Digits_Arr[L][R] @@ -516,7 +538,7 @@ type SubtractDigit< type SubtractDigitWithBorrow< L extends SingleDigits, R extends SingleDigits, - B extends "0" | "1", + B extends "0" | "1" > = B extends "1" ? SubtractDigit extends infer D extends SingleDigits ? SubtractDigit @@ -529,7 +551,7 @@ type SubtractDigitWithBorrow< type Borrow< L extends SingleDigits, R extends SingleDigits, - B extends "0" | "1", + B extends "0" | "1" > = B extends "1" ? GetBorrow, R> : GetBorrow /** @@ -537,7 +559,7 @@ type Borrow< */ type GetBorrow< L extends SingleDigits, - R extends SingleDigits, + R extends SingleDigits > = L extends keyof Sub_Borrow_Digits ? R extends keyof Sub_Borrow_Digits[L] ? Sub_Borrow_Digits[L][R] extends 1 @@ -552,15 +574,14 @@ type GetBorrow< type AddDigitWithCarry< L extends SingleDigits, R extends SingleDigits, - C extends "0" | "1", -> = - AddDigit extends infer D extends SingleDigits - ? C extends "1" - ? AddDigit extends infer D1 extends SingleDigits - ? D1 - : never - : D - : never + C extends "0" | "1" +> = AddDigit extends infer D extends SingleDigits + ? C extends "1" + ? AddDigit extends infer D1 extends SingleDigits + ? D1 + : never + : D + : never /** * Check if carry is required taking into account current digits and previous carry @@ -568,20 +589,19 @@ type AddDigitWithCarry< type Carry< L extends SingleDigits, R extends SingleDigits, - C extends "0" | "1", -> = - GetCarry extends true - ? true - : C extends "1" - ? GetCarry, "1"> - : false + C extends "0" | "1" +> = GetCarry extends true + ? true + : C extends "1" + ? GetCarry, "1"> + : false /** * Get the digit at the location of the two single digit values */ type AddDigit< L extends SingleDigits, - R extends SingleDigits, + R extends SingleDigits > = L extends keyof Add_Digits_Arr ? R extends keyof Add_Digits_Arr[L] ? Add_Digits_Arr[L][R] @@ -593,7 +613,7 @@ type AddDigit< */ type GetCarry< L extends SingleDigits, - R extends SingleDigits, + R extends SingleDigits > = L extends keyof Add_CarryDigits ? R extends keyof Add_CarryDigits[L] ? Add_CarryDigits[L][R] extends 1 diff --git a/packages/type-utils/regex.ts b/packages/type-utils/regex.ts new file mode 100644 index 0000000..86b2f4b --- /dev/null +++ b/packages/type-utils/regex.ts @@ -0,0 +1,890 @@ +import type { IgnoreAny } from "./common.js" +import type { GTE, Increment, LTE } from "./math.js" +import type { IsPartialGroup, Replace, Split, SplitGroups } from "./strings.js" + +/** + * Validate the candidate against the regex and return the candidate if there is + * a match + */ +export type ValidateRegEx< + Regex extends string, + Candidate extends string +> = IsMatch extends true + ? Candidate + : "Candidate does not match supplied Regex" + +/** + * Verify if the given candidate matches the regex + */ +export type IsMatch< + Regex extends string, + Candidate extends string +> = RegEx extends infer Tree extends RegexToken + ? RunStateMachine + : false + +/** + * Parse the regex tree from the current point down + */ +export type RegEx = IsLeaf extends true + ? CollapseRegexTokens< + ParseRegex + > extends infer Tokens extends RegexToken[] + ? Tokens extends [infer SingleToken extends RegexToken] + ? SingleToken + : RegexGroupToken + : never + : SplitAlternates extends [RegEx] // No alternates + ? CollapseRegexTokens< + TranslateGroups> + > extends infer Tokens extends RegexToken[] + ? Tokens extends [infer SingleToken extends RegexToken] + ? SingleToken + : RegexGroupToken + : never + : CollapseRegexTokens< + CollapseAlternates> + > extends infer Alternates extends RegexToken[] + ? CollapseRegexTokens< + BuildAlternates + > extends infer Tokens extends RegexToken[] + ? Tokens extends [infer SingleToken extends RegexToken] + ? SingleToken + : RegexGroupToken + : never + : never + +/** + * Verify if the number is in range after failing a match check (Min <= N <= Max) + */ +type InRange = GTE< + N, + Min +> extends true + ? Max extends -1 + ? true + : LTE extends true + ? true + : false + : false + +/** + * Take associated columns (like repetitions) + */ +type CollapseRegexTokens = Tokens extends [ + infer First extends RegexToken, + infer Second, + ...infer Rest +] + ? Second extends RegexRepeatingToken + ? Rest extends never[] + ? [RegexRepeatingToken] + : [RegexRepeatingToken, ...CollapseRegexTokens] + : Rest extends never[] + ? [First, Second] + : [First, ...CollapseRegexTokens<[Second, ...Rest]>] + : Tokens + +/** + * Parse the full regex, extracting one token at a time + */ +type ParseRegex = RegEx extends "" + ? [] + : NextToken extends [infer Token, infer Remainder extends string] + ? Remainder extends "" + ? [Token] + : ParseRegex extends infer Tokens extends unknown[] + ? [Token, ...Tokens] + : [Token] + : never + +/** + * Check to see if a regex is structural or a leaf node + */ +type IsLeaf = SplitAlternates extends [RegEx] + ? SplitCaptureGroups extends [RegEx] + ? true + : false + : false + +/** + * Fast check for SplitGroups with the correct tokens + */ +type SplitCaptureGroups = + RegEx extends `${infer _}\\(${infer _}` + ? FixCaptures< + SplitGroups< + Replace, "\\)", "$__second__$">, + "(", + ")" + > + > + : SplitGroups + +/** + * Fix mangled captures for escaped parenthesis + */ +type FixCaptures = Captures extends [ + infer Next extends string, + ...infer Rest +] + ? Rest extends [] + ? [Replace, "$__second__$", "\\)">] + : [ + Replace, "$__second__$", "\\)">, + ...FixCaptures + ] + : never + +/** + * Extract a group of tokens + */ +type ExtractGroup = IsLeaf extends true + ? CollapseRegexTokens< + ParseRegex + > extends infer Tokens extends RegexToken[] + ? ValidateGroup + : never + : [RegEx] + +type ValidateGroup = Tokens extends [ + infer SingleToken extends RegexToken +] + ? [SingleToken] + : Tokens extends [ + infer Repeating extends RegexRepeatingToken, + ...infer Rest extends RegexToken[] + ] + ? [Repeating, ...ValidateGroup] + : [RegexGroupToken] + +/** + * Build the alternates from the groups + */ +type BuildAlternates = Alternates extends [ + infer First extends RegexToken, + infer Second extends RegexToken, + ...infer Rest +] + ? BuildAlternates< + [RegexAlternateToken, ...Rest] + > extends infer Tokens extends RegexToken[] + ? Tokens + : never + : Alternates + +/** + * Collapse all of the alternates that were found + */ +type CollapseAlternates = Tokens extends [ + infer First extends string, + ...infer Rest +] + ? CollapseRegexTokens< + TranslateGroups> + > extends infer Groups extends RegexToken[] + ? Rest extends never[] + ? ValidateGroup + : CollapseAlternates extends infer Alternates extends RegexToken[] + ? Groups extends [infer Token extends RegexToken] + ? [Token, ...Alternates] + : [...ValidateGroup, ...Alternates] + : never + : never + : never + +/** + * Translate all groups + */ +type TranslateGroups = Groups extends [ + infer Next extends string, + ...infer Rest +] + ? Rest extends never[] + ? [...ExtractGroup] + : TranslateGroups extends infer Tokens extends RegexToken[] + ? [...ExtractGroup, ...Tokens] + : never + : never + +/** + * Split out all alternate groups + */ +type SplitAlternates = + RegEx extends `${infer _}\\|${infer _}` + ? FixAlternates< + Split, "|"> + > extends infer Tokens extends string[] + ? RejoinPartial + : never + : Split extends infer Tokens extends string[] + ? RejoinPartial + : never + +/** + * Fix any alternate escaping + */ +type FixAlternates = Alternates extends [ + infer Next extends string, + ...infer Rest +] + ? Rest extends never[] + ? [Replace] + : [Replace, ...FixAlternates] + : never + +/** + * Restore partial groups that might be affected by a split on '|' + */ +type RejoinPartial = Tokens extends [ + infer First extends string, + infer Second extends string, + ...infer Rest +] + ? IsPartialGroup extends true + ? RejoinPartial<[`${First}${C}${Second}`, ...Rest], C> + : [First, ...RejoinPartial<[Second, ...Rest], C>] + : Tokens + +/** + * Read the next regex token from the string + */ +type NextToken = + RegEx extends `{${infer Repeating}}${infer Unparsed}` + ? [ParseRepeating, Unparsed] + : RegEx extends `[${infer Group}]${infer Unparsed}` + ? ParseRange extends infer Token extends string + ? [RegexRangeToken, Unparsed] + : "Invalid range" + : RegEx extends `${infer Special extends REGEX_SPECIAL_TOKENS}${infer Unparsed}` + ? [CheckSpecial, Unparsed] + : RegEx extends `\\${infer Literal}${infer Unparsed}` + ? [CheckLiteral, Unparsed] + : RegEx extends `${infer Literal}${infer Unparsed}` + ? [RegexLiteralToken, Unparsed] + : never + +/** Set of special characters */ +type REGEX_SPECIAL_TOKENS = "." | "+" | "*" | "?" + +// Special ranges or tokens for matching +type REGEX_ANY = RegexRangeToken +type REGEX_WORD = RegexRangeToken> +type REGEX_DIGIT = RegexRangeToken> +type REGEX_WHITESPACE = RegexRangeToken<"\t" | " "> + +type REGEX_ONE_OR_MORE = RegexRepeatingToken +type REGEX_ZERO_OR_MORE = RegexRepeatingToken +type REGEX_ZERO_OR_ONE = RegexRepeatingToken + +/** + * Map special character sets + */ +type CheckSpecial = Special extends "." + ? REGEX_ANY + : Special extends "+" + ? REGEX_ONE_OR_MORE + : Special extends "*" + ? REGEX_ZERO_OR_MORE + : Special extends "?" + ? REGEX_ZERO_OR_ONE + : never + +/** + * Check literal escape vs supported sets + */ +type CheckLiteral = Literal extends "w" + ? REGEX_WORD + : Literal extends "s" + ? REGEX_WHITESPACE + : Literal extends "d" + ? REGEX_DIGIT + : RegexLiteralToken // Check a word + +/** + * Parse a repeating token: {2,3} + */ +type ParseRepeating = + Repeating extends `${infer Min extends number},${infer Max extends number}` + ? RegexRepeatingToken + : Repeating extends `${infer Min extends number},` + ? RegexRepeatingToken + : Repeating extends `${infer Min extends number}` + ? RegexRepeatingToken + : never + +/** + * Parse the next segment of the range + */ +type ParseRange = + Range extends `${infer First}${infer Second}${infer Third}${infer Rest}` + ? Second extends "-" + ? VerifyRange | ParseRange + : First | ParseRange<`${Second}${Third}${Rest}`> + : Range extends `${infer First}${infer Second}${infer _}` + ? First | Second + : Range extends `${infer First}${infer _}` + ? First + : never + +/** + * Verify the range is valid and fits our hard coded sets + */ +type VerifyRange< + Start extends string, + End extends string +> = CToN extends number + ? CToN extends number + ? BuildRange, CToN> extends infer R extends string + ? R + : never + : Start | "-" | End + : Start | "-" | End + +/** + * Build all the characters in a range + */ +type BuildRange< + N extends number, + End extends number, + D extends number = 0 +> = NToC extends string + ? N extends End + ? NToC + : NToC | BuildRange, End, Increment> + : never + +/** + * Valid token types + */ +type RegexToken = + | RegexLiteralToken + | RegexRangeToken + | RegexRepeatingToken + | RegexAlternateToken + | RegexGroupToken + +/** + * A group token + */ +type RegexGroupToken = { + type: "group" + group: Group +} + +/** + * An alternate token: a|b + */ +type RegexAlternateToken< + Left extends RegexToken = IgnoreAny, + Right extends RegexToken = IgnoreAny +> = { + type: "alternate" + left: Left + right: Right +} + +/** + * Represents a literal token: A + */ +type RegexLiteralToken = { + type: "literal" + literal: Literal +} + +/** + * Represents a range of characters: [a-Z] + */ +type RegexRangeToken = { + type: "range" + range: Range +} + +/** + * Represents a repeating token + */ +type RegexRepeatingToken< + Token extends RegexToken = IgnoreAny, + Minimum extends number = number, + Maximum extends number = number +> = { + type: "repeating" + token: Token + min: Minimum + max: Maximum +} + +/** + * A potential branch towards a solution + */ +type RegexValidationState< + Candidate extends string = string, + Current extends RegexToken = IgnoreAny, + Remaining extends RegexToken[] = IgnoreAny, + Depth extends number = number +> = { + candidate: Candidate + current: Current + remaining: Remaining + depth: Depth +} + +/** + * Run a literal token against the candidate + */ +type RunLiteral< + Candidate extends string, + Token extends RegexToken +> = Token extends RegexLiteralToken + ? Candidate extends `${infer _ extends Literal}${infer Remainder}` + ? Remainder + : Candidate + : Candidate + +/** + * Run a range token against the candidate + */ +type RunRange< + Candidate extends string, + Token extends RegexToken +> = Token extends RegexRangeToken + ? Candidate extends `${infer _ extends Range}${infer Remainder}` + ? Remainder + : Candidate + : Candidate + +/** + * Run a DFS exploration on a candidate token + */ +type RunStateMachine< + Candidate extends string, + Token extends RegexToken +> = GenerateStates< + Candidate, + Token +> extends infer States extends RegexValidationState[] + ? TryAllStates + : never + +/** + * Run the DFS operation across all available states + */ +type TryAllStates = States extends [ + infer Next extends RegexValidationState, + ...infer Rest +] + ? RunState extends "" // Verify we consumed the entire string + ? true + : Rest extends never[] + ? false + : TryAllStates // Check the next potential state + : never + +/** + * Check the current state for a valid result + */ +type RunState = + State extends RegexValidationState + ? Token extends RegexLiteralToken + ? DFSLiteral extends infer Result + ? VerifyResult + : false + : Token extends RegexRangeToken + ? DFSRange extends infer Result + ? VerifyResult + : false + : Token extends RegexRepeatingToken + ? DFSRepeating2 extends infer Result + ? VerifyResult + : false + : Token extends RegexAlternateToken + ? DFSAlternate extends infer Result + ? VerifyResult + : false + : Token extends RegexGroupToken + ? DFSGroup extends infer Result + ? VerifyResult + : false + : false + : false + +/** + * Verify or call further down the state result chain + */ +type VerifyResult = Result extends RegexValidationState + ? RunState extends infer R + ? R + : false + : Result + +/** + * DFS on a range node + */ +type DFSRange = + State extends RegexValidationState< + infer Candidate, + infer Token, + infer Rest, + infer _ + > + ? Token extends RegexRangeToken + ? RunRange extends infer Returning extends string + ? Returning extends Candidate + ? false + : NextState + : false + : false + : false + +/** + * Handle DFS call chain for a literal + */ +type DFSLiteral = + State extends RegexValidationState< + infer Candidate, + infer Token, + infer Rest, + infer _ + > + ? Token extends RegexLiteralToken + ? RunLiteral extends infer Returning extends string + ? Returning extends Candidate + ? false + : NextState + : false + : false + : false + +/** + * Handle the DFS call chain for a repeating token using backtracking for + * matches in range + */ +type DFSRepeating2 = + State extends RegexValidationState< + infer Candidate, + infer Token, + infer Rest, + infer N + > + ? Token extends RegexRepeatingToken + ? RunState< + RegexValidationState + > extends infer Returning extends string + ? InRange, Min, Max> extends true + ? NextState< + Candidate, + Rest + > extends infer Next extends RegexValidationState + ? RunState extends "" + ? Next + : RegexValidationState> + : RegexValidationState> // No more states + : RegexValidationState> // Keep recursive + : InRange extends true // Can we keep going ? + ? NextState + : false // No more repetition + : false // Not repeating + : false + +/** + * Run the DFS on the alternates + */ +type DFSAlternate = + State extends RegexValidationState< + infer Candidate, + infer Token, + infer Rest, + infer N + > + ? Token extends RegexAlternateToken + ? RunState< + RegexValidationState + > extends infer Result extends string + ? Result + : RunState> + : false + : false + +/** + * Run the DFS over the group + */ +type DFSGroup = + State extends RegexValidationState< + infer Candidate, + infer Token, + infer Rest, + infer _ + > + ? Token extends RegexGroupToken + ? Group extends [ + infer First extends RegexToken, + ...infer Tokens extends RegexToken[] + ] + ? RunState< + RegexValidationState + > extends infer Result extends string + ? NextState + : false + : false + : false + : false + +/** + * Generate the valid states from a given token + */ +type GenerateStates< + Candidate extends string, + Token extends RegexToken, + N extends number = 0, + Remaining extends RegexToken[] = [] +> = Token extends RegexAlternateToken + ? [ + RegexValidationState, + RegexValidationState + ] + : Token extends RegexGroupToken + ? Group extends [infer SingleToken extends RegexToken] + ? [RegexValidationState] + : Group extends [ + infer Next extends RegexToken, + ...infer Rest extends RegexToken[] + ] + ? [RegexValidationState] + : never + : [RegexValidationState] + +/** + * Get the next available state + */ +type NextState< + Candidate extends string, + Tokens extends RegexToken[], + Counter extends number = 0 +> = Tokens extends [infer SingleToken extends RegexToken] + ? RegexValidationState + : Tokens extends [ + infer NextToken extends RegexToken, + ...infer Rest extends RegexToken[] + ] + ? RegexValidationState + : Candidate + +/** + * Get the value for the character + */ +type CToN = C extends keyof CharToIdx ? CharToIdx[C] : never + +/** + * Get the character for the value + */ +type NToC = IdxToChar[N] extends [never] + ? never + : IdxToChar[N] + +/** + * Array index to character + */ +type IdxToChar = [ + "\t", + "\n", + "\r", + " ", + "!", + '"', + "#", + "$", + "%", + "&", + "'", + "(", + ")", + "*", + "+", + ",", + "-", + ".", + "/", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + ":", + ";", + "<", + "=", + ">", + "?", + "@", + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", + "Q", + "R", + "S", + "T", + "U", + "V", + "W", + "X", + "Y", + "Z", + "[", + "\\", + "]", + "^", + "_", + "`", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "{", + "|", + "}", + "~" +] + +/** + * Character to index mapping + */ +type CharToIdx = { + "\t": 0 + "\n": 1 + "\r": 2 + " ": 3 + "!": 4 + '"': 5 + "#": 6 + $: 7 + "%": 8 + "&": 9 + "'": 10 + "(": 11 + ")": 12 + "*": 13 + "+": 14 + ",": 15 + "-": 16 + ".": 17 + "/": 18 + "0": 19 + "1": 20 + "2": 21 + "3": 22 + "4": 23 + "5": 24 + "6": 25 + "7": 26 + "8": 27 + "9": 28 + ":": 29 + ";": 30 + "<": 31 + "=": 32 + ">": 33 + "?": 34 + "@": 35 + A: 36 + B: 37 + C: 38 + D: 39 + E: 40 + F: 41 + G: 42 + H: 43 + I: 44 + J: 45 + K: 46 + L: 47 + M: 48 + N: 49 + O: 50 + P: 51 + Q: 52 + R: 53 + S: 54 + T: 55 + U: 56 + V: 57 + W: 58 + X: 59 + Y: 60 + Z: 61 + "[": 62 + "\\": 63 + "]": 64 + "^": 65 + _: 66 + "`": 67 + a: 68 + b: 69 + c: 70 + d: 71 + e: 72 + f: 73 + g: 74 + h: 75 + i: 76 + j: 77 + k: 78 + l: 79 + m: 80 + n: 81 + o: 82 + p: 83 + q: 84 + r: 85 + s: 86 + t: 87 + u: 88 + v: 89 + w: 90 + x: 91 + y: 92 + z: 93 + "{": 94 + "|": 95 + "}": 96 + "~": 97 +} diff --git a/packages/type-utils/strings.ts b/packages/type-utils/strings.ts index 268fd9a..2fba5bc 100644 --- a/packages/type-utils/strings.ts +++ b/packages/type-utils/strings.ts @@ -1,36 +1,237 @@ +import type { Decrement, GT, Increment, LT } from "./math.js" + /** * Join all of the strings using the given join (default ' ') */ -export type Join = T extends [ +export type Join = T extends [ infer Next extends string, - ...infer Rest, + ...infer Rest ] ? Rest extends never[] ? Next : Rest extends string[] - ? `${Next}${N}${Join}` - : "" + ? `${Next}${Token}${Join}` + : "" : "" /** - * Trim the leading/trailing whitespace characters - * - * Have to do manually since Regex isn't supported for string template types + * Trim excess whitespace */ -export type Trim = T extends ` ${infer Rest}` +export type Trim = Original extends ` ${infer Rest}` + ? Trim + : Original extends `\n${infer Rest}` + ? Trim + : Original extends `\t${infer Rest}` ? Trim - : T extends `${infer Rest} ` - ? Trim - : T extends `\n${infer Rest}` - ? Trim - : T extends `${infer Rest}\n` - ? Trim - : T extends `\r${infer Rest}` - ? Trim - : T extends `${infer Rest}\r` - ? Trim - : T extends `\t${infer Rest}` - ? Trim - : T extends `${infer Rest}\t` - ? Trim - : T + : Original extends `\r${infer Rest}` + ? Trim + : Original extends `${infer Start} ` + ? Trim + : Original extends `${infer Start}\n` + ? Trim + : Original extends `${infer Start}\t` + ? Trim + : Original extends `${infer Start}\r` + ? Trim + : Original + +/** + * Split the string using the given token + */ +export type Split< + Original extends string, + Token extends string +> = Original extends `${infer Left}${Token}${infer Right}` + ? [Left, ...Split] + : [Original] + +/** + * Find the length of the string + */ +export type StrLen< + Original extends string, + N extends number = 0 +> = Original extends "" + ? N + : Original extends `${infer _}${infer Rest}` + ? StrLen> + : -1 + +/** + * Replace the given values in the string with another + */ +export type Replace< + Original extends string, + Token extends string, + Replacement extends string +> = Original extends `${infer Left}${Token}${infer Right}` + ? Replace extends infer R extends string + ? `${Left}${Replacement}${R}` + : never + : Original + +/** + * Find the index of the character in the string + */ +export type IndexOf< + Original extends string, + Token extends string, + N extends number = 0 +> = Original extends "" + ? -1 + : Original extends `${Token}${infer _}` + ? N + : Original extends `${infer _}${infer Rest}` + ? IndexOf> + : -1 + +/** + * Truncate the first N characters + */ +export type Truncate< + Original extends string, + N extends number, + M extends number = 0 +> = M extends N + ? Original + : Original extends `${infer _}${infer Rest}` + ? Truncate> + : Original + +/** + * Get the next (Count) characters from the Start + */ +export type Substring< + Original extends string, + Start extends number, + Count extends number = -1 +> = GT extends true + ? Truncate extends infer Truncated extends string + ? Count extends -1 + ? Truncated + : _Substring + : never + : Count extends -1 + ? Original + : _Substring + +/** + * Internal helper to build the substrings + */ +type _Substring< + Original extends string, + N extends number, + S extends string = "" +> = N extends 0 + ? S + : Original extends "" + ? S + : Original extends `${infer C}${infer Rest}` + ? _Substring, `${S}${C}`> + : never + +/** + * Split into groups with open/close tokens at barriers + */ +export type SplitGroups< + Original extends string, + OpenToken extends string, + CloseToken extends string, + N extends number = 0, + C extends string = "" +> = N extends -1 // Unbalanced + ? never + : Original extends "" + ? [] + : IndexOf extends infer OT extends number + ? IndexOf extends infer CT extends number + ? CT extends OT // both are -1 + ? N extends 0 + ? [Original] + : never + : CT extends -1 // No close, only open + ? never + : OT extends -1 // Only close, no open + ? _SplitAt extends [ + infer Left extends string, + infer Right extends string + ] + ? Decrement extends 0 + ? `${C}${Left}` extends "" + ? SplitGroups + : [`${C}${Left}`, ...SplitGroups] + : SplitGroups< + Right, + OpenToken, + CloseToken, + Decrement, + `${C}${Left}${CloseToken}` + > + : never // No Split + : _SplitAt extends [ + infer Left extends string, + infer Right extends string + ] + ? LT extends true // Open before close + ? N extends 0 + ? Left extends "" + ? SplitGroups + : [Left, ...SplitGroups] + : SplitGroups< + Right, + OpenToken, + CloseToken, + Increment, + `${C}${Left}${OpenToken}` + > + : N extends 1 + ? `${C}${Left}` extends "" + ? SplitGroups + : [`${C}${Left}`, ...SplitGroups] + : SplitGroups< + Right, + OpenToken, + CloseToken, + Decrement, + `${C}${Left}${CloseToken}` + > + : never + : never + : never + +/** + * Split the string at the least of the two indices + */ +type _SplitAt< + Original extends string, + OpenIdx extends number, + CloseIdx extends number = OpenIdx +> = LT extends true + ? OpenIdx extends 0 + ? ["", Truncate] + : [Substring, Substring>] + : CloseIdx extends 0 + ? ["", Truncate] + : [Substring, Substring>] + +/** + * Utility type to check if there is a partial or unbalanced open/close count + */ +export type IsPartialGroup< + Original extends string, + OpenToken extends string, + CloseToken extends string +> = CountTokens extends CountTokens + ? false + : true + +/** + * Count the number of times the given token appears in the string + */ +export type CountTokens< + Original extends string, + Token extends string, + N extends number = 0 +> = Original extends `${infer _}${Token}${infer Right}` + ? CountTokens> + : N From 4d005b6f66813a28f8295d665fcb2c9e4f9600f7 Mon Sep 17 00:00:00 2001 From: Nathan Northcutt Date: Wed, 14 Aug 2024 11:49:39 +0200 Subject: [PATCH 02/21] starting to work on adding where clause support --- packages/sql/index.test.ts | 2 +- packages/sql/query/parser/insert.ts | 24 +-- packages/sql/query/parser/normalize.ts | 146 +++++++++------- packages/sql/query/parser/select.ts | 107 ++++++------ packages/sql/query/parser/utils.ts | 3 + packages/sql/query/parser/values.ts | 2 +- packages/sql/query/parser/where.ts | 224 +++++++++++++++++++++++++ 7 files changed, 386 insertions(+), 122 deletions(-) create mode 100644 packages/sql/query/parser/where.ts diff --git a/packages/sql/index.test.ts b/packages/sql/index.test.ts index f88e2bf..396c21b 100644 --- a/packages/sql/index.test.ts +++ b/packages/sql/index.test.ts @@ -68,7 +68,7 @@ describe("Schema building should create valid schemas", () => { describe("Invalid queries should be rejected", () => { describe("Invalid select should be rejected", () => { it("Should reject a select with no from", () => { - const bad: ParseSQL<"SELECT column"> = "Missing FROM" + const bad: ParseSQL<"SELECT column"> = "Missing FROM clause" expect(bad).not.toBeUndefined() }) diff --git a/packages/sql/query/parser/insert.ts b/packages/sql/query/parser/insert.ts index 86388b3..d4c4d29 100644 --- a/packages/sql/query/parser/insert.ts +++ b/packages/sql/query/parser/insert.ts @@ -47,7 +47,7 @@ type ExtractInsert< InsertSQL extends string, Options extends ParserOptions > = ExtractReturning extends PartialParserResult< - infer SQL extends string, + infer SQL, infer Returning > ? Returning extends ReturningClause @@ -61,10 +61,7 @@ type ExtractInsert< type ExtractInsertValues< Current extends PartialParserResult, Options extends ParserOptions -> = Current extends PartialParserResult< - infer SQL extends string, - infer Result extends object -> +> = Current extends PartialParserResult ? SQL extends `${infer Columns} VALUES ( ${infer ValuesClause} )` ? ParseValues< ValuesClause, @@ -76,12 +73,15 @@ type ExtractInsertValues< > : ParseValues : SQL extends `${infer Columns} SELECT ${infer Select}` - ? ParseSelect extends infer S extends SelectClause + ? ParseSelect< + `SELECT ${Select}`, + Options + > extends infer S extends SelectClause ? ExtractInsertColumns< PartialParserResult>, Options > - : ParseSelect + : ParseSelect<`SELECT ${Select}`, Options> : Invalid<`VALUES or SELECT are required for INSERT`> : never @@ -91,10 +91,7 @@ type ExtractInsertValues< type ExtractInsertColumns< Current extends PartialParserResult, Options extends ParserOptions -> = Current extends PartialParserResult< - infer SQL extends string, - infer Result extends object -> +> = Current extends PartialParserResult ? SQL extends `${infer Table} ( ${infer ColumnsClause} )` ? ParseSelectedColumns< ColumnsClause, @@ -119,10 +116,7 @@ type ExtractInsertColumns< type ExtractInsertTable< Current extends PartialParserResult, Options extends ParserOptions -> = Current extends PartialParserResult< - infer SQL extends string, - infer Result extends object -> +> = Current extends PartialParserResult ? ParseTableReference extends TableReference< infer Table, infer Alias diff --git a/packages/sql/query/parser/normalize.ts b/packages/sql/query/parser/normalize.ts index bc3b43d..11ece76 100644 --- a/packages/sql/query/parser/normalize.ts +++ b/packages/sql/query/parser/normalize.ts @@ -6,31 +6,33 @@ import { NORMALIZE_TARGETS } from "./keywords.js" /** * Ensure a query has a known structure with keywords uppercase and consistent spacing */ -export type NormalizeQuery = - SplitJoin extends infer Tabs extends string - ? SplitJoin extends infer NewLines extends string - ? SplitJoin extends infer Commas extends string - ? SplitJoin extends infer OpenParen extends string - ? SplitJoin extends infer Normalized extends string - ? Trim - : never +export type NormalizeQuery = SplitJoin< + Query, + "\t" +> extends infer Tabs extends string + ? SplitJoin extends infer NewLines extends string + ? SplitJoin extends infer Commas extends string + ? SplitJoin extends infer OpenParen extends string + ? SplitJoin extends infer Normalized extends string + ? Trim : never : never : never : never + : never /** * Normalize the values by ensuring capitalization */ export type NormalizedJoin = T extends [ infer Left, - ...infer Rest, + ...infer Rest ] ? Rest extends never[] ? Check : NormalizedJoin extends infer NJ extends string - ? `${Check & string} ${NJ}` - : never + ? `${Check & string} ${NJ}` + : never : "" /** @@ -49,20 +51,21 @@ export type Extractor = [clause: U | never, remainder: string] /** * Check if T starts with S (case insensitive) */ -export type StartsWith = - NextToken extends [infer Left extends string, infer _] - ? Uppercase extends S - ? true - : false +export type StartsWith = NextToken extends [ + infer Left extends string, + infer _ +] + ? Uppercase extends S + ? true : false + : false /** * Split words based on spacing only */ -export type SplitWords = - Trim extends `${infer Left} ${infer Right}` - ? [...SplitWords, ...SplitWords] - : [Trim] +export type SplitWords = Trim extends `${infer Left} ${infer Right}` + ? [...SplitWords, ...SplitWords] + : [Trim] /** * Keep aggregating the next token until the terminator is reached @@ -71,21 +74,20 @@ export type ExtractUntil< T extends string, K extends string, N extends number = 0, - S extends string = "", -> = - NextToken extends [infer Token extends string, infer Rest extends string] - ? Rest extends "" - ? [Trim] - : Token extends "(" - ? ExtractUntil, `${S} (`> - : Token extends ")" - ? ExtractUntil, `${S} )`> - : [Token] extends [K] - ? N extends 0 - ? [Trim, Trim<`${Token} ${Rest}`>] - : ExtractUntil - : ExtractUntil - : never + S extends string = "" +> = NextToken extends [infer Token extends string, infer Rest extends string] + ? Rest extends "" + ? [Trim] + : Token extends "(" + ? ExtractUntil, `${S} (`> + : Token extends ")" + ? ExtractUntil, `${S} )`> + : [Token] extends [K] + ? N extends 0 + ? [Trim, Trim<`${Token} ${Rest}`>] + : ExtractUntil + : ExtractUntil + : never /** * Custom split that is SQL aware and respects parenthesis depth @@ -93,17 +95,16 @@ export type ExtractUntil< export type SplitSQL< T extends string, Token extends string = ",", - S extends string = "", -> = - Trim extends `${infer Left} ${Token} ${infer Right}` - ? EqualParenthesis<`${S} ${Left}`> extends true - ? SplitSQL extends infer Tokens extends string[] - ? [Trim<`${S} ${Left}`>, ...Tokens] - : Invalid<"Unequal parenthesis"> - : SplitSQL> - : EqualParenthesis<`${S} ${T}`> extends true - ? [Trim<`${S} ${T}`>] + S extends string = "" +> = Trim extends `${infer Left} ${Token} ${infer Right}` + ? EqualParenthesis<`${S} ${Left}`> extends true + ? SplitSQL extends infer Tokens extends string[] + ? [Trim<`${S} ${Left}`>, ...Tokens] : Invalid<"Unequal parenthesis"> + : SplitSQL> + : EqualParenthesis<`${S} ${T}`> extends true + ? [Trim<`${S} ${T}`>] + : Invalid<"Unequal parenthesis"> /** * This function is responsible for making sure that the query string being @@ -163,6 +164,31 @@ export function takeUntil(tokens: string[], terminal: string[]): string[] { return ret } +/** + * Extract the next tokens while one of the filters matches + * + * @param tokens The tokens to process + * @param filters The set of filters to continue consuming + * @returns The set of tokens that matched the filters + */ +export function takeWhile(tokens: string[], filters: string[]): string[] { + const ret = [] + + let cnt = 0 + + while (tokens.length > 0 && filters.indexOf(tokens[0]) >= 0 && cnt === 0) { + const token = tokens.shift()! + ret.push(token) + if (token === "(") { + cnt++ + } else if (token === ")") { + cnt-- + } + } + + return ret +} + /** * Extract the next set of parenthesis * @@ -177,7 +203,7 @@ export function extractParenthesis(tokens: string[]): string[] { throw new Error( `Invalid, does not start with a parenthesis: ${ tokens.length > 0 ? tokens[0] : "empty array" - }`, + }` ) } tokens.shift() @@ -227,26 +253,32 @@ type CountOpen = T extends `${infer _}(${infer Right}` */ type CountClosed< T, - N extends number = 0, + N extends number = 0 > = T extends `${infer _})${infer Right}` ? CountClosed> : N /** * Split and then rejoin a string */ -type SplitJoin = - SplitTrim extends infer Tokens extends string[] ? Join : never +type SplitJoin = SplitTrim< + T, + C +> extends infer Tokens extends string[] + ? Join + : never /** * Split and trim all the values */ -type SplitTrim = - Trim extends `${infer Left}${C}${infer Right}` - ? [...SplitTrim, Trim, ...SplitTrim] - : Trim extends infer S extends string - ? SplitWords extends infer Words extends string[] - ? [NormalizedJoin] - : never - : never +type SplitTrim< + T, + C extends string = "," +> = Trim extends `${infer Left}${C}${infer Right}` + ? [...SplitTrim, Trim, ...SplitTrim] + : Trim extends infer S extends string + ? SplitWords extends infer Words extends string[] + ? [NormalizedJoin] + : never + : never /** * Check if a value is a normalized keyword diff --git a/packages/sql/query/parser/select.ts b/packages/sql/query/parser/select.ts index de1e326..2b7c9bb 100644 --- a/packages/sql/query/parser/select.ts +++ b/packages/sql/query/parser/select.ts @@ -1,28 +1,26 @@ -import type { Invalid } from "@telefrek/type-utils/common.js" +import type { Flatten, Invalid } from "@telefrek/type-utils/common.js" +import type { Trim } from "@telefrek/type-utils/strings" +import type { WhereClause } from "../../ast/filtering.js" import type { NamedQuery } from "../../ast/named.js" import type { SelectClause } from "../../ast/select.js" import type { TableReference } from "../../ast/tables.js" import { parseSelectedColumns, type ParseSelectedColumns } from "./columns.js" -import { FROM_KEYS, type FromKeywords } from "./keywords.js" -import { - takeUntil, - type ExtractUntil, - type NextToken, - type SplitSQL, - type StartsWith, -} from "./normalize.js" +import type { PartialParserResult } from "./common.js" +import { FROM_KEYS } from "./keywords.js" +import { takeUntil, type SplitSQL } from "./normalize.js" import type { ParserOptions } from "./options.js" import { tryParseNamedQuery } from "./query.js" import { parseTableReference, type ParseTableReference } from "./table.js" +import { parseWhere, type ExtractWhere } from "./where.js" /** * Parse the next select statement from the string */ export type ParseSelect< - T extends string, + SelectSQL extends string, Options extends ParserOptions -> = NextToken extends ["SELECT", infer Right extends string] - ? CheckSelect> +> = SelectSQL extends `SELECT ${infer Remainder}` + ? VerifySelect> : Invalid<"Corrupt SELECT syntax"> /** @@ -35,11 +33,21 @@ export function parseSelectClause( tokens: string[], options: ParserOptions ): SelectClause { - return { - type: "SelectClause", + // Extract the core select + let select = { columns: parseSelectedColumns(takeUntil(tokens, ["FROM"])), ...parseFrom(takeUntil(tokens, FROM_KEYS), options), } + + // Parse the optional where clause + if (tokens.length > 0 && tokens[0] === "WHERE") { + select = { ...select, ...parseWhere(tokens, options) } + } + + return { + type: "SelectClause", + ...select, + } } /** @@ -84,8 +92,12 @@ function parseFrom( /** * Check to get the type information */ -type CheckSelect = T extends Partial> - ? SelectClause +type VerifySelect = T extends Partial< + SelectClause +> + ? T extends WhereClause + ? Flatten & WhereClause> + : SelectClause : T /** @@ -129,40 +141,39 @@ type CheckColumnSyntax = Columns extends [ */ type CheckColumns = CheckColumnSyntax> -/** - * Parse out the columns and then process any from information - */ -type ExtractColumns< - T extends string, +/** Extract the SelectClause from the back to the front */ +type ExtractSelect< + SelectSQL extends string, Options extends ParserOptions -> = ExtractUntil extends [ - infer Columns extends string, - infer From extends string -] - ? CheckColumns extends true - ? StartsWith extends true - ? { - columns: ParseSelectedColumns - } & ExtractFrom - : Invalid<"Failed to parse columns"> - : CheckColumns - : Invalid<"Missing FROM"> +> = ExtractWhere< + PartialParserResult, + Options +> extends PartialParserResult + ? ExtractFrom, Options> + : ExtractWhere, Options> -/** - * Extract the from information - */ +/** Extract the from clause */ type ExtractFrom< - T extends string, + Current extends PartialParserResult, Options extends ParserOptions -> = NextToken extends ["FROM", infer Clause extends string] - ? ExtractUntil extends [ - infer From extends string, - infer _ - ] - ? { - from: ParseTableReference - } - : { - from: ParseTableReference - } +> = Current extends PartialParserResult + ? SQL extends `${infer Columns}FROM ${infer FromClause}` + ? ExtractColumns< + PartialParserResult< + Trim, + Flatten }> + >, + Options + > + : Invalid<`Missing FROM clause`> + : never + +/** Extract the selected columns */ +type ExtractColumns< + Current extends PartialParserResult, + Options extends ParserOptions +> = Current extends PartialParserResult + ? CheckColumns extends true + ? Flatten }> + : CheckColumns : never diff --git a/packages/sql/query/parser/utils.ts b/packages/sql/query/parser/utils.ts index 7fc1f68..8cf3e4a 100644 --- a/packages/sql/query/parser/utils.ts +++ b/packages/sql/query/parser/utils.ts @@ -26,6 +26,9 @@ export type RemoveQuotes< : S : S +/** + * Check if the string is quoted + */ export type IsQuoted< S extends string, Options extends ParserOptions diff --git a/packages/sql/query/parser/values.ts b/packages/sql/query/parser/values.ts index 00352d3..ea874e9 100644 --- a/packages/sql/query/parser/values.ts +++ b/packages/sql/query/parser/values.ts @@ -191,7 +191,7 @@ type ExtractValueType = ExtractValue< /** * Parse out the entire value string (may be quoted) */ -type ExtractValue< +export type ExtractValue< T extends string, Quote extends string, N extends number = 0, diff --git a/packages/sql/query/parser/where.ts b/packages/sql/query/parser/where.ts new file mode 100644 index 0000000..79396a9 --- /dev/null +++ b/packages/sql/query/parser/where.ts @@ -0,0 +1,224 @@ +import type { Flatten, Invalid } from "@telefrek/type-utils/common" +import type { Join, Trim } from "@telefrek/type-utils/strings" +import type { ColumnReference } from "../../ast/columns.js" +import type { + ColumnFilter, + FilteringOperation, + LogicalExpression, + LogicalOperation, + LogicalTree, + WhereClause, +} from "../../ast/filtering.js" +import type { ValueTypes } from "../../ast/values.js" +import { parseColumnReference, type ParseColumnDetails } from "./columns.js" +import type { PartialParserResult } from "./common.js" +import { + takeUntil, + takeWhile, + type ExtractUntil, + type NextToken, + type SplitWords, +} from "./normalize.js" +import type { GetQuote, ParserOptions } from "./options.js" +import type { CheckValueType, ExtractValue } from "./values.js" + +/** + * Parse the {@link WhereClause} from the token stack + * + * @param tokens The tokens to parse + * @param options The current {@link ParserOptions} + * @returns A {@link WhereClause} if one is found + */ +export function parseWhere( + tokens: string[], + options: ParserOptions +): WhereClause | object { + if (tokens.length === 0) { + return {} + } + + if ("WHERE" !== tokens.shift()) { + throw new Error(`Invalid where tokens`) + } + + return { + where: parseLogicalExpression(tokens, options), + } +} + +/** + * + * @param tokens The current token stack + * @param _options The current {@link ParserOptions} + * @returns The next {@link LogicalExpression} from the token stack + */ +function parseLogicalExpression( + tokens: string[], + _options: ParserOptions // TODO: Pass this through for filtering ops +): LogicalExpression { + const segments = tokens.join(" ").split(/(?=[>==", "<", "=", "!"]).join(" ").trim() + const op = takeWhile(segments, ["<", ">", "=", "!"]).join("") + const right = segments.join(" ").trim() + + return { + type: "ColumnFilter", + left: parseColumnReference(left.split(" ")), + op: op as FilteringOperation, + right: { + type: "StringValue", + value: right, + }, + } +} + +/** + * Extract a {@link WhereClause} from the end of the SQL provided in the {@link PartialParserResult} + */ +export type ExtractWhere< + Current extends PartialParserResult, + Options extends ParserOptions +> = Current extends PartialParserResult + ? SQL extends `${infer QuerySegment} WHERE ${infer Where}` + ? ParseExpressionTree< + Join>, + Options + > extends infer Exp extends LogicalExpression + ? PartialParserResult>> + : PartialParserResult + : Current + : never + +/** + * Split the where statement by potential filtering operations + */ +type SplitWhere = T extends `${infer Left}<>${infer Right}` + ? [...SplitWhere, "<>", ...SplitWhere] + : T extends `${infer Left}>${infer Next}${infer Right}` + ? SplitEqual"> + : T extends `${infer Left}<${infer Next}${infer Right}` + ? SplitEqual + : T extends `${infer Left}=${infer Right}` + ? [...SplitWhere, "=", ...SplitWhere] + : SplitWords + +/** + * Split out a possible trailing '=' character + */ +type SplitEqual< + Left extends string, + Next extends string, + Right extends string, + C extends string +> = Next extends "=" + ? [...SplitWhere, `${C}=`, ...SplitWhere] + : [...SplitWhere, C, ...SplitWhere<`${Next}${Right}`>] + +/** + * Parse an expression tree + */ +type ParseExpressionTree< + SQL extends string, + Options extends ParserOptions +> = ExtractLogical extends LogicalTree< + infer Left, + infer Op, + infer Right +> + ? LogicalTree + : ParseColumnFilter extends ColumnFilter< + infer Left, + infer Op, + infer Right + > + ? ColumnFilter + : Trim extends `( ${infer Inner} )` + ? ParseExpressionTree + : Invalid<`invalid expression: ${SQL & string}`> + +/** + * Extract a {@link LogicalTree} + */ +type ExtractLogical< + SQL extends string, + Options extends ParserOptions +> = ExtractUntil extends [ + infer Left extends string, + infer Remainder extends string +] + ? NextToken extends [ + infer Operation extends string, + infer Right extends string + ] + ? [Operation] extends [LogicalOperation] + ? CheckLogicalTree< + ParseExpressionTree, + Operation, + ParseExpressionTree + > + : never + : never + : ParseColumnFilter extends ColumnFilter< + infer Left, + infer Op, + infer Right + > + ? ColumnFilter + : Invalid<`Cannot parse logical or conditional filter from ${SQL & string}`> + +/** + * Check the logical tree to ensure it's correctly formed or extract/generate an + * Invalid error message + */ +type CheckLogicalTree = Left extends LogicalExpression + ? Right extends LogicalExpression + ? Operation extends LogicalOperation + ? LogicalTree + : Invalid<"Invalid logical tree detected"> + : Right extends Invalid + ? Invalid + : Invalid<"Invalid logical tree detected"> + : Left extends Invalid + ? Invalid + : Invalid<"Invalid logical tree detected"> + +/** + * Parse out a {@link ColumnFilter} + */ +type ParseColumnFilter< + SQL extends string, + Options extends ParserOptions +> = NextToken extends [ + infer Column extends string, + infer Exp extends string +] + ? NextToken extends [infer Op extends string, infer Value extends string] + ? Op extends FilteringOperation + ? ExtractValue> extends [infer V] + ? CheckFilter< + ColumnReference>, + Op, + CheckValueType> + > + : Invalid<`Failed to column filter: ${SQL & string}`> + : Invalid<`Failed to column filter: ${SQL & string}`> + : Invalid<`Failed to column filter: ${SQL & string}`> + : Invalid<`Failed to column filter: ${SQL & string}`> + +/** + * Check that the column filter is appropriate and well formed + */ +type CheckFilter = Left extends ColumnReference< + infer Reference, + infer Alias +> + ? [Operation] extends [FilteringOperation] + ? Right extends ValueTypes + ? ColumnFilter, Operation, Right> + : Right extends Invalid + ? Invalid + : Invalid<`Invalid column filter`> + : Invalid<`Invalid column filter`> + : Left extends Invalid + ? Invalid + : Invalid<`Invalid column filter`> From 8f217f2d499ba851a5018090c597b4d83ed2a101 Mon Sep 17 00:00:00 2001 From: Nathan Northcutt Date: Thu, 15 Aug 2024 11:44:23 +0200 Subject: [PATCH 03/21] working on where clause addition --- packages/sql/README.md | 33 +++ packages/sql/ast/filtering.ts | 22 +- packages/sql/index.test.ts | 9 + packages/sql/query/builder/from.ts | 12 +- packages/sql/query/builder/insert.ts | 2 +- packages/sql/query/builder/returning.ts | 2 +- .../query/builder/{columns.ts => select.ts} | 42 ++- packages/sql/query/builder/where.ts | 279 ++++++++++++++++++ packages/sql/query/context.ts | 131 +++++--- packages/sql/query/parser/select.ts | 3 +- packages/sql/query/parser/where.ts | 9 +- packages/sql/query/visitor/common.ts | 54 ++++ packages/sql/query/visitor/types.ts | 36 +++ packages/sql/results.ts | 2 +- packages/type-utils/object.ts | 68 +++-- 15 files changed, 575 insertions(+), 129 deletions(-) create mode 100644 packages/sql/README.md rename packages/sql/query/builder/{columns.ts => select.ts} (85%) create mode 100644 packages/sql/query/builder/where.ts diff --git a/packages/sql/README.md b/packages/sql/README.md new file mode 100644 index 0000000..9d31a51 --- /dev/null +++ b/packages/sql/README.md @@ -0,0 +1,33 @@ +# Telefrek SQL + +This package is designed to showcase typescript parsing, validation and +customization capabilities when dealing with SQL queries that can be executed +against a variety of backends. It is initially being written as a tutorial +series for how to build something ambitious and new but is something I also +intend to use in future projects and will be releasing packages and updates for +in the future. + +## Structure + +There are several sub packages within this main SQL project related to the +specific areas they are handling in the parsing and query building process. The +query package contains the parsing, building and validation components that are +used to manage the queries themselves in the project. The ast is the backbone +for communication between components and represents the SQL independent of it's +parsed or re-hydrated forms. The engines represent code designed to manage and +execute queries at runtime without having to know about the individual sources. +Finally the schema packages are intended to help with managing database schemas +that are used to validate queries and represent the entities that are expected +to exist as well as their shape in the target database. + +## Testing + +For now the primary testing is done with some mostly silly tests that fail as +soon as the TypeScript compilation becomes invalid to help track down +regressions or issues with the realtime parsing system. There are other tests +that verify that the builders generate the same structures that the parsers +expect and define. Finally most of the tests are grouped into a single file to +help find performance issues when dealing with dozens or hundreds of queries +within a single file. I might eventually split more of them out but for now I +want a place to be able to stress the type system and force a lot of +recompilation. diff --git a/packages/sql/ast/filtering.ts b/packages/sql/ast/filtering.ts index f7cb7b9..a6504d1 100644 --- a/packages/sql/ast/filtering.ts +++ b/packages/sql/ast/filtering.ts @@ -1,4 +1,4 @@ -import type { Invalid } from "@telefrek/type-utils/common.js" +import type { IgnoreAny, Invalid } from "@telefrek/type-utils/common.js" import type { ColumnReference } from "./columns.js" import type { SubQuery } from "./queries.js" import type { ValueTypes } from "./values.js" @@ -10,16 +10,18 @@ import type { ValueTypes } from "./values.js" * we need to avoid. This simply tells TypeScript to leave it alone and we'll * have to deal with the potential for bad data via our ValidateLogicalTree type */ -// eslint-disable-next-line @typescript-eslint/no-explicit-any -type AnyLogicalTree = LogicalTree +type AnyLogicalTree = LogicalTree /** * Utility type to verify a LogicalTree doesn't have invalid data */ -export type ValidateLogicalTree = - Tree extends LogicalTree - ? LogicalTree - : Invalid<"Tree is not a LogicalTree"> +export type ValidateLogicalTree = Tree extends LogicalTree< + infer Left, + infer Op, + infer Right +> + ? LogicalTree + : Invalid<"Tree is not a LogicalTree"> /** * Types for building filtering trees @@ -51,7 +53,7 @@ export type LogicalOperation = "AND" | "OR" | "NOT" export type SubqueryFilter< Column extends ColumnReference = ColumnReference, Operation extends string = SubQueryFilterOperation, - Subquery extends SubQuery = SubQuery, + Subquery extends SubQuery = SubQuery > = { type: "SubqueryFilter" column: Column @@ -65,7 +67,7 @@ export type SubqueryFilter< export type LogicalTree< Left extends LogicalExpression = LogicalExpression, Operation extends string = LogicalOperation, - Right extends LogicalExpression = LogicalExpression, + Right extends LogicalExpression = LogicalExpression > = { type: "LogicalTree" left: Left @@ -88,7 +90,7 @@ export type LogicalExpression = export type ColumnFilter< Left extends ColumnReference = ColumnReference, Operation extends string = FilteringOperation, - Right extends ValueTypes | ColumnReference = ValueTypes | ColumnReference, + Right extends ValueTypes | ColumnReference = ValueTypes | ColumnReference > = { type: "ColumnFilter" left: Left diff --git a/packages/sql/index.test.ts b/packages/sql/index.test.ts index 396c21b..30893d9 100644 --- a/packages/sql/index.test.ts +++ b/packages/sql/index.test.ts @@ -122,6 +122,15 @@ describe("Query visitors should produce equivalent SQL", () => { expect(normalizeQuery(visitor.sql)).toBe(normalizeQuery(queryString)) }) + it("Should be able to handle filtering with a where clause", () => { + const queryString = "SELECT id FROM orders WHERE user_id >= 1" + const query = getDatabase(TEST_DATABASE).parseSQL(queryString) + expect(query.query.where.left.alias).toBe("user_id") + const visitor = new DefaultQueryVisitor() + visitor.visitQuery(query) + expect(normalizeQuery(visitor.sql)).toBe(normalizeQuery(queryString)) + }) + it("Should be able to return an insert with no return", () => { const queryString = "INSERT INTO users(first_name, last_name) VALUES('firstName', 'lastName')" diff --git a/packages/sql/query/builder/from.ts b/packages/sql/query/builder/from.ts index ac5b49f..e66faa0 100644 --- a/packages/sql/query/builder/from.ts +++ b/packages/sql/query/builder/from.ts @@ -13,7 +13,7 @@ import type { ParseTableReference } from "../parser/table.js" import { createSelectedColumnsBuilder, type SelectedColumnsBuilder, -} from "./columns.js" +} from "./select.js" import { buildTableReference } from "./table.js" /** @@ -21,7 +21,7 @@ import { buildTableReference } from "./table.js" */ export interface FromQueryBuilder< Context extends QueryContext, - Options extends ParserOptions, + Options extends ParserOptions > { /** * Choose a table to select data from and optionally alias it @@ -29,7 +29,7 @@ export interface FromQueryBuilder< * @param table The table or table alias to select from */ from>>( - table: Table, + table: Table ): SelectedColumnsBuilder< ActivateTableContext>, ParseTableReference @@ -44,7 +44,7 @@ export interface FromQueryBuilder< */ export function createFromQueryBuilder< Context extends QueryContext, - Options extends ParserOptions, + Options extends ParserOptions >(context: Context, options: Options): FromQueryBuilder { return new DefaultFromQueryBuilder(context, options) } @@ -52,7 +52,7 @@ export function createFromQueryBuilder< class DefaultFromQueryBuilder< Database extends SQLDatabaseSchema, Context extends QueryContext, - Options extends ParserOptions, + Options extends ParserOptions > implements FromQueryBuilder { private _context: Context @@ -64,7 +64,7 @@ class DefaultFromQueryBuilder< } from
>>( - table: Table, + table: Table ): SelectedColumnsBuilder< ActivateTableContext< Context, diff --git a/packages/sql/query/builder/insert.ts b/packages/sql/query/builder/insert.ts index 14a0b69..0f2c886 100644 --- a/packages/sql/query/builder/insert.ts +++ b/packages/sql/query/builder/insert.ts @@ -18,8 +18,8 @@ import type { import type { ParserOptions } from "../parser/options.js" import type { ParseTableReference } from "../parser/table.js" import { parseValue, type ExtractTSValueTypes } from "../parser/values.js" -import { buildColumnReference, type VerifyColumnReferences } from "./columns.js" import { createReturningBuilder, type ReturningBuilder } from "./returning.js" +import { buildColumnReference, type VerifyColumnReferences } from "./select.js" import { buildTableReference } from "./table.js" /** diff --git a/packages/sql/query/builder/returning.ts b/packages/sql/query/builder/returning.ts index f919154..cac784f 100644 --- a/packages/sql/query/builder/returning.ts +++ b/packages/sql/query/builder/returning.ts @@ -7,7 +7,7 @@ import type { } from "../../ast/queries.js" import type { SQLColumnSchema } from "../../schema/columns.js" import type { AllowAliasing, QueryAST } from "../common.js" -import { buildColumnReference, type VerifySelectColumns } from "./columns.js" +import { buildColumnReference, type VerifySelectColumns } from "./select.js" /** * An interface for specifying optional RETURNING clauses diff --git a/packages/sql/query/builder/columns.ts b/packages/sql/query/builder/select.ts similarity index 85% rename from packages/sql/query/builder/columns.ts rename to packages/sql/query/builder/select.ts index cbdaaef..cee1cf9 100644 --- a/packages/sql/query/builder/columns.ts +++ b/packages/sql/query/builder/select.ts @@ -16,13 +16,14 @@ import { } from "../common.js" import type { GetSelectableColumns, QueryContext } from "../context.js" import type { ParseColumnReference } from "../parser/columns.js" +import { where, type WhereBuilder } from "./where.js" /** * Interface that can provide the columns for a select builder */ export interface SelectedColumnsBuilder< Context extends QueryContext = QueryContext, - Table extends TableReference = TableReference, + Table extends TableReference = TableReference > extends QueryAST> { /** * Choose the columns that we want to include in the select @@ -31,7 +32,7 @@ export interface SelectedColumnsBuilder< */ columns>[]>( ...columns: AtLeastOne - ): QueryAST, Table>> + ): WhereBuilder, Table>> } /** @@ -42,7 +43,7 @@ export interface SelectedColumnsBuilder< */ export function createSelectedColumnsBuilder< Context extends QueryContext, - Table extends TableReference, + Table extends TableReference >(context: Context, from: Table): SelectedColumnsBuilder { return new DefaultSelectedColumnsBuilder(context, from) } @@ -53,7 +54,7 @@ export function createSelectedColumnsBuilder< class DefaultSelectedColumnsBuilder< Database extends SQLDatabaseSchema = SQLDatabaseSchema, Context extends QueryContext = QueryContext, - Table extends TableReference = TableReference, + Table extends TableReference = TableReference > implements SelectedColumnsBuilder { private _context: Context @@ -77,19 +78,14 @@ class DefaultSelectedColumnsBuilder< columns>[]>( ...columns: AtLeastOne - ): QueryAST, Table>> { - return { - ast: { - type: "SQLQuery", - query: { - type: "SelectClause", - from: this._from, - columns: [ - ...columns.map((r) => buildColumnReference(r as unknown as string)), - ] as VerifySelectColumns, - }, - }, - } + ): WhereBuilder, Table>> { + return where(this._context, { + type: "SelectClause", + from: this._from, + columns: [ + ...columns.map((r) => buildColumnReference(r as unknown as string)), + ] as VerifySelectColumns, + }) } } @@ -112,17 +108,17 @@ export type VerifySelectColumns = type BuildSelectColumns = Columns extends [ infer Next extends string, - ...infer Rest, + ...infer Rest ] ? Rest extends never[] ? [ParseColumnReference] : Rest extends string[] - ? [ParseColumnReference, ...BuildSelectColumns] - : never + ? [ParseColumnReference, ...BuildSelectColumns] + : never : never export function buildColumnReference( - value: T, + value: T ): ParseColumnReference { if (ALIAS_REGEX.test(value)) { const data = value.split(" AS ") @@ -142,7 +138,7 @@ export function buildColumnReference( : (unboundColumnReference(value) as unknown as ParseColumnReference) } function unboundColumnReference( - column: T, + column: T ): ColumnReference> { return { type: "ColumnReference", @@ -155,7 +151,7 @@ function unboundColumnReference( } function tableColumnReference( - column: T, + column: T ): TableColumnReferenceType { const data = column.split(".") return { diff --git a/packages/sql/query/builder/where.ts b/packages/sql/query/builder/where.ts new file mode 100644 index 0000000..0a1b99a --- /dev/null +++ b/packages/sql/query/builder/where.ts @@ -0,0 +1,279 @@ +/** + * Where query clause building + */ + +import type { Flatten } from "@telefrek/type-utils/common" +import type { + ColumnReference, + TableColumnReference, + UnboundColumnReference, +} from "../../ast/columns.js" +import type { + ColumnFilter, + FilteringOperation, + LogicalExpression, + LogicalTree, + WhereClause, +} from "../../ast/filtering.js" +import type { QueryClause, SQLQuery } from "../../ast/queries.js" +import type { ValueTypes } from "../../ast/values.js" +import type { QueryAST } from "../common.js" +import type { + ColumnType, + MatchingColumns, + QueryContext, + QueryContextColumns, +} from "../context.js" +import type { ParseColumnReference } from "../parser/columns.js" +import { type CheckValueType, parseValue } from "../parser/values.js" +import { buildColumnReference } from "./select.js" + +export function where( + context: Context, + query: Query +): WhereBuilder { + return new DefaultWhereBuilder(context, query) +} + +export interface WhereBuilder< + Context extends QueryContext, + Query extends QueryClause +> extends QueryAST { + /** + * + */ + where( + builder: (w: WhereClauseBuilder) => Exp + ): AddWhereToAST +} + +class DefaultWhereBuilder< + Context extends QueryContext, + Query extends QueryClause +> implements WhereBuilder +{ + private _context: Context + private _query: Query + + constructor(context: Context, query: Query) { + this._context = context + this._query = query + } + + where( + builder: (w: WhereClauseBuilder) => Exp + ): AddWhereToAST { + return { + ast: { + type: "SQLQuery", + query: { + ...this._query, + where: builder(whereClause(this._context)), + }, + }, + } as AddWhereToAST + } + + get ast(): SQLQuery { + return { + type: "SQLQuery", + query: this._query, + } + } +} + +export type AddWhereToAST< + Query extends QueryClause, + Exp extends LogicalExpression +> = Flatten> extends QueryClause + ? QueryAST>> + : never + +type Parameter< + Value, + Context extends QueryContext, + Column +> = Value extends `:${infer _}` + ? Value + : Value extends `$${infer _}` + ? Value + : ColumnType | MatchingColumns + +type RefType = C extends `${infer Table}.${infer Column}` + ? ColumnReference> + : ColumnReference> + +export interface WhereClauseBuilder { + and( + left: Left, + right: Right + ): LogicalTree + + or( + left: Left, + right: Right + ): LogicalTree + + filter< + Column extends QueryContextColumns, + Op extends FilteringOperation, + Value + >( + column: Column, + op: Op, + value: Parameter + ): ColumnFilter< + RefType, + Op, + CheckColumnRef> + > +} + +type CheckColumnRef = Value extends Columns + ? ParseColumnReference + : CheckValueType extends infer V extends ValueTypes + ? V + : never + +export function whereClause( + context: Context +): WhereClauseBuilder { + return new DefaultWhereClauseBuilder(context) +} + +class DefaultWhereClauseBuilder + implements WhereClauseBuilder +{ + private _context: Context + + constructor(context: Context) { + this._context = context + } + + and( + left: Left, + right: Right + ): LogicalTree { + return { + type: "LogicalTree", + left, + op: "AND", + right, + } + } + + or( + left: Left, + right: Right + ): LogicalTree { + return { + type: "LogicalTree", + left, + op: "OR", + right, + } + } + + filter< + Column extends QueryContextColumns, + Op extends FilteringOperation, + Value + >( + column: Column, + op: Op, + value: Parameter + ): ColumnFilter< + RefType, + Op, + CheckColumnRef> + > { + return buildFilter( + this._context, + column, + op, + value as Value + ) as unknown as ColumnFilter< + RefType, + Op, + CheckColumnRef> + > + } +} + +function buildFilter< + Context extends QueryContext, + Column extends string, + Operation extends FilteringOperation, + Value +>( + context: Context, + column: Column, + op: Operation, + value: Value +): ColumnFilter< + Column extends `${infer Table}.${infer Col}` + ? ColumnReference, Col> + : ColumnReference, Column>, + Operation, + CheckColumnRef> +> { + return { + type: "ColumnFilter", + // eslint-disable-next-line @typescript-eslint/no-explicit-any + left: buildColumnReference(column) as any, + op, + right: (isParameter(value) + ? { + type: "ParameterValue", + name: String(value).substring(1), + } + : isColumn(context, value) + ? buildColumnReference(value as string) + : parseValue(String(value))) as CheckColumnRef< + Value, + QueryContextColumns + >, + } +} + +function isColumn( + context: Context, + value: Value +): boolean { + if (typeof value === "string") { + if (value.indexOf(".") > 0) { + const data = value.split(".") + if (Object.hasOwn(context.active, data[0])) { + const table = Object.getOwnPropertyDescriptor( + context.active, + data[0] + )?.value + if (table !== undefined && Object.hasOwn(table["columns"], data[1])) { + return true + } + } + } else { + for (const key of Object.keys(context.active)) { + const table = Object.getOwnPropertyDescriptor( + context.active, + key + )?.value + if (table !== undefined) { + for (const col of Object.keys(table["columns"])) { + if (col === value) { + return true + } + } + } + } + } + } + return false +} + +function isParameter(value: T): boolean { + return ( + typeof value === "string" && + (value.startsWith(":") || value.startsWith("$")) + ) +} diff --git a/packages/sql/query/context.ts b/packages/sql/query/context.ts index 579b97e..07af288 100644 --- a/packages/sql/query/context.ts +++ b/packages/sql/query/context.ts @@ -6,9 +6,12 @@ import type { import { clone, type CheckDuplicateKey, + type IsUnion, + type Keys, type StringKeys, } from "@telefrek/type-utils/object.js" import type { TableReference } from "../ast/tables.js" +import type { ColumnTSType } from "../results.js" import { createColumnSchemaBuilder, type ColumnSchemaBuilderFn, @@ -30,7 +33,7 @@ import type { ParseTableReference } from "./parser/table.js" * @returns A new {@link QueryContextBuilder} */ export function createContext( - database: Database, + database: Database ): QueryContextBuilder { return new QueryContextBuilder(< QueryContext @@ -48,7 +51,7 @@ export function createContext( * @returns A new {@link QueryContextBuilder} */ export function modifyContext( - context: Context, + context: Context ): QueryContextBuilder { return new QueryContextBuilder(context) } @@ -59,7 +62,7 @@ export function modifyContext( export type QueryContext< Database extends SQLDatabaseSchema = SQLDatabaseSchema, Active extends SQLDatabaseTables = IgnoreEmpty, - Returning extends SQLColumnSchema | number = SQLColumnSchema | number, + Returning extends SQLColumnSchema | number = SQLColumnSchema | number > = { database: Database active: Active @@ -82,12 +85,46 @@ export type GetSelectableColumns = ? GetColumnNames : never +export type QueryContextColumns = + Context extends QueryContext + ? IsUnion> extends true + ? GetColumnNames + : { + [Key in Keys]: `${Keys & string}` + }[Keys] + : never + +export type ColumnType< + Context extends QueryContext, + Column +> = Context extends QueryContext< + infer _Database, + infer Active, + infer _Returning +> + ? Column extends `${infer Table}.${infer Column}` + ? ColumnTSType + : { + [Key in Keys]: Column extends keyof Active[Key]["columns"] + ? ColumnTSType + : never + }[Keys] + : never + +export type MatchingColumns = { + [K in QueryContextColumns]: Column extends K + ? never + : ColumnType extends ColumnType + ? K + : never +}[QueryContextColumns] + /** * Class used for manipulating {@link QueryContext} objects */ class QueryContextBuilder< Database extends SQLDatabaseSchema, - Context extends QueryContext = QueryContext, + Context extends QueryContext = QueryContext > { private _context: Context constructor(context: Context) { @@ -110,7 +147,7 @@ class QueryContextBuilder< */ add
( table: CheckDuplicateKey, - builder: ColumnSchemaBuilderFn | Updated, + builder: ColumnSchemaBuilderFn | Updated ): QueryContextBuilder< Database, ActivateTableContext< @@ -152,7 +189,7 @@ class QueryContextBuilder< * @template Table The table from the database to copy */ copy
( - table: CheckDuplicateTableReference, + table: CheckDuplicateTableReference ): QueryContextBuilder< Database, ActivateTableContext< @@ -164,7 +201,7 @@ class QueryContextBuilder< const t = table as unknown as Table return this.add( t.alias as CheckDuplicateKey, - this.getTableSchema(t.table), + this.getTableSchema(t.table) ) } @@ -184,7 +221,7 @@ class QueryContextBuilder< (this._context["active"] as IgnoreAny)[ table ] as unknown as SQLTableSchema - )["columns"], + )["columns"] ) } @@ -200,7 +237,7 @@ class QueryContextBuilder< * @template Schema The new return schema */ returning( - schema: Schema, + schema: Schema ): QueryContextBuilder< Database, ChangeContextReturn @@ -224,7 +261,7 @@ class QueryContextBuilder< type GetColumnNames = { [Table in StringKeys]: { [Column in StringKeys]: [Column] extends [ - GetUniqueColumns, + GetUniqueColumns ] ? Column : `${Table}.${Column}` @@ -236,7 +273,7 @@ type GetColumnNames = { */ type GetOtherColumns< Schema extends SQLDatabaseTables, - Table extends keyof Schema, + Table extends keyof Schema > = { [Key in keyof Schema]: Key extends Table ? never @@ -264,40 +301,40 @@ type UniqueKeys = { * Retrieve the schema for the given table from the database or active portions * of the context */ -type GetTableSchema = - Context extends QueryContext - ? [Table] extends [StringKeys] - ? Database["tables"][Table]["columns"] - : [Table] extends [StringKeys] - ? Active[Table]["columns"] - : never +type GetTableSchema< + Context extends QueryContext, + Table extends string +> = Context extends QueryContext + ? [Table] extends [StringKeys] + ? Database["tables"][Table]["columns"] + : [Table] extends [StringKeys] + ? Active[Table]["columns"] : never + : never /** * Utility type to check for table reference conflict with an existing table */ type CheckDuplicateTableReference< Table extends TableReference, - Tables extends SQLDatabaseTables, -> = - CheckDuplicateKey extends Table["alias"] - ? Table - : Invalid<"Table reference alias conflicts with existing table name"> + Tables extends SQLDatabaseTables +> = CheckDuplicateKey extends Table["alias"] + ? Table + : Invalid<"Table reference alias conflicts with existing table name"> /** * Utility type for retrieving the table schema from the context */ export type GetContextTableSchema< Context extends QueryContext, - Table extends string, -> = - Context extends QueryContext - ? Table extends StringKeys - ? Active[Table]["columns"] - : Table extends StringKeys - ? Database["tables"][Table]["columns"] - : never + Table extends string +> = Context extends QueryContext + ? Table extends StringKeys + ? Active[Table]["columns"] + : Table extends StringKeys + ? Database["tables"][Table]["columns"] : never + : never /** * Utility type that adds the given table and schema to the active context @@ -308,19 +345,18 @@ export type ActivateTableContext< Schema extends SQLColumnSchema = GetContextTableSchema< Context, Table["table"] - >, -> = - Context extends QueryContext< - Context["database"], - infer Active, - infer Returning > - ? QueryContext< - Context["database"], - AddTableToSchema, - Returning - > - : never +> = Context extends QueryContext< + Context["database"], + infer Active, + infer Returning +> + ? QueryContext< + Context["database"], + AddTableToSchema, + Returning + > + : never /** * Change the context return type @@ -328,8 +364,7 @@ export type ActivateTableContext< type ChangeContextReturn< Database extends SQLDatabaseSchema, Context extends QueryContext, - Returning extends SQLColumnSchema, -> = - Context extends QueryContext - ? QueryContext - : never + Returning extends SQLColumnSchema +> = Context extends QueryContext + ? QueryContext + : never diff --git a/packages/sql/query/parser/select.ts b/packages/sql/query/parser/select.ts index 2b7c9bb..d9bb382 100644 --- a/packages/sql/query/parser/select.ts +++ b/packages/sql/query/parser/select.ts @@ -6,7 +6,6 @@ import type { SelectClause } from "../../ast/select.js" import type { TableReference } from "../../ast/tables.js" import { parseSelectedColumns, type ParseSelectedColumns } from "./columns.js" import type { PartialParserResult } from "./common.js" -import { FROM_KEYS } from "./keywords.js" import { takeUntil, type SplitSQL } from "./normalize.js" import type { ParserOptions } from "./options.js" import { tryParseNamedQuery } from "./query.js" @@ -36,7 +35,7 @@ export function parseSelectClause( // Extract the core select let select = { columns: parseSelectedColumns(takeUntil(tokens, ["FROM"])), - ...parseFrom(takeUntil(tokens, FROM_KEYS), options), + ...parseFrom(tokens, options), } // Parse the optional where clause diff --git a/packages/sql/query/parser/where.ts b/packages/sql/query/parser/where.ts index 79396a9..2ecea23 100644 --- a/packages/sql/query/parser/where.ts +++ b/packages/sql/query/parser/where.ts @@ -20,7 +20,7 @@ import { type SplitWords, } from "./normalize.js" import type { GetQuote, ParserOptions } from "./options.js" -import type { CheckValueType, ExtractValue } from "./values.js" +import { parseValue, type CheckValueType, type ExtractValue } from "./values.js" /** * Parse the {@link WhereClause} from the token stack @@ -54,7 +54,7 @@ export function parseWhere( */ function parseLogicalExpression( tokens: string[], - _options: ParserOptions // TODO: Pass this through for filtering ops + options: ParserOptions // TODO: Pass this through for filtering ops ): LogicalExpression { const segments = tokens.join(" ").split(/(?=[>==", "<", "=", "!"]).join(" ").trim() @@ -65,10 +65,7 @@ function parseLogicalExpression( type: "ColumnFilter", left: parseColumnReference(left.split(" ")), op: op as FilteringOperation, - right: { - type: "StringValue", - value: right, - }, + right: parseValue(right, options.tokens.quote), } } diff --git a/packages/sql/query/visitor/common.ts b/packages/sql/query/visitor/common.ts index 4650c52..cf6e894 100644 --- a/packages/sql/query/visitor/common.ts +++ b/packages/sql/query/visitor/common.ts @@ -1,4 +1,10 @@ import type { ColumnReference } from "../../ast/columns.js" +import type { + ColumnFilter, + LogicalExpression, + LogicalTree, + WhereClause, +} from "../../ast/filtering.js" import type { InsertClause, QueryClause, @@ -67,6 +73,12 @@ export class DefaultQueryVisitor } else { throw new Error(`Unuspported named queries on SELECT...FROM`) } + + // Check WHERE + if ("where" in select) { + this.append("WHERE") + this.visitWhereClause(select as Readonly) + } } visitInsertClause(insert: Readonly): void { @@ -126,6 +138,42 @@ export class DefaultQueryVisitor } } + visitWhereClause(where: Readonly): void { + this.visitLogicalExpression(where.where as Readonly) + } + + visitLogicalExpression( + expression: Readonly + ): void { + switch (expression.type) { + case "LogicalTree": + this.visitLogicalTree(expression as Readonly) + break + case "ColumnFilter": + this.visitColumnFilter(expression as Readonly) + break + } + } + + visitLogicalTree(tree: T): void { + // TODO: Handle subquery grouping... + this.visitLogicalExpression(tree.left as Readonly) + + this.append(tree.op) + + this.visitLogicalExpression(tree.right as Readonly) + } + + visitColumnFilter(filter: T): void { + this.visitColumnReference(filter.left) + this.append(filter.op) + if (filter.right.type === "ColumnReference") { + this.visitColumnReference(filter.right) + } else { + this.visitValueType(filter.right) + } + } + visitReturning(clause: Readonly): void { this.append("RETURNING") if (Array.isArray(clause.returning)) { @@ -188,6 +236,12 @@ export class DefaultQueryVisitor case "NullValue": this.append("null") break + case "BigIntValue": + this.append(value.value.toString()) + break + case "NumberValue": + this.append(value.value.toString()) + break default: this.append(String(value.value)) break diff --git a/packages/sql/query/visitor/types.ts b/packages/sql/query/visitor/types.ts index 46d8b3d..a1e56fe 100644 --- a/packages/sql/query/visitor/types.ts +++ b/packages/sql/query/visitor/types.ts @@ -1,4 +1,10 @@ import type { ColumnReference } from "../../ast/columns.js" +import type { + ColumnFilter, + LogicalExpression, + LogicalTree, + WhereClause, +} from "../../ast/filtering.js" import type { InsertClause, QueryClause, @@ -41,6 +47,36 @@ export interface QueryAstVisitor { */ visitInsertClause(insert: Readonly): void + /** + * Visit the where clause + * + * @param where The {@link WhereClause} to visit + */ + visitWhereClause(where: Readonly): void + + /** + * Visit the logical expression + * + * @param expression the {@link LogicalExpression} to visit + */ + visitLogicalExpression( + expression: Readonly + ): void + + /** + * Visit the logical tree + * + * @param tree The {@link LogicalTree} to visit + */ + visitLogicalTree(tree: T): void + + /** + * Visit the column filter + * + * @param filter The {@link ColumnFilter} to visit + */ + visitColumnFilter(filter: T): void + /** * Visit the table reference * diff --git a/packages/sql/results.ts b/packages/sql/results.ts index da2c84d..73f7cf8 100644 --- a/packages/sql/results.ts +++ b/packages/sql/results.ts @@ -19,7 +19,7 @@ import type { TSSQLType } from "./types.js" /** * Extract the typescript type for a column */ -type ColumnTSType> = +export type ColumnTSType> = T["array"] extends [true] ? TSSQLType[] : TSSQLType /** diff --git a/packages/type-utils/object.ts b/packages/type-utils/object.ts index 374235f..f2a6d20 100644 --- a/packages/type-utils/object.ts +++ b/packages/type-utils/object.ts @@ -6,34 +6,31 @@ import type { Invalid } from "./common.js" * @returns A clone of the object */ export function clone ? V : never>( - original: T, + original: T ): T { return original instanceof Date ? (new Date(original.getTime()) as T & Date) : Array.isArray(original) - ? (original.map((item) => clone(item)) as T & U[]) - : original && typeof original === "object" - ? (Object.getOwnPropertyNames(original) as (keyof T)[]).reduce( - (o, prop) => { - const descriptor = Object.getOwnPropertyDescriptor( - original, - prop, - )! - Object.defineProperty(o, prop, { - ...descriptor, - writable: true, // Mark this as readable temporarily - }) - o[prop] = clone(original[prop]) + ? (original.map((item) => clone(item)) as T & U[]) + : original && typeof original === "object" + ? (Object.getOwnPropertyNames(original) as (keyof T)[]).reduce( + (o, prop) => { + const descriptor = Object.getOwnPropertyDescriptor(original, prop)! + Object.defineProperty(o, prop, { + ...descriptor, + writable: true, // Mark this as readable temporarily + }) + o[prop] = clone(original[prop]) - // Refreeze if necessary - if (descriptor.writable) { - Object.freeze(o[prop]) - } - return o - }, - Object.create(Object.getPrototypeOf(original)), - ) - : original + // Refreeze if necessary + if (descriptor.writable) { + Object.freeze(o[prop]) + } + return o + }, + Object.create(Object.getPrototypeOf(original)) + ) + : original } /** @@ -41,6 +38,15 @@ export function clone ? V : never>( */ export type Keys = keyof T +/** + * Verify if T is a union type + */ +export type IsUnion = ( + T extends [never] ? never : U extends T ? false : true +) extends false + ? false + : true + /** * Get all of the keys that are strings */ @@ -60,10 +66,10 @@ export type RequiredLiteralKeys = { [K in keyof T as string extends K ? never : number extends K - ? never - : object extends Pick - ? never - : K]: T[K] + ? never + : object extends Pick + ? never + : K]: T[K] } /** @@ -73,10 +79,10 @@ export type OptionalLiteralKeys = { [K in keyof T as string extends K ? never : number extends K - ? never - : object extends Pick - ? K - : never]: T[K] + ? never + : object extends Pick + ? K + : never]: T[K] } /** From 9344782eb5865b6b63f3100792dfc60131821c3e Mon Sep 17 00:00:00 2001 From: Nathan Northcutt Date: Sun, 18 Aug 2024 17:20:50 +0200 Subject: [PATCH 04/21] working on refactoring, a lot of things have to change for full where clauses, yikes --- packages/sql/ast/filtering.ts | 56 +++++++++++++----------- packages/sql/index.test.ts | 24 ++++++++--- packages/sql/query/builder/where.ts | 15 +++++++ packages/sql/query/common.ts | 3 ++ packages/sql/query/parser/normalize.ts | 60 ++++++++++++++++++++------ packages/sql/query/parser/options.ts | 31 +++++++++++-- packages/sql/query/parser/query.ts | 11 +++-- packages/sql/query/parser/where.ts | 50 ++++++++++++++------- packages/type-utils/regex.ts | 8 ++-- packages/type-utils/strings.ts | 4 +- 10 files changed, 190 insertions(+), 72 deletions(-) diff --git a/packages/sql/ast/filtering.ts b/packages/sql/ast/filtering.ts index a6504d1..7995670 100644 --- a/packages/sql/ast/filtering.ts +++ b/packages/sql/ast/filtering.ts @@ -1,28 +1,8 @@ -import type { IgnoreAny, Invalid } from "@telefrek/type-utils/common.js" +import type { IgnoreAny } from "@telefrek/type-utils/common.js" import type { ColumnReference } from "./columns.js" import type { SubQuery } from "./queries.js" import type { ValueTypes } from "./values.js" -/** - * This is a helper type to instruct TypeScript to stop exploring the recursive - * chains that come from expression trees that are nested by nature. Since a - * LogicalExpression an contain a LogicalTree, it creates a circular type which - * we need to avoid. This simply tells TypeScript to leave it alone and we'll - * have to deal with the potential for bad data via our ValidateLogicalTree type - */ -type AnyLogicalTree = LogicalTree - -/** - * Utility type to verify a LogicalTree doesn't have invalid data - */ -export type ValidateLogicalTree = Tree extends LogicalTree< - infer Left, - infer Op, - infer Right -> - ? LogicalTree - : Invalid<"Tree is not a LogicalTree"> - /** * Types for building filtering trees */ @@ -37,6 +17,21 @@ export type FilteringOperation = | "LIKE" | "ILIKE" +/** + * The default filtering operations + */ +export const DEFAULT_FILTER_OPS: FilteringOperation[] = [ + "=", + "<", + ">", + "<=", + ">=", + "!=", + "<>", + "LIKE", + "ILIKE", +] + /** * Types of subquery filtering mechanisms */ @@ -45,7 +40,17 @@ export type SubQueryFilterOperation = "IN" | "ANY" | "ALL" | "EXISTS" | "SOME" /** * Types for building logical trees */ -export type LogicalOperation = "AND" | "OR" | "NOT" +export type LogicalTreeOperation = "AND" | "OR" + +/** + * Type for handling logical negations + */ +export type LogicalNegation< + Expression extends LogicalExpression = LogicalExpression +> = { + type: "LogicalNegation" + expression: Expression +} /** * The IN filter definition @@ -66,7 +71,7 @@ export type SubqueryFilter< */ export type LogicalTree< Left extends LogicalExpression = LogicalExpression, - Operation extends string = LogicalOperation, + Operation extends string = LogicalTreeOperation, Right extends LogicalExpression = LogicalExpression > = { type: "LogicalTree" @@ -76,13 +81,14 @@ export type LogicalTree< } /** - * The valid types for building a logical expression tree + * The valid types for building a logical expression trees */ export type LogicalExpression = | ValueTypes - | AnyLogicalTree + | LogicalTree | ColumnFilter | SubqueryFilter + | LogicalNegation /** * A filter between two objects diff --git a/packages/sql/index.test.ts b/packages/sql/index.test.ts index 30893d9..4f6c422 100644 --- a/packages/sql/index.test.ts +++ b/packages/sql/index.test.ts @@ -103,7 +103,9 @@ describe("Query visitors should produce equivalent SQL", () => { const query = getDatabase(TEST_DATABASE).parseSQL(queryString) const visitor = new DefaultQueryVisitor() visitor.visitQuery(query) - expect(normalizeQuery(visitor.sql)).toBe(normalizeQuery(queryString)) + expect(normalizeQuery(visitor.sql, DefaultOptions)).toBe( + normalizeQuery(queryString, DefaultOptions) + ) }) it("Should be able to return a select with columns", () => { @@ -111,7 +113,9 @@ describe("Query visitors should produce equivalent SQL", () => { const query = getDatabase(TEST_DATABASE).parseSQL(queryString) const visitor = new DefaultQueryVisitor() visitor.visitQuery(query) - expect(normalizeQuery(visitor.sql)).toBe(normalizeQuery(queryString)) + expect(normalizeQuery(visitor.sql, DefaultOptions)).toBe( + normalizeQuery(queryString, DefaultOptions) + ) }) it("Should be able to return a select with alias", () => { @@ -119,7 +123,9 @@ describe("Query visitors should produce equivalent SQL", () => { const query = getDatabase(TEST_DATABASE).parseSQL(queryString) const visitor = new DefaultQueryVisitor() visitor.visitQuery(query) - expect(normalizeQuery(visitor.sql)).toBe(normalizeQuery(queryString)) + expect(normalizeQuery(visitor.sql, DefaultOptions)).toBe( + normalizeQuery(queryString, DefaultOptions) + ) }) it("Should be able to handle filtering with a where clause", () => { @@ -128,7 +134,9 @@ describe("Query visitors should produce equivalent SQL", () => { expect(query.query.where.left.alias).toBe("user_id") const visitor = new DefaultQueryVisitor() visitor.visitQuery(query) - expect(normalizeQuery(visitor.sql)).toBe(normalizeQuery(queryString)) + expect(normalizeQuery(visitor.sql, DefaultOptions)).toBe( + normalizeQuery(queryString, DefaultOptions) + ) }) it("Should be able to return an insert with no return", () => { @@ -137,7 +145,9 @@ describe("Query visitors should produce equivalent SQL", () => { const query = getDatabase(TEST_DATABASE).parseSQL(queryString) const visitor = new DefaultQueryVisitor() visitor.visitQuery(query) - expect(normalizeQuery(visitor.sql)).toBe(normalizeQuery(queryString)) + expect(normalizeQuery(visitor.sql, DefaultOptions)).toBe( + normalizeQuery(queryString, DefaultOptions) + ) }) it("Should be able to return an insert with a return", () => { @@ -146,7 +156,9 @@ describe("Query visitors should produce equivalent SQL", () => { const query = getDatabase(TEST_DATABASE).parseSQL(queryString) const visitor = new DefaultQueryVisitor() visitor.visitQuery(query) - expect(normalizeQuery(visitor.sql)).toBe(normalizeQuery(queryString)) + expect(normalizeQuery(visitor.sql, DefaultOptions)).toBe( + normalizeQuery(queryString, DefaultOptions) + ) }) }) diff --git a/packages/sql/query/builder/where.ts b/packages/sql/query/builder/where.ts index 0a1b99a..c9ed309 100644 --- a/packages/sql/query/builder/where.ts +++ b/packages/sql/query/builder/where.ts @@ -28,6 +28,13 @@ import type { ParseColumnReference } from "../parser/columns.js" import { type CheckValueType, parseValue } from "../parser/values.js" import { buildColumnReference } from "./select.js" +/** + * Create a where builder + * + * @param context The current context + * @param query The current query + * @returns A {@link WhereBuilder} + */ export function where( context: Context, query: Query @@ -35,18 +42,26 @@ export function where( return new DefaultWhereBuilder(context, query) } +/** + * Build a where clause + */ export interface WhereBuilder< Context extends QueryContext, Query extends QueryClause > extends QueryAST { /** + * Create a where clause * + * @param builder The clause builder */ where( builder: (w: WhereClauseBuilder) => Exp ): AddWhereToAST } +/** + * Default implementation of the {@link WhereBuilder} + */ class DefaultWhereBuilder< Context extends QueryContext, Query extends QueryClause diff --git a/packages/sql/query/common.ts b/packages/sql/query/common.ts index 85cd710..3ec08cf 100644 --- a/packages/sql/query/common.ts +++ b/packages/sql/query/common.ts @@ -18,5 +18,8 @@ export type AllowAliasing = Value | AliasedValue */ export type AliasedValue = `${Value} AS ${string}` +/** Regex for aliasing */ export const ALIAS_REGEX = /.+ AS .+/ + +/** Regex for table bound columns */ export const TABLE_BOUND_REGEX = /([^.])+\.([^.])+/ diff --git a/packages/sql/query/parser/normalize.ts b/packages/sql/query/parser/normalize.ts index 11ece76..3188bd5 100644 --- a/packages/sql/query/parser/normalize.ts +++ b/packages/sql/query/parser/normalize.ts @@ -2,25 +2,60 @@ import type { Invalid } from "@telefrek/type-utils/common.js" import type { Decrement, Increment } from "@telefrek/type-utils/math.js" import type { Join, Trim } from "@telefrek/type-utils/strings.js" import { NORMALIZE_TARGETS } from "./keywords.js" +import type { GetFilteringOperations, ParserOptions } from "./options.js" /** * Ensure a query has a known structure with keywords uppercase and consistent spacing */ -export type NormalizeQuery = SplitJoin< - Query, - "\t" -> extends infer Tabs extends string +export type NormalizeQuery< + Query extends string, + Options extends ParserOptions +> = SplitJoin extends infer Tabs extends string ? SplitJoin extends infer NewLines extends string ? SplitJoin extends infer Commas extends string ? SplitJoin extends infer OpenParen extends string ? SplitJoin extends infer Normalized extends string - ? Trim + ? NormalizeFilters, Options> : never : never : never : never : never +type NormalizeFilters< + SQL extends string, + Options extends ParserOptions +> = SplitWords extends infer Tokens extends string[] + ? CleanFilters extends infer Cleaned extends string[] + ? Join + : never + : never + +type CleanFilters = Words extends [ + infer Next extends string, + ...infer Rest +] + ? Rest extends never[] + ? [CheckFilters] + : [CheckFilters, ...CleanFilters] + : never + +type CheckFilters< + SQL extends string, + Options extends ParserOptions, + S extends string = "" +> = SQL extends "" + ? S + : SQL extends `${infer Left}${infer Rest}` + ? Left extends GetFilteringOperations + ? Rest extends `${infer Right}${infer Remaining}` + ? `${Left}${Right}` extends GetFilteringOperations + ? Trim<`${Trim} ${Left}${Right} ${Trim}`> + : Trim<`${Trim} ${Left} ${Trim}`> + : Trim<`${Trim} ${Left}`> + : CheckFilters + : Trim<`${S}${SQL}`> + /** * Normalize the values by ensuring capitalization */ @@ -43,11 +78,6 @@ export type NextToken = ? [Token, Remainder] : [Trim, ""] -/** - * Utility type for extracting clauses and remainders - */ -export type Extractor = [clause: U | never, remainder: string] - /** * Check if T starts with S (case insensitive) */ @@ -116,16 +146,22 @@ export type SplitSQL< * 4. We combine it all back together as a single collapsed string * * @param query The query string to normalize + * @param options The parsing options to use * @returns A {@link NormalizeQuery} string */ -export function normalizeQuery(query: T): NormalizeQuery { +export function normalizeQuery( + query: T, + _options: Options +): NormalizeQuery { return query .split(/\s|(?=[,()])|(?<=[,()])/g) .filter((s) => s.length > 0) .map((s) => normalizeWord(s.trim())) - .join(" ") as NormalizeQuery + .join(" ") as NormalizeQuery } +// TODO: Add the filtering for extracting filters from individual words + /** * Ensure that keywords are uppercase so we can process them correctly * diff --git a/packages/sql/query/parser/options.ts b/packages/sql/query/parser/options.ts index 75ad7dd..c01552b 100644 --- a/packages/sql/query/parser/options.ts +++ b/packages/sql/query/parser/options.ts @@ -1,4 +1,8 @@ import type { Flatten } from "@telefrek/type-utils/common" +import { + DEFAULT_FILTER_OPS, + type FilteringOperation, +} from "../../ast/filtering.js" /** * The options for what can be overridden in the parsing logic @@ -14,8 +18,12 @@ export type ParserOptions< /** * Tokens that have syntatic meaning */ -export type SyntaxTokens = { +export type SyntaxTokens< + Quote extends string = string, + FilterOps extends string = FilteringOperation +> = { quote: Quote + filters: FilterOps[] } /** @@ -33,13 +41,20 @@ type DEFAULT_TOKENS = SyntaxTokens<"'"> */ const DefaultTokens: DEFAULT_TOKENS = { quote: "'", + filters: DEFAULT_FILTER_OPS, } /** * The default options used if none are provided */ -export const DefaultOptions = createParsingOptions({ quote: "'" }, "RETURNING") +export const DefaultOptions = createParsingOptions( + { quote: "'", filters: DEFAULT_FILTER_OPS }, + "RETURNING" +) +/** + * the default parser type + */ export type DEFAULT_PARSER_OPTIONS = typeof DefaultOptions /** @@ -59,11 +74,21 @@ export type CheckFeature< */ export type GetQuote = Options extends ParserOptions - ? Tokens extends SyntaxTokens + ? Tokens extends SyntaxTokens ? Quote : never : never +/** + * Retrieve the current filter operations + */ +export type GetFilteringOperations = + Options extends ParserOptions + ? Tokens extends SyntaxTokens + ? FilterOps + : never + : never + /** * Merge the partial tokens with the default tokens */ diff --git a/packages/sql/query/parser/query.ts b/packages/sql/query/parser/query.ts index 60d7d67..c884c7d 100644 --- a/packages/sql/query/parser/query.ts +++ b/packages/sql/query/parser/query.ts @@ -28,7 +28,7 @@ export type ParseSQL< export type ParseQuery< T extends string, Options extends ParserOptions -> = NormalizeQuery extends infer Q extends string +> = NormalizeQuery extends infer Q extends string ? Q extends `SELECT ${string}` ? ParseSelect : Q extends `INSERT INTO ${string}` @@ -73,11 +73,14 @@ export class QueryParser< * @param query The query to parse * @returns A fully parsed SQL query */ - parse(query: T): ParseSQL { + parse(query: T): ParseSQL { return { type: "SQLQuery", - query: parseQueryClause(normalizeQuery(query).split(" "), this._options), - } as ParseSQL + query: parseQueryClause( + normalizeQuery(query, this._options).split(" "), + this._options + ), + } as ParseSQL } } diff --git a/packages/sql/query/parser/where.ts b/packages/sql/query/parser/where.ts index 2ecea23..bd7d28f 100644 --- a/packages/sql/query/parser/where.ts +++ b/packages/sql/query/parser/where.ts @@ -5,12 +5,16 @@ import type { ColumnFilter, FilteringOperation, LogicalExpression, - LogicalOperation, LogicalTree, + LogicalTreeOperation, WhereClause, } from "../../ast/filtering.js" import type { ValueTypes } from "../../ast/values.js" -import { parseColumnReference, type ParseColumnDetails } from "./columns.js" +import { + parseColumnReference, + type ParseColumnDetails, + type ParseColumnReference, +} from "./columns.js" import type { PartialParserResult } from "./common.js" import { takeUntil, @@ -19,9 +23,15 @@ import { type NextToken, type SplitWords, } from "./normalize.js" -import type { GetQuote, ParserOptions } from "./options.js" +import type { + GetFilteringOperations, + GetQuote, + ParserOptions, +} from "./options.js" import { parseValue, type CheckValueType, type ExtractValue } from "./values.js" +// This entire thing needs a re-write... + /** * Parse the {@link WhereClause} from the token stack * @@ -57,8 +67,8 @@ function parseLogicalExpression( options: ParserOptions // TODO: Pass this through for filtering ops ): LogicalExpression { const segments = tokens.join(" ").split(/(?=[>==", "<", "=", "!"]).join(" ").trim() - const op = takeWhile(segments, ["<", ">", "=", "!"]).join("") + const left = takeUntil(segments, options.tokens.filters).join(" ").trim() + const op = takeWhile(segments, options.tokens.filters).join("") const right = segments.join(" ").trim() return { @@ -139,7 +149,7 @@ type ParseExpressionTree< type ExtractLogical< SQL extends string, Options extends ParserOptions -> = ExtractUntil extends [ +> = ExtractUntil extends [ infer Left extends string, infer Remainder extends string ] @@ -147,7 +157,7 @@ type ExtractLogical< infer Operation extends string, infer Right extends string ] - ? [Operation] extends [LogicalOperation] + ? [Operation] extends [LogicalTreeOperation] ? CheckLogicalTree< ParseExpressionTree, Operation, @@ -169,7 +179,7 @@ type ExtractLogical< */ type CheckLogicalTree = Left extends LogicalExpression ? Right extends LogicalExpression - ? Operation extends LogicalOperation + ? Operation extends LogicalTreeOperation ? LogicalTree : Invalid<"Invalid logical tree detected"> : Right extends Invalid @@ -190,17 +200,27 @@ type ParseColumnFilter< infer Exp extends string ] ? NextToken extends [infer Op extends string, infer Value extends string] - ? Op extends FilteringOperation - ? ExtractValue> extends [infer V] + ? Op extends GetFilteringOperations + ? ExtractValue> extends [infer V extends string] ? CheckFilter< ColumnReference>, Op, - CheckValueType> + ParseValueOrReference > - : Invalid<`Failed to column filter: ${SQL & string}`> - : Invalid<`Failed to column filter: ${SQL & string}`> - : Invalid<`Failed to column filter: ${SQL & string}`> - : Invalid<`Failed to column filter: ${SQL & string}`> + : Invalid<`Failed to parse column filter: ${SQL & string}`> + : Invalid<`Failed to parse column filter: ${SQL & string}`> + : Invalid<`Failed to parse column filter: ${SQL & string}`> + : Invalid<`Failed to parse column filter: ${SQL & string}`> + +/** + * Type to try to parse a value and if not fallback and assume it is column reference + */ +type ParseValueOrReference< + SQL extends string, + Options extends ParserOptions +> = CheckValueType> extends infer V extends ValueTypes + ? V + : ParseColumnReference /** * Check that the column filter is appropriate and well formed diff --git a/packages/type-utils/regex.ts b/packages/type-utils/regex.ts index 86b2f4b..d9b41ab 100644 --- a/packages/type-utils/regex.ts +++ b/packages/type-utils/regex.ts @@ -7,7 +7,7 @@ import type { IsPartialGroup, Replace, Split, SplitGroups } from "./strings.js" * a match */ export type ValidateRegEx< - Regex extends string, + Regex extends RegexToken, Candidate extends string > = IsMatch extends true ? Candidate @@ -17,11 +17,9 @@ export type ValidateRegEx< * Verify if the given candidate matches the regex */ export type IsMatch< - Regex extends string, + Regex extends RegexToken, Candidate extends string -> = RegEx extends infer Tree extends RegexToken - ? RunStateMachine - : false +> = RunStateMachine /** * Parse the regex tree from the current point down diff --git a/packages/type-utils/strings.ts b/packages/type-utils/strings.ts index 2fba5bc..c77fb4d 100644 --- a/packages/type-utils/strings.ts +++ b/packages/type-utils/strings.ts @@ -42,8 +42,8 @@ export type Split< Original extends string, Token extends string > = Original extends `${infer Left}${Token}${infer Right}` - ? [Left, ...Split] - : [Original] + ? [Trim, ...Split] + : [Trim] /** * Find the length of the string From 5fee2c776a9be1a20a0b0e96f993615c30079a40 Mon Sep 17 00:00:00 2001 From: Nathan Northcutt Date: Sun, 18 Aug 2024 17:28:28 +0200 Subject: [PATCH 05/21] fix with test --- packages/sql/index.test.ts | 2 +- packages/sql/query/parser/normalize.ts | 24 ++++++++++++++++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/packages/sql/index.test.ts b/packages/sql/index.test.ts index 4f6c422..b2c68eb 100644 --- a/packages/sql/index.test.ts +++ b/packages/sql/index.test.ts @@ -129,7 +129,7 @@ describe("Query visitors should produce equivalent SQL", () => { }) it("Should be able to handle filtering with a where clause", () => { - const queryString = "SELECT id FROM orders WHERE user_id >= 1" + const queryString = "SELECT id FROM orders WHERE user_id >=1" const query = getDatabase(TEST_DATABASE).parseSQL(queryString) expect(query.query.where.left.alias).toBe("user_id") const visitor = new DefaultQueryVisitor() diff --git a/packages/sql/query/parser/normalize.ts b/packages/sql/query/parser/normalize.ts index 3188bd5..09cd570 100644 --- a/packages/sql/query/parser/normalize.ts +++ b/packages/sql/query/parser/normalize.ts @@ -151,16 +151,36 @@ export type SplitSQL< */ export function normalizeQuery( query: T, - _options: Options + options: Options ): NormalizeQuery { return query .split(/\s|(?=[,()])|(?<=[,()])/g) .filter((s) => s.length > 0) .map((s) => normalizeWord(s.trim())) + .map((s) => splitFilters(s, options.tokens.filters)) .join(" ") as NormalizeQuery } -// TODO: Add the filtering for extracting filters from individual words +/** + * Ensure that filters are appropriately separated out with correct spacing + * + * @param word The word to split out filters + * @param filters The list of candidate filters + * @returns The patched word with the filter correctly sorted out + */ +function splitFilters(word: string, filters: string[]): string { + const filter = filters + .filter((f) => word.indexOf(f) >= 0) + .sort((a, b) => (a.length > b.length ? -1 : 1)) + .shift() + + if (filter !== undefined) { + const data = word.split(filter) + return (data[0].trim() + ` ${filter} ` + data[1].trim()).trim() + } + + return word +} /** * Ensure that keywords are uppercase so we can process them correctly From 504ed6dbddb8b4326b6cebc77cfcbc857728da51 Mon Sep 17 00:00:00 2001 From: Nathan Northcutt Date: Mon, 19 Aug 2024 12:28:57 +0200 Subject: [PATCH 06/21] refactor out arithmetic operations and ensure all are used for normalization and parsing options --- packages/sql/ast/arithmetic.ts | 77 +++++++++++++++++ packages/sql/ast/filtering.ts | 92 +++++++++++++-------- packages/sql/ast/update.ts | 81 +----------------- packages/sql/index.test.ts | 2 +- packages/sql/query/builder/where.ts | 12 +-- packages/sql/query/parser/keywords.ts | 2 + packages/sql/query/parser/normalize.test.ts | 20 +++++ packages/sql/query/parser/normalize.ts | 30 ++++--- packages/sql/query/parser/options.ts | 81 +++++++++++++++--- packages/sql/query/parser/where.ts | 26 ++++-- packages/sql/query/visitor/common.ts | 8 +- 11 files changed, 276 insertions(+), 155 deletions(-) create mode 100644 packages/sql/ast/arithmetic.ts create mode 100644 packages/sql/query/parser/normalize.test.ts diff --git a/packages/sql/ast/arithmetic.ts b/packages/sql/ast/arithmetic.ts new file mode 100644 index 0000000..b3c656e --- /dev/null +++ b/packages/sql/ast/arithmetic.ts @@ -0,0 +1,77 @@ +import type { ColumnReference } from "./columns.js" +import type { ValueTypes } from "./values.js" + +/** + * Types for Arithmetic operations + */ +export type ArithmeticOperation = "+" | "-" | "*" | "/" | "%" | "|" | "&" | "^" + +/** + * The default arithmetic operations + */ +export const DEFAULT_ARITHMETIC_OPS: ArithmeticOperation[] = [ + "%", + "&", + "*", + "+", + "-", + "/", + "^", + "|", +] + +/** + * Types for Arithmetic assignment + */ +export type ArithmeticAssignmentOperation = `${ArithmeticOperation}=` | "=" + +/** + * The default arithmetic assignment operations + */ +export const DEFAULT_ARITHMETIC_ASSIGNMENT_OPS: ArithmeticAssignmentOperation[] = + ["%=", "&=", "*=", "+=", "-=", "/=", "^=", "|=", "="] + +/** + * An arithmetic assignment expression, ex: column += b + */ +export type ColumnArithmeticAssignment< + Column extends ColumnReference = ColumnReference, + Op extends string = ArithmeticAssignmentOperation, + Value extends ColumnReference | ValueTypes | ArithmenticExpressionTree = + | ColumnReference + | ValueTypes + | ArithmenticExpressionTree +> = { + type: "ColumnArithmeticAssignment" + column: Column + operation: Op + value: Value +} + +/** + * An arithmetic expression between two values, ex: a + b + */ +export type ArithmeticExpression< + Left extends ColumnReference | ValueTypes = ColumnReference | ValueTypes, + Op extends string = ArithmeticOperation, + Right extends ColumnReference | ValueTypes = ColumnReference | ValueTypes +> = { + type: "ArithmeticExpression" + left: Left + operation: Op + right: Right +} + +/** + * An arithmetic expression tree + */ +export type ArithmenticExpressionTree< + Left extends ArithmeticExpression = ArithmeticExpression, + Op extends string = ArithmeticOperation, + Right extends ArithmeticExpression = ArithmeticExpression +> = { + type: "ArithmeticExpressionTree" + left: Left + operation: Op + right: Right +} diff --git a/packages/sql/ast/filtering.ts b/packages/sql/ast/filtering.ts index 7995670..cb6f8a2 100644 --- a/packages/sql/ast/filtering.ts +++ b/packages/sql/ast/filtering.ts @@ -4,23 +4,14 @@ import type { SubQuery } from "./queries.js" import type { ValueTypes } from "./values.js" /** - * Types for building filtering trees + * Types for for value comparisons */ -export type FilteringOperation = - | "=" - | "<" - | ">" - | "<=" - | ">=" - | "!=" - | "<>" - | "LIKE" - | "ILIKE" +export type ComparisonOperation = "=" | "<" | ">" | "<=" | ">=" | "!=" | "<>" /** - * The default filtering operations + * The default comparison operations */ -export const DEFAULT_FILTER_OPS: FilteringOperation[] = [ +export const DEFAULT_COMPARISON_OPS: ComparisonOperation[] = [ "=", "<", ">", @@ -28,14 +19,17 @@ export const DEFAULT_FILTER_OPS: FilteringOperation[] = [ ">=", "!=", "<>", - "LIKE", - "ILIKE", ] /** - * Types of subquery filtering mechanisms + * Types of subquery filtering mechanisms (IN is a special case) */ -export type SubQueryFilterOperation = "IN" | "ANY" | "ALL" | "EXISTS" | "SOME" +export type SubQueryFilterOperation = "ANY" | "ALL" | "EXISTS" | "SOME" + +/** + * Types of logical operations + */ +export type LogicalOperation = "BETWEEN" | "LIKE" | "ILIKE" /** * Types for building logical trees @@ -52,20 +46,6 @@ export type LogicalNegation< expression: Expression } -/** - * The IN filter definition - */ -export type SubqueryFilter< - Column extends ColumnReference = ColumnReference, - Operation extends string = SubQueryFilterOperation, - Subquery extends SubQuery = SubQuery -> = { - type: "SubqueryFilter" - column: Column - query: Subquery - op: Operation -} - /** * A logical tree structure for processing groups of filters */ @@ -94,14 +74,14 @@ export type LogicalExpression = * A filter between two objects */ export type ColumnFilter< - Left extends ColumnReference = ColumnReference, - Operation extends string = FilteringOperation, - Right extends ValueTypes | ColumnReference = ValueTypes | ColumnReference + Column extends ColumnReference = ColumnReference, + Operation extends string = ComparisonOperation, + Filter extends ValueTypes | ColumnReference = ValueTypes | ColumnReference > = { type: "ColumnFilter" - left: Left + column: Column op: Operation - right: Right + filter: Filter } /** @@ -110,3 +90,43 @@ export type ColumnFilter< export type WhereClause = { where: Where } + +/** + * A filter for a column in some range + */ +export type BetweenFilter< + Column extends ColumnReference = ColumnReference, + Left extends ValueTypes = ValueTypes, + Right extends ValueTypes = ValueTypes +> = { + type: "BetweenFilter" + column: Column + left: Left + right: Right +} + +/** + * A filter for an "IN" clause that can be either a set of values or a subquery + */ +export type InFilter< + Column extends ColumnReference = ColumnReference, + Values extends SubQuery | ValueTypes[] = SubQuery | ValueTypes[] +> = { + type: "InFilter" + column: Column + values: Values +} + +/** + * A filter for a SubQuery operation + */ +export type SubqueryFilter< + Column extends ColumnReference = ColumnReference, + Operation extends string = SubQueryFilterOperation, + Subquery extends SubQuery = SubQuery +> = { + type: "SubqueryFilter" + column: Column + query: Subquery + op: Operation +} diff --git a/packages/sql/ast/update.ts b/packages/sql/ast/update.ts index 49a7495..237e964 100644 --- a/packages/sql/ast/update.ts +++ b/packages/sql/ast/update.ts @@ -1,90 +1,15 @@ -import type { Invalid, OneOrMore } from "@telefrek/type-utils/common.js" -import type { ColumnReference } from "./columns.js" +import type { OneOrMore } from "@telefrek/type-utils/common.js" +import type { ColumnArithmeticAssignment } from "./arithmetic.js" import type { TableReference } from "./tables.js" -import type { ValueTypes } from "./values.js" - -/** - * This is a helper type to instruct TypeScript to stop exploring the recursive - * chains that come from assignment trees that are nested by nature. Since an - * AssignmentTree an contain an AssignmentTree, it creates a circular type which - * we need to avoid. This simply tells TypeScript to leave it alone and we'll - * have to deal with the potential for bad data via our ValidateAssignmenTree type - */ -// eslint-disable-next-line @typescript-eslint/no-explicit-any -type AnyAssignmentTree = AssignmentTree - -/** - * Utility type to verify a LogicalTree doesn't have invalid data - */ -export type ValidateAssignmentTree = - Tree extends AssignmentTree - ? AssignmentTree - : Invalid<"Tree is not an AssignmentTree"> - -/** - * Operation to modify a column using a value - */ -export type AssignmentOperation = - | "=" - | "+" - | "-" - | "*" - | "/" - | "%" - | "&" - | "|" - | "^" - | "+=" - | "-=" - | "*=" - | "/=" - | "%=" - | "&=" - -/** - * An abstract expression for column assignment - */ -export type AssignmentExpression = - | ValueTypes - | ColumnReference - | AnyAssignmentTree - -/** - * Represents a tree of assignment operations to facilitate combinations of - * parameters, values and other column manipulations to get a final value - */ -export type AssignmentTree< - Left extends AssignmentExpression = AssignmentExpression, - Operation extends string = AssignmentOperation, - Right extends AssignmentExpression = AssignmentExpression, -> = { - type: "AssignmentTree" - left: Left - op: Operation - right: Right -} /** * Structure for an update clause */ export type UpdateClause< Table extends TableReference = TableReference, - Columns extends OneOrMore = OneOrMore, + Columns extends OneOrMore = OneOrMore > = { type: "UpdateClause" columns: Columns table: Table } - -/** - * A column can be assigned to a value that can be a combination of parameters, - * other columns or simple values - */ -export type ColumnAssignment< - Column extends ColumnReference = ColumnReference, - Assignment extends AssignmentExpression = AssignmentExpression, -> = { - type: "ColumnAssignment" - column: Column - assignment: Assignment -} diff --git a/packages/sql/index.test.ts b/packages/sql/index.test.ts index b2c68eb..088607c 100644 --- a/packages/sql/index.test.ts +++ b/packages/sql/index.test.ts @@ -131,7 +131,7 @@ describe("Query visitors should produce equivalent SQL", () => { it("Should be able to handle filtering with a where clause", () => { const queryString = "SELECT id FROM orders WHERE user_id >=1" const query = getDatabase(TEST_DATABASE).parseSQL(queryString) - expect(query.query.where.left.alias).toBe("user_id") + expect(query.query.where.column.alias).toBe("user_id") const visitor = new DefaultQueryVisitor() visitor.visitQuery(query) expect(normalizeQuery(visitor.sql, DefaultOptions)).toBe( diff --git a/packages/sql/query/builder/where.ts b/packages/sql/query/builder/where.ts index c9ed309..4521e8a 100644 --- a/packages/sql/query/builder/where.ts +++ b/packages/sql/query/builder/where.ts @@ -10,7 +10,7 @@ import type { } from "../../ast/columns.js" import type { ColumnFilter, - FilteringOperation, + ComparisonOperation, LogicalExpression, LogicalTree, WhereClause, @@ -131,7 +131,7 @@ export interface WhereClauseBuilder { filter< Column extends QueryContextColumns, - Op extends FilteringOperation, + Op extends ComparisonOperation, Value >( column: Column, @@ -191,7 +191,7 @@ class DefaultWhereClauseBuilder filter< Column extends QueryContextColumns, - Op extends FilteringOperation, + Op extends ComparisonOperation, Value >( column: Column, @@ -218,7 +218,7 @@ class DefaultWhereClauseBuilder function buildFilter< Context extends QueryContext, Column extends string, - Operation extends FilteringOperation, + Operation extends ComparisonOperation, Value >( context: Context, @@ -235,9 +235,9 @@ function buildFilter< return { type: "ColumnFilter", // eslint-disable-next-line @typescript-eslint/no-explicit-any - left: buildColumnReference(column) as any, + column: buildColumnReference(column) as any, op, - right: (isParameter(value) + filter: (isParameter(value) ? { type: "ParameterValue", name: String(value).substring(1), diff --git a/packages/sql/query/parser/keywords.ts b/packages/sql/query/parser/keywords.ts index c29ea0b..5b5a264 100644 --- a/packages/sql/query/parser/keywords.ts +++ b/packages/sql/query/parser/keywords.ts @@ -41,6 +41,8 @@ export const QUERY_KEYS = ["SELECT", "UPDATE", "INSERT", "DELETE", "WITH"] */ export const WHERE_KEYS = ["HAVING", "GROUP", "OFFSET", "LIMIT", "ORDER"] +export const EXPRESSION_KEYS = ["BETWEEN", "IN", ""] + /** * The set of keys that indicate the end of a join clause */ diff --git a/packages/sql/query/parser/normalize.test.ts b/packages/sql/query/parser/normalize.test.ts new file mode 100644 index 0000000..9bc4a3a --- /dev/null +++ b/packages/sql/query/parser/normalize.test.ts @@ -0,0 +1,20 @@ +import { normalizeQuery } from "./normalize.js" +import { DefaultOptions } from "./options.js" + +describe("SQL query strings should be appropriately normalized", () => { + describe("Filters should be appropriately handled", () => { + it("Should split out comparisons in a where clause", () => { + const query = "SELECT * FROM t WHERE id>=1 AND id=(1%2)" + expect(normalizeQuery(query, DefaultOptions)).toBe( + "SELECT * FROM t WHERE id >= 1 AND id = ( 1 % 2 )" + ) + }) + + it("Should split out set operations in an update clause", () => { + const query = "UPDATE t SET a&=1,b=(2+3/4)" + expect(normalizeQuery(query, DefaultOptions)).toBe( + "UPDATE t SET a &= 1 , b = ( 2 + 3 / 4 )" + ) + }) + }) +}) diff --git a/packages/sql/query/parser/normalize.ts b/packages/sql/query/parser/normalize.ts index 09cd570..f38b7dc 100644 --- a/packages/sql/query/parser/normalize.ts +++ b/packages/sql/query/parser/normalize.ts @@ -2,7 +2,7 @@ import type { Invalid } from "@telefrek/type-utils/common.js" import type { Decrement, Increment } from "@telefrek/type-utils/math.js" import type { Join, Trim } from "@telefrek/type-utils/strings.js" import { NORMALIZE_TARGETS } from "./keywords.js" -import type { GetFilteringOperations, ParserOptions } from "./options.js" +import type { GetNormalizationTokens, ParserOptions } from "./options.js" /** * Ensure a query has a known structure with keywords uppercase and consistent spacing @@ -47,12 +47,12 @@ type CheckFilters< > = SQL extends "" ? S : SQL extends `${infer Left}${infer Rest}` - ? Left extends GetFilteringOperations + ? Left extends GetNormalizationTokens ? Rest extends `${infer Right}${infer Remaining}` - ? `${Left}${Right}` extends GetFilteringOperations - ? Trim<`${Trim} ${Left}${Right} ${Trim}`> - : Trim<`${Trim} ${Left} ${Trim}`> - : Trim<`${Trim} ${Left}`> + ? `${Left}${Right}` extends GetNormalizationTokens + ? Trim<`${Trim} ${Left}${Right} ${CheckFilters}`> + : Trim<`${Trim} ${Left} ${CheckFilters}`> + : Trim<`${Trim} ${Left} ${CheckFilters}`> : CheckFilters : Trim<`${S}${SQL}`> @@ -153,22 +153,28 @@ export function normalizeQuery( query: T, options: Options ): NormalizeQuery { + const keys = new Set() + + options.tokens.arithmetic.forEach((t) => keys.add(t)) + options.tokens.assignments.forEach((t) => keys.add(t)) + options.tokens.comparisons.forEach((t) => keys.add(t)) + return query .split(/\s|(?=[,()])|(?<=[,()])/g) .filter((s) => s.length > 0) .map((s) => normalizeWord(s.trim())) - .map((s) => splitFilters(s, options.tokens.filters)) + .map((s) => splitKeywords(s, Array.from(keys.values()))) .join(" ") as NormalizeQuery } /** - * Ensure that filters are appropriately separated out with correct spacing + * Ensure that keywords are appropriately separated out with correct spacing * * @param word The word to split out filters * @param filters The list of candidate filters * @returns The patched word with the filter correctly sorted out */ -function splitFilters(word: string, filters: string[]): string { +function splitKeywords(word: string, filters: string[]): string { const filter = filters .filter((f) => word.indexOf(f) >= 0) .sort((a, b) => (a.length > b.length ? -1 : 1)) @@ -176,7 +182,11 @@ function splitFilters(word: string, filters: string[]): string { if (filter !== undefined) { const data = word.split(filter) - return (data[0].trim() + ` ${filter} ` + data[1].trim()).trim() + return ( + data[0].trim() + + ` ${filter} ` + + splitKeywords(data[1].trim(), filters) + ).trim() } return word diff --git a/packages/sql/query/parser/options.ts b/packages/sql/query/parser/options.ts index c01552b..019283f 100644 --- a/packages/sql/query/parser/options.ts +++ b/packages/sql/query/parser/options.ts @@ -1,7 +1,13 @@ import type { Flatten } from "@telefrek/type-utils/common" import { - DEFAULT_FILTER_OPS, - type FilteringOperation, + DEFAULT_ARITHMETIC_ASSIGNMENT_OPS, + DEFAULT_ARITHMETIC_OPS, + type ArithmeticAssignmentOperation, + type ArithmeticOperation, +} from "../../ast/arithmetic.js" +import { + DEFAULT_COMPARISON_OPS, + type ComparisonOperation, } from "../../ast/filtering.js" /** @@ -20,10 +26,14 @@ export type ParserOptions< */ export type SyntaxTokens< Quote extends string = string, - FilterOps extends string = FilteringOperation + Comparisons extends string = ComparisonOperation, + Assignments extends string = ArithmeticAssignmentOperation, + Arithmetic extends string = ArithmeticOperation > = { quote: Quote - filters: FilterOps[] + comparisons: Comparisons[] + assignments: Assignments[] + arithmetic: Arithmetic[] } /** @@ -41,14 +51,21 @@ type DEFAULT_TOKENS = SyntaxTokens<"'"> */ const DefaultTokens: DEFAULT_TOKENS = { quote: "'", - filters: DEFAULT_FILTER_OPS, + comparisons: DEFAULT_COMPARISON_OPS, + arithmetic: DEFAULT_ARITHMETIC_OPS, + assignments: DEFAULT_ARITHMETIC_ASSIGNMENT_OPS, } /** * The default options used if none are provided */ export const DefaultOptions = createParsingOptions( - { quote: "'", filters: DEFAULT_FILTER_OPS }, + { + quote: "'", + filters: DEFAULT_COMPARISON_OPS, + assignments: DEFAULT_ARITHMETIC_ASSIGNMENT_OPS, + arithmetic: DEFAULT_ARITHMETIC_OPS, + }, "RETURNING" ) @@ -74,21 +91,63 @@ export type CheckFeature< */ export type GetQuote = Options extends ParserOptions - ? Tokens extends SyntaxTokens + ? Tokens extends SyntaxTokens ? Quote : never : never /** - * Retrieve the current filter operations + * Extract all special tokens for normalization + */ +export type GetNormalizationTokens = + | GetComparisonOperations + | GetAssignmentOperations + | GetArithmeticOperations + +/** + * Retrieve the current comparison operations */ -export type GetFilteringOperations = +export type GetComparisonOperations = Options extends ParserOptions - ? Tokens extends SyntaxTokens - ? FilterOps + ? Tokens extends SyntaxTokens< + infer _, + infer ComparisonOps, + infer _, + infer _ + > + ? ComparisonOps : never : never +/** + * Retrieve the current arithmetic operations + */ +export type GetArithmeticOperations = + Options extends ParserOptions + ? Tokens extends SyntaxTokens< + infer _, + infer _, + infer _, + infer ArithmeticOps + > + ? ArithmeticOps + : never + : never + +/** + * Retrieve the current arithmetic assignment operations + */ +export type GetAssignmentOperations = + Options extends ParserOptions + ? Tokens extends SyntaxTokens< + infer _, + infer _, + infer AssignmentOps, + infer _ + > + ? AssignmentOps + : never + : never /** * Merge the partial tokens with the default tokens */ diff --git a/packages/sql/query/parser/where.ts b/packages/sql/query/parser/where.ts index bd7d28f..5a347aa 100644 --- a/packages/sql/query/parser/where.ts +++ b/packages/sql/query/parser/where.ts @@ -3,7 +3,7 @@ import type { Join, Trim } from "@telefrek/type-utils/strings" import type { ColumnReference } from "../../ast/columns.js" import type { ColumnFilter, - FilteringOperation, + ComparisonOperation, LogicalExpression, LogicalTree, LogicalTreeOperation, @@ -24,7 +24,7 @@ import { type SplitWords, } from "./normalize.js" import type { - GetFilteringOperations, + GetComparisonOperations, GetQuote, ParserOptions, } from "./options.js" @@ -67,15 +67,15 @@ function parseLogicalExpression( options: ParserOptions // TODO: Pass this through for filtering ops ): LogicalExpression { const segments = tokens.join(" ").split(/(?=[>== extends [infer Op extends string, infer Value extends string] - ? Op extends GetFilteringOperations + ? Op extends GetComparisonOperations ? ExtractValue> extends [infer V extends string] ? CheckFilter< ColumnReference>, @@ -229,7 +229,7 @@ type CheckFilter = Left extends ColumnReference< infer Reference, infer Alias > - ? [Operation] extends [FilteringOperation] + ? [Operation] extends [ComparisonOperation] ? Right extends ValueTypes ? ColumnFilter, Operation, Right> : Right extends Invalid @@ -239,3 +239,11 @@ type CheckFilter = Left extends ColumnReference< : Left extends Invalid ? Invalid : Invalid<`Invalid column filter`> + +/** + * Process: WHERE {clause} + * + * Clause can be: + * 1. Column filter: a {filter} b + * 2. Subquery filter: a [NOT] IN (subquery or values) + */ diff --git a/packages/sql/query/visitor/common.ts b/packages/sql/query/visitor/common.ts index cf6e894..128a0ae 100644 --- a/packages/sql/query/visitor/common.ts +++ b/packages/sql/query/visitor/common.ts @@ -165,12 +165,12 @@ export class DefaultQueryVisitor } visitColumnFilter(filter: T): void { - this.visitColumnReference(filter.left) + this.visitColumnReference(filter.column) this.append(filter.op) - if (filter.right.type === "ColumnReference") { - this.visitColumnReference(filter.right) + if (filter.filter.type === "ColumnReference") { + this.visitColumnReference(filter.filter) } else { - this.visitValueType(filter.right) + this.visitValueType(filter.filter) } } From d43347240bd45231cc0579e0761d70ef1081f3bd Mon Sep 17 00:00:00 2001 From: Nathan Northcutt Date: Tue, 20 Aug 2024 13:14:29 +0200 Subject: [PATCH 07/21] working on arithmetic parser as example for expressions --- packages/sql/ast/arithmetic.ts | 43 +++++---- packages/sql/query/builder/where.ts | 13 ++- packages/sql/query/parser/arithmetic.ts | 123 ++++++++++++++++++++++++ packages/sql/query/parser/columns.ts | 22 ++++- packages/sql/query/parser/normalize.ts | 92 ++++++++++-------- packages/sql/query/parser/options.ts | 2 +- packages/sql/query/parser/utils.ts | 47 ++++++++- packages/sql/query/parser/values.ts | 12 ++- packages/sql/query/parser/where.ts | 19 +--- 9 files changed, 286 insertions(+), 87 deletions(-) create mode 100644 packages/sql/query/parser/arithmetic.ts diff --git a/packages/sql/ast/arithmetic.ts b/packages/sql/ast/arithmetic.ts index b3c656e..71cec6e 100644 --- a/packages/sql/ast/arithmetic.ts +++ b/packages/sql/ast/arithmetic.ts @@ -1,3 +1,4 @@ +import type { IgnoreAny } from "@telefrek/type-utils/common" import type { ColumnReference } from "./columns.js" import type { ValueTypes } from "./values.js" @@ -37,10 +38,7 @@ export const DEFAULT_ARITHMETIC_ASSIGNMENT_OPS: ArithmeticAssignmentOperation[] export type ColumnArithmeticAssignment< Column extends ColumnReference = ColumnReference, Op extends string = ArithmeticAssignmentOperation, - Value extends ColumnReference | ValueTypes | ArithmenticExpressionTree = - | ColumnReference - | ValueTypes - | ArithmenticExpressionTree + Value extends ArithmeticExpressionType = ArithmeticExpressionType > = { type: "ColumnArithmeticAssignment" column: Column @@ -49,28 +47,37 @@ export type ColumnArithmeticAssignment< } /** - * An arithmetic expression between two values, ex: a + b + * The default type for an arithmetic expression */ -export type ArithmeticExpression< - Left extends ColumnReference | ValueTypes = ColumnReference | ValueTypes, - Op extends string = ArithmeticOperation, - Right extends ColumnReference | ValueTypes = ColumnReference | ValueTypes +export type ArithmeticExpressionType = + | ColumnReference + | ValueTypes + | ArithmeticExpression + | GroupedArithmeticExpression + +/** + * A grouped expression (surrounded by parenthesis) + */ +export type GroupedArithmeticExpression< + Expression extends ArithmeticExpression< + IgnoreAny, + string, + IgnoreAny + > = ArithmeticExpression > = { - type: "ArithmeticExpression" - left: Left - operation: Op - right: Right + type: "GroupedArithmeticExpression" + expression: Expression } /** - * An arithmetic expression tree + * An arithmetic expression between two values, ex: a + b */ -export type ArithmenticExpressionTree< - Left extends ArithmeticExpression = ArithmeticExpression, +export type ArithmeticExpression< + Left extends ArithmeticExpressionType = ArithmeticExpressionType, Op extends string = ArithmeticOperation, - Right extends ArithmeticExpression = ArithmeticExpression + Right extends ArithmeticExpressionType = ArithmeticExpressionType > = { - type: "ArithmeticExpressionTree" + type: "ArithmeticExpression" left: Left operation: Op right: Right diff --git a/packages/sql/query/builder/where.ts b/packages/sql/query/builder/where.ts index 4521e8a..821c0f8 100644 --- a/packages/sql/query/builder/where.ts +++ b/packages/sql/query/builder/where.ts @@ -132,7 +132,7 @@ export interface WhereClauseBuilder { filter< Column extends QueryContextColumns, Op extends ComparisonOperation, - Value + Value extends string | number | bigint | boolean | null | undefined >( column: Column, op: Op, @@ -144,9 +144,12 @@ export interface WhereClauseBuilder { > } -type CheckColumnRef = Value extends Columns +type CheckColumnRef< + Value extends string | number | bigint | boolean | null | undefined, + Columns extends string +> = Value extends Columns ? ParseColumnReference - : CheckValueType extends infer V extends ValueTypes + : CheckValueType<`${Value}`, "'"> extends infer V extends ValueTypes ? V : never @@ -192,7 +195,7 @@ class DefaultWhereClauseBuilder filter< Column extends QueryContextColumns, Op extends ComparisonOperation, - Value + Value extends string | number | bigint | boolean | null | undefined >( column: Column, op: Op, @@ -219,7 +222,7 @@ function buildFilter< Context extends QueryContext, Column extends string, Operation extends ComparisonOperation, - Value + Value extends string | number | bigint | boolean | null | undefined >( context: Context, column: Column, diff --git a/packages/sql/query/parser/arithmetic.ts b/packages/sql/query/parser/arithmetic.ts new file mode 100644 index 0000000..6a1b940 --- /dev/null +++ b/packages/sql/query/parser/arithmetic.ts @@ -0,0 +1,123 @@ +import type { Invalid } from "@telefrek/type-utils/common" +import type { + ArithmeticExpression, + ArithmeticExpressionType, + ColumnArithmeticAssignment, + GroupedArithmeticExpression, +} from "../../ast/arithmetic.js" +import type { ColumnReference } from "../../ast/columns.js" +import type { ValueTypes } from "../../ast/values.js" +import type { CheckEqualParenthesis, NextToken } from "./normalize.js" +import type { + GetArithmeticOperations, + GetAssignmentOperations, + ParserOptions, +} from "./options.js" +import type { ExtractGroup, ParseValueOrReference } from "./utils.js" + +/** + * Parse an {@link ArithmeticExpression} + */ +export type ParseArithmeticExpression< + SQL extends string, + Options extends ParserOptions +> = ParseNextExpression + +// Keep reading next tokens +type ParseNextExpression< + SQL extends string, + Options extends ParserOptions, + State extends ColumnReference | ValueTypes | ArithmeticExpressionType = never +> = CheckEqualParenthesis extends false + ? Invalid<"unbalanced parenthesis"> + : NextToken extends [ + infer Next extends string, + infer Remainder extends string + ] + ? Next extends GetArithmeticOperations + ? [State] extends [never] + ? Invalid<"Corrupt syntax for operation"> + : ParseNextExpression< + Remainder, + Options, + ArithmeticExpression + > + : Next extends GetAssignmentOperations + ? [State] extends [never] + ? Invalid<"Corrupt syntax for assignment"> + : State extends ColumnReference + ? ParseNextExpression< + Remainder, + Options + > extends infer Exp extends ArithmeticExpressionType + ? ColumnArithmeticAssignment + : Invalid<"Right side of assignment is invalid"> + : Invalid<"Cannot assign to anything other than a column"> + : Next extends ")" + ? Invalid<`Corrupt syntax, extra ')'`> + : Next extends "(" + ? ExtractGroup extends [ + infer Group extends string, + infer Rest extends string + ] + ? ParseNextExpression< + Group, + Options + > extends infer Exp extends ArithmeticExpression + ? [State] extends [never] + ? Rest extends "" + ? GroupedArithmeticExpression + : ParseNextExpression< + Rest, + Options, + GroupedArithmeticExpression + > + : State extends ColumnArithmeticAssignment< + infer Column, + infer Op, + never + > + ? Rest extends "" + ? ColumnArithmeticAssignment< + Column, + Op, + GroupedArithmeticExpression + > + : ParseNextExpression< + Rest, + Options, + GroupedArithmeticExpression + > extends infer Exp2 extends ArithmeticExpressionType + ? ColumnArithmeticAssignment + : Invalid<"Assignment contains invalid partial expression"> + : State extends ArithmeticExpression + ? Rest extends "" + ? ArithmeticExpression> + : ParseNextExpression< + Rest, + Options, + GroupedArithmeticExpression + > extends infer Exp2 extends ArithmeticExpressionType + ? ArithmeticExpression + : Invalid<"Corrupt expression"> + : Invalid<"State is invalid for group"> + : Invalid<"Groups must be arithmetic expressions"> + : Invalid<"Corrupt group"> + : ParseValueOrReference extends infer CRef extends + | ColumnReference + | ValueTypes + ? [State] extends [never] + ? ParseNextExpression + : State extends ColumnReference | ValueTypes + ? Invalid<"Multiple columns or values cannot be in sequence"> + : State extends ArithmeticExpression + ? Remainder extends "" + ? ArithmeticExpression + : ParseNextExpression< + Remainder, + Options, + ArithmeticExpression + > + : Invalid<"Cannot append value or column to fully formed expression"> + : Invalid<`Failed to parse any useful value from ${Next}`> + : Invalid<"Failed to parse expression"> diff --git a/packages/sql/query/parser/columns.ts b/packages/sql/query/parser/columns.ts index 21f5e2c..a579644 100644 --- a/packages/sql/query/parser/columns.ts +++ b/packages/sql/query/parser/columns.ts @@ -1,3 +1,4 @@ +import type { Invalid } from "@telefrek/type-utils/common" import type { ColumnReference, TableColumnReference, @@ -6,6 +7,7 @@ import type { import type { SelectColumns } from "../../ast/select.js" import type { SplitSQL } from "./normalize.js" import type { ParserOptions } from "./options.js" +import type { IsSingleToken } from "./utils.js" import { tryParseAlias } from "./utils.js" /** @@ -13,7 +15,7 @@ import { tryParseAlias } from "./utils.js" */ export type ParseSelectedColumns< Columns extends string, - Options extends ParserOptions, + Options extends ParserOptions > = Columns extends "*" ? Columns : ParseColumns, Options> /** @@ -21,7 +23,7 @@ export type ParseSelectedColumns< */ type ParseColumns = T extends [ infer Column extends string, - ...infer Rest, + ...infer Rest ] ? Rest extends never[] ? [ParseColumnReference] @@ -47,13 +49,23 @@ export function parseSelectedColumns(tokens: string[]): SelectColumns | "*" { return columns.map((c) => parseColumnReference(c.split(" "))) as SelectColumns } +type IsValidReference = IsSingleToken extends true + ? T extends `$${infer _}` + ? Invalid<"Columns cannot start with a $ character"> + : true + : false + /** * Utility type to parse a value as a ColumnReference */ export type ParseColumnReference = T extends `${infer ColumnDetails} AS ${infer Alias}` - ? ColumnReference, Alias> - : ColumnReference> + ? IsSingleToken extends true + ? ColumnReference, Alias> + : Invalid<"Alias cannot contain spaces"> + : IsValidReference extends true + ? ColumnReference> + : Invalid<"Column reference is invalid"> /** * Utility type to parse column details @@ -92,7 +104,7 @@ export function parseColumnReference(tokens: string[]): ColumnReference { * @returns the correct table or unbound reference */ function parseReference( - column: string, + column: string ): TableColumnReference | UnboundColumnReference { // Check for a table reference const idx = column.indexOf(".") diff --git a/packages/sql/query/parser/normalize.ts b/packages/sql/query/parser/normalize.ts index f38b7dc..797f029 100644 --- a/packages/sql/query/parser/normalize.ts +++ b/packages/sql/query/parser/normalize.ts @@ -2,7 +2,7 @@ import type { Invalid } from "@telefrek/type-utils/common.js" import type { Decrement, Increment } from "@telefrek/type-utils/math.js" import type { Join, Trim } from "@telefrek/type-utils/strings.js" import { NORMALIZE_TARGETS } from "./keywords.js" -import type { GetNormalizationTokens, ParserOptions } from "./options.js" +import type { GetOverridableTokens, ParserOptions } from "./options.js" /** * Ensure a query has a known structure with keywords uppercase and consistent spacing @@ -15,47 +15,13 @@ export type NormalizeQuery< ? SplitJoin extends infer Commas extends string ? SplitJoin extends infer OpenParen extends string ? SplitJoin extends infer Normalized extends string - ? NormalizeFilters, Options> + ? NormalizeOverrides, Options> : never : never : never : never : never -type NormalizeFilters< - SQL extends string, - Options extends ParserOptions -> = SplitWords extends infer Tokens extends string[] - ? CleanFilters extends infer Cleaned extends string[] - ? Join - : never - : never - -type CleanFilters = Words extends [ - infer Next extends string, - ...infer Rest -] - ? Rest extends never[] - ? [CheckFilters] - : [CheckFilters, ...CleanFilters] - : never - -type CheckFilters< - SQL extends string, - Options extends ParserOptions, - S extends string = "" -> = SQL extends "" - ? S - : SQL extends `${infer Left}${infer Rest}` - ? Left extends GetNormalizationTokens - ? Rest extends `${infer Right}${infer Remaining}` - ? `${Left}${Right}` extends GetNormalizationTokens - ? Trim<`${Trim} ${Left}${Right} ${CheckFilters}`> - : Trim<`${Trim} ${Left} ${CheckFilters}`> - : Trim<`${Trim} ${Left} ${CheckFilters}`> - : CheckFilters - : Trim<`${S}${SQL}`> - /** * Normalize the values by ensuring capitalization */ @@ -127,12 +93,12 @@ export type SplitSQL< Token extends string = ",", S extends string = "" > = Trim extends `${infer Left} ${Token} ${infer Right}` - ? EqualParenthesis<`${S} ${Left}`> extends true + ? CheckEqualParenthesis<`${S} ${Left}`> extends true ? SplitSQL extends infer Tokens extends string[] ? [Trim<`${S} ${Left}`>, ...Tokens] : Invalid<"Unequal parenthesis"> : SplitSQL> - : EqualParenthesis<`${S} ${T}`> extends true + : CheckEqualParenthesis<`${S} ${T}`> extends true ? [Trim<`${S} ${T}`>] : Invalid<"Unequal parenthesis"> @@ -302,10 +268,58 @@ export function extractParenthesis(tokens: string[]): string[] { return ret } +/** + * Normalize keywords that can be user provided + */ +type NormalizeOverrides< + SQL extends string, + Options extends ParserOptions +> = SplitWords extends infer Tokens extends string[] + ? CleanOverrides extends infer Cleaned extends string[] + ? Join + : never + : never + +/** + * Clean the overrideable strings + */ +type CleanOverrides = Words extends [ + infer Next extends string, + ...infer Rest +] + ? Rest extends never[] + ? [CheckOverrides] + : [CheckOverrides, ...CleanOverrides] + : never + +/** + * Check for user provided overrides + */ +type CheckOverrides< + SQL extends string, + Options extends ParserOptions, + S extends string = "" +> = SQL extends "" + ? S + : SQL extends `${infer Left}${infer Rest}` + ? Left extends GetOverridableTokens + ? Rest extends `${infer Right}${infer Remaining}` + ? `${Left}${Right}` extends GetOverridableTokens + ? Trim<`${Trim} ${Left}${Right} ${CheckOverrides< + Remaining, + Options + >}`> + : Trim<`${Trim} ${Left} ${CheckOverrides}`> + : Trim<`${Trim} ${Left} ${CheckOverrides}`> + : CheckOverrides + : Trim<`${S}${SQL}`> + /** * Test if ( matches ) counts */ -type EqualParenthesis = CountOpen extends CountClosed ? true : false +export type CheckEqualParenthesis = CountOpen extends CountClosed + ? true + : false /** * Count the ( characters diff --git a/packages/sql/query/parser/options.ts b/packages/sql/query/parser/options.ts index 019283f..5b2cd46 100644 --- a/packages/sql/query/parser/options.ts +++ b/packages/sql/query/parser/options.ts @@ -99,7 +99,7 @@ export type GetQuote = /** * Extract all special tokens for normalization */ -export type GetNormalizationTokens = +export type GetOverridableTokens = | GetComparisonOperations | GetAssignmentOperations | GetArithmeticOperations diff --git a/packages/sql/query/parser/utils.ts b/packages/sql/query/parser/utils.ts index 8cf3e4a..b50294e 100644 --- a/packages/sql/query/parser/utils.ts +++ b/packages/sql/query/parser/utils.ts @@ -1,6 +1,12 @@ +import type { Invalid } from "@telefrek/type-utils/common" +import type { Decrement, Increment } from "@telefrek/type-utils/math" +import type { Trim } from "@telefrek/type-utils/strings" import type { ReturningClause } from "../../ast/queries.js" -import { parseSelectedColumns } from "./columns.js" +import type { ValueTypes } from "../../ast/values.js" +import { parseSelectedColumns, type ParseColumnReference } from "./columns.js" +import type { NextToken } from "./normalize.js" import type { GetQuote, ParserOptions } from "./options.js" +import type { CheckValueType } from "./values.js" /** * Parse an optional alias from the stack @@ -17,6 +23,45 @@ export function tryParseAlias(tokens: string[]): string | undefined { return } +/** + * Check if the string represents a single token + */ +export type IsSingleToken = T extends `${infer _} ${infer _}` + ? false + : true + +/** + * Type to try to parse a value and if not fallback and assume it is column reference + */ +export type ParseValueOrReference< + SQL extends string, + Options extends ParserOptions +> = CheckValueType> extends infer V extends ValueTypes + ? V + : ParseColumnReference + +/** + * Extract the next full group from the current string + */ +export type ExtractGroup< + SQL extends string, + N extends number = 1, + S extends string = "" +> = NextToken extends [ + infer Next extends string, + infer Remainder extends string +] + ? Next extends ")" + ? N extends 1 + ? [`${Trim}`, Remainder] + : ExtractGroup, `${S} ${Next}`> + : Next extends "(" + ? ExtractGroup, `${S} ${Next}`> + : Remainder extends "" + ? Invalid<"Unbalanced parenthesis"> + : ExtractGroup + : Invalid<"Unbalanced parenthesis"> + export type RemoveQuotes< S extends string, Options extends ParserOptions diff --git a/packages/sql/query/parser/values.ts b/packages/sql/query/parser/values.ts index ea874e9..64428cb 100644 --- a/packages/sql/query/parser/values.ts +++ b/packages/sql/query/parser/values.ts @@ -15,6 +15,7 @@ import type { } from "../../ast/values.js" import type { NextToken, SplitSQL } from "./normalize.js" import type { GetQuote, ParserOptions } from "./options.js" +import type { IsSingleToken } from "./utils.js" /** * Parse out the value @@ -226,14 +227,21 @@ type Digits = "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" */ // TODO: Possible extension is to check that all characters for numbers are // digits and expand to bigint if over 8 characters by default -export type CheckValueType = T extends `:${infer Name}` - ? ParameterValueType +export type CheckValueType< + T extends string, + Quote extends string +> = T extends `:${infer Name}` + ? IsSingleToken extends true + ? ParameterValueType + : Invalid<"Invalid parameter names cannot contain spaces"> : T extends `$${infer _}` ? Invalid<`index position not supported`> : T extends `${Quote}${infer Contents}${Quote}` ? Contents extends `{${string}}` ? JsonValueType : StringValueType + : IsSingleToken extends false + ? Invalid<"Value cannot contain spaces"> : T extends `0x${infer _}` ? BufferValueType : Lowercase extends "null" diff --git a/packages/sql/query/parser/where.ts b/packages/sql/query/parser/where.ts index 5a347aa..8033126 100644 --- a/packages/sql/query/parser/where.ts +++ b/packages/sql/query/parser/where.ts @@ -10,11 +10,7 @@ import type { WhereClause, } from "../../ast/filtering.js" import type { ValueTypes } from "../../ast/values.js" -import { - parseColumnReference, - type ParseColumnDetails, - type ParseColumnReference, -} from "./columns.js" +import { parseColumnReference, type ParseColumnDetails } from "./columns.js" import type { PartialParserResult } from "./common.js" import { takeUntil, @@ -28,7 +24,8 @@ import type { GetQuote, ParserOptions, } from "./options.js" -import { parseValue, type CheckValueType, type ExtractValue } from "./values.js" +import type { ParseValueOrReference } from "./utils.js" +import { parseValue, type ExtractValue } from "./values.js" // This entire thing needs a re-write... @@ -212,16 +209,6 @@ type ParseColumnFilter< : Invalid<`Failed to parse column filter: ${SQL & string}`> : Invalid<`Failed to parse column filter: ${SQL & string}`> -/** - * Type to try to parse a value and if not fallback and assume it is column reference - */ -type ParseValueOrReference< - SQL extends string, - Options extends ParserOptions -> = CheckValueType> extends infer V extends ValueTypes - ? V - : ParseColumnReference - /** * Check that the column filter is appropriate and well formed */ From 7585dd690f431d4eceb4bab7157018e6c4c7a890 Mon Sep 17 00:00:00 2001 From: Nathan Northcutt Date: Tue, 20 Aug 2024 13:59:16 +0200 Subject: [PATCH 08/21] making some changes as I test alternate operations and options --- packages/sql/query/parser/arithmetic.ts | 28 ++++++++++++---------- packages/sql/query/parser/normalize.ts | 32 ++++++++++++++++--------- packages/sql/query/parser/options.ts | 22 ++++++++++++----- 3 files changed, 53 insertions(+), 29 deletions(-) diff --git a/packages/sql/query/parser/arithmetic.ts b/packages/sql/query/parser/arithmetic.ts index 6a1b940..39620db 100644 --- a/packages/sql/query/parser/arithmetic.ts +++ b/packages/sql/query/parser/arithmetic.ts @@ -1,29 +1,33 @@ import type { Invalid } from "@telefrek/type-utils/common" -import type { - ArithmeticExpression, - ArithmeticExpressionType, - ColumnArithmeticAssignment, - GroupedArithmeticExpression, +import { + type ArithmeticExpression, + type ArithmeticExpressionType, + type ColumnArithmeticAssignment, + type GroupedArithmeticExpression, } from "../../ast/arithmetic.js" import type { ColumnReference } from "../../ast/columns.js" import type { ValueTypes } from "../../ast/values.js" import type { CheckEqualParenthesis, NextToken } from "./normalize.js" -import type { - GetArithmeticOperations, - GetAssignmentOperations, - ParserOptions, +import { + type GetArithmeticOperations, + type GetAssignmentOperations, + type ParserOptions, } from "./options.js" import type { ExtractGroup, ParseValueOrReference } from "./utils.js" /** - * Parse an {@link ArithmeticExpression} + * Parse an {@link ArithmeticExpression} or {@link ColumnArithmeticAssignment} + * from the given SQL string */ export type ParseArithmeticExpression< SQL extends string, Options extends ParserOptions > = ParseNextExpression -// Keep reading next tokens +/** + * Recursive call to build the current state of the expression handling groups, + * assignments and customized parsing + */ type ParseNextExpression< SQL extends string, Options extends ParserOptions, @@ -52,7 +56,7 @@ type ParseNextExpression< > extends infer Exp extends ArithmeticExpressionType ? ColumnArithmeticAssignment : Invalid<"Right side of assignment is invalid"> - : Invalid<"Cannot assign to anything other than a column"> + : [State, Next, Remainder] // Invalid<`Cannot assign to anything other than a column`> : Next extends ")" ? Invalid<`Corrupt syntax, extra ')'`> : Next extends "(" diff --git a/packages/sql/query/parser/normalize.ts b/packages/sql/query/parser/normalize.ts index 797f029..71b9f95 100644 --- a/packages/sql/query/parser/normalize.ts +++ b/packages/sql/query/parser/normalize.ts @@ -301,18 +301,28 @@ type CheckOverrides< S extends string = "" > = SQL extends "" ? S + : GetLongestOverride extends infer Override extends string + ? Override extends "" + ? SQL extends `${infer Left}${infer Rest}` + ? CheckOverrides + : Trim<`${S}${SQL}`> + : SQL extends `${Override}${infer Rest}` + ? Trim<`${S} ${Override} ${CheckOverrides}`> + : SQL + : SQL + +type GetLongestOverride< + SQL extends string, + Options extends ParserOptions, + Prefix extends string = "", + L extends string = "" +> = SQL extends "" + ? L : SQL extends `${infer Left}${infer Rest}` - ? Left extends GetOverridableTokens - ? Rest extends `${infer Right}${infer Remaining}` - ? `${Left}${Right}` extends GetOverridableTokens - ? Trim<`${Trim} ${Left}${Right} ${CheckOverrides< - Remaining, - Options - >}`> - : Trim<`${Trim} ${Left} ${CheckOverrides}`> - : Trim<`${Trim} ${Left} ${CheckOverrides}`> - : CheckOverrides - : Trim<`${S}${SQL}`> + ? `${Prefix}${Left}` extends GetOverridableTokens + ? GetLongestOverride + : GetLongestOverride + : L /** * Test if ( matches ) counts diff --git a/packages/sql/query/parser/options.ts b/packages/sql/query/parser/options.ts index 5b2cd46..222f233 100644 --- a/packages/sql/query/parser/options.ts +++ b/packages/sql/query/parser/options.ts @@ -14,7 +14,12 @@ import { * The options for what can be overridden in the parsing logic */ export type ParserOptions< - Tokens extends SyntaxTokens = SyntaxTokens, + Tokens extends SyntaxTokens = SyntaxTokens< + string, + string, + string, + string + >, Features extends ParsingFeatures = ParsingFeatures > = { tokens: Tokens @@ -151,10 +156,15 @@ export type GetAssignmentOperations = /** * Merge the partial tokens with the default tokens */ -type MergeTokens> = Flatten< - Tokens & Omit -> extends SyntaxTokens - ? SyntaxTokens +type MergeTokens< + Tokens extends Partial> +> = Flatten> extends SyntaxTokens< + infer Quote, + infer Comparisons, + infer Assignments, + infer Arithmetic +> + ? SyntaxTokens : never /** @@ -165,7 +175,7 @@ type MergeTokens> = Flatten< * @returns A new set of {@link ParserOptions} to use */ export function createParsingOptions< - const Tokens extends Partial, + const Tokens extends Partial>, Features extends ParsingFeatures >( tokens: Tokens, From 43539366ad00c479d244e9f39fc8755b300fac39 Mon Sep 17 00:00:00 2001 From: Nathan Northcutt Date: Wed, 21 Aug 2024 15:51:01 +0200 Subject: [PATCH 09/21] refactoring arithmetic expressions for more completeness, still need to allow partial consumption of next valid assignment --- packages/sql/query/parser/arithmetic.ts | 321 ++++++++++++++++-------- packages/sql/query/parser/filtering.ts | 43 ++++ 2 files changed, 265 insertions(+), 99 deletions(-) create mode 100644 packages/sql/query/parser/filtering.ts diff --git a/packages/sql/query/parser/arithmetic.ts b/packages/sql/query/parser/arithmetic.ts index 39620db..4e07135 100644 --- a/packages/sql/query/parser/arithmetic.ts +++ b/packages/sql/query/parser/arithmetic.ts @@ -1,13 +1,12 @@ -import type { Invalid } from "@telefrek/type-utils/common" +import type { IgnoreAny, Invalid } from "@telefrek/type-utils/common" import { type ArithmeticExpression, - type ArithmeticExpressionType, type ColumnArithmeticAssignment, type GroupedArithmeticExpression, } from "../../ast/arithmetic.js" import type { ColumnReference } from "../../ast/columns.js" import type { ValueTypes } from "../../ast/values.js" -import type { CheckEqualParenthesis, NextToken } from "./normalize.js" +import type { NextToken } from "./normalize.js" import { type GetArithmeticOperations, type GetAssignmentOperations, @@ -16,112 +15,236 @@ import { import type { ExtractGroup, ParseValueOrReference } from "./utils.js" /** - * Parse an {@link ArithmeticExpression} or {@link ColumnArithmeticAssignment} - * from the given SQL string + * Get the types of tokens supported */ -export type ParseArithmeticExpression< - SQL extends string, - Options extends ParserOptions -> = ParseNextExpression +type GetTokenTypes = + | GetArithmeticOperations + | GetAssignmentOperations + | ColumnReference + | ValueTypes /** - * Recursive call to build the current state of the expression handling groups, - * assignments and customized parsing + * Get the next token value */ -type ParseNextExpression< +type ReadNextToken< SQL extends string, - Options extends ParserOptions, - State extends ColumnReference | ValueTypes | ArithmeticExpressionType = never -> = CheckEqualParenthesis extends false - ? Invalid<"unbalanced parenthesis"> - : NextToken extends [ - infer Next extends string, - infer Remainder extends string - ] - ? Next extends GetArithmeticOperations - ? [State] extends [never] - ? Invalid<"Corrupt syntax for operation"> - : ParseNextExpression< - Remainder, - Options, - ArithmeticExpression - > - : Next extends GetAssignmentOperations - ? [State] extends [never] - ? Invalid<"Corrupt syntax for assignment"> - : State extends ColumnReference - ? ParseNextExpression< - Remainder, - Options - > extends infer Exp extends ArithmeticExpressionType - ? ColumnArithmeticAssignment - : Invalid<"Right side of assignment is invalid"> - : [State, Next, Remainder] // Invalid<`Cannot assign to anything other than a column`> - : Next extends ")" - ? Invalid<`Corrupt syntax, extra ')'`> - : Next extends "(" + Options extends ParserOptions +> = NextToken extends [ + infer Token extends string, + infer Remainder extends string +] + ? Token extends GetArithmeticOperations + ? [Token, Remainder] + : Token extends GetAssignmentOperations + ? [Token, Remainder] + : Token extends ")" + ? Invalid<"Invalid syntax, extra )"> + : Token extends "(" ? ExtractGroup extends [ infer Group extends string, infer Rest extends string ] - ? ParseNextExpression< - Group, - Options - > extends infer Exp extends ArithmeticExpression - ? [State] extends [never] - ? Rest extends "" - ? GroupedArithmeticExpression - : ParseNextExpression< - Rest, - Options, - GroupedArithmeticExpression - > - : State extends ColumnArithmeticAssignment< - infer Column, - infer Op, - never - > - ? Rest extends "" - ? ColumnArithmeticAssignment< - Column, - Op, - GroupedArithmeticExpression - > - : ParseNextExpression< - Rest, - Options, - GroupedArithmeticExpression - > extends infer Exp2 extends ArithmeticExpressionType - ? ColumnArithmeticAssignment - : Invalid<"Assignment contains invalid partial expression"> - : State extends ArithmeticExpression - ? Rest extends "" - ? ArithmeticExpression> - : ParseNextExpression< - Rest, - Options, - GroupedArithmeticExpression - > extends infer Exp2 extends ArithmeticExpressionType - ? ArithmeticExpression - : Invalid<"Corrupt expression"> - : Invalid<"State is invalid for group"> - : Invalid<"Groups must be arithmetic expressions"> + ? [Group, Rest] : Invalid<"Corrupt group"> - : ParseValueOrReference extends infer CRef extends + : ParseValueOrReference extends infer CRef extends | ColumnReference | ValueTypes - ? [State] extends [never] - ? ParseNextExpression - : State extends ColumnReference | ValueTypes - ? Invalid<"Multiple columns or values cannot be in sequence"> - : State extends ArithmeticExpression + ? [CRef, Remainder] + : Invalid<"Cannot map value"> + : Invalid<"No more tokens to extract"> + +/** + * Parse the next {@link ArithmeticExpression} from the provided string + */ +type ParseNextArithmeticExpression< + SQL extends string, + Options extends ParserOptions +> = ReadNextToken extends [ + infer Token, + infer Remainder extends string +] + ? Token extends ColumnReference + ? ParseColumnExpression + : Token extends ValueTypes + ? ParseValueExpression + : Token extends GetTokenTypes + ? Invalid<"Cannot start an operation with an assignment or arithmetic sign"> + : Token extends string + ? ParseEntireArithmeticTree< + Token, + Options + > extends infer Tree extends ArithmeticExpression< + IgnoreAny, + string, + IgnoreAny + > + ? [GroupedArithmeticExpression, Remainder] + : ParseEntireArithmeticTree + : Invalid<"Invalid token"> + : ReadNextToken + +/** + * Parse expressions starting with a column + */ +type ParseColumnExpression< + SQL extends string, + Options extends ParserOptions, + Column extends ColumnReference +> = ReadNextToken extends [ + infer Token, + infer Remainder extends string +] + ? Token extends GetArithmeticOperations + ? ParseSingleArithmeticExpression< + Remainder, + Options, + ArithmeticExpression + > + : Token extends GetAssignmentOperations + ? ParseColumnAssignmentExpression< + Remainder, + Options, + ColumnArithmeticAssignment + > + : Invalid<"Column must be followed by an assignment or arithmetic operation"> + : ReadNextToken + +/** + * Parse expressions starting with a value + */ +type ParseValueExpression< + SQL extends string, + Options extends ParserOptions, + Value extends ValueTypes +> = ReadNextToken extends [ + infer Token, + infer Remainder extends string +] + ? Token extends GetArithmeticOperations + ? [ArithmeticExpression, Remainder] + : Invalid<"Value must be followed by an arithmetic operation"> + : ReadNextToken + +/** + * Parse only the next segment + */ +type ParseSingleArithmeticExpression< + SQL extends string, + Options extends ParserOptions, + Expression extends ArithmeticExpression +> = ReadNextToken extends [ + infer Token, + infer Remainder extends string +] + ? Token extends ColumnReference | ValueTypes + ? Expression extends ArithmeticExpression + ? [ArithmeticExpression, Remainder] + : Invalid<"Corrupt expression"> + : Token extends GetTokenTypes + ? Invalid<"Right hand side of expression must be a value, column or other expression"> + : Token extends string + ? ParseEntireArithmeticTree< + Token, + Options + > extends infer Right extends ArithmeticExpression< + IgnoreAny, + string, + IgnoreAny + > + ? Expression extends ArithmeticExpression + ? [ + ArithmeticExpression>, + Remainder + ] + : Invalid<"Corrupted expression"> + : ParseEntireArithmeticTree + : Invalid<"Next token is not valid"> + : ReadNextToken + +/** + * Parse a column assignment + * + * Note: To be valid, the entire remainder must be consumable... + */ +type ParseColumnAssignmentExpression< + SQL extends string, + Options extends ParserOptions, + Assignment extends ColumnArithmeticAssignment +> = ReadNextToken extends [ + infer Token, + infer Remainder extends string +] + ? Token extends ValueTypes | ColumnReference + ? Assignment extends ColumnArithmeticAssignment< + infer Column, + infer Op, + infer _ + > ? Remainder extends "" - ? ArithmeticExpression - : ParseNextExpression< - Remainder, - Options, - ArithmeticExpression - > - : Invalid<"Cannot append value or column to fully formed expression"> - : Invalid<`Failed to parse any useful value from ${Next}`> - : Invalid<"Failed to parse expression"> + ? ColumnArithmeticAssignment + : Invalid<"Cannot have assignment with trailing information"> + : Invalid<"Corrupt assignment"> + : Token extends GetTokenTypes + ? Invalid<"Right hand side of assignment must be a value, column or other expression"> + : Token extends string + ? ParseEntireArithmeticTree< + Token, + Options + > extends infer Expression extends AnyExpression + ? Assignment extends ColumnArithmeticAssignment< + infer Column, + infer Op, + infer _ + > + ? ColumnArithmeticAssignment + : Invalid<"Corrupted column assignment"> + : ParseEntireArithmeticTree + : Invalid<"Invalid grouping in column assignment"> + : ReadNextToken + +/** + * Type to prevent assumption about operations from causing mismatch + */ +type AnyExpression = + | ArithmeticExpression + | GroupedArithmeticExpression + +/** + * Consume the entire arithmetic tree + */ +type ParseEntireArithmeticTree< + SQL extends string, + Options extends ParserOptions, + Current extends AnyExpression = never +> = [Current] extends [never] + ? ParseNextArithmeticExpression extends [ + infer Expression extends AnyExpression, + infer Remainder extends string + ] + ? Remainder extends "" + ? Expression + : ParseEntireArithmeticTree + : ParseNextArithmeticExpression + : ReadNextToken extends [ + infer Token, + infer Remainder extends string + ] + ? Token extends GetArithmeticOperations + ? ParseSingleArithmeticExpression< + Remainder, + Options, + ArithmeticExpression + > extends [ + infer Expression extends AnyExpression, + infer Rest extends string + ] + ? Rest extends "" + ? Expression + : ParseEntireArithmeticTree + : ParseSingleArithmeticExpression< + Remainder, + Options, + ArithmeticExpression + > + : [SQL, Token, Remainder, Current] + : ReadNextToken diff --git a/packages/sql/query/parser/filtering.ts b/packages/sql/query/parser/filtering.ts new file mode 100644 index 0000000..64c5340 --- /dev/null +++ b/packages/sql/query/parser/filtering.ts @@ -0,0 +1,43 @@ +import type { Invalid } from "@telefrek/type-utils/common" +import type { ColumnReference } from "../../ast/columns.js" +import type { LogicalExpression } from "../../ast/filtering.js" +import type { ValueTypes } from "../../ast/values.js" +import type { CheckEqualParenthesis, NextToken } from "./normalize.js" +import type { GetComparisonOperations, ParserOptions } from "./options.js" +import type { ExtractGroup, ParseValueOrReference } from "./utils.js" + +/** + * Parser for logical expressions + */ +export type ParseLogicalExpression< + SQL extends string, + Options extends ParserOptions +> = ParseNextExpression + +type ParseNextExpression< + SQL extends string, + Options extends ParserOptions, + _State extends LogicalExpression = never +> = CheckEqualParenthesis extends false + ? Invalid<"unbalanced parenthesis"> + : NextToken extends [ + infer Token extends string, + infer Remainder extends string + ] + ? Token extends GetComparisonOperations + ? `Comparison: ${Token}` + : Token extends ")" + ? Invalid<"Corrupt syntax, extra )"> + : Token extends "(" + ? ExtractGroup extends [ + infer Group extends string, + infer _Rest extends string + ] + ? Group + : Invalid<"corrupt group"> + : ParseValueOrReference extends infer CRef extends + | ColumnReference + | ValueTypes + ? CRef + : Token + : Invalid<"failed to parse expression"> From f482da2ed04d3c944dbc7172eb8d279c73420c27 Mon Sep 17 00:00:00 2001 From: Nathan Northcutt Date: Thu, 22 Aug 2024 11:26:25 +0200 Subject: [PATCH 10/21] parse the next arithmetic chunk and return the remainder as the main entrypoint --- packages/sql/query/parser/arithmetic.ts | 42 ++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/packages/sql/query/parser/arithmetic.ts b/packages/sql/query/parser/arithmetic.ts index 4e07135..6538807 100644 --- a/packages/sql/query/parser/arithmetic.ts +++ b/packages/sql/query/parser/arithmetic.ts @@ -14,6 +14,46 @@ import { } from "./options.js" import type { ExtractGroup, ParseValueOrReference } from "./utils.js" +/** + * Extract the next valid expression chunk and the remaining string + */ +export type ParseArithmeticExpression< + SQL extends string, + Options extends ParserOptions, + Current extends AnyExpression = never +> = [Current] extends [never] + ? ParseNextArithmeticExpression extends [ + infer Expression extends AnyExpression, + infer Remainder extends string + ] + ? Remainder extends "" + ? [Expression, ""] + : ParseArithmeticExpression + : ParseNextArithmeticExpression + : ReadNextToken extends [ + infer Token, + infer Remainder extends string + ] + ? Token extends GetArithmeticOperations + ? ParseSingleArithmeticExpression< + Remainder, + Options, + ArithmeticExpression + > extends [ + infer Expression extends AnyExpression, + infer Rest extends string + ] + ? Rest extends "" + ? [Expression, ""] + : ParseArithmeticExpression + : ParseSingleArithmeticExpression< + Remainder, + Options, + ArithmeticExpression + > + : [Current, SQL] // Return + : [Current, SQL] // Return expression and remainder + /** * Get the types of tokens supported */ @@ -246,5 +286,5 @@ type ParseEntireArithmeticTree< Options, ArithmeticExpression > - : [SQL, Token, Remainder, Current] + : Invalid<"Failed to consume entire segment as arithmetic operation"> : ReadNextToken From 317c6f8edc6ad7aae61566ac05e38e2882974a02 Mon Sep 17 00:00:00 2001 From: Nathan Northcutt Date: Thu, 22 Aug 2024 11:28:42 +0200 Subject: [PATCH 11/21] refactor to cleanup --- packages/sql/query/parser/arithmetic.ts | 43 ++++++------------------- 1 file changed, 9 insertions(+), 34 deletions(-) diff --git a/packages/sql/query/parser/arithmetic.ts b/packages/sql/query/parser/arithmetic.ts index 6538807..76e5b61 100644 --- a/packages/sql/query/parser/arithmetic.ts +++ b/packages/sql/query/parser/arithmetic.ts @@ -254,37 +254,12 @@ type AnyExpression = */ type ParseEntireArithmeticTree< SQL extends string, - Options extends ParserOptions, - Current extends AnyExpression = never -> = [Current] extends [never] - ? ParseNextArithmeticExpression extends [ - infer Expression extends AnyExpression, - infer Remainder extends string - ] - ? Remainder extends "" - ? Expression - : ParseEntireArithmeticTree - : ParseNextArithmeticExpression - : ReadNextToken extends [ - infer Token, - infer Remainder extends string - ] - ? Token extends GetArithmeticOperations - ? ParseSingleArithmeticExpression< - Remainder, - Options, - ArithmeticExpression - > extends [ - infer Expression extends AnyExpression, - infer Rest extends string - ] - ? Rest extends "" - ? Expression - : ParseEntireArithmeticTree - : ParseSingleArithmeticExpression< - Remainder, - Options, - ArithmeticExpression - > - : Invalid<"Failed to consume entire segment as arithmetic operation"> - : ReadNextToken + Options extends ParserOptions +> = ParseArithmeticExpression extends [ + infer Expression, + infer Remainder extends string +] + ? Remainder extends "" + ? Expression + : Invalid<"Failed to consume the entire SQL"> + : ParseArithmeticExpression From b75cf9a71af05b49eefb9bbe64bc4d0b81b176cc Mon Sep 17 00:00:00 2001 From: Nathan Northcutt Date: Fri, 23 Aug 2024 12:45:18 +0200 Subject: [PATCH 12/21] adding options to all builders, refactoring parse value to read potential chunks --- packages/sql/query/builder/from.ts | 2 +- packages/sql/query/builder/insert.ts | 24 ++++-- packages/sql/query/builder/select.ts | 54 ++++++++---- packages/sql/query/builder/where.ts | 90 ++++++++++++-------- packages/sql/query/parser/arithmetic.ts | 36 ++++++++ packages/sql/query/parser/insert.ts | 4 +- packages/sql/query/parser/utils.ts | 24 +++++- packages/sql/query/parser/values.ts | 104 ++++++++++++++++++++++++ 8 files changed, 280 insertions(+), 58 deletions(-) diff --git a/packages/sql/query/builder/from.ts b/packages/sql/query/builder/from.ts index e66faa0..4aaf8b2 100644 --- a/packages/sql/query/builder/from.ts +++ b/packages/sql/query/builder/from.ts @@ -94,6 +94,6 @@ class DefaultFromQueryBuilder< > } - return createSelectedColumnsBuilder(context, reference) + return createSelectedColumnsBuilder(context, reference, this._options) } } diff --git a/packages/sql/query/builder/insert.ts b/packages/sql/query/builder/insert.ts index 0f2c886..06739e3 100644 --- a/packages/sql/query/builder/insert.ts +++ b/packages/sql/query/builder/insert.ts @@ -17,7 +17,7 @@ import type { } from "../context.js" import type { ParserOptions } from "../parser/options.js" import type { ParseTableReference } from "../parser/table.js" -import { parseValue, type ExtractTSValueTypes } from "../parser/values.js" +import { pv, type ExtractTSValueTypes } from "../parser/values.js" import { createReturningBuilder, type ReturningBuilder } from "./returning.js" import { buildColumnReference, type VerifyColumnReferences } from "./select.js" import { buildTableReference } from "./table.js" @@ -131,7 +131,8 @@ class DefaultInsertIntoBuilder< > { return new DefaultColumnValueBuilder( buildTableReference(table, this._options), - [] + [], + this._options ) } } @@ -147,10 +148,12 @@ class DefaultColumnValueBuilder< { private _table: Table private _columns: Columns + private _options: ParserOptions - constructor(table: Table, columns: Columns) { + constructor(table: Table, columns: Columns, options: ParserOptions) { this._table = table this._columns = columns + this._options = options } columns>[]>( @@ -163,7 +166,7 @@ class DefaultColumnValueBuilder< buildColumnReference(c) ) as unknown as VerifyColumnReferences - return new DefaultColumnValueBuilder(this._table, verified) + return new DefaultColumnValueBuilder(this._table, verified, this._options) } values>( @@ -173,7 +176,18 @@ class DefaultColumnValueBuilder< type: "InsertClause", table: this._table, columns: this._columns, - values: (values as unknown[]).map((v) => parseValue(String(v))) as Values, + values: (values as unknown[]).map((v) => + pv( + [ + String( + typeof v === "string" + ? `${this._options.tokens.quote}${v}${this._options.tokens.quote}` + : v + ), + ], + this._options + ) + ) as Values, }) } } diff --git a/packages/sql/query/builder/select.ts b/packages/sql/query/builder/select.ts index cee1cf9..dc078a4 100644 --- a/packages/sql/query/builder/select.ts +++ b/packages/sql/query/builder/select.ts @@ -16,6 +16,7 @@ import { } from "../common.js" import type { GetSelectableColumns, QueryContext } from "../context.js" import type { ParseColumnReference } from "../parser/columns.js" +import type { ParserOptions } from "../parser/options.js" import { where, type WhereBuilder } from "./where.js" /** @@ -23,7 +24,8 @@ import { where, type WhereBuilder } from "./where.js" */ export interface SelectedColumnsBuilder< Context extends QueryContext = QueryContext, - Table extends TableReference = TableReference + Table extends TableReference = TableReference, + Options extends ParserOptions = ParserOptions > extends QueryAST> { /** * Choose the columns that we want to include in the select @@ -32,7 +34,11 @@ export interface SelectedColumnsBuilder< */ columns>[]>( ...columns: AtLeastOne - ): WhereBuilder, Table>> + ): WhereBuilder< + Context, + SelectClause, Table>, + Options + > } /** @@ -43,9 +49,14 @@ export interface SelectedColumnsBuilder< */ export function createSelectedColumnsBuilder< Context extends QueryContext, - Table extends TableReference ->(context: Context, from: Table): SelectedColumnsBuilder { - return new DefaultSelectedColumnsBuilder(context, from) + Table extends TableReference, + Options extends ParserOptions +>( + context: Context, + from: Table, + options: Options +): SelectedColumnsBuilder { + return new DefaultSelectedColumnsBuilder(context, from, options) } /** @@ -54,15 +65,18 @@ export function createSelectedColumnsBuilder< class DefaultSelectedColumnsBuilder< Database extends SQLDatabaseSchema = SQLDatabaseSchema, Context extends QueryContext = QueryContext, - Table extends TableReference = TableReference -> implements SelectedColumnsBuilder + Table extends TableReference = TableReference, + Options extends ParserOptions = ParserOptions +> implements SelectedColumnsBuilder { private _context: Context private _from: Table + private _options: Options - constructor(context: Context, from: Table) { + constructor(context: Context, from: Table, options: Options) { this._context = context this._from = from + this._options = options } get ast(): SQLQuery> { @@ -78,14 +92,22 @@ class DefaultSelectedColumnsBuilder< columns>[]>( ...columns: AtLeastOne - ): WhereBuilder, Table>> { - return where(this._context, { - type: "SelectClause", - from: this._from, - columns: [ - ...columns.map((r) => buildColumnReference(r as unknown as string)), - ] as VerifySelectColumns, - }) + ): WhereBuilder< + Context, + SelectClause, Table>, + Options + > { + return where( + this._context, + { + type: "SelectClause", + from: this._from, + columns: [ + ...columns.map((r) => buildColumnReference(r as unknown as string)), + ] as VerifySelectColumns, + }, + this._options + ) } } diff --git a/packages/sql/query/builder/where.ts b/packages/sql/query/builder/where.ts index 821c0f8..dcb673a 100644 --- a/packages/sql/query/builder/where.ts +++ b/packages/sql/query/builder/where.ts @@ -25,7 +25,8 @@ import type { QueryContextColumns, } from "../context.js" import type { ParseColumnReference } from "../parser/columns.js" -import { type CheckValueType, parseValue } from "../parser/values.js" +import type { GetQuote, ParserOptions } from "../parser/options.js" +import { type CheckValueType, pv } from "../parser/values.js" import { buildColumnReference } from "./select.js" /** @@ -35,11 +36,16 @@ import { buildColumnReference } from "./select.js" * @param query The current query * @returns A {@link WhereBuilder} */ -export function where( +export function where< + Context extends QueryContext, + Query extends QueryClause, + Options extends ParserOptions +>( context: Context, - query: Query -): WhereBuilder { - return new DefaultWhereBuilder(context, query) + query: Query, + options: Options +): WhereBuilder { + return new DefaultWhereBuilder(context, query, options) } /** @@ -47,7 +53,8 @@ export function where( */ export interface WhereBuilder< Context extends QueryContext, - Query extends QueryClause + Query extends QueryClause, + Options extends ParserOptions > extends QueryAST { /** * Create a where clause @@ -55,7 +62,7 @@ export interface WhereBuilder< * @param builder The clause builder */ where( - builder: (w: WhereClauseBuilder) => Exp + builder: (w: WhereClauseBuilder) => Exp ): AddWhereToAST } @@ -64,26 +71,29 @@ export interface WhereBuilder< */ class DefaultWhereBuilder< Context extends QueryContext, - Query extends QueryClause -> implements WhereBuilder + Query extends QueryClause, + Options extends ParserOptions +> implements WhereBuilder { private _context: Context private _query: Query + private _options: Options - constructor(context: Context, query: Query) { + constructor(context: Context, query: Query, options: Options) { this._context = context this._query = query + this._options = options } where( - builder: (w: WhereClauseBuilder) => Exp + builder: (w: WhereClauseBuilder) => Exp ): AddWhereToAST { return { ast: { type: "SQLQuery", query: { ...this._query, - where: builder(whereClause(this._context)), + where: builder(whereClause(this._context, this._options)), }, }, } as AddWhereToAST @@ -118,7 +128,10 @@ type RefType = C extends `${infer Table}.${infer Column}` ? ColumnReference> : ColumnReference> -export interface WhereClauseBuilder { +export interface WhereClauseBuilder< + Context extends QueryContext, + Options extends ParserOptions +> { and( left: Left, right: Right @@ -140,32 +153,41 @@ export interface WhereClauseBuilder { ): ColumnFilter< RefType, Op, - CheckColumnRef> + CheckColumnRef, Options> > } type CheckColumnRef< Value extends string | number | bigint | boolean | null | undefined, - Columns extends string + Columns extends string, + Options extends ParserOptions > = Value extends Columns ? ParseColumnReference - : CheckValueType<`${Value}`, "'"> extends infer V extends ValueTypes + : CheckValueType< + `${Value}`, + GetQuote + > extends infer V extends ValueTypes ? V : never -export function whereClause( - context: Context -): WhereClauseBuilder { - return new DefaultWhereClauseBuilder(context) +export function whereClause< + Context extends QueryContext, + Options extends ParserOptions +>(context: Context, options: Options): WhereClauseBuilder { + return new DefaultWhereClauseBuilder(context, options) } -class DefaultWhereClauseBuilder - implements WhereClauseBuilder +class DefaultWhereClauseBuilder< + Context extends QueryContext, + Options extends ParserOptions +> implements WhereClauseBuilder { private _context: Context + private _options: Options - constructor(context: Context) { + constructor(context: Context, options: Options) { this._context = context + this._options = options } and( @@ -203,17 +225,18 @@ class DefaultWhereClauseBuilder ): ColumnFilter< RefType, Op, - CheckColumnRef> + CheckColumnRef, Options> > { - return buildFilter( + return buildFilter( this._context, column, op, - value as Value + value as Value, + this._options ) as unknown as ColumnFilter< RefType, Op, - CheckColumnRef> + CheckColumnRef, Options> > } } @@ -222,18 +245,20 @@ function buildFilter< Context extends QueryContext, Column extends string, Operation extends ComparisonOperation, - Value extends string | number | bigint | boolean | null | undefined + Value extends string | number | bigint | boolean | null | undefined, + Options extends ParserOptions >( context: Context, column: Column, op: Operation, - value: Value + value: Value, + options: Options ): ColumnFilter< Column extends `${infer Table}.${infer Col}` ? ColumnReference, Col> : ColumnReference, Column>, Operation, - CheckColumnRef> + CheckColumnRef, Options> > { return { type: "ColumnFilter", @@ -247,9 +272,10 @@ function buildFilter< } : isColumn(context, value) ? buildColumnReference(value as string) - : parseValue(String(value))) as CheckColumnRef< + : pv(String(value).split(" "), options)) as CheckColumnRef< Value, - QueryContextColumns + QueryContextColumns, + Options >, } } diff --git a/packages/sql/query/parser/arithmetic.ts b/packages/sql/query/parser/arithmetic.ts index 76e5b61..4170762 100644 --- a/packages/sql/query/parser/arithmetic.ts +++ b/packages/sql/query/parser/arithmetic.ts @@ -14,6 +14,20 @@ import { } from "./options.js" import type { ExtractGroup, ParseValueOrReference } from "./utils.js" +// export function parseArithmeticExpression( +// tokens: string[], +// options: ParserOptions +// ): AnyExpression | undefined { +// return +// } + +// function parseNextArithmeticExpression( +// tokens: string[], +// options: ParserOptions +// ): AnyExpression | undefined { +// return +// } + /** * Extract the next valid expression chunk and the remaining string */ @@ -93,6 +107,28 @@ type ReadNextToken< : Invalid<"Cannot map value"> : Invalid<"No more tokens to extract"> +export function readNextToken( + tokens: string, + options: ParserOptions +): ValueTypes | ColumnReference | string[] { + if (tokens.length === 0) { + throw new Error("No more tokens to extract") + } + + switch (true) { + case tokens[0] === ")": + throw new Error("Corrupt group") + case tokens[0] === "(": + break + case options.tokens.arithmetic.indexOf(tokens[0]) >= 0: + break + case options.tokens.assignments.indexOf(tokens[0]) >= 0: + break + } + + throw new Error("Cannot map value") +} + /** * Parse the next {@link ArithmeticExpression} from the provided string */ diff --git a/packages/sql/query/parser/insert.ts b/packages/sql/query/parser/insert.ts index d4c4d29..a281bee 100644 --- a/packages/sql/query/parser/insert.ts +++ b/packages/sql/query/parser/insert.ts @@ -17,7 +17,7 @@ import type { ExtractReturning } from "./returning.js" import type { ParseSelect } from "./select.js" import { parseTableReference, type ParseTableReference } from "./table.js" import { tryParseReturning } from "./utils.js" -import { parseValue, type ParseValues } from "./values.js" +import { pv, type ParseValues } from "./values.js" /** * Parse an insert clause @@ -191,7 +191,7 @@ function parseValuesOrSelect( return extractParenthesis(tokens) .join(" ") .split(" , ") - .map((v) => parseValue(v.trim())) as ValueTypes[] + .map((v) => pv(v.trim().split(" "), options)) as ValueTypes[] } const subquery = parseQueryClause(tokens, options) diff --git a/packages/sql/query/parser/utils.ts b/packages/sql/query/parser/utils.ts index b50294e..dc0a583 100644 --- a/packages/sql/query/parser/utils.ts +++ b/packages/sql/query/parser/utils.ts @@ -1,12 +1,17 @@ import type { Invalid } from "@telefrek/type-utils/common" import type { Decrement, Increment } from "@telefrek/type-utils/math" import type { Trim } from "@telefrek/type-utils/strings" +import type { ColumnReference } from "../../ast/columns.js" import type { ReturningClause } from "../../ast/queries.js" import type { ValueTypes } from "../../ast/values.js" -import { parseSelectedColumns, type ParseColumnReference } from "./columns.js" +import { + parseColumnReference, + parseSelectedColumns, + type ParseColumnReference, +} from "./columns.js" import type { NextToken } from "./normalize.js" import type { GetQuote, ParserOptions } from "./options.js" -import type { CheckValueType } from "./values.js" +import { pv, type CheckValueType } from "./values.js" /** * Parse an optional alias from the stack @@ -40,6 +45,21 @@ export type ParseValueOrReference< ? V : ParseColumnReference +/** + * Parse the next token as a value or reference + * + * @param tokens The current token stack + * @param options The parsing options + * + * @returns A value, reference or undefined if one cannot be read + */ +export function parseValueOrReference( + tokens: string[], + options: ParserOptions +): ValueTypes | ColumnReference { + return pv(tokens, options) ?? parseColumnReference(tokens) +} + /** * Extract the next full group from the current string */ diff --git a/packages/sql/query/parser/values.ts b/packages/sql/query/parser/values.ts index 64428cb..f13162e 100644 --- a/packages/sql/query/parser/values.ts +++ b/packages/sql/query/parser/values.ts @@ -17,6 +17,110 @@ import type { NextToken, SplitSQL } from "./normalize.js" import type { GetQuote, ParserOptions } from "./options.js" import type { IsSingleToken } from "./utils.js" +export function pv( + tokens: string[], + options: ParserOptions +): ValueTypes | undefined { + if (tokens.length === 0) { + throw new Error("Empty token stack whlie trying to read a value") + } + + // Try fixed size tokens first + switch (true) { + case tokens[0].startsWith(":"): + return { + type: "ParameterValue", + value: tokens.shift()!, + } + case tokens[0] === "true" || tokens[0] === "false": + return { + type: "BooleanValue", + value: Boolean(tokens.shift()!), + } + case isNumber(tokens[0]): + return { + type: "NumberValue", + value: Number(tokens.shift()!), + } + case isBigInt(tokens[0]): + return { + type: "BigIntValue", + value: BigInt(tokens.shift()!), + } + case tokens[0] === "null": + tokens.shift() + return { + type: "NullValue", + value: null, + } + case tokens[0].startsWith("0x"): + return { + type: "BufferValue", + value: Uint8Array.from( + Uint8Array.from( + tokens + .shift()! + .slice(2) + .match(/.{1,2}/g)! + .map((byte) => parseInt(byte, 16)) + ) + ), + } + } + + // TODO: This is probably brittle for cases where we have things like nested + // arrays but covering all edge cases right now feels like too much work + + // Check for variable size tokens + if (tokens[0].startsWith(options.tokens.quote)) { + // Read tokens until the end of the quote + for (let n = 0; n < tokens.length; ++n) { + // Check for ending but not escaped quote + if ( + tokens[n].endsWith(options.tokens.quote) && + !tokens[n].endsWith(`\\${options.tokens.quote}`) + ) { + // Read all the tokens that were used and remove the quotes + const value = tokens + .splice(0, n + 1) + .join(" ") + .slice(1, -1) + + // Check for arrays or json values + if (value.startsWith("{") && value.endsWith("}")) { + return { + type: "JsonValue", + value: JSON.parse(value), + } + } else if (value.startsWith("[") && value.endsWith("]")) { + return { + type: "ArrayValue", + value: JSON.parse(value), + } + } + + return { + type: "StringValue", + value, + } + } + } + } else if (tokens[0].startsWith("[")) { + // Keep reading until we find the end of the array + for (let n = 0; n < tokens.length; ++n) { + if (tokens[n].endsWith("]")) { + // Rip out the array portion and deserialize it into values + return { + type: "ArrayValue", + value: JSON.parse(tokens.splice(0, n + 1).join(" ")), + } + } + } + } + + return +} + /** * Parse out the value * From d66a18ceb3dfc28b0e76360e27adce606345e171 Mon Sep 17 00:00:00 2001 From: Nathan Northcutt Date: Fri, 23 Aug 2024 14:29:36 +0200 Subject: [PATCH 13/21] switching to use the value from the stack --- packages/sql/query/builder/insert.ts | 4 +- packages/sql/query/builder/where.ts | 4 +- packages/sql/query/parser/insert.ts | 4 +- packages/sql/query/parser/utils.ts | 4 +- packages/sql/query/parser/values.ts | 74 +++------------------------- packages/sql/query/parser/where.ts | 5 +- 6 files changed, 18 insertions(+), 77 deletions(-) diff --git a/packages/sql/query/builder/insert.ts b/packages/sql/query/builder/insert.ts index 06739e3..1bbad7b 100644 --- a/packages/sql/query/builder/insert.ts +++ b/packages/sql/query/builder/insert.ts @@ -17,7 +17,7 @@ import type { } from "../context.js" import type { ParserOptions } from "../parser/options.js" import type { ParseTableReference } from "../parser/table.js" -import { pv, type ExtractTSValueTypes } from "../parser/values.js" +import { parseNextValue, type ExtractTSValueTypes } from "../parser/values.js" import { createReturningBuilder, type ReturningBuilder } from "./returning.js" import { buildColumnReference, type VerifyColumnReferences } from "./select.js" import { buildTableReference } from "./table.js" @@ -177,7 +177,7 @@ class DefaultColumnValueBuilder< table: this._table, columns: this._columns, values: (values as unknown[]).map((v) => - pv( + parseNextValue( [ String( typeof v === "string" diff --git a/packages/sql/query/builder/where.ts b/packages/sql/query/builder/where.ts index dcb673a..c7108e8 100644 --- a/packages/sql/query/builder/where.ts +++ b/packages/sql/query/builder/where.ts @@ -26,7 +26,7 @@ import type { } from "../context.js" import type { ParseColumnReference } from "../parser/columns.js" import type { GetQuote, ParserOptions } from "../parser/options.js" -import { type CheckValueType, pv } from "../parser/values.js" +import { type CheckValueType, parseNextValue } from "../parser/values.js" import { buildColumnReference } from "./select.js" /** @@ -272,7 +272,7 @@ function buildFilter< } : isColumn(context, value) ? buildColumnReference(value as string) - : pv(String(value).split(" "), options)) as CheckColumnRef< + : parseNextValue(String(value).split(" "), options)) as CheckColumnRef< Value, QueryContextColumns, Options diff --git a/packages/sql/query/parser/insert.ts b/packages/sql/query/parser/insert.ts index a281bee..8db2c46 100644 --- a/packages/sql/query/parser/insert.ts +++ b/packages/sql/query/parser/insert.ts @@ -17,7 +17,7 @@ import type { ExtractReturning } from "./returning.js" import type { ParseSelect } from "./select.js" import { parseTableReference, type ParseTableReference } from "./table.js" import { tryParseReturning } from "./utils.js" -import { pv, type ParseValues } from "./values.js" +import { parseNextValue, type ParseValues } from "./values.js" /** * Parse an insert clause @@ -191,7 +191,7 @@ function parseValuesOrSelect( return extractParenthesis(tokens) .join(" ") .split(" , ") - .map((v) => pv(v.trim().split(" "), options)) as ValueTypes[] + .map((v) => parseNextValue(v.trim().split(" "), options)) as ValueTypes[] } const subquery = parseQueryClause(tokens, options) diff --git a/packages/sql/query/parser/utils.ts b/packages/sql/query/parser/utils.ts index dc0a583..48953b6 100644 --- a/packages/sql/query/parser/utils.ts +++ b/packages/sql/query/parser/utils.ts @@ -11,7 +11,7 @@ import { } from "./columns.js" import type { NextToken } from "./normalize.js" import type { GetQuote, ParserOptions } from "./options.js" -import { pv, type CheckValueType } from "./values.js" +import { parseNextValue, type CheckValueType } from "./values.js" /** * Parse an optional alias from the stack @@ -57,7 +57,7 @@ export function parseValueOrReference( tokens: string[], options: ParserOptions ): ValueTypes | ColumnReference { - return pv(tokens, options) ?? parseColumnReference(tokens) + return parseNextValue(tokens, options) ?? parseColumnReference(tokens) } /** diff --git a/packages/sql/query/parser/values.ts b/packages/sql/query/parser/values.ts index f13162e..cf81d60 100644 --- a/packages/sql/query/parser/values.ts +++ b/packages/sql/query/parser/values.ts @@ -17,7 +17,14 @@ import type { NextToken, SplitSQL } from "./normalize.js" import type { GetQuote, ParserOptions } from "./options.js" import type { IsSingleToken } from "./utils.js" -export function pv( +/** + * Try to read the next value off the token stack + * + * @param tokens The token stack to use + * @param options The options for parsing + * @returns The next value or nothing if one is not found + */ +export function parseNextValue( tokens: string[], options: ParserOptions ): ValueTypes | undefined { @@ -121,71 +128,6 @@ export function pv( return } -/** - * Parse out the value - * - * @param value The value to parse - * @param quote The quoted character - * @returns The next value or column reference identified - */ -export function parseValue(value: string, quote: string = "'"): ValueTypes { - if (value.startsWith(":")) { - return { - type: "ParameterValue", - value: value.substring(1), - } - } else if (value.startsWith("$")) { - throw new Error("Index positions for variables is not supported") - } else if (value === "true" || value === "false") { - return { - type: "BooleanValue", - value: Boolean(value), - } - } else if (isNumber(value)) { - return { - type: "NumberValue", - value: Number(value), - } - } else if (isBigInt(value)) { - return { - type: "BigIntValue", - value: BigInt(value), - } - } else if (value === "null") { - return { - type: "NullValue", - value: null, - } - } else if (value.startsWith("{")) { - return { - type: "JsonValue", - value: JSON.parse(value), - } - } else if (value.startsWith("[")) { - return { - type: "ArrayValue", - value: JSON.parse(value), - } - } else if (value.startsWith("0x")) { - return { - type: "BufferValue", - value: Uint8Array.from( - Uint8Array.from( - value - .slice(2) - .match(/.{1,2}/g)! - .map((byte) => parseInt(byte, 16)) - ) - ), - } - } else { - return { - type: "StringValue", - value: value.replaceAll(quote, ""), - } - } -} - /** * Regex to test for valid numbers */ diff --git a/packages/sql/query/parser/where.ts b/packages/sql/query/parser/where.ts index 8033126..17745f3 100644 --- a/packages/sql/query/parser/where.ts +++ b/packages/sql/query/parser/where.ts @@ -25,7 +25,7 @@ import type { ParserOptions, } from "./options.js" import type { ParseValueOrReference } from "./utils.js" -import { parseValue, type ExtractValue } from "./values.js" +import { parseNextValue, type ExtractValue } from "./values.js" // This entire thing needs a re-write... @@ -66,13 +66,12 @@ function parseLogicalExpression( const segments = tokens.join(" ").split(/(?=[>== Date: Fri, 23 Aug 2024 15:42:02 +0200 Subject: [PATCH 14/21] horrible mess but getting more clear --- packages/sql/ast/arithmetic.ts | 1 + packages/sql/query/parser/arithmetic.test.ts | 16 ++ packages/sql/query/parser/arithmetic.ts | 226 +++++++++++++++++-- 3 files changed, 221 insertions(+), 22 deletions(-) create mode 100644 packages/sql/query/parser/arithmetic.test.ts diff --git a/packages/sql/ast/arithmetic.ts b/packages/sql/ast/arithmetic.ts index 71cec6e..a6f2f77 100644 --- a/packages/sql/ast/arithmetic.ts +++ b/packages/sql/ast/arithmetic.ts @@ -54,6 +54,7 @@ export type ArithmeticExpressionType = | ValueTypes | ArithmeticExpression | GroupedArithmeticExpression + | ColumnArithmeticAssignment /** * A grouped expression (surrounded by parenthesis) diff --git a/packages/sql/query/parser/arithmetic.test.ts b/packages/sql/query/parser/arithmetic.test.ts new file mode 100644 index 0000000..d8c0771 --- /dev/null +++ b/packages/sql/query/parser/arithmetic.test.ts @@ -0,0 +1,16 @@ +import { parseArithmeticExpression } from "./arithmetic.js" +import { DefaultOptions } from "./options.js" + +describe("Arithmetic parsing should correctly extract types and values", () => { + it("Should be able to parse a simple value completely", () => { + const ret = parseArithmeticExpression("a + b", DefaultOptions) + expect(ret).not.toBeUndefined() + expect(ret.left.type).toBe("ColumnReference") + expect(ret.left.alias).toBe("a") + + expect(ret.operation).toBe("+") + + expect(ret.right.type).toBe("ColumnReference") + expect(ret.right.alias).toBe("b") + }) +}) diff --git a/packages/sql/query/parser/arithmetic.ts b/packages/sql/query/parser/arithmetic.ts index 4170762..ac041e0 100644 --- a/packages/sql/query/parser/arithmetic.ts +++ b/packages/sql/query/parser/arithmetic.ts @@ -1,6 +1,7 @@ import type { IgnoreAny, Invalid } from "@telefrek/type-utils/common" import { type ArithmeticExpression, + type ArithmeticExpressionType, type ColumnArithmeticAssignment, type GroupedArithmeticExpression, } from "../../ast/arithmetic.js" @@ -12,21 +13,79 @@ import { type GetAssignmentOperations, type ParserOptions, } from "./options.js" -import type { ExtractGroup, ParseValueOrReference } from "./utils.js" - -// export function parseArithmeticExpression( -// tokens: string[], -// options: ParserOptions -// ): AnyExpression | undefined { -// return -// } - -// function parseNextArithmeticExpression( -// tokens: string[], -// options: ParserOptions -// ): AnyExpression | undefined { -// return -// } +import { + parseValueOrReference, + type ExtractGroup, + type ParseValueOrReference, +} from "./utils.js" + +/** + * Utility to get the full expression type + */ +type GetFullExpressionType< + SQL extends string, + Options extends ParserOptions +> = ParseArithmeticExpression extends [infer Expression, ""] + ? Expression + : never + +/** + * Parse the SQL string as an expression + * @param sql The SQL to parse as an expression + * @param options The options to use + */ +export function parseArithmeticExpression< + SQL extends string, + Options extends ParserOptions +>(sql: SQL, options: Options): GetFullExpressionType + +/** + * Parse the next arithmetic expression from the stack + * + * @param tokens The current token stack + * @param options The parser options to use + * @param current The current expression if one exists + */ +export function parseArithmeticExpression< + Expression extends AnyExpression, + Options extends ParserOptions +>( + tokens: string[], + options: Options, + current?: Expression +): AnyExpression | undefined + +// Implementation +export function parseArithmeticExpression( + sql: unknown, + options: Options, + current?: AnyExpression +): unknown { + const tokens = typeof sql === "string" ? sql.split(" ") : (sql as string[]) + + // Create a copy of the tokens in case of partial reads + const copy = [...tokens] + + if (current !== undefined) { + return // fail nested for now + } + + const next = parseNextArithmeticExpression(copy, options) + if (next !== undefined) { + const fullExpression = parseArithmeticExpression( + copy, + options, + next as AnyExpression + ) + + const diff = tokens.length - copy.length + tokens.splice(0, diff) + + return fullExpression ?? next + } + + return +} /** * Extract the next valid expression chunk and the remaining string @@ -107,10 +166,16 @@ type ReadNextToken< : Invalid<"Cannot map value"> : Invalid<"No more tokens to extract"> +/** + * Read the next value from the stack + * @param tokens The current token stack + * @param options The parsing options to use + * @returns A column reference, value or group + */ export function readNextToken( - tokens: string, + tokens: string[], options: ParserOptions -): ValueTypes | ColumnReference | string[] { +): ValueTypes | ColumnReference | string[] | string | undefined { if (tokens.length === 0) { throw new Error("No more tokens to extract") } @@ -121,12 +186,12 @@ export function readNextToken( case tokens[0] === "(": break case options.tokens.arithmetic.indexOf(tokens[0]) >= 0: - break + return tokens.shift()! case options.tokens.assignments.indexOf(tokens[0]) >= 0: - break + return tokens.shift()! } - throw new Error("Cannot map value") + return parseValueOrReference(tokens, options) } /** @@ -159,6 +224,27 @@ type ParseNextArithmeticExpression< : Invalid<"Invalid token"> : ReadNextToken +function parseNextArithmeticExpression( + tokens: string[], + options: ParserOptions +): Partial | undefined { + const token = readNextToken(tokens, options) + if (token === undefined) { + return undefined + } + + // Group + if (Array.isArray(token)) { + return + } else if (typeof token === "string") { + return + } else if (token.type === "ColumnReference") { + return parseColumnExpression(tokens, options, token) + } else { + return parseValueExpression(tokens, options, token) + } +} + /** * Parse expressions starting with a column */ @@ -185,6 +271,81 @@ type ParseColumnExpression< : Invalid<"Column must be followed by an assignment or arithmetic operation"> : ReadNextToken +function parseColumnExpression( + tokens: string[], + options: ParserOptions, + column: ColumnReference +): AnyExpression | undefined { + const token = readNextToken(tokens, options) + if (typeof token === "string") { + if (options.tokens.assignments.indexOf(token) >= 0) { + return parseColumnAssignmentExpression(tokens, options, { + type: "ColumnArithmeticAssignment", + column, + operation: token, + }) + } else { + return parseSingleArithmeticExpression(tokens, options, { + type: "ArithmeticExpression", + left: column, + operation: token, + }) + } + } + + return +} + +function parseColumnAssignmentExpression< + Assignment extends Partial< + ColumnArithmeticAssignment + > +>( + tokens: string[], + options: ParserOptions, + assignment: Assignment +): ColumnArithmeticAssignment | undefined { + const token = readNextToken(tokens, options) + if (token === undefined) { + return + } + + if (typeof token === "string") { + return + } else if (Array.isArray(token)) { + return + } + + return { + ...assignment, + value: token, + } +} + +function parseSingleArithmeticExpression< + Expression extends Partial> +>( + tokens: string[], + options: ParserOptions, + expression: Expression +): ArithmeticExpression | undefined { + const token = readNextToken(tokens, options) + if (token === undefined) { + return + } + + if (typeof token === "string") { + return + } else if (Array.isArray(token)) { + return + } + + return { + ...expression, + right: token, + } +} + /** * Parse expressions starting with a value */ @@ -201,13 +362,33 @@ type ParseValueExpression< : Invalid<"Value must be followed by an arithmetic operation"> : ReadNextToken +function parseValueExpression( + tokens: string[], + options: ParserOptions, + value: ValueTypes +): Partial> | undefined { + const token = readNextToken(tokens, options) + if ( + typeof token === "string" && + options.tokens.arithmetic.indexOf(token) >= 0 + ) { + return { + type: "ArithmeticExpression", + left: value, + operation: token, + } + } + + return +} + /** * Parse only the next segment */ type ParseSingleArithmeticExpression< SQL extends string, Options extends ParserOptions, - Expression extends ArithmeticExpression + Expression extends AnyExpression > = ReadNextToken extends [ infer Token, infer Remainder extends string @@ -266,7 +447,7 @@ type ParseColumnAssignmentExpression< ? ParseEntireArithmeticTree< Token, Options - > extends infer Expression extends AnyExpression + > extends infer Expression extends ArithmeticExpressionType ? Assignment extends ColumnArithmeticAssignment< infer Column, infer Op, @@ -284,6 +465,7 @@ type ParseColumnAssignmentExpression< type AnyExpression = | ArithmeticExpression | GroupedArithmeticExpression + | ColumnArithmeticAssignment /** * Consume the entire arithmetic tree From 2d5ef6f792752ebc0e9ba2c7473efd4f0b71b39d Mon Sep 17 00:00:00 2001 From: Nathan Northcutt Date: Wed, 28 Aug 2024 15:07:32 +0200 Subject: [PATCH 15/21] fixing a few things to get ready to start parsing where clauses --- packages/sql/query/parser/arithmetic.test.ts | 85 ++++++++++++++++++-- packages/sql/query/parser/arithmetic.ts | 66 +++++++++++++-- packages/sql/query/parser/utils.ts | 26 ++++++ 3 files changed, 163 insertions(+), 14 deletions(-) diff --git a/packages/sql/query/parser/arithmetic.test.ts b/packages/sql/query/parser/arithmetic.test.ts index d8c0771..e0e39a1 100644 --- a/packages/sql/query/parser/arithmetic.test.ts +++ b/packages/sql/query/parser/arithmetic.test.ts @@ -2,15 +2,86 @@ import { parseArithmeticExpression } from "./arithmetic.js" import { DefaultOptions } from "./options.js" describe("Arithmetic parsing should correctly extract types and values", () => { - it("Should be able to parse a simple value completely", () => { - const ret = parseArithmeticExpression("a + b", DefaultOptions) + it("Should consume the correct amount of tokens", () => { + const tokens = "a + b OR c".split(" ") + + const ret = parseArithmeticExpression(tokens, DefaultOptions) + const partial = parseArithmeticExpression("a + b", DefaultOptions) expect(ret).not.toBeUndefined() - expect(ret.left.type).toBe("ColumnReference") - expect(ret.left.alias).toBe("a") + expect(ret).toStrictEqual(partial) + expect(tokens.length).toBe(2) + expect(tokens).toStrictEqual(["OR", "c"]) + }) - expect(ret.operation).toBe("+") + it("Should not consume an invalid set of tokens", () => { + const tokens = "( a + b + c / d".split(" ") - expect(ret.right.type).toBe("ColumnReference") - expect(ret.right.alias).toBe("b") + const ret = parseArithmeticExpression(tokens, DefaultOptions) + expect(ret).toBeUndefined() + expect(tokens).toStrictEqual(["(", "a", "+", "b", "+", "c", "/", "d"]) + }) + + it("Should be able to parse a group value", () => { + const ret = parseArithmeticExpression("( a + b )", DefaultOptions) + expect(ret).not.toBeUndefined() + expect(ret.type).toBe("GroupedArithmeticExpression") + expect(ret.expression).toStrictEqual({ + type: "ArithmeticExpression", + left: { + type: "ColumnReference", + reference: { + type: "UnboundColumnReference", + column: "a", + }, + alias: "a", + }, + operation: "+", + right: { + type: "ColumnReference", + reference: { + type: "UnboundColumnReference", + column: "b", + }, + alias: "b", + }, + }) + }) + + it("Should be able to parse a simple value completely", () => { + const ret = parseArithmeticExpression("a + b + c", DefaultOptions) + + expect(ret).not.toBeUndefined() + expect(ret).toStrictEqual({ + type: "ArithmeticExpression", + left: { + type: "ArithmeticExpression", + left: { + type: "ColumnReference", + reference: { + type: "UnboundColumnReference", + column: "a", + }, + alias: "a", + }, + operation: "+", + right: { + type: "ColumnReference", + reference: { + type: "UnboundColumnReference", + column: "b", + }, + alias: "b", + }, + }, + operation: "+", + right: { + type: "ColumnReference", + reference: { + type: "UnboundColumnReference", + column: "c", + }, + alias: "c", + }, + }) }) }) diff --git a/packages/sql/query/parser/arithmetic.ts b/packages/sql/query/parser/arithmetic.ts index ac041e0..6238743 100644 --- a/packages/sql/query/parser/arithmetic.ts +++ b/packages/sql/query/parser/arithmetic.ts @@ -14,6 +14,7 @@ import { type ParserOptions, } from "./options.js" import { + extractGroup, parseValueOrReference, type ExtractGroup, type ParseValueOrReference, @@ -67,7 +68,21 @@ export function parseArithmeticExpression( const copy = [...tokens] if (current !== undefined) { - return // fail nested for now + const token = readNextToken(copy, options) + if (typeof token === "string") { + // Only allow additional arithmetic + if (options.tokens.arithmetic.indexOf(token) >= 0) { + return parseSingleArithmeticExpression(copy, options, { + type: "ArithmeticExpression", + left: current, + operation: token, + }) + } + } else if (token === undefined) { + return copy.length === 0 ? current : undefined + } + + return } const next = parseNextArithmeticExpression(copy, options) @@ -87,6 +102,35 @@ export function parseArithmeticExpression( return } +/** + * Parse the entire token stack as an expression or return undefined + * + * @param tokens The current token stack + * @param options The parsing options + * @returns Either a fully consumed expression or undefined + */ +function parseGroupExpression( + tokens: string[], + options: ParserOptions +): GroupedArithmeticExpression | undefined { + const copy = [...tokens] + + const expression = parseArithmeticExpression(copy, options) + if ( + expression !== undefined && + copy.length === 0 && + expression.type === "ArithmeticExpression" + ) { + tokens.splice(0, tokens.length) + return { + type: "GroupedArithmeticExpression", + expression, + } + } + + return +} + /** * Extract the next valid expression chunk and the remaining string */ @@ -177,14 +221,15 @@ export function readNextToken( options: ParserOptions ): ValueTypes | ColumnReference | string[] | string | undefined { if (tokens.length === 0) { - throw new Error("No more tokens to extract") + return } switch (true) { case tokens[0] === ")": throw new Error("Corrupt group") case tokens[0] === "(": - break + tokens.shift() + return extractGroup(tokens) case options.tokens.arithmetic.indexOf(tokens[0]) >= 0: return tokens.shift()! case options.tokens.assignments.indexOf(tokens[0]) >= 0: @@ -230,12 +275,12 @@ function parseNextArithmeticExpression( ): Partial | undefined { const token = readNextToken(tokens, options) if (token === undefined) { - return undefined + return } // Group if (Array.isArray(token)) { - return + return parseGroupExpression(token, options) } else if (typeof token === "string") { return } else if (token.type === "ColumnReference") { @@ -313,7 +358,12 @@ function parseColumnAssignmentExpression< if (typeof token === "string") { return } else if (Array.isArray(token)) { - return + const value = parseGroupExpression(token, options) + if (value === undefined) return + return { + ...assignment, + value, + } } return { @@ -337,7 +387,9 @@ function parseSingleArithmeticExpression< if (typeof token === "string") { return } else if (Array.isArray(token)) { - return + const right = parseGroupExpression(token, options) + if (right === undefined) return + return { ...expression, right } } return { diff --git a/packages/sql/query/parser/utils.ts b/packages/sql/query/parser/utils.ts index 48953b6..add01f9 100644 --- a/packages/sql/query/parser/utils.ts +++ b/packages/sql/query/parser/utils.ts @@ -60,6 +60,32 @@ export function parseValueOrReference( return parseNextValue(tokens, options) ?? parseColumnReference(tokens) } +/** + * Extract the next token group + * + * @param tokens The current token stack + * @returns The tokens in the group + */ +export function extractGroup(tokens: string[]): string[] | undefined { + let n = 1 + const ret: string[] = [] + + for (let i = 0; i < tokens.length && n > 0; ++i) { + if (tokens[i] === "(") { + ++n + } else if (tokens[i] === ")") { + if (--n === 0) { + tokens.splice(0, i + 1) + return ret + } + } + + ret.push(tokens[i]) + } + + return +} + /** * Extract the next full group from the current string */ From 99e4b9c8ddbbd6f3ced2e5fba27cc0ec05552eed Mon Sep 17 00:00:00 2001 From: Nathan Northcutt Date: Mon, 2 Sep 2024 16:55:34 +0200 Subject: [PATCH 16/21] working on the parsing and expressions handling arithmetic and logical --- packages/sql/ast/README.md | 2 +- packages/sql/ast/arithmetic.ts | 85 -------- packages/sql/ast/combined.ts | 20 +- packages/sql/ast/expressions.ts | 195 +++++++++++++++++++ packages/sql/ast/filtering.ts | 132 ------------- packages/sql/ast/select.ts | 18 +- packages/sql/ast/update.ts | 2 +- packages/sql/ast/where.ts | 7 + packages/sql/query/builder/where.ts | 31 +-- packages/sql/query/parser/arithmetic.test.ts | 2 +- packages/sql/query/parser/arithmetic.ts | 32 ++- packages/sql/query/parser/filtering.ts | 67 ++++++- packages/sql/query/parser/options.ts | 10 +- packages/sql/query/parser/select.ts | 3 +- packages/sql/query/parser/where.test.ts | 26 +++ packages/sql/query/parser/where.ts | 76 ++++---- packages/sql/query/visitor/common.ts | 49 +++-- packages/sql/query/visitor/types.ts | 5 +- 18 files changed, 421 insertions(+), 341 deletions(-) delete mode 100644 packages/sql/ast/arithmetic.ts create mode 100644 packages/sql/ast/expressions.ts delete mode 100644 packages/sql/ast/filtering.ts create mode 100644 packages/sql/ast/where.ts create mode 100644 packages/sql/query/parser/where.test.ts diff --git a/packages/sql/ast/README.md b/packages/sql/ast/README.md index eaee58f..ccdc342 100644 --- a/packages/sql/ast/README.md +++ b/packages/sql/ast/README.md @@ -22,7 +22,7 @@ export type ColumnFilter< > = { type: "ColumnFilter" left: Left - op: Operation + operation: Operation right: Right } ``` diff --git a/packages/sql/ast/arithmetic.ts b/packages/sql/ast/arithmetic.ts deleted file mode 100644 index a6f2f77..0000000 --- a/packages/sql/ast/arithmetic.ts +++ /dev/null @@ -1,85 +0,0 @@ -import type { IgnoreAny } from "@telefrek/type-utils/common" -import type { ColumnReference } from "./columns.js" -import type { ValueTypes } from "./values.js" - -/** - * Types for Arithmetic operations - */ -export type ArithmeticOperation = "+" | "-" | "*" | "/" | "%" | "|" | "&" | "^" - -/** - * The default arithmetic operations - */ -export const DEFAULT_ARITHMETIC_OPS: ArithmeticOperation[] = [ - "%", - "&", - "*", - "+", - "-", - "/", - "^", - "|", -] - -/** - * Types for Arithmetic assignment - */ -export type ArithmeticAssignmentOperation = `${ArithmeticOperation}=` | "=" - -/** - * The default arithmetic assignment operations - */ -export const DEFAULT_ARITHMETIC_ASSIGNMENT_OPS: ArithmeticAssignmentOperation[] = - ["%=", "&=", "*=", "+=", "-=", "/=", "^=", "|=", "="] - -/** - * An arithmetic assignment expression, ex: column += b - */ -export type ColumnArithmeticAssignment< - Column extends ColumnReference = ColumnReference, - Op extends string = ArithmeticAssignmentOperation, - Value extends ArithmeticExpressionType = ArithmeticExpressionType -> = { - type: "ColumnArithmeticAssignment" - column: Column - operation: Op - value: Value -} - -/** - * The default type for an arithmetic expression - */ -export type ArithmeticExpressionType = - | ColumnReference - | ValueTypes - | ArithmeticExpression - | GroupedArithmeticExpression - | ColumnArithmeticAssignment - -/** - * A grouped expression (surrounded by parenthesis) - */ -export type GroupedArithmeticExpression< - Expression extends ArithmeticExpression< - IgnoreAny, - string, - IgnoreAny - > = ArithmeticExpression -> = { - type: "GroupedArithmeticExpression" - expression: Expression -} - -/** - * An arithmetic expression between two values, ex: a + b - */ -export type ArithmeticExpression< - Left extends ArithmeticExpressionType = ArithmeticExpressionType, - Op extends string = ArithmeticOperation, - Right extends ArithmeticExpressionType = ArithmeticExpressionType -> = { - type: "ArithmeticExpression" - left: Left - operation: Op - right: Right -} diff --git a/packages/sql/ast/combined.ts b/packages/sql/ast/combined.ts index 2d21364..de7db1c 100644 --- a/packages/sql/ast/combined.ts +++ b/packages/sql/ast/combined.ts @@ -11,10 +11,10 @@ export type CombinedSelectOperation = "UNION" | "INTERSECT" | "MINUS" | "EXCEPT" */ export type CombinedSelect< Operation extends string = CombinedSelectOperation, - Next extends SelectClause = SelectClause, + Next extends SelectClause = SelectClause > = { type: "CombinedSelect" - op: Operation + operation: Operation next: Next } @@ -22,12 +22,14 @@ export type CombinedSelect< * Utliity type to extract the keys from the initial select clause to restrict * others to having the same set of keys */ -type GetSelectKeys = Select extends SelectClause< + infer Columns, + infer _ +> + ? Columns extends "*" + ? string + : Extract + : string /** * A chain of select clauses @@ -36,7 +38,7 @@ export type CombinedSelectClause< Original extends SelectClause = SelectClause, Additions extends OneOrMore< CombinedSelect> - > = OneOrMore>>, + > = OneOrMore>> > = { type: "CombinedSelectClause" original: Original diff --git a/packages/sql/ast/expressions.ts b/packages/sql/ast/expressions.ts new file mode 100644 index 0000000..b7c32bc --- /dev/null +++ b/packages/sql/ast/expressions.ts @@ -0,0 +1,195 @@ +import type { ColumnReference } from "./columns.js" +import type { SubQuery } from "./queries.js" +import type { ValueTypes } from "./values.js" + +/** + * Types for Arithmetic operations + */ +export type ArithmeticOperation = "+" | "-" | "*" | "/" | "%" | "|" | "&" | "^" + +/** + * The default arithmetic operations + */ +export const DEFAULT_ARITHMETIC_OPS: ArithmeticOperation[] = [ + "%", + "&", + "*", + "+", + "-", + "/", + "^", + "|", +] + +/** + * Types for Arithmetic assignment + */ +export type ArithmeticAssignmentOperation = `${ArithmeticOperation}=` | "=" + +/** + * The default arithmetic assignment operations + */ +export const DEFAULT_ARITHMETIC_ASSIGNMENT_OPS: ArithmeticAssignmentOperation[] = + ["%=", "&=", "*=", "+=", "-=", "/=", "^=", "|=", "="] + +/** + * Any logical operation + */ +export type LogicalOperation = { + type: string + operation: string +} + +export function isLogicalOperation(value: unknown): value is LogicalOperation { + return ( + typeof value === "object" && + value !== null && + "type" in value && + "operation" in value && + typeof value.type === "string" && + typeof value.operation === "string" + ) +} + +/** + * Types for for value comparisons + */ +export type ComparisonOperation = "=" | "<" | ">" | "<=" | ">=" | "!=" | "<>" + +/** + * The default comparison operations + */ +export const DEFAULT_COMPARISON_OPS: ComparisonOperation[] = [ + "=", + "<", + ">", + "<=", + ">=", + "!=", + "<>", +] + +/** + * A filter between two objects + */ +export type ColumnFilter< + Column extends ColumnReference = ColumnReference, + Operation extends string = ComparisonOperation, + Filter extends LogicalExpression = LogicalExpression +> = { + type: "ColumnFilter" + column: Column + operation: Operation + filter: Filter +} + +/** + * An expression which could be an operation, value or column + */ +export type LogicalExpression = LogicalOperation | ValueTypes | ColumnReference + +/** + * Type for handling logical negations + */ +export type LogicalNegation< + Expression extends LogicalExpression = LogicalExpression +> = LogicalOperation & { + type: "LogicalNegation" + operation: "NOT" + expression: Expression +} + +/** + * An arithmetic assignment expression, ex: column += b + */ +export type ColumnArithmeticAssignment< + Column extends ColumnReference = ColumnReference, + Op extends string = ArithmeticAssignmentOperation, + Value extends LogicalExpression = LogicalExpression +> = LogicalOperation & { + type: "ColumnArithmeticAssignment" + column: Column + operation: Op + value: Value +} + +/** + * An arithmetic expression between two values, ex: a + b + */ +export type ArithmeticExpression< + Left extends LogicalExpression = LogicalExpression, + Op extends string = ArithmeticOperation, + Right extends LogicalExpression = LogicalExpression +> = LogicalOperation & { + type: "ArithmeticExpression" + left: Left + operation: Op + right: Right +} + +/** + * A grouped expression (surrounded by parenthesis) + */ +export type LogicalGroup< + Expression extends LogicalOperation = LogicalOperation +> = { + type: "LogicalGroup" + operation: "LogicalGroup" + expression: Expression +} + +/** + * A filter for a clause that matches something IN a set + */ +export type InFilter< + Column extends ColumnReference = ColumnReference, + Values extends SubQuery | ValueTypes[] = SubQuery | ValueTypes[] +> = LogicalOperation & { + type: "InFilter" + operation: "IN" + column: Column + values: Values +} + +/** + * A filter between two values + */ +export type BetweenFilter< + Column extends ColumnReference = ColumnReference, + Left extends LogicalExpression = LogicalExpression, + Right extends LogicalExpression = LogicalExpression +> = LogicalOperation & { + type: "BetweenFilter" + operation: "BETWEEN" + column: Column + left: Left + right: Right +} + +/** + * A logical tree operation + */ +export type LogicalTree< + Left extends LogicalExpression = LogicalExpression, + Operation extends string = "AND" | "OR", + Right extends LogicalExpression = LogicalExpression +> = LogicalOperation & { + type: "LogicalTree" + operation: Operation + left: Left + right: Right +} + +/** + * A subquery filter that is NOT an "IN" because of syntax restrictions + */ +export type SubqueryFilter< + Column extends ColumnReference = ColumnReference, + Operation extends string = "ANY" | "ALL" | "EXISTS" | "SOME", + Subquery extends SubQuery = SubQuery +> = LogicalOperation & { + type: "SubqueryFilter" + operation: Operation + column: Column + query: Subquery +} diff --git a/packages/sql/ast/filtering.ts b/packages/sql/ast/filtering.ts deleted file mode 100644 index cb6f8a2..0000000 --- a/packages/sql/ast/filtering.ts +++ /dev/null @@ -1,132 +0,0 @@ -import type { IgnoreAny } from "@telefrek/type-utils/common.js" -import type { ColumnReference } from "./columns.js" -import type { SubQuery } from "./queries.js" -import type { ValueTypes } from "./values.js" - -/** - * Types for for value comparisons - */ -export type ComparisonOperation = "=" | "<" | ">" | "<=" | ">=" | "!=" | "<>" - -/** - * The default comparison operations - */ -export const DEFAULT_COMPARISON_OPS: ComparisonOperation[] = [ - "=", - "<", - ">", - "<=", - ">=", - "!=", - "<>", -] - -/** - * Types of subquery filtering mechanisms (IN is a special case) - */ -export type SubQueryFilterOperation = "ANY" | "ALL" | "EXISTS" | "SOME" - -/** - * Types of logical operations - */ -export type LogicalOperation = "BETWEEN" | "LIKE" | "ILIKE" - -/** - * Types for building logical trees - */ -export type LogicalTreeOperation = "AND" | "OR" - -/** - * Type for handling logical negations - */ -export type LogicalNegation< - Expression extends LogicalExpression = LogicalExpression -> = { - type: "LogicalNegation" - expression: Expression -} - -/** - * A logical tree structure for processing groups of filters - */ -export type LogicalTree< - Left extends LogicalExpression = LogicalExpression, - Operation extends string = LogicalTreeOperation, - Right extends LogicalExpression = LogicalExpression -> = { - type: "LogicalTree" - left: Left - op: Operation - right: Right -} - -/** - * The valid types for building a logical expression trees - */ -export type LogicalExpression = - | ValueTypes - | LogicalTree - | ColumnFilter - | SubqueryFilter - | LogicalNegation - -/** - * A filter between two objects - */ -export type ColumnFilter< - Column extends ColumnReference = ColumnReference, - Operation extends string = ComparisonOperation, - Filter extends ValueTypes | ColumnReference = ValueTypes | ColumnReference -> = { - type: "ColumnFilter" - column: Column - op: Operation - filter: Filter -} - -/** - * Required structure for where clause - */ -export type WhereClause = { - where: Where -} - -/** - * A filter for a column in some range - */ -export type BetweenFilter< - Column extends ColumnReference = ColumnReference, - Left extends ValueTypes = ValueTypes, - Right extends ValueTypes = ValueTypes -> = { - type: "BetweenFilter" - column: Column - left: Left - right: Right -} - -/** - * A filter for an "IN" clause that can be either a set of values or a subquery - */ -export type InFilter< - Column extends ColumnReference = ColumnReference, - Values extends SubQuery | ValueTypes[] = SubQuery | ValueTypes[] -> = { - type: "InFilter" - column: Column - values: Values -} - -/** - * A filter for a SubQuery operation - */ -export type SubqueryFilter< - Column extends ColumnReference = ColumnReference, - Operation extends string = SubQueryFilterOperation, - Subquery extends SubQuery = SubQuery -> = { - type: "SubqueryFilter" - column: Column - query: Subquery - op: Operation -} diff --git a/packages/sql/ast/select.ts b/packages/sql/ast/select.ts index cdfd4c6..ff15ac8 100644 --- a/packages/sql/ast/select.ts +++ b/packages/sql/ast/select.ts @@ -1,6 +1,6 @@ import type { IgnoreAny, OneOrMore } from "@telefrek/type-utils/common.js" import type { ColumnReference } from "./columns.js" -import type { LogicalExpression } from "./filtering.js" +import type { LogicalExpression } from "./expressions.js" import type { NamedQuery } from "./named.js" import type { TableReference } from "./tables.js" @@ -32,7 +32,7 @@ export type ColumnAggretator = "SUM" | "COUNT" | "AVG" | "MAX" | "MIN" export type JoinExpression< Type extends string = JoinType, From extends TableReference | NamedQuery = TableReference | NamedQuery, - On extends LogicalExpression = LogicalExpression, + On extends LogicalExpression = LogicalExpression > = { type: "JoinClause" joinType: Type @@ -56,7 +56,7 @@ export type SelectColumns = [SelectedColumn, ...SelectedColumn[]] export type ColumnAggregate< Column extends ColumnReference = ColumnReference, Aggretator extends string = ColumnAggretator, - Alias extends string = string, + Alias extends string = string > = { type: "ColumnAggregate" column: Column @@ -69,7 +69,7 @@ export type ColumnAggregate< */ export type ColumnOrdering< Column extends ColumnReference = ColumnReference, - Order extends string = SortOrder, + Order extends string = SortOrder > = { type: "ColumnOrdering" column: Column @@ -81,7 +81,7 @@ export type ColumnOrdering< */ export type SelectClause< Columns extends SelectColumns | "*" = SelectColumns | "*", - From extends TableReference | AnyNamedQuery = TableReference | AnyNamedQuery, + From extends TableReference | AnyNamedQuery = TableReference | AnyNamedQuery > = { type: "SelectClause" columns: Columns @@ -93,7 +93,7 @@ export type SelectClause< * A join clause */ export type JoinClause< - Join extends OneOrMore = OneOrMore, + Join extends OneOrMore = OneOrMore > = { join: Join } @@ -103,7 +103,7 @@ export type JoinClause< */ export type LimitClause< Offset extends number = number, - Limit extends number = number, + Limit extends number = number > = { offset: Offset limit: Limit @@ -113,7 +113,7 @@ export type LimitClause< * Structure for a group by clause */ export type GroupByClause< - GroupBy extends OneOrMore = OneOrMore, + GroupBy extends OneOrMore = OneOrMore > = { groupBy: GroupBy } @@ -122,7 +122,7 @@ export type GroupByClause< * Structure for an order by clause */ export type OrderByClause< - OrderBy extends OneOrMore = OneOrMore, + OrderBy extends OneOrMore = OneOrMore > = { orderBy: OrderBy } diff --git a/packages/sql/ast/update.ts b/packages/sql/ast/update.ts index 237e964..ed1992e 100644 --- a/packages/sql/ast/update.ts +++ b/packages/sql/ast/update.ts @@ -1,5 +1,5 @@ import type { OneOrMore } from "@telefrek/type-utils/common.js" -import type { ColumnArithmeticAssignment } from "./arithmetic.js" +import type { ColumnArithmeticAssignment } from "./expressions.js" import type { TableReference } from "./tables.js" /** diff --git a/packages/sql/ast/where.ts b/packages/sql/ast/where.ts new file mode 100644 index 0000000..ee4117c --- /dev/null +++ b/packages/sql/ast/where.ts @@ -0,0 +1,7 @@ +import type { LogicalExpression } from "./expressions.js" + +export type WhereClause< + Expression extends LogicalExpression = LogicalExpression +> = { + where: Expression +} diff --git a/packages/sql/query/builder/where.ts b/packages/sql/query/builder/where.ts index c7108e8..06bf383 100644 --- a/packages/sql/query/builder/where.ts +++ b/packages/sql/query/builder/where.ts @@ -10,13 +10,12 @@ import type { } from "../../ast/columns.js" import type { ColumnFilter, - ComparisonOperation, LogicalExpression, LogicalTree, - WhereClause, -} from "../../ast/filtering.js" +} from "../../ast/expressions.js" import type { QueryClause, SQLQuery } from "../../ast/queries.js" import type { ValueTypes } from "../../ast/values.js" +import type { WhereClause } from "../../ast/where.js" import type { QueryAST } from "../common.js" import type { ColumnType, @@ -25,7 +24,11 @@ import type { QueryContextColumns, } from "../context.js" import type { ParseColumnReference } from "../parser/columns.js" -import type { GetQuote, ParserOptions } from "../parser/options.js" +import type { + GetComparisonOperations, + GetQuote, + ParserOptions, +} from "../parser/options.js" import { type CheckValueType, parseNextValue } from "../parser/values.js" import { buildColumnReference } from "./select.js" @@ -144,11 +147,11 @@ export interface WhereClauseBuilder< filter< Column extends QueryContextColumns, - Op extends ComparisonOperation, + Op extends GetComparisonOperations, Value extends string | number | bigint | boolean | null | undefined >( column: Column, - op: Op, + operation: Op, value: Parameter ): ColumnFilter< RefType, @@ -197,7 +200,7 @@ class DefaultWhereClauseBuilder< return { type: "LogicalTree", left, - op: "AND", + operation: "AND", right, } } @@ -209,18 +212,18 @@ class DefaultWhereClauseBuilder< return { type: "LogicalTree", left, - op: "OR", + operation: "OR", right, } } filter< Column extends QueryContextColumns, - Op extends ComparisonOperation, + Op extends GetComparisonOperations, Value extends string | number | bigint | boolean | null | undefined >( column: Column, - op: Op, + operation: Op, value: Parameter ): ColumnFilter< RefType, @@ -230,7 +233,7 @@ class DefaultWhereClauseBuilder< return buildFilter( this._context, column, - op, + operation, value as Value, this._options ) as unknown as ColumnFilter< @@ -244,13 +247,13 @@ class DefaultWhereClauseBuilder< function buildFilter< Context extends QueryContext, Column extends string, - Operation extends ComparisonOperation, + Operation extends GetComparisonOperations, Value extends string | number | bigint | boolean | null | undefined, Options extends ParserOptions >( context: Context, column: Column, - op: Operation, + operation: Operation, value: Value, options: Options ): ColumnFilter< @@ -264,7 +267,7 @@ function buildFilter< type: "ColumnFilter", // eslint-disable-next-line @typescript-eslint/no-explicit-any column: buildColumnReference(column) as any, - op, + operation, filter: (isParameter(value) ? { type: "ParameterValue", diff --git a/packages/sql/query/parser/arithmetic.test.ts b/packages/sql/query/parser/arithmetic.test.ts index e0e39a1..8213fa8 100644 --- a/packages/sql/query/parser/arithmetic.test.ts +++ b/packages/sql/query/parser/arithmetic.test.ts @@ -24,7 +24,7 @@ describe("Arithmetic parsing should correctly extract types and values", () => { it("Should be able to parse a group value", () => { const ret = parseArithmeticExpression("( a + b )", DefaultOptions) expect(ret).not.toBeUndefined() - expect(ret.type).toBe("GroupedArithmeticExpression") + expect(ret.type).toBe("LogicalGroup") expect(ret.expression).toStrictEqual({ type: "ArithmeticExpression", left: { diff --git a/packages/sql/query/parser/arithmetic.ts b/packages/sql/query/parser/arithmetic.ts index 6238743..2264c60 100644 --- a/packages/sql/query/parser/arithmetic.ts +++ b/packages/sql/query/parser/arithmetic.ts @@ -1,11 +1,12 @@ import type { IgnoreAny, Invalid } from "@telefrek/type-utils/common" -import { - type ArithmeticExpression, - type ArithmeticExpressionType, - type ColumnArithmeticAssignment, - type GroupedArithmeticExpression, -} from "../../ast/arithmetic.js" + import type { ColumnReference } from "../../ast/columns.js" +import type { + ArithmeticExpression, + ColumnArithmeticAssignment, + LogicalExpression, + LogicalGroup, +} from "../../ast/expressions.js" import type { ValueTypes } from "../../ast/values.js" import type { NextToken } from "./normalize.js" import { @@ -112,7 +113,7 @@ export function parseArithmeticExpression( function parseGroupExpression( tokens: string[], options: ParserOptions -): GroupedArithmeticExpression | undefined { +): LogicalGroup | undefined { const copy = [...tokens] const expression = parseArithmeticExpression(copy, options) @@ -123,7 +124,8 @@ function parseGroupExpression( ) { tokens.splice(0, tokens.length) return { - type: "GroupedArithmeticExpression", + type: "LogicalGroup", + operation: "LogicalGroup", expression, } } @@ -264,7 +266,7 @@ type ParseNextArithmeticExpression< string, IgnoreAny > - ? [GroupedArithmeticExpression, Remainder] + ? [LogicalGroup, Remainder] : ParseEntireArithmeticTree : Invalid<"Invalid token"> : ReadNextToken @@ -461,10 +463,7 @@ type ParseSingleArithmeticExpression< IgnoreAny > ? Expression extends ArithmeticExpression - ? [ - ArithmeticExpression>, - Remainder - ] + ? [ArithmeticExpression>, Remainder] : Invalid<"Corrupted expression"> : ParseEntireArithmeticTree : Invalid<"Next token is not valid"> @@ -499,7 +498,7 @@ type ParseColumnAssignmentExpression< ? ParseEntireArithmeticTree< Token, Options - > extends infer Expression extends ArithmeticExpressionType + > extends infer Expression extends ArithmeticExpression ? Assignment extends ColumnArithmeticAssignment< infer Column, infer Op, @@ -514,10 +513,7 @@ type ParseColumnAssignmentExpression< /** * Type to prevent assumption about operations from causing mismatch */ -type AnyExpression = - | ArithmeticExpression - | GroupedArithmeticExpression - | ColumnArithmeticAssignment +type AnyExpression = LogicalExpression /** * Consume the entire arithmetic tree diff --git a/packages/sql/query/parser/filtering.ts b/packages/sql/query/parser/filtering.ts index 64c5340..9e34f7c 100644 --- a/packages/sql/query/parser/filtering.ts +++ b/packages/sql/query/parser/filtering.ts @@ -1,9 +1,13 @@ import type { Invalid } from "@telefrek/type-utils/common" import type { ColumnReference } from "../../ast/columns.js" -import type { LogicalExpression } from "../../ast/filtering.js" +import type { ColumnFilter, LogicalExpression } from "../../ast/expressions.js" import type { ValueTypes } from "../../ast/values.js" import type { CheckEqualParenthesis, NextToken } from "./normalize.js" -import type { GetComparisonOperations, ParserOptions } from "./options.js" +import type { + DEFAULT_PARSER_OPTIONS, + GetComparisonOperations, + ParserOptions, +} from "./options.js" import type { ExtractGroup, ParseValueOrReference } from "./utils.js" /** @@ -14,6 +18,8 @@ export type ParseLogicalExpression< Options extends ParserOptions > = ParseNextExpression +export type t = ParseNextExpression<"a - b + c >= d", DEFAULT_PARSER_OPTIONS> + type ParseNextExpression< SQL extends string, Options extends ParserOptions, @@ -38,6 +44,61 @@ type ParseNextExpression< : ParseValueOrReference extends infer CRef extends | ColumnReference | ValueTypes - ? CRef + ? CRef extends ColumnReference + ? ParseColumnExpression + : CRef : Token : Invalid<"failed to parse expression"> + +type ParseSingleExpression< + SQL extends string, + Options extends ParserOptions, + Expression +> = ReadNextToken extends [ + infer Token, + infer Remainder extends string +] + ? [Token, Expression, Remainder] + : ReadNextToken + +type ParseColumnExpression< + SQL extends string, + Options extends ParserOptions, + Column extends ColumnReference +> = ReadNextToken extends [ + infer Token, + infer Remainder extends string +] + ? Token extends GetComparisonOperations + ? ParseSingleExpression< + Remainder, + Options, + ColumnFilter + > + : Invalid<"nope"> + : ReadNextToken + +type ReadNextToken< + SQL extends string, + Options extends ParserOptions +> = NextToken extends [ + infer Token extends string, + infer Remainder extends string +] + ? Token extends GetComparisonOperations + ? [Token, Remainder] + : Token extends ")" + ? Invalid<"Invalid syntax, extra )"> + : Token extends "(" + ? ExtractGroup extends [ + infer Group extends string, + infer Rest extends string + ] + ? [Group, Rest] + : Invalid<"Corrupt group"> + : ParseValueOrReference extends infer CRef extends + | ColumnReference + | ValueTypes + ? [CRef, Remainder] + : Invalid<"Cannot map value"> + : Invalid<"No more tokens to extract"> diff --git a/packages/sql/query/parser/options.ts b/packages/sql/query/parser/options.ts index 222f233..2fe6bf4 100644 --- a/packages/sql/query/parser/options.ts +++ b/packages/sql/query/parser/options.ts @@ -1,14 +1,12 @@ import type { Flatten } from "@telefrek/type-utils/common" import { - DEFAULT_ARITHMETIC_ASSIGNMENT_OPS, - DEFAULT_ARITHMETIC_OPS, type ArithmeticAssignmentOperation, type ArithmeticOperation, -} from "../../ast/arithmetic.js" -import { - DEFAULT_COMPARISON_OPS, type ComparisonOperation, -} from "../../ast/filtering.js" + DEFAULT_ARITHMETIC_ASSIGNMENT_OPS, + DEFAULT_ARITHMETIC_OPS, + DEFAULT_COMPARISON_OPS, +} from "../../ast/expressions.js" /** * The options for what can be overridden in the parsing logic diff --git a/packages/sql/query/parser/select.ts b/packages/sql/query/parser/select.ts index d9bb382..645e146 100644 --- a/packages/sql/query/parser/select.ts +++ b/packages/sql/query/parser/select.ts @@ -1,9 +1,9 @@ import type { Flatten, Invalid } from "@telefrek/type-utils/common.js" import type { Trim } from "@telefrek/type-utils/strings" -import type { WhereClause } from "../../ast/filtering.js" import type { NamedQuery } from "../../ast/named.js" import type { SelectClause } from "../../ast/select.js" import type { TableReference } from "../../ast/tables.js" +import type { WhereClause } from "../../ast/where.js" import { parseSelectedColumns, type ParseSelectedColumns } from "./columns.js" import type { PartialParserResult } from "./common.js" import { takeUntil, type SplitSQL } from "./normalize.js" @@ -40,6 +40,7 @@ export function parseSelectClause( // Parse the optional where clause if (tokens.length > 0 && tokens[0] === "WHERE") { + tokens.shift() select = { ...select, ...parseWhere(tokens, options) } } diff --git a/packages/sql/query/parser/where.test.ts b/packages/sql/query/parser/where.test.ts new file mode 100644 index 0000000..2bcc1dd --- /dev/null +++ b/packages/sql/query/parser/where.test.ts @@ -0,0 +1,26 @@ +import { DefaultOptions } from "./options.js" +import { parseWhere } from "./where.js" + +describe("WHERE clauses should correctly parse", () => { + it("Should parse a simple where clause", () => { + const where = parseWhere("id >= 1", DefaultOptions) + expect(where).toStrictEqual({ + where: { + type: "ColumnFilter", + filter: { + type: "NumberValue", + value: 1, + }, + operation: ">=", + column: { + type: "ColumnReference", + reference: { + type: "UnboundColumnReference", + column: "id", + }, + alias: "id", + }, + }, + }) + }) +}) diff --git a/packages/sql/query/parser/where.ts b/packages/sql/query/parser/where.ts index 17745f3..5ad4ca5 100644 --- a/packages/sql/query/parser/where.ts +++ b/packages/sql/query/parser/where.ts @@ -1,15 +1,15 @@ import type { Flatten, Invalid } from "@telefrek/type-utils/common" -import type { Join, Trim } from "@telefrek/type-utils/strings" +import type { Trim } from "@telefrek/type-utils/strings" import type { ColumnReference } from "../../ast/columns.js" + import type { ColumnFilter, ComparisonOperation, LogicalExpression, LogicalTree, - LogicalTreeOperation, - WhereClause, -} from "../../ast/filtering.js" +} from "../../ast/expressions.js" import type { ValueTypes } from "../../ast/values.js" +import type { WhereClause } from "../../ast/where.js" import { parseColumnReference, type ParseColumnDetails } from "./columns.js" import type { PartialParserResult } from "./common.js" import { @@ -17,7 +17,6 @@ import { takeWhile, type ExtractUntil, type NextToken, - type SplitWords, } from "./normalize.js" import type { GetComparisonOperations, @@ -29,6 +28,22 @@ import { parseNextValue, type ExtractValue } from "./values.js" // This entire thing needs a re-write... +type CheckWhere< + SQL extends string, + Options extends ParserOptions +> = ExtractLogical extends LogicalExpression + ? SQL + : Invalid<"Cannot parse expression"> + +type CheckExpression = Exp extends LogicalExpression + ? WhereClause + : never + +export function parseWhere( + sql: CheckWhere, + options: Options +): CheckExpression> + /** * Parse the {@link WhereClause} from the token stack * @@ -39,15 +54,21 @@ import { parseNextValue, type ExtractValue } from "./values.js" export function parseWhere( tokens: string[], options: ParserOptions +): WhereClause | object + +export function parseWhere( + sql: unknown, + options: ParserOptions ): WhereClause | object { + // Get the token stack + const tokens = Array.isArray(sql) + ? (sql as string[]) + : (sql as string).split(" ") + if (tokens.length === 0) { return {} } - if ("WHERE" !== tokens.shift()) { - throw new Error(`Invalid where tokens`) - } - return { where: parseLogicalExpression(tokens, options), } @@ -70,9 +91,9 @@ function parseLogicalExpression( return { type: "ColumnFilter", column: parseColumnReference(left.split(" ")), - op: op as ComparisonOperation, + operation: op as ComparisonOperation, filter: parseNextValue(segments, options)!, - } + } as ColumnFilter } /** @@ -84,7 +105,7 @@ export type ExtractWhere< > = Current extends PartialParserResult ? SQL extends `${infer QuerySegment} WHERE ${infer Where}` ? ParseExpressionTree< - Join>, + Where, Options > extends infer Exp extends LogicalExpression ? PartialParserResult>> @@ -92,31 +113,6 @@ export type ExtractWhere< : Current : never -/** - * Split the where statement by potential filtering operations - */ -type SplitWhere = T extends `${infer Left}<>${infer Right}` - ? [...SplitWhere, "<>", ...SplitWhere] - : T extends `${infer Left}>${infer Next}${infer Right}` - ? SplitEqual"> - : T extends `${infer Left}<${infer Next}${infer Right}` - ? SplitEqual - : T extends `${infer Left}=${infer Right}` - ? [...SplitWhere, "=", ...SplitWhere] - : SplitWords - -/** - * Split out a possible trailing '=' character - */ -type SplitEqual< - Left extends string, - Next extends string, - Right extends string, - C extends string -> = Next extends "=" - ? [...SplitWhere, `${C}=`, ...SplitWhere] - : [...SplitWhere, C, ...SplitWhere<`${Next}${Right}`>] - /** * Parse an expression tree */ @@ -145,7 +141,7 @@ type ParseExpressionTree< type ExtractLogical< SQL extends string, Options extends ParserOptions -> = ExtractUntil extends [ +> = ExtractUntil extends [ infer Left extends string, infer Remainder extends string ] @@ -153,7 +149,7 @@ type ExtractLogical< infer Operation extends string, infer Right extends string ] - ? [Operation] extends [LogicalTreeOperation] + ? [Operation] extends [LogicalTree] ? CheckLogicalTree< ParseExpressionTree, Operation, @@ -175,7 +171,7 @@ type ExtractLogical< */ type CheckLogicalTree = Left extends LogicalExpression ? Right extends LogicalExpression - ? Operation extends LogicalTreeOperation + ? Operation extends "AND" | "OR" ? LogicalTree : Invalid<"Invalid logical tree detected"> : Right extends Invalid diff --git a/packages/sql/query/visitor/common.ts b/packages/sql/query/visitor/common.ts index 128a0ae..d60e508 100644 --- a/packages/sql/query/visitor/common.ts +++ b/packages/sql/query/visitor/common.ts @@ -1,10 +1,12 @@ import type { ColumnReference } from "../../ast/columns.js" -import type { - ColumnFilter, - LogicalExpression, - LogicalTree, - WhereClause, -} from "../../ast/filtering.js" +import { + isLogicalOperation, + type ColumnFilter, + type LogicalExpression, + type LogicalOperation, + type LogicalTree, +} from "../../ast/expressions.js" + import type { InsertClause, QueryClause, @@ -14,6 +16,7 @@ import type { import type { SelectClause } from "../../ast/select.js" import type { TableReference } from "../../ast/tables.js" import type { ValueTypes } from "../../ast/values.js" +import type { WhereClause } from "../../ast/where.js" import { DefaultQueryProvider, type QueryAstVisitor } from "./types.js" /** @@ -77,7 +80,7 @@ export class DefaultQueryVisitor // Check WHERE if ("where" in select) { this.append("WHERE") - this.visitWhereClause(select as Readonly) + this.visitWhereClause(select as unknown as Readonly) } } @@ -145,13 +148,19 @@ export class DefaultQueryVisitor visitLogicalExpression( expression: Readonly ): void { - switch (expression.type) { - case "LogicalTree": - this.visitLogicalTree(expression as Readonly) - break - case "ColumnFilter": - this.visitColumnFilter(expression as Readonly) - break + if (isLogicalOperation(expression)) { + switch (expression.type) { + case "LogicalTree": + this.visitLogicalTree( + expression as LogicalOperation as Readonly + ) + break + case "ColumnFilter": + this.visitColumnFilter( + expression as LogicalOperation as Readonly + ) + break + } } } @@ -159,18 +168,20 @@ export class DefaultQueryVisitor // TODO: Handle subquery grouping... this.visitLogicalExpression(tree.left as Readonly) - this.append(tree.op) + this.append(tree.operation) this.visitLogicalExpression(tree.right as Readonly) } visitColumnFilter(filter: T): void { this.visitColumnReference(filter.column) - this.append(filter.op) - if (filter.filter.type === "ColumnReference") { - this.visitColumnReference(filter.filter) + this.append(filter.operation) + if (isLogicalOperation(filter.filter)) { + this.visitLogicalExpression(filter.filter) + } else if (filter.filter.type === "ColumnReference") { + this.visitColumnReference(filter.filter as ColumnReference) } else { - this.visitValueType(filter.filter) + this.visitValueType(filter.filter as ValueTypes) } } diff --git a/packages/sql/query/visitor/types.ts b/packages/sql/query/visitor/types.ts index a1e56fe..6b4f403 100644 --- a/packages/sql/query/visitor/types.ts +++ b/packages/sql/query/visitor/types.ts @@ -3,8 +3,8 @@ import type { ColumnFilter, LogicalExpression, LogicalTree, - WhereClause, -} from "../../ast/filtering.js" +} from "../../ast/expressions.js" + import type { InsertClause, QueryClause, @@ -14,6 +14,7 @@ import type { import type { SelectClause } from "../../ast/select.js" import type { TableReference } from "../../ast/tables.js" import type { ValueTypes } from "../../ast/values.js" +import type { WhereClause } from "../../ast/where.js" /** * A visitor for exploring the SQL AST From 9b4a07633b4a8cd7ef2e8b6cba8a7adb94559a6d Mon Sep 17 00:00:00 2001 From: Nathan Northcutt Date: Mon, 2 Sep 2024 17:03:33 +0200 Subject: [PATCH 17/21] parser should handle where, not select --- packages/sql/query/parser/select.ts | 5 +---- packages/sql/query/parser/where.ts | 6 ++++++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/sql/query/parser/select.ts b/packages/sql/query/parser/select.ts index 645e146..e841f3f 100644 --- a/packages/sql/query/parser/select.ts +++ b/packages/sql/query/parser/select.ts @@ -39,10 +39,7 @@ export function parseSelectClause( } // Parse the optional where clause - if (tokens.length > 0 && tokens[0] === "WHERE") { - tokens.shift() - select = { ...select, ...parseWhere(tokens, options) } - } + select = { ...select, ...parseWhere(tokens, options) } return { type: "SelectClause", diff --git a/packages/sql/query/parser/where.ts b/packages/sql/query/parser/where.ts index 5ad4ca5..ef1c8d0 100644 --- a/packages/sql/query/parser/where.ts +++ b/packages/sql/query/parser/where.ts @@ -69,6 +69,12 @@ export function parseWhere( return {} } + if (tokens[0] !== "WHERE") { + return {} + } + + tokens.shift() + return { where: parseLogicalExpression(tokens, options), } From 3f21cb76447aae4568c87c5530fe8925694d078e Mon Sep 17 00:00:00 2001 From: Nathan Northcutt Date: Mon, 2 Sep 2024 17:06:02 +0200 Subject: [PATCH 18/21] better --- packages/sql/query/parser/select.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/sql/query/parser/select.ts b/packages/sql/query/parser/select.ts index e841f3f..c44abaf 100644 --- a/packages/sql/query/parser/select.ts +++ b/packages/sql/query/parser/select.ts @@ -33,14 +33,12 @@ export function parseSelectClause( options: ParserOptions ): SelectClause { // Extract the core select - let select = { + const select = { columns: parseSelectedColumns(takeUntil(tokens, ["FROM"])), ...parseFrom(tokens, options), + ...parseWhere(tokens, options), } - // Parse the optional where clause - select = { ...select, ...parseWhere(tokens, options) } - return { type: "SelectClause", ...select, From 48fa0d8a9c210286c8dbe95f37e1dda434a35113 Mon Sep 17 00:00:00 2001 From: Nathan Northcutt Date: Thu, 12 Sep 2024 13:01:24 -0400 Subject: [PATCH 19/21] sync everything to fix local wsl --- packages/sql/query/parser/expressions.ts | 586 +++++++++++++++++++++++ packages/sql/query/parser/where.test.ts | 41 +- 2 files changed, 605 insertions(+), 22 deletions(-) create mode 100644 packages/sql/query/parser/expressions.ts diff --git a/packages/sql/query/parser/expressions.ts b/packages/sql/query/parser/expressions.ts new file mode 100644 index 0000000..351ec50 --- /dev/null +++ b/packages/sql/query/parser/expressions.ts @@ -0,0 +1,586 @@ +import type { IgnoreAny, Invalid } from "@telefrek/type-utils/common" + +import type { ColumnReference } from "../../ast/columns.js" +import type { + ArithmeticExpression, + ColumnArithmeticAssignment, + ColumnFilter, + LogicalExpression, + LogicalGroup, +} from "../../ast/expressions.js" +import type { ValueTypes } from "../../ast/values.js" +import type { NextToken } from "./normalize.js" +import { + type GetArithmeticOperations, + type GetAssignmentOperations, + type GetComparisonOperations, + type GetOverridableTokens, + type ParserOptions, +} from "./options.js" +import { + extractGroup, + parseValueOrReference, + type ExtractGroup, + type ParseValueOrReference, +} from "./utils.js" + +/** + * Utility to get the full expression type + */ +type GetFullExpressionType< + SQL extends string, + Options extends ParserOptions +> = ParseArithmeticExpression extends [infer Expression, ""] + ? Expression + : never + +/** + * Parse the SQL string as an expression + * @param sql The SQL to parse as an expression + * @param options The options to use + */ +export function parseArithmeticExpression< + SQL extends string, + Options extends ParserOptions +>(sql: SQL, options: Options): GetFullExpressionType + +/** + * Parse the next arithmetic expression from the stack + * + * @param tokens The current token stack + * @param options The parser options to use + * @param current The current expression if one exists + */ +export function parseArithmeticExpression< + Expression extends AnyExpression, + Options extends ParserOptions +>( + tokens: string[], + options: Options, + current?: Expression +): AnyExpression | undefined + +// Implementation +export function parseArithmeticExpression( + sql: unknown, + options: Options, + current?: AnyExpression +): unknown { + const tokens = typeof sql === "string" ? sql.split(" ") : (sql as string[]) + + // Create a copy of the tokens in case of partial reads + const copy = [...tokens] + + if (current !== undefined) { + const token = readNextToken(copy, options) + if (typeof token === "string") { + // Only allow additional arithmetic + if (options.tokens.arithmetic.indexOf(token) >= 0) { + return parseSingleArithmeticExpression(copy, options, { + type: "ArithmeticExpression", + left: current, + operation: token, + }) + } + } else if (token === undefined) { + return copy.length === 0 ? current : undefined + } + + return + } + + const next = parseNextArithmeticExpression(copy, options) + if (next !== undefined) { + const fullExpression = parseArithmeticExpression( + copy, + options, + next as AnyExpression + ) + + const diff = tokens.length - copy.length + tokens.splice(0, diff) + + return fullExpression ?? next + } + + return +} + +/** + * Parse the entire token stack as an expression or return undefined + * + * @param tokens The current token stack + * @param options The parsing options + * @returns Either a fully consumed expression or undefined + */ +function parseGroupExpression( + tokens: string[], + options: ParserOptions +): LogicalGroup | undefined { + const copy = [...tokens] + + const expression = parseArithmeticExpression(copy, options) + if ( + expression !== undefined && + copy.length === 0 && + expression.type === "ArithmeticExpression" + ) { + tokens.splice(0, tokens.length) + return { + type: "LogicalGroup", + operation: "LogicalGroup", + expression, + } + } + + return +} + +/** + * Extract the next valid expression chunk and the remaining string + */ +export type ParseArithmeticExpression< + SQL extends string, + Options extends ParserOptions, + Current extends AnyExpression = never +> = [Current] extends [never] + ? ParseNextArithmeticExpression extends [ + infer Expression extends AnyExpression, + infer Remainder extends string + ] + ? Remainder extends "" + ? [Expression, ""] + : ParseArithmeticExpression + : ParseNextArithmeticExpression + : ReadNextToken extends [ + infer Token, + infer Remainder extends string + ] + ? Token extends GetArithmeticOperations + ? ParseSingleArithmeticExpression< + Remainder, + Options, + ArithmeticExpression + > extends [ + infer Expression extends AnyExpression, + infer Rest extends string + ] + ? Rest extends "" + ? [Expression, ""] + : ParseArithmeticExpression + : ParseSingleArithmeticExpression< + Remainder, + Options, + ArithmeticExpression + > + : [Current, SQL] // Return + : [Current, SQL] // Return expression and remainder + +/** + * Get the types of tokens supported + */ +type GetTokenTypes = + | GetArithmeticOperations + | GetAssignmentOperations + | ColumnReference + | ValueTypes + +/** + * Get the next token value + */ +type ReadNextToken< + SQL extends string, + Options extends ParserOptions +> = NextToken extends [ + infer Token extends string, + infer Remainder extends string +] + ? Token extends GetOverridableTokens + ? [Token, Remainder] + : Token extends ")" + ? Invalid<"Invalid syntax, extra )"> + : Token extends "(" + ? ExtractGroup extends [ + infer Group extends string, + infer Rest extends string + ] + ? [Group, Rest] + : Invalid<"Corrupt group"> + : ParseValueOrReference extends infer CRef extends + | ColumnReference + | ValueTypes + ? [CRef, Remainder] + : Invalid<"Cannot map value"> + : Invalid<"No more tokens to extract"> + +/** + * Read the next value from the stack + * @param tokens The current token stack + * @param options The parsing options to use + * @returns A column reference, value or group + */ +export function readNextToken( + tokens: string[], + options: ParserOptions +): ValueTypes | ColumnReference | string[] | string | undefined { + if (tokens.length === 0) { + return + } + + switch (true) { + case tokens[0] === ")": + throw new Error("Corrupt group") + case tokens[0] === "(": + tokens.shift() + return extractGroup(tokens) + case options.tokens.arithmetic.indexOf(tokens[0]) >= 0: + return tokens.shift()! + case options.tokens.assignments.indexOf(tokens[0]) >= 0: + return tokens.shift()! + } + + return parseValueOrReference(tokens, options) +} + +/** + * Parse the next {@link ArithmeticExpression} from the provided string + */ +type ParseNextArithmeticExpression< + SQL extends string, + Options extends ParserOptions +> = ReadNextToken extends [ + infer Token, + infer Remainder extends string +] + ? Token extends ColumnReference + ? ParseColumnExpression + : Token extends ValueTypes + ? ParseValueExpression + : Token extends GetTokenTypes + ? Invalid<"Cannot start an operation with an assignment or arithmetic sign"> + : Token extends string + ? ParseEntireArithmeticTree< + Token, + Options + > extends infer Tree extends ArithmeticExpression< + IgnoreAny, + string, + IgnoreAny + > + ? [LogicalGroup, Remainder] + : ParseEntireArithmeticTree + : Invalid<"Invalid token"> + : ReadNextToken + +function parseNextArithmeticExpression( + tokens: string[], + options: ParserOptions +): Partial | undefined { + const token = readNextToken(tokens, options) + if (token === undefined) { + return + } + + // Group + if (Array.isArray(token)) { + return parseGroupExpression(token, options) + } else if (typeof token === "string") { + return + } else if (token.type === "ColumnReference") { + return parseColumnExpression(tokens, options, token) + } else { + return parseValueExpression(tokens, options, token) + } +} + +/** + * Parse expressions starting with a column + */ +type ParseColumnExpression< + SQL extends string, + Options extends ParserOptions, + Column extends ColumnReference +> = ReadNextToken extends [ + infer Token, + infer Remainder extends string +] + ? Token extends GetArithmeticOperations + ? ParseSingleArithmeticExpression< + Remainder, + Options, + ArithmeticExpression + > + : Token extends GetAssignmentOperations + ? ParseColumnAssignmentExpression< + Remainder, + Options, + ColumnArithmeticAssignment + > + : Token extends GetComparisonOperations + ? ParseColumnFilter> + : Invalid<"Column must be followed by an assignment or arithmetic operation"> + : ReadNextToken + +type ParseColumnFilter< + SQL extends string, + Options extends ParserOptions, + Comparison extends ColumnFilter +> = Comparison extends ColumnFilter + ? ReadNextToken extends [ + infer Token, + infer Remainder extends string + ] + ? Token extends ColumnReference + ? [ColumnFilter, Remainder] + : Token extends ValueTypes + ? [ColumnFilter, Remainder] + : Token extends string + ? Invalid<"foo"> + : Invalid<"Invalid Token type"> + : ReadNextToken + : Invalid<"Corrupt ColumnFilter"> + +function parseColumnExpression( + tokens: string[], + options: ParserOptions, + column: ColumnReference +): AnyExpression | undefined { + const token = readNextToken(tokens, options) + if (typeof token === "string") { + if (options.tokens.assignments.indexOf(token) >= 0) { + return parseColumnAssignmentExpression(tokens, options, { + type: "ColumnArithmeticAssignment", + column, + operation: token, + }) + } else { + return parseSingleArithmeticExpression(tokens, options, { + type: "ArithmeticExpression", + left: column, + operation: token, + }) + } + } + + return +} + +function parseColumnAssignmentExpression< + Assignment extends Partial< + ColumnArithmeticAssignment + > +>( + tokens: string[], + options: ParserOptions, + assignment: Assignment +): ColumnArithmeticAssignment | undefined { + const token = readNextToken(tokens, options) + if (token === undefined) { + return + } + + if (typeof token === "string") { + return + } else if (Array.isArray(token)) { + const value = parseGroupExpression(token, options) + if (value === undefined) return + return { + ...assignment, + value, + } as ColumnArithmeticAssignment + } + + return { + ...assignment, + value: token, + } as ColumnArithmeticAssignment +} + +function parseSingleArithmeticExpression< + Expression extends Partial> +>( + tokens: string[], + options: ParserOptions, + expression: Expression +): ArithmeticExpression | undefined { + const token = readNextToken(tokens, options) + if (token === undefined) { + return + } + + if (typeof token === "string") { + return + } else if (Array.isArray(token)) { + const right = parseGroupExpression(token, options) + if (right === undefined) return + return { ...expression, right } as ArithmeticExpression< + IgnoreAny, + string, + IgnoreAny + > + } + + return { + ...expression, + right: token, + } as ArithmeticExpression +} + +/** + * Parse expressions starting with a value + */ +type ParseValueExpression< + SQL extends string, + Options extends ParserOptions, + Value extends ValueTypes +> = ReadNextToken extends [ + infer Token, + infer Remainder extends string +] + ? Token extends GetArithmeticOperations + ? [ArithmeticExpression, Remainder] + : Invalid<"Value must be followed by an arithmetic operation"> + : ReadNextToken + +function parseValueExpression( + tokens: string[], + options: ParserOptions, + value: ValueTypes +): Partial> | undefined { + const token = readNextToken(tokens, options) + if ( + typeof token === "string" && + options.tokens.arithmetic.indexOf(token) >= 0 + ) { + return { + type: "ArithmeticExpression", + left: value, + operation: token, + } + } + + return +} + +/** + * Parse only the next segment + */ +type ParseSingleArithmeticExpression< + SQL extends string, + Options extends ParserOptions, + Expression extends AnyExpression +> = ReadNextToken extends [ + infer Token, + infer Remainder extends string +] + ? Token extends ColumnReference | ValueTypes + ? Expression extends ArithmeticExpression + ? [ArithmeticExpression, Remainder] + : Invalid<"Corrupt expression"> + : Token extends GetTokenTypes + ? Invalid<"Right hand side of expression must be a value, column or other expression"> + : Token extends string + ? ParseEntireArithmeticTree< + Token, + Options + > extends infer Right extends ArithmeticExpression< + IgnoreAny, + string, + IgnoreAny + > + ? Expression extends ArithmeticExpression + ? [ArithmeticExpression>, Remainder] + : Invalid<"Corrupted expression"> + : ParseEntireArithmeticTree + : Invalid<"Next token is not valid"> + : ReadNextToken + +/** + * Parse a column assignment + * + * Note: To be valid, the entire remainder must be consumable... + */ +type ParseColumnAssignmentExpression< + SQL extends string, + Options extends ParserOptions, + Assignment extends ColumnArithmeticAssignment +> = ReadNextToken extends [ + infer Token, + infer Remainder extends string +] + ? Token extends ValueTypes | ColumnReference + ? Assignment extends ColumnArithmeticAssignment< + infer Column, + infer Op, + infer _ + > + ? Remainder extends "" + ? ColumnArithmeticAssignment + : Invalid<"Cannot have assignment with trailing information"> + : Invalid<"Corrupt assignment"> + : Token extends GetTokenTypes + ? Invalid<"Right hand side of assignment must be a value, column or other expression"> + : Token extends string + ? ParseEntireArithmeticTree< + Token, + Options + > extends infer Expression extends ArithmeticExpression + ? Assignment extends ColumnArithmeticAssignment< + infer Column, + infer Op, + infer _ + > + ? ColumnArithmeticAssignment + : Invalid<"Corrupted column assignment"> + : ParseEntireArithmeticTree + : Invalid<"Invalid grouping in column assignment"> + : ReadNextToken + +/** + * Type to prevent assumption about operations from causing mismatch + */ +type AnyExpression = LogicalExpression + +/** + * Consume the entire arithmetic tree + */ +type ParseEntireArithmeticTree< + SQL extends string, + Options extends ParserOptions +> = ParseArithmeticExpression extends [ + infer Expression, + infer Remainder extends string +] + ? Remainder extends "" + ? Expression + : Invalid<"Failed to consume the entire SQL"> + : ParseArithmeticExpression + +// Start by parsing the next unit (column, value, token) +// Get next "operator" +// Parse the next chunk, repeat until done + +export type ParseNextLogicalExpression< + SQL extends string, + Options extends ParserOptions, + Previous extends LogicalExpression = never +> = ReadNextToken extends [ + infer Token, + infer Remainder extends string +] + ? Token extends ColumnReference + ? [Token, Remainder] + : Token extends ValueTypes + ? [Token, Remainder] + : Token extends GetAssignmentOperations + ? Previous extends ColumnReference + ? [ColumnArithmeticAssignment, Remainder] + : Invalid<"Cannot use assignment on non-column"> + : Token extends GetArithmeticOperations + ? [ArithmeticExpression, Remainder] + : Token extends GetComparisonOperations + ? Previous extends ColumnReference + ? [ColumnFilter, Remainder] + : Invalid<"Cannot use comparison on non-column"> + : Invalid<"Corrupt token type"> + : ReadNextToken diff --git a/packages/sql/query/parser/where.test.ts b/packages/sql/query/parser/where.test.ts index 2bcc1dd..44fd136 100644 --- a/packages/sql/query/parser/where.test.ts +++ b/packages/sql/query/parser/where.test.ts @@ -1,26 +1,23 @@ -import { DefaultOptions } from "./options.js" -import { parseWhere } from "./where.js" - describe("WHERE clauses should correctly parse", () => { it("Should parse a simple where clause", () => { - const where = parseWhere("id >= 1", DefaultOptions) - expect(where).toStrictEqual({ - where: { - type: "ColumnFilter", - filter: { - type: "NumberValue", - value: 1, - }, - operation: ">=", - column: { - type: "ColumnReference", - reference: { - type: "UnboundColumnReference", - column: "id", - }, - alias: "id", - }, - }, - }) + // const where = parseWhere("id >= 1", DefaultOptions) + // expect(where).toStrictEqual({ + // where: { + // type: "ColumnFilter", + // filter: { + // type: "NumberValue", + // value: 1, + // }, + // operation: ">=", + // column: { + // type: "ColumnReference", + // reference: { + // type: "UnboundColumnReference", + // column: "id", + // }, + // alias: "id", + // }, + // }, + // }) }) }) From 4bc5a303251d79503a59a9706feecb8473f22d55 Mon Sep 17 00:00:00 2001 From: Nathan Northcutt Date: Mon, 16 Sep 2024 17:03:47 -0400 Subject: [PATCH 20/21] sometimes easier to just delete and begin again with an easier solution --- packages/sql/query/parser/arithmetic.test.ts | 2 +- packages/sql/query/parser/arithmetic.ts | 531 ------------------- packages/sql/query/parser/expressions.ts | 364 ++++--------- 3 files changed, 110 insertions(+), 787 deletions(-) delete mode 100644 packages/sql/query/parser/arithmetic.ts diff --git a/packages/sql/query/parser/arithmetic.test.ts b/packages/sql/query/parser/arithmetic.test.ts index 8213fa8..d74d43b 100644 --- a/packages/sql/query/parser/arithmetic.test.ts +++ b/packages/sql/query/parser/arithmetic.test.ts @@ -1,4 +1,4 @@ -import { parseArithmeticExpression } from "./arithmetic.js" +import { parseArithmeticExpression } from "./expressions.js" import { DefaultOptions } from "./options.js" describe("Arithmetic parsing should correctly extract types and values", () => { diff --git a/packages/sql/query/parser/arithmetic.ts b/packages/sql/query/parser/arithmetic.ts deleted file mode 100644 index 2264c60..0000000 --- a/packages/sql/query/parser/arithmetic.ts +++ /dev/null @@ -1,531 +0,0 @@ -import type { IgnoreAny, Invalid } from "@telefrek/type-utils/common" - -import type { ColumnReference } from "../../ast/columns.js" -import type { - ArithmeticExpression, - ColumnArithmeticAssignment, - LogicalExpression, - LogicalGroup, -} from "../../ast/expressions.js" -import type { ValueTypes } from "../../ast/values.js" -import type { NextToken } from "./normalize.js" -import { - type GetArithmeticOperations, - type GetAssignmentOperations, - type ParserOptions, -} from "./options.js" -import { - extractGroup, - parseValueOrReference, - type ExtractGroup, - type ParseValueOrReference, -} from "./utils.js" - -/** - * Utility to get the full expression type - */ -type GetFullExpressionType< - SQL extends string, - Options extends ParserOptions -> = ParseArithmeticExpression extends [infer Expression, ""] - ? Expression - : never - -/** - * Parse the SQL string as an expression - * @param sql The SQL to parse as an expression - * @param options The options to use - */ -export function parseArithmeticExpression< - SQL extends string, - Options extends ParserOptions ->(sql: SQL, options: Options): GetFullExpressionType - -/** - * Parse the next arithmetic expression from the stack - * - * @param tokens The current token stack - * @param options The parser options to use - * @param current The current expression if one exists - */ -export function parseArithmeticExpression< - Expression extends AnyExpression, - Options extends ParserOptions ->( - tokens: string[], - options: Options, - current?: Expression -): AnyExpression | undefined - -// Implementation -export function parseArithmeticExpression( - sql: unknown, - options: Options, - current?: AnyExpression -): unknown { - const tokens = typeof sql === "string" ? sql.split(" ") : (sql as string[]) - - // Create a copy of the tokens in case of partial reads - const copy = [...tokens] - - if (current !== undefined) { - const token = readNextToken(copy, options) - if (typeof token === "string") { - // Only allow additional arithmetic - if (options.tokens.arithmetic.indexOf(token) >= 0) { - return parseSingleArithmeticExpression(copy, options, { - type: "ArithmeticExpression", - left: current, - operation: token, - }) - } - } else if (token === undefined) { - return copy.length === 0 ? current : undefined - } - - return - } - - const next = parseNextArithmeticExpression(copy, options) - if (next !== undefined) { - const fullExpression = parseArithmeticExpression( - copy, - options, - next as AnyExpression - ) - - const diff = tokens.length - copy.length - tokens.splice(0, diff) - - return fullExpression ?? next - } - - return -} - -/** - * Parse the entire token stack as an expression or return undefined - * - * @param tokens The current token stack - * @param options The parsing options - * @returns Either a fully consumed expression or undefined - */ -function parseGroupExpression( - tokens: string[], - options: ParserOptions -): LogicalGroup | undefined { - const copy = [...tokens] - - const expression = parseArithmeticExpression(copy, options) - if ( - expression !== undefined && - copy.length === 0 && - expression.type === "ArithmeticExpression" - ) { - tokens.splice(0, tokens.length) - return { - type: "LogicalGroup", - operation: "LogicalGroup", - expression, - } - } - - return -} - -/** - * Extract the next valid expression chunk and the remaining string - */ -export type ParseArithmeticExpression< - SQL extends string, - Options extends ParserOptions, - Current extends AnyExpression = never -> = [Current] extends [never] - ? ParseNextArithmeticExpression extends [ - infer Expression extends AnyExpression, - infer Remainder extends string - ] - ? Remainder extends "" - ? [Expression, ""] - : ParseArithmeticExpression - : ParseNextArithmeticExpression - : ReadNextToken extends [ - infer Token, - infer Remainder extends string - ] - ? Token extends GetArithmeticOperations - ? ParseSingleArithmeticExpression< - Remainder, - Options, - ArithmeticExpression - > extends [ - infer Expression extends AnyExpression, - infer Rest extends string - ] - ? Rest extends "" - ? [Expression, ""] - : ParseArithmeticExpression - : ParseSingleArithmeticExpression< - Remainder, - Options, - ArithmeticExpression - > - : [Current, SQL] // Return - : [Current, SQL] // Return expression and remainder - -/** - * Get the types of tokens supported - */ -type GetTokenTypes = - | GetArithmeticOperations - | GetAssignmentOperations - | ColumnReference - | ValueTypes - -/** - * Get the next token value - */ -type ReadNextToken< - SQL extends string, - Options extends ParserOptions -> = NextToken extends [ - infer Token extends string, - infer Remainder extends string -] - ? Token extends GetArithmeticOperations - ? [Token, Remainder] - : Token extends GetAssignmentOperations - ? [Token, Remainder] - : Token extends ")" - ? Invalid<"Invalid syntax, extra )"> - : Token extends "(" - ? ExtractGroup extends [ - infer Group extends string, - infer Rest extends string - ] - ? [Group, Rest] - : Invalid<"Corrupt group"> - : ParseValueOrReference extends infer CRef extends - | ColumnReference - | ValueTypes - ? [CRef, Remainder] - : Invalid<"Cannot map value"> - : Invalid<"No more tokens to extract"> - -/** - * Read the next value from the stack - * @param tokens The current token stack - * @param options The parsing options to use - * @returns A column reference, value or group - */ -export function readNextToken( - tokens: string[], - options: ParserOptions -): ValueTypes | ColumnReference | string[] | string | undefined { - if (tokens.length === 0) { - return - } - - switch (true) { - case tokens[0] === ")": - throw new Error("Corrupt group") - case tokens[0] === "(": - tokens.shift() - return extractGroup(tokens) - case options.tokens.arithmetic.indexOf(tokens[0]) >= 0: - return tokens.shift()! - case options.tokens.assignments.indexOf(tokens[0]) >= 0: - return tokens.shift()! - } - - return parseValueOrReference(tokens, options) -} - -/** - * Parse the next {@link ArithmeticExpression} from the provided string - */ -type ParseNextArithmeticExpression< - SQL extends string, - Options extends ParserOptions -> = ReadNextToken extends [ - infer Token, - infer Remainder extends string -] - ? Token extends ColumnReference - ? ParseColumnExpression - : Token extends ValueTypes - ? ParseValueExpression - : Token extends GetTokenTypes - ? Invalid<"Cannot start an operation with an assignment or arithmetic sign"> - : Token extends string - ? ParseEntireArithmeticTree< - Token, - Options - > extends infer Tree extends ArithmeticExpression< - IgnoreAny, - string, - IgnoreAny - > - ? [LogicalGroup, Remainder] - : ParseEntireArithmeticTree - : Invalid<"Invalid token"> - : ReadNextToken - -function parseNextArithmeticExpression( - tokens: string[], - options: ParserOptions -): Partial | undefined { - const token = readNextToken(tokens, options) - if (token === undefined) { - return - } - - // Group - if (Array.isArray(token)) { - return parseGroupExpression(token, options) - } else if (typeof token === "string") { - return - } else if (token.type === "ColumnReference") { - return parseColumnExpression(tokens, options, token) - } else { - return parseValueExpression(tokens, options, token) - } -} - -/** - * Parse expressions starting with a column - */ -type ParseColumnExpression< - SQL extends string, - Options extends ParserOptions, - Column extends ColumnReference -> = ReadNextToken extends [ - infer Token, - infer Remainder extends string -] - ? Token extends GetArithmeticOperations - ? ParseSingleArithmeticExpression< - Remainder, - Options, - ArithmeticExpression - > - : Token extends GetAssignmentOperations - ? ParseColumnAssignmentExpression< - Remainder, - Options, - ColumnArithmeticAssignment - > - : Invalid<"Column must be followed by an assignment or arithmetic operation"> - : ReadNextToken - -function parseColumnExpression( - tokens: string[], - options: ParserOptions, - column: ColumnReference -): AnyExpression | undefined { - const token = readNextToken(tokens, options) - if (typeof token === "string") { - if (options.tokens.assignments.indexOf(token) >= 0) { - return parseColumnAssignmentExpression(tokens, options, { - type: "ColumnArithmeticAssignment", - column, - operation: token, - }) - } else { - return parseSingleArithmeticExpression(tokens, options, { - type: "ArithmeticExpression", - left: column, - operation: token, - }) - } - } - - return -} - -function parseColumnAssignmentExpression< - Assignment extends Partial< - ColumnArithmeticAssignment - > ->( - tokens: string[], - options: ParserOptions, - assignment: Assignment -): ColumnArithmeticAssignment | undefined { - const token = readNextToken(tokens, options) - if (token === undefined) { - return - } - - if (typeof token === "string") { - return - } else if (Array.isArray(token)) { - const value = parseGroupExpression(token, options) - if (value === undefined) return - return { - ...assignment, - value, - } - } - - return { - ...assignment, - value: token, - } -} - -function parseSingleArithmeticExpression< - Expression extends Partial> ->( - tokens: string[], - options: ParserOptions, - expression: Expression -): ArithmeticExpression | undefined { - const token = readNextToken(tokens, options) - if (token === undefined) { - return - } - - if (typeof token === "string") { - return - } else if (Array.isArray(token)) { - const right = parseGroupExpression(token, options) - if (right === undefined) return - return { ...expression, right } - } - - return { - ...expression, - right: token, - } -} - -/** - * Parse expressions starting with a value - */ -type ParseValueExpression< - SQL extends string, - Options extends ParserOptions, - Value extends ValueTypes -> = ReadNextToken extends [ - infer Token, - infer Remainder extends string -] - ? Token extends GetArithmeticOperations - ? [ArithmeticExpression, Remainder] - : Invalid<"Value must be followed by an arithmetic operation"> - : ReadNextToken - -function parseValueExpression( - tokens: string[], - options: ParserOptions, - value: ValueTypes -): Partial> | undefined { - const token = readNextToken(tokens, options) - if ( - typeof token === "string" && - options.tokens.arithmetic.indexOf(token) >= 0 - ) { - return { - type: "ArithmeticExpression", - left: value, - operation: token, - } - } - - return -} - -/** - * Parse only the next segment - */ -type ParseSingleArithmeticExpression< - SQL extends string, - Options extends ParserOptions, - Expression extends AnyExpression -> = ReadNextToken extends [ - infer Token, - infer Remainder extends string -] - ? Token extends ColumnReference | ValueTypes - ? Expression extends ArithmeticExpression - ? [ArithmeticExpression, Remainder] - : Invalid<"Corrupt expression"> - : Token extends GetTokenTypes - ? Invalid<"Right hand side of expression must be a value, column or other expression"> - : Token extends string - ? ParseEntireArithmeticTree< - Token, - Options - > extends infer Right extends ArithmeticExpression< - IgnoreAny, - string, - IgnoreAny - > - ? Expression extends ArithmeticExpression - ? [ArithmeticExpression>, Remainder] - : Invalid<"Corrupted expression"> - : ParseEntireArithmeticTree - : Invalid<"Next token is not valid"> - : ReadNextToken - -/** - * Parse a column assignment - * - * Note: To be valid, the entire remainder must be consumable... - */ -type ParseColumnAssignmentExpression< - SQL extends string, - Options extends ParserOptions, - Assignment extends ColumnArithmeticAssignment -> = ReadNextToken extends [ - infer Token, - infer Remainder extends string -] - ? Token extends ValueTypes | ColumnReference - ? Assignment extends ColumnArithmeticAssignment< - infer Column, - infer Op, - infer _ - > - ? Remainder extends "" - ? ColumnArithmeticAssignment - : Invalid<"Cannot have assignment with trailing information"> - : Invalid<"Corrupt assignment"> - : Token extends GetTokenTypes - ? Invalid<"Right hand side of assignment must be a value, column or other expression"> - : Token extends string - ? ParseEntireArithmeticTree< - Token, - Options - > extends infer Expression extends ArithmeticExpression - ? Assignment extends ColumnArithmeticAssignment< - infer Column, - infer Op, - infer _ - > - ? ColumnArithmeticAssignment - : Invalid<"Corrupted column assignment"> - : ParseEntireArithmeticTree - : Invalid<"Invalid grouping in column assignment"> - : ReadNextToken - -/** - * Type to prevent assumption about operations from causing mismatch - */ -type AnyExpression = LogicalExpression - -/** - * Consume the entire arithmetic tree - */ -type ParseEntireArithmeticTree< - SQL extends string, - Options extends ParserOptions -> = ParseArithmeticExpression extends [ - infer Expression, - infer Remainder extends string -] - ? Remainder extends "" - ? Expression - : Invalid<"Failed to consume the entire SQL"> - : ParseArithmeticExpression diff --git a/packages/sql/query/parser/expressions.ts b/packages/sql/query/parser/expressions.ts index 351ec50..88dc226 100644 --- a/packages/sql/query/parser/expressions.ts +++ b/packages/sql/query/parser/expressions.ts @@ -7,6 +7,9 @@ import type { ColumnFilter, LogicalExpression, LogicalGroup, + LogicalNegation, + LogicalOperation, + LogicalTree, } from "../../ast/expressions.js" import type { ValueTypes } from "../../ast/values.js" import type { NextToken } from "./normalize.js" @@ -30,7 +33,7 @@ import { type GetFullExpressionType< SQL extends string, Options extends ParserOptions -> = ParseArithmeticExpression extends [infer Expression, ""] +> = ParseExpressionTree extends [infer Expression, ""] ? Expression : never @@ -52,19 +55,19 @@ export function parseArithmeticExpression< * @param current The current expression if one exists */ export function parseArithmeticExpression< - Expression extends AnyExpression, + Expression extends LogicalExpression, Options extends ParserOptions >( tokens: string[], options: Options, current?: Expression -): AnyExpression | undefined +): LogicalExpression | undefined // Implementation export function parseArithmeticExpression( sql: unknown, options: Options, - current?: AnyExpression + current?: LogicalExpression ): unknown { const tokens = typeof sql === "string" ? sql.split(" ") : (sql as string[]) @@ -94,7 +97,7 @@ export function parseArithmeticExpression( const fullExpression = parseArithmeticExpression( copy, options, - next as AnyExpression + next as LogicalExpression ) const diff = tokens.length - copy.length @@ -136,55 +139,6 @@ function parseGroupExpression( return } -/** - * Extract the next valid expression chunk and the remaining string - */ -export type ParseArithmeticExpression< - SQL extends string, - Options extends ParserOptions, - Current extends AnyExpression = never -> = [Current] extends [never] - ? ParseNextArithmeticExpression extends [ - infer Expression extends AnyExpression, - infer Remainder extends string - ] - ? Remainder extends "" - ? [Expression, ""] - : ParseArithmeticExpression - : ParseNextArithmeticExpression - : ReadNextToken extends [ - infer Token, - infer Remainder extends string - ] - ? Token extends GetArithmeticOperations - ? ParseSingleArithmeticExpression< - Remainder, - Options, - ArithmeticExpression - > extends [ - infer Expression extends AnyExpression, - infer Rest extends string - ] - ? Rest extends "" - ? [Expression, ""] - : ParseArithmeticExpression - : ParseSingleArithmeticExpression< - Remainder, - Options, - ArithmeticExpression - > - : [Current, SQL] // Return - : [Current, SQL] // Return expression and remainder - -/** - * Get the types of tokens supported - */ -type GetTokenTypes = - | GetArithmeticOperations - | GetAssignmentOperations - | ColumnReference - | ValueTypes - /** * Get the next token value */ @@ -196,6 +150,8 @@ type ReadNextToken< infer Remainder extends string ] ? Token extends GetOverridableTokens + ? [Token, Remainder] + : Token extends "AND" | "OR" | "NOT" ? [Token, Remainder] : Token extends ")" ? Invalid<"Invalid syntax, extra )"> @@ -242,40 +198,10 @@ export function readNextToken( return parseValueOrReference(tokens, options) } -/** - * Parse the next {@link ArithmeticExpression} from the provided string - */ -type ParseNextArithmeticExpression< - SQL extends string, - Options extends ParserOptions -> = ReadNextToken extends [ - infer Token, - infer Remainder extends string -] - ? Token extends ColumnReference - ? ParseColumnExpression - : Token extends ValueTypes - ? ParseValueExpression - : Token extends GetTokenTypes - ? Invalid<"Cannot start an operation with an assignment or arithmetic sign"> - : Token extends string - ? ParseEntireArithmeticTree< - Token, - Options - > extends infer Tree extends ArithmeticExpression< - IgnoreAny, - string, - IgnoreAny - > - ? [LogicalGroup, Remainder] - : ParseEntireArithmeticTree - : Invalid<"Invalid token"> - : ReadNextToken - function parseNextArithmeticExpression( tokens: string[], options: ParserOptions -): Partial | undefined { +): Partial | undefined { const token = readNextToken(tokens, options) if (token === undefined) { return @@ -293,58 +219,11 @@ function parseNextArithmeticExpression( } } -/** - * Parse expressions starting with a column - */ -type ParseColumnExpression< - SQL extends string, - Options extends ParserOptions, - Column extends ColumnReference -> = ReadNextToken extends [ - infer Token, - infer Remainder extends string -] - ? Token extends GetArithmeticOperations - ? ParseSingleArithmeticExpression< - Remainder, - Options, - ArithmeticExpression - > - : Token extends GetAssignmentOperations - ? ParseColumnAssignmentExpression< - Remainder, - Options, - ColumnArithmeticAssignment - > - : Token extends GetComparisonOperations - ? ParseColumnFilter> - : Invalid<"Column must be followed by an assignment or arithmetic operation"> - : ReadNextToken - -type ParseColumnFilter< - SQL extends string, - Options extends ParserOptions, - Comparison extends ColumnFilter -> = Comparison extends ColumnFilter - ? ReadNextToken extends [ - infer Token, - infer Remainder extends string - ] - ? Token extends ColumnReference - ? [ColumnFilter, Remainder] - : Token extends ValueTypes - ? [ColumnFilter, Remainder] - : Token extends string - ? Invalid<"foo"> - : Invalid<"Invalid Token type"> - : ReadNextToken - : Invalid<"Corrupt ColumnFilter"> - function parseColumnExpression( tokens: string[], options: ParserOptions, column: ColumnReference -): AnyExpression | undefined { +): LogicalExpression | undefined { const token = readNextToken(tokens, options) if (typeof token === "string") { if (options.tokens.assignments.indexOf(token) >= 0) { @@ -426,22 +305,6 @@ function parseSingleArithmeticExpression< } as ArithmeticExpression } -/** - * Parse expressions starting with a value - */ -type ParseValueExpression< - SQL extends string, - Options extends ParserOptions, - Value extends ValueTypes -> = ReadNextToken extends [ - infer Token, - infer Remainder extends string -] - ? Token extends GetArithmeticOperations - ? [ArithmeticExpression, Remainder] - : Invalid<"Value must be followed by an arithmetic operation"> - : ReadNextToken - function parseValueExpression( tokens: string[], options: ParserOptions, @@ -462,125 +325,116 @@ function parseValueExpression( return } -/** - * Parse only the next segment - */ -type ParseSingleArithmeticExpression< - SQL extends string, - Options extends ParserOptions, - Expression extends AnyExpression -> = ReadNextToken extends [ - infer Token, - infer Remainder extends string -] - ? Token extends ColumnReference | ValueTypes - ? Expression extends ArithmeticExpression - ? [ArithmeticExpression, Remainder] - : Invalid<"Corrupt expression"> - : Token extends GetTokenTypes - ? Invalid<"Right hand side of expression must be a value, column or other expression"> - : Token extends string - ? ParseEntireArithmeticTree< - Token, - Options - > extends infer Right extends ArithmeticExpression< - IgnoreAny, - string, - IgnoreAny - > - ? Expression extends ArithmeticExpression - ? [ArithmeticExpression>, Remainder] - : Invalid<"Corrupted expression"> - : ParseEntireArithmeticTree - : Invalid<"Next token is not valid"> - : ReadNextToken +// Start by parsing the next unit (column, value, token) +// Get next "operator" +// Parse the next chunk, repeat until done -/** - * Parse a column assignment - * - * Note: To be valid, the entire remainder must be consumable... - */ -type ParseColumnAssignmentExpression< +type ParseNextLogicalObject< SQL extends string, - Options extends ParserOptions, - Assignment extends ColumnArithmeticAssignment + Options extends ParserOptions > = ReadNextToken extends [ infer Token, infer Remainder extends string ] - ? Token extends ValueTypes | ColumnReference - ? Assignment extends ColumnArithmeticAssignment< - infer Column, - infer Op, - infer _ - > - ? Remainder extends "" - ? ColumnArithmeticAssignment - : Invalid<"Cannot have assignment with trailing information"> - : Invalid<"Corrupt assignment"> - : Token extends GetTokenTypes - ? Invalid<"Right hand side of assignment must be a value, column or other expression"> + ? Token extends GetAssignmentOperations + ? [ColumnArithmeticAssignment, Remainder] + : Token extends GetArithmeticOperations + ? [ArithmeticExpression, Remainder] + : Token extends GetComparisonOperations + ? [ColumnFilter, Remainder] + : Token extends ColumnReference + ? [Token, Remainder] + : Token extends ValueTypes + ? [Token, Remainder] + : Token extends "AND" | "OR" + ? [LogicalTree, Remainder] + : Token extends "NOT" + ? [LogicalNegation, Remainder] : Token extends string - ? ParseEntireArithmeticTree< - Token, - Options - > extends infer Expression extends ArithmeticExpression - ? Assignment extends ColumnArithmeticAssignment< - infer Column, - infer Op, - infer _ - > - ? ColumnArithmeticAssignment - : Invalid<"Corrupted column assignment"> - : ParseEntireArithmeticTree - : Invalid<"Invalid grouping in column assignment"> + ? ParseExpressionTree extends [ + infer Exp extends LogicalOperation, + "" + ] + ? [LogicalGroup, Remainder] + : Invalid<"Failed to process group"> + : Invalid<"Cannot process token"> : ReadNextToken -/** - * Type to prevent assumption about operations from causing mismatch - */ -type AnyExpression = LogicalExpression - -/** - * Consume the entire arithmetic tree - */ -type ParseEntireArithmeticTree< +export type ParseExpressionTree< SQL extends string, Options extends ParserOptions -> = ParseArithmeticExpression extends [ - infer Expression, +> = ParseAllExpressionTokens extends [ + infer Exp extends LogicalExpression[], infer Remainder extends string ] - ? Remainder extends "" - ? Expression - : Invalid<"Failed to consume the entire SQL"> - : ParseArithmeticExpression + ? CollapseExpressions extends infer Consolidated extends LogicalExpression + ? [Consolidated, Remainder] + : CollapseExpressions + : Invalid<"Failed"> -// Start by parsing the next unit (column, value, token) -// Get next "operator" -// Parse the next chunk, repeat until done - -export type ParseNextLogicalExpression< +type ParseAllExpressionTokens< SQL extends string, Options extends ParserOptions, - Previous extends LogicalExpression = never -> = ReadNextToken extends [ - infer Token, - infer Remainder extends string -] - ? Token extends ColumnReference - ? [Token, Remainder] - : Token extends ValueTypes - ? [Token, Remainder] - : Token extends GetAssignmentOperations - ? Previous extends ColumnReference - ? [ColumnArithmeticAssignment, Remainder] - : Invalid<"Cannot use assignment on non-column"> - : Token extends GetArithmeticOperations - ? [ArithmeticExpression, Remainder] - : Token extends GetComparisonOperations - ? Previous extends ColumnReference - ? [ColumnFilter, Remainder] - : Invalid<"Cannot use comparison on non-column"> - : Invalid<"Corrupt token type"> - : ReadNextToken + Current extends LogicalExpression[] = [] +> = SQL extends "" + ? [Current, SQL] + : ParseNextLogicalObject extends [ + infer Token extends LogicalExpression, + infer Remainder extends string + ] + ? ParseAllExpressionTokens extends [ + infer Tokens, + infer R extends string + ] + ? [Tokens, R] + : [[Token], Remainder] + : [Current, SQL] + +type CollapseExpressions = + Expressions extends [ + ...infer Rest extends LogicalExpression[], + infer First extends LogicalExpression, + infer Second extends LogicalExpression + ] + ? First extends ColumnArithmeticAssignment + ? CollapseExpressions< + [...Rest, ColumnArithmeticAssignment] + > + : First extends ColumnFilter + ? CollapseExpressions<[...Rest, ColumnFilter]> + : First extends ArithmeticExpression + ? CollapseExpressions< + [...Rest, ArithmeticExpression] + > + : First extends LogicalNegation + ? CollapseExpressions<[...Rest, LogicalNegation]> + : First extends LogicalTree + ? CollapseExpressions<[...Rest, LogicalTree]> + : Second extends ColumnArithmeticAssignment< + never, + infer Token, + infer Right + > + ? First extends ColumnReference + ? CollapseExpressions< + [...Rest, ColumnArithmeticAssignment] + > + : Invalid<"Cannot do assignment on non-column reference"> + : Second extends ColumnFilter + ? First extends ColumnReference + ? CollapseExpressions<[...Rest, ColumnFilter]> + : Invalid<"Cannot do column filtering on non-column reference"> + : Second extends ArithmeticExpression + ? CollapseExpressions< + [...Rest, ArithmeticExpression] + > + : Second extends LogicalTree + ? CollapseExpressions< + [...Rest, First] + > extends infer Exp extends LogicalExpression + ? LogicalTree + : Invalid<"failed to parse tree"> + : Expressions + : Expressions extends [infer Exp extends LogicalExpression] + ? Exp + : Expressions From 98e8d70ee65332b398203cd888006f2d69d42b4d Mon Sep 17 00:00:00 2001 From: Nathan Northcutt Date: Mon, 16 Sep 2024 17:52:09 -0400 Subject: [PATCH 21/21] starting to work on refactoring and greatly simplifying parsing of where --- packages/sql/query/parser/expressions.test.ts | 15 +++ packages/sql/query/parser/expressions.ts | 121 +++++++++++++++++- 2 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 packages/sql/query/parser/expressions.test.ts diff --git a/packages/sql/query/parser/expressions.test.ts b/packages/sql/query/parser/expressions.test.ts new file mode 100644 index 0000000..59f69bf --- /dev/null +++ b/packages/sql/query/parser/expressions.test.ts @@ -0,0 +1,15 @@ +import { parseAllExpressionTokens } from "./expressions.js" +import { normalizeQuery } from "./normalize.js" +import { DefaultOptions } from "./options.js" + +describe("Expression parsing should work for all value types", () => { + it("Should handle parsing something", () => { + const tokens = normalizeQuery("a + b OR c + d", DefaultOptions).split(" ") + const result = JSON.stringify( + parseAllExpressionTokens(tokens, DefaultOptions), + undefined, + 2 + ) + expect(result).not.toBeUndefined() + }) +}) diff --git a/packages/sql/query/parser/expressions.ts b/packages/sql/query/parser/expressions.ts index 88dc226..0e2213b 100644 --- a/packages/sql/query/parser/expressions.ts +++ b/packages/sql/query/parser/expressions.ts @@ -193,6 +193,10 @@ export function readNextToken( return tokens.shift()! case options.tokens.assignments.indexOf(tokens[0]) >= 0: return tokens.shift()! + case options.tokens.comparisons.indexOf(tokens[0]) >= 0: + return tokens.shift()! + case ["AND", "OR", "NOT"].indexOf(tokens[0]) >= 0: + return tokens.shift()! } return parseValueOrReference(tokens, options) @@ -360,6 +364,10 @@ type ParseNextLogicalObject< : Invalid<"Cannot process token"> : ReadNextToken +/** + * Main entrypoint that parses as much of the SQL as an expression tree as + * possible while returning the remainder + */ export type ParseExpressionTree< SQL extends string, Options extends ParserOptions @@ -370,8 +378,114 @@ export type ParseExpressionTree< ? CollapseExpressions extends infer Consolidated extends LogicalExpression ? [Consolidated, Remainder] : CollapseExpressions - : Invalid<"Failed"> + : Invalid<"Failed to parse an expression"> + +export function parseAllExpressionTokens( + tokens: string[], + options: ParserOptions +): LogicalExpression | undefined { + const expressions: Partial[] = [] + + let next = readNextToken(tokens, options) + while (next !== undefined) { + if (typeof next === "string") { + if (options.tokens.arithmetic.indexOf(next) >= 0) { + expressions.push({ type: "ArithmeticExpression", operation: next }) + } else if (options.tokens.assignments.indexOf(next) >= 0) { + expressions.push({ + type: "ColumnArithmeticAssignment", + operation: next, + }) + } else if (options.tokens.comparisons.indexOf(next) >= 0) { + expressions.push({ type: "ColumnFilter", operation: next }) + } else if (next === "NOT") { + expressions.push({ type: "LogicalNegation" }) + } else if (next === "AND" || next === "OR") { + expressions.push({ type: "LogicalTree", operation: next }) + } + } else if (Array.isArray(next)) { + const exp = parseAllExpressionTokens(next, options) + if (exp !== undefined) { + expressions.push(exp) + } else throw new Error("oops") + } else { + expressions.push(next) + } + + if (tokens.length === 0) break + next = readNextToken(tokens, options) + } + + return aggregateExpressions(expressions) +} +function aggregateExpressions( + expressions: Partial[] +): LogicalExpression | undefined { + while (expressions.length > 1) { + const second = expressions.pop()! + const first = expressions.pop()! + + switch (first.type ?? "undefined") { + case "ColumnArighmeticAssignment": + ;(first as ColumnArithmeticAssignment).value = + second as LogicalExpression + expressions.push(first) + continue + case "ColumnFilter": + ;(first as ColumnFilter).filter = second as LogicalExpression + expressions.push(first) + continue + case "ArithmeticExpression": + ;(first as ArithmeticExpression).right = second as LogicalExpression + expressions.push(first) + continue + case "LogicalNegation": + ;(first as LogicalNegation).expression = second as LogicalExpression + expressions.push(first) + continue + case "LogicalTree": + ;(first as LogicalTree).right = second as LogicalExpression + expressions.push(first) + continue + } + + switch (second.type ?? "undefined") { + case "ColumnArithmeticAssignment": + ;(second as ColumnArithmeticAssignment).column = + first as ColumnReference + expressions.push(second) + continue + case "ColumnFilter": + ;(second as ColumnFilter).column = first as ColumnReference + expressions.push(second) + continue + case "ArithmeticExpression": + ;(second as ArithmeticExpression).left = first as LogicalExpression + expressions.push(second) + continue + case "LogicalTree": { + expressions.push(first) + const left = aggregateExpressions(expressions) + if (left === undefined) + throw new Error("Failed to parse left side of tree") + ;(second as LogicalTree).left = left + return second as LogicalExpression + } + } + } + + if (expressions.length === 1) { + return expressions[0] as LogicalExpression + } + + return +} + +/** + * Parse out all of the valid individual expression tokens and return them with + * the remainding string + */ type ParseAllExpressionTokens< SQL extends string, Options extends ParserOptions, @@ -390,6 +504,9 @@ type ParseAllExpressionTokens< : [[Token], Remainder] : [Current, SQL] +/** + * Collapse a series of logical expressions into a single expression + */ type CollapseExpressions = Expressions extends [ ...infer Rest extends LogicalExpression[], @@ -437,4 +554,4 @@ type CollapseExpressions = : Expressions : Expressions extends [infer Exp extends LogicalExpression] ? Exp - : Expressions + : Invalid<"Corrupt or empty expression group">