From c2ba7ebec7be028e97f9b9eaaccccbebedd7655d Mon Sep 17 00:00:00 2001 From: Noel Phillips Date: Thu, 10 Sep 2026 14:25:19 -0500 Subject: [PATCH] Fix: digit-leading generalized-identifier segment mislexed as numeric literal A run of digits inside an unquoted [fieldName] selector, immediately preceded by whitespace and followed by "." then a non-digit identifier character (e.g. [Maintenance vendors 20230329.Department]), was mislexed: readNumericLiteral greedily consumed the digit run as a NumericLiteral token, leaving a bare "." that tokenizeDefault has no valid interpretation for and throws on. TaskUtils.tryLexParse (the entry point actually used by consumers) surfaces this as a Lex-stage error via errorLineMap, even though Lexer.tryLex + Lexer.trySnapshot called directly happen to still succeed. This is a real-world pattern, not a contrived edge case: it's exactly the naming convention Table.ExpandTableColumn itself generates for an expanded/merged column ("."), so any M document with a merge/expand step whose other query's display name ends in digits right before the generated "." hits this. Fix: when a digit-first token's matched numeric span is immediately followed by "." then a non-digit identifier-part character (the one sequence a genuine numeric literal can never produce, since a decimal point is always followed by more digits), re-read the whole span as an identifier instead - mirroring the digit-tolerant continuation that identifier tokens starting with a letter already get via IdentifierUtils.getIdentifierLength. Verified against the full existing lexer/parser test suite (399 -> 402 library tests, 102 resource tests) with zero regressions, plus the two real-world repro cases. --- src/powerquery-parser/lexer/lexer.ts | 84 +++++++++++++++++++- src/test/libraryTest/lexer/lexSimple.test.ts | 37 +++++++++ 2 files changed, 119 insertions(+), 2 deletions(-) diff --git a/src/powerquery-parser/lexer/lexer.ts b/src/powerquery-parser/lexer/lexer.ts index 3770788c..11e800f3 100644 --- a/src/powerquery-parser/lexer/lexer.ts +++ b/src/powerquery-parser/lexer/lexer.ts @@ -852,16 +852,18 @@ function tokenizeDefault(line: TLine, lineNumber: number, positionStart: number, if (chr2 === "x" || chr2 === "X") { token = readHexLiteral(text, lineNumber, positionStart, locale); } else { - token = readNumericLiteral(text, lineNumber, positionStart, locale); + token = readNumericLiteralOrDigitLeadingIdentifier(text, lineNumber, positionStart, locale); } } else if ("1" <= chr1 && chr1 <= "9") { - token = readNumericLiteral(text, lineNumber, positionStart, locale); + token = readNumericLiteralOrDigitLeadingIdentifier(text, lineNumber, positionStart, locale); } else if (chr1 === ".") { const chr2: string | undefined = text[positionStart + 1]; if (chr2 === undefined) { throw new LexError.UnexpectedEofError(graphemePositionFrom(text, lineNumber, positionStart), locale); } else if ("1" <= chr2 && chr2 <= "9") { + // Unlike the digit-first branches above there's no ambiguity here, + // as an identifier can't start with `.` token = readNumericLiteral(text, lineNumber, positionStart, locale); } else if (chr2 === ".") { const chr3: string | undefined = text[positionStart + 2]; @@ -993,6 +995,84 @@ function readNumericLiteral(text: string, lineNumber: number, positionStart: num return readTokenFrom(Token.LineTokenKind.NumericLiteral, text, positionStart, positionEnd); } +// A run of digits is ambiguous with the start of a generalized identifier, eg. `123.Bar`. +// The two are distinguished by what follows the digit run: +// * A numeric literal's digits are followed by whitespace, punctuation, or eof. +// * An identifier's digits are followed by `.` then a non-digit identifier part character, +// which a numeric literal can't produce as a decimal point is always followed by more digits. +function readNumericLiteralOrDigitLeadingIdentifier( + text: string, + lineNumber: number, + positionStart: number, + locale: string, +): Token.LineToken { + const numericPositionEnd: number | undefined = indexOfRegexEnd(Pattern.Numeric, text, positionStart); + + if (numericPositionEnd === undefined) { + throw new LexError.ExpectedError( + graphemePositionFrom(text, lineNumber, positionStart), + LexError.ExpectedKind.Numeric, + locale, + ); + } + + if (text[numericPositionEnd] === ".") { + const chrAfterDot: string | undefined = text[numericPositionEnd + 1]; + + const isDotLeadingIdentifierContinuation: boolean = + chrAfterDot !== undefined && + chrAfterDot !== "." && + StringUtils.regexMatchLength(Pattern.IdentifierPartCharacters, text, numericPositionEnd + 1) !== undefined; + + if (isDotLeadingIdentifierContinuation) { + const identifierPositionEnd: number | undefined = indexOfDigitLeadingIdentifierEnd(text, positionStart); + + if (identifierPositionEnd !== undefined) { + return readTokenFrom(Token.LineTokenKind.Identifier, text, positionStart, identifierPositionEnd); + } + } + } + + return readTokenFrom(Token.LineTokenKind.NumericLiteral, text, positionStart, numericPositionEnd); +} + +// Mirrors IdentifierUtils.getIdentifierLength's continuation loop, +// minus the identifier start character check which would reject a leading digit. +function indexOfDigitLeadingIdentifierEnd(text: string, positionStart: number): number | undefined { + const textLength: number = text.length; + let index: number = positionStart; + + while (index < textLength) { + const currentChr: string = StringUtils.assertGet(text, index); + + if (currentChr === ".") { + const nextChr: string | undefined = text[index + 1]; + + if (nextChr === undefined || nextChr === ".") { + break; + } + + index += 1; + + continue; + } + + const matchLength: number | undefined = StringUtils.regexMatchLength( + Pattern.IdentifierPartCharacters, + text, + index, + ); + + if (matchLength === undefined) { + break; + } + + index += matchLength; + } + + return index !== positionStart ? index : undefined; +} + function readLineComment(text: string, positionStart: number): Token.LineToken { return readRestOfLine(Token.LineTokenKind.LineComment, text, positionStart); } diff --git a/src/test/libraryTest/lexer/lexSimple.test.ts b/src/test/libraryTest/lexer/lexSimple.test.ts index 9a357a74..dfe21aba 100644 --- a/src/test/libraryTest/lexer/lexSimple.test.ts +++ b/src/test/libraryTest/lexer/lexSimple.test.ts @@ -156,6 +156,43 @@ type assertGetSnapshotAbridgedTokens(text, expected, false); }); + it(`digit-leading identifier segment followed by "." and more identifier characters`, () => { + // `123.` can't be a numeric literal as a decimal point is always followed by more digits, + // eg. the `.` names Table.ExpandTableColumn generates. + const text: string = `123.Bar`; + + const expected: ReadonlyArray<[Language.Token.TokenKind, string]> = [ + [Language.Token.TokenKind.Identifier, `123.Bar`], + ]; + + assertGetSnapshotAbridgedTokens(text, expected, false); + }); + + it(`digit-leading identifier segment inside a field selector, alongside a genuine numeric literal`, () => { + const text: string = `[a] = 123.Bar`; + + const expected: ReadonlyArray<[Language.Token.TokenKind, string]> = [ + [Language.Token.TokenKind.LeftBracket, `[`], + [Language.Token.TokenKind.Identifier, `a`], + [Language.Token.TokenKind.RightBracket, `]`], + [Language.Token.TokenKind.Equal, `=`], + [Language.Token.TokenKind.Identifier, `123.Bar`], + ]; + + assertGetSnapshotAbridgedTokens(text, expected, false); + }); + + it(`digit run still lexes as NumericLiteral when not followed by a dotted identifier continuation`, () => { + const text: string = `123 456.789`; + + const expected: ReadonlyArray<[Language.Token.TokenKind, string]> = [ + [Language.Token.TokenKind.NumericLiteral, `123`], + [Language.Token.TokenKind.NumericLiteral, `456.789`], + ]; + + assertGetSnapshotAbridgedTokens(text, expected, false); + }); + it(`operator-or-punctuation`, () => { const text: string = ` ,