From 097ce586796df67ac8a42b1ba0c45778617da219 Mon Sep 17 00:00:00 2001 From: JordanBoltonMN Date: Mon, 4 Aug 2025 10:30:01 -0500 Subject: [PATCH 01/15] initial commit --- src/powerquery-parser/parser/parseBehavior.ts | 8 + src/powerquery-parser/parser/parseSettings.ts | 4 +- .../parser/parseState/parseState.ts | 2 + .../parser/parseState/parseStateUtils.ts | 4 +- src/powerquery-parser/parser/parser/parser.ts | 7 - .../parser/parser/parserUtils.ts | 321 +++++++++---- .../combinatorialParserV2.ts | 2 - .../parser/parsers/naiveParseSteps.ts | 79 +-- .../parser/parsers/recursiveDescentParser.ts | 2 - src/powerquery-parser/settings.ts | 4 +- .../libraryTest/parser/parseBehavior.test.ts | 133 +++++ .../libraryTest/parser/parseSimple.test.ts | 454 ++++++------------ .../libraryTest/parser/parserTestUtils.ts | 204 ++++++++ 13 files changed, 729 insertions(+), 495 deletions(-) create mode 100644 src/powerquery-parser/parser/parseBehavior.ts create mode 100644 src/test/libraryTest/parser/parseBehavior.test.ts create mode 100644 src/test/libraryTest/parser/parserTestUtils.ts diff --git a/src/powerquery-parser/parser/parseBehavior.ts b/src/powerquery-parser/parser/parseBehavior.ts new file mode 100644 index 00000000..a95523b0 --- /dev/null +++ b/src/powerquery-parser/parser/parseBehavior.ts @@ -0,0 +1,8 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +export enum ParseBehavior { + ParseAll = "ParseAll", + ParseExpression = "ParseExpression", + ParseSection = "ParseSection", +} diff --git a/src/powerquery-parser/parser/parseSettings.ts b/src/powerquery-parser/parser/parseSettings.ts index c9ae8311..670e4220 100644 --- a/src/powerquery-parser/parser/parseSettings.ts +++ b/src/powerquery-parser/parser/parseSettings.ts @@ -4,12 +4,14 @@ import { Ast } from "../language"; import { CommonSettings } from "../common"; import { LexerSnapshot } from "../lexer"; +import { ParseBehavior } from "./parseBehavior"; import { Parser } from "./parser"; import { ParseState } from "./parseState"; export interface ParseSettings extends CommonSettings { + readonly parseBehavior: ParseBehavior; readonly parser: Parser; - readonly newParseState: (lexerSnapshot: LexerSnapshot, overrides: Partial | undefined) => ParseState; + readonly newParseState: (lexerSnapshot: LexerSnapshot, overrides?: Partial) => ParseState; readonly parserEntryPoint: | ((state: ParseState, parser: Parser, correlationId: number | undefined) => Promise) | undefined; diff --git a/src/powerquery-parser/parser/parseState/parseState.ts b/src/powerquery-parser/parser/parseState/parseState.ts index 7edb5b33..e04e302f 100644 --- a/src/powerquery-parser/parser/parseState/parseState.ts +++ b/src/powerquery-parser/parser/parseState/parseState.ts @@ -4,6 +4,7 @@ import { Disambiguation } from "../disambiguation"; import { ICancellationToken } from "../../common"; import { LexerSnapshot } from "../../lexer"; +import { ParseBehavior } from "../parseBehavior"; import { ParseContext } from ".."; import { Token } from "../../language"; import { TraceManager } from "../../common/trace"; @@ -13,6 +14,7 @@ export interface ParseState { readonly disambiguationBehavior: Disambiguation.DismabiguationBehavior; readonly lexerSnapshot: LexerSnapshot; readonly locale: string; + readonly parseBehavior: ParseBehavior; readonly traceManager: TraceManager; contextState: ParseContext.State; currentContextNode: ParseContext.TNode | undefined; diff --git a/src/powerquery-parser/parser/parseState/parseStateUtils.ts b/src/powerquery-parser/parser/parseState/parseStateUtils.ts index e4c3e5eb..a6321df4 100644 --- a/src/powerquery-parser/parser/parseState/parseStateUtils.ts +++ b/src/powerquery-parser/parser/parseState/parseStateUtils.ts @@ -8,10 +8,11 @@ import { NoOpTraceManagerInstance, Trace } from "../../common/trace"; import { DefaultLocale } from "../../localization"; import { Disambiguation } from "../disambiguation"; import { LexerSnapshot } from "../../lexer"; +import { ParseBehavior } from "../parseBehavior"; import { ParseState } from "./parseState"; import { SequenceKind } from "../error"; -export function newState(lexerSnapshot: LexerSnapshot, overrides: Partial | undefined): ParseState { +export function newState(lexerSnapshot: LexerSnapshot, overrides?: Partial): ParseState { const tokenIndex: number = overrides?.tokenIndex ?? 0; const currentToken: Token.Token | undefined = lexerSnapshot.tokens[tokenIndex]; const currentTokenKind: Token.TokenKind | undefined = currentToken?.kind; @@ -30,6 +31,7 @@ export function newState(lexerSnapshot: LexerSnapshot, overrides: Partial Ast.IdentifierExpression; - // 12.2.1 Documents - readonly readDocument: ( - state: ParseState, - parser: Parser, - correlationId: number | undefined, - ) => Promise; - // 12.2.2 Section Documents readonly readSectionDocument: ( state: ParseState, diff --git a/src/powerquery-parser/parser/parser/parserUtils.ts b/src/powerquery-parser/parser/parser/parserUtils.ts index 7ac31d81..edc8aa0f 100644 --- a/src/powerquery-parser/parser/parser/parserUtils.ts +++ b/src/powerquery-parser/parser/parser/parserUtils.ts @@ -1,14 +1,15 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -import { Assert, CommonError, ResultUtils } from "../../common"; +import { Assert, CommonError, Result, ResultUtils } from "../../common"; import { NodeIdMap, NodeIdMapUtils } from "../nodeIdMap"; import { ParseContext, ParseContextUtils } from "../context"; import { ParseError, ParseSettings } from ".."; -import { Parser, ParseStateCheckpoint, TriedParse } from "./parser"; +import { ParseOk, Parser, ParseStateCheckpoint, TriedParse } from "./parser"; import { ParseState, ParseStateUtils } from "../parseState"; import { Ast } from "../../language"; import { LexerSnapshot } from "../../lexer"; +import { ParseBehavior } from "../parseBehavior"; import { Trace } from "../../common/trace"; export async function tryParse(parseSettings: ParseSettings, lexerSnapshot: LexerSnapshot): Promise { @@ -23,101 +24,13 @@ export async function tryParse(parseSettings: ParseSettings, lexerSnapshot: Lexe initialCorrelationId: trace.id, }; - const parserEntryPoint: - | ((state: ParseState, parser: Parser, correlationId: number | undefined) => Promise) - | undefined = updatedSettings?.parserEntryPoint; + const result: TriedParse = updatedSettings.parserEntryPoint + ? await tryParseEntryPoint(updatedSettings.parserEntryPoint, updatedSettings, lexerSnapshot) + : await tryParseDocument(updatedSettings, lexerSnapshot); - if (parserEntryPoint === undefined) { - return await tryParseDocument(updatedSettings, lexerSnapshot); - } - - const parseState: ParseState = updatedSettings.newParseState(lexerSnapshot, defaultOverrides(updatedSettings)); - - try { - const root: Ast.TNode = await parserEntryPoint(parseState, updatedSettings.parser, trace.id); - ParseStateUtils.assertIsDoneParsing(parseState); - - return ResultUtils.ok({ - lexerSnapshot, - root, - state: parseState, - }); - } catch (caught: unknown) { - Assert.isInstanceofError(caught); - CommonError.throwIfCancellationError(caught); - - return ResultUtils.error(ensureParseError(parseState, caught, updatedSettings.locale)); - } -} - -// Attempts to parse the document both as an expression and section document. -// Whichever attempt consumed the most tokens is the one returned. Ties go to expression documents. -export async function tryParseDocument( - parseSettings: ParseSettings, - lexerSnapshot: LexerSnapshot, -): Promise { - const trace: Trace = parseSettings.traceManager.entry( - ParserUtilsTraceConstant.ParserUtils, - tryParseDocument.name, - parseSettings.initialCorrelationId, - ); - - let root: Ast.TNode; - - const expressionDocumentState: ParseState = parseSettings.newParseState( - lexerSnapshot, - defaultOverrides(parseSettings), - ); - - try { - root = await parseSettings.parser.readExpression(expressionDocumentState, parseSettings.parser, trace.id); - ParseStateUtils.assertIsDoneParsing(expressionDocumentState); - trace.exit(); - - return ResultUtils.ok({ - lexerSnapshot, - root, - state: expressionDocumentState, - }); - } catch (expressionDocumentError: unknown) { - Assert.isInstanceofError(expressionDocumentError); - CommonError.throwIfCancellationError(expressionDocumentError); - - const sectionDocumentState: ParseState = parseSettings.newParseState( - lexerSnapshot, - defaultOverrides(parseSettings), - ); - - try { - root = await parseSettings.parser.readSectionDocument(sectionDocumentState, parseSettings.parser, trace.id); - ParseStateUtils.assertIsDoneParsing(sectionDocumentState); - trace.exit(); - - return ResultUtils.ok({ - lexerSnapshot, - root, - state: sectionDocumentState, - }); - } catch (sectionDocumentError: unknown) { - Assert.isInstanceofError(sectionDocumentError); - CommonError.throwIfCancellationError(expressionDocumentError); - - let betterParsedState: ParseState; - let betterParsedError: Error; - - if (expressionDocumentState.tokenIndex >= sectionDocumentState.tokenIndex) { - betterParsedState = expressionDocumentState; - betterParsedError = expressionDocumentError; - } else { - betterParsedState = sectionDocumentState; - betterParsedError = sectionDocumentError; - } + trace.exit(); - trace.exit(); - - return ResultUtils.error(ensureParseError(betterParsedState, betterParsedError, parseSettings.locale)); - } - } + return result; } // If you have a custom parser + parser state, @@ -208,5 +121,223 @@ function defaultOverrides(parseSettings: ParseSettings): Partial { locale: parseSettings.locale, cancellationToken: parseSettings.cancellationToken, traceManager: parseSettings.traceManager, + parseBehavior: parseSettings.parseBehavior, }; } + +async function tryParseEntryPoint( + parserEntryPoint: (state: ParseState, parser: Parser, correlationId: number | undefined) => Promise, + parseSettings: ParseSettings, + lexerSnapshot: LexerSnapshot, +): Promise { + const trace: Trace = parseSettings.traceManager.entry( + ParserUtilsTraceConstant.ParserUtils, + tryParseEntryPoint.name, + parseSettings.initialCorrelationId, + ); + + const parseState: ParseState = parseSettings.newParseState(lexerSnapshot, defaultOverrides(parseSettings)); + + try { + const root: Ast.TNode = await parserEntryPoint(parseState, parseSettings.parser, trace.id); + ParseStateUtils.assertIsDoneParsing(parseState); + + trace.exit(); + + return ResultUtils.ok({ + lexerSnapshot, + root, + state: parseState, + }); + } catch (caught: unknown) { + Assert.isInstanceofError(caught); + CommonError.throwIfCancellationError(caught); + + const result: TriedParse = ResultUtils.error(ensureParseError(parseState, caught, parseSettings.locale)); + + trace.exit(); + + return result; + } +} + +// Attempts to parse the document both as an expression and section document. +// Whichever attempt consumed the most tokens is the one returned. Ties go to expression documents. +async function tryParseDocument(parseSettings: ParseSettings, lexerSnapshot: LexerSnapshot): Promise { + switch (parseSettings.parseBehavior) { + case ParseBehavior.ParseAll: + return await tryParseExpressionDocumentOrSectionDocument(parseSettings, lexerSnapshot); + + case ParseBehavior.ParseExpression: { + const expressionParseResult: InternalTriedParse = await tryParseExpressionDocument( + parseSettings, + lexerSnapshot, + ); + + return ResultUtils.isOk(expressionParseResult) + ? expressionParseResult + : ResultUtils.error(expressionParseResult.error.innerError); + } + + case ParseBehavior.ParseSection: { + const sectionParseResult: InternalTriedParse = await tryParseSectionDocument(parseSettings, lexerSnapshot); + + return ResultUtils.isOk(sectionParseResult) + ? sectionParseResult + : ResultUtils.error(sectionParseResult.error.innerError); + } + + default: + Assert.isNever(parseSettings.parseBehavior); + } +} + +async function tryParseExpressionDocumentOrSectionDocument( + parseSettings: ParseSettings, + lexerSnapshot: LexerSnapshot, +): Promise { + const trace: Trace = parseSettings.traceManager.entry( + ParserUtilsTraceConstant.ParserUtils, + tryParseExpressionDocumentOrSectionDocument.name, + parseSettings.initialCorrelationId, + ); + + try { + const parseExpressionResult: InternalTriedParse = await tryParseExpressionDocument( + parseSettings, + lexerSnapshot, + ); + + if (ResultUtils.isOk(parseExpressionResult)) { + trace.exit(); + + return parseExpressionResult; + } + + // If the expression parse failed, try parsing as a section document. + const parseSectionResult: InternalTriedParse = await tryParseSectionDocument(parseSettings, lexerSnapshot); + + if (ResultUtils.isOk(parseSectionResult)) { + trace.exit(); + + return parseSectionResult; + } + + // If both parse attempts fail then return the instance with the most tokens consumed. + // On ties fallback to the expression parse attempt. + const errorResult: TriedParse = ResultUtils.error( + parseExpressionResult.error.tokensConsumed >= parseSectionResult.error.tokensConsumed + ? parseExpressionResult.error.innerError + : parseSectionResult.error.innerError, + ); + + trace.exit(); + + return errorResult; + } catch (error: unknown) { + Assert.isInstanceofError(error); + CommonError.throwIfCancellationError(error); + + const result: TriedParse = ResultUtils.error( + ensureParseError(parseSettings.newParseState(lexerSnapshot), error, parseSettings.locale), + ); + + trace.exit(); + + return result; + } +} + +async function tryParseExpressionDocument( + parseSettings: ParseSettings, + lexerSnapshot: LexerSnapshot, +): Promise { + const trace: Trace = parseSettings.traceManager.entry( + ParserUtilsTraceConstant.ParserUtils, + tryParseExpressionDocument.name, + parseSettings.initialCorrelationId, + ); + + const parseState: ParseState = parseSettings.newParseState(lexerSnapshot, defaultOverrides(parseSettings)); + + try { + const root: Ast.TExpression = await parseSettings.parser.readExpression( + parseState, + parseSettings.parser, + trace.id, + ); + + ParseStateUtils.assertIsDoneParsing(parseState); + trace.exit(); + + return ResultUtils.ok({ + lexerSnapshot, + root, + state: parseState, + }); + } catch (error: unknown) { + Assert.isInstanceofError(error); + CommonError.throwIfCancellationError(error); + + const result: InternalTriedParse = ResultUtils.error({ + innerError: ensureParseError(parseState, error, parseSettings.locale), + tokensConsumed: parseState.tokenIndex, + }); + + trace.exit(); + + return result; + } +} + +async function tryParseSectionDocument( + parseSettings: ParseSettings, + lexerSnapshot: LexerSnapshot, +): Promise { + const trace: Trace = parseSettings.traceManager.entry( + ParserUtilsTraceConstant.ParserUtils, + tryParseSectionDocument.name, + parseSettings.initialCorrelationId, + ); + + const parseState: ParseState = parseSettings.newParseState(lexerSnapshot, defaultOverrides(parseSettings)); + + try { + const root: Ast.Section = await parseSettings.parser.readSectionDocument( + parseState, + parseSettings.parser, + trace.id, + ); + + ParseStateUtils.assertIsDoneParsing(parseState); + trace.exit(); + + return ResultUtils.ok({ + lexerSnapshot, + root, + state: parseState, + }); + } catch (error: unknown) { + Assert.isInstanceofError(error); + CommonError.throwIfCancellationError(error); + + const result: InternalTriedParse = ResultUtils.error({ + innerError: ensureParseError(parseState, error, parseSettings.locale), + tokensConsumed: parseState.tokenIndex, + }); + + trace.exit(); + + return result; + } +} + +// Adds `tokensConsumed` to the TriedParse type. +// This is used to determine which parse attempt should be returned +// when both an expression and section document are attempted. +type InternalTriedParse = Result; + +interface InternalTriedParseError { + readonly innerError: ParseError.TParseError; + readonly tokensConsumed: number; +} diff --git a/src/powerquery-parser/parser/parsers/combinatorialParserV2/combinatorialParserV2.ts b/src/powerquery-parser/parser/parsers/combinatorialParserV2/combinatorialParserV2.ts index a8daf630..6b538de6 100644 --- a/src/powerquery-parser/parser/parsers/combinatorialParserV2/combinatorialParserV2.ts +++ b/src/powerquery-parser/parser/parsers/combinatorialParserV2/combinatorialParserV2.ts @@ -31,8 +31,6 @@ export const CombinatorialParserV2: Parser = { readGeneralizedIdentifier: NaiveParseSteps.readGeneralizedIdentifier, readKeyword: NaiveParseSteps.readKeyword, - readDocument: NaiveParseSteps.readDocument, - readSectionDocument: NaiveParseSteps.readSectionDocument, readSectionMembers: NaiveParseSteps.readSectionMembers, readSectionMember: NaiveParseSteps.readSectionMember, diff --git a/src/powerquery-parser/parser/parsers/naiveParseSteps.ts b/src/powerquery-parser/parser/parsers/naiveParseSteps.ts index f6833858..1017eb54 100644 --- a/src/powerquery-parser/parser/parsers/naiveParseSteps.ts +++ b/src/powerquery-parser/parser/parsers/naiveParseSteps.ts @@ -4,7 +4,7 @@ import { Assert, CommonError, Result, ResultUtils } from "../../common"; import { Ast, AstUtils, Constant, ConstantUtils, IdentifierUtils, Token } from "../../language"; import { Disambiguation, DisambiguationUtils } from "../disambiguation"; -import { NaiveParseSteps, ParseContext, ParseContextUtils, ParseError } from ".."; +import { NaiveParseSteps, ParseError } from ".."; import { Parser, ParseStateCheckpoint } from "../parser"; import { ParseState, ParseStateUtils } from "../parseState"; import { Trace, TraceConstant } from "../../common/trace"; @@ -197,83 +197,6 @@ export function readKeyword( return identifierExpression; } -// -------------------------------------- -// ---------- 12.2.1 Documents ---------- -// -------------------------------------- - -export async function readDocument( - state: ParseState, - parser: Parser, - correlationId: number | undefined, -): Promise { - const trace: Trace = state.traceManager.entry(NaiveTraceConstant.Parse, readDocument.name, correlationId, { - [NaiveTraceConstant.TokenIndex]: state.tokenIndex, - }); - - state.cancellationToken?.throwIfCancelled(); - - let document: Ast.TDocument; - - // Try parsing as an Expression document first. - // If Expression document fails (including UnusedTokensRemainError) then try parsing a SectionDocument. - // If both fail then return the error which parsed more tokens. - try { - document = await parser.readExpression(state, parser, trace.id); - ParseStateUtils.assertIsDoneParsing(state); - } catch (expressionError: unknown) { - Assert.isInstanceofError(expressionError); - CommonError.throwIfCancellationError(expressionError); - - // Fast backup deletes context state, but we want to preserve it for the case - // where both parsing an expression and section document error out. - const expressionCheckpoint: ParseStateCheckpoint = await parser.checkpoint(state); - const expressionErrorContextState: ParseContext.State = state.contextState; - - // Reset the parser's state. - state.tokenIndex = 0; - state.contextState = ParseContextUtils.newState(); - state.currentContextNode = undefined; - - if (state.lexerSnapshot.tokens.length) { - state.currentToken = state.lexerSnapshot.tokens[0]; - state.currentTokenKind = state.currentToken?.kind; - } - - try { - document = await readSectionDocument(state, parser, trace.id); - ParseStateUtils.assertIsDoneParsing(state); - } catch (sectionError: unknown) { - Assert.isInstanceofError(sectionError); - CommonError.throwIfCancellationError(sectionError); - - let triedError: Error; - - if (expressionCheckpoint.tokenIndex > /* sectionErrorState */ state.tokenIndex) { - triedError = expressionError; - await parser.restoreCheckpoint(state, expressionCheckpoint); - // eslint-disable-next-line require-atomic-updates - state.contextState = expressionErrorContextState; - } else { - triedError = sectionError; - } - - trace.exit({ - [NaiveTraceConstant.TokenIndex]: state.tokenIndex, - [TraceConstant.IsThrowing]: true, - }); - - throw triedError; - } - } - - trace.exit({ - [NaiveTraceConstant.TokenIndex]: state.tokenIndex, - [TraceConstant.IsThrowing]: false, - }); - - return document; -} - // ---------------------------------------------- // ---------- 12.2.2 Section Documents ---------- // ---------------------------------------------- diff --git a/src/powerquery-parser/parser/parsers/recursiveDescentParser.ts b/src/powerquery-parser/parser/parsers/recursiveDescentParser.ts index 6d53f512..923ea84a 100644 --- a/src/powerquery-parser/parser/parsers/recursiveDescentParser.ts +++ b/src/powerquery-parser/parser/parsers/recursiveDescentParser.ts @@ -15,8 +15,6 @@ export const RecursiveDescentParser: Parser = { readGeneralizedIdentifier: NaiveParseSteps.readGeneralizedIdentifier, readKeyword: NaiveParseSteps.readKeyword, - readDocument: NaiveParseSteps.readDocument, - readSectionDocument: NaiveParseSteps.readSectionDocument, readSectionMembers: NaiveParseSteps.readSectionMembers, readSectionMember: NaiveParseSteps.readSectionMember, diff --git a/src/powerquery-parser/settings.ts b/src/powerquery-parser/settings.ts index 9980b6a7..289b469d 100644 --- a/src/powerquery-parser/settings.ts +++ b/src/powerquery-parser/settings.ts @@ -5,16 +5,18 @@ import { CombinatorialParserV2, ParseSettings, ParseState, ParseStateUtils } fro import { LexerSnapshot, LexSettings } from "./lexer"; import { DefaultLocale } from "./localization"; import { NoOpTraceManagerInstance } from "./common/trace"; +import { ParseBehavior } from "./parser/parseBehavior"; export type Settings = LexSettings & ParseSettings; export const DefaultSettings: Settings = { - newParseState: (lexerSnapshot: LexerSnapshot, overrides: Partial | undefined) => + newParseState: (lexerSnapshot: LexerSnapshot, overrides?: Partial) => ParseStateUtils.newState(lexerSnapshot, overrides), locale: DefaultLocale, cancellationToken: undefined, initialCorrelationId: undefined, parserEntryPoint: undefined, + parseBehavior: ParseBehavior.ParseAll, parser: CombinatorialParserV2, traceManager: NoOpTraceManagerInstance, }; diff --git a/src/test/libraryTest/parser/parseBehavior.test.ts b/src/test/libraryTest/parser/parseBehavior.test.ts new file mode 100644 index 00000000..dc995e45 --- /dev/null +++ b/src/test/libraryTest/parser/parseBehavior.test.ts @@ -0,0 +1,133 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import "mocha"; +import { expect } from "chai"; + +import * as ParserTestUtils from "./parserTestUtils"; +import { Assert, DefaultSettings, Task, TaskUtils } from "../../../powerquery-parser"; +import { NodeKind } from "../../../powerquery-parser/language/ast/ast"; +import { ParseBehavior } from "../../../powerquery-parser/parser/parseBehavior"; +import { ParseError } from "../../../powerquery-parser/parser"; + +describe("ParseBehavior", () => { + async function runParseBehaviorTest(params: { + readonly parseBehavior: ParseBehavior; + readonly text: string; + readonly expectedAbridgedNodes: ReadonlyArray; + readonly expectedStatus: "ExpectedAnyTokenKindError" | "ExpectedTokenKindError" | "ParseStageOk"; + }): Promise { + const result: Task.ParseTaskOk | Task.ParseTaskParseError = await ParserTestUtils.runAbridgedNodeTest( + params.text, + params.expectedAbridgedNodes, + { + astOnly: true, + settings: { + ...DefaultSettings, + parseBehavior: params.parseBehavior, + }, + }, + ); + + switch (params.expectedStatus) { + case "ExpectedAnyTokenKindError": + TaskUtils.assertIsParseStageError(result); + expect(result.error.innerError).to.be.instanceOf(ParseError.ExpectedAnyTokenKindError); + break; + + case "ExpectedTokenKindError": + TaskUtils.assertIsParseStageError(result); + expect(result.error.innerError).to.be.instanceOf(ParseError.ExpectedTokenKindError); + break; + + case "ParseStageOk": + TaskUtils.assertIsParseStageOk(result); + break; + + default: + Assert.isNever(params.expectedStatus); + } + + return result; + } + + it(`1 // with ParseAll`, async () => { + await runParseBehaviorTest({ + parseBehavior: ParseBehavior.ParseAll, + text: `1`, + expectedAbridgedNodes: [[NodeKind.LiteralExpression, undefined]], + expectedStatus: "ParseStageOk", + }); + }); + + it(`1 // with ParseExpression`, async () => { + await runParseBehaviorTest({ + parseBehavior: ParseBehavior.ParseExpression, + text: `1`, + expectedAbridgedNodes: [[NodeKind.LiteralExpression, undefined]], + expectedStatus: "ParseStageOk", + }); + }); + + it(`1 // with ParseSection`, async () => { + await runParseBehaviorTest({ + parseBehavior: ParseBehavior.ParseSection, + text: `1`, + expectedAbridgedNodes: [], + expectedStatus: "ExpectedTokenKindError", + }); + }); + + it(`section Foo; shared Bar = 1; // with ParseAll`, async () => { + await runParseBehaviorTest({ + parseBehavior: ParseBehavior.ParseAll, + text: `section Foo; shared Bar = 1;`, + expectedAbridgedNodes: [ + [NodeKind.Section, undefined], + [NodeKind.Constant, 1], + [NodeKind.Identifier, 2], + [NodeKind.Constant, 3], + [NodeKind.ArrayWrapper, 4], + [NodeKind.SectionMember, 0], + [NodeKind.Constant, 1], + [NodeKind.IdentifierPairedExpression, 2], + [NodeKind.Identifier, 0], + [NodeKind.Constant, 1], + [NodeKind.LiteralExpression, 2], + [NodeKind.Constant, 3], + ], + expectedStatus: "ParseStageOk", + }); + }); + + it(`section Foo; shared Bar = 1; // with ParseExpression`, async () => { + await runParseBehaviorTest({ + parseBehavior: ParseBehavior.ParseExpression, + text: `section Foo; shared Bar = 1;`, + expectedAbridgedNodes: [], + expectedStatus: "ExpectedAnyTokenKindError", + }); + }); + + it(`section Foo; shared Bar = 1; // with ParseSection`, async () => { + await runParseBehaviorTest({ + parseBehavior: ParseBehavior.ParseSection, + text: `section Foo; shared Bar = 1;`, + expectedAbridgedNodes: [ + [NodeKind.Section, undefined], + [NodeKind.Constant, 1], + [NodeKind.Identifier, 2], + [NodeKind.Constant, 3], + [NodeKind.ArrayWrapper, 4], + [NodeKind.SectionMember, 0], + [NodeKind.Constant, 1], + [NodeKind.IdentifierPairedExpression, 2], + [NodeKind.Identifier, 0], + [NodeKind.Constant, 1], + [NodeKind.LiteralExpression, 2], + [NodeKind.Constant, 3], + ], + expectedStatus: "ParseStageOk", + }); + }); +}); diff --git a/src/test/libraryTest/parser/parseSimple.test.ts b/src/test/libraryTest/parser/parseSimple.test.ts index 6f6c64ab..4a383764 100644 --- a/src/test/libraryTest/parser/parseSimple.test.ts +++ b/src/test/libraryTest/parser/parseSimple.test.ts @@ -2,184 +2,14 @@ // Licensed under the MIT license. import "mocha"; -import { expect } from "chai"; -import { Assert, DefaultLocale, DefaultSettings, ResultUtils, Task, TaskUtils, Traverse } from "../../.."; +import * as ParserTestUtils from "./parserTestUtils"; import { Ast, Constant } from "../../../powerquery-parser/language"; -import { NodeIdMap, TXorNode, XorNodeUtils } from "../../../powerquery-parser/parser"; -import { AssertTestUtils } from "../../testUtils"; -import { NoOpTraceManagerInstance } from "../../../powerquery-parser/common/trace"; - -type AbridgedNode = [Ast.NodeKind, number | undefined]; - -type CollectAbridgeNodeState = Traverse.ITraversalState; - -interface NthNodeOfKindState extends Traverse.ITraversalState { - readonly nodeKind: Ast.NodeKind; - readonly nthRequired: number; - nthCounter: number; -} - -async function collectAbridgeNodeFromXor( - nodeIdMapCollection: NodeIdMap.Collection, - root: TXorNode, -): Promise> { - const state: CollectAbridgeNodeState = { - locale: DefaultLocale, - result: [], - cancellationToken: undefined, - initialCorrelationId: undefined, - traceManager: NoOpTraceManagerInstance, - }; - - const triedTraverse: Traverse.TriedTraverse = await Traverse.tryTraverseXor< - CollectAbridgeNodeState, - AbridgedNode[] - >( - state, - nodeIdMapCollection, - root, - Traverse.VisitNodeStrategy.BreadthFirst, - collectAbridgeXorNodeVisit, - Traverse.assertGetAllXorChildren, - undefined, - ); - - ResultUtils.assertIsOk(triedTraverse); - - return triedTraverse.value; -} - -async function assertGetNthNodeOfKind( - text: string, - nodeKind: Ast.NodeKind, - nthRequired: number, -): Promise { - const parseTaskOk: Task.ParseTaskOk = await AssertTestUtils.assertGetLexParseOk(DefaultSettings, text); - - const state: NthNodeOfKindState = { - locale: DefaultLocale, - result: undefined, - nodeKind, - nthCounter: 0, - nthRequired, - cancellationToken: undefined, - initialCorrelationId: undefined, - traceManager: NoOpTraceManagerInstance, - }; - - const triedTraverse: Traverse.TriedTraverse = await Traverse.tryTraverseAst< - NthNodeOfKindState, - Ast.TNode | undefined - >( - state, - parseTaskOk.nodeIdMapCollection, - parseTaskOk.ast, - Traverse.VisitNodeStrategy.BreadthFirst, - nthNodeVisit, - Traverse.assertGetAllAstChildren, - nthNodeEarlyExit, - ); - - ResultUtils.assertIsOk(triedTraverse); - - return Assert.asDefined(triedTraverse.value) as N; -} - -// eslint-disable-next-line require-await -async function collectAbridgeXorNodeVisit(state: CollectAbridgeNodeState, xorNode: TXorNode): Promise { - state.result.push([xorNode.node.kind, xorNode.node.attributeIndex]); -} - -// eslint-disable-next-line require-await -async function nthNodeVisit(state: NthNodeOfKindState, node: Ast.TNode): Promise { - if (node.kind === state.nodeKind) { - state.nthCounter += 1; - - if (state.nthCounter === state.nthRequired) { - state.result = node; - } - } -} - -// eslint-disable-next-line require-await -async function nthNodeEarlyExit(state: NthNodeOfKindState, _: Ast.TNode): Promise { - return state.nthCounter === state.nthRequired; -} - -function validateNodeIdMapCollection(nodeIdMapCollection: NodeIdMap.Collection, root: TXorNode): void { - const astNodeIds: Set = new Set(nodeIdMapCollection.astNodeById.keys()); - const contextNodeIds: Set = new Set(nodeIdMapCollection.contextNodeById.keys()); - const allNodeIds: Set = new Set([...astNodeIds].concat([...contextNodeIds])); - - expect(nodeIdMapCollection.parentIdById).to.not.have.key(root.node.id.toString()); - - expect(nodeIdMapCollection.parentIdById.size).to.equal( - allNodeIds.size - 1, - "parentIdById should have one less entry than allNodeIds", - ); - - expect(astNodeIds.size + contextNodeIds.size).to.equal( - allNodeIds.size, - "allNodeIds should be a union of astNodeIds and contextNodeIds", - ); - - for (const [childId, parentId] of nodeIdMapCollection.parentIdById.entries()) { - expect(allNodeIds).to.include(childId, "keys for parentIdById should be in allNodeIds"); - expect(allNodeIds).to.include(parentId, "values for parentIdById should be in allNodeIds"); - } - - for (const [parentId, childrenIds] of nodeIdMapCollection.childIdsById.entries()) { - expect(allNodeIds).to.include(parentId, "keys for childIdsById should be in allNodeIds"); - - for (const childId of childrenIds) { - expect(allNodeIds).to.include(childId, "childIds should be in allNodeIds"); - - if (astNodeIds.has(parentId)) { - expect(astNodeIds).to.include(childId, "if a parent is an astNode then so should be its children"); - } - } - } -} describe("Parser.AbridgedNode", () => { - async function runAbridgedNodeTest(text: string, expected: ReadonlyArray): Promise { - const triedLexParse: Task.TriedLexParseTask = await TaskUtils.tryLexParse(DefaultSettings, text); - - let root: TXorNode; - let nodeIdMapCollection: NodeIdMap.Collection; - - if (TaskUtils.isParseStageOk(triedLexParse)) { - root = XorNodeUtils.boxAst(triedLexParse.ast); - nodeIdMapCollection = triedLexParse.nodeIdMapCollection; - } else if (TaskUtils.isParseStageParseError(triedLexParse)) { - root = XorNodeUtils.boxContext(Assert.asDefined(triedLexParse.parseState.contextState.root)); - nodeIdMapCollection = triedLexParse.nodeIdMapCollection; - } else { - throw new Error(`expected isParseStageOk/isParseStageParseError`); - } - - validateNodeIdMapCollection(nodeIdMapCollection, root); - - const actual: ReadonlyArray = await collectAbridgeNodeFromXor(nodeIdMapCollection, root); - expect(actual).to.deep.equal(expected); - } - - async function runAbridgedNodeAndOperatorTest( - text: string, - constant: Constant.TConstant, - expected: ReadonlyArray, - ): Promise { - await runAbridgedNodeTest(text, expected); - - const operatorNode: Ast.TConstant = await assertGetNthNodeOfKind(text, Ast.NodeKind.Constant, 1); - - expect(operatorNode.constantKind).to.equal(constant); - } - describe(`${Ast.NodeKind.ArithmeticExpression}`, () => { it(`1 &`, async () => { - await runAbridgedNodeTest(`1 &`, [ + await ParserTestUtils.runAbridgedNodeTest(`1 &`, [ [Ast.NodeKind.LogicalExpression, undefined], [Ast.NodeKind.LiteralExpression, 0], [Ast.NodeKind.ArithmeticExpression, 1], @@ -189,7 +19,7 @@ describe("Parser.AbridgedNode", () => { }); it(`1 *`, async () => { - await runAbridgedNodeTest(`1 *`, [ + await ParserTestUtils.runAbridgedNodeTest(`1 *`, [ [Ast.NodeKind.LogicalExpression, undefined], [Ast.NodeKind.LiteralExpression, 0], [Ast.NodeKind.ArithmeticExpression, 1], @@ -199,7 +29,7 @@ describe("Parser.AbridgedNode", () => { }); it(`1 /`, async () => { - await runAbridgedNodeTest(`1 /`, [ + await ParserTestUtils.runAbridgedNodeTest(`1 /`, [ [Ast.NodeKind.LogicalExpression, undefined], [Ast.NodeKind.LiteralExpression, 0], [Ast.NodeKind.ArithmeticExpression, 1], @@ -209,7 +39,7 @@ describe("Parser.AbridgedNode", () => { }); it(`1 +`, async () => { - await runAbridgedNodeTest(`1 +`, [ + await ParserTestUtils.runAbridgedNodeTest(`1 +`, [ [Ast.NodeKind.LogicalExpression, undefined], [Ast.NodeKind.LiteralExpression, 0], [Ast.NodeKind.ArithmeticExpression, 1], @@ -219,7 +49,7 @@ describe("Parser.AbridgedNode", () => { }); it(`1 -`, async () => { - await runAbridgedNodeTest(`1 -`, [ + await ParserTestUtils.runAbridgedNodeTest(`1 -`, [ [Ast.NodeKind.LogicalExpression, undefined], [Ast.NodeKind.LiteralExpression, 0], [Ast.NodeKind.ArithmeticExpression, 1], @@ -229,7 +59,7 @@ describe("Parser.AbridgedNode", () => { }); it(`1 & 2`, async () => { - await runAbridgedNodeAndOperatorTest(`1 & 2`, Constant.ArithmeticOperator.And, [ + await ParserTestUtils.runAbridgedNodeAndOperatorTest(`1 & 2`, Constant.ArithmeticOperator.And, [ [Ast.NodeKind.ArithmeticExpression, undefined], [Ast.NodeKind.LiteralExpression, 0], [Ast.NodeKind.Constant, 1], @@ -238,7 +68,7 @@ describe("Parser.AbridgedNode", () => { }); it(`1 * 2`, async () => { - await runAbridgedNodeAndOperatorTest(`1 * 2`, Constant.ArithmeticOperator.Multiplication, [ + await ParserTestUtils.runAbridgedNodeAndOperatorTest(`1 * 2`, Constant.ArithmeticOperator.Multiplication, [ [Ast.NodeKind.ArithmeticExpression, undefined], [Ast.NodeKind.LiteralExpression, 0], [Ast.NodeKind.Constant, 1], @@ -247,7 +77,7 @@ describe("Parser.AbridgedNode", () => { }); it(`1 / 2`, async () => { - await runAbridgedNodeAndOperatorTest(`1 / 2`, Constant.ArithmeticOperator.Division, [ + await ParserTestUtils.runAbridgedNodeAndOperatorTest(`1 / 2`, Constant.ArithmeticOperator.Division, [ [Ast.NodeKind.ArithmeticExpression, undefined], [Ast.NodeKind.LiteralExpression, 0], [Ast.NodeKind.Constant, 1], @@ -256,7 +86,7 @@ describe("Parser.AbridgedNode", () => { }); it(`1 + 2`, async () => { - await runAbridgedNodeAndOperatorTest(`1 + 2`, Constant.ArithmeticOperator.Addition, [ + await ParserTestUtils.runAbridgedNodeAndOperatorTest(`1 + 2`, Constant.ArithmeticOperator.Addition, [ [Ast.NodeKind.ArithmeticExpression, undefined], [Ast.NodeKind.LiteralExpression, 0], [Ast.NodeKind.Constant, 1], @@ -265,7 +95,7 @@ describe("Parser.AbridgedNode", () => { }); it(`1 - 2`, async () => { - await runAbridgedNodeAndOperatorTest(`1 - 2`, Constant.ArithmeticOperator.Subtraction, [ + await ParserTestUtils.runAbridgedNodeAndOperatorTest(`1 - 2`, Constant.ArithmeticOperator.Subtraction, [ [Ast.NodeKind.ArithmeticExpression, undefined], [Ast.NodeKind.LiteralExpression, 0], [Ast.NodeKind.Constant, 1], @@ -274,7 +104,7 @@ describe("Parser.AbridgedNode", () => { }); it(`1 + 2 + 3 + 4`, async () => { - await runAbridgedNodeTest(`1 + 2 + 3 + 4`, [ + await ParserTestUtils.runAbridgedNodeTest(`1 + 2 + 3 + 4`, [ [Ast.NodeKind.ArithmeticExpression, undefined], [Ast.NodeKind.ArithmeticExpression, 0], [Ast.NodeKind.ArithmeticExpression, 0], @@ -291,7 +121,7 @@ describe("Parser.AbridgedNode", () => { describe(`${Ast.NodeKind.AsExpression}`, () => { it(`1 as`, async () => { - await runAbridgedNodeTest(`1 as`, [ + await ParserTestUtils.runAbridgedNodeTest(`1 as`, [ [Ast.NodeKind.LogicalExpression, undefined], [Ast.NodeKind.LiteralExpression, 0], [Ast.NodeKind.AsExpression, 1], @@ -301,7 +131,7 @@ describe("Parser.AbridgedNode", () => { }); it(`1 as number`, async () => { - await runAbridgedNodeTest(`1 as number`, [ + await ParserTestUtils.runAbridgedNodeTest(`1 as number`, [ [Ast.NodeKind.AsExpression, undefined], [Ast.NodeKind.LiteralExpression, 0], [Ast.NodeKind.Constant, 1], @@ -310,7 +140,7 @@ describe("Parser.AbridgedNode", () => { }); it(`1 as number as logical`, async () => { - await runAbridgedNodeTest(`1 as number as logical`, [ + await ParserTestUtils.runAbridgedNodeTest(`1 as number as logical`, [ [Ast.NodeKind.AsExpression, undefined], [Ast.NodeKind.AsExpression, 0], [Ast.NodeKind.LiteralExpression, 0], @@ -322,7 +152,7 @@ describe("Parser.AbridgedNode", () => { }); it(`type function (x as number) as number`, async () => { - await runAbridgedNodeTest(`type function (x as number) as number`, [ + await ParserTestUtils.runAbridgedNodeTest(`type function (x as number) as number`, [ [Ast.NodeKind.TypePrimaryType, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.FunctionType, 1], @@ -349,7 +179,7 @@ describe("Parser.AbridgedNode", () => { // Ast.Ast.NodeKind.Csv covered by many it(`${Ast.NodeKind.EachExpression}`, async () => { - await runAbridgedNodeTest(`each 1`, [ + await ParserTestUtils.runAbridgedNodeTest(`each 1`, [ [Ast.NodeKind.EachExpression, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.LiteralExpression, 1], @@ -358,7 +188,7 @@ describe("Parser.AbridgedNode", () => { describe(`${Ast.NodeKind.EqualityExpression}`, () => { it(`1 = 2`, async () => { - await runAbridgedNodeAndOperatorTest(`1 = 2`, Constant.EqualityOperator.EqualTo, [ + await ParserTestUtils.runAbridgedNodeAndOperatorTest(`1 = 2`, Constant.EqualityOperator.EqualTo, [ [Ast.NodeKind.EqualityExpression, undefined], [Ast.NodeKind.LiteralExpression, 0], [Ast.NodeKind.Constant, 1], @@ -367,7 +197,7 @@ describe("Parser.AbridgedNode", () => { }); it(`1 <> 2`, async () => { - await runAbridgedNodeAndOperatorTest(`1 <> 2`, Constant.EqualityOperator.NotEqualTo, [ + await ParserTestUtils.runAbridgedNodeAndOperatorTest(`1 <> 2`, Constant.EqualityOperator.NotEqualTo, [ [Ast.NodeKind.EqualityExpression, undefined], [Ast.NodeKind.LiteralExpression, 0], [Ast.NodeKind.Constant, 1], @@ -378,7 +208,7 @@ describe("Parser.AbridgedNode", () => { describe(`${Ast.NodeKind.ErrorHandlingExpression}`, () => { it(`try 1`, async () => { - await runAbridgedNodeTest(`try 1`, [ + await ParserTestUtils.runAbridgedNodeTest(`try 1`, [ [Ast.NodeKind.ErrorHandlingExpression, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.LiteralExpression, 1], @@ -386,7 +216,7 @@ describe("Parser.AbridgedNode", () => { }); it(`try 1 otherwise 2`, async () => { - await runAbridgedNodeTest(`try 1 otherwise 2`, [ + await ParserTestUtils.runAbridgedNodeTest(`try 1 otherwise 2`, [ [Ast.NodeKind.ErrorHandlingExpression, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.LiteralExpression, 1], @@ -397,7 +227,7 @@ describe("Parser.AbridgedNode", () => { }); it(`try 1 catch () => 1`, async () => { - await runAbridgedNodeTest(`try 1 catch () => 1`, [ + await ParserTestUtils.runAbridgedNodeTest(`try 1 catch () => 1`, [ [Ast.NodeKind.ErrorHandlingExpression, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.LiteralExpression, 1], @@ -414,7 +244,7 @@ describe("Parser.AbridgedNode", () => { }); it(`try 1 catch (x) => 1`, async () => { - await runAbridgedNodeTest(`try 1 catch (x) => 1`, [ + await ParserTestUtils.runAbridgedNodeTest(`try 1 catch (x) => 1`, [ [Ast.NodeKind.ErrorHandlingExpression, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.LiteralExpression, 1], @@ -435,7 +265,7 @@ describe("Parser.AbridgedNode", () => { }); it(`${Ast.NodeKind.ErrorRaisingExpression}`, async () => { - await runAbridgedNodeTest(`error 1`, [ + await ParserTestUtils.runAbridgedNodeTest(`error 1`, [ [Ast.NodeKind.ErrorRaisingExpression, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.LiteralExpression, 1], @@ -444,7 +274,7 @@ describe("Parser.AbridgedNode", () => { describe(`${Ast.NodeKind.FieldProjection}`, () => { it(`x[[y]]`, async () => { - await runAbridgedNodeTest(`x[[y]]`, [ + await ParserTestUtils.runAbridgedNodeTest(`x[[y]]`, [ [Ast.NodeKind.RecursivePrimaryExpression, undefined], [Ast.NodeKind.IdentifierExpression, 0], [Ast.NodeKind.Identifier, 1], @@ -462,7 +292,7 @@ describe("Parser.AbridgedNode", () => { }); it(`x[[y], [z]]`, async () => { - await runAbridgedNodeTest(`x[[y], [z]]`, [ + await ParserTestUtils.runAbridgedNodeTest(`x[[y], [z]]`, [ [Ast.NodeKind.RecursivePrimaryExpression, undefined], [Ast.NodeKind.IdentifierExpression, 0], [Ast.NodeKind.Identifier, 1], @@ -486,7 +316,7 @@ describe("Parser.AbridgedNode", () => { }); it(`x[[y]]?`, async () => { - await runAbridgedNodeTest(`x[[y]]?`, [ + await ParserTestUtils.runAbridgedNodeTest(`x[[y]]?`, [ [Ast.NodeKind.RecursivePrimaryExpression, undefined], [Ast.NodeKind.IdentifierExpression, 0], [Ast.NodeKind.Identifier, 1], @@ -507,7 +337,7 @@ describe("Parser.AbridgedNode", () => { describe(`${Ast.NodeKind.FieldSelector}`, () => { it(`[x]`, async () => { - await runAbridgedNodeTest(`[x]`, [ + await ParserTestUtils.runAbridgedNodeTest(`[x]`, [ [Ast.NodeKind.FieldSelector, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.GeneralizedIdentifier, 1], @@ -516,7 +346,7 @@ describe("Parser.AbridgedNode", () => { }); it(`[x]?`, async () => { - await runAbridgedNodeTest(`[x]?`, [ + await ParserTestUtils.runAbridgedNodeTest(`[x]?`, [ [Ast.NodeKind.FieldSelector, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.GeneralizedIdentifier, 1], @@ -528,7 +358,7 @@ describe("Parser.AbridgedNode", () => { describe(`${Ast.NodeKind.FieldSpecification}`, () => { it(`type [x]`, async () => { - await runAbridgedNodeTest(`type [x]`, [ + await ParserTestUtils.runAbridgedNodeTest(`type [x]`, [ [Ast.NodeKind.TypePrimaryType, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.RecordType, 1], @@ -543,7 +373,7 @@ describe("Parser.AbridgedNode", () => { }); it(`type [optional x]`, async () => { - await runAbridgedNodeTest(`type [optional x]`, [ + await ParserTestUtils.runAbridgedNodeTest(`type [optional x]`, [ [Ast.NodeKind.TypePrimaryType, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.RecordType, 1], @@ -559,7 +389,7 @@ describe("Parser.AbridgedNode", () => { }); it(`type [x = number]`, async () => { - await runAbridgedNodeTest(`type [x = number]`, [ + await ParserTestUtils.runAbridgedNodeTest(`type [x = number]`, [ [Ast.NodeKind.TypePrimaryType, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.RecordType, 1], @@ -579,7 +409,7 @@ describe("Parser.AbridgedNode", () => { describe(`${Ast.NodeKind.FieldSpecificationList}`, () => { it(`type []`, async () => { - await runAbridgedNodeTest(`type []`, [ + await ParserTestUtils.runAbridgedNodeTest(`type []`, [ [Ast.NodeKind.TypePrimaryType, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.RecordType, 1], @@ -591,7 +421,7 @@ describe("Parser.AbridgedNode", () => { }); it(`type table []`, async () => { - await runAbridgedNodeTest(`type table []`, [ + await ParserTestUtils.runAbridgedNodeTest(`type table []`, [ [Ast.NodeKind.TypePrimaryType, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.TableType, 1], @@ -604,7 +434,7 @@ describe("Parser.AbridgedNode", () => { }); it(`${Ast.NodeKind.FieldSpecificationList}`, async () => { - await runAbridgedNodeTest(`type [x]`, [ + await ParserTestUtils.runAbridgedNodeTest(`type [x]`, [ [Ast.NodeKind.TypePrimaryType, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.RecordType, 1], @@ -619,7 +449,7 @@ describe("Parser.AbridgedNode", () => { }); it(`type [x, ...]`, async () => { - await runAbridgedNodeTest(`type [x, ...]`, [ + await ParserTestUtils.runAbridgedNodeTest(`type [x, ...]`, [ [Ast.NodeKind.TypePrimaryType, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.RecordType, 1], @@ -640,7 +470,7 @@ describe("Parser.AbridgedNode", () => { describe(`${Ast.NodeKind.FunctionExpression}`, () => { it(`() => 1`, async () => { - await runAbridgedNodeTest(`() => 1`, [ + await ParserTestUtils.runAbridgedNodeTest(`() => 1`, [ [Ast.NodeKind.FunctionExpression, undefined], [Ast.NodeKind.ParameterList, 0], [Ast.NodeKind.Constant, 0], @@ -652,7 +482,7 @@ describe("Parser.AbridgedNode", () => { }); it(`(x) => 1`, async () => { - await runAbridgedNodeTest(`(x) => 1`, [ + await ParserTestUtils.runAbridgedNodeTest(`(x) => 1`, [ [Ast.NodeKind.FunctionExpression, undefined], [Ast.NodeKind.ParameterList, 0], [Ast.NodeKind.Constant, 0], @@ -667,7 +497,7 @@ describe("Parser.AbridgedNode", () => { }); it(`(x, y, z) => 1`, async () => { - await runAbridgedNodeTest(`(x, y, z) => 1`, [ + await ParserTestUtils.runAbridgedNodeTest(`(x, y, z) => 1`, [ [Ast.NodeKind.FunctionExpression, undefined], [Ast.NodeKind.ParameterList, 0], [Ast.NodeKind.Constant, 0], @@ -690,7 +520,7 @@ describe("Parser.AbridgedNode", () => { }); it(`(optional x) => 1`, async () => { - await runAbridgedNodeTest(`(optional x) => 1`, [ + await ParserTestUtils.runAbridgedNodeTest(`(optional x) => 1`, [ [Ast.NodeKind.FunctionExpression, undefined], [Ast.NodeKind.ParameterList, 0], [Ast.NodeKind.Constant, 0], @@ -706,7 +536,7 @@ describe("Parser.AbridgedNode", () => { }); it(`(x as nullable text) => 1`, async () => { - await runAbridgedNodeTest(`(x as nullable text) => 1`, [ + await ParserTestUtils.runAbridgedNodeTest(`(x as nullable text) => 1`, [ [Ast.NodeKind.FunctionExpression, undefined], [Ast.NodeKind.ParameterList, 0], [Ast.NodeKind.Constant, 0], @@ -726,7 +556,7 @@ describe("Parser.AbridgedNode", () => { }); it(`(x) as number => x`, async () => { - await runAbridgedNodeTest(`(x) as number => x`, [ + await ParserTestUtils.runAbridgedNodeTest(`(x) as number => x`, [ [Ast.NodeKind.FunctionExpression, undefined], [Ast.NodeKind.ParameterList, 0], [Ast.NodeKind.Constant, 0], @@ -745,7 +575,7 @@ describe("Parser.AbridgedNode", () => { }); it(`(x as number) as number => x`, async () => { - await runAbridgedNodeTest(`(x as number) as number => x`, [ + await ParserTestUtils.runAbridgedNodeTest(`(x as number) as number => x`, [ [Ast.NodeKind.FunctionExpression, undefined], [Ast.NodeKind.ParameterList, 0], [Ast.NodeKind.Constant, 0], @@ -767,7 +597,7 @@ describe("Parser.AbridgedNode", () => { }); it(`(x as number) as nullable number => x`, async () => { - await runAbridgedNodeTest(`(x as number) as nullable number => x`, [ + await ParserTestUtils.runAbridgedNodeTest(`(x as number) as nullable number => x`, [ [Ast.NodeKind.FunctionExpression, undefined], [Ast.NodeKind.ParameterList, 0], [Ast.NodeKind.Constant, 0], @@ -791,7 +621,7 @@ describe("Parser.AbridgedNode", () => { }); it(`let Fn = () as nullable text => "asd" in Fn`, async () => { - await runAbridgedNodeTest(`let Fn = () as nullable text => "asd" in Fn`, [ + await ParserTestUtils.runAbridgedNodeTest(`let Fn = () as nullable text => "asd" in Fn`, [ [Ast.NodeKind.LetExpression, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.ArrayWrapper, 1], @@ -820,7 +650,7 @@ describe("Parser.AbridgedNode", () => { describe(`${Ast.NodeKind.FunctionType}`, () => { it(`type function () as number`, async () => { - await runAbridgedNodeTest(`type function () as number`, [ + await ParserTestUtils.runAbridgedNodeTest(`type function () as number`, [ [Ast.NodeKind.TypePrimaryType, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.FunctionType, 1], @@ -836,7 +666,7 @@ describe("Parser.AbridgedNode", () => { }); it(`type function (x as number) as number`, async () => { - await runAbridgedNodeTest(`type function (x as number) as number`, [ + await ParserTestUtils.runAbridgedNodeTest(`type function (x as number) as number`, [ [Ast.NodeKind.TypePrimaryType, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.FunctionType, 1], @@ -862,7 +692,7 @@ describe("Parser.AbridgedNode", () => { describe(`${Ast.NodeKind.GeneralizedIdentifier}`, () => { it(`[foo bar]`, async () => { - await runAbridgedNodeTest(`[foo bar]`, [ + await ParserTestUtils.runAbridgedNodeTest(`[foo bar]`, [ [Ast.NodeKind.FieldSelector, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.GeneralizedIdentifier, 1], @@ -871,7 +701,7 @@ describe("Parser.AbridgedNode", () => { }); it(`[1]`, async () => { - await runAbridgedNodeTest(`[1]`, [ + await ParserTestUtils.runAbridgedNodeTest(`[1]`, [ [Ast.NodeKind.FieldSelector, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.GeneralizedIdentifier, 1], @@ -880,7 +710,7 @@ describe("Parser.AbridgedNode", () => { }); it(`[a.1]`, async () => { - await runAbridgedNodeTest(`[a.1]`, [ + await ParserTestUtils.runAbridgedNodeTest(`[a.1]`, [ [Ast.NodeKind.FieldSelector, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.GeneralizedIdentifier, 1], @@ -889,7 +719,7 @@ describe("Parser.AbridgedNode", () => { }); it(`[#"a""" = 1]`, async () => { - await runAbridgedNodeTest(`[#"a""" = 1]`, [ + await ParserTestUtils.runAbridgedNodeTest(`[#"a""" = 1]`, [ [Ast.NodeKind.RecordExpression, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.ArrayWrapper, 1], @@ -904,7 +734,7 @@ describe("Parser.AbridgedNode", () => { }); it(`Ast.Ast.NodeKind.GeneralizedIdentifierPairedAnyLiteral`, async () => { - await runAbridgedNodeTest(`[x=1] section;`, [ + await ParserTestUtils.runAbridgedNodeTest(`[x=1] section;`, [ [Ast.NodeKind.Section, undefined], [Ast.NodeKind.RecordLiteral, 0], [Ast.NodeKind.Constant, 0], @@ -922,7 +752,7 @@ describe("Parser.AbridgedNode", () => { }); it(`${Ast.NodeKind.GeneralizedIdentifierPairedExpression}`, async () => { - await runAbridgedNodeTest(`[x=1]`, [ + await ParserTestUtils.runAbridgedNodeTest(`[x=1]`, [ [Ast.NodeKind.RecordExpression, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.ArrayWrapper, 1], @@ -939,7 +769,7 @@ describe("Parser.AbridgedNode", () => { describe(`${Ast.NodeKind.IdentifierExpression}`, () => { it(`@foo`, async () => { - await runAbridgedNodeTest(`@foo`, [ + await ParserTestUtils.runAbridgedNodeTest(`@foo`, [ [Ast.NodeKind.IdentifierExpression, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.Identifier, 1], @@ -947,7 +777,7 @@ describe("Parser.AbridgedNode", () => { }); it(`零`, async () => { - await runAbridgedNodeTest(`零`, [ + await ParserTestUtils.runAbridgedNodeTest(`零`, [ [Ast.NodeKind.IdentifierExpression, undefined], [Ast.NodeKind.Identifier, 1], ]); @@ -955,7 +785,7 @@ describe("Parser.AbridgedNode", () => { }); it(`${Ast.NodeKind.IdentifierPairedExpression}`, async () => { - await runAbridgedNodeTest(`section; x = 1;`, [ + await ParserTestUtils.runAbridgedNodeTest(`section; x = 1;`, [ [Ast.NodeKind.Section, undefined], [Ast.NodeKind.Constant, 1], [Ast.NodeKind.Constant, 3], @@ -970,7 +800,7 @@ describe("Parser.AbridgedNode", () => { }); it(`${Ast.NodeKind.IfExpression}`, async () => { - await runAbridgedNodeTest(`if x then x else x`, [ + await ParserTestUtils.runAbridgedNodeTest(`if x then x else x`, [ [Ast.NodeKind.IfExpression, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.IdentifierExpression, 1], @@ -985,7 +815,7 @@ describe("Parser.AbridgedNode", () => { }); it(`${Ast.NodeKind.InvokeExpression}`, async () => { - await runAbridgedNodeTest(`foo()`, [ + await ParserTestUtils.runAbridgedNodeTest(`foo()`, [ [Ast.NodeKind.RecursivePrimaryExpression, undefined], [Ast.NodeKind.IdentifierExpression, 0], [Ast.NodeKind.Identifier, 1], @@ -999,7 +829,7 @@ describe("Parser.AbridgedNode", () => { describe(`${Ast.NodeKind.IsExpression}`, () => { it(`1 is`, async () => { - await runAbridgedNodeTest(`1 is`, [ + await ParserTestUtils.runAbridgedNodeTest(`1 is`, [ [Ast.NodeKind.LogicalExpression, undefined], [Ast.NodeKind.LiteralExpression, 0], [Ast.NodeKind.IsExpression, 1], @@ -1009,7 +839,7 @@ describe("Parser.AbridgedNode", () => { }); it(`1 is number`, async () => { - await runAbridgedNodeTest(`1 is number`, [ + await ParserTestUtils.runAbridgedNodeTest(`1 is number`, [ [Ast.NodeKind.IsExpression, undefined], [Ast.NodeKind.LiteralExpression, 0], [Ast.NodeKind.Constant, 1], @@ -1018,7 +848,7 @@ describe("Parser.AbridgedNode", () => { }); it(`1 is number is number`, async () => { - await runAbridgedNodeTest(`1 is number is number`, [ + await ParserTestUtils.runAbridgedNodeTest(`1 is number is number`, [ [Ast.NodeKind.IsExpression, undefined], [Ast.NodeKind.IsExpression, 0], [Ast.NodeKind.LiteralExpression, 0], @@ -1031,7 +861,7 @@ describe("Parser.AbridgedNode", () => { }); it(`${Ast.NodeKind.ItemAccessExpression}`, async () => { - await runAbridgedNodeTest(`x{1}`, [ + await ParserTestUtils.runAbridgedNodeTest(`x{1}`, [ [Ast.NodeKind.RecursivePrimaryExpression, undefined], [Ast.NodeKind.IdentifierExpression, 0], [Ast.NodeKind.Identifier, 1], @@ -1044,7 +874,7 @@ describe("Parser.AbridgedNode", () => { }); it(`${Ast.NodeKind.ItemAccessExpression} optional`, async () => { - await runAbridgedNodeTest(`x{1}?`, [ + await ParserTestUtils.runAbridgedNodeTest(`x{1}?`, [ [Ast.NodeKind.RecursivePrimaryExpression, undefined], [Ast.NodeKind.IdentifierExpression, 0], [Ast.NodeKind.Identifier, 1], @@ -1059,14 +889,14 @@ describe("Parser.AbridgedNode", () => { describe(`keywords`, () => { it(`#sections`, async () => { - await runAbridgedNodeTest(`#sections`, [ + await ParserTestUtils.runAbridgedNodeTest(`#sections`, [ [Ast.NodeKind.IdentifierExpression, undefined], [Ast.NodeKind.Identifier, 1], ]); }); it(`#shared`, async () => { - await runAbridgedNodeTest(`#shared`, [ + await ParserTestUtils.runAbridgedNodeTest(`#shared`, [ [Ast.NodeKind.IdentifierExpression, undefined], [Ast.NodeKind.Identifier, 1], ]); @@ -1075,7 +905,7 @@ describe("Parser.AbridgedNode", () => { describe(`${Ast.NodeKind.LetExpression}`, () => { it(`let in 1`, async () => { - await runAbridgedNodeTest(`let in 1`, [ + await ParserTestUtils.runAbridgedNodeTest(`let in 1`, [ [Ast.NodeKind.LetExpression, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.ArrayWrapper, 1], @@ -1085,7 +915,7 @@ describe("Parser.AbridgedNode", () => { }); it(`let x = 1 in x`, async () => { - await runAbridgedNodeTest(`let x = 1 in x`, [ + await ParserTestUtils.runAbridgedNodeTest(`let x = 1 in x`, [ [Ast.NodeKind.LetExpression, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.ArrayWrapper, 1], @@ -1101,7 +931,7 @@ describe("Parser.AbridgedNode", () => { }); it(`let x = 1 in try x`, async () => { - await runAbridgedNodeTest(`let x = 1 in try x`, [ + await ParserTestUtils.runAbridgedNodeTest(`let x = 1 in try x`, [ [Ast.NodeKind.LetExpression, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.ArrayWrapper, 1], @@ -1119,7 +949,7 @@ describe("Parser.AbridgedNode", () => { }); it(`let a = let argh`, async () => { - await runAbridgedNodeTest(`let a = let argh`, [ + await ParserTestUtils.runAbridgedNodeTest(`let a = let argh`, [ [Ast.NodeKind.LetExpression, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.ArrayWrapper, 1], @@ -1140,7 +970,7 @@ describe("Parser.AbridgedNode", () => { describe(`${Ast.NodeKind.ListExpression}`, () => { it(`{}`, async () => { - await runAbridgedNodeTest(`{}`, [ + await ParserTestUtils.runAbridgedNodeTest(`{}`, [ [Ast.NodeKind.ListExpression, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.ArrayWrapper, 1], @@ -1149,7 +979,7 @@ describe("Parser.AbridgedNode", () => { }); it(`{1, 2}`, async () => { - await runAbridgedNodeTest(`{1, 2}`, [ + await ParserTestUtils.runAbridgedNodeTest(`{1, 2}`, [ [Ast.NodeKind.ListExpression, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.ArrayWrapper, 1], @@ -1163,7 +993,7 @@ describe("Parser.AbridgedNode", () => { }); it(`{1..2}`, async () => { - await runAbridgedNodeTest(`{1..2}`, [ + await ParserTestUtils.runAbridgedNodeTest(`{1..2}`, [ [Ast.NodeKind.ListExpression, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.ArrayWrapper, 1], @@ -1177,7 +1007,7 @@ describe("Parser.AbridgedNode", () => { }); it(`{1..2, 3..4}`, async () => { - await runAbridgedNodeTest(`{1..2, 3..4}`, [ + await ParserTestUtils.runAbridgedNodeTest(`{1..2, 3..4}`, [ [Ast.NodeKind.ListExpression, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.ArrayWrapper, 1], @@ -1197,7 +1027,7 @@ describe("Parser.AbridgedNode", () => { }); it(`{1, 2..3}`, async () => { - await runAbridgedNodeTest(`{1, 2..3}`, [ + await ParserTestUtils.runAbridgedNodeTest(`{1, 2..3}`, [ [Ast.NodeKind.ListExpression, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.ArrayWrapper, 1], @@ -1214,7 +1044,7 @@ describe("Parser.AbridgedNode", () => { }); it(`{1..2, 3}`, async () => { - await runAbridgedNodeTest(`{1..2, 3}`, [ + await ParserTestUtils.runAbridgedNodeTest(`{1..2, 3}`, [ [Ast.NodeKind.ListExpression, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.ArrayWrapper, 1], @@ -1231,7 +1061,7 @@ describe("Parser.AbridgedNode", () => { }); it(`let x = 1, y = {x..2} in y`, async () => { - await runAbridgedNodeTest(`let x = 1, y = {x..2} in y`, [ + await ParserTestUtils.runAbridgedNodeTest(`let x = 1, y = {x..2} in y`, [ [Ast.NodeKind.LetExpression, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.ArrayWrapper, 1], @@ -1264,7 +1094,7 @@ describe("Parser.AbridgedNode", () => { describe(`${Ast.NodeKind.ListLiteral}`, () => { it(`[foo = {1}] section;`, async () => { - await runAbridgedNodeTest(`[foo = {1}] section;`, [ + await ParserTestUtils.runAbridgedNodeTest(`[foo = {1}] section;`, [ [Ast.NodeKind.Section, undefined], [Ast.NodeKind.RecordLiteral, 0], [Ast.NodeKind.Constant, 0], @@ -1287,7 +1117,7 @@ describe("Parser.AbridgedNode", () => { }); it(`[foo = {}] section;`, async () => { - await runAbridgedNodeTest(`[foo = {}] section;`, [ + await ParserTestUtils.runAbridgedNodeTest(`[foo = {}] section;`, [ [Ast.NodeKind.Section, undefined], [Ast.NodeKind.RecordLiteral, 0], [Ast.NodeKind.Constant, 0], @@ -1309,7 +1139,7 @@ describe("Parser.AbridgedNode", () => { }); it(`${Ast.NodeKind.ListType}`, async () => { - await runAbridgedNodeTest(`type {number}`, [ + await ParserTestUtils.runAbridgedNodeTest(`type {number}`, [ [Ast.NodeKind.TypePrimaryType, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.ListType, 1], @@ -1321,69 +1151,69 @@ describe("Parser.AbridgedNode", () => { describe(`${Ast.NodeKind.LiteralExpression}`, () => { it(`true`, async () => { - await runAbridgedNodeTest(`true`, [[Ast.NodeKind.LiteralExpression, undefined]]); + await ParserTestUtils.runAbridgedNodeTest(`true`, [[Ast.NodeKind.LiteralExpression, undefined]]); }); it(`false`, async () => { - await runAbridgedNodeTest(`false`, [[Ast.NodeKind.LiteralExpression, undefined]]); + await ParserTestUtils.runAbridgedNodeTest(`false`, [[Ast.NodeKind.LiteralExpression, undefined]]); }); it(`1`, async () => { - await runAbridgedNodeTest(`1`, [[Ast.NodeKind.LiteralExpression, undefined]]); + await ParserTestUtils.runAbridgedNodeTest(`1`, [[Ast.NodeKind.LiteralExpression, undefined]]); }); it(`0x1`, async () => { - await runAbridgedNodeTest(`0x1`, [[Ast.NodeKind.LiteralExpression, undefined]]); + await ParserTestUtils.runAbridgedNodeTest(`0x1`, [[Ast.NodeKind.LiteralExpression, undefined]]); }); it(`0X1`, async () => { - await runAbridgedNodeTest(`0X1`, [[Ast.NodeKind.LiteralExpression, undefined]]); + await ParserTestUtils.runAbridgedNodeTest(`0X1`, [[Ast.NodeKind.LiteralExpression, undefined]]); }); it(`1.2`, async () => { - await runAbridgedNodeTest(`1.2`, [[Ast.NodeKind.LiteralExpression, undefined]]); + await ParserTestUtils.runAbridgedNodeTest(`1.2`, [[Ast.NodeKind.LiteralExpression, undefined]]); }); it(`.1`, async () => { - await runAbridgedNodeTest(".1", [[Ast.NodeKind.LiteralExpression, undefined]]); + await ParserTestUtils.runAbridgedNodeTest(".1", [[Ast.NodeKind.LiteralExpression, undefined]]); }); it(`1e2`, async () => { - await runAbridgedNodeTest("1e2", [[Ast.NodeKind.LiteralExpression, undefined]]); + await ParserTestUtils.runAbridgedNodeTest("1e2", [[Ast.NodeKind.LiteralExpression, undefined]]); }); it(`1e+2`, async () => { - await runAbridgedNodeTest("1e+2", [[Ast.NodeKind.LiteralExpression, undefined]]); + await ParserTestUtils.runAbridgedNodeTest("1e+2", [[Ast.NodeKind.LiteralExpression, undefined]]); }); it(`1e-2`, async () => { - await runAbridgedNodeTest("1e-2", [[Ast.NodeKind.LiteralExpression, undefined]]); + await ParserTestUtils.runAbridgedNodeTest("1e-2", [[Ast.NodeKind.LiteralExpression, undefined]]); }); it(`#nan`, async () => { - await runAbridgedNodeTest(`#nan`, [[Ast.NodeKind.LiteralExpression, undefined]]); + await ParserTestUtils.runAbridgedNodeTest(`#nan`, [[Ast.NodeKind.LiteralExpression, undefined]]); }); it(`#infinity`, async () => { - await runAbridgedNodeTest(`#infinity`, [[Ast.NodeKind.LiteralExpression, undefined]]); + await ParserTestUtils.runAbridgedNodeTest(`#infinity`, [[Ast.NodeKind.LiteralExpression, undefined]]); }); it(`""`, async () => { - await runAbridgedNodeTest(`""`, [[Ast.NodeKind.LiteralExpression, undefined]]); + await ParserTestUtils.runAbridgedNodeTest(`""`, [[Ast.NodeKind.LiteralExpression, undefined]]); }); it(`""""`, async () => { - await runAbridgedNodeTest(`""""`, [[Ast.NodeKind.LiteralExpression, undefined]]); + await ParserTestUtils.runAbridgedNodeTest(`""""`, [[Ast.NodeKind.LiteralExpression, undefined]]); }); it(`null`, async () => { - await runAbridgedNodeTest(`null`, [[Ast.NodeKind.LiteralExpression, undefined]]); + await ParserTestUtils.runAbridgedNodeTest(`null`, [[Ast.NodeKind.LiteralExpression, undefined]]); }); }); describe(`${Ast.NodeKind.LogicalExpression}`, () => { it(`true and true`, async () => { - await runAbridgedNodeTest(`true and true`, [ + await ParserTestUtils.runAbridgedNodeTest(`true and true`, [ [Ast.NodeKind.LogicalExpression, undefined], [Ast.NodeKind.LiteralExpression, 0], [Ast.NodeKind.Constant, 1], @@ -1392,7 +1222,7 @@ describe("Parser.AbridgedNode", () => { }); it(`true or true`, async () => { - await runAbridgedNodeTest(`true or true`, [ + await ParserTestUtils.runAbridgedNodeTest(`true or true`, [ [Ast.NodeKind.LogicalExpression, undefined], [Ast.NodeKind.LiteralExpression, 0], [Ast.NodeKind.Constant, 1], @@ -1402,7 +1232,7 @@ describe("Parser.AbridgedNode", () => { }); it(`${Ast.NodeKind.MetadataExpression}`, async () => { - await runAbridgedNodeTest(`1 meta 1`, [ + await ParserTestUtils.runAbridgedNodeTest(`1 meta 1`, [ [Ast.NodeKind.MetadataExpression, undefined], [Ast.NodeKind.LiteralExpression, 0], [Ast.NodeKind.Constant, 1], @@ -1411,14 +1241,14 @@ describe("Parser.AbridgedNode", () => { }); it(`${Ast.NodeKind.NotImplementedExpression}`, async () => { - await runAbridgedNodeTest(`...`, [ + await ParserTestUtils.runAbridgedNodeTest(`...`, [ [Ast.NodeKind.NotImplementedExpression, undefined], [Ast.NodeKind.Constant, 0], ]); }); it(`${Ast.NodeKind.NullablePrimitiveType}`, async () => { - await runAbridgedNodeTest(`1 is nullable number`, [ + await ParserTestUtils.runAbridgedNodeTest(`1 is nullable number`, [ [Ast.NodeKind.IsExpression, undefined], [Ast.NodeKind.LiteralExpression, 0], [Ast.NodeKind.Constant, 1], @@ -1429,7 +1259,7 @@ describe("Parser.AbridgedNode", () => { }); it(`${Ast.NodeKind.NullableType}`, async () => { - await runAbridgedNodeTest(`type nullable number`, [ + await ParserTestUtils.runAbridgedNodeTest(`type nullable number`, [ [Ast.NodeKind.TypePrimaryType, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.NullableType, 1], @@ -1440,7 +1270,7 @@ describe("Parser.AbridgedNode", () => { describe(`${Ast.NodeKind.NullCoalescingExpression}`, () => { it(`1 ?? a`, async () => { - await runAbridgedNodeTest(`1 ?? a`, [ + await ParserTestUtils.runAbridgedNodeTest(`1 ?? a`, [ [Ast.NodeKind.NullCoalescingExpression, undefined], [Ast.NodeKind.LiteralExpression, 0], [Ast.NodeKind.Constant, 1], @@ -1450,7 +1280,7 @@ describe("Parser.AbridgedNode", () => { }); it(`1 ?? 2 ?? 3`, async () => { - await runAbridgedNodeTest(`1 ?? 2 ?? 3`, [ + await ParserTestUtils.runAbridgedNodeTest(`1 ?? 2 ?? 3`, [ [Ast.NodeKind.NullCoalescingExpression, undefined], [Ast.NodeKind.LiteralExpression, 0], [Ast.NodeKind.Constant, 1], @@ -1470,7 +1300,7 @@ describe("Parser.AbridgedNode", () => { describe(`${Ast.NodeKind.ParenthesizedExpression}`, () => { it(`(1)`, async () => { - await runAbridgedNodeTest(`(1)`, [ + await ParserTestUtils.runAbridgedNodeTest(`(1)`, [ [Ast.NodeKind.ParenthesizedExpression, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.LiteralExpression, 1], @@ -1479,7 +1309,7 @@ describe("Parser.AbridgedNode", () => { }); it(`(1) + 1`, async () => { - await runAbridgedNodeTest(`(1) + 1`, [ + await ParserTestUtils.runAbridgedNodeTest(`(1) + 1`, [ [Ast.NodeKind.ArithmeticExpression, undefined], [Ast.NodeKind.ParenthesizedExpression, 0], [Ast.NodeKind.Constant, 0], @@ -1491,7 +1321,7 @@ describe("Parser.AbridgedNode", () => { }); it(`(if true then true else false) and true`, async () => { - await runAbridgedNodeTest(`(if true then true else false) and true`, [ + await ParserTestUtils.runAbridgedNodeTest(`(if true then true else false) and true`, [ [Ast.NodeKind.LogicalExpression, undefined], [Ast.NodeKind.ParenthesizedExpression, 0], [Ast.NodeKind.Constant, 0], @@ -1509,7 +1339,7 @@ describe("Parser.AbridgedNode", () => { }); it(`((1)) and true`, async () => { - await runAbridgedNodeTest(`((1)) and true`, [ + await ParserTestUtils.runAbridgedNodeTest(`((1)) and true`, [ [Ast.NodeKind.LogicalExpression, undefined], [Ast.NodeKind.ParenthesizedExpression, 0], [Ast.NodeKind.Constant, 0], @@ -1526,7 +1356,7 @@ describe("Parser.AbridgedNode", () => { describe(`${Ast.NodeKind.PrimitiveType}`, () => { it(`1 as time`, async () => { - await runAbridgedNodeTest(`1 as time`, [ + await ParserTestUtils.runAbridgedNodeTest(`1 as time`, [ [Ast.NodeKind.AsExpression, undefined], [Ast.NodeKind.LiteralExpression, 0], [Ast.NodeKind.Constant, 1], @@ -1537,7 +1367,7 @@ describe("Parser.AbridgedNode", () => { describe(`${Ast.NodeKind.RecordExpression}`, () => { it(`[x=1]`, async () => { - await runAbridgedNodeTest(`[x=1]`, [ + await ParserTestUtils.runAbridgedNodeTest(`[x=1]`, [ [Ast.NodeKind.RecordExpression, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.ArrayWrapper, 1], @@ -1551,7 +1381,7 @@ describe("Parser.AbridgedNode", () => { }); it(`[]`, async () => { - await runAbridgedNodeTest(`[]`, [ + await ParserTestUtils.runAbridgedNodeTest(`[]`, [ [Ast.NodeKind.RecordExpression, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.ArrayWrapper, 1], @@ -1564,7 +1394,7 @@ describe("Parser.AbridgedNode", () => { describe(`${Ast.NodeKind.RecordType}`, () => { it(`type [x]`, async () => { - await runAbridgedNodeTest(`type [x]`, [ + await ParserTestUtils.runAbridgedNodeTest(`type [x]`, [ [Ast.NodeKind.TypePrimaryType, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.RecordType, 1], @@ -1579,7 +1409,7 @@ describe("Parser.AbridgedNode", () => { }); it(`type [x, ...]`, async () => { - await runAbridgedNodeTest(`type [x, ...]`, [ + await ParserTestUtils.runAbridgedNodeTest(`type [x, ...]`, [ [Ast.NodeKind.TypePrimaryType, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.RecordType, 1], @@ -1600,7 +1430,7 @@ describe("Parser.AbridgedNode", () => { describe(`${Ast.NodeKind.RelationalExpression}`, () => { it(`1 > 2`, async () => { - await runAbridgedNodeAndOperatorTest(`1 > 2`, Constant.RelationalOperator.GreaterThan, [ + await ParserTestUtils.runAbridgedNodeAndOperatorTest(`1 > 2`, Constant.RelationalOperator.GreaterThan, [ [Ast.NodeKind.RelationalExpression, undefined], [Ast.NodeKind.LiteralExpression, 0], [Ast.NodeKind.Constant, 1], @@ -1609,16 +1439,20 @@ describe("Parser.AbridgedNode", () => { }); it(`1 >= 2`, async () => { - await runAbridgedNodeAndOperatorTest(`1 >= 2`, Constant.RelationalOperator.GreaterThanEqualTo, [ - [Ast.NodeKind.RelationalExpression, undefined], - [Ast.NodeKind.LiteralExpression, 0], - [Ast.NodeKind.Constant, 1], - [Ast.NodeKind.LiteralExpression, 2], - ]); + await ParserTestUtils.runAbridgedNodeAndOperatorTest( + `1 >= 2`, + Constant.RelationalOperator.GreaterThanEqualTo, + [ + [Ast.NodeKind.RelationalExpression, undefined], + [Ast.NodeKind.LiteralExpression, 0], + [Ast.NodeKind.Constant, 1], + [Ast.NodeKind.LiteralExpression, 2], + ], + ); }); it(`1 < 2`, async () => { - await runAbridgedNodeAndOperatorTest(`1 < 2`, Constant.RelationalOperator.LessThan, [ + await ParserTestUtils.runAbridgedNodeAndOperatorTest(`1 < 2`, Constant.RelationalOperator.LessThan, [ [Ast.NodeKind.RelationalExpression, undefined], [Ast.NodeKind.LiteralExpression, 0], [Ast.NodeKind.Constant, 1], @@ -1627,18 +1461,22 @@ describe("Parser.AbridgedNode", () => { }); it(`1 <= 2`, async () => { - await runAbridgedNodeAndOperatorTest(`1 <= 2`, Constant.RelationalOperator.LessThanEqualTo, [ - [Ast.NodeKind.RelationalExpression, undefined], - [Ast.NodeKind.LiteralExpression, 0], - [Ast.NodeKind.Constant, 1], - [Ast.NodeKind.LiteralExpression, 2], - ]); + await ParserTestUtils.runAbridgedNodeAndOperatorTest( + `1 <= 2`, + Constant.RelationalOperator.LessThanEqualTo, + [ + [Ast.NodeKind.RelationalExpression, undefined], + [Ast.NodeKind.LiteralExpression, 0], + [Ast.NodeKind.Constant, 1], + [Ast.NodeKind.LiteralExpression, 2], + ], + ); }); }); describe(`${Ast.NodeKind.Section}`, () => { it(`section;`, async () => { - await runAbridgedNodeTest(`section;`, [ + await ParserTestUtils.runAbridgedNodeTest(`section;`, [ [Ast.NodeKind.Section, undefined], [Ast.NodeKind.Constant, 1], [Ast.NodeKind.Constant, 3], @@ -1647,7 +1485,7 @@ describe("Parser.AbridgedNode", () => { }); it(`[] section;`, async () => { - await runAbridgedNodeTest(`[] section;`, [ + await ParserTestUtils.runAbridgedNodeTest(`[] section;`, [ [Ast.NodeKind.Section, undefined], [Ast.NodeKind.RecordLiteral, 0], [Ast.NodeKind.Constant, 0], @@ -1660,7 +1498,7 @@ describe("Parser.AbridgedNode", () => { }); it(`section foo;`, async () => { - await runAbridgedNodeTest(`section foo;`, [ + await ParserTestUtils.runAbridgedNodeTest(`section foo;`, [ [Ast.NodeKind.Section, undefined], [Ast.NodeKind.Constant, 1], [Ast.NodeKind.Identifier, 2], @@ -1670,7 +1508,7 @@ describe("Parser.AbridgedNode", () => { }); it(`section; x = 1;`, async () => { - await runAbridgedNodeTest(`section; x = 1;`, [ + await ParserTestUtils.runAbridgedNodeTest(`section; x = 1;`, [ [Ast.NodeKind.Section, undefined], [Ast.NodeKind.Constant, 1], [Ast.NodeKind.Constant, 3], @@ -1685,7 +1523,7 @@ describe("Parser.AbridgedNode", () => { }); it(`section; x = 1; y = 2;`, async () => { - await runAbridgedNodeTest(`section; x = 1; y = 2;`, [ + await ParserTestUtils.runAbridgedNodeTest(`section; x = 1; y = 2;`, [ [Ast.NodeKind.Section, undefined], [Ast.NodeKind.Constant, 1], [Ast.NodeKind.Constant, 3], @@ -1708,7 +1546,7 @@ describe("Parser.AbridgedNode", () => { describe(`${Ast.NodeKind.SectionMember}`, () => { it(`section; x = 1;`, async () => { - await runAbridgedNodeTest(`section; x = 1;`, [ + await ParserTestUtils.runAbridgedNodeTest(`section; x = 1;`, [ [Ast.NodeKind.Section, undefined], [Ast.NodeKind.Constant, 1], [Ast.NodeKind.Constant, 3], @@ -1723,7 +1561,7 @@ describe("Parser.AbridgedNode", () => { }); it(`section; [] x = 1;`, async () => { - await runAbridgedNodeTest(`section; [] x = 1;`, [ + await ParserTestUtils.runAbridgedNodeTest(`section; [] x = 1;`, [ [Ast.NodeKind.Section, undefined], [Ast.NodeKind.Constant, 1], [Ast.NodeKind.Constant, 3], @@ -1742,7 +1580,7 @@ describe("Parser.AbridgedNode", () => { }); it(`section; shared x = 1;`, async () => { - await runAbridgedNodeTest(`section; shared x = 1;`, [ + await ParserTestUtils.runAbridgedNodeTest(`section; shared x = 1;`, [ [Ast.NodeKind.Section, undefined], [Ast.NodeKind.Constant, 1], [Ast.NodeKind.Constant, 3], @@ -1760,7 +1598,7 @@ describe("Parser.AbridgedNode", () => { describe(`${Ast.NodeKind.TableType}`, () => { it(`type table [x]`, async () => { - await runAbridgedNodeTest(`type table [x]`, [ + await ParserTestUtils.runAbridgedNodeTest(`type table [x]`, [ [Ast.NodeKind.TypePrimaryType, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.TableType, 1], @@ -1776,7 +1614,7 @@ describe("Parser.AbridgedNode", () => { }); it(`type table (x)`, async () => { - await runAbridgedNodeTest(`type table (x)`, [ + await ParserTestUtils.runAbridgedNodeTest(`type table (x)`, [ [Ast.NodeKind.TypePrimaryType, undefined], [Ast.NodeKind.Constant, 0], [Ast.NodeKind.TableType, 1], @@ -1794,7 +1632,7 @@ describe("Parser.AbridgedNode", () => { describe(`${Ast.NodeKind.UnaryExpression}`, () => { it(`-1`, async () => { - await runAbridgedNodeAndOperatorTest(`-1`, Constant.UnaryOperator.Negative, [ + await ParserTestUtils.runAbridgedNodeAndOperatorTest(`-1`, Constant.UnaryOperator.Negative, [ [Ast.NodeKind.UnaryExpression, undefined], [Ast.NodeKind.ArrayWrapper, 0], [Ast.NodeKind.Constant, 0], @@ -1803,7 +1641,7 @@ describe("Parser.AbridgedNode", () => { }); it(`not 1`, async () => { - await runAbridgedNodeAndOperatorTest(`not 1`, Constant.UnaryOperator.Not, [ + await ParserTestUtils.runAbridgedNodeAndOperatorTest(`not 1`, Constant.UnaryOperator.Not, [ [Ast.NodeKind.UnaryExpression, undefined], [Ast.NodeKind.ArrayWrapper, 0], [Ast.NodeKind.Constant, 0], @@ -1812,7 +1650,7 @@ describe("Parser.AbridgedNode", () => { }); it(`+1`, async () => { - await runAbridgedNodeAndOperatorTest(`+1`, Constant.UnaryOperator.Positive, [ + await ParserTestUtils.runAbridgedNodeAndOperatorTest(`+1`, Constant.UnaryOperator.Positive, [ [Ast.NodeKind.UnaryExpression, undefined], [Ast.NodeKind.ArrayWrapper, 0], [Ast.NodeKind.Constant, 0], diff --git a/src/test/libraryTest/parser/parserTestUtils.ts b/src/test/libraryTest/parser/parserTestUtils.ts new file mode 100644 index 00000000..87fd291a --- /dev/null +++ b/src/test/libraryTest/parser/parserTestUtils.ts @@ -0,0 +1,204 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import "mocha"; +import { expect } from "chai"; + +import { Assert, DefaultLocale, DefaultSettings, ResultUtils, Task, TaskUtils, Traverse } from "../../.."; +import { Ast, Constant } from "../../../powerquery-parser/language"; +import { NodeIdMap, ParseSettings, TXorNode, XorNodeUtils } from "../../../powerquery-parser/parser"; +import { AssertTestUtils } from "../../testUtils"; +import { LexSettings } from "../../../powerquery-parser/lexer"; +import { NoOpTraceManagerInstance } from "../../../powerquery-parser/common/trace"; + +export type AbridgedNode = [Ast.NodeKind, number | undefined]; + +type CollectAbridgeNodeState = Traverse.ITraversalState; + +interface NthNodeOfKindState extends Traverse.ITraversalState { + readonly nodeKind: Ast.NodeKind; + readonly nthRequired: number; + nthCounter: number; +} + +export async function runAbridgedNodeTest( + text: string, + expected: ReadonlyArray, + options?: { + readonly astOnly?: boolean; + readonly settings: LexSettings & ParseSettings; + }, +): Promise { + const triedLexParse: Task.TriedLexParseTask = await TaskUtils.tryLexParse( + options?.settings ?? DefaultSettings, + text, + ); + + let root: TXorNode; + let nodeIdMapCollection: NodeIdMap.Collection; + + if (TaskUtils.isParseStageOk(triedLexParse)) { + root = XorNodeUtils.boxAst(triedLexParse.ast); + nodeIdMapCollection = triedLexParse.nodeIdMapCollection; + } else if (TaskUtils.isParseStageParseError(triedLexParse)) { + root = XorNodeUtils.boxContext(Assert.asDefined(triedLexParse.parseState.contextState.root)); + nodeIdMapCollection = triedLexParse.nodeIdMapCollection; + } else { + throw new Error(`expected isParseStageOk/isParseStageParseError`); + } + + validateNodeIdMapCollection(nodeIdMapCollection, root); + + const actual: ReadonlyArray = await collectAbridgeNodeFromXor( + nodeIdMapCollection, + root, + options?.astOnly ?? false, + ); + + expect(actual).to.deep.equal(expected); + + return triedLexParse; +} + +export async function runAbridgedNodeAndOperatorTest( + text: string, + constant: Constant.TConstant, + expected: ReadonlyArray, +): Promise { + await runAbridgedNodeTest(text, expected); + + const operatorNode: Ast.TConstant = await assertGetNthNodeOfKind(text, Ast.NodeKind.Constant, 1); + + expect(operatorNode.constantKind).to.equal(constant); +} + +export async function assertGetNthNodeOfKind( + text: string, + nodeKind: Ast.NodeKind, + nthRequired: number, +): Promise { + const parseTaskOk: Task.ParseTaskOk = await AssertTestUtils.assertGetLexParseOk(DefaultSettings, text); + + const state: NthNodeOfKindState = { + locale: DefaultLocale, + result: undefined, + nodeKind, + nthCounter: 0, + nthRequired, + cancellationToken: undefined, + initialCorrelationId: undefined, + traceManager: NoOpTraceManagerInstance, + }; + + const triedTraverse: Traverse.TriedTraverse = await Traverse.tryTraverseAst< + NthNodeOfKindState, + Ast.TNode | undefined + >( + state, + parseTaskOk.nodeIdMapCollection, + parseTaskOk.ast, + Traverse.VisitNodeStrategy.BreadthFirst, + nthNodeVisit, + Traverse.assertGetAllAstChildren, + nthNodeEarlyExit, + ); + + ResultUtils.assertIsOk(triedTraverse); + + return Assert.asDefined(triedTraverse.value) as N; +} + +async function collectAbridgeNodeFromXor( + nodeIdMapCollection: NodeIdMap.Collection, + root: TXorNode, + astOnly: boolean, +): Promise> { + const state: CollectAbridgeNodeState = { + locale: DefaultLocale, + result: [], + cancellationToken: undefined, + initialCorrelationId: undefined, + traceManager: NoOpTraceManagerInstance, + }; + + const triedTraverse: Traverse.TriedTraverse = await Traverse.tryTraverseXor< + CollectAbridgeNodeState, + AbridgedNode[] + >( + state, + nodeIdMapCollection, + root, + Traverse.VisitNodeStrategy.BreadthFirst, + (state: CollectAbridgeNodeState, xorNode: TXorNode) => collectAbridgeXorNodeVisit(state, xorNode, astOnly), + Traverse.assertGetAllXorChildren, + undefined, + ); + + ResultUtils.assertIsOk(triedTraverse); + + return triedTraverse.value; +} + +// eslint-disable-next-line require-await +async function collectAbridgeXorNodeVisit( + state: CollectAbridgeNodeState, + xorNode: TXorNode, + astOnly: boolean, +): Promise { + if (astOnly && !XorNodeUtils.isAst(xorNode)) { + return; + } + + state.result.push([xorNode.node.kind, xorNode.node.attributeIndex]); +} + +// eslint-disable-next-line require-await +async function nthNodeVisit(state: NthNodeOfKindState, node: Ast.TNode): Promise { + if (node.kind === state.nodeKind) { + state.nthCounter += 1; + + if (state.nthCounter === state.nthRequired) { + state.result = node; + } + } +} + +// eslint-disable-next-line require-await +async function nthNodeEarlyExit(state: NthNodeOfKindState, _: Ast.TNode): Promise { + return state.nthCounter === state.nthRequired; +} + +function validateNodeIdMapCollection(nodeIdMapCollection: NodeIdMap.Collection, root: TXorNode): void { + const astNodeIds: Set = new Set(nodeIdMapCollection.astNodeById.keys()); + const contextNodeIds: Set = new Set(nodeIdMapCollection.contextNodeById.keys()); + const allNodeIds: Set = new Set([...astNodeIds].concat([...contextNodeIds])); + + expect(nodeIdMapCollection.parentIdById).to.not.have.key(root.node.id.toString()); + + expect(nodeIdMapCollection.parentIdById.size).to.equal( + allNodeIds.size - 1, + "parentIdById should have one less entry than allNodeIds", + ); + + expect(astNodeIds.size + contextNodeIds.size).to.equal( + allNodeIds.size, + "allNodeIds should be a union of astNodeIds and contextNodeIds", + ); + + for (const [childId, parentId] of nodeIdMapCollection.parentIdById.entries()) { + expect(allNodeIds).to.include(childId, "keys for parentIdById should be in allNodeIds"); + expect(allNodeIds).to.include(parentId, "values for parentIdById should be in allNodeIds"); + } + + for (const [parentId, childrenIds] of nodeIdMapCollection.childIdsById.entries()) { + expect(allNodeIds).to.include(parentId, "keys for childIdsById should be in allNodeIds"); + + for (const childId of childrenIds) { + expect(allNodeIds).to.include(childId, "childIds should be in allNodeIds"); + + if (astNodeIds.has(parentId)) { + expect(astNodeIds).to.include(childId, "if a parent is an astNode then so should be its children"); + } + } + } +} From a358669cd60f5990180f7a9b3fbe81b0b39da2f3 Mon Sep 17 00:00:00 2001 From: JordanBoltonMN Date: Mon, 4 Aug 2025 12:45:43 -0500 Subject: [PATCH 02/15] incrementing version --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index a478cfba..690e2888 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@microsoft/powerquery-parser", - "version": "0.16.1", + "version": "0.17.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@microsoft/powerquery-parser", - "version": "0.16.1", + "version": "0.17.0", "license": "MIT", "dependencies": { "grapheme-splitter": "^1.0.4", diff --git a/package.json b/package.json index 112ff476..0933673b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@microsoft/powerquery-parser", - "version": "0.16.1", + "version": "0.17.0", "description": "A parser for the Power Query/M formula language.", "author": "Microsoft", "license": "MIT", From 180da311d0595ddacc8fdbf7b08067fd26784dfd Mon Sep 17 00:00:00 2001 From: JordanBoltonMN Date: Mon, 4 Aug 2025 13:03:01 -0500 Subject: [PATCH 03/15] adjusting name to be explicit --- src/powerquery-parser/parser/parseBehavior.ts | 2 +- .../parser/parseState/parseStateUtils.ts | 2 +- src/powerquery-parser/parser/parser/parserUtils.ts | 2 +- src/powerquery-parser/settings.ts | 2 +- src/test/libraryTest/parser/parseBehavior.test.ts | 8 ++++---- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/powerquery-parser/parser/parseBehavior.ts b/src/powerquery-parser/parser/parseBehavior.ts index a95523b0..aedfb6e2 100644 --- a/src/powerquery-parser/parser/parseBehavior.ts +++ b/src/powerquery-parser/parser/parseBehavior.ts @@ -2,7 +2,7 @@ // Licensed under the MIT license. export enum ParseBehavior { - ParseAll = "ParseAll", + ParseEitherExpressionOrSection = "ParseEitherExpressionOrSection", ParseExpression = "ParseExpression", ParseSection = "ParseSection", } diff --git a/src/powerquery-parser/parser/parseState/parseStateUtils.ts b/src/powerquery-parser/parser/parseState/parseStateUtils.ts index a6321df4..74d60886 100644 --- a/src/powerquery-parser/parser/parseState/parseStateUtils.ts +++ b/src/powerquery-parser/parser/parseState/parseStateUtils.ts @@ -31,7 +31,7 @@ export function newState(lexerSnapshot: LexerSnapshot, overrides?: Partial { switch (parseSettings.parseBehavior) { - case ParseBehavior.ParseAll: + case ParseBehavior.ParseEitherExpressionOrSection: return await tryParseExpressionDocumentOrSectionDocument(parseSettings, lexerSnapshot); case ParseBehavior.ParseExpression: { diff --git a/src/powerquery-parser/settings.ts b/src/powerquery-parser/settings.ts index 289b469d..be7c0bcf 100644 --- a/src/powerquery-parser/settings.ts +++ b/src/powerquery-parser/settings.ts @@ -16,7 +16,7 @@ export const DefaultSettings: Settings = { cancellationToken: undefined, initialCorrelationId: undefined, parserEntryPoint: undefined, - parseBehavior: ParseBehavior.ParseAll, + parseBehavior: ParseBehavior.ParseEitherExpressionOrSection, parser: CombinatorialParserV2, traceManager: NoOpTraceManagerInstance, }; diff --git a/src/test/libraryTest/parser/parseBehavior.test.ts b/src/test/libraryTest/parser/parseBehavior.test.ts index dc995e45..d56ce742 100644 --- a/src/test/libraryTest/parser/parseBehavior.test.ts +++ b/src/test/libraryTest/parser/parseBehavior.test.ts @@ -51,9 +51,9 @@ describe("ParseBehavior", () => { return result; } - it(`1 // with ParseAll`, async () => { + it(`1 // with ParseEitherExpressionOrSection`, async () => { await runParseBehaviorTest({ - parseBehavior: ParseBehavior.ParseAll, + parseBehavior: ParseBehavior.ParseEitherExpressionOrSection, text: `1`, expectedAbridgedNodes: [[NodeKind.LiteralExpression, undefined]], expectedStatus: "ParseStageOk", @@ -78,9 +78,9 @@ describe("ParseBehavior", () => { }); }); - it(`section Foo; shared Bar = 1; // with ParseAll`, async () => { + it(`section Foo; shared Bar = 1; // with ParseEitherExpressionOrSection`, async () => { await runParseBehaviorTest({ - parseBehavior: ParseBehavior.ParseAll, + parseBehavior: ParseBehavior.ParseEitherExpressionOrSection, text: `section Foo; shared Bar = 1;`, expectedAbridgedNodes: [ [NodeKind.Section, undefined], From 1cf1b82c04f4be6d29e25d4ebc2274baa2313c85 Mon Sep 17 00:00:00 2001 From: JordanBoltonMN Date: Mon, 4 Aug 2025 13:21:43 -0500 Subject: [PATCH 04/15] Update src/powerquery-parser/parser/parser/parserUtils.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/powerquery-parser/parser/parser/parserUtils.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/powerquery-parser/parser/parser/parserUtils.ts b/src/powerquery-parser/parser/parser/parserUtils.ts index 3117c4fd..9781d118 100644 --- a/src/powerquery-parser/parser/parser/parserUtils.ts +++ b/src/powerquery-parser/parser/parser/parserUtils.ts @@ -332,9 +332,9 @@ async function tryParseSectionDocument( } } -// Adds `tokensConsumed` to the TriedParse type. -// This is used to determine which parse attempt should be returned -// when both an expression and section document are attempted. +// Internal type: Used specifically for comparing parse attempts when both an expression and section document are attempted. +// Not a general extension of TriedParse; do not use outside this context. +// Adds `tokensConsumed` to the TriedParse type to help determine which parse attempt should be returned. type InternalTriedParse = Result; interface InternalTriedParseError { From dfe30290ec60321dbbe839ad93563f7992681489 Mon Sep 17 00:00:00 2001 From: JordanBoltonMN Date: Mon, 4 Aug 2025 15:06:44 -0500 Subject: [PATCH 05/15] ran 'npm audit fix' --- package-lock.json | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/package-lock.json b/package-lock.json index 690e2888..b6adbe93 100644 --- a/package-lock.json +++ b/package-lock.json @@ -892,10 +892,11 @@ } }, "node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", "dev": true, + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -2330,10 +2331,11 @@ "dev": true }, "node_modules/mocha/node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", "dev": true, + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" } @@ -4167,9 +4169,9 @@ "dev": true }, "brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", "dev": true, "requires": { "balanced-match": "^1.0.0", @@ -5205,9 +5207,9 @@ "dev": true }, "brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", "dev": true, "requires": { "balanced-match": "^1.0.0" From 34daf4ad9a4433b221f868e3ab3a66dfae1f0ecb Mon Sep 17 00:00:00 2001 From: JordanBoltonMN Date: Mon, 4 Aug 2025 15:07:32 -0500 Subject: [PATCH 06/15] fixing lint issue from copilot --- src/powerquery-parser/parser/parser/parserUtils.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/powerquery-parser/parser/parser/parserUtils.ts b/src/powerquery-parser/parser/parser/parserUtils.ts index 9781d118..077970f0 100644 --- a/src/powerquery-parser/parser/parser/parserUtils.ts +++ b/src/powerquery-parser/parser/parser/parserUtils.ts @@ -332,7 +332,8 @@ async function tryParseSectionDocument( } } -// Internal type: Used specifically for comparing parse attempts when both an expression and section document are attempted. +// Note: Internal type: +// Used specifically for comparing parse attempts when both an expression and section document are attempted. // Not a general extension of TriedParse; do not use outside this context. // Adds `tokensConsumed` to the TriedParse type to help determine which parse attempt should be returned. type InternalTriedParse = Result; From e93cd62d6b006e42cbf7992faef5a51f6076c983 Mon Sep 17 00:00:00 2001 From: JordanBoltonMN Date: Mon, 4 Aug 2025 15:07:50 -0500 Subject: [PATCH 07/15] removing ':' --- src/powerquery-parser/parser/parser/parserUtils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/powerquery-parser/parser/parser/parserUtils.ts b/src/powerquery-parser/parser/parser/parserUtils.ts index 077970f0..e7b47a8b 100644 --- a/src/powerquery-parser/parser/parser/parserUtils.ts +++ b/src/powerquery-parser/parser/parser/parserUtils.ts @@ -332,7 +332,7 @@ async function tryParseSectionDocument( } } -// Note: Internal type: +// Note: Internal type // Used specifically for comparing parse attempts when both an expression and section document are attempted. // Not a general extension of TriedParse; do not use outside this context. // Adds `tokensConsumed` to the TriedParse type to help determine which parse attempt should be returned. From 6fdbe2f8ce2a2e5519c60c98aa6246f4f73af765 Mon Sep 17 00:00:00 2001 From: JordanBoltonMN Date: Wed, 6 Aug 2025 10:22:56 -0500 Subject: [PATCH 08/15] about to work on IdentifierUtilsOptions --- .../language/identifierUtils.ts | 169 ++++++++++++++++-- .../parser/nodeIdMap/nodeIdMapIterator.ts | 21 ++- src/test/libraryTest/identifierUtils.test.ts | 4 +- 3 files changed, 177 insertions(+), 17 deletions(-) diff --git a/src/powerquery-parser/language/identifierUtils.ts b/src/powerquery-parser/language/identifierUtils.ts index 92154e01..8072f690 100644 --- a/src/powerquery-parser/language/identifierUtils.ts +++ b/src/powerquery-parser/language/identifierUtils.ts @@ -1,14 +1,15 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -import { Assert, Pattern, StringUtils } from "../common"; +import { Assert, CommonError, Pattern, Result, ResultUtils, StringUtils } from "../common"; export enum IdentifierKind { Generalized = "Generalized", + GeneralizedWithQuotes = "GeneralizedWithQuotes", Invalid = "Invalid", - Quote = "Quote", - QuoteRequired = "QuoteRequired", Regular = "Regular", + RegularWithQuotes = "RegularWithQuotes", + RegularWithRequiredQuotes = "RegularWithRequiredQuotes", } // Assuming the text is a quoted identifier, finds the quotes that enclose the identifier. @@ -21,6 +22,38 @@ export function findQuotedIdentifierQuotes(text: string, index: number): StringU return StringUtils.findQuotes(text, index + 1); } +export function getAllowedIdentifiers(text: string, isGeneralizedIdentifierAllowed: boolean): ReadonlyArray { + const quotedAndUnquoted: TQuotedAndUnquoted | undefined = getQuotedAndUnquoted(text); + + if (quotedAndUnquoted === undefined) { + return []; + } + + switch (quotedAndUnquoted.identifierKind) { + case IdentifierKind.Generalized: + quotedAndUnquoted.withoutQuotes; + + return isGeneralizedIdentifierAllowed + ? [quotedAndUnquoted.withQuotes, quotedAndUnquoted.withoutQuotes] + : []; + + case IdentifierKind.Invalid: + return []; + + case IdentifierKind.RegularWithQuotes: + return [quotedAndUnquoted.withQuotes, quotedAndUnquoted.withoutQuotes]; + + case IdentifierKind.RegularWithRequiredQuotes: + return [quotedAndUnquoted.withQuotes]; + + case IdentifierKind.Regular: + return [quotedAndUnquoted.withoutQuotes, quotedAndUnquoted.withQuotes]; + + default: + throw Assert.isNever(quotedAndUnquoted); + } +} + // Determines what kind of identifier the text is. // It's possible that the text is a partially completed identifier, // which is why we have the `allowTrailingPeriod` parameter. @@ -28,7 +61,10 @@ export function getIdentifierKind(text: string, allowTrailingPeriod: boolean): I if (isRegularIdentifier(text, allowTrailingPeriod)) { return IdentifierKind.Regular; } else if (isQuotedIdentifier(text)) { - return isRegularIdentifier(text.slice(2, -1), false) ? IdentifierKind.Quote : IdentifierKind.QuoteRequired; + if (isRegularIdentifier) + return isRegularIdentifier(text.slice(2, -1), false) + ? IdentifierKind.RegularWithQuotes + : IdentifierKind.RegularWithRequiredQuotes; } else if (isGeneralizedIdentifier(text)) { return IdentifierKind.Generalized; } else { @@ -63,6 +99,32 @@ export function getIdentifierLength(text: string, index: number, allowTrailingPe break; case IdentifierRegexpState.RegularIdentifier: + if (text[index] === ".") { + const nextChr: string | undefined = text[index + 1]; + + // If the last character is a period + if (nextChr === undefined) { + // If we allow trailing period, we can consider it part of the identifier. + if (allowTrailingPeriod) { + index += 1; + } + // Else we are done. + else { + state = IdentifierRegexpState.Done; + } + } + // Else if it's two sequential periods, we are done. + else if (nextChr === ".") { + state = IdentifierRegexpState.Done; + } + // Else if it's a single period followed by a potentially valid identifier character. + else { + index += 1; + } + + break; + } + // Don't consider `..` or `...` part of an identifier. if (allowTrailingPeriod && text[index] === "." && text[index + 1] !== ".") { index += 1; @@ -146,18 +208,105 @@ export function isQuotedIdentifier(text: string): boolean { } // Removes the quotes from a quoted identifier if possible. -export function normalizeIdentifier(text: string): string { - if (isQuotedIdentifier(text)) { - const stripped: string = text.slice(2, -1); +export function getNormalizedIdentifier( + text: string, + isGeneralizedIdentifierAllowed: boolean, +): Result { + const quotedAndUnquoted: TQuotedAndUnquoted = getQuotedAndUnquoted(text); + + if (quotedAndUnquoted.identifierKind === IdentifierKind.Invalid) { + return ResultUtils.error(new CommonError.InvariantError(`The text "${text}" is not a valid identifier.`)); + } - return isRegularIdentifier(stripped, false) ? stripped : text; - } else { - return text; + // Validate a generalized identifier is allowed in this context. + if (quotedAndUnquoted.identifierKind === IdentifierKind.Generalized && !isGeneralizedIdentifierAllowed) { + return ResultUtils.error( + new CommonError.InvariantError( + `The text "${text}" is a generalized identifier, but it is not allowed in this context.`, + ), + ); } + + // Prefer without quotes if it exists. + return ResultUtils.ok(quotedAndUnquoted.withoutQuotes ?? quotedAndUnquoted.withQuotes); } +function getQuotedAndUnquoted(text: string): TQuotedAndUnquoted { + const identifierKind: IdentifierKind = getIdentifierKind(text, /* allowTrailingPeriod */ false); + + switch (identifierKind) { + case IdentifierKind.Generalized: + return { + identifierKind, + withoutQuotes: insertQuotes(text), + withQuotes: text, + }; + + case IdentifierKind.Invalid: + return { + identifierKind, + withoutQuotes: undefined, + withQuotes: undefined, + }; + + case IdentifierKind.RegularWithQuotes: + return { + identifierKind, + withoutQuotes: stripQuotes(text), + withQuotes: text, + }; + + case IdentifierKind.RegularWithRequiredQuotes: + return { + identifierKind, + withoutQuotes: undefined, + withQuotes: text, + }; + + case IdentifierKind.Regular: + return { + identifierKind, + withoutQuotes: text, + withQuotes: insertQuotes(text), + }; + + default: + throw Assert.isNever(identifierKind); + } +} + +interface IQuotedAndUnquoted< + TKind extends IdentifierKind, + TWithQuotes extends string | undefined, + TWithoutQuotes extends string | undefined, +> { + readonly identifierKind: TKind; + readonly withQuotes: TWithQuotes; + readonly withoutQuotes: TWithoutQuotes; +} + +type TQuotedAndUnquoted = + | IQuotedAndUnquoted + | IQuotedAndUnquoted + | IQuotedAndUnquoted + | IQuotedAndUnquoted + | IQuotedAndUnquoted; + const enum IdentifierRegexpState { Done = "Done", RegularIdentifier = "RegularIdentifier", Start = "Start", } + +function insertQuotes(text: string): string { + return `#"${text}"`; +} + +function stripQuotes(text: string): string { + return text.slice(2, -1); +} + +interface IdentifierUtilsOptions { + readonly allowTrailingPeriod?: boolean; + readonly isGeneralizedIdentifierAllowed?: boolean; +} diff --git a/src/powerquery-parser/parser/nodeIdMap/nodeIdMapIterator.ts b/src/powerquery-parser/parser/nodeIdMap/nodeIdMapIterator.ts index 7f4d565b..169d5413 100644 --- a/src/powerquery-parser/parser/nodeIdMap/nodeIdMapIterator.ts +++ b/src/powerquery-parser/parser/nodeIdMap/nodeIdMapIterator.ts @@ -1,9 +1,9 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +import { Assert, ResultUtils } from "../../common"; import { Ast, Constant, IdentifierUtils } from "../../language"; import { NodeIdMap, NodeIdMapUtils, TXorNode, XorNodeKind, XorNodeUtils } from "."; -import { Assert } from "../../common"; import { parameterIdentifier } from "./nodeIdMapUtils"; import { XorNode } from "./xorNode"; @@ -342,7 +342,9 @@ export function iterFieldSpecificationList( keyLiteral, optional, value, - normalizedKeyLiteral: IdentifierUtils.normalizeIdentifier(keyLiteral), + normalizedKeyLiteral: ResultUtils.assertOk( + IdentifierUtils.getNormalizedIdentifier(keyLiteral, /* isGeneralizedIdentifierAllowed */ true), + ), pairKind: PairKind.FieldSpecification, source: fieldSpecification, }); @@ -380,6 +382,7 @@ export function iterLetExpression( nodeIdMapCollection, arrayWrapper, PairKind.LetExpression, + /* isGeneralizedIdentifierAllowed */ false, ); } @@ -410,6 +413,7 @@ export function iterRecord( nodeIdMapCollection, arrayWrapper, PairKind.Record, + /* isGeneralizedIdentifierAllowed */ true, ); } @@ -450,7 +454,9 @@ export function iterSection( source: XorNodeUtils.boxAst(namePairedExpression), key: namePairedExpression.key, keyLiteral, - normalizedKeyLiteral: IdentifierUtils.normalizeIdentifier(keyLiteral), + normalizedKeyLiteral: ResultUtils.assertOk( + IdentifierUtils.getNormalizedIdentifier(keyLiteral, /* isGeneralizedIdentifierAllowed */ true), + ), value: XorNodeUtils.boxAst(namePairedExpression.value), pairKind: PairKind.SectionMember, }; @@ -504,7 +510,9 @@ export function iterSection( source: keyValuePair, key, keyLiteral, - normalizedKeyLiteral: IdentifierUtils.normalizeIdentifier(keyLiteral), + normalizedKeyLiteral: ResultUtils.assertOk( + IdentifierUtils.getNormalizedIdentifier(keyLiteral, /* isGeneralizedIdentifierAllowed */ true), + ), value: NodeIdMapUtils.nthChildXor(nodeIdMapCollection, keyValuePairNodeId, 2), pairKind: PairKind.SectionMember, }); @@ -520,6 +528,7 @@ function iterKeyValuePairs< nodeIdMapCollection: NodeIdMap.Collection, arrayWrapper: TXorNode, pairKind: TKeyValuePair["pairKind"], + isGeneralizedIdentifierAllowed: boolean, ): ReadonlyArray { const partial: KVP[] = []; @@ -539,7 +548,9 @@ function iterKeyValuePairs< source: keyValuePair, key, keyLiteral, - normalizedKeyLiteral: IdentifierUtils.normalizeIdentifier(keyLiteral), + normalizedKeyLiteral: ResultUtils.assertOk( + IdentifierUtils.getNormalizedIdentifier(keyLiteral, isGeneralizedIdentifierAllowed), + ), value: NodeIdMapUtils.nthChildXor(nodeIdMapCollection, keyValuePair.node.id, 2), pairKind, } as KVP); diff --git a/src/test/libraryTest/identifierUtils.test.ts b/src/test/libraryTest/identifierUtils.test.ts index e592b737..dfa853d0 100644 --- a/src/test/libraryTest/identifierUtils.test.ts +++ b/src/test/libraryTest/identifierUtils.test.ts @@ -12,9 +12,9 @@ describe("IdentifierUtils", () => { it(`foo`, () => expect(IdentifierUtils.isRegularIdentifier("foo", false), "should be true").to.be.true); it(`foo`, () => expect(IdentifierUtils.isRegularIdentifier("foo", true), "should be true").to.be.true); it(`foo.`, () => expect(IdentifierUtils.isRegularIdentifier("foo.", true), "should be true").to.be.true); - it(`foo.1`, () => expect(IdentifierUtils.isRegularIdentifier("foo.1", true), "should be true").to.be.true); + it(`foo.1`, () => expect(IdentifierUtils.isRegularIdentifier("foo.1", false), "should be true").to.be.true); - it(`foo.bar123`, () => + it(`WIP foo.bar123`, () => expect(IdentifierUtils.isRegularIdentifier("foo.bar123", true), "should be true").to.be.true); }); From aef00854f766e68aa1064f1237a54d8584a0b848 Mon Sep 17 00:00:00 2001 From: JordanBoltonMN Date: Wed, 6 Aug 2025 16:41:19 -0500 Subject: [PATCH 09/15] perhaps complete --- .../language/identifierUtils.ts | 206 ++++++++++------- .../language/type/typeUtils/isEqualType.ts | 2 +- src/powerquery-parser/lexer/lexer.ts | 5 +- .../parser/nodeIdMap/nodeIdMapIterator.ts | 14 +- .../parser/parsers/naiveParseSteps.ts | 6 +- src/test/libraryTest/identifierUtils.test.ts | 218 +++++++++++++++--- .../language/typeUtils/isEqualType.test.ts | 62 +++++ 7 files changed, 382 insertions(+), 131 deletions(-) create mode 100644 src/test/libraryTest/language/typeUtils/isEqualType.test.ts diff --git a/src/powerquery-parser/language/identifierUtils.ts b/src/powerquery-parser/language/identifierUtils.ts index 8072f690..1e9ef3d8 100644 --- a/src/powerquery-parser/language/identifierUtils.ts +++ b/src/powerquery-parser/language/identifierUtils.ts @@ -12,18 +12,16 @@ export enum IdentifierKind { RegularWithRequiredQuotes = "RegularWithRequiredQuotes", } -// Assuming the text is a quoted identifier, finds the quotes that enclose the identifier. -// Otherwise returns undefined. -export function findQuotedIdentifierQuotes(text: string, index: number): StringUtils.FoundQuotes | undefined { - if (text[index] !== "#") { - return undefined; - } - - return StringUtils.findQuotes(text, index + 1); +export interface IdentifierUtilsOptions { + readonly allowGeneralizedIdentifier?: boolean; + readonly allowTrailingPeriod?: boolean; } -export function getAllowedIdentifiers(text: string, isGeneralizedIdentifierAllowed: boolean): ReadonlyArray { - const quotedAndUnquoted: TQuotedAndUnquoted | undefined = getQuotedAndUnquoted(text); +export function getAllowedIdentifiers(text: string, options?: IdentifierUtilsOptions): ReadonlyArray { + const allowGeneralizedIdentifier: boolean = + options?.allowGeneralizedIdentifier ?? DefaultallowGeneralizedIdentifier; + + const quotedAndUnquoted: TQuotedAndUnquoted | undefined = getQuotedAndUnquoted(text, options); if (quotedAndUnquoted === undefined) { return []; @@ -31,11 +29,8 @@ export function getAllowedIdentifiers(text: string, isGeneralizedIdentifierAllow switch (quotedAndUnquoted.identifierKind) { case IdentifierKind.Generalized: - quotedAndUnquoted.withoutQuotes; - - return isGeneralizedIdentifierAllowed - ? [quotedAndUnquoted.withQuotes, quotedAndUnquoted.withoutQuotes] - : []; + case IdentifierKind.GeneralizedWithQuotes: + return allowGeneralizedIdentifier ? [quotedAndUnquoted.withQuotes, quotedAndUnquoted.withoutQuotes] : []; case IdentifierKind.Invalid: return []; @@ -57,23 +52,37 @@ export function getAllowedIdentifiers(text: string, isGeneralizedIdentifierAllow // Determines what kind of identifier the text is. // It's possible that the text is a partially completed identifier, // which is why we have the `allowTrailingPeriod` parameter. -export function getIdentifierKind(text: string, allowTrailingPeriod: boolean): IdentifierKind { - if (isRegularIdentifier(text, allowTrailingPeriod)) { +export function getIdentifierKind(text: string, options?: IdentifierUtilsOptions): IdentifierKind { + const allowGeneralizedIdentifier: boolean = + options?.allowGeneralizedIdentifier ?? DefaultallowGeneralizedIdentifier; + + if (isRegularIdentifier(text, options)) { return IdentifierKind.Regular; - } else if (isQuotedIdentifier(text)) { - if (isRegularIdentifier) - return isRegularIdentifier(text.slice(2, -1), false) - ? IdentifierKind.RegularWithQuotes - : IdentifierKind.RegularWithRequiredQuotes; - } else if (isGeneralizedIdentifier(text)) { + } else if (allowGeneralizedIdentifier && isGeneralizedIdentifier(text)) { return IdentifierKind.Generalized; + } + // If the identifier is quoted it's either: + // - a regular identifier with quotes, + // - a generalized identifier with quotes, + else if (isQuotedIdentifier(text)) { + const stripped: string = stripQuotes(text); + + if (isRegularIdentifier(stripped, options)) { + return IdentifierKind.RegularWithQuotes; + } else if (isGeneralizedIdentifier(stripped) && allowGeneralizedIdentifier) { + return IdentifierKind.GeneralizedWithQuotes; + } else { + return IdentifierKind.RegularWithRequiredQuotes; + } } else { return IdentifierKind.Invalid; } } -// Assuming the text is an identifier, returns the length of the identifier. -export function getIdentifierLength(text: string, index: number, allowTrailingPeriod: boolean): number | undefined { +// I'd prefer if this was internal, but it's used by the lexer so it's marked as public. +// Returns the length of the identifier starting at the given index. +export function getIdentifierLength(text: string, index: number, options?: IdentifierUtilsOptions): number | undefined { + const allowTrailingPeriod: boolean = options?.allowTrailingPeriod ?? DefaultAllowTrailingPeriod; const startingIndex: number = index; const textLength: number = text.length; @@ -153,8 +162,69 @@ export function getIdentifierLength(text: string, index: number, allowTrailingPe return index !== startingIndex ? index - startingIndex : undefined; } +// Removes the quotes from a quoted identifier if possible. +export function getNormalizedIdentifier( + text: string, + options?: IdentifierUtilsOptions, +): Result { + const allowGeneralizedIdentifier: boolean = + options?.allowGeneralizedIdentifier ?? DefaultallowGeneralizedIdentifier; + + const quotedAndUnquoted: TQuotedAndUnquoted = getQuotedAndUnquoted(text, options); + + if (quotedAndUnquoted.identifierKind === IdentifierKind.Invalid) { + return ResultUtils.error(new CommonError.InvariantError(`The text "${text}" is not a valid identifier.`)); + } + + // Validate a generalized identifier is allowed in this context. + if (quotedAndUnquoted.identifierKind === IdentifierKind.Generalized && !allowGeneralizedIdentifier) { + return ResultUtils.error( + new CommonError.InvariantError( + `The text "${text}" is a generalized identifier, but it is not allowed in this context.`, + ), + ); + } + + // Prefer without quotes if it exists. + return ResultUtils.ok(quotedAndUnquoted.withoutQuotes ?? quotedAndUnquoted.withQuotes); +} + +interface IQuotedAndUnquoted< + TKind extends IdentifierKind, + TWithQuotes extends string | undefined, + TWithoutQuotes extends string | undefined, +> { + readonly identifierKind: TKind; + readonly withQuotes: TWithQuotes; + readonly withoutQuotes: TWithoutQuotes; +} + +type TQuotedAndUnquoted = + | IQuotedAndUnquoted + | IQuotedAndUnquoted + | IQuotedAndUnquoted + | IQuotedAndUnquoted + | IQuotedAndUnquoted + | IQuotedAndUnquoted; + +const enum IdentifierRegexpState { + Done = "Done", + RegularIdentifier = "RegularIdentifier", + Start = "Start", +} + +// Assuming the text is a quoted identifier, finds the quotes that enclose the identifier. +// Otherwise returns undefined. +function findQuotedIdentifierQuotes(text: string, index: number): StringUtils.FoundQuotes | undefined { + if (text[index] !== "#") { + return undefined; + } + + return StringUtils.findQuotes(text, index + 1); +} + // Assuming the text is a generalized identifier, returns the length of the identifier. -export function getGeneralizedIdentifierLength(text: string, index: number): number | undefined { +function getGeneralizedIdentifierLength(text: string, index: number): number | undefined { const startingIndex: number = index; const textLength: number = text.length; @@ -195,50 +265,21 @@ export function getGeneralizedIdentifierLength(text: string, index: number): num return index !== startingIndex ? index - startingIndex : undefined; } -export function isGeneralizedIdentifier(text: string): boolean { - return getGeneralizedIdentifierLength(text, 0) === text.length; -} - -export function isRegularIdentifier(text: string, allowTrailingPeriod: boolean): boolean { - return getIdentifierLength(text, 0, allowTrailingPeriod) === text.length; -} - -export function isQuotedIdentifier(text: string): boolean { - return findQuotedIdentifierQuotes(text, 0) !== undefined; -} - -// Removes the quotes from a quoted identifier if possible. -export function getNormalizedIdentifier( - text: string, - isGeneralizedIdentifierAllowed: boolean, -): Result { - const quotedAndUnquoted: TQuotedAndUnquoted = getQuotedAndUnquoted(text); - - if (quotedAndUnquoted.identifierKind === IdentifierKind.Invalid) { - return ResultUtils.error(new CommonError.InvariantError(`The text "${text}" is not a valid identifier.`)); - } - - // Validate a generalized identifier is allowed in this context. - if (quotedAndUnquoted.identifierKind === IdentifierKind.Generalized && !isGeneralizedIdentifierAllowed) { - return ResultUtils.error( - new CommonError.InvariantError( - `The text "${text}" is a generalized identifier, but it is not allowed in this context.`, - ), - ); - } - - // Prefer without quotes if it exists. - return ResultUtils.ok(quotedAndUnquoted.withoutQuotes ?? quotedAndUnquoted.withQuotes); -} - -function getQuotedAndUnquoted(text: string): TQuotedAndUnquoted { - const identifierKind: IdentifierKind = getIdentifierKind(text, /* allowTrailingPeriod */ false); +function getQuotedAndUnquoted(text: string, options?: IdentifierUtilsOptions): TQuotedAndUnquoted { + const identifierKind: IdentifierKind = getIdentifierKind(text, options); switch (identifierKind) { case IdentifierKind.Generalized: return { identifierKind, - withoutQuotes: insertQuotes(text), + withoutQuotes: text, + withQuotes: insertQuotes(text), + }; + + case IdentifierKind.GeneralizedWithQuotes: + return { + identifierKind, + withoutQuotes: stripQuotes(text), withQuotes: text, }; @@ -275,38 +316,25 @@ function getQuotedAndUnquoted(text: string): TQuotedAndUnquoted { } } -interface IQuotedAndUnquoted< - TKind extends IdentifierKind, - TWithQuotes extends string | undefined, - TWithoutQuotes extends string | undefined, -> { - readonly identifierKind: TKind; - readonly withQuotes: TWithQuotes; - readonly withoutQuotes: TWithoutQuotes; +function insertQuotes(text: string): string { + return `#"${text}"`; } -type TQuotedAndUnquoted = - | IQuotedAndUnquoted - | IQuotedAndUnquoted - | IQuotedAndUnquoted - | IQuotedAndUnquoted - | IQuotedAndUnquoted; +function isGeneralizedIdentifier(text: string): boolean { + return text.length > 0 && getGeneralizedIdentifierLength(text, 0) === text.length; +} -const enum IdentifierRegexpState { - Done = "Done", - RegularIdentifier = "RegularIdentifier", - Start = "Start", +function isRegularIdentifier(text: string, options?: IdentifierUtilsOptions): boolean { + return text.length > 0 && getIdentifierLength(text, 0, options) === text.length; } -function insertQuotes(text: string): string { - return `#"${text}"`; +function isQuotedIdentifier(text: string): boolean { + return findQuotedIdentifierQuotes(text, 0) !== undefined; } function stripQuotes(text: string): string { return text.slice(2, -1); } -interface IdentifierUtilsOptions { - readonly allowTrailingPeriod?: boolean; - readonly isGeneralizedIdentifierAllowed?: boolean; -} +const DefaultAllowTrailingPeriod: boolean = false; +const DefaultallowGeneralizedIdentifier: boolean = false; diff --git a/src/powerquery-parser/language/type/typeUtils/isEqualType.ts b/src/powerquery-parser/language/type/typeUtils/isEqualType.ts index 9b02e9bf..0352db4b 100644 --- a/src/powerquery-parser/language/type/typeUtils/isEqualType.ts +++ b/src/powerquery-parser/language/type/typeUtils/isEqualType.ts @@ -22,7 +22,7 @@ export function isEqualType(left: Type.TPowerQueryType, right: Type.TPowerQueryT } export function isEqualFunctionParameter(left: Type.FunctionParameter, right: Type.FunctionParameter): boolean { - return left.isNullable !== right.isNullable || left.isOptional !== right.isOptional || left.type !== right.type; + return left.isNullable === right.isNullable && left.isOptional === right.isOptional && left.type === right.type; } export function isEqualFunctionSignature( diff --git a/src/powerquery-parser/lexer/lexer.ts b/src/powerquery-parser/lexer/lexer.ts index 494db56d..958e9f92 100644 --- a/src/powerquery-parser/lexer/lexer.ts +++ b/src/powerquery-parser/lexer/lexer.ts @@ -1015,7 +1015,6 @@ function readKeyword(text: string, lineNumber: number, positionStart: number, lo function readKeywordHelper(text: string, currentPosition: number): Token.LineToken | undefined { const identifierPositionStart: number = text[currentPosition] === "#" ? currentPosition + 1 : currentPosition; - const identifierPositionEnd: number | undefined = indexOfIdentifierEnd(text, identifierPositionStart); if (identifierPositionEnd === undefined) { @@ -1137,7 +1136,9 @@ function indexOfRegexEnd(pattern: RegExp, text: string, positionStart: number): } function indexOfIdentifierEnd(text: string, positionStart: number): number | undefined { - const length: number | undefined = IdentifierUtils.getIdentifierLength(text, positionStart, true); + const length: number | undefined = IdentifierUtils.getIdentifierLength(text, positionStart, { + allowTrailingPeriod: true, + }); return length !== undefined ? positionStart + length : undefined; } diff --git a/src/powerquery-parser/parser/nodeIdMap/nodeIdMapIterator.ts b/src/powerquery-parser/parser/nodeIdMap/nodeIdMapIterator.ts index 169d5413..3aac622e 100644 --- a/src/powerquery-parser/parser/nodeIdMap/nodeIdMapIterator.ts +++ b/src/powerquery-parser/parser/nodeIdMap/nodeIdMapIterator.ts @@ -343,7 +343,7 @@ export function iterFieldSpecificationList( optional, value, normalizedKeyLiteral: ResultUtils.assertOk( - IdentifierUtils.getNormalizedIdentifier(keyLiteral, /* isGeneralizedIdentifierAllowed */ true), + IdentifierUtils.getNormalizedIdentifier(keyLiteral, { allowGeneralizedIdentifier: true }), ), pairKind: PairKind.FieldSpecification, source: fieldSpecification, @@ -382,7 +382,7 @@ export function iterLetExpression( nodeIdMapCollection, arrayWrapper, PairKind.LetExpression, - /* isGeneralizedIdentifierAllowed */ false, + { allowGeneralizedIdentifier: false }, ); } @@ -413,7 +413,7 @@ export function iterRecord( nodeIdMapCollection, arrayWrapper, PairKind.Record, - /* isGeneralizedIdentifierAllowed */ true, + { allowGeneralizedIdentifier: true }, ); } @@ -455,7 +455,7 @@ export function iterSection( key: namePairedExpression.key, keyLiteral, normalizedKeyLiteral: ResultUtils.assertOk( - IdentifierUtils.getNormalizedIdentifier(keyLiteral, /* isGeneralizedIdentifierAllowed */ true), + IdentifierUtils.getNormalizedIdentifier(keyLiteral, { allowGeneralizedIdentifier: true }), ), value: XorNodeUtils.boxAst(namePairedExpression.value), pairKind: PairKind.SectionMember, @@ -511,7 +511,7 @@ export function iterSection( key, keyLiteral, normalizedKeyLiteral: ResultUtils.assertOk( - IdentifierUtils.getNormalizedIdentifier(keyLiteral, /* isGeneralizedIdentifierAllowed */ true), + IdentifierUtils.getNormalizedIdentifier(keyLiteral, { allowGeneralizedIdentifier: true }), ), value: NodeIdMapUtils.nthChildXor(nodeIdMapCollection, keyValuePairNodeId, 2), pairKind: PairKind.SectionMember, @@ -528,7 +528,7 @@ function iterKeyValuePairs< nodeIdMapCollection: NodeIdMap.Collection, arrayWrapper: TXorNode, pairKind: TKeyValuePair["pairKind"], - isGeneralizedIdentifierAllowed: boolean, + identifierUtilsOptions: IdentifierUtils.IdentifierUtilsOptions, ): ReadonlyArray { const partial: KVP[] = []; @@ -549,7 +549,7 @@ function iterKeyValuePairs< key, keyLiteral, normalizedKeyLiteral: ResultUtils.assertOk( - IdentifierUtils.getNormalizedIdentifier(keyLiteral, isGeneralizedIdentifierAllowed), + IdentifierUtils.getNormalizedIdentifier(keyLiteral, identifierUtilsOptions), ), value: NodeIdMapUtils.nthChildXor(nodeIdMapCollection, keyValuePair.node.id, 2), pairKind, diff --git a/src/powerquery-parser/parser/parsers/naiveParseSteps.ts b/src/powerquery-parser/parser/parsers/naiveParseSteps.ts index 1017eb54..1ae59704 100644 --- a/src/powerquery-parser/parser/parsers/naiveParseSteps.ts +++ b/src/powerquery-parser/parser/parsers/naiveParseSteps.ts @@ -121,7 +121,11 @@ export async function readGeneralizedIdentifier( const contiguousIdentifierStartIndex: number = tokens[tokenRangeStartIndex].positionStart.codeUnit; const contiguousIdentifierEndIndex: number = tokens[tokenRangeEndIndex - 1].positionEnd.codeUnit; const literal: string = lexerSnapshot.text.slice(contiguousIdentifierStartIndex, contiguousIdentifierEndIndex); - const literalKind: IdentifierUtils.IdentifierKind = IdentifierUtils.getIdentifierKind(literal, true); + + const literalKind: IdentifierUtils.IdentifierKind = IdentifierUtils.getIdentifierKind(literal, { + allowTrailingPeriod: true, + allowGeneralizedIdentifier: true, + }); if (literalKind === IdentifierUtils.IdentifierKind.Invalid) { trace.exit({ diff --git a/src/test/libraryTest/identifierUtils.test.ts b/src/test/libraryTest/identifierUtils.test.ts index dfa853d0..a0d52138 100644 --- a/src/test/libraryTest/identifierUtils.test.ts +++ b/src/test/libraryTest/identifierUtils.test.ts @@ -4,54 +4,210 @@ import "mocha"; import { expect } from "chai"; +import { CommonError, Result, ResultUtils } from "../../powerquery-parser"; +import { IdentifierKind } from "../../powerquery-parser/language/identifierUtils"; import { IdentifierUtils } from "../../powerquery-parser/language"; describe("IdentifierUtils", () => { - describe(`isRegularIdentifier`, () => { - describe(`valid`, () => { - it(`foo`, () => expect(IdentifierUtils.isRegularIdentifier("foo", false), "should be true").to.be.true); - it(`foo`, () => expect(IdentifierUtils.isRegularIdentifier("foo", true), "should be true").to.be.true); - it(`foo.`, () => expect(IdentifierUtils.isRegularIdentifier("foo.", true), "should be true").to.be.true); - it(`foo.1`, () => expect(IdentifierUtils.isRegularIdentifier("foo.1", false), "should be true").to.be.true); + function createIdentifierUtilsOptions( + allowTrailingPeriod?: boolean, + allowGeneralizedIdentifier?: boolean, + ): IdentifierUtils.IdentifierUtilsOptions { + return { + allowTrailingPeriod: allowTrailingPeriod ?? false, + allowGeneralizedIdentifier: allowGeneralizedIdentifier ?? false, + }; + } - it(`WIP foo.bar123`, () => - expect(IdentifierUtils.isRegularIdentifier("foo.bar123", true), "should be true").to.be.true); + describe(`getIdentifierKind`, () => { + function runGetIdentifierKindTest(params: { + readonly text: string; + readonly expected: IdentifierKind; + readonly allowTrailingPeriod?: boolean; + readonly allowGeneralizedIdentifier?: boolean; + }): void { + const text: string = params.text; + + const identifierUtilsOptions: IdentifierUtils.IdentifierUtilsOptions = createIdentifierUtilsOptions( + params.allowTrailingPeriod, + params.allowGeneralizedIdentifier, + ); + + it(`${text} with ${JSON.stringify(identifierUtilsOptions)}`, () => { + const actual: IdentifierKind = IdentifierUtils.getIdentifierKind(text, identifierUtilsOptions); + expect(actual).to.equal(params.expected); + }); + } + + runGetIdentifierKindTest({ + text: "foo", + expected: IdentifierKind.Regular, }); - describe(`invalid`, () => { - it(`foo.`, () => expect(IdentifierUtils.isRegularIdentifier("foo.", false), "should be false").to.be.false); + runGetIdentifierKindTest({ + text: "", + expected: IdentifierKind.Invalid, + }); + + runGetIdentifierKindTest({ + text: "foo.", + expected: IdentifierKind.Regular, + allowTrailingPeriod: true, + }); + + runGetIdentifierKindTest({ + text: "foo.", + expected: IdentifierKind.Invalid, + allowTrailingPeriod: false, + }); + + runGetIdentifierKindTest({ + text: "foo.bar", + expected: IdentifierKind.Regular, }); - }); - describe(`isGeneralizedIdentifier`, () => { - describe(`valid`, () => { - it("a", () => expect(IdentifierUtils.isGeneralizedIdentifier("a"), "should be true").to.be.true); - it("a.1", () => expect(IdentifierUtils.isGeneralizedIdentifier("a.1"), "should be true").to.be.true); - it("a b", () => expect(IdentifierUtils.isGeneralizedIdentifier("a b"), "should be true").to.be.true); + runGetIdentifierKindTest({ + text: "foo.1", + expected: IdentifierKind.Regular, }); - describe(`invalid`, () => { - it("a..1", () => expect(IdentifierUtils.isGeneralizedIdentifier("a..1"), "should be false").to.be.false); + runGetIdentifierKindTest({ + text: "with space", + expected: IdentifierKind.Invalid, + }); + + runGetIdentifierKindTest({ + text: "with space", + expected: IdentifierKind.Generalized, + allowGeneralizedIdentifier: true, + }); + + runGetIdentifierKindTest({ + text: '#"quoteNotNeeded"', + expected: IdentifierKind.RegularWithQuotes, + }); + + runGetIdentifierKindTest({ + text: '#"quote needed"', + expected: IdentifierKind.RegularWithRequiredQuotes, + }); + + runGetIdentifierKindTest({ + text: '#"quoted generalized identifier"', + expected: IdentifierKind.GeneralizedWithQuotes, + allowGeneralizedIdentifier: true, }); }); - describe(`isQuotedIdentifier`, () => { - describe(`valid`, () => { - it(`#"foo"`, () => expect(IdentifierUtils.isQuotedIdentifier(`#"foo"`), "should be true").to.be.true); - it(`#""`, () => expect(IdentifierUtils.isQuotedIdentifier(`#""`), "should be true").to.be.true); - it(`#""""`, () => expect(IdentifierUtils.isQuotedIdentifier(`#""""`), "should be true").to.be.true); + describe(`getNormalizedIdentifier`, () => { + function runGetNormalizedIdentifierTest(params: { + readonly text: string; + readonly expectedSuccess: string | undefined; + readonly allowGeneralizedIdentifier?: boolean; + readonly allowTrailingPeriod?: boolean; + }): void { + const text: string = params.text; + + const identifierUtilsOptions: IdentifierUtils.IdentifierUtilsOptions = createIdentifierUtilsOptions( + params.allowTrailingPeriod, + params.allowGeneralizedIdentifier, + ); - it(`#"a""b""c"`, () => - expect(IdentifierUtils.isQuotedIdentifier(`#"a""b""c"`), "should be true").to.be.true); + const actual: Result = IdentifierUtils.getNormalizedIdentifier( + text, + identifierUtilsOptions, + ); + + if (params.expectedSuccess !== undefined) { + ResultUtils.assertIsOk(actual); + expect(actual.value).to.equal(params.expectedSuccess); + } else { + ResultUtils.assertIsError(actual); + } + } + + it("foo", () => { + runGetNormalizedIdentifierTest({ + text: "foo", + expectedSuccess: "foo", + }); + }); + + it("[empty string]", () => { + runGetNormalizedIdentifierTest({ + text: "", + expectedSuccess: undefined, + }); + }); + + it("foo. // allowTrailingPeriod - true", () => { + runGetNormalizedIdentifierTest({ + text: "foo.", + expectedSuccess: "foo.", + allowTrailingPeriod: true, + }); + }); + + it("foo. // allowTrailingPeriod - false", () => { + runGetNormalizedIdentifierTest({ + text: "foo.", + expectedSuccess: undefined, + allowTrailingPeriod: false, + }); + }); + + it("foo.bar", () => { + runGetNormalizedIdentifierTest({ + text: "foo.bar", + expectedSuccess: "foo.bar", + }); + }); + + it("foo.1", () => { + runGetNormalizedIdentifierTest({ + text: "foo.1", + expectedSuccess: "foo.1", + }); + }); + + it("with space // allowGeneralizedIdentifier - false", () => { + runGetNormalizedIdentifierTest({ + text: "with space", + allowGeneralizedIdentifier: false, + expectedSuccess: undefined, + }); + }); + + it("with space // allowGeneralizedIdentifier - true", () => { + runGetNormalizedIdentifierTest({ + text: "with space", + expectedSuccess: "with space", + allowGeneralizedIdentifier: true, + }); + }); + + it(`#"regularIdentifierWithUnneededQuotes" // allowGeneralizedIdentifier - false`, () => { + runGetNormalizedIdentifierTest({ + text: '#"regularIdentifierWithUnneededQuotes"', + expectedSuccess: "regularIdentifierWithUnneededQuotes", + allowGeneralizedIdentifier: false, + }); + }); - it(`#"""b""c"`, () => expect(IdentifierUtils.isQuotedIdentifier(`#"""b""c"`), "should be true").to.be.true); - it(`#"a""b"""`, () => expect(IdentifierUtils.isQuotedIdentifier(`#"a""b"""`), "should be true").to.be.true); - it(`#"bar.1"`, () => expect(IdentifierUtils.isQuotedIdentifier(`#"foo"`), "should be true").to.be.true); + it(`#"quoted regular identifier" // allowGeneralizedIdentifier - false`, () => { + runGetNormalizedIdentifierTest({ + text: '#"quoted regular identifier"', + expectedSuccess: `#"quoted regular identifier"`, + allowGeneralizedIdentifier: false, + }); }); - describe(`invalid`, () => { - it(`#"`, () => expect(IdentifierUtils.isGeneralizedIdentifier(`#"`), "should be false").to.be.false); - it(`""`, () => expect(IdentifierUtils.isGeneralizedIdentifier(`""`), "should be false").to.be.false); + it(`#"quoted generalized identifier" // allowGeneralizedIdentifier - true`, () => { + runGetNormalizedIdentifierTest({ + text: '#"quoted generalized identifier"', + expectedSuccess: "quoted generalized identifier", + allowGeneralizedIdentifier: true, + }); }); }); }); diff --git a/src/test/libraryTest/language/typeUtils/isEqualType.test.ts b/src/test/libraryTest/language/typeUtils/isEqualType.test.ts new file mode 100644 index 00000000..01babeb7 --- /dev/null +++ b/src/test/libraryTest/language/typeUtils/isEqualType.test.ts @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import "mocha"; +import { expect } from "chai"; + +import { Type, TypeUtils } from "../../../../powerquery-parser/language"; +import { NoOpTraceManagerInstance } from "../../../../powerquery-parser/common/trace"; +import { TPowerQueryType } from "../../../../powerquery-parser/language/type/type"; + +describe(`TypeUtils.isEqualType`, () => { + function runTest(params: { + readonly left: TPowerQueryType; + readonly right: TPowerQueryType; + readonly expected: boolean; + }): void { + const actual: boolean = TypeUtils.isEqualType(params.left, params.right); + expect(actual).to.equal(params.expected); + } + + describe(`${Type.TypeKind.Function}`, () => { + it(`null is not compatible`, () => { + const type: TPowerQueryType = TypeUtils.definedFunction( + false, + [ + { + isNullable: false, + isOptional: true, + type: Type.TypeKind.Text, + nameLiteral: `x`, + }, + ], + TypeUtils.anyUnion( + [ + { + isNullable: false, + kind: Type.TypeKind.Number, + literal: `1`, + extendedKind: Type.ExtendedTypeKind.NumberLiteral, + normalizedLiteral: 1, + }, + { + isNullable: false, + kind: Type.TypeKind.Number, + literal: `2`, + extendedKind: Type.ExtendedTypeKind.NumberLiteral, + normalizedLiteral: 2, + }, + ], + NoOpTraceManagerInstance, + undefined, + ), + ); + + runTest({ + left: type, + right: type, + expected: true, + }); + }); + }); +}); From b97eeb39fe5034a7691d35cc58c3f74bcbd19b50 Mon Sep 17 00:00:00 2001 From: JordanBoltonMN Date: Thu, 7 Aug 2025 08:36:38 -0500 Subject: [PATCH 10/15] some touch ups --- .../language/identifierUtils.ts | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/powerquery-parser/language/identifierUtils.ts b/src/powerquery-parser/language/identifierUtils.ts index cd40b0ba..f074c885 100644 --- a/src/powerquery-parser/language/identifierUtils.ts +++ b/src/powerquery-parser/language/identifierUtils.ts @@ -17,6 +17,9 @@ export interface IdentifierUtilsOptions { readonly allowTrailingPeriod?: boolean; } +// Identifiers have multiple forms that can be used interchangeably. +// For example, if you have `[key = 1]`, you can use `key` or `#""key""`. +// The `getAllowedIdentifiers` function returns all the forms of the identifier that are allowed in the current context. export function getAllowedIdentifiers(text: string, options?: IdentifierUtilsOptions): ReadonlyArray { const allowGeneralizedIdentifier: boolean = options?.allowGeneralizedIdentifier ?? DefaultallowGeneralizedIdentifier; @@ -49,9 +52,15 @@ export function getAllowedIdentifiers(text: string, options?: IdentifierUtilsOpt } } -// Determines what kind of identifier the text is. -// It's possible that the text is a partially completed identifier, -// which is why we have the `allowTrailingPeriod` parameter. +// An identifier can have multiple forms: +// - Regular: `foo` +// - Regular with quotes: `#""foo""` +// - Regular with required quotes: `#""foo bar""` +// - Regular with required quotes is used when the identifier has spaces or special characters, +// and when generalized identifiers are not allowed. +// - Generalized: `foo bar` +// - Generalized with quotes: `#""foo bar""` +// - Invalid: `foo..bar` export function getIdentifierKind(text: string, options?: IdentifierUtilsOptions): IdentifierKind { const allowGeneralizedIdentifier: boolean = options?.allowGeneralizedIdentifier ?? DefaultallowGeneralizedIdentifier; @@ -163,6 +172,7 @@ export function getIdentifierLength(text: string, index: number, options?: Ident } // Removes the quotes from a quoted identifier if possible. +// When given an invalid identifier, returns undefined. export function getNormalizedIdentifier(text: string, options?: IdentifierUtilsOptions): string | undefined { const allowGeneralizedIdentifier: boolean = options?.allowGeneralizedIdentifier ?? DefaultallowGeneralizedIdentifier; @@ -206,8 +216,8 @@ const enum IdentifierRegexpState { Start = "Start", } -// Assuming the text is a quoted identifier, finds the quotes that enclose the identifier. -// Otherwise returns undefined. +// Finds the locations of quotes in a quoted identifier. +// Returns undefined if the identifier is not quoted. function findQuotedIdentifierQuotes(text: string, index: number): StringUtils.FoundQuotes | undefined { if (text[index] !== "#") { return undefined; @@ -258,6 +268,7 @@ function getGeneralizedIdentifierLength(text: string, index: number): number | u return index !== startingIndex ? index - startingIndex : undefined; } +// Returns the quoted and unquoted versions of the identifier (if applicable). function getQuotedAndUnquoted(text: string, options?: IdentifierUtilsOptions): TQuotedAndUnquoted { const identifierKind: IdentifierKind = getIdentifierKind(text, options); From b898a0aec0bae3fea0815859f67d288a53aab3f9 Mon Sep 17 00:00:00 2001 From: JordanBoltonMN Date: Thu, 7 Aug 2025 09:39:05 -0500 Subject: [PATCH 11/15] adding 'allowRecursive' to getAllowedIdentifiers --- .../language/identifierUtils.ts | 49 +++-- .../parser/nodeIdMap/nodeIdMapIterator.ts | 2 +- src/test/libraryTest/identifierUtils.test.ts | 175 ++++++++++++++---- 3 files changed, 181 insertions(+), 45 deletions(-) diff --git a/src/powerquery-parser/language/identifierUtils.ts b/src/powerquery-parser/language/identifierUtils.ts index f074c885..498b7fcb 100644 --- a/src/powerquery-parser/language/identifierUtils.ts +++ b/src/powerquery-parser/language/identifierUtils.ts @@ -12,15 +12,19 @@ export enum IdentifierKind { RegularWithRequiredQuotes = "RegularWithRequiredQuotes", } -export interface IdentifierUtilsOptions { +export interface CommonIdentifierUtilsOptions { readonly allowGeneralizedIdentifier?: boolean; readonly allowTrailingPeriod?: boolean; } +export interface GetAllowedIdentifiersOptions extends CommonIdentifierUtilsOptions { + readonly allowRecursive?: boolean; +} + // Identifiers have multiple forms that can be used interchangeably. // For example, if you have `[key = 1]`, you can use `key` or `#""key""`. // The `getAllowedIdentifiers` function returns all the forms of the identifier that are allowed in the current context. -export function getAllowedIdentifiers(text: string, options?: IdentifierUtilsOptions): ReadonlyArray { +export function getAllowedIdentifiers(text: string, options?: GetAllowedIdentifiersOptions): ReadonlyArray { const allowGeneralizedIdentifier: boolean = options?.allowGeneralizedIdentifier ?? DefaultallowGeneralizedIdentifier; @@ -30,26 +34,39 @@ export function getAllowedIdentifiers(text: string, options?: IdentifierUtilsOpt return []; } + let result: string[]; + switch (quotedAndUnquoted.identifierKind) { case IdentifierKind.Generalized: case IdentifierKind.GeneralizedWithQuotes: - return allowGeneralizedIdentifier ? [quotedAndUnquoted.withQuotes, quotedAndUnquoted.withoutQuotes] : []; + result = allowGeneralizedIdentifier ? [quotedAndUnquoted.withQuotes, quotedAndUnquoted.withoutQuotes] : []; + break; case IdentifierKind.Invalid: - return []; + result = []; + break; case IdentifierKind.RegularWithQuotes: - return [quotedAndUnquoted.withQuotes, quotedAndUnquoted.withoutQuotes]; + result = [quotedAndUnquoted.withQuotes, quotedAndUnquoted.withoutQuotes]; + break; case IdentifierKind.RegularWithRequiredQuotes: - return [quotedAndUnquoted.withQuotes]; + result = [quotedAndUnquoted.withQuotes]; + break; case IdentifierKind.Regular: - return [quotedAndUnquoted.withoutQuotes, quotedAndUnquoted.withQuotes]; + result = [quotedAndUnquoted.withoutQuotes, quotedAndUnquoted.withQuotes]; + break; default: throw Assert.isNever(quotedAndUnquoted); } + + if (options?.allowRecursive) { + result = result.concat(result.map((value: string) => prefixInclusiveConstant(value))); + } + + return result; } // An identifier can have multiple forms: @@ -61,7 +78,7 @@ export function getAllowedIdentifiers(text: string, options?: IdentifierUtilsOpt // - Generalized: `foo bar` // - Generalized with quotes: `#""foo bar""` // - Invalid: `foo..bar` -export function getIdentifierKind(text: string, options?: IdentifierUtilsOptions): IdentifierKind { +export function getIdentifierKind(text: string, options?: CommonIdentifierUtilsOptions): IdentifierKind { const allowGeneralizedIdentifier: boolean = options?.allowGeneralizedIdentifier ?? DefaultallowGeneralizedIdentifier; @@ -90,7 +107,11 @@ export function getIdentifierKind(text: string, options?: IdentifierUtilsOptions // I'd prefer if this was internal, but it's used by the lexer so it's marked as public. // Returns the length of the identifier starting at the given index. -export function getIdentifierLength(text: string, index: number, options?: IdentifierUtilsOptions): number | undefined { +export function getIdentifierLength( + text: string, + index: number, + options?: CommonIdentifierUtilsOptions, +): number | undefined { const allowTrailingPeriod: boolean = options?.allowTrailingPeriod ?? DefaultAllowTrailingPeriod; const startingIndex: number = index; const textLength: number = text.length; @@ -173,7 +194,7 @@ export function getIdentifierLength(text: string, index: number, options?: Ident // Removes the quotes from a quoted identifier if possible. // When given an invalid identifier, returns undefined. -export function getNormalizedIdentifier(text: string, options?: IdentifierUtilsOptions): string | undefined { +export function getNormalizedIdentifier(text: string, options?: CommonIdentifierUtilsOptions): string | undefined { const allowGeneralizedIdentifier: boolean = options?.allowGeneralizedIdentifier ?? DefaultallowGeneralizedIdentifier; @@ -269,7 +290,7 @@ function getGeneralizedIdentifierLength(text: string, index: number): number | u } // Returns the quoted and unquoted versions of the identifier (if applicable). -function getQuotedAndUnquoted(text: string, options?: IdentifierUtilsOptions): TQuotedAndUnquoted { +function getQuotedAndUnquoted(text: string, options?: CommonIdentifierUtilsOptions): TQuotedAndUnquoted { const identifierKind: IdentifierKind = getIdentifierKind(text, options); switch (identifierKind) { @@ -324,11 +345,15 @@ function insertQuotes(text: string): string { return `#"${text}"`; } +function prefixInclusiveConstant(text: string): string { + return `@${text}`; +} + function isGeneralizedIdentifier(text: string): boolean { return text.length > 0 && getGeneralizedIdentifierLength(text, 0) === text.length; } -function isRegularIdentifier(text: string, options?: IdentifierUtilsOptions): boolean { +function isRegularIdentifier(text: string, options?: CommonIdentifierUtilsOptions): boolean { return text.length > 0 && getIdentifierLength(text, 0, options) === text.length; } diff --git a/src/powerquery-parser/parser/nodeIdMap/nodeIdMapIterator.ts b/src/powerquery-parser/parser/nodeIdMap/nodeIdMapIterator.ts index 2cc5da71..4d485c37 100644 --- a/src/powerquery-parser/parser/nodeIdMap/nodeIdMapIterator.ts +++ b/src/powerquery-parser/parser/nodeIdMap/nodeIdMapIterator.ts @@ -531,7 +531,7 @@ function iterKeyValuePairs< nodeIdMapCollection: NodeIdMap.Collection, arrayWrapper: TXorNode, pairKind: TKeyValuePair["pairKind"], - identifierUtilsOptions: IdentifierUtils.IdentifierUtilsOptions, + identifierUtilsOptions: IdentifierUtils.CommonIdentifierUtilsOptions, ): ReadonlyArray { const partial: KVP[] = []; diff --git a/src/test/libraryTest/identifierUtils.test.ts b/src/test/libraryTest/identifierUtils.test.ts index 8221710c..c7c24b82 100644 --- a/src/test/libraryTest/identifierUtils.test.ts +++ b/src/test/libraryTest/identifierUtils.test.ts @@ -8,34 +8,148 @@ import { IdentifierKind } from "../../powerquery-parser/language/identifierUtils import { IdentifierUtils } from "../../powerquery-parser/language"; describe("IdentifierUtils", () => { - function createIdentifierUtilsOptions( - allowTrailingPeriod?: boolean, - allowGeneralizedIdentifier?: boolean, - ): IdentifierUtils.IdentifierUtilsOptions { + function createCommonIdentifierUtilsOptions( + overrides?: Partial, + ): IdentifierUtils.CommonIdentifierUtilsOptions { return { - allowTrailingPeriod: allowTrailingPeriod ?? false, - allowGeneralizedIdentifier: allowGeneralizedIdentifier ?? false, + allowTrailingPeriod: false, + allowGeneralizedIdentifier: false, + ...overrides, + }; + } + + function createGetAllowedIdentifiersOptions( + overrides?: Partial, + ): IdentifierUtils.GetAllowedIdentifiersOptions { + return { + allowRecursive: false, + ...overrides, }; } + describe(`getAllowedIdentifiers`, () => { + function getAllowedIdentifiersTest(params: { + readonly text: string; + readonly expected: ReadonlyArray; + readonly options?: Partial; + }): void { + const text: string = params.text; + + const options: IdentifierUtils.GetAllowedIdentifiersOptions = createGetAllowedIdentifiersOptions( + params.options, + ); + + const actual: ReadonlyArray = IdentifierUtils.getAllowedIdentifiers(text, options); + expect(actual).to.have.members(params.expected); + } + + it("foo", () => { + getAllowedIdentifiersTest({ + text: "foo", + expected: ["foo", `#"foo"`], + }); + }); + + it("[empty string]", () => { + getAllowedIdentifiersTest({ + text: "", + expected: [], + }); + }); + + it("foo. // allowTrailingPeriod - true", () => { + getAllowedIdentifiersTest({ + text: "foo.", + options: { allowTrailingPeriod: true }, + expected: ["foo.", `#"foo."`], + }); + }); + + it("foo. // allowTrailingPeriod - false", () => { + getAllowedIdentifiersTest({ + text: "foo.", + options: { allowTrailingPeriod: false }, + expected: [], + }); + }); + + it("foo.bar", () => { + getAllowedIdentifiersTest({ + text: "foo.bar", + expected: ["foo.bar", `#"foo.bar"`], + }); + }); + + it("foo.1", () => { + getAllowedIdentifiersTest({ + text: "foo.1", + expected: ["foo.1", `#"foo.1"`], + }); + }); + + it("with space // allowGeneralizedIdentifier - false", () => { + getAllowedIdentifiersTest({ + text: "with space", + options: { allowGeneralizedIdentifier: false }, + expected: [], + }); + }); + + it("with space // allowGeneralizedIdentifier - true", () => { + getAllowedIdentifiersTest({ + text: "with space", + options: { allowGeneralizedIdentifier: true }, + expected: ["with space", `#"with space"`], + }); + }); + + it(`#"regularIdentifierWithUnneededQuotes" // allowGeneralizedIdentifier - false`, () => { + getAllowedIdentifiersTest({ + text: '#"regularIdentifierWithUnneededQuotes"', + options: { allowGeneralizedIdentifier: false }, + expected: ["regularIdentifierWithUnneededQuotes", `#"regularIdentifierWithUnneededQuotes"`], + }); + }); + + it(`#"quoted regular identifier" // allowGeneralizedIdentifier - false`, () => { + getAllowedIdentifiersTest({ + text: '#"quoted regular identifier"', + options: { allowGeneralizedIdentifier: false }, + expected: [`#"quoted regular identifier"`], + }); + }); + + it(`#"quoted generalized identifier" // allowGeneralizedIdentifier - true`, () => { + getAllowedIdentifiersTest({ + text: '#"quoted generalized identifier"', + options: { allowGeneralizedIdentifier: true }, + expected: ["quoted generalized identifier", `#"quoted generalized identifier"`], + }); + }); + + it("foo // allowRecursive - true", () => { + getAllowedIdentifiersTest({ + text: "foo", + options: { allowRecursive: true }, + expected: ["foo", `#"foo"`, "@foo", `@#"foo"`], + }); + }); + }); + describe(`getIdentifierKind`, () => { function runGetIdentifierKindTest(params: { readonly text: string; readonly expected: IdentifierKind; - readonly allowTrailingPeriod?: boolean; - readonly allowGeneralizedIdentifier?: boolean; + readonly options?: Partial; }): void { const text: string = params.text; - const identifierUtilsOptions: IdentifierUtils.IdentifierUtilsOptions = createIdentifierUtilsOptions( - params.allowTrailingPeriod, - params.allowGeneralizedIdentifier, + const options: IdentifierUtils.CommonIdentifierUtilsOptions = createCommonIdentifierUtilsOptions( + params.options, ); - it(`${text} with ${JSON.stringify(identifierUtilsOptions)}`, () => { - const actual: IdentifierKind = IdentifierUtils.getIdentifierKind(text, identifierUtilsOptions); - expect(actual).to.equal(params.expected); - }); + const actual: IdentifierKind = IdentifierUtils.getIdentifierKind(text, options); + expect(actual).to.equal(params.expected); } runGetIdentifierKindTest({ @@ -50,14 +164,14 @@ describe("IdentifierUtils", () => { runGetIdentifierKindTest({ text: "foo.", + options: { allowTrailingPeriod: true }, expected: IdentifierKind.Regular, - allowTrailingPeriod: true, }); runGetIdentifierKindTest({ text: "foo.", + options: { allowTrailingPeriod: false }, expected: IdentifierKind.Invalid, - allowTrailingPeriod: false, }); runGetIdentifierKindTest({ @@ -77,8 +191,8 @@ describe("IdentifierUtils", () => { runGetIdentifierKindTest({ text: "with space", + options: { allowGeneralizedIdentifier: true }, expected: IdentifierKind.Generalized, - allowGeneralizedIdentifier: true, }); runGetIdentifierKindTest({ @@ -93,8 +207,8 @@ describe("IdentifierUtils", () => { runGetIdentifierKindTest({ text: '#"quoted generalized identifier"', + options: { allowGeneralizedIdentifier: true }, expected: IdentifierKind.GeneralizedWithQuotes, - allowGeneralizedIdentifier: true, }); }); @@ -102,15 +216,12 @@ describe("IdentifierUtils", () => { function runGetNormalizedIdentifierTest(params: { readonly text: string; readonly expectedSuccess: string | undefined; - readonly allowGeneralizedIdentifier?: boolean; - readonly allowTrailingPeriod?: boolean; + readonly options?: Partial; }): void { const text: string = params.text; - const identifierUtilsOptions: IdentifierUtils.IdentifierUtilsOptions = createIdentifierUtilsOptions( - params.allowTrailingPeriod, - params.allowGeneralizedIdentifier, - ); + const identifierUtilsOptions: IdentifierUtils.CommonIdentifierUtilsOptions = + createCommonIdentifierUtilsOptions(params.options); const actual: string | undefined = IdentifierUtils.getNormalizedIdentifier(text, identifierUtilsOptions); @@ -138,16 +249,16 @@ describe("IdentifierUtils", () => { it("foo. // allowTrailingPeriod - true", () => { runGetNormalizedIdentifierTest({ text: "foo.", + options: { allowTrailingPeriod: true }, expectedSuccess: "foo.", - allowTrailingPeriod: true, }); }); it("foo. // allowTrailingPeriod - false", () => { runGetNormalizedIdentifierTest({ text: "foo.", + options: { allowTrailingPeriod: false }, expectedSuccess: undefined, - allowTrailingPeriod: false, }); }); @@ -168,7 +279,7 @@ describe("IdentifierUtils", () => { it("with space // allowGeneralizedIdentifier - false", () => { runGetNormalizedIdentifierTest({ text: "with space", - allowGeneralizedIdentifier: false, + options: { allowGeneralizedIdentifier: false }, expectedSuccess: undefined, }); }); @@ -176,32 +287,32 @@ describe("IdentifierUtils", () => { it("with space // allowGeneralizedIdentifier - true", () => { runGetNormalizedIdentifierTest({ text: "with space", + options: { allowGeneralizedIdentifier: true }, expectedSuccess: "with space", - allowGeneralizedIdentifier: true, }); }); it(`#"regularIdentifierWithUnneededQuotes" // allowGeneralizedIdentifier - false`, () => { runGetNormalizedIdentifierTest({ text: '#"regularIdentifierWithUnneededQuotes"', + options: { allowGeneralizedIdentifier: false }, expectedSuccess: "regularIdentifierWithUnneededQuotes", - allowGeneralizedIdentifier: false, }); }); it(`#"quoted regular identifier" // allowGeneralizedIdentifier - false`, () => { runGetNormalizedIdentifierTest({ text: '#"quoted regular identifier"', + options: { allowGeneralizedIdentifier: false }, expectedSuccess: `#"quoted regular identifier"`, - allowGeneralizedIdentifier: false, }); }); it(`#"quoted generalized identifier" // allowGeneralizedIdentifier - true`, () => { runGetNormalizedIdentifierTest({ text: '#"quoted generalized identifier"', + options: { allowGeneralizedIdentifier: true }, expectedSuccess: "quoted generalized identifier", - allowGeneralizedIdentifier: true, }); }); }); From 2f8c6cf046a0470a50cc1092f0a477bf541c96c3 Mon Sep 17 00:00:00 2001 From: JordanBoltonMN Date: Fri, 8 Aug 2025 17:14:35 -0500 Subject: [PATCH 12/15] couple fixes --- src/powerquery-parser/language/identifierUtils.ts | 8 ++++---- .../language/type/typeUtils/isEqualType.ts | 7 ++++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/powerquery-parser/language/identifierUtils.ts b/src/powerquery-parser/language/identifierUtils.ts index 498b7fcb..b2e74117 100644 --- a/src/powerquery-parser/language/identifierUtils.ts +++ b/src/powerquery-parser/language/identifierUtils.ts @@ -26,7 +26,7 @@ export interface GetAllowedIdentifiersOptions extends CommonIdentifierUtilsOptio // The `getAllowedIdentifiers` function returns all the forms of the identifier that are allowed in the current context. export function getAllowedIdentifiers(text: string, options?: GetAllowedIdentifiersOptions): ReadonlyArray { const allowGeneralizedIdentifier: boolean = - options?.allowGeneralizedIdentifier ?? DefaultallowGeneralizedIdentifier; + options?.allowGeneralizedIdentifier ?? DefaultAllowGeneralizedIdentifier; const quotedAndUnquoted: TQuotedAndUnquoted | undefined = getQuotedAndUnquoted(text, options); @@ -80,7 +80,7 @@ export function getAllowedIdentifiers(text: string, options?: GetAllowedIdentifi // - Invalid: `foo..bar` export function getIdentifierKind(text: string, options?: CommonIdentifierUtilsOptions): IdentifierKind { const allowGeneralizedIdentifier: boolean = - options?.allowGeneralizedIdentifier ?? DefaultallowGeneralizedIdentifier; + options?.allowGeneralizedIdentifier ?? DefaultAllowGeneralizedIdentifier; if (isRegularIdentifier(text, options)) { return IdentifierKind.Regular; @@ -196,7 +196,7 @@ export function getIdentifierLength( // When given an invalid identifier, returns undefined. export function getNormalizedIdentifier(text: string, options?: CommonIdentifierUtilsOptions): string | undefined { const allowGeneralizedIdentifier: boolean = - options?.allowGeneralizedIdentifier ?? DefaultallowGeneralizedIdentifier; + options?.allowGeneralizedIdentifier ?? DefaultAllowGeneralizedIdentifier; const quotedAndUnquoted: TQuotedAndUnquoted = getQuotedAndUnquoted(text, options); @@ -366,4 +366,4 @@ function stripQuotes(text: string): string { } const DefaultAllowTrailingPeriod: boolean = false; -const DefaultallowGeneralizedIdentifier: boolean = false; +const DefaultAllowGeneralizedIdentifier: boolean = false; diff --git a/src/powerquery-parser/language/type/typeUtils/isEqualType.ts b/src/powerquery-parser/language/type/typeUtils/isEqualType.ts index 0352db4b..4a097487 100644 --- a/src/powerquery-parser/language/type/typeUtils/isEqualType.ts +++ b/src/powerquery-parser/language/type/typeUtils/isEqualType.ts @@ -22,7 +22,12 @@ export function isEqualType(left: Type.TPowerQueryType, right: Type.TPowerQueryT } export function isEqualFunctionParameter(left: Type.FunctionParameter, right: Type.FunctionParameter): boolean { - return left.isNullable === right.isNullable && left.isOptional === right.isOptional && left.type === right.type; + return ( + left.nameLiteral === right.nameLiteral && + left.isNullable === right.isNullable && + left.isOptional === right.isOptional && + left.type === right.type + ); } export function isEqualFunctionSignature( From c9c2dcdb3b3aa05c95a0f15c82fb78cd447544f4 Mon Sep 17 00:00:00 2001 From: JordanBoltonMN Date: Mon, 11 Aug 2025 09:56:59 -0500 Subject: [PATCH 13/15] simplifying logic and adding tests --- .../language/identifierUtils.ts | 57 +++----- src/test/libraryTest/identifierUtils.test.ts | 127 +++++++++++++----- 2 files changed, 111 insertions(+), 73 deletions(-) diff --git a/src/powerquery-parser/language/identifierUtils.ts b/src/powerquery-parser/language/identifierUtils.ts index b2e74117..26c78783 100644 --- a/src/powerquery-parser/language/identifierUtils.ts +++ b/src/powerquery-parser/language/identifierUtils.ts @@ -137,52 +137,37 @@ export function getIdentifierLength( break; - case IdentifierRegexpState.RegularIdentifier: - if (text[index] === ".") { + // We should allow a single period as part of the identifier, + // but only if it's not the last character and not followed by another period. + // Allow an exception for when it's the last character and allowTrailingPeriod is true. + case IdentifierRegexpState.RegularIdentifier: { + const currentChr: string | undefined = text[index]; + + if (currentChr === undefined) { + state = IdentifierRegexpState.Done; + } else if (currentChr === ".") { const nextChr: string | undefined = text[index + 1]; - // If the last character is a period - if (nextChr === undefined) { - // If we allow trailing period, we can consider it part of the identifier. - if (allowTrailingPeriod) { - index += 1; - } - // Else we are done. - else { - state = IdentifierRegexpState.Done; - } - } - // Else if it's two sequential periods, we are done. - else if (nextChr === ".") { - state = IdentifierRegexpState.Done; - } - // Else if it's a single period followed by a potentially valid identifier character. - else { + // If we have a single period we might include it as part of the identifier when: + // 1. It's not the last character and not followed by another period + // 2. It's the last character and allowTrailingPeriod is true + if ((nextChr && nextChr !== ".") || (nextChr === undefined && allowTrailingPeriod)) { index += 1; + } else { + state = IdentifierRegexpState.Done; } - - break; - } - - // Don't consider `..` or `...` part of an identifier. - if (allowTrailingPeriod && text[index] === "." && text[index + 1] !== ".") { - index += 1; - } - - matchLength = StringUtils.regexMatchLength(Pattern.IdentifierPartCharacters, text, index); - - if (matchLength === undefined) { - state = IdentifierRegexpState.Done; } else { - index += matchLength; + matchLength = StringUtils.regexMatchLength(Pattern.IdentifierPartCharacters, text, index); - // Don't consider `..` or `...` part of an identifier. - if (allowTrailingPeriod && text[index] === "." && text[index + 1] !== ".") { - index += 1; + if (matchLength === undefined) { + state = IdentifierRegexpState.Done; + } else { + index += matchLength; } } break; + } default: throw Assert.isNever(state); diff --git a/src/test/libraryTest/identifierUtils.test.ts b/src/test/libraryTest/identifierUtils.test.ts index c7c24b82..9d30f000 100644 --- a/src/test/libraryTest/identifierUtils.test.ts +++ b/src/test/libraryTest/identifierUtils.test.ts @@ -152,63 +152,116 @@ describe("IdentifierUtils", () => { expect(actual).to.equal(params.expected); } - runGetIdentifierKindTest({ - text: "foo", - expected: IdentifierKind.Regular, + it("foo", () => { + runGetIdentifierKindTest({ + text: "foo", + expected: IdentifierKind.Regular, + }); + }); + + it("", () => { + runGetIdentifierKindTest({ + text: "", + expected: IdentifierKind.Invalid, + }); + }); + + it("foo. // allowTrailingPeriod - true", () => { + runGetIdentifierKindTest({ + text: "foo.", + options: { allowTrailingPeriod: true }, + expected: IdentifierKind.Regular, + }); + }); + + it("foo. // allowTrailingPeriod - false", () => { + runGetIdentifierKindTest({ + text: "foo.", + options: { allowTrailingPeriod: false }, + expected: IdentifierKind.Invalid, + }); + }); + + it("foo.bar", () => { + runGetIdentifierKindTest({ + text: "foo.bar", + expected: IdentifierKind.Regular, + }); }); - runGetIdentifierKindTest({ - text: "", - expected: IdentifierKind.Invalid, + it("foo..bar", () => { + runGetIdentifierKindTest({ + text: "foo..bar", + expected: IdentifierKind.Invalid, + }); }); - runGetIdentifierKindTest({ - text: "foo.", - options: { allowTrailingPeriod: true }, - expected: IdentifierKind.Regular, + it("foo.bar.baz.green.eggs.and.ham", () => { + runGetIdentifierKindTest({ + text: "foo.bar.baz.green.eggs.and.ham", + expected: IdentifierKind.Regular, + }); }); - runGetIdentifierKindTest({ - text: "foo.", - options: { allowTrailingPeriod: false }, - expected: IdentifierKind.Invalid, + it("foo.bar.baz.green.eggs.and.ham. // allowTrailingPeriod - true", () => { + runGetIdentifierKindTest({ + text: "foo.bar.baz.green.eggs.and.ham", + options: { allowTrailingPeriod: true }, + expected: IdentifierKind.Regular, + }); }); - runGetIdentifierKindTest({ - text: "foo.bar", - expected: IdentifierKind.Regular, + it("foo.bar.baz.green.eggs.and.ham. // allowTrailingPeriod - false", () => { + runGetIdentifierKindTest({ + text: "foo.bar.baz.green.eggs.and.ham.", + options: { allowTrailingPeriod: false }, + expected: IdentifierKind.Invalid, + }); }); - runGetIdentifierKindTest({ - text: "foo.1", - expected: IdentifierKind.Regular, + it("foo.1", () => { + runGetIdentifierKindTest({ + text: "foo.1", + expected: IdentifierKind.Regular, + }); }); - runGetIdentifierKindTest({ - text: "with space", - expected: IdentifierKind.Invalid, + it("with space // allowGeneralizedIdentifier - false", () => { + runGetIdentifierKindTest({ + text: "with space", + options: { allowGeneralizedIdentifier: false }, + expected: IdentifierKind.Invalid, + }); }); - runGetIdentifierKindTest({ - text: "with space", - options: { allowGeneralizedIdentifier: true }, - expected: IdentifierKind.Generalized, + it("with space // allowGeneralizedIdentifier - true", () => { + runGetIdentifierKindTest({ + text: "with space", + options: { allowGeneralizedIdentifier: true }, + expected: IdentifierKind.Generalized, + }); }); - runGetIdentifierKindTest({ - text: '#"quoteNotNeeded"', - expected: IdentifierKind.RegularWithQuotes, + it("with space", () => { + runGetIdentifierKindTest({ + text: '#"quoteNotNeeded"', + expected: IdentifierKind.RegularWithQuotes, + }); }); - runGetIdentifierKindTest({ - text: '#"quote needed"', - expected: IdentifierKind.RegularWithRequiredQuotes, + it(`#"quote needed"`, () => { + runGetIdentifierKindTest({ + text: '#"quote needed"', + expected: IdentifierKind.RegularWithRequiredQuotes, + }); }); - runGetIdentifierKindTest({ - text: '#"quoted generalized identifier"', - options: { allowGeneralizedIdentifier: true }, - expected: IdentifierKind.GeneralizedWithQuotes, + it(`#"quoted generalized identifier"' // allowGeneralizedIdentifier - true`, () => { + runGetIdentifierKindTest({ + text: '#"quoted generalized identifier"', + options: { allowGeneralizedIdentifier: true }, + expected: IdentifierKind.GeneralizedWithQuotes, + }); }); }); From d216361c0fd3911414180f2eff0f7d6ce424a50c Mon Sep 17 00:00:00 2001 From: JordanBoltonMN Date: Mon, 11 Aug 2025 10:08:00 -0500 Subject: [PATCH 14/15] renamed insertQuotes --- src/powerquery-parser/language/identifierUtils.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/powerquery-parser/language/identifierUtils.ts b/src/powerquery-parser/language/identifierUtils.ts index 26c78783..7cf0be57 100644 --- a/src/powerquery-parser/language/identifierUtils.ts +++ b/src/powerquery-parser/language/identifierUtils.ts @@ -283,7 +283,7 @@ function getQuotedAndUnquoted(text: string, options?: CommonIdentifierUtilsOptio return { identifierKind, withoutQuotes: text, - withQuotes: insertQuotes(text), + withQuotes: makeQuoted(text), }; case IdentifierKind.GeneralizedWithQuotes: @@ -318,7 +318,7 @@ function getQuotedAndUnquoted(text: string, options?: CommonIdentifierUtilsOptio return { identifierKind, withoutQuotes: text, - withQuotes: insertQuotes(text), + withQuotes: makeQuoted(text), }; default: @@ -326,7 +326,7 @@ function getQuotedAndUnquoted(text: string, options?: CommonIdentifierUtilsOptio } } -function insertQuotes(text: string): string { +function makeQuoted(text: string): string { return `#"${text}"`; } From 238bda3c26f19e4fabb1b4620dc835809bd31e28 Mon Sep 17 00:00:00 2001 From: JordanBoltonMN Date: Mon, 11 Aug 2025 11:23:14 -0500 Subject: [PATCH 15/15] tweaking TQuotedAndUnquoted type --- .../language/identifierUtils.ts | 69 +++++++++++-------- 1 file changed, 42 insertions(+), 27 deletions(-) diff --git a/src/powerquery-parser/language/identifierUtils.ts b/src/powerquery-parser/language/identifierUtils.ts index 7cf0be57..995983e2 100644 --- a/src/powerquery-parser/language/identifierUtils.ts +++ b/src/powerquery-parser/language/identifierUtils.ts @@ -185,36 +185,54 @@ export function getNormalizedIdentifier(text: string, options?: CommonIdentifier const quotedAndUnquoted: TQuotedAndUnquoted = getQuotedAndUnquoted(text, options); - if (quotedAndUnquoted.identifierKind === IdentifierKind.Invalid) { - return undefined; - } + switch (quotedAndUnquoted.identifierKind) { + case IdentifierKind.Regular: + case IdentifierKind.RegularWithQuotes: + return quotedAndUnquoted.withoutQuotes; - // Validate a generalized identifier is allowed in this context. - if (quotedAndUnquoted.identifierKind === IdentifierKind.Generalized && !allowGeneralizedIdentifier) { - return undefined; - } + case IdentifierKind.GeneralizedWithQuotes: + case IdentifierKind.Generalized: + return allowGeneralizedIdentifier ? quotedAndUnquoted.withoutQuotes : undefined; - // Prefer without quotes if it exists. - return quotedAndUnquoted.withoutQuotes ?? quotedAndUnquoted.withQuotes; -} + case IdentifierKind.Invalid: + return undefined; + + case IdentifierKind.RegularWithRequiredQuotes: + return quotedAndUnquoted.withQuotes; -interface IQuotedAndUnquoted< - TKind extends IdentifierKind, - TWithQuotes extends string | undefined, - TWithoutQuotes extends string | undefined, -> { - readonly identifierKind: TKind; - readonly withQuotes: TWithQuotes; - readonly withoutQuotes: TWithoutQuotes; + default: + throw Assert.isNever(quotedAndUnquoted); + } } type TQuotedAndUnquoted = - | IQuotedAndUnquoted - | IQuotedAndUnquoted - | IQuotedAndUnquoted - | IQuotedAndUnquoted - | IQuotedAndUnquoted - | IQuotedAndUnquoted; + | { + readonly identifierKind: IdentifierKind.Generalized; + readonly withQuotes: string; + readonly withoutQuotes: string; + } + | { + readonly identifierKind: IdentifierKind.GeneralizedWithQuotes; + readonly withQuotes: string; + readonly withoutQuotes: string; + } + | { + readonly identifierKind: IdentifierKind.Invalid; + } + | { + readonly identifierKind: IdentifierKind.Regular; + readonly withQuotes: string; + readonly withoutQuotes: string; + } + | { + readonly identifierKind: IdentifierKind.RegularWithQuotes; + readonly withQuotes: string; + readonly withoutQuotes: string; + } + | { + readonly identifierKind: IdentifierKind.RegularWithRequiredQuotes; + readonly withQuotes: string; + }; const enum IdentifierRegexpState { Done = "Done", @@ -296,8 +314,6 @@ function getQuotedAndUnquoted(text: string, options?: CommonIdentifierUtilsOptio case IdentifierKind.Invalid: return { identifierKind, - withoutQuotes: undefined, - withQuotes: undefined, }; case IdentifierKind.RegularWithQuotes: @@ -310,7 +326,6 @@ function getQuotedAndUnquoted(text: string, options?: CommonIdentifierUtilsOptio case IdentifierKind.RegularWithRequiredQuotes: return { identifierKind, - withoutQuotes: undefined, withQuotes: text, };