perhaps complete

Author: JordanBoltonMN

src/powerquery-parser/language/identifierUtils.ts

@@ -12,30 +12,25 @@ export enum IdentifierKind {
     RegularWithRequiredQuotes = "RegularWithRequiredQuotes",
 }
 
-// Assuming the text is a quoted identifier, finds the quotes that enclose the identifier.
-// Otherwise returns undefined.
-export function findQuotedIdentifierQuotes(text: string, index: number): StringUtils.FoundQuotes | undefined {
-    if (text[index] !== "#") {
-        return undefined;
-    }
-
-    return StringUtils.findQuotes(text, index + 1);
+export interface IdentifierUtilsOptions {
+    readonly allowGeneralizedIdentifier?: boolean;
+    readonly allowTrailingPeriod?: boolean;
 }
 
-export function getAllowedIdentifiers(text: string, isGeneralizedIdentifierAllowed: boolean): ReadonlyArray<string> {
-    const quotedAndUnquoted: TQuotedAndUnquoted | undefined = getQuotedAndUnquoted(text);
+export function getAllowedIdentifiers(text: string, options?: IdentifierUtilsOptions): ReadonlyArray<string> {
+    const allowGeneralizedIdentifier: boolean =
+        options?.allowGeneralizedIdentifier ?? DefaultallowGeneralizedIdentifier;
+
+    const quotedAndUnquoted: TQuotedAndUnquoted | undefined = getQuotedAndUnquoted(text, options);
 
     if (quotedAndUnquoted === undefined) {
         return [];
     }
 
     switch (quotedAndUnquoted.identifierKind) {
         case IdentifierKind.Generalized:
-            quotedAndUnquoted.withoutQuotes;
-
-            return isGeneralizedIdentifierAllowed
-                ? [quotedAndUnquoted.withQuotes, quotedAndUnquoted.withoutQuotes]
-                : [];
+        case IdentifierKind.GeneralizedWithQuotes:
+            return allowGeneralizedIdentifier ? [quotedAndUnquoted.withQuotes, quotedAndUnquoted.withoutQuotes] : [];
 
         case IdentifierKind.Invalid:
             return [];
@@ -57,23 +52,37 @@ export function getAllowedIdentifiers(text: string, isGeneralizedIdentifierAllow
 // Determines what kind of identifier the text is.
 // It's possible that the text is a partially completed identifier,
 // which is why we have the `allowTrailingPeriod` parameter.
-export function getIdentifierKind(text: string, allowTrailingPeriod: boolean): IdentifierKind {
-    if (isRegularIdentifier(text, allowTrailingPeriod)) {
+export function getIdentifierKind(text: string, options?: IdentifierUtilsOptions): IdentifierKind {
+    const allowGeneralizedIdentifier: boolean =
+        options?.allowGeneralizedIdentifier ?? DefaultallowGeneralizedIdentifier;
+
+    if (isRegularIdentifier(text, options)) {
         return IdentifierKind.Regular;
-    } else if (isQuotedIdentifier(text)) {
-        if (isRegularIdentifier)
-            return isRegularIdentifier(text.slice(2, -1), false)
-                ? IdentifierKind.RegularWithQuotes
-                : IdentifierKind.RegularWithRequiredQuotes;
-    } else if (isGeneralizedIdentifier(text)) {
+    } else if (allowGeneralizedIdentifier && isGeneralizedIdentifier(text)) {
         return IdentifierKind.Generalized;
+    }
+    // If the identifier is quoted it's either:
+    // - a regular identifier with quotes,
+    // - a generalized identifier with quotes,
+    else if (isQuotedIdentifier(text)) {
+        const stripped: string = stripQuotes(text);
+
+        if (isRegularIdentifier(stripped, options)) {
+            return IdentifierKind.RegularWithQuotes;
+        } else if (isGeneralizedIdentifier(stripped) && allowGeneralizedIdentifier) {
+            return IdentifierKind.GeneralizedWithQuotes;
+        } else {
+            return IdentifierKind.RegularWithRequiredQuotes;
+        }
     } else {
         return IdentifierKind.Invalid;
     }
 }
 
-// Assuming the text is an identifier, returns the length of the identifier.
-export function getIdentifierLength(text: string, index: number, allowTrailingPeriod: boolean): number | undefined {
+// I'd prefer if this was internal, but it's used by the lexer so it's marked as public.
+// Returns the length of the identifier starting at the given index.
+export function getIdentifierLength(text: string, index: number, options?: IdentifierUtilsOptions): number | undefined {
+    const allowTrailingPeriod: boolean = options?.allowTrailingPeriod ?? DefaultAllowTrailingPeriod;
     const startingIndex: number = index;
     const textLength: number = text.length;
 
@@ -153,8 +162,69 @@ export function getIdentifierLength(text: string, index: number, allowTrailingPe
     return index !== startingIndex ? index - startingIndex : undefined;
 }
 
+// Removes the quotes from a quoted identifier if possible.
+export function getNormalizedIdentifier(
+    text: string,
+    options?: IdentifierUtilsOptions,
+): Result<string, CommonError.InvariantError> {
+    const allowGeneralizedIdentifier: boolean =
+        options?.allowGeneralizedIdentifier ?? DefaultallowGeneralizedIdentifier;
+
+    const quotedAndUnquoted: TQuotedAndUnquoted = getQuotedAndUnquoted(text, options);
+
+    if (quotedAndUnquoted.identifierKind === IdentifierKind.Invalid) {
+        return ResultUtils.error(new CommonError.InvariantError(`The text "${text}" is not a valid identifier.`));
+    }
+
+    // Validate a generalized identifier is allowed in this context.
+    if (quotedAndUnquoted.identifierKind === IdentifierKind.Generalized && !allowGeneralizedIdentifier) {
+        return ResultUtils.error(
+            new CommonError.InvariantError(
+                `The text "${text}" is a generalized identifier, but it is not allowed in this context.`,
+            ),
+        );
+    }
+
+    // Prefer without quotes if it exists.
+    return ResultUtils.ok(quotedAndUnquoted.withoutQuotes ?? quotedAndUnquoted.withQuotes);
+}
+
+interface IQuotedAndUnquoted<
+    TKind extends IdentifierKind,
+    TWithQuotes extends string | undefined,
+    TWithoutQuotes extends string | undefined,
+> {
+    readonly identifierKind: TKind;
+    readonly withQuotes: TWithQuotes;
+    readonly withoutQuotes: TWithoutQuotes;
+}
+
+type TQuotedAndUnquoted =
+    | IQuotedAndUnquoted<IdentifierKind.Generalized, string, string>
+    | IQuotedAndUnquoted<IdentifierKind.GeneralizedWithQuotes, string, string>
+    | IQuotedAndUnquoted<IdentifierKind.Invalid, undefined, undefined>
+    | IQuotedAndUnquoted<IdentifierKind.RegularWithQuotes, string, string>
+    | IQuotedAndUnquoted<IdentifierKind.RegularWithRequiredQuotes, string, undefined>
+    | IQuotedAndUnquoted<IdentifierKind.Regular, string, string>;
+
+const enum IdentifierRegexpState {
+    Done = "Done",
+    RegularIdentifier = "RegularIdentifier",
+    Start = "Start",
+}
+
+// Assuming the text is a quoted identifier, finds the quotes that enclose the identifier.
+// Otherwise returns undefined.
+function findQuotedIdentifierQuotes(text: string, index: number): StringUtils.FoundQuotes | undefined {
+    if (text[index] !== "#") {
+        return undefined;
+    }
+
+    return StringUtils.findQuotes(text, index + 1);
+}
+
 // Assuming the text is a generalized identifier, returns the length of the identifier.
-export function getGeneralizedIdentifierLength(text: string, index: number): number | undefined {
+function getGeneralizedIdentifierLength(text: string, index: number): number | undefined {
     const startingIndex: number = index;
     const textLength: number = text.length;
 
@@ -195,50 +265,21 @@ export function getGeneralizedIdentifierLength(text: string, index: number): num
     return index !== startingIndex ? index - startingIndex : undefined;
 }
 
-export function isGeneralizedIdentifier(text: string): boolean {
-    return getGeneralizedIdentifierLength(text, 0) === text.length;
-}
-
-export function isRegularIdentifier(text: string, allowTrailingPeriod: boolean): boolean {
-    return getIdentifierLength(text, 0, allowTrailingPeriod) === text.length;
-}
-
-export function isQuotedIdentifier(text: string): boolean {
-    return findQuotedIdentifierQuotes(text, 0) !== undefined;
-}
-
-// Removes the quotes from a quoted identifier if possible.
-export function getNormalizedIdentifier(
-    text: string,
-    isGeneralizedIdentifierAllowed: boolean,
-): Result<string, CommonError.InvariantError> {
-    const quotedAndUnquoted: TQuotedAndUnquoted = getQuotedAndUnquoted(text);
-
-    if (quotedAndUnquoted.identifierKind === IdentifierKind.Invalid) {
-        return ResultUtils.error(new CommonError.InvariantError(`The text "${text}" is not a valid identifier.`));
-    }
-
-    // Validate a generalized identifier is allowed in this context.
-    if (quotedAndUnquoted.identifierKind === IdentifierKind.Generalized && !isGeneralizedIdentifierAllowed) {
-        return ResultUtils.error(
-            new CommonError.InvariantError(
-                `The text "${text}" is a generalized identifier, but it is not allowed in this context.`,
-            ),
-        );
-    }
-
-    // Prefer without quotes if it exists.
-    return ResultUtils.ok(quotedAndUnquoted.withoutQuotes ?? quotedAndUnquoted.withQuotes);
-}
-
-function getQuotedAndUnquoted(text: string): TQuotedAndUnquoted {
-    const identifierKind: IdentifierKind = getIdentifierKind(text, /* allowTrailingPeriod */ false);
+function getQuotedAndUnquoted(text: string, options?: IdentifierUtilsOptions): TQuotedAndUnquoted {
+    const identifierKind: IdentifierKind = getIdentifierKind(text, options);
 
     switch (identifierKind) {
         case IdentifierKind.Generalized:
             return {
                 identifierKind,
-                withoutQuotes: insertQuotes(text),
+                withoutQuotes: text,
+                withQuotes: insertQuotes(text),
+            };
+
+        case IdentifierKind.GeneralizedWithQuotes:
+            return {
+                identifierKind,
+                withoutQuotes: stripQuotes(text),
                 withQuotes: text,
             };
 
@@ -275,38 +316,25 @@ function getQuotedAndUnquoted(text: string): TQuotedAndUnquoted {
     }
 }
 
-interface IQuotedAndUnquoted<
-    TKind extends IdentifierKind,
-    TWithQuotes extends string | undefined,
-    TWithoutQuotes extends string | undefined,
-> {
-    readonly identifierKind: TKind;
-    readonly withQuotes: TWithQuotes;
-    readonly withoutQuotes: TWithoutQuotes;
+function insertQuotes(text: string): string {
+    return `#"${text}"`;
 }
 
-type TQuotedAndUnquoted =
-    | IQuotedAndUnquoted<IdentifierKind.Generalized, string, string>
-    | IQuotedAndUnquoted<IdentifierKind.Invalid, undefined, undefined>
-    | IQuotedAndUnquoted<IdentifierKind.RegularWithQuotes, string, string>
-    | IQuotedAndUnquoted<IdentifierKind.RegularWithRequiredQuotes, string, undefined>
-    | IQuotedAndUnquoted<IdentifierKind.Regular, string, string>;
+function isGeneralizedIdentifier(text: string): boolean {
+    return text.length > 0 && getGeneralizedIdentifierLength(text, 0) === text.length;
+}
 
-const enum IdentifierRegexpState {
-    Done = "Done",
-    RegularIdentifier = "RegularIdentifier",
-    Start = "Start",
+function isRegularIdentifier(text: string, options?: IdentifierUtilsOptions): boolean {
+    return text.length > 0 && getIdentifierLength(text, 0, options) === text.length;
 }
 
-function insertQuotes(text: string): string {
-    return `#"${text}"`;
+function isQuotedIdentifier(text: string): boolean {
+    return findQuotedIdentifierQuotes(text, 0) !== undefined;
 }
 
 function stripQuotes(text: string): string {
     return text.slice(2, -1);
 }
 
-interface IdentifierUtilsOptions {
-    readonly allowTrailingPeriod?: boolean;
-    readonly isGeneralizedIdentifierAllowed?: boolean;
-}
+const DefaultAllowTrailingPeriod: boolean = false;
+const DefaultallowGeneralizedIdentifier: boolean = false;

src/powerquery-parser/language/type/typeUtils/isEqualType.ts

@@ -22,7 +22,7 @@ export function isEqualType(left: Type.TPowerQueryType, right: Type.TPowerQueryT
 }
 
 export function isEqualFunctionParameter(left: Type.FunctionParameter, right: Type.FunctionParameter): boolean {
-    return left.isNullable !== right.isNullable || left.isOptional !== right.isOptional || left.type !== right.type;
+    return left.isNullable === right.isNullable && left.isOptional === right.isOptional && left.type === right.type;
 }
 
 export function isEqualFunctionSignature(

src/powerquery-parser/lexer/lexer.ts

@@ -1015,7 +1015,6 @@ function readKeyword(text: string, lineNumber: number, positionStart: number, lo
 
 function readKeywordHelper(text: string, currentPosition: number): Token.LineToken | undefined {
     const identifierPositionStart: number = text[currentPosition] === "#" ? currentPosition + 1 : currentPosition;
-
     const identifierPositionEnd: number | undefined = indexOfIdentifierEnd(text, identifierPositionStart);
 
     if (identifierPositionEnd === undefined) {
@@ -1137,7 +1136,9 @@ function indexOfRegexEnd(pattern: RegExp, text: string, positionStart: number):
 }
 
 function indexOfIdentifierEnd(text: string, positionStart: number): number | undefined {
-    const length: number | undefined = IdentifierUtils.getIdentifierLength(text, positionStart, true);
+    const length: number | undefined = IdentifierUtils.getIdentifierLength(text, positionStart, {
+        allowTrailingPeriod: true,
+    });
 
     return length !== undefined ? positionStart + length : undefined;
 }

src/powerquery-parser/parser/nodeIdMap/nodeIdMapIterator.ts

@@ -343,7 +343,7 @@ export function iterFieldSpecificationList(
             optional,
             value,
             normalizedKeyLiteral: ResultUtils.assertOk(
-                IdentifierUtils.getNormalizedIdentifier(keyLiteral, /* isGeneralizedIdentifierAllowed */ true),
+                IdentifierUtils.getNormalizedIdentifier(keyLiteral, { allowGeneralizedIdentifier: true }),
             ),
             pairKind: PairKind.FieldSpecification,
             source: fieldSpecification,
@@ -382,7 +382,7 @@ export function iterLetExpression(
         nodeIdMapCollection,
         arrayWrapper,
         PairKind.LetExpression,
-        /* isGeneralizedIdentifierAllowed */ false,
+        { allowGeneralizedIdentifier: false },
     );
 }
 
@@ -413,7 +413,7 @@ export function iterRecord(
         nodeIdMapCollection,
         arrayWrapper,
         PairKind.Record,
-        /* isGeneralizedIdentifierAllowed */ true,
+        { allowGeneralizedIdentifier: true },
     );
 }
 
@@ -455,7 +455,7 @@ export function iterSection(
                 key: namePairedExpression.key,
                 keyLiteral,
                 normalizedKeyLiteral: ResultUtils.assertOk(
-                    IdentifierUtils.getNormalizedIdentifier(keyLiteral, /* isGeneralizedIdentifierAllowed */ true),
+                    IdentifierUtils.getNormalizedIdentifier(keyLiteral, { allowGeneralizedIdentifier: true }),
                 ),
                 value: XorNodeUtils.boxAst(namePairedExpression.value),
                 pairKind: PairKind.SectionMember,
@@ -511,7 +511,7 @@ export function iterSection(
             key,
             keyLiteral,
             normalizedKeyLiteral: ResultUtils.assertOk(
-                IdentifierUtils.getNormalizedIdentifier(keyLiteral, /* isGeneralizedIdentifierAllowed */ true),
+                IdentifierUtils.getNormalizedIdentifier(keyLiteral, { allowGeneralizedIdentifier: true }),
             ),
             value: NodeIdMapUtils.nthChildXor(nodeIdMapCollection, keyValuePairNodeId, 2),
             pairKind: PairKind.SectionMember,
@@ -528,7 +528,7 @@ function iterKeyValuePairs<
     nodeIdMapCollection: NodeIdMap.Collection,
     arrayWrapper: TXorNode,
     pairKind: TKeyValuePair["pairKind"],
-    isGeneralizedIdentifierAllowed: boolean,
+    identifierUtilsOptions: IdentifierUtils.IdentifierUtilsOptions,
 ): ReadonlyArray<KVP> {
     const partial: KVP[] = [];
 
@@ -549,7 +549,7 @@ function iterKeyValuePairs<
             key,
             keyLiteral,
             normalizedKeyLiteral: ResultUtils.assertOk(
-                IdentifierUtils.getNormalizedIdentifier(keyLiteral, isGeneralizedIdentifierAllowed),
+                IdentifierUtils.getNormalizedIdentifier(keyLiteral, identifierUtilsOptions),
             ),
             value: NodeIdMapUtils.nthChildXor(nodeIdMapCollection, keyValuePair.node.id, 2),
             pairKind,

src/powerquery-parser/parser/parsers/naiveParseSteps.ts

@@ -121,7 +121,11 @@ export async function readGeneralizedIdentifier(
     const contiguousIdentifierStartIndex: number = tokens[tokenRangeStartIndex].positionStart.codeUnit;
     const contiguousIdentifierEndIndex: number = tokens[tokenRangeEndIndex - 1].positionEnd.codeUnit;
     const literal: string = lexerSnapshot.text.slice(contiguousIdentifierStartIndex, contiguousIdentifierEndIndex);
-    const literalKind: IdentifierUtils.IdentifierKind = IdentifierUtils.getIdentifierKind(literal, true);
+
+    const literalKind: IdentifierUtils.IdentifierKind = IdentifierUtils.getIdentifierKind(literal, {
+        allowTrailingPeriod: true,
+        allowGeneralizedIdentifier: true,
+    });
 
     if (literalKind === IdentifierUtils.IdentifierKind.Invalid) {
         trace.exit({