diff --git a/eslint_src.json b/eslint_src.json index 328febcee..dc1c72151 100644 --- a/eslint_src.json +++ b/eslint_src.json @@ -48,7 +48,6 @@ "no-extend-native": 2, "no-global-assign": 2, "no-extra-bind": 2, - "no-redeclare": 2, "array-bracket-spacing": 2, "brace-style": [ 1, diff --git a/fluent-syntax/src/errors.js b/fluent-syntax/src/errors.js index f721d25a4..b13c301e3 100644 --- a/fluent-syntax/src/errors.js +++ b/fluent-syntax/src/errors.js @@ -62,8 +62,6 @@ function getErrorMessage(code, args) { return "Positional arguments must not follow named arguments"; case "E0022": return "Named arguments must be unique"; - case "E0023": - return "VariantLists are only allowed inside of other VariantLists."; case "E0024": return "Cannot access variants of a message."; case "E0025": { diff --git a/fluent-syntax/src/parser.js b/fluent-syntax/src/parser.js index 1f8651a66..965a45aa7 100644 --- a/fluent-syntax/src/parser.js +++ b/fluent-syntax/src/parser.js @@ -43,7 +43,7 @@ export default class FluentParser { // Poor man's decorators. const methodNames = [ "getComment", "getMessage", "getTerm", "getAttribute", "getIdentifier", - "getVariant", "getNumber", "getValue", "getPattern", "getVariantList", + "getVariant", "getNumber", "getPattern", "getVariantList", "getTextElement", "getPlaceable", "getExpression", "getSelectorExpression", "getCallArg", "getString", "getLiteral" ]; @@ -68,7 +68,9 @@ export default class FluentParser { // they should parse as standalone when they're followed by Junk. // Consequently, we only attach Comments once we know that the Message // or the Term parsed successfully. - if (entry.type === "Comment" && blankLines === 0 && ps.currentChar) { + if (entry.type === "Comment" + && blankLines.length === 0 + && ps.currentChar) { // Stash the comment and decide what to do with it in the next pass. lastComment = entry; continue; @@ -200,7 +202,7 @@ export default class FluentParser { } } - if (ps.isNextLineComment(level, {skip: false})) { + if (ps.isNextLineComment(level)) { content += ps.currentChar; ps.next(); } else { @@ -229,19 +231,14 @@ export default class FluentParser { ps.skipBlankInline(); ps.expectChar("="); - if (ps.isValueStart({skip: true})) { - var pattern = this.getPattern(ps); - } - - if (ps.isNextLineAttributeStart({skip: true})) { - var attrs = this.getAttributes(ps); - } + const value = this.maybeGetValue(ps, {allowVariantList: false}); + const attrs = this.getAttributes(ps); - if (pattern === undefined && attrs === undefined) { + if (value === null && attrs.length === 0) { throw new ParseError("E0005", id.name); } - return new AST.Message(id, pattern, attrs); + return new AST.Message(id, value, attrs); } getTerm(ps) { @@ -251,16 +248,16 @@ export default class FluentParser { ps.skipBlankInline(); ps.expectChar("="); - if (ps.isValueStart({skip: true})) { - var value = this.getValue(ps); - } else { + // XXX Once https://github.com/projectfluent/fluent/pull/220 lands, + // getTerm will be the only place where VariantLists are still legal. Move + // the code from getPatternOrVariantList up to here then, and remove the + // allowVariantList switch. + const value = this.maybeGetValue(ps, {allowVariantList: true}); + if (value === null) { throw new ParseError("E0006", id.name); } - if (ps.isNextLineAttributeStart({skip: true})) { - var attrs = this.getAttributes(ps); - } - + const attrs = this.getAttributes(ps); return new AST.Term(id, value, attrs); } @@ -272,24 +269,22 @@ export default class FluentParser { ps.skipBlankInline(); ps.expectChar("="); - if (ps.isValueStart({skip: true})) { - const value = this.getPattern(ps); - return new AST.Attribute(key, value); + const value = this.maybeGetValue(ps, {allowVariantList: false}); + if (value === null) { + throw new ParseError("E0012"); } - throw new ParseError("E0012"); + return new AST.Attribute(key, value); } getAttributes(ps) { const attrs = []; - - while (true) { + ps.peekBlank(); + while (ps.isAttributeStart()) { + ps.skipToPeek(); const attr = this.getAttribute(ps); attrs.push(attr); - - if (!ps.isNextLineAttributeStart({skip: true})) { - break; - } + ps.peekBlank(); } return attrs; } @@ -321,7 +316,7 @@ export default class FluentParser { return this.getIdentifier(ps); } - getVariant(ps, hasDefault) { + getVariant(ps, {hasDefault, allowVariantList}) { let defaultIndex = false; if (ps.currentChar === "*") { @@ -340,36 +335,39 @@ export default class FluentParser { const key = this.getVariantKey(ps); ps.skipBlank(); - ps.expectChar("]"); - if (ps.isValueStart({skip: true})) { - const value = this.getValue(ps); - return new AST.Variant(key, value, defaultIndex); + // XXX We need to pass allowVariantList all the way down to here because + // nested VariantLists in Terms are legal for now. + const value = this.maybeGetValue(ps, {allowVariantList}); + if (value === null) { + throw new ParseError("E0012"); } - throw new ParseError("E0012"); + return new AST.Variant(key, value, defaultIndex); } - getVariants(ps) { + getVariants(ps, {allowVariantList}) { const variants = []; let hasDefault = false; - while (true) { - const variant = this.getVariant(ps, hasDefault); + ps.skipBlank(); + while (ps.isVariantStart()) { + const variant = this.getVariant(ps, {allowVariantList, hasDefault}); if (variant.default) { hasDefault = true; } variants.push(variant); - - if (!ps.isNextLineVariantStart({skip: false})) { - break; - } + ps.expectLineEnd(); ps.skipBlank(); } + if (variants.length === 0) { + throw new ParseError("E0011"); + } + if (!hasDefault) { throw new ParseError("E0010"); } @@ -411,65 +409,171 @@ export default class FluentParser { return new AST.NumberLiteral(num); } - getValue(ps) { - if (ps.currentChar === "{") { + // maybeGetValue distinguishes between patterns which start on the same line + // as the identifier (a.k.a. inline signleline patterns and inline multiline + // patterns) and patterns which start on a new line (a.k.a. block multiline + // patterns). The distinction is important for the dedentation logic: the + // indent of the first line of a block pattern must be taken into account when + // calculating the maximum common indent. + maybeGetValue(ps, {allowVariantList}) { + ps.peekBlankInline(); + if (ps.isValueStart()) { + ps.skipToPeek(); + return this.getPatternOrVariantList( + ps, {isBlock: false, allowVariantList}); + } + + ps.peekBlankBlock(); + if (ps.isValueContinuation()) { + ps.skipToPeek(); + return this.getPatternOrVariantList( + ps, {isBlock: true, allowVariantList}); + } + + return null; + } + + // Parse a VariantList (if allowed) or a Pattern. + getPatternOrVariantList(ps, {isBlock, allowVariantList}) { + ps.peekBlankInline(); + if (allowVariantList && ps.currentPeek === "{") { + const start = ps.peekOffset; ps.peek(); ps.peekBlankInline(); - if (ps.isNextLineVariantStart({skip: false})) { - return this.getVariantList(ps); + if (ps.currentPeek === EOL) { + ps.peekBlank(); + if (ps.isVariantStart()) { + ps.resetPeek(start); + ps.skipToPeek(); + return this.getVariantList(ps, {allowVariantList}); + } } - - ps.resetPeek(); } - return this.getPattern(ps); + ps.resetPeek(); + const pattern = this.getPattern(ps, {isBlock}); + return pattern; } getVariantList(ps) { ps.expectChar("{"); - ps.skipBlankInline(); - ps.expectLineEnd(); - ps.skipBlank(); - const variants = this.getVariants(ps); - ps.expectLineEnd(); - ps.skipBlank(); + var variants = this.getVariants(ps, {allowVariantList: true}); ps.expectChar("}"); return new AST.VariantList(variants); } - getPattern(ps) { + getPattern(ps, {isBlock}) { const elements = []; + if (isBlock) { + // A block pattern is a pattern which starts on a new line. Store and + // measure the indent of this first line for the dedentation logic. + const blankStart = ps.index; + const firstIndent = ps.skipBlankInline(); + elements.push(this.getIndent(ps, firstIndent, blankStart)); + var commonIndentLength = firstIndent.length; + } else { + var commonIndentLength = Infinity; + } let ch; - while ((ch = ps.currentChar)) { + elements: while ((ch = ps.currentChar)) { + switch (ch) { + case EOL: { + const blankStart = ps.index; + const blankLines = ps.peekBlankBlock(); + if (ps.isValueContinuation()) { + ps.skipToPeek(); + const indent = ps.skipBlankInline(); + commonIndentLength = Math.min(commonIndentLength, indent.length); + elements.push(this.getIndent(ps, blankLines + indent, blankStart)); + continue elements; + } - // The end condition for getPattern's while loop is a newline - // which is not followed by a valid pattern continuation. - if (ch === EOL && !ps.isNextLineValue({skip: false})) { - break; + // The end condition for getPattern's while loop is a newline + // which is not followed by a valid pattern continuation. + ps.resetPeek(); + break elements; + } + case "{": + elements.push(this.getPlaceable(ps)); + continue elements; + case "}": + throw new ParseError("E0027"); + default: + const element = this.getTextElement(ps); + elements.push(element); } + } - if (ch === "{") { - const element = this.getPlaceable(ps); - elements.push(element); - } else if (ch === "}") { - throw new ParseError("E0027"); - } else { - const element = this.getTextElement(ps); - elements.push(element); + const dedented = this.dedent(elements, commonIndentLength); + return new AST.Pattern(dedented); + } + + // Create a token representing an indent. It's not part of the AST and it will + // be trimmed and merged into adjacent TextElements, or turned into a new + // TextElement, if it's surrounded by two Placeables. + getIndent(ps, value, start) { + return { + type: "Indent", + span: {start, end: ps.index}, + value, + }; + } + + // Dedent a list of elements by removing the maximum common indent from the + // beginning of text lines. The common indent is calculated in getPattern. + dedent(elements, commonIndent) { + const trimmed = []; + + for (let element of elements) { + if (element.type === "Placeable") { + trimmed.push(element); + continue; + } + + if (element.type === "Indent") { + // Strip common indent. + element.value = element.value.slice( + 0, element.value.length - commonIndent); + if (element.value.length === 0) { + continue; + } + } + + let prev = trimmed[trimmed.length - 1]; + if (prev && prev.type === "TextElement") { + // Join adjacent TextElements by replacing them with their sum. + const sum = new AST.TextElement(prev.value + element.value); + if (this.withSpans) { + sum.addSpan(prev.span.start, element.span.end); + } + trimmed[trimmed.length - 1] = sum; + continue; } + + if (element.type === "Indent") { + // If the indent hasn't been merged into a preceding TextElement, + // convert it into a new TextElement. + const textElement = new AST.TextElement(element.value); + if (this.withSpans) { + textElement.addSpan(element.span.start, element.span.end); + } + element = textElement; + } + + trimmed.push(element); } - // Trim trailing whitespace. - const lastElement = elements[elements.length - 1]; + // Trim trailing whitespace from the Pattern. + const lastElement = trimmed[trimmed.length - 1]; if (lastElement.type === "TextElement") { lastElement.value = lastElement.value.replace(trailingWSRe, ""); - if (lastElement.value === "") { - elements.pop(); + if (lastElement.value.length === 0) { + trimmed.pop(); } } - return new AST.Pattern(elements); + return trimmed; } getTextElement(ps) { @@ -482,15 +586,7 @@ export default class FluentParser { } if (ch === EOL) { - if (!ps.isNextLineValue({skip: false})) { - return new AST.TextElement(buffer); - } - - ps.next(); - ps.skipBlankInline(); - - buffer += EOL; - continue; + return new AST.TextElement(buffer); } buffer += ch; @@ -530,16 +626,14 @@ export default class FluentParser { getPlaceable(ps) { ps.expectChar("{"); + ps.skipBlank(); const expression = this.getExpression(ps); ps.expectChar("}"); return new AST.Placeable(expression); } getExpression(ps) { - ps.skipBlank(); - const selector = this.getSelectorExpression(ps); - ps.skipBlank(); if (ps.currentChar === "-") { @@ -567,28 +661,14 @@ export default class FluentParser { ps.skipBlankInline(); ps.expectLineEnd(); - ps.skipBlank(); - - const variants = this.getVariants(ps); - ps.skipBlank(); - - if (variants.length === 0) { - throw new ParseError("E0011"); - } - - // VariantLists are only allowed in other VariantLists. - if (variants.some(v => v.value.type === "VariantList")) { - throw new ParseError("E0023"); - } + const variants = this.getVariants(ps, {allowVariantList: false}); return new AST.SelectExpression(selector, variants); } else if (selector.type === "AttributeExpression" && selector.ref.type === "TermReference") { throw new ParseError("E0019"); } - ps.skipBlank(); - return selector; } @@ -596,6 +676,7 @@ export default class FluentParser { if (ps.currentChar === "{") { return this.getPlaceable(ps); } + const literal = this.getLiteral(ps); if (literal.type !== "MessageReference" diff --git a/fluent-syntax/src/stream.js b/fluent-syntax/src/stream.js index c95190aa5..2f474be99 100644 --- a/fluent-syntax/src/stream.js +++ b/fluent-syntax/src/stream.js @@ -67,61 +67,44 @@ export const EOF = undefined; const SPECIAL_LINE_START_CHARS = ["}", ".", "[", "*"]; export class FluentParserStream extends ParserStream { - skipBlankInline() { - while (this.currentChar === " ") { - this.next(); - } - } - peekBlankInline() { + const start = this.index + this.peekOffset; while (this.currentPeek === " ") { this.peek(); } + return this.string.slice(start, this.index + this.peekOffset); } - skipBlankBlock() { - let lineCount = 0; - while (true) { - this.peekBlankInline(); - - if (this.currentPeek === EOL) { - this.skipToPeek(); - this.next(); - lineCount++; - continue; - } - - if (this.currentPeek === EOF) { - // Consume any inline blanks before the EOF. - this.skipToPeek(); - return lineCount; - } - - // Any other char; reset to column 1 on this line. - this.resetPeek(); - return lineCount; - } + skipBlankInline() { + const blank = this.peekBlankInline(); + this.skipToPeek(); + return blank; } peekBlankBlock() { + let blank = ""; while (true) { const lineStart = this.peekOffset; - this.peekBlankInline(); - if (this.currentPeek === EOL) { + blank += EOL; this.peek(); - } else { - this.resetPeek(lineStart); - break; + continue; + } + if (this.currentPeek === EOF) { + // Treat the blank line at EOF as a blank block. + return blank; } + // Any other char; reset to column 1 on this line. + this.resetPeek(lineStart); + return blank; } } - skipBlank() { - while (this.currentChar === " " || this.currentChar === EOL) { - this.next(); - } + skipBlankBlock() { + const blank = this.peekBlankBlock(); + this.skipToPeek(); + return blank; } peekBlank() { @@ -130,6 +113,11 @@ export class FluentParserStream extends ParserStream { } } + skipBlank() { + this.peekBlank(); + this.skipToPeek(); + } + expectChar(ch) { if (this.currentChar === ch) { this.next(); @@ -204,29 +192,39 @@ export class FluentParserStream extends ParserStream { return !includes(SPECIAL_LINE_START_CHARS, ch); } - isValueStart({skip = true}) { - if (skip === false) throw new Error("Unimplemented"); + isValueStart() { + // Inline Patterns may start with any char. + const ch = this.currentPeek; + return ch !== EOL && ch !== EOF; + } + isValueContinuation() { + const column1 = this.peekOffset; this.peekBlankInline(); - const ch = this.currentPeek; - // Inline Patterns may start with any char. - if (ch !== EOF && ch !== EOL) { - this.skipToPeek(); + if (this.currentPeek === "{") { + this.resetPeek(column1); return true; } - return this.isNextLineValue({skip}); + if (this.peekOffset - column1 === 0) { + return false; + } + + if (this.isCharPatternContinuation(this.currentPeek)) { + this.resetPeek(column1); + return true; + } + + return false; } // -1 - any // 0 - comment // 1 - group comment // 2 - resource comment - isNextLineComment(level = -1, {skip = false}) { - if (skip === true) throw new Error("Unimplemented"); - - if (this.currentPeek !== EOL) { + isNextLineComment(level = -1) { + if (this.currentChar !== EOL) { return false; } @@ -254,70 +252,21 @@ export class FluentParserStream extends ParserStream { return false; } - isNextLineVariantStart({skip = false}) { - if (skip === true) throw new Error("Unimplemented"); - - if (this.currentPeek !== EOL) { - return false; - } - - this.peekBlank(); - + isVariantStart() { + const currentPeekOffset = this.peekOffset; if (this.currentPeek === "*") { this.peek(); } - if (this.currentPeek === "[") { - this.resetPeek(); + this.resetPeek(currentPeekOffset); return true; } - this.resetPeek(); + this.resetPeek(currentPeekOffset); return false; } - isNextLineAttributeStart({skip = true}) { - if (skip === false) throw new Error("Unimplemented"); - - this.peekBlank(); - - if (this.currentPeek === ".") { - this.skipToPeek(); - return true; - } - - this.resetPeek(); - return false; - } - - isNextLineValue({skip = true}) { - if (this.currentPeek !== EOL) { - return false; - } - - this.peekBlankBlock(); - - const ptr = this.peekOffset; - - this.peekBlankInline(); - - if (this.currentPeek !== "{") { - if (this.peekOffset - ptr === 0) { - this.resetPeek(); - return false; - } - - if (!this.isCharPatternContinuation(this.currentPeek)) { - this.resetPeek(); - return false; - } - } - - if (skip) { - this.skipToPeek(); - } else { - this.resetPeek(); - } - return true; + isAttributeStart() { + return this.currentPeek === "."; } skipToNextEntryStart(junkStart) { diff --git a/fluent-syntax/test/fixtures_behavior/select_expression_without_variants.ftl b/fluent-syntax/test/fixtures_behavior/select_expression_without_variants.ftl index 7e43355d1..8eea7b423 100644 --- a/fluent-syntax/test/fixtures_behavior/select_expression_without_variants.ftl +++ b/fluent-syntax/test/fixtures_behavior/select_expression_without_variants.ftl @@ -3,4 +3,4 @@ key = { $foo -> } key = { $foo -> } -# ~ERROR E0003, pos 39, args "[" +# ~ERROR E0011, pos 39 diff --git a/fluent-syntax/test/fixtures_behavior/selector_expression_ends_abruptly.ftl b/fluent-syntax/test/fixtures_behavior/selector_expression_ends_abruptly.ftl index a11a18144..8cb339853 100644 --- a/fluent-syntax/test/fixtures_behavior/selector_expression_ends_abruptly.ftl +++ b/fluent-syntax/test/fixtures_behavior/selector_expression_ends_abruptly.ftl @@ -1,2 +1,2 @@ key = { $foo -> -# ~ERROR E0003, pos 16, args "[" +# ~ERROR E0011, pos 16 diff --git a/fluent-syntax/test/fixtures_behavior/variant_lists.ftl b/fluent-syntax/test/fixtures_behavior/variant_lists.ftl index 10f5cab3d..2f3cc5b8d 100644 --- a/fluent-syntax/test/fixtures_behavior/variant_lists.ftl +++ b/fluent-syntax/test/fixtures_behavior/variant_lists.ftl @@ -4,7 +4,7 @@ message1 = *[one] One } -# ~ERROR E0023, pos 123 +# ~ERROR E0014, pos 97 message2 = { $sel -> *[one] { @@ -24,7 +24,7 @@ message2 = } } -# ~ERROR E0023, pos 318 +# ~ERROR E0014, pos 292 -term3 = { $sel -> *[one] { diff --git a/fluent-syntax/test/fixtures_reference/multiline_values.ftl b/fluent-syntax/test/fixtures_reference/multiline_values.ftl index 4dd811548..e3739bb5c 100644 --- a/fluent-syntax/test/fixtures_reference/multiline_values.ftl +++ b/fluent-syntax/test/fixtures_reference/multiline_values.ftl @@ -33,3 +33,28 @@ key07 = {"A multiline value"} starting and ending {"with a placeable"} key08 = Leading and trailing whitespace. + +key09 = zero + three + two + one + zero + +key10 = + two + zero + four + +key11 = + + + two + zero + +key12 = +{"."} + four + +key13 = + four +{"."} diff --git a/fluent-syntax/test/fixtures_reference/multiline_values.json b/fluent-syntax/test/fixtures_reference/multiline_values.json index 645a09b38..17fb6f8e4 100644 --- a/fluent-syntax/test/fixtures_reference/multiline_values.json +++ b/fluent-syntax/test/fixtures_reference/multiline_values.json @@ -102,7 +102,7 @@ "elements": [ { "type": "TextElement", - "value": "A multiline value with non-standard\n\nindentation." + "value": "A multiline value with non-standard\n\n indentation." } ] }, @@ -212,6 +212,110 @@ }, "attributes": [], "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key09" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "zero\n three\n two\n one\nzero" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key10" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": " two\nzero\n four" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key11" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": " two\nzero" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key12" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "." + } + }, + { + "type": "TextElement", + "value": "\n four" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key13" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": " four\n" + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "." + } + } + ] + }, + "attributes": [], + "comment": null } ] } diff --git a/fluent-syntax/test/fixtures_structure/blank_lines.json b/fluent-syntax/test/fixtures_structure/blank_lines.json index 14fab74f6..998e7da64 100644 --- a/fluent-syntax/test/fixtures_structure/blank_lines.json +++ b/fluent-syntax/test/fixtures_structure/blank_lines.json @@ -112,7 +112,7 @@ ], "span": { "type": "Span", - "start": 130, + "start": 126, "end": 153 } }, @@ -150,7 +150,7 @@ ], "span": { "type": "Span", - "start": 240, + "start": 236, "end": 267 } }, @@ -196,7 +196,7 @@ ], "span": { "type": "Span", - "start": 339, + "start": 335, "end": 347 } }, diff --git a/fluent-syntax/test/fixtures_structure/crlf.json b/fluent-syntax/test/fixtures_structure/crlf.json index b72afa33f..9461bfaa2 100644 --- a/fluent-syntax/test/fixtures_structure/crlf.json +++ b/fluent-syntax/test/fixtures_structure/crlf.json @@ -65,7 +65,7 @@ ], "span": { "type": "Span", - "start": 35, + "start": 31, "end": 58 } }, diff --git a/fluent-syntax/test/fixtures_structure/leading_dots.json b/fluent-syntax/test/fixtures_structure/leading_dots.json index 837427b45..d2e234c38 100644 --- a/fluent-syntax/test/fixtures_structure/leading_dots.json +++ b/fluent-syntax/test/fixtures_structure/leading_dots.json @@ -175,7 +175,7 @@ ], "span": { "type": "Span", - "start": 61, + "start": 57, "end": 71 } }, @@ -514,7 +514,7 @@ ], "span": { "type": "Span", - "start": 428, + "start": 424, "end": 484 } }, @@ -565,7 +565,7 @@ ], "span": { "type": "Span", - "start": 515, + "start": 511, "end": 516 } }, @@ -696,7 +696,7 @@ ], "span": { "type": "Span", - "start": 589, + "start": 580, "end": 599 } }, @@ -822,7 +822,7 @@ ], "span": { "type": "Span", - "start": 669, + "start": 657, "end": 679 } }, @@ -836,7 +836,7 @@ ], "span": { "type": "Span", - "start": 614, + "start": 615, "end": 684 } }, @@ -849,7 +849,7 @@ ], "span": { "type": "Span", - "start": 613, + "start": 609, "end": 685 } }, diff --git a/fluent-syntax/test/fixtures_structure/multiline_pattern.json b/fluent-syntax/test/fixtures_structure/multiline_pattern.json index 1793692b6..8f5c7fadd 100644 --- a/fluent-syntax/test/fixtures_structure/multiline_pattern.json +++ b/fluent-syntax/test/fixtures_structure/multiline_pattern.json @@ -65,7 +65,7 @@ ], "span": { "type": "Span", - "start": 47, + "start": 43, "end": 72 } }, @@ -103,7 +103,7 @@ ], "span": { "type": "Span", - "start": 171, + "start": 167, "end": 176 } }, @@ -173,7 +173,7 @@ ], "span": { "type": "Span", - "start": 311, + "start": 307, "end": 316 } }, diff --git a/fluent-syntax/test/fixtures_structure/placeable_at_eol.json b/fluent-syntax/test/fixtures_structure/placeable_at_eol.json index f79d7454b..7018303eb 100644 --- a/fluent-syntax/test/fixtures_structure/placeable_at_eol.json +++ b/fluent-syntax/test/fixtures_structure/placeable_at_eol.json @@ -61,7 +61,7 @@ ], "span": { "type": "Span", - "start": 11, + "start": 7, "end": 131 } }, @@ -124,7 +124,7 @@ ], "span": { "type": "Span", - "start": 144, + "start": 140, "end": 184 } }, diff --git a/fluent-syntax/test/fixtures_structure/sparse-messages.json b/fluent-syntax/test/fixtures_structure/sparse-messages.json index bdeb917b5..32780f68c 100644 --- a/fluent-syntax/test/fixtures_structure/sparse-messages.json +++ b/fluent-syntax/test/fixtures_structure/sparse-messages.json @@ -27,7 +27,7 @@ ], "span": { "type": "Span", - "start": 12, + "start": 8, "end": 17 } }, @@ -122,7 +122,7 @@ ], "span": { "type": "Span", - "start": 63, + "start": 59, "end": 104 } }, @@ -322,7 +322,7 @@ ], "span": { "type": "Span", - "start": 154, + "start": 155, "end": 208 } }, diff --git a/fluent-syntax/test/fixtures_structure/term.json b/fluent-syntax/test/fixtures_structure/term.json index 6da90f3bb..c36ec4509 100644 --- a/fluent-syntax/test/fixtures_structure/term.json +++ b/fluent-syntax/test/fixtures_structure/term.json @@ -218,7 +218,7 @@ ], "span": { "type": "Span", - "start": 131, + "start": 127, "end": 171 } }, @@ -481,7 +481,7 @@ ], "span": { "type": "Span", - "start": 198, + "start": 199, "end": 436 } }, @@ -494,7 +494,7 @@ ], "span": { "type": "Span", - "start": 197, + "start": 193, "end": 437 } }, diff --git a/fluent-syntax/test/fixtures_structure/variant_with_empty_pattern.json b/fluent-syntax/test/fixtures_structure/variant_with_empty_pattern.json index 19fe137d2..2b61ee990 100644 --- a/fluent-syntax/test/fixtures_structure/variant_with_empty_pattern.json +++ b/fluent-syntax/test/fixtures_structure/variant_with_empty_pattern.json @@ -77,7 +77,7 @@ ], "span": { "type": "Span", - "start": 12, + "start": 13, "end": 41 } }, @@ -90,7 +90,7 @@ ], "span": { "type": "Span", - "start": 11, + "start": 7, "end": 42 } }, diff --git a/fluent/src/resolver.js b/fluent/src/resolver.js index f2a216d25..b1a29a0d1 100644 --- a/fluent/src/resolver.js +++ b/fluent/src/resolver.js @@ -305,6 +305,8 @@ function Type(env, expr) { switch (expr.type) { + case "str": + return expr.value; case "num": return new FluentNumber(expr.value); case "var": diff --git a/fluent/src/resource.js b/fluent/src/resource.js index b457a38ec..d1b6421c4 100644 --- a/fluent/src/resource.js +++ b/fluent/src/resource.js @@ -27,11 +27,13 @@ const RE_STRING_RUN = /([^\\"\n\r]*)/y; const RE_UNICODE_ESCAPE = /\\u([a-fA-F0-9]{4})/y; const RE_STRING_ESCAPE = /\\([\\"])/y; -// Used for trimming TextElements and indents. With the /m flag, the $ matches -// the end of every line. -const RE_TRAILING_SPACES = / +$/mg; -// CRLFs are normalized to LF. -const RE_CRLF = /\r\n/g; +// Used for trimming TextElements and indents. +const RE_LEADING_NEWLINES = /^\n+/; +const RE_TRAILING_SPACES = / +$/; +// Used in makeIndent to strip spaces from blank lines and normalize CRLF to LF. +const RE_BLANK_LINES = / *\r?\n/g; +// Used in makeIndent to measure the indentation. +const RE_INDENT = /( *)$/; // Common tokens. const TOKEN_BRACE_OPEN = /{\s*/y; @@ -180,42 +182,41 @@ export default class FluentResource extends Map { // If there's a placeable on the first line, parse a complex pattern. if (source[cursor] === "{" || source[cursor] === "}") { - return first - // Re-use the text parsed above, if possible. - ? parsePatternElements(first) - : parsePatternElements(); + // Re-use the text parsed above, if possible. + return parsePatternElements(first ? [first] : [], Infinity); } // RE_TEXT_VALUE stops at newlines. Only continue parsing the pattern if // what comes after the newline is indented. let indent = parseIndent(); if (indent) { - return first + if (first) { // If there's text on the first line, the blank block is part of the - // translation content. - ? parsePatternElements(first, trim(indent)) - // Otherwise, we're dealing with a block pattern. The blank block is - // the leading whitespace; discard it. - : parsePatternElements(); + // translation content in its entirety. + return parsePatternElements([first, indent], indent.length); + } + // Otherwise, we're dealing with a block pattern, i.e. a pattern which + // starts on a new line. Discrad the leading newlines but keep the + // inline indent; it will be used by the dedentation logic. + indent.value = trim(indent.value, RE_LEADING_NEWLINES); + return parsePatternElements([indent], indent.length); } if (first) { // It was just a simple inline text after all. - return trim(first); + return trim(first, RE_TRAILING_SPACES); } return null; } // Parse a complex pattern as an array of elements. - function parsePatternElements(...elements) { + function parsePatternElements(elements = [], commonIndent) { let placeableCount = 0; - let needsTrimming = false; while (true) { if (test(RE_TEXT_RUN)) { elements.push(match(RE_TEXT_RUN)); - needsTrimming = true; continue; } @@ -224,7 +225,6 @@ export default class FluentResource extends Map { throw new FluentError("Too many placeables"); } elements.push(parsePlaceable()); - needsTrimming = false; continue; } @@ -234,23 +234,34 @@ export default class FluentResource extends Map { let indent = parseIndent(); if (indent) { - elements.push(trim(indent)); - needsTrimming = false; + elements.push(indent); + commonIndent = Math.min(commonIndent, indent.length); continue; } break; } - if (needsTrimming) { - // Trim the trailing whitespace of the last element if it's a - // TextElement. Use a flag rather than a typeof check to tell - // TextElements and StringLiterals apart (both are strings). - let lastIndex = elements.length - 1; - elements[lastIndex] = trim(elements[lastIndex]); + let lastIndex = elements.length - 1; + // Trim the trailing spaces in the last element if it's a TextElement. + if (typeof elements[lastIndex] === "string") { + elements[lastIndex] = trim(elements[lastIndex], RE_TRAILING_SPACES); } - return elements; + let baked = []; + for (let element of elements) { + if (element.type === "indent") { + // Dedent indented lines by the maximum common indent. + element = element.value.slice(0, element.value.length - commonIndent); + } else if (element.type === "str") { + // Optimize StringLiterals into their value. + element = element.value; + } + if (element) { + baked.push(element); + } + } + return baked; } function parsePlaceable() { @@ -401,7 +412,7 @@ export default class FluentResource extends Map { } if (consumeChar("\"")) { - return value; + return {type: "str", value}; } // We've reached an EOL of EOF. @@ -447,7 +458,7 @@ export default class FluentResource extends Map { case "{": // Placeables don't require indentation (in EBNF: block-placeable). // Continue the Pattern. - return source.slice(start, cursor).replace(RE_CRLF, "\n"); + return makeIndent(source.slice(start, cursor)); } // If the first character on the line is not one of the special characters @@ -456,7 +467,7 @@ export default class FluentResource extends Map { if (source[cursor - 1] === " ") { // It's an indented text character (in EBNF: indented-char). Continue // the Pattern. - return source.slice(start, cursor).replace(RE_CRLF, "\n"); + return makeIndent(source.slice(start, cursor)); } // A not-indented text character is likely the identifier of the next @@ -464,9 +475,16 @@ export default class FluentResource extends Map { return false; } - // Trim spaces trailing on every line of text. - function trim(text) { - return text.replace(RE_TRAILING_SPACES, ""); + // Trim blanks in text according to the given regex. + function trim(text, re) { + return text.replace(re, ""); + } + + // Normalize a blank block and extract the indent details. + function makeIndent(blank) { + let value = blank.replace(RE_BLANK_LINES, "\n"); + let length = RE_INDENT.exec(blank)[1].length; + return {type: "indent", value, length}; } } } diff --git a/fluent/test/fixtures_reference/call_expressions.json b/fluent/test/fixtures_reference/call_expressions.json index 3e0fe30e5..431a25b5a 100644 --- a/fluent/test/fixtures_reference/call_expressions.json +++ b/fluent/test/fixtures_reference/call_expressions.json @@ -11,7 +11,10 @@ "type": "num", "value": "1" }, - "a", + { + "type": "str", + "value": "a" + }, { "type": "ref", "name": "msg" @@ -38,7 +41,10 @@ { "type": "narg", "name": "y", - "value": "Y" + "value": { + "type": "str", + "value": "Y" + } } ] } @@ -62,7 +68,10 @@ { "type": "narg", "name": "y", - "value": "Y" + "value": { + "type": "str", + "value": "Y" + } } ] } @@ -79,7 +88,10 @@ "type": "num", "value": "1" }, - "a", + { + "type": "str", + "value": "a" + }, { "type": "ref", "name": "msg" @@ -95,7 +107,10 @@ { "type": "narg", "name": "y", - "value": "Y" + "value": { + "type": "str", + "value": "Y" + } } ] } @@ -120,11 +135,17 @@ "value": "1" } }, - "a", + { + "type": "str", + "value": "a" + }, { "type": "narg", "name": "y", - "value": "Y" + "value": { + "type": "str", + "value": "Y" + } }, { "type": "ref", @@ -152,7 +173,10 @@ { "type": "narg", "name": "x", - "value": "X" + "value": { + "type": "str", + "value": "X" + } } ] } @@ -165,7 +189,10 @@ "name": "FUN" }, "args": [ - "a", + { + "type": "str", + "value": "a" + }, { "type": "ref", "name": "msg" @@ -199,7 +226,10 @@ "name": "FUN" }, "args": [ - "a", + { + "type": "str", + "value": "a" + }, { "type": "ref", "name": "msg" @@ -223,7 +253,10 @@ "name": "FUN" }, "args": [ - "a", + { + "type": "str", + "value": "a" + }, { "type": "ref", "name": "msg" @@ -272,7 +305,10 @@ "name": "FUN" }, "args": [ - "a" + { + "type": "str", + "value": "a" + } ] } ], diff --git a/fluent/test/fixtures_reference/cr.json b/fluent/test/fixtures_reference/cr.json index 6c6f39082..88e772c02 100644 --- a/fluent/test/fixtures_reference/cr.json +++ b/fluent/test/fixtures_reference/cr.json @@ -3,6 +3,7 @@ "err02": "Value 02", "err03": { "value": [ + "\r\r", "Value 03", "\r", "Continued" diff --git a/fluent/test/fixtures_reference/messages.json b/fluent/test/fixtures_reference/messages.json index 49d97ae73..03f07b796 100644 --- a/fluent/test/fixtures_reference/messages.json +++ b/fluent/test/fixtures_reference/messages.json @@ -26,7 +26,5 @@ "attr1": "Attribute 1" } }, - "key06": [ - "" - ] + "key06": [] } diff --git a/fluent/test/fixtures_reference/multiline_values.json b/fluent/test/fixtures_reference/multiline_values.json index a4cb0a170..c6d153fcb 100644 --- a/fluent/test/fixtures_reference/multiline_values.json +++ b/fluent/test/fixtures_reference/multiline_values.json @@ -35,7 +35,7 @@ }, "key05": [ "A multiline value with non-standard", - "\n\n", + "\n\n ", "indentation." ], "key06": [ @@ -53,5 +53,41 @@ " starting and ending ", "with a placeable" ], - "key08": "Leading and trailing whitespace." + "key08": "Leading and trailing whitespace.", + "key09": [ + "zero", + "\n ", + "three", + "\n ", + "two", + "\n ", + "one", + "\n", + "zero" + ], + "key10": [ + " ", + "two", + "\n", + "zero", + "\n ", + "four" + ], + "key11": [ + " ", + "two", + "\n", + "zero" + ], + "key12": [ + ".", + "\n ", + "four" + ], + "key13": [ + " ", + "four", + "\n", + "." + ] } diff --git a/fluent/test/fixtures_reference/select_expressions.json b/fluent/test/fixtures_reference/select_expressions.json index 6881a51a7..65ed7c426 100644 --- a/fluent/test/fixtures_reference/select_expressions.json +++ b/fluent/test/fixtures_reference/select_expressions.json @@ -21,7 +21,6 @@ { "key": "other", "value": [ - "", "Other" ] } @@ -79,9 +78,7 @@ "variants": [ { "key": "one", - "value": [ - "" - ] + "value": [] } ], "star": 0 diff --git a/fluent/test/fixtures_reference/select_indent.json b/fluent/test/fixtures_reference/select_indent.json index 52b4c28bd..c8e21fea3 100644 --- a/fluent/test/fixtures_reference/select_indent.json +++ b/fluent/test/fixtures_reference/select_indent.json @@ -229,7 +229,6 @@ } ], "star": 0 - }, - "" + } ] } diff --git a/fluent/test/fixtures_reference/terms.json b/fluent/test/fixtures_reference/terms.json index 81e77bc5c..0195fe337 100644 --- a/fluent/test/fixtures_reference/terms.json +++ b/fluent/test/fixtures_reference/terms.json @@ -5,9 +5,7 @@ "attr": "Attribute" } }, - "-term02": [ - "" - ], + "-term02": [], "-term03": { "value": null, "attrs": { diff --git a/fluent/test/fixtures_structure/variant_with_empty_pattern.json b/fluent/test/fixtures_structure/variant_with_empty_pattern.json index 3ffa51e1b..23d0fb0ca 100644 --- a/fluent/test/fixtures_structure/variant_with_empty_pattern.json +++ b/fluent/test/fixtures_structure/variant_with_empty_pattern.json @@ -9,9 +9,7 @@ "variants": [ { "key": "one", - "value": [ - "" - ] + "value": [] } ], "star": 0 diff --git a/fluent/test/fixtures_structure/whitespace_leading.json b/fluent/test/fixtures_structure/whitespace_leading.json index 7a69cc773..96fc498e8 100644 --- a/fluent/test/fixtures_structure/whitespace_leading.json +++ b/fluent/test/fixtures_structure/whitespace_leading.json @@ -2,7 +2,6 @@ "key1": "Value", "key2": "  Value", "key3": [ - "", " Value" ], "key4": [