From d009ea21c6d58d3d824a20060c3ee62fe16aa53d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Thu, 15 Nov 2018 14:42:23 +0100 Subject: [PATCH 01/23] Support astral Unicode characters in TextElements and StringLiterals --- eslint_src.json | 11 -- .../test/fixtures_reference/astral.ftl | 20 ++ .../test/fixtures_reference/astral.json | 174 ++++++++++++++++++ fluent/src/resource.js | 8 +- fluent/test/fixtures_reference/astral.json | 19 ++ fluent/test/parser_behavior_test.js | 4 +- fluent/test/parser_reference_test.js | 2 +- fluent/test/parser_structure_test.js | 2 +- 8 files changed, 224 insertions(+), 16 deletions(-) create mode 100644 fluent-syntax/test/fixtures_reference/astral.ftl create mode 100644 fluent-syntax/test/fixtures_reference/astral.json create mode 100644 fluent/test/fixtures_reference/astral.json diff --git a/eslint_src.json b/eslint_src.json index 252b3cd95..328febcee 100644 --- a/eslint_src.json +++ b/eslint_src.json @@ -32,17 +32,6 @@ ], "no-implied-eval": 2, "no-loop-func": 2, - "no-magic-numbers": [ - 1, - { - "ignore": [ - -1, - 0, - 1, - 2 - ] - } - ], "no-useless-call": 2, "no-useless-concat": 2, "no-delete-var": 2, diff --git a/fluent-syntax/test/fixtures_reference/astral.ftl b/fluent-syntax/test/fixtures_reference/astral.ftl new file mode 100644 index 000000000..b77e32e38 --- /dev/null +++ b/fluent-syntax/test/fixtures_reference/astral.ftl @@ -0,0 +1,20 @@ +face-with-tears-of-joy = ๐Ÿ˜‚ +tetragram-for-centre = ๐Œ† + +surrogates-in-text = \uD83D\uDE02 +surrogates-in-string = {"\uD83D\uDE02"} +surrogates-in-adjacent-strings = {"\uD83D"}{"\uDE02"} + +emoji-in-text = A face ๐Ÿ˜‚ with tears of joy. +emoji-in-string = {"A face ๐Ÿ˜‚ with tears of joy."} + +# ERROR Invalid identifier +err-๐Ÿ˜‚ = Value + +# ERROR Invalid expression +err-invalid-expression = { ๐Ÿ˜‚ } + +# ERROR Invalid variant key +err-invalid-variant-key = { $sel -> + *[๐Ÿ˜‚] Value +} diff --git a/fluent-syntax/test/fixtures_reference/astral.json b/fluent-syntax/test/fixtures_reference/astral.json new file mode 100644 index 000000000..6056fb727 --- /dev/null +++ b/fluent-syntax/test/fixtures_reference/astral.json @@ -0,0 +1,174 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "face-with-tears-of-joy" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "๐Ÿ˜‚" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "tetragram-for-centre" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "๐Œ†" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "surrogates-in-text" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "\\uD83D\\uDE02" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "surrogates-in-string" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "\\uD83D\\uDE02" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "surrogates-in-adjacent-strings" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "\\uD83D" + } + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "\\uDE02" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "emoji-in-text" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "A face ๐Ÿ˜‚ with tears of joy." + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "emoji-in-string" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "A face ๐Ÿ˜‚ with tears of joy." + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR Invalid identifier" + }, + { + "type": "Junk", + "annotations": [], + "content": "err-๐Ÿ˜‚ = Value\n" + }, + { + "type": "Comment", + "content": "ERROR Invalid expression" + }, + { + "type": "Junk", + "annotations": [], + "content": "err-invalid-expression = { ๐Ÿ˜‚ }\n" + }, + { + "type": "Comment", + "content": "ERROR Invalid variant key" + }, + { + "type": "Junk", + "annotations": [], + "content": "err-invalid-variant-key = { $sel ->\n *[๐Ÿ˜‚] Value\n}\n" + } + ] +} diff --git a/fluent/src/resource.js b/fluent/src/resource.js index a7c6b5675..62025ad8c 100644 --- a/fluent/src/resource.js +++ b/fluent/src/resource.js @@ -420,7 +420,13 @@ export default class FluentResource extends Map { function parseEscapeSequence(reSpecialized) { if (test(RE_UNICODE_ESCAPE)) { let sequence = match(RE_UNICODE_ESCAPE); - return String.fromCodePoint(parseInt(sequence, 16)); + let codepoint = parseInt(sequence, 16); + return codepoint <= 0xD7FF || 0xE000 <= codepoint + // It's a Unicode scalar value. + ? String.fromCodePoint(codepoint) + // Lonely surrogates can cause trouble when the parsing result is + // saved using UTF-8. Use U+FFFD REPLACEMENT CHARACTER instead. + : "๏ฟฝ"; } if (test(reSpecialized)) { diff --git a/fluent/test/fixtures_reference/astral.json b/fluent/test/fixtures_reference/astral.json new file mode 100644 index 000000000..9a9469f0c --- /dev/null +++ b/fluent/test/fixtures_reference/astral.json @@ -0,0 +1,19 @@ +{ + "face-with-tears-of-joy": "๐Ÿ˜‚", + "tetragram-for-centre": "๐Œ†", + "surrogates-in-text": [ + "๏ฟฝ", + "๏ฟฝ" + ], + "surrogates-in-string": [ + "๏ฟฝ๏ฟฝ" + ], + "surrogates-in-adjacent-strings": [ + "๏ฟฝ", + "๏ฟฝ" + ], + "emoji-in-text": "A face ๐Ÿ˜‚ with tears of joy.", + "emoji-in-string": [ + "A face ๐Ÿ˜‚ with tears of joy." + ] +} diff --git a/fluent/test/parser_behavior_test.js b/fluent/test/parser_behavior_test.js index 239a4b47c..53a169ed2 100644 --- a/fluent/test/parser_behavior_test.js +++ b/fluent/test/parser_behavior_test.js @@ -47,8 +47,8 @@ readdir(ftlFixtures, function(err, filenames) { if (expected.attributes) { assert(hasAttrs(entry), `Expected ${id} to have attributes`); assert.deepEqual( - Object.keys(expected.attributes), - Object.keys(entry.attrs) + Object.keys(entry.attrs), + Object.keys(expected.attributes) ); } else { assert(!hasAttrs(entry), `Expected ${id} to have zero attributes`); diff --git a/fluent/test/parser_reference_test.js b/fluent/test/parser_reference_test.js index 1058fb85d..de69b757b 100644 --- a/fluent/test/parser_reference_test.js +++ b/fluent/test/parser_reference_test.js @@ -30,7 +30,7 @@ readdir(ftlFixtures, function(err, filenames) { ); const resource = FluentResource.fromString(ftl); assert.deepEqual( - JSON.parse(expected), toObject(resource), + toObject(resource), JSON.parse(expected), 'Actual Annotations don\'t match the expected ones' ); }); diff --git a/fluent/test/parser_structure_test.js b/fluent/test/parser_structure_test.js index 8c406c5b5..8462192e7 100644 --- a/fluent/test/parser_structure_test.js +++ b/fluent/test/parser_structure_test.js @@ -30,7 +30,7 @@ readdir(ftlFixtures, function(err, filenames) { ); const resource = FluentResource.fromString(ftl); assert.deepEqual( - JSON.parse(expected), toObject(resource), + toObject(resource), JSON.parse(expected), 'Actual Annotations don\'t match the expected ones' ); }); From 5941f1bbf98300a193b61d0e69e2918dc6ec5e54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Thu, 15 Nov 2018 13:51:49 +0100 Subject: [PATCH 02/23] Don't store the term sigil in the identifier name --- fluent-syntax/src/errors.js | 2 +- fluent-syntax/src/parser.js | 20 +-- fluent-syntax/src/serializer.js | 33 ++++- fluent-syntax/test/fixtures_behavior/term.ftl | 4 +- .../fixtures_reference/call_expressions.json | 2 +- .../test/fixtures_reference/comments.json | 2 +- .../member_expressions.json | 2 +- .../fixtures_reference/mixed_entries.json | 2 +- .../reference_expressions.json | 2 +- .../select_expressions.json | 2 +- .../test/fixtures_reference/terms.json | 4 +- .../test/fixtures_reference/variables.ftl | 17 +++ .../test/fixtures_reference/variables.json | 132 ++++++++++++++++++ .../test/fixtures_reference/variant_keys.json | 8 +- .../fixtures_reference/variant_lists.json | 8 +- .../fixtures_reference/variants_indent.json | 8 +- .../fixtures_structure/sparse-messages.json | 8 +- .../test/fixtures_structure/term.json | 24 ++-- .../term_with_empty_pattern.json | 4 +- .../test/fixtures_behavior/make_fixtures.js | 20 +-- fluent/test/fixtures_reference/variables.json | 32 +++++ 21 files changed, 268 insertions(+), 68 deletions(-) create mode 100644 fluent-syntax/test/fixtures_reference/variables.ftl create mode 100644 fluent-syntax/test/fixtures_reference/variables.json create mode 100644 fluent/test/fixtures_reference/variables.json diff --git a/fluent-syntax/src/errors.js b/fluent-syntax/src/errors.js index df0c71db9..81caf7c73 100644 --- a/fluent-syntax/src/errors.js +++ b/fluent-syntax/src/errors.js @@ -28,7 +28,7 @@ function getErrorMessage(code, args) { } case "E0006": { const [id] = args; - return `Expected term "${id}" to have a value`; + return `Expected term "-${id}" to have a value`; } case "E0007": return "Keyword cannot end with a whitespace"; diff --git a/fluent-syntax/src/parser.js b/fluent-syntax/src/parser.js index c0b0e4449..58877f01b 100644 --- a/fluent-syntax/src/parser.js +++ b/fluent-syntax/src/parser.js @@ -39,10 +39,9 @@ export default class FluentParser { // Poor man's decorators. const methodNames = [ "getComment", "getMessage", "getTerm", "getAttribute", "getIdentifier", - "getTermIdentifier", "getVariant", "getNumber", - "getValue", "getPattern", "getVariantList", "getTextElement", - "getPlaceable", "getExpression", "getSelectorExpression", "getCallArg", - "getString", "getLiteral" + "getVariant", "getNumber", "getValue", "getPattern", "getVariantList", + "getTextElement", "getPlaceable", "getExpression", + "getSelectorExpression", "getCallArg", "getString", "getLiteral" ]; for (const name of methodNames) { this[name] = withSpan(this[name]); @@ -242,7 +241,8 @@ export default class FluentParser { } getTerm(ps) { - const id = this.getTermIdentifier(ps); + ps.expectChar("-"); + const id = this.getIdentifier(ps); ps.skipBlankInline(); ps.expectChar("="); @@ -301,13 +301,6 @@ export default class FluentParser { return new AST.Identifier(name); } - getTermIdentifier(ps) { - ps.expectChar("-"); - const id = this.getIdentifier(ps); - return new AST.Identifier(`-${id.name}`); - - } - getVariantKey(ps) { const ch = ps.currentChar; @@ -777,7 +770,8 @@ export default class FluentParser { } if (ch === "-") { - const id = this.getTermIdentifier(ps); + ps.next(); + const id = this.getIdentifier(ps); return new AST.TermReference(id); } diff --git a/fluent-syntax/src/serializer.js b/fluent-syntax/src/serializer.js index 91cc8d635..618496b7f 100644 --- a/fluent-syntax/src/serializer.js +++ b/fluent-syntax/src/serializer.js @@ -44,8 +44,9 @@ export default class FluentSerializer { serializeEntry(entry, state = 0) { switch (entry.type) { case "Message": - case "Term": return serializeMessage(entry); + case "Term": + return serializeTerm(entry); case "Comment": if (state & HAS_ENTRIES) { return `\n${serializeComment(entry, "#")}\n`; @@ -95,8 +96,7 @@ function serializeMessage(message) { parts.push(serializeComment(message.comment)); } - parts.push(serializeIdentifier(message.id)); - parts.push(" ="); + parts.push(`${serializeIdentifier(message.id)} =`); if (message.value) { parts.push(serializeValue(message.value)); @@ -111,6 +111,25 @@ function serializeMessage(message) { } +function serializeTerm(term) { + const parts = []; + + if (term.comment) { + parts.push(serializeComment(term.comment)); + } + + parts.push(`-${serializeIdentifier(term.id)} =`); + parts.push(serializeValue(term.value)); + + for (const attribute of term.attributes) { + parts.push(serializeAttribute(attribute)); + } + + parts.push("\n"); + return parts.join(""); +} + + function serializeAttribute(attribute) { const id = serializeIdentifier(attribute.id); const value = indent(serializeValue(attribute.value)); @@ -202,8 +221,9 @@ function serializeExpression(expr) { case "NumberLiteral": return serializeNumberLiteral(expr); case "MessageReference": - case "TermReference": return serializeMessageReference(expr); + case "TermReference": + return serializeTermReference(expr); case "VariableReference": return serializeVariableReference(expr); case "AttributeExpression": @@ -237,6 +257,11 @@ function serializeMessageReference(expr) { } +function serializeTermReference(expr) { + return `-${serializeIdentifier(expr.id)}`; +} + + function serializeVariableReference(expr) { return `$${serializeIdentifier(expr.id)}`; } diff --git a/fluent-syntax/test/fixtures_behavior/term.ftl b/fluent-syntax/test/fixtures_behavior/term.ftl index 272e30397..58426d638 100644 --- a/fluent-syntax/test/fixtures_behavior/term.ftl +++ b/fluent-syntax/test/fixtures_behavior/term.ftl @@ -25,8 +25,8 @@ err4 = { -brand() } # ~ERROR E0008, pos 339 -err5 = -# ~ERROR E0006, pos 351, args "-err5" +# ~ERROR E0006, pos 351, args "err5" -err6 = .attr = Attribute -# ~ERROR E0006, pos 360, args "-err6" +# ~ERROR E0006, pos 360, args "err6" diff --git a/fluent-syntax/test/fixtures_reference/call_expressions.json b/fluent-syntax/test/fixtures_reference/call_expressions.json index ae9029a05..a860260fe 100644 --- a/fluent-syntax/test/fixtures_reference/call_expressions.json +++ b/fluent-syntax/test/fixtures_reference/call_expressions.json @@ -548,7 +548,7 @@ "type": "TermReference", "id": { "type": "Identifier", - "name": "-msg" + "name": "msg" } } ], diff --git a/fluent-syntax/test/fixtures_reference/comments.json b/fluent-syntax/test/fixtures_reference/comments.json index 484f0b155..c28115ac7 100644 --- a/fluent-syntax/test/fixtures_reference/comments.json +++ b/fluent-syntax/test/fixtures_reference/comments.json @@ -30,7 +30,7 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-term" + "name": "term" }, "value": { "type": "Pattern", diff --git a/fluent-syntax/test/fixtures_reference/member_expressions.json b/fluent-syntax/test/fixtures_reference/member_expressions.json index 939ff5e80..a95879011 100644 --- a/fluent-syntax/test/fixtures_reference/member_expressions.json +++ b/fluent-syntax/test/fixtures_reference/member_expressions.json @@ -18,7 +18,7 @@ "type": "TermReference", "id": { "type": "Identifier", - "name": "-term" + "name": "term" } }, "key": { diff --git a/fluent-syntax/test/fixtures_reference/mixed_entries.json b/fluent-syntax/test/fixtures_reference/mixed_entries.json index 4e349a5ab..8f334f4fa 100644 --- a/fluent-syntax/test/fixtures_reference/mixed_entries.json +++ b/fluent-syntax/test/fixtures_reference/mixed_entries.json @@ -13,7 +13,7 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-brand-name" + "name": "brand-name" }, "value": { "type": "Pattern", diff --git a/fluent-syntax/test/fixtures_reference/reference_expressions.json b/fluent-syntax/test/fixtures_reference/reference_expressions.json index 080e54f1e..47e4dfab0 100644 --- a/fluent-syntax/test/fixtures_reference/reference_expressions.json +++ b/fluent-syntax/test/fixtures_reference/reference_expressions.json @@ -40,7 +40,7 @@ "type": "TermReference", "id": { "type": "Identifier", - "name": "-term" + "name": "term" } } } diff --git a/fluent-syntax/test/fixtures_reference/select_expressions.json b/fluent-syntax/test/fixtures_reference/select_expressions.json index 3dac7c79d..13f5b71bc 100644 --- a/fluent-syntax/test/fixtures_reference/select_expressions.json +++ b/fluent-syntax/test/fixtures_reference/select_expressions.json @@ -92,7 +92,7 @@ "type": "TermReference", "id": { "type": "Identifier", - "name": "-term" + "name": "term" } }, "name": { diff --git a/fluent-syntax/test/fixtures_reference/terms.json b/fluent-syntax/test/fixtures_reference/terms.json index 790e6073c..f63122e57 100644 --- a/fluent-syntax/test/fixtures_reference/terms.json +++ b/fluent-syntax/test/fixtures_reference/terms.json @@ -5,7 +5,7 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-term01" + "name": "term01" }, "value": { "type": "Pattern", @@ -40,7 +40,7 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-term02" + "name": "term02" }, "value": { "type": "Pattern", diff --git a/fluent-syntax/test/fixtures_reference/variables.ftl b/fluent-syntax/test/fixtures_reference/variables.ftl new file mode 100644 index 000000000..6c3436929 --- /dev/null +++ b/fluent-syntax/test/fixtures_reference/variables.ftl @@ -0,0 +1,17 @@ +key01 = {$var} +key02 = { $var } +key03 = { + $var +} +key04 = { +$var} + + +## Errors + +# ERROR Missing variable identifier +err01 = {$} +# ERROR Double $$ +err02 = {$$var} +# ERROR Invalid first char of the identifier +err03 = {$-var} diff --git a/fluent-syntax/test/fixtures_reference/variables.json b/fluent-syntax/test/fixtures_reference/variables.json new file mode 100644 index 000000000..58682e5ba --- /dev/null +++ b/fluent-syntax/test/fixtures_reference/variables.json @@ -0,0 +1,132 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key01" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "var" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key02" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "var" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key03" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "var" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key04" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "var" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "GroupComment", + "content": "Errors" + }, + { + "type": "Comment", + "content": "ERROR Missing variable identifier" + }, + { + "type": "Junk", + "annotations": [], + "content": "err01 = {$}\n" + }, + { + "type": "Comment", + "content": "ERROR Double $$" + }, + { + "type": "Junk", + "annotations": [], + "content": "err02 = {$$var}\n" + }, + { + "type": "Comment", + "content": "ERROR Invalid first char of the identifier" + }, + { + "type": "Junk", + "annotations": [], + "content": "err03 = {$-var}\n" + } + ] +} diff --git a/fluent-syntax/test/fixtures_reference/variant_keys.json b/fluent-syntax/test/fixtures_reference/variant_keys.json index f85abe669..e8c60b002 100644 --- a/fluent-syntax/test/fixtures_reference/variant_keys.json +++ b/fluent-syntax/test/fixtures_reference/variant_keys.json @@ -5,7 +5,7 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-simple-identifier" + "name": "simple-identifier" }, "value": { "type": "VariantList", @@ -36,7 +36,7 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-identifier-surrounded-by-whitespace" + "name": "identifier-surrounded-by-whitespace" }, "value": { "type": "VariantList", @@ -67,7 +67,7 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-int-number" + "name": "int-number" }, "value": { "type": "VariantList", @@ -98,7 +98,7 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-float-number" + "name": "float-number" }, "value": { "type": "VariantList", diff --git a/fluent-syntax/test/fixtures_reference/variant_lists.json b/fluent-syntax/test/fixtures_reference/variant_lists.json index 9713e098e..d10484046 100644 --- a/fluent-syntax/test/fixtures_reference/variant_lists.json +++ b/fluent-syntax/test/fixtures_reference/variant_lists.json @@ -5,7 +5,7 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-variant-list-in-term" + "name": "variant-list-in-term" }, "value": { "type": "VariantList", @@ -36,7 +36,7 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-variant-list-in-term-attr" + "name": "variant-list-in-term-attr" }, "value": { "type": "Pattern", @@ -97,7 +97,7 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-nested-variant-list-in-term" + "name": "nested-variant-list-in-term" }, "value": { "type": "VariantList", @@ -141,7 +141,7 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-nested-select" + "name": "nested-select" }, "value": { "type": "VariantList", diff --git a/fluent-syntax/test/fixtures_reference/variants_indent.json b/fluent-syntax/test/fixtures_reference/variants_indent.json index ad5c1723a..7b987d487 100644 --- a/fluent-syntax/test/fixtures_reference/variants_indent.json +++ b/fluent-syntax/test/fixtures_reference/variants_indent.json @@ -5,7 +5,7 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-variants-1tbs" + "name": "variants-1tbs" }, "value": { "type": "VariantList", @@ -36,7 +36,7 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-variants-allman" + "name": "variants-allman" }, "value": { "type": "VariantList", @@ -67,7 +67,7 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-variants-gnu" + "name": "variants-gnu" }, "value": { "type": "VariantList", @@ -98,7 +98,7 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-variants-no-indent" + "name": "variants-no-indent" }, "value": { "type": "VariantList", diff --git a/fluent-syntax/test/fixtures_structure/sparse-messages.json b/fluent-syntax/test/fixtures_structure/sparse-messages.json index 90f8fdb31..bdeb917b5 100644 --- a/fluent-syntax/test/fixtures_structure/sparse-messages.json +++ b/fluent-syntax/test/fixtures_structure/sparse-messages.json @@ -351,10 +351,10 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-key7", + "name": "key7", "span": { "type": "Span", - "start": 211, + "start": 212, "end": 216 } }, @@ -417,10 +417,10 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-key8", + "name": "key8", "span": { "type": "Span", - "start": 254, + "start": 255, "end": 259 } }, diff --git a/fluent-syntax/test/fixtures_structure/term.json b/fluent-syntax/test/fixtures_structure/term.json index f9cf94d4f..6da90f3bb 100644 --- a/fluent-syntax/test/fixtures_structure/term.json +++ b/fluent-syntax/test/fixtures_structure/term.json @@ -5,10 +5,10 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-brand-name", + "name": "brand-name", "span": { "type": "Span", - "start": 0, + "start": 1, "end": 11 } }, @@ -172,10 +172,10 @@ "type": "TermReference", "id": { "type": "Identifier", - "name": "-brand-name", + "name": "brand-name", "span": { "type": "Span", - "start": 145, + "start": 146, "end": 156 } }, @@ -254,10 +254,10 @@ "type": "TermReference", "id": { "type": "Identifier", - "name": "-brand-name", + "name": "brand-name", "span": { "type": "Span", - "start": 199, + "start": 200, "end": 210 } }, @@ -303,10 +303,10 @@ "type": "TermReference", "id": { "type": "Identifier", - "name": "-brand-name", + "name": "brand-name", "span": { "type": "Span", - "start": 243, + "start": 244, "end": 254 } }, @@ -365,10 +365,10 @@ "type": "TermReference", "id": { "type": "Identifier", - "name": "-brand-name", + "name": "brand-name", "span": { "type": "Span", - "start": 311, + "start": 312, "end": 322 } }, @@ -436,10 +436,10 @@ "type": "TermReference", "id": { "type": "Identifier", - "name": "-brand-name", + "name": "brand-name", "span": { "type": "Span", - "start": 385, + "start": 386, "end": 396 } }, diff --git a/fluent-syntax/test/fixtures_structure/term_with_empty_pattern.json b/fluent-syntax/test/fixtures_structure/term_with_empty_pattern.json index 66b7e06b7..13e018ec8 100644 --- a/fluent-syntax/test/fixtures_structure/term_with_empty_pattern.json +++ b/fluent-syntax/test/fixtures_structure/term_with_empty_pattern.json @@ -8,7 +8,7 @@ "type": "Annotation", "code": "E0006", "args": [ - "-foo" + "foo" ], "message": "Expected term \"-foo\" to have a value", "span": { @@ -32,7 +32,7 @@ "type": "Annotation", "code": "E0006", "args": [ - "-bar" + "bar" ], "message": "Expected term \"-bar\" to have a value", "span": { diff --git a/fluent/test/fixtures_behavior/make_fixtures.js b/fluent/test/fixtures_behavior/make_fixtures.js index 10e05f35e..591fdab90 100644 --- a/fluent/test/fixtures_behavior/make_fixtures.js +++ b/fluent/test/fixtures_behavior/make_fixtures.js @@ -36,22 +36,22 @@ function isLocalizable(entry) { function printEntries(source) { const {body} = parser.parse(source); - const messages = body.filter(isLocalizable); - const entries = {}; - for (const msg of messages) { - const entry = entries[msg.id.name] = {}; + for (const entry of body.filter(isLocalizable)) { + const id = entry.type === 'Term' + ? `-${entry.id.name}` : entry.id.name; + const expected = entries[id] = {}; - if (msg.value !== null) { - entry.value = true; + if (entry.value !== null) { + expected.value = true; } - if (msg.attributes.length > 0) { - entry.attributes = {}; + if (entry.attributes.length > 0) { + expected.attributes = {}; - for (const attr of msg.attributes) { - entry.attributes[attr.id.name] = true + for (const attr of entry.attributes) { + expected.attributes[attr.id.name] = true } } diff --git a/fluent/test/fixtures_reference/variables.json b/fluent/test/fixtures_reference/variables.json new file mode 100644 index 000000000..6f0de9904 --- /dev/null +++ b/fluent/test/fixtures_reference/variables.json @@ -0,0 +1,32 @@ +{ + "key01": [ + { + "type": "var", + "name": "var" + } + ], + "key02": [ + { + "type": "var", + "name": "var" + } + ], + "key03": [ + { + "type": "var", + "name": "var" + } + ], + "key04": [ + { + "type": "var", + "name": "var" + } + ], + "err03": [ + { + "type": "var", + "name": "-var" + } + ] +} From bf3f6c8caaba9f86bb377f3f3d8eff217f214036 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Thu, 15 Nov 2018 15:59:03 +0100 Subject: [PATCH 03/23] Treat backslash as normal char in TextElements --- fluent-syntax/src/parser.js | 12 +- .../fixtures_behavior/escape_sequences.ftl | 22 +- .../fixtures_reference/escaped_characters.ftl | 23 +- .../escaped_characters.json | 156 +++++- .../fixtures_structure/escape_sequences.ftl | 23 +- .../fixtures_structure/escape_sequences.json | 458 +++++++++++++++--- fluent-syntax/test/serializer_test.js | 6 +- fluent/src/resource.js | 43 +- .../fixtures_behavior/escape_sequences.json | 20 +- fluent/test/fixtures_reference/astral.json | 5 +- .../escaped_characters.json | 40 +- .../fixtures_structure/escape_sequences.json | 40 +- 12 files changed, 676 insertions(+), 172 deletions(-) diff --git a/fluent-syntax/src/parser.js b/fluent-syntax/src/parser.js index 58877f01b..14066cbda 100644 --- a/fluent-syntax/src/parser.js +++ b/fluent-syntax/src/parser.js @@ -487,12 +487,6 @@ export default class FluentParser { continue; } - if (ch === "\\") { - ps.next(); - buffer += this.getEscapeSequence(ps); - continue; - } - buffer += ch; ps.next(); } @@ -500,10 +494,10 @@ export default class FluentParser { return new AST.TextElement(buffer); } - getEscapeSequence(ps, specials = ["{", "\\"]) { + getEscapeSequence(ps) { const next = ps.currentChar; - if (specials.includes(next)) { + if (next === "\\" || next === "\"") { ps.next(); return `\\${next}`; } @@ -731,7 +725,7 @@ export default class FluentParser { let ch; while ((ch = ps.takeChar(x => x !== '"' && x !== EOL))) { if (ch === "\\") { - val += this.getEscapeSequence(ps, ["{", "\\", "\""]); + val += this.getEscapeSequence(ps); } else { val += ch; } diff --git a/fluent-syntax/test/fixtures_behavior/escape_sequences.ftl b/fluent-syntax/test/fixtures_behavior/escape_sequences.ftl index d0bbb85ae..801923be2 100644 --- a/fluent-syntax/test/fixtures_behavior/escape_sequences.ftl +++ b/fluent-syntax/test/fixtures_behavior/escape_sequences.ftl @@ -1,10 +1,16 @@ -# ~ERROR E0025, pos 8, args "A" -key1 = \A +## Backslash is a regular character in text elements. +key01 = \A +key02 = \u0041 +key03 = \\u0041 +key04 = \u000z +key05 = \{Value} -# ~ERROR E0026, pos 23, args "000z" -key2 = \u000z +key06 = {"Escaped \" quote"} +key07 = {"Escaped \\ backslash"} +key08 = {"Escaped \u0041 A"} -key3 = \{Escaped} -key4 = {"Escaped \" quote"} -key5 = \u0041 -key6 = \\u0041 +# ~ERROR E0025, pos 232, args "A" +key09 = {"\A"} + +# ~ERROR E0026, pos 252, args "000z" +key10 = {"\u000z"} diff --git a/fluent-syntax/test/fixtures_reference/escaped_characters.ftl b/fluent-syntax/test/fixtures_reference/escaped_characters.ftl index d3a5a078f..3c64fcef7 100644 --- a/fluent-syntax/test/fixtures_reference/escaped_characters.ftl +++ b/fluent-syntax/test/fixtures_reference/escaped_characters.ftl @@ -1,9 +1,22 @@ -backslash = Value with \\ (an escaped backslash) -closing-brace = Value with \{ (a closing brace) -unicode-escape = \u0041 -escaped-unicode = \\u0041 +## Literal text +text-backslash-one = Value with \ a backslash +text-backslash-two = Value with \\ two backslashes +text-backslash-brace = Value with \{placeable} +text-backslash-u = \u0041 +text-backslash-backslash-u = \\u0041 -## String Expressions +## String literals quote-in-string = {"\""} backslash-in-string = {"\\"} +# ERROR Mismatched quote mismatched-quote = {"\\""} +# ERROR Unknown escape +unknown-escape = {"\x"} + +## Unicode escapes +string-unicode-sequence = {"\u0041"} +string-escaped-unicode = {"\\u0041"} + +## Literal braces +brace-open = An opening {"{"} brace. +brace-close = A closing } brace. diff --git a/fluent-syntax/test/fixtures_reference/escaped_characters.json b/fluent-syntax/test/fixtures_reference/escaped_characters.json index 560277517..6c26f82ee 100644 --- a/fluent-syntax/test/fixtures_reference/escaped_characters.json +++ b/fluent-syntax/test/fixtures_reference/escaped_characters.json @@ -1,18 +1,22 @@ { "type": "Resource", "body": [ + { + "type": "GroupComment", + "content": "Literal text" + }, { "type": "Message", "id": { "type": "Identifier", - "name": "backslash" + "name": "text-backslash-one" }, "value": { "type": "Pattern", "elements": [ { "type": "TextElement", - "value": "Value with \\\\ (an escaped backslash)" + "value": "Value with \\ a backslash" } ] }, @@ -23,14 +27,14 @@ "type": "Message", "id": { "type": "Identifier", - "name": "closing-brace" + "name": "text-backslash-two" }, "value": { "type": "Pattern", "elements": [ { "type": "TextElement", - "value": "Value with \\{ (a closing brace)" + "value": "Value with \\\\ two backslashes" } ] }, @@ -41,7 +45,35 @@ "type": "Message", "id": { "type": "Identifier", - "name": "unicode-escape" + "name": "text-backslash-brace" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value with \\" + }, + { + "type": "Placeable", + "expression": { + "type": "MessageReference", + "id": { + "type": "Identifier", + "name": "placeable" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "text-backslash-u" }, "value": { "type": "Pattern", @@ -59,7 +91,7 @@ "type": "Message", "id": { "type": "Identifier", - "name": "escaped-unicode" + "name": "text-backslash-backslash-u" }, "value": { "type": "Pattern", @@ -75,7 +107,7 @@ }, { "type": "GroupComment", - "content": "String Expressions" + "content": "String literals" }, { "type": "Message", @@ -119,10 +151,120 @@ "attributes": [], "comment": null }, + { + "type": "Comment", + "content": "ERROR Mismatched quote" + }, { "type": "Junk", "annotations": [], "content": "mismatched-quote = {\"\\\\\"\"}\n" + }, + { + "type": "Comment", + "content": "ERROR Unknown escape" + }, + { + "type": "Junk", + "annotations": [], + "content": "unknown-escape = {\"\\x\"}\n" + }, + { + "type": "GroupComment", + "content": "Unicode escapes" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "string-unicode-sequence" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "\\u0041" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "string-escaped-unicode" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "\\\\u0041" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "GroupComment", + "content": "Literal braces" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "brace-open" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "An opening " + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "{" + } + }, + { + "type": "TextElement", + "value": " brace." + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "brace-close" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "A closing } brace." + } + ] + }, + "attributes": [], + "comment": null } ] } diff --git a/fluent-syntax/test/fixtures_structure/escape_sequences.ftl b/fluent-syntax/test/fixtures_structure/escape_sequences.ftl index e91de86bd..3c64fcef7 100644 --- a/fluent-syntax/test/fixtures_structure/escape_sequences.ftl +++ b/fluent-syntax/test/fixtures_structure/escape_sequences.ftl @@ -1,9 +1,22 @@ -backslash = Value with \\ (an escaped backslash) -closing-brace = Value with \{ (an opening brace) -unicode-escape = \u0041 -escaped-unicode = \\u0041 +## Literal text +text-backslash-one = Value with \ a backslash +text-backslash-two = Value with \\ two backslashes +text-backslash-brace = Value with \{placeable} +text-backslash-u = \u0041 +text-backslash-backslash-u = \\u0041 -## String Expressions +## String literals quote-in-string = {"\""} backslash-in-string = {"\\"} +# ERROR Mismatched quote mismatched-quote = {"\\""} +# ERROR Unknown escape +unknown-escape = {"\x"} + +## Unicode escapes +string-unicode-sequence = {"\u0041"} +string-escaped-unicode = {"\\u0041"} + +## Literal braces +brace-open = An opening {"{"} brace. +brace-close = A closing } brace. diff --git a/fluent-syntax/test/fixtures_structure/escape_sequences.json b/fluent-syntax/test/fixtures_structure/escape_sequences.json index f49683f67..3259d024f 100644 --- a/fluent-syntax/test/fixtures_structure/escape_sequences.json +++ b/fluent-syntax/test/fixtures_structure/escape_sequences.json @@ -1,15 +1,24 @@ { "type": "Resource", "body": [ + { + "type": "GroupComment", + "content": "Literal text", + "span": { + "type": "Span", + "start": 0, + "end": 15 + } + }, { "type": "Message", "id": { "type": "Identifier", - "name": "backslash", + "name": "text-backslash-one", "span": { "type": "Span", - "start": 0, - "end": 9 + "start": 16, + "end": 34 } }, "value": { @@ -17,37 +26,37 @@ "elements": [ { "type": "TextElement", - "value": "Value with \\\\ (an escaped backslash)", + "value": "Value with \\ a backslash", "span": { "type": "Span", - "start": 12, - "end": 48 + "start": 37, + "end": 61 } } ], "span": { "type": "Span", - "start": 12, - "end": 48 + "start": 37, + "end": 61 } }, "attributes": [], "comment": null, "span": { "type": "Span", - "start": 0, - "end": 48 + "start": 16, + "end": 61 } }, { "type": "Message", "id": { "type": "Identifier", - "name": "closing-brace", + "name": "text-backslash-two", "span": { "type": "Span", - "start": 49, - "end": 62 + "start": 62, + "end": 80 } }, "value": { @@ -55,37 +64,100 @@ "elements": [ { "type": "TextElement", - "value": "Value with \\{ (an opening brace)", + "value": "Value with \\\\ two backslashes", "span": { "type": "Span", - "start": 65, - "end": 97 + "start": 83, + "end": 112 } } ], "span": { "type": "Span", - "start": 65, - "end": 97 + "start": 83, + "end": 112 } }, "attributes": [], "comment": null, "span": { "type": "Span", - "start": 49, - "end": 97 + "start": 62, + "end": 112 } }, { "type": "Message", "id": { "type": "Identifier", - "name": "unicode-escape", + "name": "text-backslash-brace", "span": { "type": "Span", - "start": 98, - "end": 112 + "start": 113, + "end": 133 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value with \\", + "span": { + "type": "Span", + "start": 136, + "end": 148 + } + }, + { + "type": "Placeable", + "expression": { + "type": "MessageReference", + "id": { + "type": "Identifier", + "name": "placeable", + "span": { + "type": "Span", + "start": 149, + "end": 158 + } + }, + "span": { + "type": "Span", + "start": 149, + "end": 158 + } + }, + "span": { + "type": "Span", + "start": 148, + "end": 159 + } + } + ], + "span": { + "type": "Span", + "start": 136, + "end": 159 + } + }, + "attributes": [], + "comment": null, + "span": { + "type": "Span", + "start": 113, + "end": 159 + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "text-backslash-u", + "span": { + "type": "Span", + "start": 160, + "end": 176 } }, "value": { @@ -96,34 +168,34 @@ "value": "\\u0041", "span": { "type": "Span", - "start": 115, - "end": 121 + "start": 179, + "end": 185 } } ], "span": { "type": "Span", - "start": 115, - "end": 121 + "start": 179, + "end": 185 } }, "attributes": [], "comment": null, "span": { "type": "Span", - "start": 98, - "end": 121 + "start": 160, + "end": 185 } }, { "type": "Message", "id": { "type": "Identifier", - "name": "escaped-unicode", + "name": "text-backslash-backslash-u", "span": { "type": "Span", - "start": 122, - "end": 137 + "start": 186, + "end": 212 } }, "value": { @@ -134,32 +206,32 @@ "value": "\\\\u0041", "span": { "type": "Span", - "start": 140, - "end": 147 + "start": 215, + "end": 222 } } ], "span": { "type": "Span", - "start": 140, - "end": 147 + "start": 215, + "end": 222 } }, "attributes": [], "comment": null, "span": { "type": "Span", - "start": 122, - "end": 147 + "start": 186, + "end": 222 } }, { "type": "GroupComment", - "content": "String Expressions", + "content": "String literals", "span": { "type": "Span", - "start": 149, - "end": 170 + "start": 224, + "end": 242 } }, { @@ -169,8 +241,8 @@ "name": "quote-in-string", "span": { "type": "Span", - "start": 171, - "end": 186 + "start": 243, + "end": 258 } }, "value": { @@ -183,29 +255,29 @@ "value": "\\\"", "span": { "type": "Span", - "start": 190, - "end": 194 + "start": 262, + "end": 266 } }, "span": { "type": "Span", - "start": 189, - "end": 195 + "start": 261, + "end": 267 } } ], "span": { "type": "Span", - "start": 189, - "end": 195 + "start": 261, + "end": 267 } }, "attributes": [], "comment": null, "span": { "type": "Span", - "start": 171, - "end": 195 + "start": 243, + "end": 267 } }, { @@ -215,8 +287,8 @@ "name": "backslash-in-string", "span": { "type": "Span", - "start": 196, - "end": 215 + "start": 268, + "end": 287 } }, "value": { @@ -229,29 +301,38 @@ "value": "\\\\", "span": { "type": "Span", - "start": 219, - "end": 223 + "start": 291, + "end": 295 } }, "span": { "type": "Span", - "start": 218, - "end": 224 + "start": 290, + "end": 296 } } ], "span": { "type": "Span", - "start": 218, - "end": 224 + "start": 290, + "end": 296 } }, "attributes": [], "comment": null, "span": { "type": "Span", - "start": 196, - "end": 224 + "start": 268, + "end": 296 + } + }, + { + "type": "Comment", + "content": "ERROR Mismatched quote", + "span": { + "type": "Span", + "start": 297, + "end": 321 } }, { @@ -266,22 +347,267 @@ "message": "Expected token: \"}\"", "span": { "type": "Span", - "start": 249, - "end": 249 + "start": 346, + "end": 346 } } ], "content": "mismatched-quote = {\"\\\\\"\"}\n", "span": { "type": "Span", - "start": 225, - "end": 252 + "start": 322, + "end": 349 + } + }, + { + "type": "Comment", + "content": "ERROR Unknown escape", + "span": { + "type": "Span", + "start": 349, + "end": 371 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0025", + "args": [ + "x" + ], + "message": "Unknown escape sequence: \\x.", + "span": { + "type": "Span", + "start": 392, + "end": 392 + } + } + ], + "content": "unknown-escape = {\"\\x\"}\n\n", + "span": { + "type": "Span", + "start": 372, + "end": 397 + } + }, + { + "type": "GroupComment", + "content": "Unicode escapes", + "span": { + "type": "Span", + "start": 397, + "end": 415 + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "string-unicode-sequence", + "span": { + "type": "Span", + "start": 416, + "end": 439 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "\\u0041", + "span": { + "type": "Span", + "start": 443, + "end": 451 + } + }, + "span": { + "type": "Span", + "start": 442, + "end": 452 + } + } + ], + "span": { + "type": "Span", + "start": 442, + "end": 452 + } + }, + "attributes": [], + "comment": null, + "span": { + "type": "Span", + "start": 416, + "end": 452 + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "string-escaped-unicode", + "span": { + "type": "Span", + "start": 453, + "end": 475 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "\\\\u0041", + "span": { + "type": "Span", + "start": 479, + "end": 488 + } + }, + "span": { + "type": "Span", + "start": 478, + "end": 489 + } + } + ], + "span": { + "type": "Span", + "start": 478, + "end": 489 + } + }, + "attributes": [], + "comment": null, + "span": { + "type": "Span", + "start": 453, + "end": 489 + } + }, + { + "type": "GroupComment", + "content": "Literal braces", + "span": { + "type": "Span", + "start": 491, + "end": 508 + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "brace-open", + "span": { + "type": "Span", + "start": 509, + "end": 519 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "An opening ", + "span": { + "type": "Span", + "start": 522, + "end": 533 + } + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "{", + "span": { + "type": "Span", + "start": 534, + "end": 537 + } + }, + "span": { + "type": "Span", + "start": 533, + "end": 538 + } + }, + { + "type": "TextElement", + "value": " brace.", + "span": { + "type": "Span", + "start": 538, + "end": 545 + } + } + ], + "span": { + "type": "Span", + "start": 522, + "end": 545 + } + }, + "attributes": [], + "comment": null, + "span": { + "type": "Span", + "start": 509, + "end": 545 + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "brace-close", + "span": { + "type": "Span", + "start": 546, + "end": 557 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "A closing } brace.", + "span": { + "type": "Span", + "start": 560, + "end": 578 + } + } + ], + "span": { + "type": "Span", + "start": 560, + "end": 578 + } + }, + "attributes": [], + "comment": null, + "span": { + "type": "Span", + "start": 546, + "end": 578 } } ], "span": { "type": "Span", "start": 0, - "end": 252 + "end": 579 } } diff --git a/fluent-syntax/test/serializer_test.js b/fluent-syntax/test/serializer_test.js index d4b775db2..cb32b9caa 100644 --- a/fluent-syntax/test/serializer_test.js +++ b/fluent-syntax/test/serializer_test.js @@ -457,9 +457,9 @@ suite("Serialize resource", function() { assert.equal(pretty(input), input); }); - test("Escaped special char in TextElement", function() { + test("Backslash in TextElement", function() { const input = ftl` - foo = \\{Escaped} + foo = \\{ placeable } `; assert.equal(pretty(input), input); }); @@ -473,7 +473,7 @@ suite("Serialize resource", function() { test("Unicode escape sequence", function() { const input = ftl` - foo = \\u0065 + foo = { "\\u0065" } `; assert.equal(pretty(input), input); }); diff --git a/fluent/src/resource.js b/fluent/src/resource.js index 62025ad8c..19ea001cc 100644 --- a/fluent/src/resource.js +++ b/fluent/src/resource.js @@ -15,19 +15,17 @@ const RE_IDENTIFIER = /(-?[a-zA-Z][a-zA-Z0-9_-]*)/y; const RE_NUMBER_LITERAL = /(-?[0-9]+(\.[0-9]+)?)/y; // A "run" is a sequence of text or string literal characters which don't -// require any special handling. For TextElements such special characters are: -// { (starts a placeable), \ (starts an escape sequence), and line breaks which -// require additional logic to check if the next line is indented. For -// StringLiterals they are: \ (starts an escape sequence), " (ends the -// literal), and line breaks which are not allowed in StringLiterals. Also note -// that string runs may be empty, but text runs may not. -const RE_TEXT_RUN = /([^\\{\n\r]+)/y; +// require any special handling. For TextElements such special characters are: { +// (starts a placeable), and line breaks which require additional logic to check +// if the next line is indented. For StringLiterals they are: \ (starts an +// escape sequence), " (ends the literal), and line breaks which are not allowed +// in StringLiterals. Note that string runs may be empty; text runs may not. +const RE_TEXT_RUN = /([^{\n\r]+)/y; const RE_STRING_RUN = /([^\\"\n\r]*)/y; // Escape sequences. const RE_UNICODE_ESCAPE = /\\u([a-fA-F0-9]{4})/y; const RE_STRING_ESCAPE = /\\([\\"])/y; -const RE_TEXT_ESCAPE = /\\([\\{])/y; // Used for trimming TextElements and indents. With the /m flag, the $ matches // the end of every line. @@ -180,15 +178,12 @@ export default class FluentResource extends Map { var first = match(RE_TEXT_RUN); } - // If there's a backslash escape or a placeable on the first line, fall - // back to parsing a complex pattern. - switch (source[cursor]) { - case "{": - case "\\": - return first - // Re-use the text parsed above, if possible. - ? parsePatternElements(first) - : parsePatternElements(); + // If there's a placeable on the first line, parse a complex pattern. + if (source[cursor] === "{") { + return first + // Re-use the text parsed above, if possible. + ? parsePatternElements(first) + : parsePatternElements(); } // RE_TEXT_VALUE stops at newlines. Only continue parsing the pattern if @@ -240,12 +235,6 @@ export default class FluentResource extends Map { continue; } - if (source[cursor] === "\\") { - elements.push(parseEscapeSequence(RE_TEXT_ESCAPE)); - needsTrimming = false; - continue; - } - break; } @@ -403,7 +392,7 @@ export default class FluentResource extends Map { value += match(RE_STRING_RUN); if (source[cursor] === "\\") { - value += parseEscapeSequence(RE_STRING_ESCAPE); + value += parseEscapeSequence(); continue; } @@ -417,7 +406,7 @@ export default class FluentResource extends Map { } // Unescape known escape sequences. - function parseEscapeSequence(reSpecialized) { + function parseEscapeSequence() { if (test(RE_UNICODE_ESCAPE)) { let sequence = match(RE_UNICODE_ESCAPE); let codepoint = parseInt(sequence, 16); @@ -429,8 +418,8 @@ export default class FluentResource extends Map { : "๏ฟฝ"; } - if (test(reSpecialized)) { - return match(reSpecialized); + if (test(RE_STRING_ESCAPE)) { + return match(RE_STRING_ESCAPE); } throw new FluentError("Unknown escape sequence"); diff --git a/fluent/test/fixtures_behavior/escape_sequences.json b/fluent/test/fixtures_behavior/escape_sequences.json index 638f1bd08..95c0834db 100644 --- a/fluent/test/fixtures_behavior/escape_sequences.json +++ b/fluent/test/fixtures_behavior/escape_sequences.json @@ -1,14 +1,26 @@ { - "key3": { + "key01": { "value": true }, - "key4": { + "key02": { "value": true }, - "key5": { + "key03": { "value": true }, - "key6": { + "key04": { + "value": true + }, + "key05": { + "value": true + }, + "key06": { + "value": true + }, + "key07": { + "value": true + }, + "key08": { "value": true } } diff --git a/fluent/test/fixtures_reference/astral.json b/fluent/test/fixtures_reference/astral.json index 9a9469f0c..5c4850b60 100644 --- a/fluent/test/fixtures_reference/astral.json +++ b/fluent/test/fixtures_reference/astral.json @@ -1,10 +1,7 @@ { "face-with-tears-of-joy": "๐Ÿ˜‚", "tetragram-for-centre": "๐Œ†", - "surrogates-in-text": [ - "๏ฟฝ", - "๏ฟฝ" - ], + "surrogates-in-text": "\\uD83D\\uDE02", "surrogates-in-string": [ "๏ฟฝ๏ฟฝ" ], diff --git a/fluent/test/fixtures_reference/escaped_characters.json b/fluent/test/fixtures_reference/escaped_characters.json index 1b358bc60..c371cab9f 100644 --- a/fluent/test/fixtures_reference/escaped_characters.json +++ b/fluent/test/fixtures_reference/escaped_characters.json @@ -1,25 +1,31 @@ { - "backslash": [ - "Value with ", - "\\", - " (an escaped backslash)" - ], - "closing-brace": [ - "Value with ", - "{", - " (a closing brace)" - ], - "unicode-escape": [ - "A" - ], - "escaped-unicode": [ - "\\", - "u0041" + "text-backslash-one": "Value with \\ a backslash", + "text-backslash-two": "Value with \\\\ two backslashes", + "text-backslash-brace": [ + "Value with \\", + { + "type": "ref", + "name": "placeable" + } ], + "text-backslash-u": "\\u0041", + "text-backslash-backslash-u": "\\\\u0041", "quote-in-string": [ "\"" ], "backslash-in-string": [ "\\" - ] + ], + "string-unicode-sequence": [ + "A" + ], + "string-escaped-unicode": [ + "\\u0041" + ], + "brace-open": [ + "An opening ", + "{", + " brace." + ], + "brace-close": "A closing } brace." } diff --git a/fluent/test/fixtures_structure/escape_sequences.json b/fluent/test/fixtures_structure/escape_sequences.json index 22d8f1cd4..c371cab9f 100644 --- a/fluent/test/fixtures_structure/escape_sequences.json +++ b/fluent/test/fixtures_structure/escape_sequences.json @@ -1,25 +1,31 @@ { - "backslash": [ - "Value with ", - "\\", - " (an escaped backslash)" - ], - "closing-brace": [ - "Value with ", - "{", - " (an opening brace)" - ], - "unicode-escape": [ - "A" - ], - "escaped-unicode": [ - "\\", - "u0041" + "text-backslash-one": "Value with \\ a backslash", + "text-backslash-two": "Value with \\\\ two backslashes", + "text-backslash-brace": [ + "Value with \\", + { + "type": "ref", + "name": "placeable" + } ], + "text-backslash-u": "\\u0041", + "text-backslash-backslash-u": "\\\\u0041", "quote-in-string": [ "\"" ], "backslash-in-string": [ "\\" - ] + ], + "string-unicode-sequence": [ + "A" + ], + "string-escaped-unicode": [ + "\\u0041" + ], + "brace-open": [ + "An opening ", + "{", + " brace." + ], + "brace-close": "A closing } brace." } From 1758b959e167ba307fdb95332118a73d9e7255fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Fri, 16 Nov 2018 13:45:36 +0100 Subject: [PATCH 04/23] End comment lines on EOLs recognized by JS RegExp --- fluent-syntax/.gitattributes | 1 + fluent-syntax/src/parser.js | 8 +++-- fluent-syntax/test/fixtures_reference/cr.ftl | 1 + fluent-syntax/test/fixtures_reference/cr.json | 10 ++++++ .../test/fixtures_reference/crlf.ftl | 11 +++++-- .../test/fixtures_reference/crlf.json | 32 +++++++++++++++++-- fluent/test/fixtures_reference/cr.json | 23 +++++++++++++ fluent/test/fixtures_reference/crlf.json | 22 ++++++++++--- 8 files changed, 97 insertions(+), 11 deletions(-) create mode 100644 fluent-syntax/test/fixtures_reference/cr.ftl create mode 100644 fluent-syntax/test/fixtures_reference/cr.json create mode 100644 fluent/test/fixtures_reference/cr.json diff --git a/fluent-syntax/.gitattributes b/fluent-syntax/.gitattributes index 9e0125713..dc3c32a74 100644 --- a/fluent-syntax/.gitattributes +++ b/fluent-syntax/.gitattributes @@ -1,2 +1,3 @@ test/fixtures_reference/crlf.ftl eol=crlf +test/fixtures_reference/cr.ftl eol=cr test/fixtures_structure/crlf.ftl eol=crlf diff --git a/fluent-syntax/src/parser.js b/fluent-syntax/src/parser.js index 14066cbda..bb4f60311 100644 --- a/fluent-syntax/src/parser.js +++ b/fluent-syntax/src/parser.js @@ -6,6 +6,10 @@ import { ParseError } from "./errors"; const trailingWSRe = /[ \t\n\r]+$/; +// The Fluent Syntax spec uses /.*/ to parse comment lines. It matches all +// characters except the following ones, which are considered line endings by +// the regex engine. +const COMMENT_EOL = ["\n", "\r", "\u2028", "\u2029"]; function withSpan(fn) { @@ -188,10 +192,10 @@ export default class FluentParser { level = i; } - if (ps.currentChar !== EOL) { + if (!COMMENT_EOL.includes(ps.currentChar)) { ps.expectChar(" "); let ch; - while ((ch = ps.takeChar(x => x !== EOL))) { + while ((ch = ps.takeChar(x => !COMMENT_EOL.includes(x)))) { content += ch; } } diff --git a/fluent-syntax/test/fixtures_reference/cr.ftl b/fluent-syntax/test/fixtures_reference/cr.ftl new file mode 100644 index 000000000..549c662ae --- /dev/null +++ b/fluent-syntax/test/fixtures_reference/cr.ftl @@ -0,0 +1 @@ +### This entire file uses CR as EOL. err01 = Value 01 err02 = Value 02 err03 = Value 03 Continued .title = Title err04 = { "str err05 = { $sel -> } \ No newline at end of file diff --git a/fluent-syntax/test/fixtures_reference/cr.json b/fluent-syntax/test/fixtures_reference/cr.json new file mode 100644 index 000000000..afec8119a --- /dev/null +++ b/fluent-syntax/test/fixtures_reference/cr.json @@ -0,0 +1,10 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Junk", + "annotations": [], + "content": "### This entire file uses CR as EOL.\r\rerr01 = Value 01\rerr02 = Value 02\r\rerr03 =\r\r Value 03\r Continued\r\r .title = Title\r\rerr04 = { \"str\r\rerr05 = { $sel -> }\r" + } + ] +} diff --git a/fluent-syntax/test/fixtures_reference/crlf.ftl b/fluent-syntax/test/fixtures_reference/crlf.ftl index 7349195be..df3a02c5f 100644 --- a/fluent-syntax/test/fixtures_reference/crlf.ftl +++ b/fluent-syntax/test/fixtures_reference/crlf.ftl @@ -1,7 +1,14 @@ + key01 = Value 01 key02 = + Value 02 Continued -# ERROR (Missing value or attributes) -key03 + .title = Title + +# ERROR Unclosed StringLiteral +err03 = { "str + +# ERROR Missing newline after ->. +err04 = { $sel -> } diff --git a/fluent-syntax/test/fixtures_reference/crlf.json b/fluent-syntax/test/fixtures_reference/crlf.json index a36838a0b..6b324cd4e 100644 --- a/fluent-syntax/test/fixtures_reference/crlf.json +++ b/fluent-syntax/test/fixtures_reference/crlf.json @@ -34,17 +34,43 @@ } ] }, - "attributes": [], + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "title" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Title" + } + ] + } + } + ], "comment": null }, { "type": "Comment", - "content": "ERROR (Missing value or attributes)" + "content": "ERROR Unclosed StringLiteral" + }, + { + "type": "Junk", + "annotations": [], + "content": "err03 = { \"str\r\n" + }, + { + "type": "Comment", + "content": "ERROR Missing newline after ->." }, { "type": "Junk", "annotations": [], - "content": "key03\n" + "content": "err04 = { $sel -> }\r\n" } ] } diff --git a/fluent/test/fixtures_reference/cr.json b/fluent/test/fixtures_reference/cr.json new file mode 100644 index 000000000..6c6f39082 --- /dev/null +++ b/fluent/test/fixtures_reference/cr.json @@ -0,0 +1,23 @@ +{ + "err01": "Value 01", + "err02": "Value 02", + "err03": { + "value": [ + "Value 03", + "\r", + "Continued" + ], + "attrs": { + "title": "Title" + } + }, + "err05": [ + { + "type": "select", + "selector": { + "type": "var", + "name": "sel" + } + } + ] +} diff --git a/fluent/test/fixtures_reference/crlf.json b/fluent/test/fixtures_reference/crlf.json index 65d3a327e..aa9ccd77c 100644 --- a/fluent/test/fixtures_reference/crlf.json +++ b/fluent/test/fixtures_reference/crlf.json @@ -1,8 +1,22 @@ { "key01": "Value 01", - "key02": [ - "Value 02", - "\n", - "Continued" + "key02": { + "value": [ + "Value 02", + "\n", + "Continued" + ], + "attrs": { + "title": "Title" + } + }, + "err04": [ + { + "type": "select", + "selector": { + "type": "var", + "name": "sel" + } + } ] } From eb0493b3293b58452c18bb28a8571b51f67beec0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Fri, 16 Nov 2018 14:13:14 +0100 Subject: [PATCH 05/23] Forbid the closing brace in text --- fluent-syntax/src/errors.js | 2 ++ fluent-syntax/src/parser.js | 4 ++- .../placeable_in_placeable.ftl | 3 +- .../fixtures_reference/escaped_characters.ftl | 2 +- .../escaped_characters.json | 13 ++++++- .../test/fixtures_reference/placeables.ftl | 12 +++++++ .../test/fixtures_reference/placeables.json | 36 +++++++++++++++++++ .../fixtures_reference/select_expressions.ftl | 5 +++ .../select_expressions.json | 9 +++++ .../fixtures_structure/escape_sequences.ftl | 2 +- .../fixtures_structure/escape_sequences.json | 36 ++++++++++++++++--- fluent/src/resource.js | 8 +++-- .../placeable_in_placeable.json | 3 -- .../escaped_characters.json | 6 +++- .../fixtures_structure/escape_sequences.json | 6 +++- 15 files changed, 130 insertions(+), 17 deletions(-) diff --git a/fluent-syntax/src/errors.js b/fluent-syntax/src/errors.js index 81caf7c73..f721d25a4 100644 --- a/fluent-syntax/src/errors.js +++ b/fluent-syntax/src/errors.js @@ -74,6 +74,8 @@ function getErrorMessage(code, args) { const [char] = args; return `Invalid Unicode escape sequence: \\u${char}.`; } + case "E0027": + return "Unbalanced closing brace in TextElement."; default: return code; } diff --git a/fluent-syntax/src/parser.js b/fluent-syntax/src/parser.js index bb4f60311..1f8651a66 100644 --- a/fluent-syntax/src/parser.js +++ b/fluent-syntax/src/parser.js @@ -452,6 +452,8 @@ export default class FluentParser { if (ch === "{") { const element = this.getPlaceable(ps); elements.push(element); + } else if (ch === "}") { + throw new ParseError("E0027"); } else { const element = this.getTextElement(ps); elements.push(element); @@ -475,7 +477,7 @@ export default class FluentParser { let ch; while ((ch = ps.currentChar)) { - if (ch === "{") { + if (ch === "{" || ch === "}") { return new AST.TextElement(buffer); } diff --git a/fluent-syntax/test/fixtures_behavior/placeable_in_placeable.ftl b/fluent-syntax/test/fixtures_behavior/placeable_in_placeable.ftl index 7ece456be..827c93e72 100644 --- a/fluent-syntax/test/fixtures_behavior/placeable_in_placeable.ftl +++ b/fluent-syntax/test/fixtures_behavior/placeable_in_placeable.ftl @@ -7,8 +7,9 @@ key2 = { { foo } } # { foo } # } -key4 = { { foo } # ~ERROR E0003, pos 96, args "}" +key4 = { { foo } +# ~ERROR E0027, pos 111 key5 = { foo } } diff --git a/fluent-syntax/test/fixtures_reference/escaped_characters.ftl b/fluent-syntax/test/fixtures_reference/escaped_characters.ftl index 3c64fcef7..5242a4bcb 100644 --- a/fluent-syntax/test/fixtures_reference/escaped_characters.ftl +++ b/fluent-syntax/test/fixtures_reference/escaped_characters.ftl @@ -19,4 +19,4 @@ string-escaped-unicode = {"\\u0041"} ## Literal braces brace-open = An opening {"{"} brace. -brace-close = A closing } brace. +brace-close = A closing {"}"} brace. diff --git a/fluent-syntax/test/fixtures_reference/escaped_characters.json b/fluent-syntax/test/fixtures_reference/escaped_characters.json index 6c26f82ee..26b1974b0 100644 --- a/fluent-syntax/test/fixtures_reference/escaped_characters.json +++ b/fluent-syntax/test/fixtures_reference/escaped_characters.json @@ -259,7 +259,18 @@ "elements": [ { "type": "TextElement", - "value": "A closing } brace." + "value": "A closing " + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "}" + } + }, + { + "type": "TextElement", + "value": " brace." } ] }, diff --git a/fluent-syntax/test/fixtures_reference/placeables.ftl b/fluent-syntax/test/fixtures_reference/placeables.ftl index c0e515b67..7a1b280f4 100644 --- a/fluent-syntax/test/fixtures_reference/placeables.ftl +++ b/fluent-syntax/test/fixtures_reference/placeables.ftl @@ -1,3 +1,15 @@ nested-placeable = {{{1}}} padded-placeable = { 1 } sparse-placeable = { { 1 } } + +# ERROR Unmatched opening brace +unmatched-open1 = { 1 + +# ERROR Unmatched opening brace +unmatched-open2 = {{ 1 } + +# ERROR Unmatched closing brace +unmatched-close1 = 1 } + +# ERROR Unmatched closing brace +unmatched-close2 = { 1 }} diff --git a/fluent-syntax/test/fixtures_reference/placeables.json b/fluent-syntax/test/fixtures_reference/placeables.json index c0e7cc9e1..ff7fac8a7 100644 --- a/fluent-syntax/test/fixtures_reference/placeables.json +++ b/fluent-syntax/test/fixtures_reference/placeables.json @@ -72,6 +72,42 @@ }, "attributes": [], "comment": null + }, + { + "type": "Comment", + "content": "ERROR Unmatched opening brace" + }, + { + "type": "Junk", + "annotations": [], + "content": "unmatched-open1 = { 1\n" + }, + { + "type": "Comment", + "content": "ERROR Unmatched opening brace" + }, + { + "type": "Junk", + "annotations": [], + "content": "unmatched-open2 = {{ 1 }\n" + }, + { + "type": "Comment", + "content": "ERROR Unmatched closing brace" + }, + { + "type": "Junk", + "annotations": [], + "content": "unmatched-close1 = 1 }\n" + }, + { + "type": "Comment", + "content": "ERROR Unmatched closing brace" + }, + { + "type": "Junk", + "annotations": [], + "content": "unmatched-close2 = { 1 }}\n" } ] } diff --git a/fluent-syntax/test/fixtures_reference/select_expressions.ftl b/fluent-syntax/test/fixtures_reference/select_expressions.ftl index 3c54f57c2..859c01a04 100644 --- a/fluent-syntax/test/fixtures_reference/select_expressions.ftl +++ b/fluent-syntax/test/fixtures_reference/select_expressions.ftl @@ -34,3 +34,8 @@ nested-variant-list = *[two] Value } } + +# ERROR Missing line end after variant list +missing-line-end = + { 1 -> + *[one] One} diff --git a/fluent-syntax/test/fixtures_reference/select_expressions.json b/fluent-syntax/test/fixtures_reference/select_expressions.json index 13f5b71bc..4c782006a 100644 --- a/fluent-syntax/test/fixtures_reference/select_expressions.json +++ b/fluent-syntax/test/fixtures_reference/select_expressions.json @@ -257,6 +257,15 @@ "type": "Junk", "annotations": [], "content": "nested-variant-list =\n { 1 ->\n *[one] {\n *[two] Value\n }\n }\n" + }, + { + "type": "Comment", + "content": "ERROR Missing line end after variant list" + }, + { + "type": "Junk", + "annotations": [], + "content": "missing-line-end =\n { 1 ->\n *[one] One}\n" } ] } diff --git a/fluent-syntax/test/fixtures_structure/escape_sequences.ftl b/fluent-syntax/test/fixtures_structure/escape_sequences.ftl index 3c64fcef7..5242a4bcb 100644 --- a/fluent-syntax/test/fixtures_structure/escape_sequences.ftl +++ b/fluent-syntax/test/fixtures_structure/escape_sequences.ftl @@ -19,4 +19,4 @@ string-escaped-unicode = {"\\u0041"} ## Literal braces brace-open = An opening {"{"} brace. -brace-close = A closing } brace. +brace-close = A closing {"}"} brace. diff --git a/fluent-syntax/test/fixtures_structure/escape_sequences.json b/fluent-syntax/test/fixtures_structure/escape_sequences.json index 3259d024f..e9e851c7b 100644 --- a/fluent-syntax/test/fixtures_structure/escape_sequences.json +++ b/fluent-syntax/test/fixtures_structure/escape_sequences.json @@ -582,18 +582,44 @@ "elements": [ { "type": "TextElement", - "value": "A closing } brace.", + "value": "A closing ", "span": { "type": "Span", "start": 560, - "end": 578 + "end": 570 + } + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "}", + "span": { + "type": "Span", + "start": 571, + "end": 574 + } + }, + "span": { + "type": "Span", + "start": 570, + "end": 575 + } + }, + { + "type": "TextElement", + "value": " brace.", + "span": { + "type": "Span", + "start": 575, + "end": 582 } } ], "span": { "type": "Span", "start": 560, - "end": 578 + "end": 582 } }, "attributes": [], @@ -601,13 +627,13 @@ "span": { "type": "Span", "start": 546, - "end": 578 + "end": 582 } } ], "span": { "type": "Span", "start": 0, - "end": 579 + "end": 583 } } diff --git a/fluent/src/resource.js b/fluent/src/resource.js index 19ea001cc..b457a38ec 100644 --- a/fluent/src/resource.js +++ b/fluent/src/resource.js @@ -20,7 +20,7 @@ const RE_NUMBER_LITERAL = /(-?[0-9]+(\.[0-9]+)?)/y; // if the next line is indented. For StringLiterals they are: \ (starts an // escape sequence), " (ends the literal), and line breaks which are not allowed // in StringLiterals. Note that string runs may be empty; text runs may not. -const RE_TEXT_RUN = /([^{\n\r]+)/y; +const RE_TEXT_RUN = /([^{}\n\r]+)/y; const RE_STRING_RUN = /([^\\"\n\r]*)/y; // Escape sequences. @@ -179,7 +179,7 @@ export default class FluentResource extends Map { } // If there's a placeable on the first line, parse a complex pattern. - if (source[cursor] === "{") { + if (source[cursor] === "{" || source[cursor] === "}") { return first // Re-use the text parsed above, if possible. ? parsePatternElements(first) @@ -228,6 +228,10 @@ export default class FluentResource extends Map { continue; } + if (source[cursor] === "}") { + throw new FluentError("Unbalanced closing brace"); + } + let indent = parseIndent(); if (indent) { elements.push(trim(indent)); diff --git a/fluent/test/fixtures_behavior/placeable_in_placeable.json b/fluent/test/fixtures_behavior/placeable_in_placeable.json index 824346581..0e54ab88d 100644 --- a/fluent/test/fixtures_behavior/placeable_in_placeable.json +++ b/fluent/test/fixtures_behavior/placeable_in_placeable.json @@ -4,8 +4,5 @@ }, "key2": { "value": true - }, - "key5": { - "value": true } } diff --git a/fluent/test/fixtures_reference/escaped_characters.json b/fluent/test/fixtures_reference/escaped_characters.json index c371cab9f..cc1619930 100644 --- a/fluent/test/fixtures_reference/escaped_characters.json +++ b/fluent/test/fixtures_reference/escaped_characters.json @@ -27,5 +27,9 @@ "{", " brace." ], - "brace-close": "A closing } brace." + "brace-close": [ + "A closing ", + "}", + " brace." + ] } diff --git a/fluent/test/fixtures_structure/escape_sequences.json b/fluent/test/fixtures_structure/escape_sequences.json index c371cab9f..cc1619930 100644 --- a/fluent/test/fixtures_structure/escape_sequences.json +++ b/fluent/test/fixtures_structure/escape_sequences.json @@ -27,5 +27,9 @@ "{", " brace." ], - "brace-close": "A closing } brace." + "brace-close": [ + "A closing ", + "}", + " brace." + ] } From b449a668ea6ba4266a4b67c97a260eeb72cd0867 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Wed, 28 Nov 2018 10:09:56 +0100 Subject: [PATCH 06/23] Syntax 0.8, part 2: Dedent multiline text to preserve content indentation (#309) * Dedent fluent-syntax/src * Dedent fluent-syntax/test * Dedent fluent/src * Dedent fluent/test * Refactor skipBlank* methods to use peekBlank* * Refactor maybeGetValue to take allowVariantList --- eslint_src.json | 1 - fluent-syntax/src/errors.js | 2 - fluent-syntax/src/parser.js | 283 +++++++++++------- fluent-syntax/src/stream.js | 155 ++++------ .../select_expression_without_variants.ftl | 2 +- .../selector_expression_ends_abruptly.ftl | 2 +- .../test/fixtures_behavior/variant_lists.ftl | 4 +- .../fixtures_reference/multiline_values.ftl | 25 ++ .../fixtures_reference/multiline_values.json | 106 ++++++- .../test/fixtures_structure/blank_lines.json | 6 +- .../test/fixtures_structure/crlf.json | 2 +- .../test/fixtures_structure/leading_dots.json | 14 +- .../fixtures_structure/multiline_pattern.json | 6 +- .../fixtures_structure/placeable_at_eol.json | 4 +- .../fixtures_structure/sparse-messages.json | 6 +- .../test/fixtures_structure/term.json | 6 +- .../variant_with_empty_pattern.json | 4 +- fluent/src/resolver.js | 2 + fluent/src/resource.js | 88 +++--- .../fixtures_reference/call_expressions.json | 60 +++- fluent/test/fixtures_reference/cr.json | 1 + fluent/test/fixtures_reference/messages.json | 4 +- .../fixtures_reference/multiline_values.json | 40 ++- .../select_expressions.json | 5 +- .../fixtures_reference/select_indent.json | 3 +- fluent/test/fixtures_reference/terms.json | 4 +- .../variant_with_empty_pattern.json | 4 +- .../whitespace_leading.json | 1 - 28 files changed, 539 insertions(+), 301 deletions(-) diff --git a/eslint_src.json b/eslint_src.json index 328febcee..dc1c72151 100644 --- a/eslint_src.json +++ b/eslint_src.json @@ -48,7 +48,6 @@ "no-extend-native": 2, "no-global-assign": 2, "no-extra-bind": 2, - "no-redeclare": 2, "array-bracket-spacing": 2, "brace-style": [ 1, diff --git a/fluent-syntax/src/errors.js b/fluent-syntax/src/errors.js index f721d25a4..b13c301e3 100644 --- a/fluent-syntax/src/errors.js +++ b/fluent-syntax/src/errors.js @@ -62,8 +62,6 @@ function getErrorMessage(code, args) { return "Positional arguments must not follow named arguments"; case "E0022": return "Named arguments must be unique"; - case "E0023": - return "VariantLists are only allowed inside of other VariantLists."; case "E0024": return "Cannot access variants of a message."; case "E0025": { diff --git a/fluent-syntax/src/parser.js b/fluent-syntax/src/parser.js index 1f8651a66..965a45aa7 100644 --- a/fluent-syntax/src/parser.js +++ b/fluent-syntax/src/parser.js @@ -43,7 +43,7 @@ export default class FluentParser { // Poor man's decorators. const methodNames = [ "getComment", "getMessage", "getTerm", "getAttribute", "getIdentifier", - "getVariant", "getNumber", "getValue", "getPattern", "getVariantList", + "getVariant", "getNumber", "getPattern", "getVariantList", "getTextElement", "getPlaceable", "getExpression", "getSelectorExpression", "getCallArg", "getString", "getLiteral" ]; @@ -68,7 +68,9 @@ export default class FluentParser { // they should parse as standalone when they're followed by Junk. // Consequently, we only attach Comments once we know that the Message // or the Term parsed successfully. - if (entry.type === "Comment" && blankLines === 0 && ps.currentChar) { + if (entry.type === "Comment" + && blankLines.length === 0 + && ps.currentChar) { // Stash the comment and decide what to do with it in the next pass. lastComment = entry; continue; @@ -200,7 +202,7 @@ export default class FluentParser { } } - if (ps.isNextLineComment(level, {skip: false})) { + if (ps.isNextLineComment(level)) { content += ps.currentChar; ps.next(); } else { @@ -229,19 +231,14 @@ export default class FluentParser { ps.skipBlankInline(); ps.expectChar("="); - if (ps.isValueStart({skip: true})) { - var pattern = this.getPattern(ps); - } - - if (ps.isNextLineAttributeStart({skip: true})) { - var attrs = this.getAttributes(ps); - } + const value = this.maybeGetValue(ps, {allowVariantList: false}); + const attrs = this.getAttributes(ps); - if (pattern === undefined && attrs === undefined) { + if (value === null && attrs.length === 0) { throw new ParseError("E0005", id.name); } - return new AST.Message(id, pattern, attrs); + return new AST.Message(id, value, attrs); } getTerm(ps) { @@ -251,16 +248,16 @@ export default class FluentParser { ps.skipBlankInline(); ps.expectChar("="); - if (ps.isValueStart({skip: true})) { - var value = this.getValue(ps); - } else { + // XXX Once https://github.com/projectfluent/fluent/pull/220 lands, + // getTerm will be the only place where VariantLists are still legal. Move + // the code from getPatternOrVariantList up to here then, and remove the + // allowVariantList switch. + const value = this.maybeGetValue(ps, {allowVariantList: true}); + if (value === null) { throw new ParseError("E0006", id.name); } - if (ps.isNextLineAttributeStart({skip: true})) { - var attrs = this.getAttributes(ps); - } - + const attrs = this.getAttributes(ps); return new AST.Term(id, value, attrs); } @@ -272,24 +269,22 @@ export default class FluentParser { ps.skipBlankInline(); ps.expectChar("="); - if (ps.isValueStart({skip: true})) { - const value = this.getPattern(ps); - return new AST.Attribute(key, value); + const value = this.maybeGetValue(ps, {allowVariantList: false}); + if (value === null) { + throw new ParseError("E0012"); } - throw new ParseError("E0012"); + return new AST.Attribute(key, value); } getAttributes(ps) { const attrs = []; - - while (true) { + ps.peekBlank(); + while (ps.isAttributeStart()) { + ps.skipToPeek(); const attr = this.getAttribute(ps); attrs.push(attr); - - if (!ps.isNextLineAttributeStart({skip: true})) { - break; - } + ps.peekBlank(); } return attrs; } @@ -321,7 +316,7 @@ export default class FluentParser { return this.getIdentifier(ps); } - getVariant(ps, hasDefault) { + getVariant(ps, {hasDefault, allowVariantList}) { let defaultIndex = false; if (ps.currentChar === "*") { @@ -340,36 +335,39 @@ export default class FluentParser { const key = this.getVariantKey(ps); ps.skipBlank(); - ps.expectChar("]"); - if (ps.isValueStart({skip: true})) { - const value = this.getValue(ps); - return new AST.Variant(key, value, defaultIndex); + // XXX We need to pass allowVariantList all the way down to here because + // nested VariantLists in Terms are legal for now. + const value = this.maybeGetValue(ps, {allowVariantList}); + if (value === null) { + throw new ParseError("E0012"); } - throw new ParseError("E0012"); + return new AST.Variant(key, value, defaultIndex); } - getVariants(ps) { + getVariants(ps, {allowVariantList}) { const variants = []; let hasDefault = false; - while (true) { - const variant = this.getVariant(ps, hasDefault); + ps.skipBlank(); + while (ps.isVariantStart()) { + const variant = this.getVariant(ps, {allowVariantList, hasDefault}); if (variant.default) { hasDefault = true; } variants.push(variant); - - if (!ps.isNextLineVariantStart({skip: false})) { - break; - } + ps.expectLineEnd(); ps.skipBlank(); } + if (variants.length === 0) { + throw new ParseError("E0011"); + } + if (!hasDefault) { throw new ParseError("E0010"); } @@ -411,65 +409,171 @@ export default class FluentParser { return new AST.NumberLiteral(num); } - getValue(ps) { - if (ps.currentChar === "{") { + // maybeGetValue distinguishes between patterns which start on the same line + // as the identifier (a.k.a. inline signleline patterns and inline multiline + // patterns) and patterns which start on a new line (a.k.a. block multiline + // patterns). The distinction is important for the dedentation logic: the + // indent of the first line of a block pattern must be taken into account when + // calculating the maximum common indent. + maybeGetValue(ps, {allowVariantList}) { + ps.peekBlankInline(); + if (ps.isValueStart()) { + ps.skipToPeek(); + return this.getPatternOrVariantList( + ps, {isBlock: false, allowVariantList}); + } + + ps.peekBlankBlock(); + if (ps.isValueContinuation()) { + ps.skipToPeek(); + return this.getPatternOrVariantList( + ps, {isBlock: true, allowVariantList}); + } + + return null; + } + + // Parse a VariantList (if allowed) or a Pattern. + getPatternOrVariantList(ps, {isBlock, allowVariantList}) { + ps.peekBlankInline(); + if (allowVariantList && ps.currentPeek === "{") { + const start = ps.peekOffset; ps.peek(); ps.peekBlankInline(); - if (ps.isNextLineVariantStart({skip: false})) { - return this.getVariantList(ps); + if (ps.currentPeek === EOL) { + ps.peekBlank(); + if (ps.isVariantStart()) { + ps.resetPeek(start); + ps.skipToPeek(); + return this.getVariantList(ps, {allowVariantList}); + } } - - ps.resetPeek(); } - return this.getPattern(ps); + ps.resetPeek(); + const pattern = this.getPattern(ps, {isBlock}); + return pattern; } getVariantList(ps) { ps.expectChar("{"); - ps.skipBlankInline(); - ps.expectLineEnd(); - ps.skipBlank(); - const variants = this.getVariants(ps); - ps.expectLineEnd(); - ps.skipBlank(); + var variants = this.getVariants(ps, {allowVariantList: true}); ps.expectChar("}"); return new AST.VariantList(variants); } - getPattern(ps) { + getPattern(ps, {isBlock}) { const elements = []; + if (isBlock) { + // A block pattern is a pattern which starts on a new line. Store and + // measure the indent of this first line for the dedentation logic. + const blankStart = ps.index; + const firstIndent = ps.skipBlankInline(); + elements.push(this.getIndent(ps, firstIndent, blankStart)); + var commonIndentLength = firstIndent.length; + } else { + var commonIndentLength = Infinity; + } let ch; - while ((ch = ps.currentChar)) { + elements: while ((ch = ps.currentChar)) { + switch (ch) { + case EOL: { + const blankStart = ps.index; + const blankLines = ps.peekBlankBlock(); + if (ps.isValueContinuation()) { + ps.skipToPeek(); + const indent = ps.skipBlankInline(); + commonIndentLength = Math.min(commonIndentLength, indent.length); + elements.push(this.getIndent(ps, blankLines + indent, blankStart)); + continue elements; + } - // The end condition for getPattern's while loop is a newline - // which is not followed by a valid pattern continuation. - if (ch === EOL && !ps.isNextLineValue({skip: false})) { - break; + // The end condition for getPattern's while loop is a newline + // which is not followed by a valid pattern continuation. + ps.resetPeek(); + break elements; + } + case "{": + elements.push(this.getPlaceable(ps)); + continue elements; + case "}": + throw new ParseError("E0027"); + default: + const element = this.getTextElement(ps); + elements.push(element); } + } - if (ch === "{") { - const element = this.getPlaceable(ps); - elements.push(element); - } else if (ch === "}") { - throw new ParseError("E0027"); - } else { - const element = this.getTextElement(ps); - elements.push(element); + const dedented = this.dedent(elements, commonIndentLength); + return new AST.Pattern(dedented); + } + + // Create a token representing an indent. It's not part of the AST and it will + // be trimmed and merged into adjacent TextElements, or turned into a new + // TextElement, if it's surrounded by two Placeables. + getIndent(ps, value, start) { + return { + type: "Indent", + span: {start, end: ps.index}, + value, + }; + } + + // Dedent a list of elements by removing the maximum common indent from the + // beginning of text lines. The common indent is calculated in getPattern. + dedent(elements, commonIndent) { + const trimmed = []; + + for (let element of elements) { + if (element.type === "Placeable") { + trimmed.push(element); + continue; + } + + if (element.type === "Indent") { + // Strip common indent. + element.value = element.value.slice( + 0, element.value.length - commonIndent); + if (element.value.length === 0) { + continue; + } + } + + let prev = trimmed[trimmed.length - 1]; + if (prev && prev.type === "TextElement") { + // Join adjacent TextElements by replacing them with their sum. + const sum = new AST.TextElement(prev.value + element.value); + if (this.withSpans) { + sum.addSpan(prev.span.start, element.span.end); + } + trimmed[trimmed.length - 1] = sum; + continue; } + + if (element.type === "Indent") { + // If the indent hasn't been merged into a preceding TextElement, + // convert it into a new TextElement. + const textElement = new AST.TextElement(element.value); + if (this.withSpans) { + textElement.addSpan(element.span.start, element.span.end); + } + element = textElement; + } + + trimmed.push(element); } - // Trim trailing whitespace. - const lastElement = elements[elements.length - 1]; + // Trim trailing whitespace from the Pattern. + const lastElement = trimmed[trimmed.length - 1]; if (lastElement.type === "TextElement") { lastElement.value = lastElement.value.replace(trailingWSRe, ""); - if (lastElement.value === "") { - elements.pop(); + if (lastElement.value.length === 0) { + trimmed.pop(); } } - return new AST.Pattern(elements); + return trimmed; } getTextElement(ps) { @@ -482,15 +586,7 @@ export default class FluentParser { } if (ch === EOL) { - if (!ps.isNextLineValue({skip: false})) { - return new AST.TextElement(buffer); - } - - ps.next(); - ps.skipBlankInline(); - - buffer += EOL; - continue; + return new AST.TextElement(buffer); } buffer += ch; @@ -530,16 +626,14 @@ export default class FluentParser { getPlaceable(ps) { ps.expectChar("{"); + ps.skipBlank(); const expression = this.getExpression(ps); ps.expectChar("}"); return new AST.Placeable(expression); } getExpression(ps) { - ps.skipBlank(); - const selector = this.getSelectorExpression(ps); - ps.skipBlank(); if (ps.currentChar === "-") { @@ -567,28 +661,14 @@ export default class FluentParser { ps.skipBlankInline(); ps.expectLineEnd(); - ps.skipBlank(); - - const variants = this.getVariants(ps); - ps.skipBlank(); - - if (variants.length === 0) { - throw new ParseError("E0011"); - } - - // VariantLists are only allowed in other VariantLists. - if (variants.some(v => v.value.type === "VariantList")) { - throw new ParseError("E0023"); - } + const variants = this.getVariants(ps, {allowVariantList: false}); return new AST.SelectExpression(selector, variants); } else if (selector.type === "AttributeExpression" && selector.ref.type === "TermReference") { throw new ParseError("E0019"); } - ps.skipBlank(); - return selector; } @@ -596,6 +676,7 @@ export default class FluentParser { if (ps.currentChar === "{") { return this.getPlaceable(ps); } + const literal = this.getLiteral(ps); if (literal.type !== "MessageReference" diff --git a/fluent-syntax/src/stream.js b/fluent-syntax/src/stream.js index c95190aa5..2f474be99 100644 --- a/fluent-syntax/src/stream.js +++ b/fluent-syntax/src/stream.js @@ -67,61 +67,44 @@ export const EOF = undefined; const SPECIAL_LINE_START_CHARS = ["}", ".", "[", "*"]; export class FluentParserStream extends ParserStream { - skipBlankInline() { - while (this.currentChar === " ") { - this.next(); - } - } - peekBlankInline() { + const start = this.index + this.peekOffset; while (this.currentPeek === " ") { this.peek(); } + return this.string.slice(start, this.index + this.peekOffset); } - skipBlankBlock() { - let lineCount = 0; - while (true) { - this.peekBlankInline(); - - if (this.currentPeek === EOL) { - this.skipToPeek(); - this.next(); - lineCount++; - continue; - } - - if (this.currentPeek === EOF) { - // Consume any inline blanks before the EOF. - this.skipToPeek(); - return lineCount; - } - - // Any other char; reset to column 1 on this line. - this.resetPeek(); - return lineCount; - } + skipBlankInline() { + const blank = this.peekBlankInline(); + this.skipToPeek(); + return blank; } peekBlankBlock() { + let blank = ""; while (true) { const lineStart = this.peekOffset; - this.peekBlankInline(); - if (this.currentPeek === EOL) { + blank += EOL; this.peek(); - } else { - this.resetPeek(lineStart); - break; + continue; + } + if (this.currentPeek === EOF) { + // Treat the blank line at EOF as a blank block. + return blank; } + // Any other char; reset to column 1 on this line. + this.resetPeek(lineStart); + return blank; } } - skipBlank() { - while (this.currentChar === " " || this.currentChar === EOL) { - this.next(); - } + skipBlankBlock() { + const blank = this.peekBlankBlock(); + this.skipToPeek(); + return blank; } peekBlank() { @@ -130,6 +113,11 @@ export class FluentParserStream extends ParserStream { } } + skipBlank() { + this.peekBlank(); + this.skipToPeek(); + } + expectChar(ch) { if (this.currentChar === ch) { this.next(); @@ -204,29 +192,39 @@ export class FluentParserStream extends ParserStream { return !includes(SPECIAL_LINE_START_CHARS, ch); } - isValueStart({skip = true}) { - if (skip === false) throw new Error("Unimplemented"); + isValueStart() { + // Inline Patterns may start with any char. + const ch = this.currentPeek; + return ch !== EOL && ch !== EOF; + } + isValueContinuation() { + const column1 = this.peekOffset; this.peekBlankInline(); - const ch = this.currentPeek; - // Inline Patterns may start with any char. - if (ch !== EOF && ch !== EOL) { - this.skipToPeek(); + if (this.currentPeek === "{") { + this.resetPeek(column1); return true; } - return this.isNextLineValue({skip}); + if (this.peekOffset - column1 === 0) { + return false; + } + + if (this.isCharPatternContinuation(this.currentPeek)) { + this.resetPeek(column1); + return true; + } + + return false; } // -1 - any // 0 - comment // 1 - group comment // 2 - resource comment - isNextLineComment(level = -1, {skip = false}) { - if (skip === true) throw new Error("Unimplemented"); - - if (this.currentPeek !== EOL) { + isNextLineComment(level = -1) { + if (this.currentChar !== EOL) { return false; } @@ -254,70 +252,21 @@ export class FluentParserStream extends ParserStream { return false; } - isNextLineVariantStart({skip = false}) { - if (skip === true) throw new Error("Unimplemented"); - - if (this.currentPeek !== EOL) { - return false; - } - - this.peekBlank(); - + isVariantStart() { + const currentPeekOffset = this.peekOffset; if (this.currentPeek === "*") { this.peek(); } - if (this.currentPeek === "[") { - this.resetPeek(); + this.resetPeek(currentPeekOffset); return true; } - this.resetPeek(); + this.resetPeek(currentPeekOffset); return false; } - isNextLineAttributeStart({skip = true}) { - if (skip === false) throw new Error("Unimplemented"); - - this.peekBlank(); - - if (this.currentPeek === ".") { - this.skipToPeek(); - return true; - } - - this.resetPeek(); - return false; - } - - isNextLineValue({skip = true}) { - if (this.currentPeek !== EOL) { - return false; - } - - this.peekBlankBlock(); - - const ptr = this.peekOffset; - - this.peekBlankInline(); - - if (this.currentPeek !== "{") { - if (this.peekOffset - ptr === 0) { - this.resetPeek(); - return false; - } - - if (!this.isCharPatternContinuation(this.currentPeek)) { - this.resetPeek(); - return false; - } - } - - if (skip) { - this.skipToPeek(); - } else { - this.resetPeek(); - } - return true; + isAttributeStart() { + return this.currentPeek === "."; } skipToNextEntryStart(junkStart) { diff --git a/fluent-syntax/test/fixtures_behavior/select_expression_without_variants.ftl b/fluent-syntax/test/fixtures_behavior/select_expression_without_variants.ftl index 7e43355d1..8eea7b423 100644 --- a/fluent-syntax/test/fixtures_behavior/select_expression_without_variants.ftl +++ b/fluent-syntax/test/fixtures_behavior/select_expression_without_variants.ftl @@ -3,4 +3,4 @@ key = { $foo -> } key = { $foo -> } -# ~ERROR E0003, pos 39, args "[" +# ~ERROR E0011, pos 39 diff --git a/fluent-syntax/test/fixtures_behavior/selector_expression_ends_abruptly.ftl b/fluent-syntax/test/fixtures_behavior/selector_expression_ends_abruptly.ftl index a11a18144..8cb339853 100644 --- a/fluent-syntax/test/fixtures_behavior/selector_expression_ends_abruptly.ftl +++ b/fluent-syntax/test/fixtures_behavior/selector_expression_ends_abruptly.ftl @@ -1,2 +1,2 @@ key = { $foo -> -# ~ERROR E0003, pos 16, args "[" +# ~ERROR E0011, pos 16 diff --git a/fluent-syntax/test/fixtures_behavior/variant_lists.ftl b/fluent-syntax/test/fixtures_behavior/variant_lists.ftl index 10f5cab3d..2f3cc5b8d 100644 --- a/fluent-syntax/test/fixtures_behavior/variant_lists.ftl +++ b/fluent-syntax/test/fixtures_behavior/variant_lists.ftl @@ -4,7 +4,7 @@ message1 = *[one] One } -# ~ERROR E0023, pos 123 +# ~ERROR E0014, pos 97 message2 = { $sel -> *[one] { @@ -24,7 +24,7 @@ message2 = } } -# ~ERROR E0023, pos 318 +# ~ERROR E0014, pos 292 -term3 = { $sel -> *[one] { diff --git a/fluent-syntax/test/fixtures_reference/multiline_values.ftl b/fluent-syntax/test/fixtures_reference/multiline_values.ftl index 4dd811548..e3739bb5c 100644 --- a/fluent-syntax/test/fixtures_reference/multiline_values.ftl +++ b/fluent-syntax/test/fixtures_reference/multiline_values.ftl @@ -33,3 +33,28 @@ key07 = {"A multiline value"} starting and ending {"with a placeable"} key08 = Leading and trailing whitespace. + +key09 = zero + three + two + one + zero + +key10 = + two + zero + four + +key11 = + + + two + zero + +key12 = +{"."} + four + +key13 = + four +{"."} diff --git a/fluent-syntax/test/fixtures_reference/multiline_values.json b/fluent-syntax/test/fixtures_reference/multiline_values.json index 645a09b38..17fb6f8e4 100644 --- a/fluent-syntax/test/fixtures_reference/multiline_values.json +++ b/fluent-syntax/test/fixtures_reference/multiline_values.json @@ -102,7 +102,7 @@ "elements": [ { "type": "TextElement", - "value": "A multiline value with non-standard\n\nindentation." + "value": "A multiline value with non-standard\n\n indentation." } ] }, @@ -212,6 +212,110 @@ }, "attributes": [], "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key09" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "zero\n three\n two\n one\nzero" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key10" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": " two\nzero\n four" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key11" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": " two\nzero" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key12" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "." + } + }, + { + "type": "TextElement", + "value": "\n four" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key13" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": " four\n" + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "." + } + } + ] + }, + "attributes": [], + "comment": null } ] } diff --git a/fluent-syntax/test/fixtures_structure/blank_lines.json b/fluent-syntax/test/fixtures_structure/blank_lines.json index 14fab74f6..998e7da64 100644 --- a/fluent-syntax/test/fixtures_structure/blank_lines.json +++ b/fluent-syntax/test/fixtures_structure/blank_lines.json @@ -112,7 +112,7 @@ ], "span": { "type": "Span", - "start": 130, + "start": 126, "end": 153 } }, @@ -150,7 +150,7 @@ ], "span": { "type": "Span", - "start": 240, + "start": 236, "end": 267 } }, @@ -196,7 +196,7 @@ ], "span": { "type": "Span", - "start": 339, + "start": 335, "end": 347 } }, diff --git a/fluent-syntax/test/fixtures_structure/crlf.json b/fluent-syntax/test/fixtures_structure/crlf.json index b72afa33f..9461bfaa2 100644 --- a/fluent-syntax/test/fixtures_structure/crlf.json +++ b/fluent-syntax/test/fixtures_structure/crlf.json @@ -65,7 +65,7 @@ ], "span": { "type": "Span", - "start": 35, + "start": 31, "end": 58 } }, diff --git a/fluent-syntax/test/fixtures_structure/leading_dots.json b/fluent-syntax/test/fixtures_structure/leading_dots.json index 837427b45..d2e234c38 100644 --- a/fluent-syntax/test/fixtures_structure/leading_dots.json +++ b/fluent-syntax/test/fixtures_structure/leading_dots.json @@ -175,7 +175,7 @@ ], "span": { "type": "Span", - "start": 61, + "start": 57, "end": 71 } }, @@ -514,7 +514,7 @@ ], "span": { "type": "Span", - "start": 428, + "start": 424, "end": 484 } }, @@ -565,7 +565,7 @@ ], "span": { "type": "Span", - "start": 515, + "start": 511, "end": 516 } }, @@ -696,7 +696,7 @@ ], "span": { "type": "Span", - "start": 589, + "start": 580, "end": 599 } }, @@ -822,7 +822,7 @@ ], "span": { "type": "Span", - "start": 669, + "start": 657, "end": 679 } }, @@ -836,7 +836,7 @@ ], "span": { "type": "Span", - "start": 614, + "start": 615, "end": 684 } }, @@ -849,7 +849,7 @@ ], "span": { "type": "Span", - "start": 613, + "start": 609, "end": 685 } }, diff --git a/fluent-syntax/test/fixtures_structure/multiline_pattern.json b/fluent-syntax/test/fixtures_structure/multiline_pattern.json index 1793692b6..8f5c7fadd 100644 --- a/fluent-syntax/test/fixtures_structure/multiline_pattern.json +++ b/fluent-syntax/test/fixtures_structure/multiline_pattern.json @@ -65,7 +65,7 @@ ], "span": { "type": "Span", - "start": 47, + "start": 43, "end": 72 } }, @@ -103,7 +103,7 @@ ], "span": { "type": "Span", - "start": 171, + "start": 167, "end": 176 } }, @@ -173,7 +173,7 @@ ], "span": { "type": "Span", - "start": 311, + "start": 307, "end": 316 } }, diff --git a/fluent-syntax/test/fixtures_structure/placeable_at_eol.json b/fluent-syntax/test/fixtures_structure/placeable_at_eol.json index f79d7454b..7018303eb 100644 --- a/fluent-syntax/test/fixtures_structure/placeable_at_eol.json +++ b/fluent-syntax/test/fixtures_structure/placeable_at_eol.json @@ -61,7 +61,7 @@ ], "span": { "type": "Span", - "start": 11, + "start": 7, "end": 131 } }, @@ -124,7 +124,7 @@ ], "span": { "type": "Span", - "start": 144, + "start": 140, "end": 184 } }, diff --git a/fluent-syntax/test/fixtures_structure/sparse-messages.json b/fluent-syntax/test/fixtures_structure/sparse-messages.json index bdeb917b5..32780f68c 100644 --- a/fluent-syntax/test/fixtures_structure/sparse-messages.json +++ b/fluent-syntax/test/fixtures_structure/sparse-messages.json @@ -27,7 +27,7 @@ ], "span": { "type": "Span", - "start": 12, + "start": 8, "end": 17 } }, @@ -122,7 +122,7 @@ ], "span": { "type": "Span", - "start": 63, + "start": 59, "end": 104 } }, @@ -322,7 +322,7 @@ ], "span": { "type": "Span", - "start": 154, + "start": 155, "end": 208 } }, diff --git a/fluent-syntax/test/fixtures_structure/term.json b/fluent-syntax/test/fixtures_structure/term.json index 6da90f3bb..c36ec4509 100644 --- a/fluent-syntax/test/fixtures_structure/term.json +++ b/fluent-syntax/test/fixtures_structure/term.json @@ -218,7 +218,7 @@ ], "span": { "type": "Span", - "start": 131, + "start": 127, "end": 171 } }, @@ -481,7 +481,7 @@ ], "span": { "type": "Span", - "start": 198, + "start": 199, "end": 436 } }, @@ -494,7 +494,7 @@ ], "span": { "type": "Span", - "start": 197, + "start": 193, "end": 437 } }, diff --git a/fluent-syntax/test/fixtures_structure/variant_with_empty_pattern.json b/fluent-syntax/test/fixtures_structure/variant_with_empty_pattern.json index 19fe137d2..2b61ee990 100644 --- a/fluent-syntax/test/fixtures_structure/variant_with_empty_pattern.json +++ b/fluent-syntax/test/fixtures_structure/variant_with_empty_pattern.json @@ -77,7 +77,7 @@ ], "span": { "type": "Span", - "start": 12, + "start": 13, "end": 41 } }, @@ -90,7 +90,7 @@ ], "span": { "type": "Span", - "start": 11, + "start": 7, "end": 42 } }, diff --git a/fluent/src/resolver.js b/fluent/src/resolver.js index f2a216d25..b1a29a0d1 100644 --- a/fluent/src/resolver.js +++ b/fluent/src/resolver.js @@ -305,6 +305,8 @@ function Type(env, expr) { switch (expr.type) { + case "str": + return expr.value; case "num": return new FluentNumber(expr.value); case "var": diff --git a/fluent/src/resource.js b/fluent/src/resource.js index b457a38ec..d1b6421c4 100644 --- a/fluent/src/resource.js +++ b/fluent/src/resource.js @@ -27,11 +27,13 @@ const RE_STRING_RUN = /([^\\"\n\r]*)/y; const RE_UNICODE_ESCAPE = /\\u([a-fA-F0-9]{4})/y; const RE_STRING_ESCAPE = /\\([\\"])/y; -// Used for trimming TextElements and indents. With the /m flag, the $ matches -// the end of every line. -const RE_TRAILING_SPACES = / +$/mg; -// CRLFs are normalized to LF. -const RE_CRLF = /\r\n/g; +// Used for trimming TextElements and indents. +const RE_LEADING_NEWLINES = /^\n+/; +const RE_TRAILING_SPACES = / +$/; +// Used in makeIndent to strip spaces from blank lines and normalize CRLF to LF. +const RE_BLANK_LINES = / *\r?\n/g; +// Used in makeIndent to measure the indentation. +const RE_INDENT = /( *)$/; // Common tokens. const TOKEN_BRACE_OPEN = /{\s*/y; @@ -180,42 +182,41 @@ export default class FluentResource extends Map { // If there's a placeable on the first line, parse a complex pattern. if (source[cursor] === "{" || source[cursor] === "}") { - return first - // Re-use the text parsed above, if possible. - ? parsePatternElements(first) - : parsePatternElements(); + // Re-use the text parsed above, if possible. + return parsePatternElements(first ? [first] : [], Infinity); } // RE_TEXT_VALUE stops at newlines. Only continue parsing the pattern if // what comes after the newline is indented. let indent = parseIndent(); if (indent) { - return first + if (first) { // If there's text on the first line, the blank block is part of the - // translation content. - ? parsePatternElements(first, trim(indent)) - // Otherwise, we're dealing with a block pattern. The blank block is - // the leading whitespace; discard it. - : parsePatternElements(); + // translation content in its entirety. + return parsePatternElements([first, indent], indent.length); + } + // Otherwise, we're dealing with a block pattern, i.e. a pattern which + // starts on a new line. Discrad the leading newlines but keep the + // inline indent; it will be used by the dedentation logic. + indent.value = trim(indent.value, RE_LEADING_NEWLINES); + return parsePatternElements([indent], indent.length); } if (first) { // It was just a simple inline text after all. - return trim(first); + return trim(first, RE_TRAILING_SPACES); } return null; } // Parse a complex pattern as an array of elements. - function parsePatternElements(...elements) { + function parsePatternElements(elements = [], commonIndent) { let placeableCount = 0; - let needsTrimming = false; while (true) { if (test(RE_TEXT_RUN)) { elements.push(match(RE_TEXT_RUN)); - needsTrimming = true; continue; } @@ -224,7 +225,6 @@ export default class FluentResource extends Map { throw new FluentError("Too many placeables"); } elements.push(parsePlaceable()); - needsTrimming = false; continue; } @@ -234,23 +234,34 @@ export default class FluentResource extends Map { let indent = parseIndent(); if (indent) { - elements.push(trim(indent)); - needsTrimming = false; + elements.push(indent); + commonIndent = Math.min(commonIndent, indent.length); continue; } break; } - if (needsTrimming) { - // Trim the trailing whitespace of the last element if it's a - // TextElement. Use a flag rather than a typeof check to tell - // TextElements and StringLiterals apart (both are strings). - let lastIndex = elements.length - 1; - elements[lastIndex] = trim(elements[lastIndex]); + let lastIndex = elements.length - 1; + // Trim the trailing spaces in the last element if it's a TextElement. + if (typeof elements[lastIndex] === "string") { + elements[lastIndex] = trim(elements[lastIndex], RE_TRAILING_SPACES); } - return elements; + let baked = []; + for (let element of elements) { + if (element.type === "indent") { + // Dedent indented lines by the maximum common indent. + element = element.value.slice(0, element.value.length - commonIndent); + } else if (element.type === "str") { + // Optimize StringLiterals into their value. + element = element.value; + } + if (element) { + baked.push(element); + } + } + return baked; } function parsePlaceable() { @@ -401,7 +412,7 @@ export default class FluentResource extends Map { } if (consumeChar("\"")) { - return value; + return {type: "str", value}; } // We've reached an EOL of EOF. @@ -447,7 +458,7 @@ export default class FluentResource extends Map { case "{": // Placeables don't require indentation (in EBNF: block-placeable). // Continue the Pattern. - return source.slice(start, cursor).replace(RE_CRLF, "\n"); + return makeIndent(source.slice(start, cursor)); } // If the first character on the line is not one of the special characters @@ -456,7 +467,7 @@ export default class FluentResource extends Map { if (source[cursor - 1] === " ") { // It's an indented text character (in EBNF: indented-char). Continue // the Pattern. - return source.slice(start, cursor).replace(RE_CRLF, "\n"); + return makeIndent(source.slice(start, cursor)); } // A not-indented text character is likely the identifier of the next @@ -464,9 +475,16 @@ export default class FluentResource extends Map { return false; } - // Trim spaces trailing on every line of text. - function trim(text) { - return text.replace(RE_TRAILING_SPACES, ""); + // Trim blanks in text according to the given regex. + function trim(text, re) { + return text.replace(re, ""); + } + + // Normalize a blank block and extract the indent details. + function makeIndent(blank) { + let value = blank.replace(RE_BLANK_LINES, "\n"); + let length = RE_INDENT.exec(blank)[1].length; + return {type: "indent", value, length}; } } } diff --git a/fluent/test/fixtures_reference/call_expressions.json b/fluent/test/fixtures_reference/call_expressions.json index 3e0fe30e5..431a25b5a 100644 --- a/fluent/test/fixtures_reference/call_expressions.json +++ b/fluent/test/fixtures_reference/call_expressions.json @@ -11,7 +11,10 @@ "type": "num", "value": "1" }, - "a", + { + "type": "str", + "value": "a" + }, { "type": "ref", "name": "msg" @@ -38,7 +41,10 @@ { "type": "narg", "name": "y", - "value": "Y" + "value": { + "type": "str", + "value": "Y" + } } ] } @@ -62,7 +68,10 @@ { "type": "narg", "name": "y", - "value": "Y" + "value": { + "type": "str", + "value": "Y" + } } ] } @@ -79,7 +88,10 @@ "type": "num", "value": "1" }, - "a", + { + "type": "str", + "value": "a" + }, { "type": "ref", "name": "msg" @@ -95,7 +107,10 @@ { "type": "narg", "name": "y", - "value": "Y" + "value": { + "type": "str", + "value": "Y" + } } ] } @@ -120,11 +135,17 @@ "value": "1" } }, - "a", + { + "type": "str", + "value": "a" + }, { "type": "narg", "name": "y", - "value": "Y" + "value": { + "type": "str", + "value": "Y" + } }, { "type": "ref", @@ -152,7 +173,10 @@ { "type": "narg", "name": "x", - "value": "X" + "value": { + "type": "str", + "value": "X" + } } ] } @@ -165,7 +189,10 @@ "name": "FUN" }, "args": [ - "a", + { + "type": "str", + "value": "a" + }, { "type": "ref", "name": "msg" @@ -199,7 +226,10 @@ "name": "FUN" }, "args": [ - "a", + { + "type": "str", + "value": "a" + }, { "type": "ref", "name": "msg" @@ -223,7 +253,10 @@ "name": "FUN" }, "args": [ - "a", + { + "type": "str", + "value": "a" + }, { "type": "ref", "name": "msg" @@ -272,7 +305,10 @@ "name": "FUN" }, "args": [ - "a" + { + "type": "str", + "value": "a" + } ] } ], diff --git a/fluent/test/fixtures_reference/cr.json b/fluent/test/fixtures_reference/cr.json index 6c6f39082..88e772c02 100644 --- a/fluent/test/fixtures_reference/cr.json +++ b/fluent/test/fixtures_reference/cr.json @@ -3,6 +3,7 @@ "err02": "Value 02", "err03": { "value": [ + "\r\r", "Value 03", "\r", "Continued" diff --git a/fluent/test/fixtures_reference/messages.json b/fluent/test/fixtures_reference/messages.json index 49d97ae73..03f07b796 100644 --- a/fluent/test/fixtures_reference/messages.json +++ b/fluent/test/fixtures_reference/messages.json @@ -26,7 +26,5 @@ "attr1": "Attribute 1" } }, - "key06": [ - "" - ] + "key06": [] } diff --git a/fluent/test/fixtures_reference/multiline_values.json b/fluent/test/fixtures_reference/multiline_values.json index a4cb0a170..c6d153fcb 100644 --- a/fluent/test/fixtures_reference/multiline_values.json +++ b/fluent/test/fixtures_reference/multiline_values.json @@ -35,7 +35,7 @@ }, "key05": [ "A multiline value with non-standard", - "\n\n", + "\n\n ", "indentation." ], "key06": [ @@ -53,5 +53,41 @@ " starting and ending ", "with a placeable" ], - "key08": "Leading and trailing whitespace." + "key08": "Leading and trailing whitespace.", + "key09": [ + "zero", + "\n ", + "three", + "\n ", + "two", + "\n ", + "one", + "\n", + "zero" + ], + "key10": [ + " ", + "two", + "\n", + "zero", + "\n ", + "four" + ], + "key11": [ + " ", + "two", + "\n", + "zero" + ], + "key12": [ + ".", + "\n ", + "four" + ], + "key13": [ + " ", + "four", + "\n", + "." + ] } diff --git a/fluent/test/fixtures_reference/select_expressions.json b/fluent/test/fixtures_reference/select_expressions.json index 6881a51a7..65ed7c426 100644 --- a/fluent/test/fixtures_reference/select_expressions.json +++ b/fluent/test/fixtures_reference/select_expressions.json @@ -21,7 +21,6 @@ { "key": "other", "value": [ - "", "Other" ] } @@ -79,9 +78,7 @@ "variants": [ { "key": "one", - "value": [ - "" - ] + "value": [] } ], "star": 0 diff --git a/fluent/test/fixtures_reference/select_indent.json b/fluent/test/fixtures_reference/select_indent.json index 52b4c28bd..c8e21fea3 100644 --- a/fluent/test/fixtures_reference/select_indent.json +++ b/fluent/test/fixtures_reference/select_indent.json @@ -229,7 +229,6 @@ } ], "star": 0 - }, - "" + } ] } diff --git a/fluent/test/fixtures_reference/terms.json b/fluent/test/fixtures_reference/terms.json index 81e77bc5c..0195fe337 100644 --- a/fluent/test/fixtures_reference/terms.json +++ b/fluent/test/fixtures_reference/terms.json @@ -5,9 +5,7 @@ "attr": "Attribute" } }, - "-term02": [ - "" - ], + "-term02": [], "-term03": { "value": null, "attrs": { diff --git a/fluent/test/fixtures_structure/variant_with_empty_pattern.json b/fluent/test/fixtures_structure/variant_with_empty_pattern.json index 3ffa51e1b..23d0fb0ca 100644 --- a/fluent/test/fixtures_structure/variant_with_empty_pattern.json +++ b/fluent/test/fixtures_structure/variant_with_empty_pattern.json @@ -9,9 +9,7 @@ "variants": [ { "key": "one", - "value": [ - "" - ] + "value": [] } ], "star": 0 diff --git a/fluent/test/fixtures_structure/whitespace_leading.json b/fluent/test/fixtures_structure/whitespace_leading.json index 7a69cc773..96fc498e8 100644 --- a/fluent/test/fixtures_structure/whitespace_leading.json +++ b/fluent/test/fixtures_structure/whitespace_leading.json @@ -2,7 +2,6 @@ "key1": "Value", "key2": "ย  Value", "key3": [ - "", " Value" ], "key4": [ From 30d13b4a5b7d6510e7de3df86072c29758020e53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Thu, 22 Nov 2018 15:57:31 +0100 Subject: [PATCH 07/23] Add FunctionReference --- fluent-syntax/src/ast.js | 15 +- fluent-syntax/src/parser.js | 2 +- fluent-syntax/src/serializer.js | 15 +- .../fixtures_reference/call_expressions.ftl | 16 ++ .../fixtures_reference/call_expressions.json | 255 ++++++++++++++---- .../select_expressions.json | 7 +- .../expressions_call_args.json | 12 +- .../fixtures_reference/call_expressions.json | 40 +++ 8 files changed, 286 insertions(+), 76 deletions(-) diff --git a/fluent-syntax/src/ast.js b/fluent-syntax/src/ast.js index 55a3f251a..0751b44ce 100644 --- a/fluent-syntax/src/ast.js +++ b/fluent-syntax/src/ast.js @@ -135,6 +135,14 @@ export class VariableReference extends Expression { } } +export class FunctionReference extends Expression { + constructor(id) { + super(); + this.type = "FunctionReference"; + this.id = id; + } +} + export class SelectExpression extends Expression { constructor(selector, variants) { super(); @@ -236,13 +244,6 @@ export class ResourceComment extends BaseComment { } } -export class Function extends Identifier { - constructor(name) { - super(name); - this.type = "Function"; - } -} - export class Junk extends SyntaxNode { constructor(content) { super(); diff --git a/fluent-syntax/src/parser.js b/fluent-syntax/src/parser.js index 965a45aa7..91e8d9e7c 100644 --- a/fluent-syntax/src/parser.js +++ b/fluent-syntax/src/parser.js @@ -718,7 +718,7 @@ export default class FluentParser { ps.expectChar(")"); - const func = new AST.Function(literal.id.name); + const func = new AST.FunctionReference(literal.id); if (this.withSpans) { func.addSpan(literal.span.start, literal.span.end); } diff --git a/fluent-syntax/src/serializer.js b/fluent-syntax/src/serializer.js index 618496b7f..33db27ae4 100644 --- a/fluent-syntax/src/serializer.js +++ b/fluent-syntax/src/serializer.js @@ -221,7 +221,8 @@ function serializeExpression(expr) { case "NumberLiteral": return serializeNumberLiteral(expr); case "MessageReference": - return serializeMessageReference(expr); + case "FunctionReference": + return serializeIdentifier(expr.id); case "TermReference": return serializeTermReference(expr); case "VariableReference": @@ -252,11 +253,6 @@ function serializeNumberLiteral(expr) { } -function serializeMessageReference(expr) { - return serializeIdentifier(expr.id); -} - - function serializeTermReference(expr) { return `-${serializeIdentifier(expr.id)}`; } @@ -296,7 +292,7 @@ function serializeVariantExpression(expr) { function serializeCallExpression(expr) { - const fun = serializeFunction(expr.callee); + const fun = serializeExpression(expr.callee); const positional = expr.positional.map(serializeExpression).join(", "); const named = expr.named.map(serializeNamedArgument).join(", "); if (expr.positional.length > 0 && expr.named.length > 0) { @@ -339,8 +335,3 @@ function serializeVariantKey(key) { throw new Error(`Unknown variant key type: ${key.type}`); } } - - -function serializeFunction(fun) { - return fun.name; -} diff --git a/fluent-syntax/test/fixtures_reference/call_expressions.ftl b/fluent-syntax/test/fixtures_reference/call_expressions.ftl index 065626422..9ed69bd1b 100644 --- a/fluent-syntax/test/fixtures_reference/call_expressions.ftl +++ b/fluent-syntax/test/fixtures_reference/call_expressions.ftl @@ -1,3 +1,19 @@ +## Callees + +function-callee = {FUNCTION()} + +# ERROR Equivalent to a MessageReference callee. +mixed-case-callee = {Function()} + +# ERROR MessageReference is not a valid callee. +message-callee = {message()} +# ERROR TermReference is not a valid callee. +term-callee = {-term()} +# ERROR VariableReference is not a valid callee. +variable-callee = {$variable()} + +## Arguments + positional-args = {FUN(1, "a", msg)} named-args = {FUN(x: 1, y: "Y")} dense-named-args = {FUN(x:1, y:"Y")} diff --git a/fluent-syntax/test/fixtures_reference/call_expressions.json b/fluent-syntax/test/fixtures_reference/call_expressions.json index a860260fe..e5a451938 100644 --- a/fluent-syntax/test/fixtures_reference/call_expressions.json +++ b/fluent-syntax/test/fixtures_reference/call_expressions.json @@ -1,6 +1,79 @@ { "type": "Resource", "body": [ + { + "type": "GroupComment", + "content": "Callees" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "function-callee" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUNCTION" + } + }, + "positional": [], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR Equivalent to a MessageReference callee." + }, + { + "type": "Junk", + "annotations": [], + "content": "mixed-case-callee = {Function()}\n" + }, + { + "type": "Comment", + "content": "ERROR MessageReference is not a valid callee." + }, + { + "type": "Junk", + "annotations": [], + "content": "message-callee = {message()}\n" + }, + { + "type": "Comment", + "content": "ERROR TermReference is not a valid callee." + }, + { + "type": "Junk", + "annotations": [], + "content": "term-callee = {-term()}\n" + }, + { + "type": "Comment", + "content": "ERROR VariableReference is not a valid callee." + }, + { + "type": "Junk", + "annotations": [], + "content": "variable-callee = {$variable()}\n" + }, + { + "type": "GroupComment", + "content": "Arguments" + }, { "type": "Message", "id": { @@ -15,8 +88,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -57,8 +133,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [], "named": [ @@ -106,8 +185,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [], "named": [ @@ -155,8 +237,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -242,8 +327,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -292,8 +380,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [], "named": [] @@ -318,8 +409,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -368,8 +462,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -418,8 +515,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [], "named": [] @@ -444,8 +544,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -475,8 +578,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -506,8 +612,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -540,8 +649,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -574,8 +686,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -608,15 +723,21 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { "type": "CallExpression", "callee": { - "type": "Function", - "name": "OTHER" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "OTHER" + } }, "positional": [], "named": [] @@ -644,8 +765,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [], "named": [ @@ -682,8 +806,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -720,8 +847,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -751,8 +881,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -790,8 +923,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -829,8 +965,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -864,8 +1003,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [ { @@ -912,8 +1054,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [], "named": [ @@ -972,8 +1117,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [], "named": [ @@ -1010,8 +1158,11 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FUN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } }, "positional": [], "named": [ diff --git a/fluent-syntax/test/fixtures_reference/select_expressions.json b/fluent-syntax/test/fixtures_reference/select_expressions.json index 4c782006a..6bd9a698c 100644 --- a/fluent-syntax/test/fixtures_reference/select_expressions.json +++ b/fluent-syntax/test/fixtures_reference/select_expressions.json @@ -17,8 +17,11 @@ "selector": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "BUILTIN" + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "BUILTIN" + } }, "positional": [], "named": [] diff --git a/fluent-syntax/test/fixtures_structure/expressions_call_args.json b/fluent-syntax/test/fixtures_structure/expressions_call_args.json index a4ea0b03a..c7be7f94e 100644 --- a/fluent-syntax/test/fixtures_structure/expressions_call_args.json +++ b/fluent-syntax/test/fixtures_structure/expressions_call_args.json @@ -20,8 +20,16 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FOO", + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FOO", + "span": { + "type": "Span", + "start": 8, + "end": 11 + } + }, "span": { "type": "Span", "start": 8, diff --git a/fluent/test/fixtures_reference/call_expressions.json b/fluent/test/fixtures_reference/call_expressions.json index 431a25b5a..5f7c8ad63 100644 --- a/fluent/test/fixtures_reference/call_expressions.json +++ b/fluent/test/fixtures_reference/call_expressions.json @@ -1,4 +1,44 @@ { + "function-callee": [ + { + "type": "call", + "callee": { + "type": "func", + "name": "FUNCTION" + }, + "args": [] + } + ], + "mixed-case-callee": [ + { + "type": "call", + "callee": { + "type": "func", + "name": "Function" + }, + "args": [] + } + ], + "message-callee": [ + { + "type": "call", + "callee": { + "type": "func", + "name": "message" + }, + "args": [] + } + ], + "term-callee": [ + { + "type": "call", + "callee": { + "type": "func", + "name": "-term" + }, + "args": [] + } + ], "positional-args": [ { "type": "call", From 50f226ffc8e1d688c5210971989587c34baedaf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Thu, 22 Nov 2018 16:10:27 +0100 Subject: [PATCH 08/23] Inline trivial serialization functions --- fluent-syntax/src/serializer.js | 74 +++++++-------------------------- 1 file changed, 14 insertions(+), 60 deletions(-) diff --git a/fluent-syntax/src/serializer.js b/fluent-syntax/src/serializer.js index 33db27ae4..dd9f499e1 100644 --- a/fluent-syntax/src/serializer.js +++ b/fluent-syntax/src/serializer.js @@ -96,7 +96,7 @@ function serializeMessage(message) { parts.push(serializeComment(message.comment)); } - parts.push(`${serializeIdentifier(message.id)} =`); + parts.push(`${message.id.name} =`); if (message.value) { parts.push(serializeValue(message.value)); @@ -118,7 +118,7 @@ function serializeTerm(term) { parts.push(serializeComment(term.comment)); } - parts.push(`-${serializeIdentifier(term.id)} =`); + parts.push(`-${term.id.name} =`); parts.push(serializeValue(term.value)); for (const attribute of term.attributes) { @@ -131,9 +131,8 @@ function serializeTerm(term) { function serializeAttribute(attribute) { - const id = serializeIdentifier(attribute.id); const value = indent(serializeValue(attribute.value)); - return `\n .${id} =${value}`; + return `\n .${attribute.id.name} =${value}`; } @@ -184,7 +183,7 @@ function serializeVariant(variant) { function serializeElement(element) { switch (element.type) { case "TextElement": - return serializeTextElement(element); + return element.value; case "Placeable": return serializePlaceable(element); default: @@ -193,11 +192,6 @@ function serializeElement(element) { } -function serializeTextElement(text) { - return text.value; -} - - function serializePlaceable(placeable) { const expr = placeable.expression; @@ -217,16 +211,16 @@ function serializePlaceable(placeable) { function serializeExpression(expr) { switch (expr.type) { case "StringLiteral": - return serializeStringLiteral(expr); + return `"${expr.value}"`; case "NumberLiteral": - return serializeNumberLiteral(expr); + return expr.value; case "MessageReference": case "FunctionReference": - return serializeIdentifier(expr.id); + return expr.id.name; case "TermReference": - return serializeTermReference(expr); + return `-${expr.id.name}`; case "VariableReference": - return serializeVariableReference(expr); + return `$${expr.id.name}`; case "AttributeExpression": return serializeAttributeExpression(expr); case "VariantExpression": @@ -243,26 +237,6 @@ function serializeExpression(expr) { } -function serializeStringLiteral(expr) { - return `"${expr.value}"`; -} - - -function serializeNumberLiteral(expr) { - return expr.value; -} - - -function serializeTermReference(expr) { - return `-${serializeIdentifier(expr.id)}`; -} - - -function serializeVariableReference(expr) { - return `$${serializeIdentifier(expr.id)}`; -} - - function serializeSelectExpression(expr) { const parts = []; const selector = `${serializeExpression(expr.selector)} ->`; @@ -279,8 +253,7 @@ function serializeSelectExpression(expr) { function serializeAttributeExpression(expr) { const ref = serializeExpression(expr.ref); - const name = serializeIdentifier(expr.name); - return `${ref}.${name}`; + return `${ref}.${expr.name.name}`; } @@ -303,35 +276,16 @@ function serializeCallExpression(expr) { function serializeNamedArgument(arg) { - const name = serializeIdentifier(arg.name); - const value = serializeArgumentValue(arg.value); - return `${name}: ${value}`; -} - - -function serializeArgumentValue(argval) { - switch (argval.type) { - case "StringLiteral": - return serializeStringLiteral(argval); - case "NumberLiteral": - return serializeNumberLiteral(argval); - default: - throw new Error(`Unknown argument type: ${argval.type}`); - } + const value = serializeExpression(arg.value); + return `${arg.name.name}: ${value}`; } -function serializeIdentifier(identifier) { - return identifier.name; -} - function serializeVariantKey(key) { switch (key.type) { case "Identifier": - return serializeIdentifier(key); - case "NumberLiteral": - return serializeNumberLiteral(key); + return key.name; default: - throw new Error(`Unknown variant key type: ${key.type}`); + return serializeExpression(key); } } From fb5cbeffdedb809413c3414fd96299e7425c26ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Thu, 22 Nov 2018 16:22:17 +0100 Subject: [PATCH 09/23] Store unescaped content in StringLiteral.value and raw content in .raw --- fluent-syntax/src/ast.js | 3 ++- fluent-syntax/src/parser.js | 25 +++++++++++++------ fluent-syntax/src/serializer.js | 2 +- .../test/fixtures_reference/astral.json | 10 +++++--- .../fixtures_reference/call_expressions.json | 9 +++++++ .../escaped_characters.json | 14 ++++++++--- .../test/fixtures_reference/leading_dots.json | 8 ++++++ .../literal_expressions.json | 1 + .../test/fixtures_reference/messages.json | 1 + .../fixtures_reference/multiline_values.json | 8 ++++++ .../select_expressions.json | 2 ++ .../test/fixtures_reference/terms.json | 1 + .../fixtures_structure/escape_sequences.json | 14 ++++++++--- .../test/fixtures_structure/leading_dots.json | 7 ++++++ .../variant_with_empty_pattern.json | 1 + .../whitespace_leading.json | 2 ++ .../whitespace_trailing.json | 1 + 17 files changed, 89 insertions(+), 20 deletions(-) diff --git a/fluent-syntax/src/ast.js b/fluent-syntax/src/ast.js index 0751b44ce..fea7823c7 100644 --- a/fluent-syntax/src/ast.js +++ b/fluent-syntax/src/ast.js @@ -96,9 +96,10 @@ export class Placeable extends PatternElement { export class Expression extends SyntaxNode {} export class StringLiteral extends Expression { - constructor(value) { + constructor(raw, value) { super(); this.type = "StringLiteral"; + this.raw = raw; this.value = value; } } diff --git a/fluent-syntax/src/parser.js b/fluent-syntax/src/parser.js index 91e8d9e7c..89376f267 100644 --- a/fluent-syntax/src/parser.js +++ b/fluent-syntax/src/parser.js @@ -601,7 +601,7 @@ export default class FluentParser { if (next === "\\" || next === "\"") { ps.next(); - return `\\${next}`; + return [`\\${next}`, next]; } if (next === "u") { @@ -618,7 +618,15 @@ export default class FluentParser { sequence += ch; } - return `\\u${sequence}`; + const codepoint = parseInt(sequence, 16); + const unescaped = codepoint <= 0xD7FF || 0xE000 <= codepoint + // It's a Unicode scalar value. + ? String.fromCodePoint(codepoint) + // Escape sequences reresenting surrogate code points are well-formed + // but invalid in Fluent. Replace them with U+FFFD REPLACEMENT + // CHARACTER. + : "๏ฟฝ"; + return [`\\u${sequence}`, unescaped]; } throw new ParseError("E0025", next); @@ -805,16 +813,20 @@ export default class FluentParser { } getString(ps) { - let val = ""; + let raw = ""; + let value = ""; ps.expectChar("\""); let ch; while ((ch = ps.takeChar(x => x !== '"' && x !== EOL))) { if (ch === "\\") { - val += this.getEscapeSequence(ps); + const [sequence, unescaped] = this.getEscapeSequence(ps); + raw += sequence; + value += unescaped; } else { - val += ch; + raw += ch; + value += ch; } } @@ -824,8 +836,7 @@ export default class FluentParser { ps.expectChar("\""); - return new AST.StringLiteral(val); - + return new AST.StringLiteral(raw, value); } getLiteral(ps) { diff --git a/fluent-syntax/src/serializer.js b/fluent-syntax/src/serializer.js index dd9f499e1..48ef8d745 100644 --- a/fluent-syntax/src/serializer.js +++ b/fluent-syntax/src/serializer.js @@ -211,7 +211,7 @@ function serializePlaceable(placeable) { function serializeExpression(expr) { switch (expr.type) { case "StringLiteral": - return `"${expr.value}"`; + return `"${expr.raw}"`; case "NumberLiteral": return expr.value; case "MessageReference": diff --git a/fluent-syntax/test/fixtures_reference/astral.json b/fluent-syntax/test/fixtures_reference/astral.json index 6056fb727..6f748c9c0 100644 --- a/fluent-syntax/test/fixtures_reference/astral.json +++ b/fluent-syntax/test/fixtures_reference/astral.json @@ -68,7 +68,8 @@ "type": "Placeable", "expression": { "type": "StringLiteral", - "value": "\\uD83D\\uDE02" + "raw": "\\uD83D\\uDE02", + "value": "๏ฟฝ๏ฟฝ" } } ] @@ -89,14 +90,16 @@ "type": "Placeable", "expression": { "type": "StringLiteral", - "value": "\\uD83D" + "raw": "\\uD83D", + "value": "๏ฟฝ" } }, { "type": "Placeable", "expression": { "type": "StringLiteral", - "value": "\\uDE02" + "raw": "\\uDE02", + "value": "๏ฟฝ" } } ] @@ -135,6 +138,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "A face ๐Ÿ˜‚ with tears of joy.", "value": "A face ๐Ÿ˜‚ with tears of joy." } } diff --git a/fluent-syntax/test/fixtures_reference/call_expressions.json b/fluent-syntax/test/fixtures_reference/call_expressions.json index e5a451938..0a3ae0697 100644 --- a/fluent-syntax/test/fixtures_reference/call_expressions.json +++ b/fluent-syntax/test/fixtures_reference/call_expressions.json @@ -101,6 +101,7 @@ }, { "type": "StringLiteral", + "raw": "a", "value": "a" }, { @@ -160,6 +161,7 @@ }, "value": { "type": "StringLiteral", + "raw": "Y", "value": "Y" } } @@ -212,6 +214,7 @@ }, "value": { "type": "StringLiteral", + "raw": "Y", "value": "Y" } } @@ -250,6 +253,7 @@ }, { "type": "StringLiteral", + "raw": "a", "value": "a" }, { @@ -280,6 +284,7 @@ }, "value": { "type": "StringLiteral", + "raw": "Y", "value": "Y" } } @@ -336,6 +341,7 @@ "positional": [ { "type": "StringLiteral", + "raw": "a", "value": "a" }, { @@ -418,6 +424,7 @@ "positional": [ { "type": "StringLiteral", + "raw": "a", "value": "a" }, { @@ -471,6 +478,7 @@ "positional": [ { "type": "StringLiteral", + "raw": "a", "value": "a" }, { @@ -587,6 +595,7 @@ "positional": [ { "type": "StringLiteral", + "raw": "a", "value": "a" } ], diff --git a/fluent-syntax/test/fixtures_reference/escaped_characters.json b/fluent-syntax/test/fixtures_reference/escaped_characters.json index 26b1974b0..91de49944 100644 --- a/fluent-syntax/test/fixtures_reference/escaped_characters.json +++ b/fluent-syntax/test/fixtures_reference/escaped_characters.json @@ -122,7 +122,8 @@ "type": "Placeable", "expression": { "type": "StringLiteral", - "value": "\\\"" + "raw": "\\\"", + "value": "\"" } } ] @@ -143,7 +144,8 @@ "type": "Placeable", "expression": { "type": "StringLiteral", - "value": "\\\\" + "raw": "\\\\", + "value": "\\" } } ] @@ -186,7 +188,8 @@ "type": "Placeable", "expression": { "type": "StringLiteral", - "value": "\\u0041" + "raw": "\\u0041", + "value": "A" } } ] @@ -207,7 +210,8 @@ "type": "Placeable", "expression": { "type": "StringLiteral", - "value": "\\\\u0041" + "raw": "\\\\u0041", + "value": "\\u0041" } } ] @@ -236,6 +240,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "{", "value": "{" } }, @@ -265,6 +270,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "}", "value": "}" } }, diff --git a/fluent-syntax/test/fixtures_reference/leading_dots.json b/fluent-syntax/test/fixtures_reference/leading_dots.json index 6459cb57c..1787d0ee0 100644 --- a/fluent-syntax/test/fixtures_reference/leading_dots.json +++ b/fluent-syntax/test/fixtures_reference/leading_dots.json @@ -50,6 +50,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": "." } }, @@ -75,6 +76,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": "." } }, @@ -104,6 +106,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": "." } }, @@ -133,6 +136,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": "." } }, @@ -229,6 +233,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": "." } }, @@ -316,6 +321,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": "." } }, @@ -377,6 +383,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": "." } }, @@ -463,6 +470,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": "." } }, diff --git a/fluent-syntax/test/fixtures_reference/literal_expressions.json b/fluent-syntax/test/fixtures_reference/literal_expressions.json index a979bc623..da73e1311 100644 --- a/fluent-syntax/test/fixtures_reference/literal_expressions.json +++ b/fluent-syntax/test/fixtures_reference/literal_expressions.json @@ -14,6 +14,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "abc", "value": "abc" } } diff --git a/fluent-syntax/test/fixtures_reference/messages.json b/fluent-syntax/test/fixtures_reference/messages.json index 1a5b5c46f..fa8dcbd7b 100644 --- a/fluent-syntax/test/fixtures_reference/messages.json +++ b/fluent-syntax/test/fixtures_reference/messages.json @@ -218,6 +218,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "", "value": "" } } diff --git a/fluent-syntax/test/fixtures_reference/multiline_values.json b/fluent-syntax/test/fixtures_reference/multiline_values.json index 17fb6f8e4..4d3dd033e 100644 --- a/fluent-syntax/test/fixtures_reference/multiline_values.json +++ b/fluent-syntax/test/fixtures_reference/multiline_values.json @@ -126,6 +126,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "placeables", "value": "placeables" } }, @@ -137,6 +138,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "at", "value": "at" } }, @@ -148,6 +150,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "of lines", "value": "of lines" } }, @@ -155,6 +158,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": "." } } @@ -176,6 +180,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "A multiline value", "value": "A multiline value" } }, @@ -187,6 +192,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "with a placeable", "value": "with a placeable" } } @@ -280,6 +286,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": "." } }, @@ -309,6 +316,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": "." } } diff --git a/fluent-syntax/test/fixtures_reference/select_expressions.json b/fluent-syntax/test/fixtures_reference/select_expressions.json index 6bd9a698c..61e18fa5c 100644 --- a/fluent-syntax/test/fixtures_reference/select_expressions.json +++ b/fluent-syntax/test/fixtures_reference/select_expressions.json @@ -57,6 +57,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "", "value": "" } }, @@ -169,6 +170,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "", "value": "" } } diff --git a/fluent-syntax/test/fixtures_reference/terms.json b/fluent-syntax/test/fixtures_reference/terms.json index f63122e57..d3eff8065 100644 --- a/fluent-syntax/test/fixtures_reference/terms.json +++ b/fluent-syntax/test/fixtures_reference/terms.json @@ -49,6 +49,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "", "value": "" } } diff --git a/fluent-syntax/test/fixtures_structure/escape_sequences.json b/fluent-syntax/test/fixtures_structure/escape_sequences.json index e9e851c7b..dccf18cb5 100644 --- a/fluent-syntax/test/fixtures_structure/escape_sequences.json +++ b/fluent-syntax/test/fixtures_structure/escape_sequences.json @@ -252,7 +252,8 @@ "type": "Placeable", "expression": { "type": "StringLiteral", - "value": "\\\"", + "raw": "\\\"", + "value": "\"", "span": { "type": "Span", "start": 262, @@ -298,7 +299,8 @@ "type": "Placeable", "expression": { "type": "StringLiteral", - "value": "\\\\", + "raw": "\\\\", + "value": "\\", "span": { "type": "Span", "start": 291, @@ -419,7 +421,8 @@ "type": "Placeable", "expression": { "type": "StringLiteral", - "value": "\\u0041", + "raw": "\\u0041", + "value": "A", "span": { "type": "Span", "start": 443, @@ -465,7 +468,8 @@ "type": "Placeable", "expression": { "type": "StringLiteral", - "value": "\\\\u0041", + "raw": "\\\\u0041", + "value": "\\u0041", "span": { "type": "Span", "start": 479, @@ -529,6 +533,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "{", "value": "{", "span": { "type": "Span", @@ -593,6 +598,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "}", "value": "}", "span": { "type": "Span", diff --git a/fluent-syntax/test/fixtures_structure/leading_dots.json b/fluent-syntax/test/fixtures_structure/leading_dots.json index d2e234c38..f95994c10 100644 --- a/fluent-syntax/test/fixtures_structure/leading_dots.json +++ b/fluent-syntax/test/fixtures_structure/leading_dots.json @@ -95,6 +95,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": ".", "span": { "type": "Span", @@ -150,6 +151,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": ".", "span": { "type": "Span", @@ -214,6 +216,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": ".", "span": { "type": "Span", @@ -278,6 +281,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": ".", "span": { "type": "Span", @@ -489,6 +493,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": ".", "span": { "type": "Span", @@ -671,6 +676,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": ".", "span": { "type": "Span", @@ -797,6 +803,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": ".", "span": { "type": "Span", diff --git a/fluent-syntax/test/fixtures_structure/variant_with_empty_pattern.json b/fluent-syntax/test/fixtures_structure/variant_with_empty_pattern.json index 2b61ee990..c1d77e6dd 100644 --- a/fluent-syntax/test/fixtures_structure/variant_with_empty_pattern.json +++ b/fluent-syntax/test/fixtures_structure/variant_with_empty_pattern.json @@ -47,6 +47,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "", "value": "", "span": { "type": "Span", diff --git a/fluent-syntax/test/fixtures_structure/whitespace_leading.json b/fluent-syntax/test/fixtures_structure/whitespace_leading.json index 168eee92f..1ee9e575c 100644 --- a/fluent-syntax/test/fixtures_structure/whitespace_leading.json +++ b/fluent-syntax/test/fixtures_structure/whitespace_leading.json @@ -111,6 +111,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "", "value": "", "span": { "type": "Span", @@ -166,6 +167,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": " ", "value": " ", "span": { "type": "Span", diff --git a/fluent-syntax/test/fixtures_structure/whitespace_trailing.json b/fluent-syntax/test/fixtures_structure/whitespace_trailing.json index 69e33deb4..0a58feab0 100644 --- a/fluent-syntax/test/fixtures_structure/whitespace_trailing.json +++ b/fluent-syntax/test/fixtures_structure/whitespace_trailing.json @@ -192,6 +192,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": " ", "value": " ", "span": { "type": "Span", From a7c58ea5c4716a4f4580173a81bfe8acc1f09fd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Mon, 26 Nov 2018 14:16:43 +0100 Subject: [PATCH 10/23] (fluent) Print missing variables with the $ sigil --- fluent/src/resolver.js | 4 ++-- fluent/test/arguments_test.js | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/fluent/src/resolver.js b/fluent/src/resolver.js index b1a29a0d1..f69d61432 100644 --- a/fluent/src/resolver.js +++ b/fluent/src/resolver.js @@ -363,7 +363,7 @@ function VariableReference(env, {name}) { if (!args || !args.hasOwnProperty(name)) { errors.push(new ReferenceError(`Unknown variable: ${name}`)); - return new FluentNone(name); + return new FluentNone(`$${name}`); } const arg = args[name]; @@ -387,7 +387,7 @@ function VariableReference(env, {name}) { errors.push( new TypeError(`Unsupported variable type: ${name}, ${typeof arg}`) ); - return new FluentNone(name); + return new FluentNone(`$${name}`); } } diff --git a/fluent/test/arguments_test.js b/fluent/test/arguments_test.js index 4e7a452d0..ef51e6086 100644 --- a/fluent/test/arguments_test.js +++ b/fluent/test/arguments_test.js @@ -101,49 +101,49 @@ suite('Variables', function() { test('falls back to argument\'s name if it\'s missing', function() { const msg = bundle.getMessage('foo'); const val = bundle.format(msg, {}, errs); - assert.equal(val, 'arg'); + assert.equal(val, '$arg'); assert(errs[0] instanceof ReferenceError); // unknown variable }); test('cannot be arrays', function() { const msg = bundle.getMessage('foo'); const val = bundle.format(msg, { arg: [1, 2, 3] }, errs); - assert.equal(val, 'arg'); + assert.equal(val, '$arg'); assert(errs[0] instanceof TypeError); // unsupported variable type }); test('cannot be a dict-like object', function() { const msg = bundle.getMessage('foo'); const val = bundle.format(msg, { arg: { prop: 1 } }, errs); - assert.equal(val, 'arg'); + assert.equal(val, '$arg'); assert(errs[0] instanceof TypeError); // unsupported variable type }); test('cannot be a boolean', function() { const msg = bundle.getMessage('foo'); const val = bundle.format(msg, { arg: true }, errs); - assert.equal(val, 'arg'); + assert.equal(val, '$arg'); assert(errs[0] instanceof TypeError); // unsupported variable type }); test('cannot be undefined', function() { const msg = bundle.getMessage('foo'); const val = bundle.format(msg, { arg: undefined }, errs); - assert.equal(val, 'arg'); + assert.equal(val, '$arg'); assert(errs[0] instanceof TypeError); // unsupported variable type }); test('cannot be null', function() { const msg = bundle.getMessage('foo'); const val = bundle.format(msg, { arg: null }, errs); - assert.equal(val, 'arg'); + assert.equal(val, '$arg'); assert(errs[0] instanceof TypeError); // unsupported variable type }); test('cannot be a function', function() { const msg = bundle.getMessage('foo'); const val = bundle.format(msg, { arg: () => null }, errs); - assert.equal(val, 'arg'); + assert.equal(val, '$arg'); assert(errs[0] instanceof TypeError); // unsupported variable type }); }); From 8d5306d8a03ac57eb3d26bcdea32808ee98d5f73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Mon, 26 Nov 2018 14:15:15 +0100 Subject: [PATCH 11/23] (fluent-syntax) Add parameterized terms --- fluent-syntax/src/errors.js | 2 +- fluent-syntax/src/parser.js | 67 +++--- fluent-syntax/src/serializer.js | 6 +- fluent-syntax/test/fixtures_behavior/term.ftl | 13 +- .../fixtures_reference/call_expressions.ftl | 3 +- .../fixtures_reference/call_expressions.json | 38 +++- .../fixtures_reference/select_expressions.ftl | 16 +- .../select_expressions.json | 22 +- .../fixtures_reference/term_parameters.ftl | 8 + .../fixtures_reference/term_parameters.json | 203 ++++++++++++++++++ fluent-syntax/test/serializer_test.js | 7 + 11 files changed, 328 insertions(+), 57 deletions(-) create mode 100644 fluent-syntax/test/fixtures_reference/term_parameters.ftl create mode 100644 fluent-syntax/test/fixtures_reference/term_parameters.json diff --git a/fluent-syntax/src/errors.js b/fluent-syntax/src/errors.js index b13c301e3..1cc8f1c57 100644 --- a/fluent-syntax/src/errors.js +++ b/fluent-syntax/src/errors.js @@ -51,7 +51,7 @@ function getErrorMessage(code, args) { case "E0016": return "Message references cannot be used as selectors"; case "E0017": - return "Variants cannot be used as selectors"; + return "Terms cannot be used as selectors"; case "E0018": return "Attributes of messages cannot be used as selectors"; case "E0019": diff --git a/fluent-syntax/src/parser.js b/fluent-syntax/src/parser.js index 89376f267..61a06d7c7 100644 --- a/fluent-syntax/src/parser.js +++ b/fluent-syntax/src/parser.js @@ -655,12 +655,18 @@ export default class FluentParser { throw new ParseError("E0016"); } - if (selector.type === "AttributeExpression" && - selector.ref.type === "MessageReference") { + if (selector.type === "AttributeExpression" + && selector.ref.type === "MessageReference") { throw new ParseError("E0018"); } - if (selector.type === "VariantExpression") { + if (selector.type === "TermReference" + || selector.type === "VariantExpression") { + throw new ParseError("E0017"); + } + + if (selector.type === "CallExpression" + && selector.callee.type === "TermReference") { throw new ParseError("E0017"); } @@ -672,8 +678,8 @@ export default class FluentParser { const variants = this.getVariants(ps, {allowVariantList: false}); return new AST.SelectExpression(selector, variants); - } else if (selector.type === "AttributeExpression" && - selector.ref.type === "TermReference") { + } else if (selector.type === "AttributeExpression" + && selector.ref.type === "TermReference") { throw new ParseError("E0019"); } @@ -685,60 +691,59 @@ export default class FluentParser { return this.getPlaceable(ps); } - const literal = this.getLiteral(ps); - - if (literal.type !== "MessageReference" - && literal.type !== "TermReference") { - return literal; + const selector = this.getLiteral(ps); + switch (selector.type) { + case "StringLiteral": + case "NumberLiteral": + case "VariableReference": + return selector; } - const ch = ps.currentChar; - - if (ch === ".") { + if (ps.currentChar === ".") { ps.next(); - const attr = this.getIdentifier(ps); - return new AST.AttributeExpression(literal, attr); + return new AST.AttributeExpression(selector, attr); } - if (ch === "[") { + if (ps.currentChar === "[") { ps.next(); - if (literal.type === "MessageReference") { + if (selector.type === "MessageReference") { throw new ParseError("E0024"); } const key = this.getVariantKey(ps); - ps.expectChar("]"); - - return new AST.VariantExpression(literal, key); + return new AST.VariantExpression(selector, key); } - if (ch === "(") { + if (ps.currentChar === "(") { ps.next(); - if (!/^[A-Z][A-Z_?-]*$/.test(literal.id.name)) { - throw new ParseError("E0008"); + if (selector.type === "MessageReference") { + if (/^[A-Z][A-Z_?-]*$/.test(selector.id.name)) { + // The callee is a Function. + var func = new AST.FunctionReference(selector.id); + if (this.withSpans) { + func.addSpan(selector.span.start, selector.span.end); + } + } else { + // Messages can't be callees. + throw new ParseError("E0008"); + } } const args = this.getCallArgs(ps); - ps.expectChar(")"); - const func = new AST.FunctionReference(literal.id); - if (this.withSpans) { - func.addSpan(literal.span.start, literal.span.end); - } - return new AST.CallExpression( - func, + func || selector, args.positional, args.named, ); } - return literal; + return selector; } getCallArg(ps) { diff --git a/fluent-syntax/src/serializer.js b/fluent-syntax/src/serializer.js index 48ef8d745..3c2c10a57 100644 --- a/fluent-syntax/src/serializer.js +++ b/fluent-syntax/src/serializer.js @@ -265,13 +265,13 @@ function serializeVariantExpression(expr) { function serializeCallExpression(expr) { - const fun = serializeExpression(expr.callee); + const callee = serializeExpression(expr.callee); const positional = expr.positional.map(serializeExpression).join(", "); const named = expr.named.map(serializeNamedArgument).join(", "); if (expr.positional.length > 0 && expr.named.length > 0) { - return `${fun}(${positional}, ${named})`; + return `${callee}(${positional}, ${named})`; } - return `${fun}(${positional || named})`; + return `${callee}(${positional || named})`; } diff --git a/fluent-syntax/test/fixtures_behavior/term.ftl b/fluent-syntax/test/fixtures_behavior/term.ftl index 58426d638..303225fde 100644 --- a/fluent-syntax/test/fixtures_behavior/term.ftl +++ b/fluent-syntax/test/fixtures_behavior/term.ftl @@ -11,22 +11,21 @@ key2 = key3 = Test { -brand-short-name[accusative] } +key4 = { -brand() } + +# ~ERROR E0004, pos 306, args "0-9" err1 = { $foo -> [one] Foo *[-other] Foo 2 } -# ~ERROR E0004, pos 285, args "0-9" +# ~ERROR E0004, pos 336, args "a-zA-Z" err2 = { $-foo } -# ~ERROR E0004, pos 315, args "a-zA-Z" - -err4 = { -brand() } -# ~ERROR E0008, pos 339 --err5 = # ~ERROR E0006, pos 351, args "err5" +-err5 = +# ~ERROR E0006, pos 360, args "err6" -err6 = .attr = Attribute -# ~ERROR E0006, pos 360, args "err6" diff --git a/fluent-syntax/test/fixtures_reference/call_expressions.ftl b/fluent-syntax/test/fixtures_reference/call_expressions.ftl index 9ed69bd1b..a4f61da40 100644 --- a/fluent-syntax/test/fixtures_reference/call_expressions.ftl +++ b/fluent-syntax/test/fixtures_reference/call_expressions.ftl @@ -1,14 +1,13 @@ ## Callees function-callee = {FUNCTION()} +term-callee = {-term()} # ERROR Equivalent to a MessageReference callee. mixed-case-callee = {Function()} # ERROR MessageReference is not a valid callee. message-callee = {message()} -# ERROR TermReference is not a valid callee. -term-callee = {-term()} # ERROR VariableReference is not a valid callee. variable-callee = {$variable()} diff --git a/fluent-syntax/test/fixtures_reference/call_expressions.json b/fluent-syntax/test/fixtures_reference/call_expressions.json index 0a3ae0697..4cc36d404 100644 --- a/fluent-syntax/test/fixtures_reference/call_expressions.json +++ b/fluent-syntax/test/fixtures_reference/call_expressions.json @@ -34,6 +34,35 @@ "attributes": [], "comment": null }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "term-callee" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + }, + "positional": [], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, { "type": "Comment", "content": "ERROR Equivalent to a MessageReference callee." @@ -52,15 +81,6 @@ "annotations": [], "content": "message-callee = {message()}\n" }, - { - "type": "Comment", - "content": "ERROR TermReference is not a valid callee." - }, - { - "type": "Junk", - "annotations": [], - "content": "term-callee = {-term()}\n" - }, { "type": "Comment", "content": "ERROR VariableReference is not a valid callee." diff --git a/fluent-syntax/test/fixtures_reference/select_expressions.ftl b/fluent-syntax/test/fixtures_reference/select_expressions.ftl index 859c01a04..ac888262b 100644 --- a/fluent-syntax/test/fixtures_reference/select_expressions.ftl +++ b/fluent-syntax/test/fixtures_reference/select_expressions.ftl @@ -4,17 +4,29 @@ new-messages = *[other] {""}Other } -valid-selector = +valid-selector-term-attribute = { -term.case -> *[key] value } # ERROR -invalid-selector = +invalid-selector-term-value = + { -term -> + *[key] value + } + +# ERROR +invalid-selector-term-variant = { -term[case] -> *[key] value } +# ERROR +invalid-selector-term-call = + { -term(case: "nominative") -> + *[key] value + } + empty-variant = { 1 -> *[one] {""} diff --git a/fluent-syntax/test/fixtures_reference/select_expressions.json b/fluent-syntax/test/fixtures_reference/select_expressions.json index 61e18fa5c..7cce4070d 100644 --- a/fluent-syntax/test/fixtures_reference/select_expressions.json +++ b/fluent-syntax/test/fixtures_reference/select_expressions.json @@ -81,7 +81,7 @@ "type": "Message", "id": { "type": "Identifier", - "name": "valid-selector" + "name": "valid-selector-term-attribute" }, "value": { "type": "Pattern", @@ -137,7 +137,25 @@ { "type": "Junk", "annotations": [], - "content": "invalid-selector =\n { -term[case] ->\n *[key] value\n }\n" + "content": "invalid-selector-term-value =\n { -term ->\n *[key] value\n }\n" + }, + { + "type": "Comment", + "content": "ERROR" + }, + { + "type": "Junk", + "annotations": [], + "content": "invalid-selector-term-variant =\n { -term[case] ->\n *[key] value\n }\n" + }, + { + "type": "Comment", + "content": "ERROR" + }, + { + "type": "Junk", + "annotations": [], + "content": "invalid-selector-term-call =\n { -term(case: \"nominative\") ->\n *[key] value\n }\n" }, { "type": "Message", diff --git a/fluent-syntax/test/fixtures_reference/term_parameters.ftl b/fluent-syntax/test/fixtures_reference/term_parameters.ftl new file mode 100644 index 000000000..614423611 --- /dev/null +++ b/fluent-syntax/test/fixtures_reference/term_parameters.ftl @@ -0,0 +1,8 @@ +-term = { $arg -> + *[key] Value +} + +key01 = { -term } +key02 = { -term() } +key03 = { -term(arg: 1) } +key04 = { -term("positional", narg1: 1, narg2: 2) } diff --git a/fluent-syntax/test/fixtures_reference/term_parameters.json b/fluent-syntax/test/fixtures_reference/term_parameters.json new file mode 100644 index 000000000..f9f09613e --- /dev/null +++ b/fluent-syntax/test/fixtures_reference/term_parameters.json @@ -0,0 +1,203 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "term" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "arg" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key01" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key02" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + }, + "positional": [], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key03" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + }, + "positional": [], + "named": [ + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "arg" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key04" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + }, + "positional": [ + { + "type": "StringLiteral", + "raw": "positional", + "value": "positional" + } + ], + "named": [ + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "narg1" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + }, + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "narg2" + }, + "value": { + "type": "NumberLiteral", + "value": "2" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + } + ] +} diff --git a/fluent-syntax/test/serializer_test.js b/fluent-syntax/test/serializer_test.js index cb32b9caa..54c537b32 100644 --- a/fluent-syntax/test/serializer_test.js +++ b/fluent-syntax/test/serializer_test.js @@ -450,6 +450,13 @@ suite("Serialize resource", function() { assert.equal(pretty(input), input); }); + test("macro call", function() { + const input = ftl` + foo = { -term() } + `; + assert.equal(pretty(input), input); + }); + test("nested placeables", function() { const input = ftl` foo = {{ FOO() }} From ecf32071f3f81e49f5c32e549fc77cf53c504257 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Mon, 26 Nov 2018 14:16:58 +0100 Subject: [PATCH 12/23] (fluent) Add parameterized terms --- fluent/src/resolver.js | 58 ++- fluent/src/resource.js | 3 +- fluent/test/fixtures_behavior/term.json | 3 + .../fixtures_reference/call_expressions.json | 140 +++---- .../select_expressions.json | 53 ++- .../fixtures_reference/term_parameters.json | 84 +++++ .../expressions_call_args.json | 4 +- fluent/test/macros_test.js | 347 ++++++++++++++++++ 8 files changed, 601 insertions(+), 91 deletions(-) create mode 100644 fluent/test/fixtures_reference/term_parameters.json create mode 100644 fluent/test/macros_test.js diff --git a/fluent/src/resolver.js b/fluent/src/resolver.js index f69d61432..7eeb08cba 100644 --- a/fluent/src/resolver.js +++ b/fluent/src/resolver.js @@ -29,9 +29,10 @@ * instantly resolve to the value of the `brand-name` message. Instead, we * want to get the message object and look for its `nominative` variant. * - * All other expressions (except for `FunctionReference` which is only used in - * `CallExpression`) resolve to an instance of `FluentType`. The caller should - * use the `toString` method to convert the instance to a native value. + * All other expressions (except for `FunctionReference` which is only used as + * a callee in `FunctionExpression`) resolve to an instance of `FluentType`. + * The caller should use the `toString` method to convert the instance to a + * native value. * * * All functions in this file pass around a special object called `env`. @@ -138,12 +139,12 @@ function DefaultMember(env, members, star) { */ function MessageReference(env, {name}) { const { bundle, errors } = env; - const message = name.startsWith("-") + const message = name[0] === "-" ? bundle._terms.get(name) : bundle._messages.get(name); if (!message) { - const err = name.startsWith("-") + const err = name[0] === "-" ? new ReferenceError(`Unknown term: ${name}`) : new ReferenceError(`Unknown message: ${name}`); errors.push(err); @@ -311,13 +312,15 @@ function Type(env, expr) { return new FluentNumber(expr.value); case "var": return VariableReference(env, expr); - case "func": - return FunctionReference(env, expr); case "call": - return CallExpression(env, expr); + return expr.ref.name[0] === "-" + ? MacroExpression(env, expr) + : FunctionExpression(env, expr); case "ref": { const message = MessageReference(env, expr); - return Type(env, message); + return expr.name[0] === "-" + ? Type({...env, args: {}}, message) + : Type(env, message); } case "getattr": { const attr = AttributeExpression(env, expr); @@ -429,16 +432,15 @@ function FunctionReference(env, {name}) { * Resolver environment object. * @param {Object} expr * An expression to be resolved. - * @param {Object} expr.callee + * @param {Object} expr.ref * FTL Function object. * @param {Array} expr.args * FTL Function argument list. * @returns {FluentType} * @private */ -function CallExpression(env, {callee, args}) { - const func = FunctionReference(env, callee); - +function FunctionExpression(env, {ref, args}) { + const func = FunctionReference(env, ref); if (func instanceof FluentNone) { return func; } @@ -462,6 +464,36 @@ function CallExpression(env, {callee, args}) { } } +/** + * Resolve a call to a Term with key-value arguments. + * + * @param {Object} env + * Resolver environment object. + * @param {Object} expr + * An expression to be resolved. + * @param {Object} expr.ref + * FTL Function object. + * @param {Array} expr.args + * FTL Function argument list. + * @returns {FluentType} + * @private + */ +function MacroExpression(env, {ref, args}) { + const callee = MessageReference(env, ref); + if (callee instanceof FluentNone) { + return callee; + } + + const keyargs = {}; + for (const arg of args) { + if (arg.type === "narg") { + keyargs[arg.name] = Type(env, arg.value); + } + } + + return Type({...env, args: keyargs}, callee); +} + /** * Resolve a pattern (a complex string with placeables). * diff --git a/fluent/src/resource.js b/fluent/src/resource.js index d1b6421c4..7afa41a3e 100644 --- a/fluent/src/resource.js +++ b/fluent/src/resource.js @@ -311,8 +311,7 @@ export default class FluentResource extends Map { } if (consumeToken(TOKEN_PAREN_OPEN)) { - let callee = {...ref, type: "func"}; - return {type: "call", callee, args: parseArguments()}; + return {type: "call", ref, args: parseArguments()}; } return ref; diff --git a/fluent/test/fixtures_behavior/term.json b/fluent/test/fixtures_behavior/term.json index e8825fb2b..f90a9c246 100644 --- a/fluent/test/fixtures_behavior/term.json +++ b/fluent/test/fixtures_behavior/term.json @@ -13,5 +13,8 @@ }, "key3": { "value": true + }, + "key4": { + "value": true } } diff --git a/fluent/test/fixtures_reference/call_expressions.json b/fluent/test/fixtures_reference/call_expressions.json index 5f7c8ad63..10682530e 100644 --- a/fluent/test/fixtures_reference/call_expressions.json +++ b/fluent/test/fixtures_reference/call_expressions.json @@ -2,39 +2,39 @@ "function-callee": [ { "type": "call", - "callee": { - "type": "func", + "ref": { + "type": "ref", "name": "FUNCTION" }, "args": [] } ], - "mixed-case-callee": [ + "term-callee": [ { "type": "call", - "callee": { - "type": "func", - "name": "Function" + "ref": { + "type": "ref", + "name": "-term" }, "args": [] } ], - "message-callee": [ + "mixed-case-callee": [ { "type": "call", - "callee": { - "type": "func", - "name": "message" + "ref": { + "type": "ref", + "name": "Function" }, "args": [] } ], - "term-callee": [ + "message-callee": [ { "type": "call", - "callee": { - "type": "func", - "name": "-term" + "ref": { + "type": "ref", + "name": "message" }, "args": [] } @@ -42,8 +42,8 @@ "positional-args": [ { "type": "call", - "callee": { - "type": "func", + "ref": { + "type": "ref", "name": "FUN" }, "args": [ @@ -65,8 +65,8 @@ "named-args": [ { "type": "call", - "callee": { - "type": "func", + "ref": { + "type": "ref", "name": "FUN" }, "args": [ @@ -92,8 +92,8 @@ "dense-named-args": [ { "type": "call", - "callee": { - "type": "func", + "ref": { + "type": "ref", "name": "FUN" }, "args": [ @@ -119,8 +119,8 @@ "mixed-args": [ { "type": "call", - "callee": { - "type": "func", + "ref": { + "type": "ref", "name": "FUN" }, "args": [ @@ -158,8 +158,8 @@ "shuffled-args": [ { "type": "call", - "callee": { - "type": "func", + "ref": { + "type": "ref", "name": "FUN" }, "args": [ @@ -197,8 +197,8 @@ "duplicate-named-args": [ { "type": "call", - "callee": { - "type": "func", + "ref": { + "type": "ref", "name": "FUN" }, "args": [ @@ -224,8 +224,8 @@ "sparse-inline-call": [ { "type": "call", - "callee": { - "type": "func", + "ref": { + "type": "ref", "name": "FUN" }, "args": [ @@ -251,8 +251,8 @@ "empty-inline-call": [ { "type": "call", - "callee": { - "type": "func", + "ref": { + "type": "ref", "name": "FUN" }, "args": [] @@ -261,8 +261,8 @@ "multiline-call": [ { "type": "call", - "callee": { - "type": "func", + "ref": { + "type": "ref", "name": "FUN" }, "args": [ @@ -288,8 +288,8 @@ "sparse-multiline-call": [ { "type": "call", - "callee": { - "type": "func", + "ref": { + "type": "ref", "name": "FUN" }, "args": [ @@ -315,8 +315,8 @@ "empty-multiline-call": [ { "type": "call", - "callee": { - "type": "func", + "ref": { + "type": "ref", "name": "FUN" }, "args": [] @@ -325,8 +325,8 @@ "unindented-arg-number": [ { "type": "call", - "callee": { - "type": "func", + "ref": { + "type": "ref", "name": "FUN" }, "args": [ @@ -340,8 +340,8 @@ "unindented-arg-string": [ { "type": "call", - "callee": { - "type": "func", + "ref": { + "type": "ref", "name": "FUN" }, "args": [ @@ -355,8 +355,8 @@ "unindented-arg-msg-ref": [ { "type": "call", - "callee": { - "type": "func", + "ref": { + "type": "ref", "name": "FUN" }, "args": [ @@ -370,8 +370,8 @@ "unindented-arg-term-ref": [ { "type": "call", - "callee": { - "type": "func", + "ref": { + "type": "ref", "name": "FUN" }, "args": [ @@ -385,8 +385,8 @@ "unindented-arg-var-ref": [ { "type": "call", - "callee": { - "type": "func", + "ref": { + "type": "ref", "name": "FUN" }, "args": [ @@ -400,15 +400,15 @@ "unindented-arg-call": [ { "type": "call", - "callee": { - "type": "func", + "ref": { + "type": "ref", "name": "FUN" }, "args": [ { "type": "call", - "callee": { - "type": "func", + "ref": { + "type": "ref", "name": "OTHER" }, "args": [] @@ -419,8 +419,8 @@ "unindented-named-arg": [ { "type": "call", - "callee": { - "type": "func", + "ref": { + "type": "ref", "name": "FUN" }, "args": [ @@ -438,8 +438,8 @@ "unindented-closing-paren": [ { "type": "call", - "callee": { - "type": "func", + "ref": { + "type": "ref", "name": "FUN" }, "args": [ @@ -453,8 +453,8 @@ "one-argument": [ { "type": "call", - "callee": { - "type": "func", + "ref": { + "type": "ref", "name": "FUN" }, "args": [ @@ -468,8 +468,8 @@ "many-arguments": [ { "type": "call", - "callee": { - "type": "func", + "ref": { + "type": "ref", "name": "FUN" }, "args": [ @@ -491,8 +491,8 @@ "inline-sparse-args": [ { "type": "call", - "callee": { - "type": "func", + "ref": { + "type": "ref", "name": "FUN" }, "args": [ @@ -514,8 +514,8 @@ "mulitline-args": [ { "type": "call", - "callee": { - "type": "func", + "ref": { + "type": "ref", "name": "FUN" }, "args": [ @@ -533,8 +533,8 @@ "mulitline-sparse-args": [ { "type": "call", - "callee": { - "type": "func", + "ref": { + "type": "ref", "name": "FUN" }, "args": [ @@ -552,8 +552,8 @@ "sparse-named-arg": [ { "type": "call", - "callee": { - "type": "func", + "ref": { + "type": "ref", "name": "FUN" }, "args": [ @@ -587,8 +587,8 @@ "unindented-colon": [ { "type": "call", - "callee": { - "type": "func", + "ref": { + "type": "ref", "name": "FUN" }, "args": [ @@ -606,8 +606,8 @@ "unindented-value": [ { "type": "call", - "callee": { - "type": "func", + "ref": { + "type": "ref", "name": "FUN" }, "args": [ diff --git a/fluent/test/fixtures_reference/select_expressions.json b/fluent/test/fixtures_reference/select_expressions.json index 65ed7c426..8b204d856 100644 --- a/fluent/test/fixtures_reference/select_expressions.json +++ b/fluent/test/fixtures_reference/select_expressions.json @@ -4,8 +4,8 @@ "type": "select", "selector": { "type": "call", - "callee": { - "type": "func", + "ref": { + "type": "ref", "name": "BUILTIN" }, "args": [] @@ -28,7 +28,7 @@ "star": 1 } ], - "valid-selector": [ + "valid-selector-term-attribute": [ { "type": "select", "selector": { @@ -48,7 +48,23 @@ "star": 0 } ], - "invalid-selector": [ + "invalid-selector-term-value": [ + { + "type": "select", + "selector": { + "type": "ref", + "name": "-term" + }, + "variants": [ + { + "key": "key", + "value": "value" + } + ], + "star": 0 + } + ], + "invalid-selector-term-variant": [ { "type": "select", "selector": { @@ -68,6 +84,35 @@ "star": 0 } ], + "invalid-selector-term-call": [ + { + "type": "select", + "selector": { + "type": "call", + "ref": { + "type": "ref", + "name": "-term" + }, + "args": [ + { + "type": "narg", + "name": "case", + "value": { + "type": "str", + "value": "nominative" + } + } + ] + }, + "variants": [ + { + "key": "key", + "value": "value" + } + ], + "star": 0 + } + ], "empty-variant": [ { "type": "select", diff --git a/fluent/test/fixtures_reference/term_parameters.json b/fluent/test/fixtures_reference/term_parameters.json new file mode 100644 index 000000000..78764b97c --- /dev/null +++ b/fluent/test/fixtures_reference/term_parameters.json @@ -0,0 +1,84 @@ +{ + "-term": [ + { + "type": "select", + "selector": { + "type": "var", + "name": "arg" + }, + "variants": [ + { + "key": "key", + "value": "Value" + } + ], + "star": 0 + } + ], + "key01": [ + { + "type": "ref", + "name": "-term" + } + ], + "key02": [ + { + "type": "call", + "ref": { + "type": "ref", + "name": "-term" + }, + "args": [] + } + ], + "key03": [ + { + "type": "call", + "ref": { + "type": "ref", + "name": "-term" + }, + "args": [ + { + "type": "narg", + "name": "arg", + "value": { + "type": "num", + "value": "1" + } + } + ] + } + ], + "key04": [ + { + "type": "call", + "ref": { + "type": "ref", + "name": "-term" + }, + "args": [ + { + "type": "str", + "value": "positional" + }, + { + "type": "narg", + "name": "narg1", + "value": { + "type": "num", + "value": "1" + } + }, + { + "type": "narg", + "name": "narg2", + "value": { + "type": "num", + "value": "2" + } + } + ] + } + ] +} diff --git a/fluent/test/fixtures_structure/expressions_call_args.json b/fluent/test/fixtures_structure/expressions_call_args.json index 0739b1d84..a15662ec0 100644 --- a/fluent/test/fixtures_structure/expressions_call_args.json +++ b/fluent/test/fixtures_structure/expressions_call_args.json @@ -2,8 +2,8 @@ "key": [ { "type": "call", - "callee": { - "type": "func", + "ref": { + "type": "ref", "name": "FOO" }, "args": [ diff --git a/fluent/test/macros_test.js b/fluent/test/macros_test.js new file mode 100644 index 000000000..e1e9c775d --- /dev/null +++ b/fluent/test/macros_test.js @@ -0,0 +1,347 @@ +"use strict"; + +import assert from "assert"; + +import FluentBundle from "../src/bundle"; +import { ftl } from "../src/util"; + +suite("Macros", function() { + let bundle, errs; + + setup(function() { + errs = []; + }); + + suite("References and calls", function(){ + suiteSetup(function() { + bundle = new FluentBundle("en-US", { + useIsolating: false, + }); + bundle.addMessages(ftl` + foo = Foo + message-call = {foo()} + + -bar = Bar + term-ref = {-bar} + term-call = {-bar()} + `); + }); + + test("messages cannot be parameterized", function() { + const msg = bundle.getMessage("message-call"); + const val = bundle.format(msg, {}, errs); + assert.equal(val, "foo()"); + assert.equal(errs.length, 1); + }); + + test("terms can be referenced without parens", function() { + const msg = bundle.getMessage("term-ref"); + const val = bundle.format(msg, {}, errs); + assert.equal(val, "Bar"); + assert.equal(errs.length, 0); + }); + + test("terms can be parameterized", function() { + const msg = bundle.getMessage("term-call"); + const val = bundle.format(msg, {}, errs); + assert.equal(val, "Bar"); + assert.equal(errs.length, 0); + }); + }); + + suite("Passing arguments", function(){ + suiteSetup(function() { + bundle = new FluentBundle("en-US", { + useIsolating: false, + }); + bundle.addMessages(ftl` + -foo = Foo {$arg} + + ref-foo = {-foo} + call-foo-no-args = {-foo()} + call-foo-with-expected-arg = {-foo(arg: 1)} + call-foo-with-other-arg = {-foo(other: 3)} + `); + }); + + test("Not parameterized, no externals", function() { + const msg = bundle.getMessage("ref-foo"); + const val = bundle.format(msg, {}, errs); + assert.equal(val, "Foo $arg"); + assert.equal(errs.length, 1); + }); + + test("Not parameterized but with externals", function() { + const msg = bundle.getMessage("ref-foo"); + const val = bundle.format(msg, {arg: 1}, errs); + assert.equal(val, "Foo $arg"); + assert.equal(errs.length, 1); + }); + + test("No arguments, no externals", function() { + const msg = bundle.getMessage("call-foo-no-args"); + const val = bundle.format(msg, {}, errs); + assert.equal(val, "Foo $arg"); + assert.equal(errs.length, 1); + }); + + test("No arguments, but with externals", function() { + const msg = bundle.getMessage("call-foo-no-args"); + const val = bundle.format(msg, {arg: 1}, errs); + assert.equal(val, "Foo $arg"); + assert.equal(errs.length, 1); + }); + + test("With expected args, no externals", function() { + const msg = bundle.getMessage("call-foo-with-expected-arg"); + const val = bundle.format(msg, {}, errs); + assert.equal(val, "Foo 1"); + assert.equal(errs.length, 0); + }); + + test("With expected args, and with externals", function() { + const msg = bundle.getMessage("call-foo-with-expected-arg"); + const val = bundle.format(msg, {arg: 5}, errs); + assert.equal(val, "Foo 1"); + assert.equal(errs.length, 0); + }); + + test("With other args, no externals", function() { + const msg = bundle.getMessage("call-foo-with-other-arg"); + const val = bundle.format(msg, {}, errs); + assert.equal(val, "Foo $arg"); + assert.equal(errs.length, 1); + }); + + test("With other args, and with externals", function() { + const msg = bundle.getMessage("call-foo-with-other-arg"); + const val = bundle.format(msg, {arg: 5}, errs); + assert.equal(val, "Foo $arg"); + assert.equal(errs.length, 1); + }); + }); + + suite("Nesting message references", function(){ + suiteSetup(function() { + bundle = new FluentBundle("en-US", { + useIsolating: false, + }); + bundle.addMessages(ftl` + foo = Foo {$arg} + -bar = {foo} + ref-bar = {-bar} + call-bar = {-bar()} + call-bar-with-arg = {-bar(arg: 1)} + `); + }); + + test("No parameterization, no externals", function() { + const msg = bundle.getMessage("ref-bar"); + const val = bundle.format(msg, {}, errs); + assert.equal(val, "Foo $arg"); + assert.equal(errs.length, 1); + }); + + test("No parameterization, but with externals", function() { + const msg = bundle.getMessage("ref-bar"); + const val = bundle.format(msg, {arg: 5}, errs); + assert.equal(val, "Foo $arg"); + assert.equal(errs.length, 1); + }); + + test("No arguments, no externals", function() { + const msg = bundle.getMessage("call-bar"); + const val = bundle.format(msg, {}, errs); + assert.equal(val, "Foo $arg"); + assert.equal(errs.length, 1); + }); + + test("No arguments, but with externals", function() { + const msg = bundle.getMessage("call-bar"); + const val = bundle.format(msg, {arg: 5}, errs); + assert.equal(val, "Foo $arg"); + assert.equal(errs.length, 1); + }); + + test("With arguments, no externals", function() { + const msg = bundle.getMessage("call-bar-with-arg"); + const val = bundle.format(msg, {}, errs); + assert.equal(val, "Foo 1"); + assert.equal(errs.length, 0); + + }); + test("With arguments and with externals", function() { + const msg = bundle.getMessage("call-bar-with-arg"); + const val = bundle.format(msg, {arg: 5}, errs); + assert.equal(val, "Foo 1"); + assert.equal(errs.length, 0); + }); + }); + + suite("Nesting term references", function(){ + suiteSetup(function() { + bundle = new FluentBundle("en-US", { + useIsolating: false, + }); + bundle.addMessages(ftl` + -foo = Foo {$arg} + -bar = {-foo} + -baz = {-foo()} + -qux = {-foo(arg: 1)} + + ref-bar = {-bar} + ref-baz = {-baz} + ref-qux = {-qux} + + call-bar-no-args = {-bar()} + call-baz-no-args = {-baz()} + call-qux-no-args = {-qux()} + + call-bar-with-arg = {-bar(arg: 2)} + call-baz-with-arg = {-baz(arg: 2)} + call-qux-with-arg = {-qux(arg: 2)} + call-qux-with-other = {-qux(other: 3)} + `); + }); + + test("No parameterization, no parameterization, no externals", function() { + const msg = bundle.getMessage("ref-bar"); + const val = bundle.format(msg, {}, errs); + assert.equal(val, "Foo $arg"); + assert.equal(errs.length, 1); + }); + + test("No parameterization, no parameterization, with externals", function() { + const msg = bundle.getMessage("ref-bar"); + const val = bundle.format(msg, {arg: 5}, errs); + assert.equal(val, "Foo $arg"); + assert.equal(errs.length, 1); + }); + + test("No parameterization, no arguments, no externals", function() { + const msg = bundle.getMessage("ref-baz"); + const val = bundle.format(msg, {}, errs); + assert.equal(val, "Foo $arg"); + assert.equal(errs.length, 1); + }); + + test("No parameterization, no arguments, with externals", function() { + const msg = bundle.getMessage("ref-baz"); + const val = bundle.format(msg, {arg: 5}, errs); + assert.equal(val, "Foo $arg"); + assert.equal(errs.length, 1); + }); + + test("No parameterization, with arguments, no externals", function() { + const msg = bundle.getMessage("ref-qux"); + const val = bundle.format(msg, {}, errs); + assert.equal(val, "Foo 1"); + assert.equal(errs.length, 0); + }); + + test("No parameterization, with arguments, with externals", function() { + const msg = bundle.getMessage("ref-qux"); + const val = bundle.format(msg, {arg: 5}, errs); + assert.equal(val, "Foo 1"); + assert.equal(errs.length, 0); + }); + + test("No arguments, no parameterization, no externals", function() { + const msg = bundle.getMessage("call-bar-no-args"); + const val = bundle.format(msg, {}, errs); + assert.equal(val, "Foo $arg"); + assert.equal(errs.length, 1); + }); + + test("No arguments, no parameterization, with externals", function() { + const msg = bundle.getMessage("call-bar-no-args"); + const val = bundle.format(msg, {arg: 5}, errs); + assert.equal(val, "Foo $arg"); + assert.equal(errs.length, 1); + }); + + test("No arguments, no arguments, no externals", function() { + const msg = bundle.getMessage("call-baz-no-args"); + const val = bundle.format(msg, {}, errs); + assert.equal(val, "Foo $arg"); + assert.equal(errs.length, 1); + }); + + test("No arguments, no arguments, with externals", function() { + const msg = bundle.getMessage("call-baz-no-args"); + const val = bundle.format(msg, {arg: 5}, errs); + assert.equal(val, "Foo $arg"); + assert.equal(errs.length, 1); + }); + + test("No arguments, with arguments, no externals", function() { + const msg = bundle.getMessage("call-qux-no-args"); + const val = bundle.format(msg, {}, errs); + assert.equal(val, "Foo 1"); + assert.equal(errs.length, 0); + }); + + test("No arguments, with arguments, with externals", function() { + const msg = bundle.getMessage("call-qux-no-args"); + const val = bundle.format(msg, {arg: 5}, errs); + assert.equal(val, "Foo 1"); + assert.equal(errs.length, 0); + }); + + test("With arguments, no parameterization, no externals", function() { + const msg = bundle.getMessage("call-bar-with-arg"); + const val = bundle.format(msg, {}, errs); + assert.equal(val, "Foo $arg"); + assert.equal(errs.length, 1); + }); + + test("With arguments, no parameterization, with externals", function() { + const msg = bundle.getMessage("call-bar-with-arg"); + const val = bundle.format(msg, {arg: 5}, errs); + assert.equal(val, "Foo $arg"); + assert.equal(errs.length, 1); + }); + + test("With arguments, no arguments, no externals", function() { + const msg = bundle.getMessage("call-baz-with-arg"); + const val = bundle.format(msg, {}, errs); + assert.equal(val, "Foo $arg"); + assert.equal(errs.length, 1); + }); + + test("With arguments, no arguments, with externals", function() { + const msg = bundle.getMessage("call-baz-with-arg"); + const val = bundle.format(msg, {arg: 5}, errs); + assert.equal(val, "Foo $arg"); + assert.equal(errs.length, 1); + }); + + test("With arguments, with arguments, no externals", function() { + const msg = bundle.getMessage("call-qux-with-arg"); + const val = bundle.format(msg, {}, errs); + assert.equal(val, "Foo 1"); + assert.equal(errs.length, 0); + }); + + test("With arguments, with arguments, with externals", function() { + const msg = bundle.getMessage("call-qux-with-arg"); + const val = bundle.format(msg, {arg: 5}, errs); + assert.equal(val, "Foo 1"); + assert.equal(errs.length, 0); + }); + + test("With unexpected arguments, with arguments, no externals", function() { + const msg = bundle.getMessage("call-qux-with-other"); + const val = bundle.format(msg, {}, errs); + assert.equal(val, "Foo 1"); + assert.equal(errs.length, 0); + }); + + test("With unexpected arguments, with arguments, with externals", function() { + const msg = bundle.getMessage("call-qux-with-other"); + const val = bundle.format(msg, {arg: 5}, errs); + assert.equal(val, "Foo 1"); + assert.equal(errs.length, 0); + }); + }); +}); From f21097394ac653d72eab9d47927b70fb6a22709c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Tue, 27 Nov 2018 13:31:56 +0100 Subject: [PATCH 13/23] Update reference fixtures wrt. Junk content Port fixture changes from: https://github.com/projectfluent/fluent/commit/bf8fff4333c8b3a3182fb4658368c8b756d9c29e --- .../test/fixtures_reference/astral.json | 4 +- .../fixtures_reference/call_expressions.json | 20 +++++-- .../test/fixtures_reference/crlf.json | 2 +- .../escaped_characters.json | 2 +- .../test/fixtures_reference/junk.ftl | 19 ++++++- .../test/fixtures_reference/junk.json | 57 ++++++++++++++++++- .../test/fixtures_reference/leading_dots.json | 12 ++-- .../member_expressions.json | 7 ++- .../test/fixtures_reference/messages.json | 2 +- .../fixtures_reference/mixed_entries.json | 4 +- .../test/fixtures_reference/placeables.json | 6 +- .../select_expressions.json | 8 +-- .../fixtures_reference/select_indent.json | 7 ++- .../test/fixtures_reference/tab.json | 4 +- .../test/fixtures_reference/terms.json | 8 +-- .../test/fixtures_reference/variant_keys.json | 4 +- .../fixtures_reference/variant_lists.json | 8 +-- fluent-syntax/test/reference_test.js | 9 +-- 18 files changed, 137 insertions(+), 46 deletions(-) diff --git a/fluent-syntax/test/fixtures_reference/astral.json b/fluent-syntax/test/fixtures_reference/astral.json index 6f748c9c0..b69743cad 100644 --- a/fluent-syntax/test/fixtures_reference/astral.json +++ b/fluent-syntax/test/fixtures_reference/astral.json @@ -154,7 +154,7 @@ { "type": "Junk", "annotations": [], - "content": "err-๐Ÿ˜‚ = Value\n" + "content": "err-๐Ÿ˜‚ = Value\n\n" }, { "type": "Comment", @@ -163,7 +163,7 @@ { "type": "Junk", "annotations": [], - "content": "err-invalid-expression = { ๐Ÿ˜‚ }\n" + "content": "err-invalid-expression = { ๐Ÿ˜‚ }\n\n" }, { "type": "Comment", diff --git a/fluent-syntax/test/fixtures_reference/call_expressions.json b/fluent-syntax/test/fixtures_reference/call_expressions.json index 4cc36d404..286ea4f39 100644 --- a/fluent-syntax/test/fixtures_reference/call_expressions.json +++ b/fluent-syntax/test/fixtures_reference/call_expressions.json @@ -70,7 +70,7 @@ { "type": "Junk", "annotations": [], - "content": "mixed-case-callee = {Function()}\n" + "content": "mixed-case-callee = {Function()}\n\n" }, { "type": "Comment", @@ -88,7 +88,7 @@ { "type": "Junk", "annotations": [], - "content": "variable-callee = {$variable()}\n" + "content": "variable-callee = {$variable()}\n\n" }, { "type": "GroupComment", @@ -323,7 +323,7 @@ { "type": "Junk", "annotations": [], - "content": "shuffled-args = {FUN(1, x: 1, \"a\", y: \"Y\", msg)}\n" + "content": "shuffled-args = {FUN(1, x: 1, \"a\", y: \"Y\", msg)}\n\n" }, { "type": "Comment", @@ -332,7 +332,7 @@ { "type": "Junk", "annotations": [], - "content": "duplicate-named-args = {FUN(x: 1, x: \"X\")}\n" + "content": "duplicate-named-args = {FUN(x: 1, x: \"X\")}\n\n\n" }, { "type": "GroupComment", @@ -1063,7 +1063,17 @@ { "type": "Junk", "annotations": [], - "content": "one-argument = {FUN(1,,)}\nmissing-arg = {FUN(,)}\nmissing-sparse-arg = {FUN( , )}\n" + "content": "one-argument = {FUN(1,,)}\n" + }, + { + "type": "Junk", + "annotations": [], + "content": "missing-arg = {FUN(,)}\n" + }, + { + "type": "Junk", + "annotations": [], + "content": "missing-sparse-arg = {FUN( , )}\n\n\n" }, { "type": "GroupComment", diff --git a/fluent-syntax/test/fixtures_reference/crlf.json b/fluent-syntax/test/fixtures_reference/crlf.json index 6b324cd4e..58d26a777 100644 --- a/fluent-syntax/test/fixtures_reference/crlf.json +++ b/fluent-syntax/test/fixtures_reference/crlf.json @@ -61,7 +61,7 @@ { "type": "Junk", "annotations": [], - "content": "err03 = { \"str\r\n" + "content": "err03 = { \"str\r\n\r\n" }, { "type": "Comment", diff --git a/fluent-syntax/test/fixtures_reference/escaped_characters.json b/fluent-syntax/test/fixtures_reference/escaped_characters.json index 91de49944..e05c0efe6 100644 --- a/fluent-syntax/test/fixtures_reference/escaped_characters.json +++ b/fluent-syntax/test/fixtures_reference/escaped_characters.json @@ -169,7 +169,7 @@ { "type": "Junk", "annotations": [], - "content": "unknown-escape = {\"\\x\"}\n" + "content": "unknown-escape = {\"\\x\"}\n\n" }, { "type": "GroupComment", diff --git a/fluent-syntax/test/fixtures_reference/junk.ftl b/fluent-syntax/test/fixtures_reference/junk.ftl index 62fc2ea1b..b0b0c5f3b 100644 --- a/fluent-syntax/test/fixtures_reference/junk.ftl +++ b/fluent-syntax/test/fixtures_reference/junk.ftl @@ -1,4 +1,21 @@ +## Two adjacent Junks. +err01 = {1x} +err02 = {2x} + +# A single Junk. +err03 = {1x +2 + +# A single Junk. ฤ…=Invalid identifier ฤ‡=Another one -key01 = { +# The COMMENT ends this junk. +err04 = { +# COMMENT + +# The COMMENT ends this junk. +# The closing brace is a separate Junk. +err04 = { +# COMMENT +} diff --git a/fluent-syntax/test/fixtures_reference/junk.json b/fluent-syntax/test/fixtures_reference/junk.json index 530ff3d5a..15e62a4db 100644 --- a/fluent-syntax/test/fixtures_reference/junk.json +++ b/fluent-syntax/test/fixtures_reference/junk.json @@ -1,15 +1,68 @@ { "type": "Resource", "body": [ + { + "type": "GroupComment", + "content": "Two adjacent Junks." + }, + { + "type": "Junk", + "annotations": [], + "content": "err01 = {1x}\n" + }, + { + "type": "Junk", + "annotations": [], + "content": "err02 = {2x}\n\n" + }, + { + "type": "Comment", + "content": "A single Junk." + }, + { + "type": "Junk", + "annotations": [], + "content": "err03 = {1x\n2\n\n" + }, + { + "type": "Comment", + "content": "A single Junk." + }, { "type": "Junk", "annotations": [], - "content": "ฤ…=Invalid identifier\nฤ‡=Another one\n" + "content": "ฤ…=Invalid identifier\nฤ‡=Another one\n\n" + }, + { + "type": "Comment", + "content": "The COMMENT ends this junk." + }, + { + "type": "Junk", + "annotations": [], + "content": "err04 = {\n" + }, + { + "type": "Comment", + "content": "COMMENT" + }, + { + "type": "Comment", + "content": "The COMMENT ends this junk.\nThe closing brace is a separate Junk." + }, + { + "type": "Junk", + "annotations": [], + "content": "err04 = {\n" + }, + { + "type": "Comment", + "content": "COMMENT" }, { "type": "Junk", "annotations": [], - "content": "key01 = {\n" + "content": "}\n" } ] } diff --git a/fluent-syntax/test/fixtures_reference/leading_dots.json b/fluent-syntax/test/fixtures_reference/leading_dots.json index 1787d0ee0..a185d4896 100644 --- a/fluent-syntax/test/fixtures_reference/leading_dots.json +++ b/fluent-syntax/test/fixtures_reference/leading_dots.json @@ -173,7 +173,7 @@ { "type": "Junk", "annotations": [], - "content": " .Continued\n" + "content": " .Continued\n\n" }, { "type": "Comment", @@ -182,7 +182,7 @@ { "type": "Junk", "annotations": [], - "content": "key08 =\n .Value\n" + "content": "key08 =\n .Value\n\n" }, { "type": "Comment", @@ -191,7 +191,7 @@ { "type": "Junk", "annotations": [], - "content": "key09 =\n .Value\n Continued\n" + "content": "key09 =\n .Value\n Continued\n\n" }, { "type": "Message", @@ -410,7 +410,7 @@ { "type": "Junk", "annotations": [], - "content": "key16 =\n { 1 ->\n *[one]\n .Value\n }\n" + "content": "key16 =\n { 1 ->\n *[one]\n .Value\n }\n\n" }, { "type": "Comment", @@ -419,7 +419,7 @@ { "type": "Junk", "annotations": [], - "content": "key17 =\n { 1 ->\n *[one] Value\n .Continued\n }\n" + "content": "key17 =\n { 1 ->\n *[one] Value\n .Continued\n }\n\n" }, { "type": "Comment", @@ -428,7 +428,7 @@ { "type": "Junk", "annotations": [], - "content": "key18 =\n.Value\n" + "content": "key18 =\n.Value\n\n" }, { "type": "Message", diff --git a/fluent-syntax/test/fixtures_reference/member_expressions.json b/fluent-syntax/test/fixtures_reference/member_expressions.json index a95879011..541bd795a 100644 --- a/fluent-syntax/test/fixtures_reference/member_expressions.json +++ b/fluent-syntax/test/fixtures_reference/member_expressions.json @@ -70,7 +70,12 @@ { "type": "Junk", "annotations": [], - "content": "variant-expression = {msg[case]}\nattribute-expression = {-term.attr}\n" + "content": "variant-expression = {msg[case]}\n" + }, + { + "type": "Junk", + "annotations": [], + "content": "attribute-expression = {-term.attr}\n" } ] } diff --git a/fluent-syntax/test/fixtures_reference/messages.json b/fluent-syntax/test/fixtures_reference/messages.json index fa8dcbd7b..7d779d3e2 100644 --- a/fluent-syntax/test/fixtures_reference/messages.json +++ b/fluent-syntax/test/fixtures_reference/messages.json @@ -234,7 +234,7 @@ { "type": "Junk", "annotations": [], - "content": "key07 =\n" + "content": "key07 =\n\n" }, { "type": "Comment", diff --git a/fluent-syntax/test/fixtures_reference/mixed_entries.json b/fluent-syntax/test/fixtures_reference/mixed_entries.json index 8f334f4fa..a9dc501f6 100644 --- a/fluent-syntax/test/fixtures_reference/mixed_entries.json +++ b/fluent-syntax/test/fixtures_reference/mixed_entries.json @@ -61,7 +61,7 @@ { "type": "Junk", "annotations": [], - "content": "ฤ…=Invalid identifier\nฤ‡=Another one\n" + "content": "ฤ…=Invalid identifier\nฤ‡=Another one\n\n" }, { "type": "Message", @@ -91,7 +91,7 @@ { "type": "Junk", "annotations": [], - "content": " .attr = Dangling attribute\n" + "content": " .attr = Dangling attribute\n\n" }, { "type": "Message", diff --git a/fluent-syntax/test/fixtures_reference/placeables.json b/fluent-syntax/test/fixtures_reference/placeables.json index ff7fac8a7..7d67d940b 100644 --- a/fluent-syntax/test/fixtures_reference/placeables.json +++ b/fluent-syntax/test/fixtures_reference/placeables.json @@ -80,7 +80,7 @@ { "type": "Junk", "annotations": [], - "content": "unmatched-open1 = { 1\n" + "content": "unmatched-open1 = { 1\n\n" }, { "type": "Comment", @@ -89,7 +89,7 @@ { "type": "Junk", "annotations": [], - "content": "unmatched-open2 = {{ 1 }\n" + "content": "unmatched-open2 = {{ 1 }\n\n" }, { "type": "Comment", @@ -98,7 +98,7 @@ { "type": "Junk", "annotations": [], - "content": "unmatched-close1 = 1 }\n" + "content": "unmatched-close1 = 1 }\n\n" }, { "type": "Comment", diff --git a/fluent-syntax/test/fixtures_reference/select_expressions.json b/fluent-syntax/test/fixtures_reference/select_expressions.json index 7cce4070d..4e84c0c2a 100644 --- a/fluent-syntax/test/fixtures_reference/select_expressions.json +++ b/fluent-syntax/test/fixtures_reference/select_expressions.json @@ -137,7 +137,7 @@ { "type": "Junk", "annotations": [], - "content": "invalid-selector-term-value =\n { -term ->\n *[key] value\n }\n" + "content": "invalid-selector-term-value =\n { -term ->\n *[key] value\n }\n\n" }, { "type": "Comment", @@ -146,7 +146,7 @@ { "type": "Junk", "annotations": [], - "content": "invalid-selector-term-variant =\n { -term[case] ->\n *[key] value\n }\n" + "content": "invalid-selector-term-variant =\n { -term[case] ->\n *[key] value\n }\n\n" }, { "type": "Comment", @@ -155,7 +155,7 @@ { "type": "Junk", "annotations": [], - "content": "invalid-selector-term-call =\n { -term(case: \"nominative\") ->\n *[key] value\n }\n" + "content": "invalid-selector-term-call =\n { -term(case: \"nominative\") ->\n *[key] value\n }\n\n" }, { "type": "Message", @@ -279,7 +279,7 @@ { "type": "Junk", "annotations": [], - "content": "nested-variant-list =\n { 1 ->\n *[one] {\n *[two] Value\n }\n }\n" + "content": "nested-variant-list =\n { 1 ->\n *[one] {\n *[two] Value\n }\n }\n\n" }, { "type": "Comment", diff --git a/fluent-syntax/test/fixtures_reference/select_indent.json b/fluent-syntax/test/fixtures_reference/select_indent.json index 03109cb0f..d8c0fa880 100644 --- a/fluent-syntax/test/fixtures_reference/select_indent.json +++ b/fluent-syntax/test/fixtures_reference/select_indent.json @@ -548,7 +548,12 @@ { "type": "Junk", "annotations": [], - "content": "select-no-indent-multiline = { $selector ->\n *[key] Value\nContinued without indent.\n}\n" + "content": "select-no-indent-multiline = { $selector ->\n *[key] Value\n" + }, + { + "type": "Junk", + "annotations": [], + "content": "Continued without indent.\n}\n\n" }, { "type": "Message", diff --git a/fluent-syntax/test/fixtures_reference/tab.json b/fluent-syntax/test/fixtures_reference/tab.json index fe438ab43..714eb947e 100644 --- a/fluent-syntax/test/fixtures_reference/tab.json +++ b/fluent-syntax/test/fixtures_reference/tab.json @@ -29,7 +29,7 @@ { "type": "Junk", "annotations": [], - "content": "key02\t= Value 02\n" + "content": "key02\t= Value 02\n\n" }, { "type": "Comment", @@ -38,7 +38,7 @@ { "type": "Junk", "annotations": [], - "content": "key03 =\n\tThis line isn't properly indented.\n" + "content": "key03 =\n\tThis line isn't properly indented.\n\n" }, { "type": "Message", diff --git a/fluent-syntax/test/fixtures_reference/terms.json b/fluent-syntax/test/fixtures_reference/terms.json index d3eff8065..2321283f0 100644 --- a/fluent-syntax/test/fixtures_reference/terms.json +++ b/fluent-syntax/test/fixtures_reference/terms.json @@ -65,7 +65,7 @@ { "type": "Junk", "annotations": [], - "content": "-term03 =\n .attr = Attribute\n" + "content": "-term03 =\n .attr = Attribute\n\n" }, { "type": "Comment", @@ -74,7 +74,7 @@ { "type": "Junk", "annotations": [], - "content": "-term04 = \n .attr1 = Attribute 1\n" + "content": "-term04 = \n .attr1 = Attribute 1\n\n" }, { "type": "Comment", @@ -83,7 +83,7 @@ { "type": "Junk", "annotations": [], - "content": "-term05 =\n" + "content": "-term05 =\n\n" }, { "type": "Comment", @@ -92,7 +92,7 @@ { "type": "Junk", "annotations": [], - "content": "-term06 = \n" + "content": "-term06 = \n\n" }, { "type": "Comment", diff --git a/fluent-syntax/test/fixtures_reference/variant_keys.json b/fluent-syntax/test/fixtures_reference/variant_keys.json index e8c60b002..cf752e433 100644 --- a/fluent-syntax/test/fixtures_reference/variant_keys.json +++ b/fluent-syntax/test/fixtures_reference/variant_keys.json @@ -132,7 +132,7 @@ { "type": "Junk", "annotations": [], - "content": "-invalid-identifier =\n {\n *[two words] value\n }\n" + "content": "-invalid-identifier =\n {\n *[two words] value\n }\n\n" }, { "type": "Comment", @@ -141,7 +141,7 @@ { "type": "Junk", "annotations": [], - "content": "-invalid-int =\n {\n *[1 apple] value\n }\n" + "content": "-invalid-int =\n {\n *[1 apple] value\n }\n\n" }, { "type": "Comment", diff --git a/fluent-syntax/test/fixtures_reference/variant_lists.json b/fluent-syntax/test/fixtures_reference/variant_lists.json index d10484046..aaac9f7b0 100644 --- a/fluent-syntax/test/fixtures_reference/variant_lists.json +++ b/fluent-syntax/test/fixtures_reference/variant_lists.json @@ -56,7 +56,7 @@ { "type": "Junk", "annotations": [], - "content": " .attr =\n {\n *[key] Value\n }\n" + "content": " .attr =\n {\n *[key] Value\n }\n\n" }, { "type": "Comment", @@ -65,7 +65,7 @@ { "type": "Junk", "annotations": [], - "content": "variant-list-in-message =\n {\n *[key] Value\n }\n" + "content": "variant-list-in-message =\n {\n *[key] Value\n }\n\n" }, { "type": "Message", @@ -91,7 +91,7 @@ { "type": "Junk", "annotations": [], - "content": " .attr =\n {\n *[key] Value\n }\n" + "content": " .attr =\n {\n *[key] Value\n }\n\n" }, { "type": "Term", @@ -200,7 +200,7 @@ { "type": "Junk", "annotations": [], - "content": "nested-select-then-variant-list =\n {\n *[one] { 2 ->\n *[two] {\n *[three] Value\n }\n }\n }\n" + "content": "nested-select-then-variant-list =\n {\n *[one] { 2 ->\n *[two] {\n *[three] Value\n }\n }\n }\n\n" }, { "type": "Comment", diff --git a/fluent-syntax/test/reference_test.js b/fluent-syntax/test/reference_test.js index a6cd8404e..39cd7ad32 100644 --- a/fluent-syntax/test/reference_test.js +++ b/fluent-syntax/test/reference_test.js @@ -46,10 +46,11 @@ readdir(fixtures, function(err, filenames) { const ref = JSON.parse(expected) const ast = parser.parse(ftl); - // Ignore Junk which is parsed differently by the tooling parser, and - // which doesn't carry spans nor annotations in the reference parser. - ref.body = ref.body.filter(entry => entry.type !== "Junk"); - ast.body = ast.body.filter(entry => entry.type !== "Junk"); + // Only compare Junk content and ignore annotations, which carry error + // messages and positions. The reference parser doesn't produce + // annotations at the moment. + ast.body = ast.body.map(entry => entry.type === "Junk" ? + {...entry, annotations: []} : entry); assert.deepEqual( ast, ref, From 20444375ea332585396fd1f08bc589b2151d99d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Tue, 27 Nov 2018 13:34:31 +0100 Subject: [PATCH 14/23] Update reference fixtures wrt. allowing all Unicode characters Port reference fixtures from: https://github.com/projectfluent/fluent/commit/741be047bb6bd55c3e761c0d2fcdd8f5dc360b6b --- .../test/fixtures_reference/any_char.ftl | 8 +++ .../test/fixtures_reference/any_char.json | 68 +++++++++++++++++++ fluent/test/fixtures_reference/any_char.json | 5 ++ 3 files changed, 81 insertions(+) create mode 100644 fluent-syntax/test/fixtures_reference/any_char.ftl create mode 100644 fluent-syntax/test/fixtures_reference/any_char.json create mode 100644 fluent/test/fixtures_reference/any_char.json diff --git a/fluent-syntax/test/fixtures_reference/any_char.ftl b/fluent-syntax/test/fixtures_reference/any_char.ftl new file mode 100644 index 000000000..6966a0dae --- /dev/null +++ b/fluent-syntax/test/fixtures_reference/any_char.ftl @@ -0,0 +1,8 @@ +# โ†“ BEL, U+0007 +control0 = abcdef + +# โ†“ DEL, U+007F +delete = abcdef + +# โ†“ BPM, U+0082 +control1 = abcย‚def diff --git a/fluent-syntax/test/fixtures_reference/any_char.json b/fluent-syntax/test/fixtures_reference/any_char.json new file mode 100644 index 000000000..07e7dc4b2 --- /dev/null +++ b/fluent-syntax/test/fixtures_reference/any_char.json @@ -0,0 +1,68 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "control0" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "abc\u0007def" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": " โ†“ BEL, U+0007" + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "delete" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "abcdef" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": " โ†“ DEL, U+007F" + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "control1" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "abcย‚def" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": " โ†“ BPM, U+0082" + } + } + ] +} diff --git a/fluent/test/fixtures_reference/any_char.json b/fluent/test/fixtures_reference/any_char.json new file mode 100644 index 000000000..21eb2e4c3 --- /dev/null +++ b/fluent/test/fixtures_reference/any_char.json @@ -0,0 +1,5 @@ +{ + "control0": "abc\u0007def", + "delete": "abcdef", + "control1": "abcย‚def" +} From 758a0a233c32b4ebad8fad71e4e900be7e75a1f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Tue, 27 Nov 2018 17:28:31 +0100 Subject: [PATCH 15/23] (fluent-syntax) Parameterized term attributes --- fluent-syntax/src/errors.js | 2 +- fluent-syntax/src/parser.js | 28 +- .../fixtures_reference/call_expressions.ftl | 13 - .../fixtures_reference/call_expressions.json | 89 ------ .../fixtures_reference/callee_expressions.ftl | 46 +++ .../callee_expressions.json | 270 ++++++++++++++++++ .../fixtures_reference/member_expressions.ftl | 32 ++- .../member_expressions.json | 122 +++++++- .../test/fixtures_reference/messages.ftl | 2 + .../test/fixtures_reference/messages.json | 20 +- .../reference_expressions.ftl | 31 +- .../reference_expressions.json | 92 +++++- 12 files changed, 607 insertions(+), 140 deletions(-) create mode 100644 fluent-syntax/test/fixtures_reference/callee_expressions.ftl create mode 100644 fluent-syntax/test/fixtures_reference/callee_expressions.json diff --git a/fluent-syntax/src/errors.js b/fluent-syntax/src/errors.js index 1cc8f1c57..9b8154fc5 100644 --- a/fluent-syntax/src/errors.js +++ b/fluent-syntax/src/errors.js @@ -33,7 +33,7 @@ function getErrorMessage(code, args) { case "E0007": return "Keyword cannot end with a whitespace"; case "E0008": - return "The callee has to be a simple, upper-case identifier"; + return "The callee has to be an upper-case identifier or a term"; case "E0009": return "The key has to be a simple identifier"; case "E0010": diff --git a/fluent-syntax/src/parser.js b/fluent-syntax/src/parser.js index 61a06d7c7..5ad68c96f 100644 --- a/fluent-syntax/src/parser.js +++ b/fluent-syntax/src/parser.js @@ -678,11 +678,18 @@ export default class FluentParser { const variants = this.getVariants(ps, {allowVariantList: false}); return new AST.SelectExpression(selector, variants); - } else if (selector.type === "AttributeExpression" + } + + if (selector.type === "AttributeExpression" && selector.ref.type === "TermReference") { throw new ParseError("E0019"); } + if (selector.type === "CallExpression" + && selector.callee.type === "AttributeExpression") { + throw new ParseError("E0019"); + } + return selector; } @@ -691,7 +698,7 @@ export default class FluentParser { return this.getPlaceable(ps); } - const selector = this.getLiteral(ps); + let selector = this.getLiteral(ps); switch (selector.type) { case "StringLiteral": case "NumberLiteral": @@ -699,12 +706,6 @@ export default class FluentParser { return selector; } - if (ps.currentChar === ".") { - ps.next(); - const attr = this.getIdentifier(ps); - return new AST.AttributeExpression(selector, attr); - } - if (ps.currentChar === "[") { ps.next(); @@ -717,6 +718,12 @@ export default class FluentParser { return new AST.VariantExpression(selector, key); } + if (ps.currentChar === ".") { + ps.next(); + const attr = this.getIdentifier(ps); + selector = new AST.AttributeExpression(selector, attr); + } + if (ps.currentChar === "(") { ps.next(); @@ -733,6 +740,11 @@ export default class FluentParser { } } + if (selector.type === "AttributeExpression" + && selector.ref.type === "MessageReference") { + throw new ParseError("E0008"); + } + const args = this.getCallArgs(ps); ps.expectChar(")"); diff --git a/fluent-syntax/test/fixtures_reference/call_expressions.ftl b/fluent-syntax/test/fixtures_reference/call_expressions.ftl index a4f61da40..19e76b7e2 100644 --- a/fluent-syntax/test/fixtures_reference/call_expressions.ftl +++ b/fluent-syntax/test/fixtures_reference/call_expressions.ftl @@ -1,16 +1,3 @@ -## Callees - -function-callee = {FUNCTION()} -term-callee = {-term()} - -# ERROR Equivalent to a MessageReference callee. -mixed-case-callee = {Function()} - -# ERROR MessageReference is not a valid callee. -message-callee = {message()} -# ERROR VariableReference is not a valid callee. -variable-callee = {$variable()} - ## Arguments positional-args = {FUN(1, "a", msg)} diff --git a/fluent-syntax/test/fixtures_reference/call_expressions.json b/fluent-syntax/test/fixtures_reference/call_expressions.json index 286ea4f39..4dd734831 100644 --- a/fluent-syntax/test/fixtures_reference/call_expressions.json +++ b/fluent-syntax/test/fixtures_reference/call_expressions.json @@ -1,95 +1,6 @@ { "type": "Resource", "body": [ - { - "type": "GroupComment", - "content": "Callees" - }, - { - "type": "Message", - "id": { - "type": "Identifier", - "name": "function-callee" - }, - "value": { - "type": "Pattern", - "elements": [ - { - "type": "Placeable", - "expression": { - "type": "CallExpression", - "callee": { - "type": "FunctionReference", - "id": { - "type": "Identifier", - "name": "FUNCTION" - } - }, - "positional": [], - "named": [] - } - } - ] - }, - "attributes": [], - "comment": null - }, - { - "type": "Message", - "id": { - "type": "Identifier", - "name": "term-callee" - }, - "value": { - "type": "Pattern", - "elements": [ - { - "type": "Placeable", - "expression": { - "type": "CallExpression", - "callee": { - "type": "TermReference", - "id": { - "type": "Identifier", - "name": "term" - } - }, - "positional": [], - "named": [] - } - } - ] - }, - "attributes": [], - "comment": null - }, - { - "type": "Comment", - "content": "ERROR Equivalent to a MessageReference callee." - }, - { - "type": "Junk", - "annotations": [], - "content": "mixed-case-callee = {Function()}\n\n" - }, - { - "type": "Comment", - "content": "ERROR MessageReference is not a valid callee." - }, - { - "type": "Junk", - "annotations": [], - "content": "message-callee = {message()}\n" - }, - { - "type": "Comment", - "content": "ERROR VariableReference is not a valid callee." - }, - { - "type": "Junk", - "annotations": [], - "content": "variable-callee = {$variable()}\n\n" - }, { "type": "GroupComment", "content": "Arguments" diff --git a/fluent-syntax/test/fixtures_reference/callee_expressions.ftl b/fluent-syntax/test/fixtures_reference/callee_expressions.ftl new file mode 100644 index 000000000..637a2e4d7 --- /dev/null +++ b/fluent-syntax/test/fixtures_reference/callee_expressions.ftl @@ -0,0 +1,46 @@ +## Callees in placeables. + +function-callee-placeable = {FUNCTION()} +term-callee-placeable = {-term()} + +# ERROR Messages cannot be parameterized. +message-callee-placeable = {message()} +# ERROR Equivalent to a MessageReference callee. +mixed-case-callee-placeable = {Function()} +# ERROR Message attributes cannot be parameterized. +message-attr-callee-placeable = {message.attr()} +# ERROR Term attributes may not be used in Placeables. +term-attr-callee-placeable = {-term.attr()} +# ERROR Variables cannot be parameterized. +variable-callee-placeable = {$variable()} + + +## Callees in selectors. + +function-callee-selector = {FUNCTION() -> + *[key] Value +} +term-attr-callee-selector = {-term.attr() -> + *[key] Value +} + +# ERROR Messages cannot be parameterized. +message-callee-selector = {message() -> + *[key] Value +} +# ERROR Equivalent to a MessageReference callee. +mixed-case-callee-selector = {Function() -> + *[key] Value +} +# ERROR Message attributes cannot be parameterized. +message-attr-callee-selector = {message.attr() -> + *[key] Value +} +# ERROR Term values may not be used as selectors. +term-callee-selector = {-term() -> + *[key] Value +} +# ERROR Variables cannot be parameterized. +variable-callee-selector = {$variable() -> + *[key] Value +} diff --git a/fluent-syntax/test/fixtures_reference/callee_expressions.json b/fluent-syntax/test/fixtures_reference/callee_expressions.json new file mode 100644 index 000000000..50cdaeb41 --- /dev/null +++ b/fluent-syntax/test/fixtures_reference/callee_expressions.json @@ -0,0 +1,270 @@ +{ + "type": "Resource", + "body": [ + { + "type": "GroupComment", + "content": "Callees in placeables." + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "function-callee-placeable" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUNCTION" + } + }, + "positional": [], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "term-callee-placeable" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + }, + "positional": [], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR Messages cannot be parameterized." + }, + { + "type": "Junk", + "annotations": [], + "content": "message-callee-placeable = {message()}\n" + }, + { + "type": "Comment", + "content": "ERROR Equivalent to a MessageReference callee." + }, + { + "type": "Junk", + "annotations": [], + "content": "mixed-case-callee-placeable = {Function()}\n" + }, + { + "type": "Comment", + "content": "ERROR Message attributes cannot be parameterized." + }, + { + "type": "Junk", + "annotations": [], + "content": "message-attr-callee-placeable = {message.attr()}\n" + }, + { + "type": "Comment", + "content": "ERROR Term attributes may not be used in Placeables." + }, + { + "type": "Junk", + "annotations": [], + "content": "term-attr-callee-placeable = {-term.attr()}\n" + }, + { + "type": "Comment", + "content": "ERROR Variables cannot be parameterized." + }, + { + "type": "Junk", + "annotations": [], + "content": "variable-callee-placeable = {$variable()}\n\n\n" + }, + { + "type": "GroupComment", + "content": "Callees in selectors." + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "function-callee-selector" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUNCTION" + } + }, + "positional": [], + "named": [] + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "term-attr-callee-selector" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "CallExpression", + "callee": { + "type": "AttributeExpression", + "ref": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + }, + "name": { + "type": "Identifier", + "name": "attr" + } + }, + "positional": [], + "named": [] + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR Messages cannot be parameterized." + }, + { + "type": "Junk", + "annotations": [], + "content": "message-callee-selector = {message() ->\n *[key] Value\n}\n" + }, + { + "type": "Comment", + "content": "ERROR Equivalent to a MessageReference callee." + }, + { + "type": "Junk", + "annotations": [], + "content": "mixed-case-callee-selector = {Function() ->\n *[key] Value\n}\n" + }, + { + "type": "Comment", + "content": "ERROR Message attributes cannot be parameterized." + }, + { + "type": "Junk", + "annotations": [], + "content": "message-attr-callee-selector = {message.attr() ->\n *[key] Value\n}\n" + }, + { + "type": "Comment", + "content": "ERROR Term values may not be used as selectors." + }, + { + "type": "Junk", + "annotations": [], + "content": "term-callee-selector = {-term() ->\n *[key] Value\n}\n" + }, + { + "type": "Comment", + "content": "ERROR Variables cannot be parameterized." + }, + { + "type": "Junk", + "annotations": [], + "content": "variable-callee-selector = {$variable() ->\n *[key] Value\n}\n" + } + ] +} diff --git a/fluent-syntax/test/fixtures_reference/member_expressions.ftl b/fluent-syntax/test/fixtures_reference/member_expressions.ftl index 09e09f3fc..b4a93922b 100644 --- a/fluent-syntax/test/fixtures_reference/member_expressions.ftl +++ b/fluent-syntax/test/fixtures_reference/member_expressions.ftl @@ -1,6 +1,28 @@ -variant-expression = {-term[case]} -attribute-expression = {msg.attr} +## Member expressions in placeables. -## Invalid syntax -variant-expression = {msg[case]} -attribute-expression = {-term.attr} +message-attribute-expression-placeable = {msg.attr} +term-variant-expression-placeable = {-term[case]} + +# ERROR Message values cannot be VariantLists +message-variant-expression-placeable = {msg[case]} +# ERROR Term attributes may not be used for interpolation. +term-attribute-expression-placeable = {-term.attr} + +## Member expressions in selectors. + +term-attribute-expression-selector = {-term.attr -> + *[key] Value +} + +# ERROR Message attributes may not be used as selector. +message-attribute-expression-selector = {msg.attr -> + *[key] Value +} +# ERROR Term values may not be used as selector. +term-variant-expression-selector = {-term[case] -> + *[key] Value +} +# ERROR Message values cannot be VariantLists +message-variant-expression-selector = {msg[case] -> + *[key] Value +} diff --git a/fluent-syntax/test/fixtures_reference/member_expressions.json b/fluent-syntax/test/fixtures_reference/member_expressions.json index 541bd795a..f6890e56f 100644 --- a/fluent-syntax/test/fixtures_reference/member_expressions.json +++ b/fluent-syntax/test/fixtures_reference/member_expressions.json @@ -1,11 +1,15 @@ { "type": "Resource", "body": [ + { + "type": "GroupComment", + "content": "Member expressions in placeables." + }, { "type": "Message", "id": { "type": "Identifier", - "name": "variant-expression" + "name": "message-attribute-expression-placeable" }, "value": { "type": "Pattern", @@ -13,17 +17,17 @@ { "type": "Placeable", "expression": { - "type": "VariantExpression", + "type": "AttributeExpression", "ref": { - "type": "TermReference", + "type": "MessageReference", "id": { "type": "Identifier", - "name": "term" + "name": "msg" } }, - "key": { + "name": { "type": "Identifier", - "name": "case" + "name": "attr" } } } @@ -36,7 +40,7 @@ "type": "Message", "id": { "type": "Identifier", - "name": "attribute-expression" + "name": "term-variant-expression-placeable" }, "value": { "type": "Pattern", @@ -44,17 +48,17 @@ { "type": "Placeable", "expression": { - "type": "AttributeExpression", + "type": "VariantExpression", "ref": { - "type": "MessageReference", + "type": "TermReference", "id": { "type": "Identifier", - "name": "msg" + "name": "term" } }, - "name": { + "key": { "type": "Identifier", - "name": "attr" + "name": "case" } } } @@ -63,19 +67,107 @@ "attributes": [], "comment": null }, + { + "type": "Comment", + "content": "ERROR Message values cannot be VariantLists" + }, + { + "type": "Junk", + "annotations": [], + "content": "message-variant-expression-placeable = {msg[case]}\n" + }, + { + "type": "Comment", + "content": "ERROR Term attributes may not be used for interpolation." + }, + { + "type": "Junk", + "annotations": [], + "content": "term-attribute-expression-placeable = {-term.attr}\n\n" + }, { "type": "GroupComment", - "content": "Invalid syntax" + "content": "Member expressions in selectors." + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "term-attribute-expression-selector" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "AttributeExpression", + "ref": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + }, + "name": { + "type": "Identifier", + "name": "attr" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR Message attributes may not be used as selector." }, { "type": "Junk", "annotations": [], - "content": "variant-expression = {msg[case]}\n" + "content": "message-attribute-expression-selector = {msg.attr ->\n *[key] Value\n}\n" + }, + { + "type": "Comment", + "content": "ERROR Term values may not be used as selector." + }, + { + "type": "Junk", + "annotations": [], + "content": "term-variant-expression-selector = {-term[case] ->\n *[key] Value\n}\n" + }, + { + "type": "Comment", + "content": "ERROR Message values cannot be VariantLists" }, { "type": "Junk", "annotations": [], - "content": "attribute-expression = {-term.attr}\n" + "content": "message-variant-expression-selector = {msg[case] ->\n *[key] Value\n}\n" } ] } diff --git a/fluent-syntax/test/fixtures_reference/messages.ftl b/fluent-syntax/test/fixtures_reference/messages.ftl index 0ade3cace..00b4ab468 100644 --- a/fluent-syntax/test/fixtures_reference/messages.ftl +++ b/fluent-syntax/test/fixtures_reference/messages.ftl @@ -25,3 +25,5 @@ key07 = # JUNK Missing = key08 + +KEY09 = Value 09 diff --git a/fluent-syntax/test/fixtures_reference/messages.json b/fluent-syntax/test/fixtures_reference/messages.json index 7d779d3e2..cdbe5c93c 100644 --- a/fluent-syntax/test/fixtures_reference/messages.json +++ b/fluent-syntax/test/fixtures_reference/messages.json @@ -243,7 +243,25 @@ { "type": "Junk", "annotations": [], - "content": "key08\n" + "content": "key08\n\n" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "KEY09" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value 09" + } + ] + }, + "attributes": [], + "comment": null } ] } diff --git a/fluent-syntax/test/fixtures_reference/reference_expressions.ftl b/fluent-syntax/test/fixtures_reference/reference_expressions.ftl index ab27cbe09..9c2e9c543 100644 --- a/fluent-syntax/test/fixtures_reference/reference_expressions.ftl +++ b/fluent-syntax/test/fixtures_reference/reference_expressions.ftl @@ -1,5 +1,28 @@ -message-reference = {msg} -term-reference = {-term} -variable-reference = {$var} +## Reference expressions in placeables. -not-call-expression = {FUN} +message-reference-placeable = {msg} +term-reference-placeable = {-term} +variable-reference-placeable = {$var} + +# ERROR Function references are invalid outside of call expressions. +function-reference-placeable = {FUN} + + +## Reference expressions in selectors. + +variable-reference-selector = {$var -> + *[key] Value +} + +# ERROR Message values may not be used as selectors. +message-reference-selector = {msg -> + *[key] Value +} +# ERROR Term values may not be used as selectors. +term-reference-selector = {-term -> + *[key] Value +} +# ERROR Function references are invalid outside of call expressions. +function-expression-selector = {FUN -> + *[key] Value +} diff --git a/fluent-syntax/test/fixtures_reference/reference_expressions.json b/fluent-syntax/test/fixtures_reference/reference_expressions.json index 47e4dfab0..65c9d4cc2 100644 --- a/fluent-syntax/test/fixtures_reference/reference_expressions.json +++ b/fluent-syntax/test/fixtures_reference/reference_expressions.json @@ -1,11 +1,15 @@ { "type": "Resource", "body": [ + { + "type": "GroupComment", + "content": "Reference expressions in placeables." + }, { "type": "Message", "id": { "type": "Identifier", - "name": "message-reference" + "name": "message-reference-placeable" }, "value": { "type": "Pattern", @@ -29,7 +33,7 @@ "type": "Message", "id": { "type": "Identifier", - "name": "term-reference" + "name": "term-reference-placeable" }, "value": { "type": "Pattern", @@ -53,7 +57,7 @@ "type": "Message", "id": { "type": "Identifier", - "name": "variable-reference" + "name": "variable-reference-placeable" }, "value": { "type": "Pattern", @@ -77,7 +81,7 @@ "type": "Message", "id": { "type": "Identifier", - "name": "not-call-expression" + "name": "function-reference-placeable" }, "value": { "type": "Pattern", @@ -95,7 +99,87 @@ ] }, "attributes": [], + "comment": { + "type": "Comment", + "content": "ERROR Function references are invalid outside of call expressions." + } + }, + { + "type": "GroupComment", + "content": "Reference expressions in selectors." + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "variable-reference-selector" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "var" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], "comment": null + }, + { + "type": "Comment", + "content": "ERROR Message values may not be used as selectors." + }, + { + "type": "Junk", + "annotations": [], + "content": "message-reference-selector = {msg ->\n *[key] Value\n}\n" + }, + { + "type": "Comment", + "content": "ERROR Term values may not be used as selectors." + }, + { + "type": "Junk", + "annotations": [], + "content": "term-reference-selector = {-term ->\n *[key] Value\n}\n" + }, + { + "type": "Comment", + "content": "ERROR Function references are invalid outside of call expressions." + }, + { + "type": "Junk", + "annotations": [], + "content": "function-expression-selector = {FUN ->\n *[key] Value\n}\n" } ] } From 9b7849d6d204d77edf0b5b91781f0548f186b64d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Mon, 26 Nov 2018 18:31:35 +0100 Subject: [PATCH 16/23] (fluent) Use references with extra data rather than member expressions --- fluent/src/resolver.js | 448 +++++------------- fluent/src/resource.js | 54 +-- .../fixtures_reference/call_expressions.json | 306 +++++------- .../callee_expressions.json | 184 +++++++ fluent/test/fixtures_reference/cr.json | 4 +- fluent/test/fixtures_reference/crlf.json | 4 +- .../escaped_characters.json | 4 +- .../member_expressions.json | 100 +++- fluent/test/fixtures_reference/messages.json | 3 +- .../reference_expressions.json | 98 +++- .../select_expressions.json | 39 +- .../fixtures_reference/select_indent.json | 52 +- .../fixtures_reference/term_parameters.json | 34 +- fluent/test/fixtures_reference/variables.json | 22 +- fluent/test/fixtures_structure/crlf.json | 4 +- .../fixtures_structure/escape_sequences.json | 4 +- .../expressions_call_args.json | 8 +- .../fixtures_structure/placeable_at_eol.json | 12 +- .../fixtures_structure/sparse-messages.json | 4 +- fluent/test/fixtures_structure/term.json | 35 +- .../whitespace_trailing.json | 4 +- fluent/test/macros_test.js | 92 ++++ fluent/test/values_ref_test.js | 16 +- 23 files changed, 862 insertions(+), 669 deletions(-) create mode 100644 fluent/test/fixtures_reference/callee_expressions.json diff --git a/fluent/src/resolver.js b/fluent/src/resolver.js index 7eeb08cba..f32fe9060 100644 --- a/fluent/src/resolver.js +++ b/fluent/src/resolver.js @@ -17,23 +17,8 @@ * translation as possible. In rare situations where the resolver didn't know * how to recover from an error it will return an instance of `FluentNone`. * - * `MessageReference`, `VariantExpression`, `AttributeExpression` and - * `SelectExpression` resolve to raw Runtime Entries objects and the result of - * the resolution needs to be passed into `Type` to get their real value. - * This is useful for composing expressions. Consider: - * - * brand-name[nominative] - * - * which is a `VariantExpression` with properties `id: MessageReference` and - * `key: Keyword`. If `MessageReference` was resolved eagerly, it would - * instantly resolve to the value of the `brand-name` message. Instead, we - * want to get the message object and look for its `nominative` variant. - * - * All other expressions (except for `FunctionReference` which is only used as - * a callee in `FunctionExpression`) resolve to an instance of `FluentType`. - * The caller should use the `toString` method to convert the instance to a - * native value. - * + * All expressions resolve to an instance of `FluentType`. The caller should + * use the `toString` method to convert the instance to a native value. * * All functions in this file pass around a special object called `env`. * This object stores a set of elements used by all resolve functions: @@ -62,20 +47,7 @@ const FSI = "\u2068"; const PDI = "\u2069"; -/** - * Helper for matching a variant key to the given selector. - * - * Used in SelectExpressions and VariantExpressions. - * - * @param {FluentBundle} bundle - * Resolver environment object. - * @param {FluentType} key - * The key of the currently considered variant. - * @param {FluentType} selector - * The selector based om which the correct variant should be chosen. - * @returns {FluentType} - * @private - */ +// Helper: match a variant key to the given selector. function match(bundle, selector, key) { if (key === selector) { // Both are strings. @@ -100,23 +72,10 @@ function match(bundle, selector, key) { return false; } -/** - * Helper for choosing the default value from a set of members. - * - * Used in SelectExpressions and Type. - * - * @param {Object} env - * Resolver environment object. - * @param {Object} members - * Hash map of variants from which the default value is to be selected. - * @param {Number} star - * The index of the default variant. - * @returns {FluentType} - * @private - */ -function DefaultMember(env, members, star) { - if (members[star]) { - return members[star]; +// Helper: resolve the default variant from a list of variants. +function getDefault(env, variants, star) { + if (variants[star]) { + return Type(env, variants[star]); } const { errors } = env; @@ -124,176 +83,34 @@ function DefaultMember(env, members, star) { return new FluentNone(); } - -/** - * Resolve a reference to another message. - * - * @param {Object} env - * Resolver environment object. - * @param {Object} id - * The identifier of the message to be resolved. - * @param {String} id.name - * The name of the identifier. - * @returns {FluentType} - * @private - */ -function MessageReference(env, {name}) { - const { bundle, errors } = env; - const message = name[0] === "-" - ? bundle._terms.get(name) - : bundle._messages.get(name); - - if (!message) { - const err = name[0] === "-" - ? new ReferenceError(`Unknown term: ${name}`) - : new ReferenceError(`Unknown message: ${name}`); - errors.push(err); - return new FluentNone(name); - } - - return message; -} - -/** - * Resolve a variant expression to the variant object. - * - * @param {Object} env - * Resolver environment object. - * @param {Object} expr - * An expression to be resolved. - * @param {Object} expr.ref - * An Identifier of a message for which the variant is resolved. - * @param {Object} expr.id.name - * Name a message for which the variant is resolved. - * @param {Object} expr.key - * Variant key to be resolved. - * @returns {FluentType} - * @private - */ -function VariantExpression(env, {ref, selector}) { - const message = MessageReference(env, ref); - if (message instanceof FluentNone) { - return message; - } - - const { bundle, errors } = env; - const sel = Type(env, selector); - const value = message.value || message; - - function isVariantList(node) { - return Array.isArray(node) && - node[0].type === "select" && - node[0].selector === null; - } - - if (isVariantList(value)) { - // Match the specified key against keys of each variant, in order. - for (const variant of value[0].variants) { - const key = Type(env, variant.key); - if (match(env.bundle, sel, key)) { - return variant; +// Helper: resolve arguments to a call expression. +function getArguments(env, args) { + const positional = []; + const named = {}; + + if (args) { + for (const arg of args) { + if (arg.type === "narg") { + named[arg.name] = Type(env, arg.value); + } else { + positional.push(Type(env, arg)); } } } - errors.push( - new ReferenceError(`Unknown variant: ${sel.toString(bundle)}`)); - return Type(env, message); + return [positional, named]; } - -/** - * Resolve an attribute expression to the attribute object. - * - * @param {Object} env - * Resolver environment object. - * @param {Object} expr - * An expression to be resolved. - * @param {String} expr.ref - * An ID of a message for which the attribute is resolved. - * @param {String} expr.name - * Name of the attribute to be resolved. - * @returns {FluentType} - * @private - */ -function AttributeExpression(env, {ref, name}) { - const message = MessageReference(env, ref); - if (message instanceof FluentNone) { - return message; - } - - if (message.attrs) { - // Match the specified name against keys of each attribute. - for (const attrName in message.attrs) { - if (name === attrName) { - return message.attrs[name]; - } - } - } - - const { errors } = env; - errors.push(new ReferenceError(`Unknown attribute: ${name}`)); - return Type(env, message); -} - -/** - * Resolve a select expression to the member object. - * - * @param {Object} env - * Resolver environment object. - * @param {Object} expr - * An expression to be resolved. - * @param {String} expr.selector - * Selector expression - * @param {Array} expr.variants - * List of variants for the select expression. - * @param {Number} expr.star - * Index of the default variant. - * @returns {FluentType} - * @private - */ -function SelectExpression(env, {selector, variants, star}) { - if (selector === null) { - return DefaultMember(env, variants, star); - } - - let sel = Type(env, selector); - if (sel instanceof FluentNone) { - return DefaultMember(env, variants, star); - } - - // Match the selector against keys of each variant, in order. - for (const variant of variants) { - const key = Type(env, variant.key); - if (match(env.bundle, sel, key)) { - return variant; - } - } - - return DefaultMember(env, variants, star); -} - - -/** - * Resolve expression to a Fluent type. - * - * JavaScript strings are a special case. Since they natively have the - * `toString` method they can be used as if they were a Fluent type without - * paying the cost of creating a instance of one. - * - * @param {Object} env - * Resolver environment object. - * @param {Object} expr - * An expression object to be resolved into a Fluent type. - * @returns {FluentType} - * @private - */ +// Resolve an expression to a Fluent type. function Type(env, expr) { - // A fast-path for strings which are the most common case, and for - // `FluentNone` which doesn't require any additional logic. + // A fast-path for strings which are the most common case. Since they + // natively have the `toString` method they can be used as if they were + // a FluentType instance without incurring the cost of creating one. if (typeof expr === "string") { return env.bundle._transform(expr); } + + // A fast-path for `FluentNone` which doesn't require any additional logic. if (expr instanceof FluentNone) { return expr; } @@ -304,7 +121,6 @@ function Type(env, expr) { return Pattern(env, expr); } - switch (expr.type) { case "str": return expr.value; @@ -312,28 +128,14 @@ function Type(env, expr) { return new FluentNumber(expr.value); case "var": return VariableReference(env, expr); - case "call": - return expr.ref.name[0] === "-" - ? MacroExpression(env, expr) - : FunctionExpression(env, expr); - case "ref": { - const message = MessageReference(env, expr); - return expr.name[0] === "-" - ? Type({...env, args: {}}, message) - : Type(env, message); - } - case "getattr": { - const attr = AttributeExpression(env, expr); - return Type(env, attr); - } - case "getvar": { - const variant = VariantExpression(env, expr); - return Type(env, variant); - } - case "select": { - const member = SelectExpression(env, expr); - return Type(env, member); - } + case "term": + return TermReference({...env, args: {}}, expr); + case "ref": + return expr.args + ? FunctionReference(env, expr) + : MessageReference(env, expr); + case "select": + return SelectExpression(env, expr); case undefined: { // If it's a node with a value, resolve the value. if (expr.value !== null && expr.value !== undefined) { @@ -349,18 +151,7 @@ function Type(env, expr) { } } -/** - * Resolve a reference to a variable. - * - * @param {Object} env - * Resolver environment object. - * @param {Object} expr - * An expression to be resolved. - * @param {String} expr.name - * Name of an argument to be returned. - * @returns {FluentType} - * @private - */ +// Resolve a reference to a variable. function VariableReference(env, {name}) { const { args, errors } = env; @@ -394,22 +185,76 @@ function VariableReference(env, {name}) { } } -/** - * Resolve a reference to a function. - * - * @param {Object} env - * Resolver environment object. - * @param {Object} expr - * An expression to be resolved. - * @param {String} expr.name - * Name of the function to be returned. - * @returns {Function} - * @private - */ -function FunctionReference(env, {name}) { - // Some functions are built-in. Others may be provided by the runtime via +// Resolve a reference to another message. +function MessageReference(env, {name, attr}) { + const {bundle, errors} = env; + const message = bundle._messages.get(name); + if (!message) { + const err = new ReferenceError(`Unknown message: ${name}`); + errors.push(err); + return new FluentNone(name); + } + + if (attr) { + const attribute = message.attrs && message.attrs[attr]; + if (attribute) { + return Type(env, attribute); + } + errors.push(new ReferenceError(`Unknown attribute: ${attr}`)); + return Type(env, message); + } + + return Type(env, message); +} + +// Resolve a call to a Term with key-value arguments. +function TermReference(env, {name, attr, selector, args}) { + const {bundle, errors} = env; + + const id = `-${name}`; + const term = bundle._terms.get(id); + if (!term) { + const err = new ReferenceError(`Unknown term: ${id}`); + errors.push(err); + return new FluentNone(id); + } + + // Every TermReference has its own args. + const [, keyargs] = getArguments(env, args); + const local = {...env, args: keyargs}; + + if (attr) { + const attribute = term.attrs && term.attrs[attr]; + if (attribute) { + return Type(local, attribute); + } + errors.push(new ReferenceError(`Unknown attribute: ${attr}`)); + return Type(local, term); + } + + const variantList = getVariantList(term); + if (selector && variantList) { + return SelectExpression(local, {...variantList, selector}); + } + + return Type(local, term); +} + +// Helper: convert a value into a variant list, if possible. +function getVariantList(term) { + const value = term.value || term; + return Array.isArray(value) + && value[0].type === "select" + && value[0].selector === null + ? value[0] + : null; +} + +// Resolve a call to a Function with positional and key-value arguments. +function FunctionReference(env, {name, args}) { + // Some functions are built-in. Others may be provided by the runtime via // the `FluentBundle` constructor. - const { bundle: { _functions }, errors } = env; + const {bundle: {_functions}, errors} = env; const func = _functions[name] || builtins[name]; if (!func) { @@ -422,88 +267,39 @@ function FunctionReference(env, {name}) { return new FluentNone(`${name}()`); } - return func; -} - -/** - * Resolve a call to a Function with positional and key-value arguments. - * - * @param {Object} env - * Resolver environment object. - * @param {Object} expr - * An expression to be resolved. - * @param {Object} expr.ref - * FTL Function object. - * @param {Array} expr.args - * FTL Function argument list. - * @returns {FluentType} - * @private - */ -function FunctionExpression(env, {ref, args}) { - const func = FunctionReference(env, ref); - if (func instanceof FluentNone) { - return func; - } - - const posargs = []; - const keyargs = {}; - - for (const arg of args) { - if (arg.type === "narg") { - keyargs[arg.name] = Type(env, arg.value); - } else { - posargs.push(Type(env, arg)); - } - } - try { - return func(posargs, keyargs); + return func(...getArguments(env, args)); } catch (e) { // XXX Report errors. return new FluentNone(); } } -/** - * Resolve a call to a Term with key-value arguments. - * - * @param {Object} env - * Resolver environment object. - * @param {Object} expr - * An expression to be resolved. - * @param {Object} expr.ref - * FTL Function object. - * @param {Array} expr.args - * FTL Function argument list. - * @returns {FluentType} - * @private - */ -function MacroExpression(env, {ref, args}) { - const callee = MessageReference(env, ref); - if (callee instanceof FluentNone) { - return callee; +// Resolve a select expression to the member object. +function SelectExpression(env, {selector, variants, star}) { + if (selector === null) { + return getDefault(env, variants, star); + } + + let sel = Type(env, selector); + if (sel instanceof FluentNone) { + const variant = getDefault(env, variants, star); + return Type(env, variant); } - const keyargs = {}; - for (const arg of args) { - if (arg.type === "narg") { - keyargs[arg.name] = Type(env, arg.value); + // Match the selector against keys of each variant, in order. + for (const variant of variants) { + const key = Type(env, variant.key); + if (match(env.bundle, sel, key)) { + return Type(env, variant); } } - return Type({...env, args: keyargs}, callee); + const variant = getDefault(env, variants, star); + return Type(env, variant); } -/** - * Resolve a pattern (a complex string with placeables). - * - * @param {Object} env - * Resolver environment object. - * @param {Array} ptn - * Array of pattern elements. - * @returns {Array} - * @private - */ +// Resolve a pattern (a complex string with placeables). function Pattern(env, ptn) { const { bundle, dirty, errors } = env; diff --git a/fluent/src/resource.js b/fluent/src/resource.js index 7afa41a3e..4d424aa2c 100644 --- a/fluent/src/resource.js +++ b/fluent/src/resource.js @@ -2,17 +2,17 @@ import FluentError from "./error.js"; // This regex is used to iterate through the beginnings of messages and terms. // With the /m flag, the ^ matches at the beginning of every line. -const RE_MESSAGE_START = /^(-?[a-zA-Z][a-zA-Z0-9_-]*) *= */mg; +const RE_MESSAGE_START = /^(-?[a-zA-Z][\w-]*) *= */mg; // Both Attributes and Variants are parsed in while loops. These regexes are // used to break out of them. -const RE_ATTRIBUTE_START = /\.([a-zA-Z][a-zA-Z0-9_-]*) *= */y; +const RE_ATTRIBUTE_START = /\.([a-zA-Z][\w-]*) *= */y; // [^] matches all characters, including newlines. // XXX Use /s (dotall) when it's widely supported. const RE_VARIANT_START = /\*?\[[^]*?] */y; -const RE_IDENTIFIER = /(-?[a-zA-Z][a-zA-Z0-9_-]*)/y; const RE_NUMBER_LITERAL = /(-?[0-9]+(\.[0-9]+)?)/y; +const RE_REFERENCE = /([$-])?([a-zA-Z][\w-]*)(?:\.([a-zA-Z][\w-]*))?/y; // A "run" is a sequence of text or string literal characters which don't // require any special handling. For TextElements such special characters are: { @@ -40,7 +40,7 @@ const TOKEN_BRACE_OPEN = /{\s*/y; const TOKEN_BRACE_CLOSE = /\s*}/y; const TOKEN_BRACKET_OPEN = /\[\s*/y; const TOKEN_BRACKET_CLOSE = /\s*]/y; -const TOKEN_PAREN_OPEN = /\(\s*/y; +const TOKEN_PAREN_OPEN = /\s*\(\s*/y; const TOKEN_ARROW = /\s*->\s*/y; const TOKEN_COLON = /\s*:\s*/y; // Note the optional comma. As a deviation from the Fluent EBNF, the parser @@ -134,7 +134,7 @@ export default class FluentResource extends Map { return false; } - // Execute a regex, advance the cursor, and return the capture group. + // Execute a regex, advance the cursor, and return all capture groups. function match(re) { re.lastIndex = cursor; let result = re.exec(source); @@ -142,7 +142,12 @@ export default class FluentResource extends Map { throw new FluentError(`Expected ${re.toString()}`); } cursor = re.lastIndex; - return result[1]; + return result; + } + + // Execute a regex, advance the cursor, and return the capture group. + function match1(re) { + return match(re)[1]; } function parseMessage() { @@ -163,7 +168,7 @@ export default class FluentResource extends Map { let attrs = {}; while (test(RE_ATTRIBUTE_START)) { - let name = match(RE_ATTRIBUTE_START); + let name = match1(RE_ATTRIBUTE_START); let value = parsePattern(); if (value === null) { throw new FluentError("Expected attribute value"); @@ -177,7 +182,7 @@ export default class FluentResource extends Map { function parsePattern() { // First try to parse any simple text on the same line as the id. if (test(RE_TEXT_RUN)) { - var first = match(RE_TEXT_RUN); + var first = match1(RE_TEXT_RUN); } // If there's a placeable on the first line, parse a complex pattern. @@ -216,7 +221,7 @@ export default class FluentResource extends Map { while (true) { if (test(RE_TEXT_RUN)) { - elements.push(match(RE_TEXT_RUN)); + elements.push(match1(RE_TEXT_RUN)); continue; } @@ -294,27 +299,20 @@ export default class FluentResource extends Map { return parsePlaceable(); } - if (consumeChar("$")) { - return {type: "var", name: match(RE_IDENTIFIER)}; - } - - if (test(RE_IDENTIFIER)) { - let ref = {type: "ref", name: match(RE_IDENTIFIER)}; - - if (consumeChar(".")) { - let name = match(RE_IDENTIFIER); - return {type: "getattr", ref, name}; - } + if (test(RE_REFERENCE)) { + let [, sigil, name, attr = null] = match(RE_REFERENCE); + let type = {"$": "var", "-": "term"}[sigil] || "ref"; if (source[cursor] === "[") { - return {type: "getvar", ref, selector: parseVariantKey()}; + // DEPRECATED VariantExpressions will be removed before 1.0. + return {type, name, selector: parseVariantKey()}; } if (consumeToken(TOKEN_PAREN_OPEN)) { - return {type: "call", ref, args: parseArguments()}; + return {type, name, attr, args: parseArguments()}; } - return ref; + return {type, name, attr, args: null}; } return parseLiteral(); @@ -378,7 +376,7 @@ export default class FluentResource extends Map { consumeToken(TOKEN_BRACKET_OPEN, FluentError); let key = test(RE_NUMBER_LITERAL) ? parseNumberLiteral() - : match(RE_IDENTIFIER); + : match(RE_REFERENCE)[2]; consumeToken(TOKEN_BRACKET_CLOSE, FluentError); return key; } @@ -396,14 +394,14 @@ export default class FluentResource extends Map { } function parseNumberLiteral() { - return {type: "num", value: match(RE_NUMBER_LITERAL)}; + return {type: "num", value: match1(RE_NUMBER_LITERAL)}; } function parseStringLiteral() { consumeChar("\"", FluentError); let value = ""; while (true) { - value += match(RE_STRING_RUN); + value += match1(RE_STRING_RUN); if (source[cursor] === "\\") { value += parseEscapeSequence(); @@ -422,7 +420,7 @@ export default class FluentResource extends Map { // Unescape known escape sequences. function parseEscapeSequence() { if (test(RE_UNICODE_ESCAPE)) { - let sequence = match(RE_UNICODE_ESCAPE); + let sequence = match1(RE_UNICODE_ESCAPE); let codepoint = parseInt(sequence, 16); return codepoint <= 0xD7FF || 0xE000 <= codepoint // It's a Unicode scalar value. @@ -433,7 +431,7 @@ export default class FluentResource extends Map { } if (test(RE_STRING_ESCAPE)) { - return match(RE_STRING_ESCAPE); + return match1(RE_STRING_ESCAPE); } throw new FluentError("Unknown escape sequence"); diff --git a/fluent/test/fixtures_reference/call_expressions.json b/fluent/test/fixtures_reference/call_expressions.json index 10682530e..9334ff18c 100644 --- a/fluent/test/fixtures_reference/call_expressions.json +++ b/fluent/test/fixtures_reference/call_expressions.json @@ -1,51 +1,9 @@ { - "function-callee": [ - { - "type": "call", - "ref": { - "type": "ref", - "name": "FUNCTION" - }, - "args": [] - } - ], - "term-callee": [ - { - "type": "call", - "ref": { - "type": "ref", - "name": "-term" - }, - "args": [] - } - ], - "mixed-case-callee": [ - { - "type": "call", - "ref": { - "type": "ref", - "name": "Function" - }, - "args": [] - } - ], - "message-callee": [ - { - "type": "call", - "ref": { - "type": "ref", - "name": "message" - }, - "args": [] - } - ], "positional-args": [ { - "type": "call", - "ref": { - "type": "ref", - "name": "FUN" - }, + "type": "ref", + "name": "FUN", + "attr": null, "args": [ { "type": "num", @@ -57,18 +15,18 @@ }, { "type": "ref", - "name": "msg" + "name": "msg", + "attr": null, + "args": null } ] } ], "named-args": [ { - "type": "call", - "ref": { - "type": "ref", - "name": "FUN" - }, + "type": "ref", + "name": "FUN", + "attr": null, "args": [ { "type": "narg", @@ -91,11 +49,9 @@ ], "dense-named-args": [ { - "type": "call", - "ref": { - "type": "ref", - "name": "FUN" - }, + "type": "ref", + "name": "FUN", + "attr": null, "args": [ { "type": "narg", @@ -118,11 +74,9 @@ ], "mixed-args": [ { - "type": "call", - "ref": { - "type": "ref", - "name": "FUN" - }, + "type": "ref", + "name": "FUN", + "attr": null, "args": [ { "type": "num", @@ -134,7 +88,9 @@ }, { "type": "ref", - "name": "msg" + "name": "msg", + "attr": null, + "args": null }, { "type": "narg", @@ -157,11 +113,9 @@ ], "shuffled-args": [ { - "type": "call", - "ref": { - "type": "ref", - "name": "FUN" - }, + "type": "ref", + "name": "FUN", + "attr": null, "args": [ { "type": "num", @@ -189,18 +143,18 @@ }, { "type": "ref", - "name": "msg" + "name": "msg", + "attr": null, + "args": null } ] } ], "duplicate-named-args": [ { - "type": "call", - "ref": { - "type": "ref", - "name": "FUN" - }, + "type": "ref", + "name": "FUN", + "attr": null, "args": [ { "type": "narg", @@ -223,11 +177,9 @@ ], "sparse-inline-call": [ { - "type": "call", - "ref": { - "type": "ref", - "name": "FUN" - }, + "type": "ref", + "name": "FUN", + "attr": null, "args": [ { "type": "str", @@ -235,7 +187,9 @@ }, { "type": "ref", - "name": "msg" + "name": "msg", + "attr": null, + "args": null }, { "type": "narg", @@ -250,21 +204,17 @@ ], "empty-inline-call": [ { - "type": "call", - "ref": { - "type": "ref", - "name": "FUN" - }, + "type": "ref", + "name": "FUN", + "attr": null, "args": [] } ], "multiline-call": [ { - "type": "call", - "ref": { - "type": "ref", - "name": "FUN" - }, + "type": "ref", + "name": "FUN", + "attr": null, "args": [ { "type": "str", @@ -272,7 +222,9 @@ }, { "type": "ref", - "name": "msg" + "name": "msg", + "attr": null, + "args": null }, { "type": "narg", @@ -287,11 +239,9 @@ ], "sparse-multiline-call": [ { - "type": "call", - "ref": { - "type": "ref", - "name": "FUN" - }, + "type": "ref", + "name": "FUN", + "attr": null, "args": [ { "type": "str", @@ -299,7 +249,9 @@ }, { "type": "ref", - "name": "msg" + "name": "msg", + "attr": null, + "args": null }, { "type": "narg", @@ -314,21 +266,17 @@ ], "empty-multiline-call": [ { - "type": "call", - "ref": { - "type": "ref", - "name": "FUN" - }, + "type": "ref", + "name": "FUN", + "attr": null, "args": [] } ], "unindented-arg-number": [ { - "type": "call", - "ref": { - "type": "ref", - "name": "FUN" - }, + "type": "ref", + "name": "FUN", + "attr": null, "args": [ { "type": "num", @@ -339,11 +287,9 @@ ], "unindented-arg-string": [ { - "type": "call", - "ref": { - "type": "ref", - "name": "FUN" - }, + "type": "ref", + "name": "FUN", + "attr": null, "args": [ { "type": "str", @@ -354,63 +300,59 @@ ], "unindented-arg-msg-ref": [ { - "type": "call", - "ref": { - "type": "ref", - "name": "FUN" - }, + "type": "ref", + "name": "FUN", + "attr": null, "args": [ { "type": "ref", - "name": "msg" + "name": "msg", + "attr": null, + "args": null } ] } ], "unindented-arg-term-ref": [ { - "type": "call", - "ref": { - "type": "ref", - "name": "FUN" - }, + "type": "ref", + "name": "FUN", + "attr": null, "args": [ { - "type": "ref", - "name": "-msg" + "type": "term", + "name": "msg", + "attr": null, + "args": null } ] } ], "unindented-arg-var-ref": [ { - "type": "call", - "ref": { - "type": "ref", - "name": "FUN" - }, + "type": "ref", + "name": "FUN", + "attr": null, "args": [ { "type": "var", - "name": "var" + "name": "var", + "attr": null, + "args": null } ] } ], "unindented-arg-call": [ { - "type": "call", - "ref": { - "type": "ref", - "name": "FUN" - }, + "type": "ref", + "name": "FUN", + "attr": null, "args": [ { - "type": "call", - "ref": { - "type": "ref", - "name": "OTHER" - }, + "type": "ref", + "name": "OTHER", + "attr": null, "args": [] } ] @@ -418,11 +360,9 @@ ], "unindented-named-arg": [ { - "type": "call", - "ref": { - "type": "ref", - "name": "FUN" - }, + "type": "ref", + "name": "FUN", + "attr": null, "args": [ { "type": "narg", @@ -437,26 +377,24 @@ ], "unindented-closing-paren": [ { - "type": "call", - "ref": { - "type": "ref", - "name": "FUN" - }, + "type": "ref", + "name": "FUN", + "attr": null, "args": [ { "type": "ref", - "name": "x" + "name": "x", + "attr": null, + "args": null } ] } ], "one-argument": [ { - "type": "call", - "ref": { - "type": "ref", - "name": "FUN" - }, + "type": "ref", + "name": "FUN", + "attr": null, "args": [ { "type": "num", @@ -467,11 +405,9 @@ ], "many-arguments": [ { - "type": "call", - "ref": { - "type": "ref", - "name": "FUN" - }, + "type": "ref", + "name": "FUN", + "attr": null, "args": [ { "type": "num", @@ -490,11 +426,9 @@ ], "inline-sparse-args": [ { - "type": "call", - "ref": { - "type": "ref", - "name": "FUN" - }, + "type": "ref", + "name": "FUN", + "attr": null, "args": [ { "type": "num", @@ -513,11 +447,9 @@ ], "mulitline-args": [ { - "type": "call", - "ref": { - "type": "ref", - "name": "FUN" - }, + "type": "ref", + "name": "FUN", + "attr": null, "args": [ { "type": "num", @@ -532,11 +464,9 @@ ], "mulitline-sparse-args": [ { - "type": "call", - "ref": { - "type": "ref", - "name": "FUN" - }, + "type": "ref", + "name": "FUN", + "attr": null, "args": [ { "type": "num", @@ -551,11 +481,9 @@ ], "sparse-named-arg": [ { - "type": "call", - "ref": { - "type": "ref", - "name": "FUN" - }, + "type": "ref", + "name": "FUN", + "attr": null, "args": [ { "type": "narg", @@ -586,11 +514,9 @@ ], "unindented-colon": [ { - "type": "call", - "ref": { - "type": "ref", - "name": "FUN" - }, + "type": "ref", + "name": "FUN", + "attr": null, "args": [ { "type": "narg", @@ -605,11 +531,9 @@ ], "unindented-value": [ { - "type": "call", - "ref": { - "type": "ref", - "name": "FUN" - }, + "type": "ref", + "name": "FUN", + "attr": null, "args": [ { "type": "narg", diff --git a/fluent/test/fixtures_reference/callee_expressions.json b/fluent/test/fixtures_reference/callee_expressions.json new file mode 100644 index 000000000..0db21c11a --- /dev/null +++ b/fluent/test/fixtures_reference/callee_expressions.json @@ -0,0 +1,184 @@ +{ + "function-callee-placeable": [ + { + "type": "ref", + "name": "FUNCTION", + "attr": null, + "args": [] + } + ], + "term-callee-placeable": [ + { + "type": "term", + "name": "term", + "attr": null, + "args": [] + } + ], + "message-callee-placeable": [ + { + "type": "ref", + "name": "message", + "attr": null, + "args": [] + } + ], + "mixed-case-callee-placeable": [ + { + "type": "ref", + "name": "Function", + "attr": null, + "args": [] + } + ], + "message-attr-callee-placeable": [ + { + "type": "ref", + "name": "message", + "attr": "attr", + "args": [] + } + ], + "term-attr-callee-placeable": [ + { + "type": "term", + "name": "term", + "attr": "attr", + "args": [] + } + ], + "variable-callee-placeable": [ + { + "type": "var", + "name": "variable", + "attr": null, + "args": [] + } + ], + "function-callee-selector": [ + { + "type": "select", + "selector": { + "type": "ref", + "name": "FUNCTION", + "attr": null, + "args": [] + }, + "variants": [ + { + "key": "key", + "value": "Value" + } + ], + "star": 0 + } + ], + "term-attr-callee-selector": [ + { + "type": "select", + "selector": { + "type": "term", + "name": "term", + "attr": "attr", + "args": [] + }, + "variants": [ + { + "key": "key", + "value": "Value" + } + ], + "star": 0 + } + ], + "message-callee-selector": [ + { + "type": "select", + "selector": { + "type": "ref", + "name": "message", + "attr": null, + "args": [] + }, + "variants": [ + { + "key": "key", + "value": "Value" + } + ], + "star": 0 + } + ], + "mixed-case-callee-selector": [ + { + "type": "select", + "selector": { + "type": "ref", + "name": "Function", + "attr": null, + "args": [] + }, + "variants": [ + { + "key": "key", + "value": "Value" + } + ], + "star": 0 + } + ], + "message-attr-callee-selector": [ + { + "type": "select", + "selector": { + "type": "ref", + "name": "message", + "attr": "attr", + "args": [] + }, + "variants": [ + { + "key": "key", + "value": "Value" + } + ], + "star": 0 + } + ], + "term-callee-selector": [ + { + "type": "select", + "selector": { + "type": "term", + "name": "term", + "attr": null, + "args": [] + }, + "variants": [ + { + "key": "key", + "value": "Value" + } + ], + "star": 0 + } + ], + "variable-callee-selector": [ + { + "type": "select", + "selector": { + "type": "var", + "name": "variable", + "attr": null, + "args": [] + }, + "variants": [ + { + "key": "key", + "value": "Value" + } + ], + "star": 0 + } + ] +} diff --git a/fluent/test/fixtures_reference/cr.json b/fluent/test/fixtures_reference/cr.json index 88e772c02..19382381e 100644 --- a/fluent/test/fixtures_reference/cr.json +++ b/fluent/test/fixtures_reference/cr.json @@ -17,7 +17,9 @@ "type": "select", "selector": { "type": "var", - "name": "sel" + "name": "sel", + "attr": null, + "args": null } } ] diff --git a/fluent/test/fixtures_reference/crlf.json b/fluent/test/fixtures_reference/crlf.json index aa9ccd77c..f3b058a2d 100644 --- a/fluent/test/fixtures_reference/crlf.json +++ b/fluent/test/fixtures_reference/crlf.json @@ -15,7 +15,9 @@ "type": "select", "selector": { "type": "var", - "name": "sel" + "name": "sel", + "attr": null, + "args": null } } ] diff --git a/fluent/test/fixtures_reference/escaped_characters.json b/fluent/test/fixtures_reference/escaped_characters.json index cc1619930..b5046fdb7 100644 --- a/fluent/test/fixtures_reference/escaped_characters.json +++ b/fluent/test/fixtures_reference/escaped_characters.json @@ -5,7 +5,9 @@ "Value with \\", { "type": "ref", - "name": "placeable" + "name": "placeable", + "attr": null, + "args": null } ], "text-backslash-u": "\\u0041", diff --git a/fluent/test/fixtures_reference/member_expressions.json b/fluent/test/fixtures_reference/member_expressions.json index 5d53602fd..b7d5eb7b9 100644 --- a/fluent/test/fixtures_reference/member_expressions.json +++ b/fluent/test/fixtures_reference/member_expressions.json @@ -1,22 +1,102 @@ { - "variant-expression": [ + "message-attribute-expression-placeable": [ { - "type": "getvar", - "ref": { + "type": "ref", + "name": "msg", + "attr": "attr", + "args": null + } + ], + "term-variant-expression-placeable": [ + { + "type": "term", + "name": "term", + "selector": "case" + } + ], + "message-variant-expression-placeable": [ + { + "type": "ref", + "name": "msg", + "selector": "case" + } + ], + "term-attribute-expression-placeable": [ + { + "type": "term", + "name": "term", + "attr": "attr", + "args": null + } + ], + "term-attribute-expression-selector": [ + { + "type": "select", + "selector": { + "type": "term", + "name": "term", + "attr": "attr", + "args": null + }, + "variants": [ + { + "key": "key", + "value": "Value" + } + ], + "star": 0 + } + ], + "message-attribute-expression-selector": [ + { + "type": "select", + "selector": { "type": "ref", - "name": "msg" + "name": "msg", + "attr": "attr", + "args": null }, - "selector": "case" + "variants": [ + { + "key": "key", + "value": "Value" + } + ], + "star": 0 + } + ], + "term-variant-expression-selector": [ + { + "type": "select", + "selector": { + "type": "term", + "name": "term", + "selector": "case" + }, + "variants": [ + { + "key": "key", + "value": "Value" + } + ], + "star": 0 } ], - "attribute-expression": [ + "message-variant-expression-selector": [ { - "type": "getattr", - "ref": { + "type": "select", + "selector": { "type": "ref", - "name": "-term" + "name": "msg", + "selector": "case" }, - "name": "attr" + "variants": [ + { + "key": "key", + "value": "Value" + } + ], + "star": 0 } ] } diff --git a/fluent/test/fixtures_reference/messages.json b/fluent/test/fixtures_reference/messages.json index 03f07b796..9e6a598a8 100644 --- a/fluent/test/fixtures_reference/messages.json +++ b/fluent/test/fixtures_reference/messages.json @@ -26,5 +26,6 @@ "attr1": "Attribute 1" } }, - "key06": [] + "key06": [], + "KEY09": "Value 09" } diff --git a/fluent/test/fixtures_reference/reference_expressions.json b/fluent/test/fixtures_reference/reference_expressions.json index c6734e51c..8d7aca81b 100644 --- a/fluent/test/fixtures_reference/reference_expressions.json +++ b/fluent/test/fixtures_reference/reference_expressions.json @@ -1,26 +1,106 @@ { - "message-reference": [ + "message-reference-placeable": [ { "type": "ref", - "name": "msg" + "name": "msg", + "attr": null, + "args": null } ], - "term-reference": [ + "term-reference-placeable": [ { - "type": "ref", - "name": "-term" + "type": "term", + "name": "term", + "attr": null, + "args": null } ], - "variable-reference": [ + "variable-reference-placeable": [ { "type": "var", - "name": "var" + "name": "var", + "attr": null, + "args": null } ], - "not-call-expression": [ + "function-reference-placeable": [ { "type": "ref", - "name": "FUN" + "name": "FUN", + "attr": null, + "args": null + } + ], + "variable-reference-selector": [ + { + "type": "select", + "selector": { + "type": "var", + "name": "var", + "attr": null, + "args": null + }, + "variants": [ + { + "key": "key", + "value": "Value" + } + ], + "star": 0 + } + ], + "message-reference-selector": [ + { + "type": "select", + "selector": { + "type": "ref", + "name": "msg", + "attr": null, + "args": null + }, + "variants": [ + { + "key": "key", + "value": "Value" + } + ], + "star": 0 + } + ], + "term-reference-selector": [ + { + "type": "select", + "selector": { + "type": "term", + "name": "term", + "attr": null, + "args": null + }, + "variants": [ + { + "key": "key", + "value": "Value" + } + ], + "star": 0 + } + ], + "function-expression-selector": [ + { + "type": "select", + "selector": { + "type": "ref", + "name": "FUN", + "attr": null, + "args": null + }, + "variants": [ + { + "key": "key", + "value": "Value" + } + ], + "star": 0 } ] } diff --git a/fluent/test/fixtures_reference/select_expressions.json b/fluent/test/fixtures_reference/select_expressions.json index 8b204d856..52ace6831 100644 --- a/fluent/test/fixtures_reference/select_expressions.json +++ b/fluent/test/fixtures_reference/select_expressions.json @@ -3,11 +3,9 @@ { "type": "select", "selector": { - "type": "call", - "ref": { - "type": "ref", - "name": "BUILTIN" - }, + "type": "ref", + "name": "BUILTIN", + "attr": null, "args": [] }, "variants": [ @@ -32,12 +30,10 @@ { "type": "select", "selector": { - "type": "getattr", - "ref": { - "type": "ref", - "name": "-term" - }, - "name": "case" + "type": "term", + "name": "term", + "attr": "case", + "args": null }, "variants": [ { @@ -52,8 +48,10 @@ { "type": "select", "selector": { - "type": "ref", - "name": "-term" + "type": "term", + "name": "term", + "attr": null, + "args": null }, "variants": [ { @@ -68,11 +66,8 @@ { "type": "select", "selector": { - "type": "getvar", - "ref": { - "type": "ref", - "name": "-term" - }, + "type": "term", + "name": "term", "selector": "case" }, "variants": [ @@ -88,11 +83,9 @@ { "type": "select", "selector": { - "type": "call", - "ref": { - "type": "ref", - "name": "-term" - }, + "type": "term", + "name": "term", + "attr": null, "args": [ { "type": "narg", diff --git a/fluent/test/fixtures_reference/select_indent.json b/fluent/test/fixtures_reference/select_indent.json index c8e21fea3..94e57c450 100644 --- a/fluent/test/fixtures_reference/select_indent.json +++ b/fluent/test/fixtures_reference/select_indent.json @@ -4,7 +4,9 @@ "type": "select", "selector": { "type": "var", - "name": "selector" + "name": "selector", + "attr": null, + "args": null }, "variants": [ { @@ -20,7 +22,9 @@ "type": "select", "selector": { "type": "var", - "name": "selector" + "name": "selector", + "attr": null, + "args": null }, "variants": [ { @@ -36,7 +40,9 @@ "type": "select", "selector": { "type": "var", - "name": "selector" + "name": "selector", + "attr": null, + "args": null }, "variants": [ { @@ -52,7 +58,9 @@ "type": "select", "selector": { "type": "var", - "name": "selector" + "name": "selector", + "attr": null, + "args": null }, "variants": [ { @@ -68,7 +76,9 @@ "type": "select", "selector": { "type": "var", - "name": "selector" + "name": "selector", + "attr": null, + "args": null }, "variants": [ { @@ -84,7 +94,9 @@ "type": "select", "selector": { "type": "var", - "name": "selector" + "name": "selector", + "attr": null, + "args": null }, "variants": [ { @@ -100,7 +112,9 @@ "type": "select", "selector": { "type": "var", - "name": "selector" + "name": "selector", + "attr": null, + "args": null }, "variants": [ { @@ -116,7 +130,9 @@ "type": "select", "selector": { "type": "var", - "name": "selector" + "name": "selector", + "attr": null, + "args": null }, "variants": [ { @@ -132,7 +148,9 @@ "type": "select", "selector": { "type": "var", - "name": "selector" + "name": "selector", + "attr": null, + "args": null }, "variants": [ { @@ -148,7 +166,9 @@ "type": "select", "selector": { "type": "var", - "name": "selector" + "name": "selector", + "attr": null, + "args": null }, "variants": [ { @@ -168,7 +188,9 @@ "type": "select", "selector": { "type": "var", - "name": "selector" + "name": "selector", + "attr": null, + "args": null }, "variants": [ { @@ -196,7 +218,9 @@ "type": "select", "selector": { "type": "var", - "name": "selector" + "name": "selector", + "attr": null, + "args": null }, "variants": [ { @@ -216,7 +240,9 @@ "type": "select", "selector": { "type": "var", - "name": "selector" + "name": "selector", + "attr": null, + "args": null }, "variants": [ { diff --git a/fluent/test/fixtures_reference/term_parameters.json b/fluent/test/fixtures_reference/term_parameters.json index 78764b97c..81d435d09 100644 --- a/fluent/test/fixtures_reference/term_parameters.json +++ b/fluent/test/fixtures_reference/term_parameters.json @@ -4,7 +4,9 @@ "type": "select", "selector": { "type": "var", - "name": "arg" + "name": "arg", + "attr": null, + "args": null }, "variants": [ { @@ -17,27 +19,25 @@ ], "key01": [ { - "type": "ref", - "name": "-term" + "type": "term", + "name": "term", + "attr": null, + "args": null } ], "key02": [ { - "type": "call", - "ref": { - "type": "ref", - "name": "-term" - }, + "type": "term", + "name": "term", + "attr": null, "args": [] } ], "key03": [ { - "type": "call", - "ref": { - "type": "ref", - "name": "-term" - }, + "type": "term", + "name": "term", + "attr": null, "args": [ { "type": "narg", @@ -52,11 +52,9 @@ ], "key04": [ { - "type": "call", - "ref": { - "type": "ref", - "name": "-term" - }, + "type": "term", + "name": "term", + "attr": null, "args": [ { "type": "str", diff --git a/fluent/test/fixtures_reference/variables.json b/fluent/test/fixtures_reference/variables.json index 6f0de9904..38038db6a 100644 --- a/fluent/test/fixtures_reference/variables.json +++ b/fluent/test/fixtures_reference/variables.json @@ -2,31 +2,33 @@ "key01": [ { "type": "var", - "name": "var" + "name": "var", + "attr": null, + "args": null } ], "key02": [ { "type": "var", - "name": "var" + "name": "var", + "attr": null, + "args": null } ], "key03": [ { "type": "var", - "name": "var" + "name": "var", + "attr": null, + "args": null } ], "key04": [ { "type": "var", - "name": "var" - } - ], - "err03": [ - { - "type": "var", - "name": "-var" + "name": "var", + "attr": null, + "args": null } ] } diff --git a/fluent/test/fixtures_structure/crlf.json b/fluent/test/fixtures_structure/crlf.json index aa9ccd77c..f3b058a2d 100644 --- a/fluent/test/fixtures_structure/crlf.json +++ b/fluent/test/fixtures_structure/crlf.json @@ -15,7 +15,9 @@ "type": "select", "selector": { "type": "var", - "name": "sel" + "name": "sel", + "attr": null, + "args": null } } ] diff --git a/fluent/test/fixtures_structure/escape_sequences.json b/fluent/test/fixtures_structure/escape_sequences.json index cc1619930..b5046fdb7 100644 --- a/fluent/test/fixtures_structure/escape_sequences.json +++ b/fluent/test/fixtures_structure/escape_sequences.json @@ -5,7 +5,9 @@ "Value with \\", { "type": "ref", - "name": "placeable" + "name": "placeable", + "attr": null, + "args": null } ], "text-backslash-u": "\\u0041", diff --git a/fluent/test/fixtures_structure/expressions_call_args.json b/fluent/test/fixtures_structure/expressions_call_args.json index a15662ec0..031801867 100644 --- a/fluent/test/fixtures_structure/expressions_call_args.json +++ b/fluent/test/fixtures_structure/expressions_call_args.json @@ -1,11 +1,9 @@ { "key": [ { - "type": "call", - "ref": { - "type": "ref", - "name": "FOO" - }, + "type": "ref", + "name": "FOO", + "attr": null, "args": [ { "type": "narg", diff --git a/fluent/test/fixtures_structure/placeable_at_eol.json b/fluent/test/fixtures_structure/placeable_at_eol.json index 6841b74db..9fe6f6c59 100644 --- a/fluent/test/fixtures_structure/placeable_at_eol.json +++ b/fluent/test/fixtures_structure/placeable_at_eol.json @@ -3,7 +3,9 @@ "A multiline message with a ", { "type": "ref", - "name": "placeable" + "name": "placeable", + "attr": null, + "args": null }, "\n", "at the end of line. The message should", @@ -14,14 +16,18 @@ "A multiline message with a ", { "type": "ref", - "name": "placeable" + "name": "placeable", + "attr": null, + "args": null } ], "key3": [ "A singleline message with a ", { "type": "ref", - "name": "placeable" + "name": "placeable", + "attr": null, + "args": null } ] } diff --git a/fluent/test/fixtures_structure/sparse-messages.json b/fluent/test/fixtures_structure/sparse-messages.json index c88af5396..687ede0ae 100644 --- a/fluent/test/fixtures_structure/sparse-messages.json +++ b/fluent/test/fixtures_structure/sparse-messages.json @@ -28,7 +28,9 @@ "type": "select", "selector": { "type": "var", - "name": "sel" + "name": "sel", + "attr": null, + "args": null }, "variants": [ { diff --git a/fluent/test/fixtures_structure/term.json b/fluent/test/fixtures_structure/term.json index 37d6ac102..4e4aedd14 100644 --- a/fluent/test/fixtures_structure/term.json +++ b/fluent/test/fixtures_structure/term.json @@ -24,11 +24,8 @@ "update-command": [ "Zaktualizuj ", { - "type": "getvar", - "ref": { - "type": "ref", - "name": "-brand-name" - }, + "type": "term", + "name": "brand-name", "selector": "accusative" }, "." @@ -37,20 +34,20 @@ { "type": "select", "selector": { - "type": "getattr", - "ref": { - "type": "ref", - "name": "-brand-name" - }, - "name": "gender" + "type": "term", + "name": "brand-name", + "attr": "gender", + "args": null }, "variants": [ { "key": "masculine", "value": [ { - "type": "ref", - "name": "-brand-name" + "type": "term", + "name": "brand-name", + "attr": null, + "args": null }, " zostaล‚ pomyล›lnie zaktualizowany." ] @@ -59,8 +56,10 @@ "key": "feminine", "value": [ { - "type": "ref", - "name": "-brand-name" + "type": "term", + "name": "brand-name", + "attr": null, + "args": null }, " zostaล‚a pomyล›lnie zaktualizowana." ] @@ -70,8 +69,10 @@ "value": [ "Program ", { - "type": "ref", - "name": "-brand-name" + "type": "term", + "name": "brand-name", + "attr": null, + "args": null }, " zostaล‚ pomyล›lnie zaktualizowany." ] diff --git a/fluent/test/fixtures_structure/whitespace_trailing.json b/fluent/test/fixtures_structure/whitespace_trailing.json index 381195600..8e7512c7d 100644 --- a/fluent/test/fixtures_structure/whitespace_trailing.json +++ b/fluent/test/fixtures_structure/whitespace_trailing.json @@ -5,7 +5,9 @@ "Value ", { "type": "ref", - "name": "placeable" + "name": "placeable", + "attr": null, + "args": null }, "." ], diff --git a/fluent/test/macros_test.js b/fluent/test/macros_test.js index e1e9c775d..1f8d82f31 100644 --- a/fluent/test/macros_test.js +++ b/fluent/test/macros_test.js @@ -344,4 +344,96 @@ suite("Macros", function() { assert.equal(errs.length, 0); }); }); + + suite("Parameterized term attributes", function(){ + suiteSetup(function() { + bundle = new FluentBundle("en-US", { + useIsolating: false, + }); + bundle.addMessages(ftl` + -ship = Ship + .gender = {$style -> + *[traditional] neuter + [chicago] feminine + } + + ref-attr = {-ship.gender -> + *[masculine] He + [feminine] She + [neuter] It + } + call-attr-no-args = {-ship.gender() -> + *[masculine] He + [feminine] She + [neuter] It + } + call-attr-with-expected-arg = {-ship.gender(style: "chicago") -> + *[masculine] He + [feminine] She + [neuter] It + } + call-attr-with-other-arg = {-ship.gender(other: 3) -> + *[masculine] He + [feminine] She + [neuter] It + } + `); + }); + + test("Not parameterized, no externals", function() { + const msg = bundle.getMessage("ref-attr"); + const val = bundle.format(msg, {}, errs); + assert.equal(val, "It"); + assert.equal(errs.length, 1); + }); + + test("Not parameterized but with externals", function() { + const msg = bundle.getMessage("ref-attr"); + const val = bundle.format(msg, {style: "chicago"}, errs); + assert.equal(val, "It"); + assert.equal(errs.length, 1); + }); + + test("No arguments, no externals", function() { + const msg = bundle.getMessage("call-attr-no-args"); + const val = bundle.format(msg, {}, errs); + assert.equal(val, "It"); + assert.equal(errs.length, 1); + }); + + test("No arguments, but with externals", function() { + const msg = bundle.getMessage("call-attr-no-args"); + const val = bundle.format(msg, {style: "chicago"}, errs); + assert.equal(val, "It"); + assert.equal(errs.length, 1); + }); + + test("With expected args, no externals", function() { + const msg = bundle.getMessage("call-attr-with-expected-arg"); + const val = bundle.format(msg, {}, errs); + assert.equal(val, "She"); + assert.equal(errs.length, 0); + }); + + test("With expected args, and with externals", function() { + const msg = bundle.getMessage("call-attr-with-expected-arg"); + const val = bundle.format(msg, {style: "chicago"}, errs); + assert.equal(val, "She"); + assert.equal(errs.length, 0); + }); + + test("With other args, no externals", function() { + const msg = bundle.getMessage("call-attr-with-other-arg"); + const val = bundle.format(msg, {}, errs); + assert.equal(val, "It"); + assert.equal(errs.length, 1); + }); + + test("With other args, and with externals", function() { + const msg = bundle.getMessage("call-attr-with-other-arg"); + const val = bundle.format(msg, {style: "chicago"}, errs); + assert.equal(val, "It"); + assert.equal(errs.length, 1); + }); + }); }); diff --git a/fluent/test/values_ref_test.js b/fluent/test/values_ref_test.js index 4992d6827..10695eed9 100644 --- a/fluent/test/values_ref_test.js +++ b/fluent/test/values_ref_test.js @@ -12,12 +12,12 @@ suite('Referencing values', function(){ bundle = new FluentBundle('en-US', { useIsolating: false }); bundle.addMessages(ftl` key1 = Value 1 - key2 = { + -key2 = { [a] A2 *[b] B2 } key3 = Value { 3 } - key4 = { + -key4 = { [a] A{ 4 } *[b] B{ 4 } } @@ -26,16 +26,16 @@ suite('Referencing values', function(){ .b = B5 ref1 = { key1 } - ref2 = { key2 } + ref2 = { -key2 } ref3 = { key3 } - ref4 = { key4 } + ref4 = { -key4 } ref5 = { key5 } - ref6 = { key2[a] } - ref7 = { key2[b] } + ref6 = { -key2[a] } + ref7 = { -key2[b] } - ref8 = { key4[a] } - ref9 = { key4[b] } + ref8 = { -key4[a] } + ref9 = { -key4[b] } ref10 = { key5.a } ref11 = { key5.b } From df1de5eb70d6b05b09891f20f83f9f07597d3d6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Fri, 30 Nov 2018 13:02:30 +0100 Subject: [PATCH 17/23] (fluent) Parse variant keys in a strict way --- .../test/fixtures_structure/variant_keys.ftl | 61 +++ .../test/fixtures_structure/variant_keys.json | 500 ++++++++++++++++++ fluent/src/resource.js | 10 +- .../test/fixtures_structure/variant_keys.json | 40 ++ 4 files changed, 605 insertions(+), 6 deletions(-) create mode 100644 fluent-syntax/test/fixtures_structure/variant_keys.ftl create mode 100644 fluent-syntax/test/fixtures_structure/variant_keys.json create mode 100644 fluent/test/fixtures_structure/variant_keys.json diff --git a/fluent-syntax/test/fixtures_structure/variant_keys.ftl b/fluent-syntax/test/fixtures_structure/variant_keys.ftl new file mode 100644 index 000000000..fc0f241ca --- /dev/null +++ b/fluent-syntax/test/fixtures_structure/variant_keys.ftl @@ -0,0 +1,61 @@ +key01 = { $sel -> + *[ + key + ] Value +} + +key02 = { $sel -> + *[ + key + ] + + Value +} + +err01 = { $sel -> + *["key"] Value +} + +err02 = { $sel -> + *[-key] Value +} + +err03 = { $sel -> + *[-key.attr] Value +} + +err04 = { $sel -> + *[-key()] Value +} + +err05 = { $sel -> + *[-key.attr()] Value +} + +err06 = { $sel -> + *[key.attr] Value +} + +err07 = { $sel -> + *[$key] Value +} + +err08 = { $sel -> + *[FUNC()] Value +} + +err09 = { $sel -> + *[{key}] Value +} + +err10 = { $sel -> + *[{"key"}] Value +} + +err11 = { $sel -> + *[{3.14}] Value +} + +err12 = { $sel -> + *[{$key}] Value +} diff --git a/fluent-syntax/test/fixtures_structure/variant_keys.json b/fluent-syntax/test/fixtures_structure/variant_keys.json new file mode 100644 index 000000000..0115ee51c --- /dev/null +++ b/fluent-syntax/test/fixtures_structure/variant_keys.json @@ -0,0 +1,500 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key01", + "span": { + "type": "Span", + "start": 0, + "end": 5 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "sel", + "span": { + "type": "Span", + "start": 11, + "end": 14 + } + }, + "span": { + "type": "Span", + "start": 10, + "end": 14 + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key", + "span": { + "type": "Span", + "start": 33, + "end": 36 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value", + "span": { + "type": "Span", + "start": 43, + "end": 48 + } + } + ], + "span": { + "type": "Span", + "start": 43, + "end": 48 + } + }, + "default": true, + "span": { + "type": "Span", + "start": 22, + "end": 48 + } + } + ], + "span": { + "type": "Span", + "start": 10, + "end": 49 + } + }, + "span": { + "type": "Span", + "start": 8, + "end": 50 + } + } + ], + "span": { + "type": "Span", + "start": 8, + "end": 50 + } + }, + "attributes": [], + "comment": null, + "span": { + "type": "Span", + "start": 0, + "end": 50 + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key02", + "span": { + "type": "Span", + "start": 52, + "end": 57 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "sel", + "span": { + "type": "Span", + "start": 63, + "end": 66 + } + }, + "span": { + "type": "Span", + "start": 62, + "end": 66 + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key", + "span": { + "type": "Span", + "start": 85, + "end": 88 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value", + "span": { + "type": "Span", + "start": 104, + "end": 109 + } + } + ], + "span": { + "type": "Span", + "start": 100, + "end": 109 + } + }, + "default": true, + "span": { + "type": "Span", + "start": 74, + "end": 109 + } + } + ], + "span": { + "type": "Span", + "start": 62, + "end": 110 + } + }, + "span": { + "type": "Span", + "start": 60, + "end": 111 + } + } + ], + "span": { + "type": "Span", + "start": 60, + "end": 111 + } + }, + "attributes": [], + "comment": null, + "span": { + "type": "Span", + "start": 52, + "end": 111 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0004", + "args": [ + "a-zA-Z" + ], + "message": "Expected a character from range: \"a-zA-Z\"", + "span": { + "type": "Span", + "start": 137, + "end": 137 + } + } + ], + "content": "err01 = { $sel ->\n *[\"key\"] Value\n}\n\n", + "span": { + "type": "Span", + "start": 113, + "end": 153 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0004", + "args": [ + "0-9" + ], + "message": "Expected a character from range: \"0-9\"", + "span": { + "type": "Span", + "start": 178, + "end": 178 + } + } + ], + "content": "err02 = { $sel ->\n *[-key] Value\n}\n\n", + "span": { + "type": "Span", + "start": 153, + "end": 192 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0004", + "args": [ + "0-9" + ], + "message": "Expected a character from range: \"0-9\"", + "span": { + "type": "Span", + "start": 217, + "end": 217 + } + } + ], + "content": "err03 = { $sel ->\n *[-key.attr] Value\n}\n\n", + "span": { + "type": "Span", + "start": 192, + "end": 236 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0004", + "args": [ + "0-9" + ], + "message": "Expected a character from range: \"0-9\"", + "span": { + "type": "Span", + "start": 261, + "end": 261 + } + } + ], + "content": "err04 = { $sel ->\n *[-key()] Value\n}\n\n", + "span": { + "type": "Span", + "start": 236, + "end": 277 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0004", + "args": [ + "0-9" + ], + "message": "Expected a character from range: \"0-9\"", + "span": { + "type": "Span", + "start": 302, + "end": 302 + } + } + ], + "content": "err05 = { $sel ->\n *[-key.attr()] Value\n}\n\n", + "span": { + "type": "Span", + "start": 277, + "end": 323 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0003", + "args": [ + "]" + ], + "message": "Expected token: \"]\"", + "span": { + "type": "Span", + "start": 350, + "end": 350 + } + } + ], + "content": "err06 = { $sel ->\n *[key.attr] Value\n}\n\n", + "span": { + "type": "Span", + "start": 323, + "end": 366 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0004", + "args": [ + "a-zA-Z" + ], + "message": "Expected a character from range: \"a-zA-Z\"", + "span": { + "type": "Span", + "start": 390, + "end": 390 + } + } + ], + "content": "err07 = { $sel ->\n *[$key] Value\n}\n\n", + "span": { + "type": "Span", + "start": 366, + "end": 405 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0003", + "args": [ + "]" + ], + "message": "Expected token: \"]\"", + "span": { + "type": "Span", + "start": 433, + "end": 433 + } + } + ], + "content": "err08 = { $sel ->\n *[FUNC()] Value\n}\n\n", + "span": { + "type": "Span", + "start": 405, + "end": 446 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0004", + "args": [ + "a-zA-Z" + ], + "message": "Expected a character from range: \"a-zA-Z\"", + "span": { + "type": "Span", + "start": 470, + "end": 470 + } + } + ], + "content": "err09 = { $sel ->\n *[{key}] Value\n}\n\n", + "span": { + "type": "Span", + "start": 446, + "end": 486 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0004", + "args": [ + "a-zA-Z" + ], + "message": "Expected a character from range: \"a-zA-Z\"", + "span": { + "type": "Span", + "start": 510, + "end": 510 + } + } + ], + "content": "err10 = { $sel ->\n *[{\"key\"}] Value\n}\n\n", + "span": { + "type": "Span", + "start": 486, + "end": 528 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0004", + "args": [ + "a-zA-Z" + ], + "message": "Expected a character from range: \"a-zA-Z\"", + "span": { + "type": "Span", + "start": 552, + "end": 552 + } + } + ], + "content": "err11 = { $sel ->\n *[{3.14}] Value\n}\n\n", + "span": { + "type": "Span", + "start": 528, + "end": 569 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0004", + "args": [ + "a-zA-Z" + ], + "message": "Expected a character from range: \"a-zA-Z\"", + "span": { + "type": "Span", + "start": 593, + "end": 593 + } + } + ], + "content": "err12 = { $sel ->\n *[{$key}] Value\n}\n", + "span": { + "type": "Span", + "start": 569, + "end": 609 + } + } + ], + "span": { + "type": "Span", + "start": 0, + "end": 609 + } +} diff --git a/fluent/src/resource.js b/fluent/src/resource.js index 4d424aa2c..1b5aeb215 100644 --- a/fluent/src/resource.js +++ b/fluent/src/resource.js @@ -7,11 +7,10 @@ const RE_MESSAGE_START = /^(-?[a-zA-Z][\w-]*) *= */mg; // Both Attributes and Variants are parsed in while loops. These regexes are // used to break out of them. const RE_ATTRIBUTE_START = /\.([a-zA-Z][\w-]*) *= */y; -// [^] matches all characters, including newlines. -// XXX Use /s (dotall) when it's widely supported. -const RE_VARIANT_START = /\*?\[[^]*?] */y; +const RE_VARIANT_START = /\*?\[/y; const RE_NUMBER_LITERAL = /(-?[0-9]+(\.[0-9]+)?)/y; +const RE_IDENTIFIER = /([a-zA-Z][\w-]*)/y; const RE_REFERENCE = /([$-])?([a-zA-Z][\w-]*)(?:\.([a-zA-Z][\w-]*))?/y; // A "run" is a sequence of text or string literal characters which don't @@ -39,7 +38,7 @@ const RE_INDENT = /( *)$/; const TOKEN_BRACE_OPEN = /{\s*/y; const TOKEN_BRACE_CLOSE = /\s*}/y; const TOKEN_BRACKET_OPEN = /\[\s*/y; -const TOKEN_BRACKET_CLOSE = /\s*]/y; +const TOKEN_BRACKET_CLOSE = /\s*] */y; const TOKEN_PAREN_OPEN = /\s*\(\s*/y; const TOKEN_ARROW = /\s*->\s*/y; const TOKEN_COLON = /\s*:\s*/y; @@ -361,7 +360,6 @@ export default class FluentResource extends Map { } let key = parseVariantKey(); - cursor = RE_VARIANT_START.lastIndex; let value = parsePattern(); if (value === null) { throw new FluentError("Expected variant value"); @@ -376,7 +374,7 @@ export default class FluentResource extends Map { consumeToken(TOKEN_BRACKET_OPEN, FluentError); let key = test(RE_NUMBER_LITERAL) ? parseNumberLiteral() - : match(RE_REFERENCE)[2]; + : match1(RE_IDENTIFIER); consumeToken(TOKEN_BRACKET_CLOSE, FluentError); return key; } diff --git a/fluent/test/fixtures_structure/variant_keys.json b/fluent/test/fixtures_structure/variant_keys.json new file mode 100644 index 000000000..fbd2deb54 --- /dev/null +++ b/fluent/test/fixtures_structure/variant_keys.json @@ -0,0 +1,40 @@ +{ + "key01": [ + { + "type": "select", + "selector": { + "type": "var", + "name": "sel", + "attr": null, + "args": null + }, + "variants": [ + { + "key": "key", + "value": "Value" + } + ], + "star": 0 + } + ], + "key02": [ + { + "type": "select", + "selector": { + "type": "var", + "name": "sel", + "attr": null, + "args": null + }, + "variants": [ + { + "key": "key", + "value": [ + "Value" + ] + } + ], + "star": 0 + } + ] +} From cb22f54cc5662698bb52b8e4a979667bb87c4304 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Fri, 30 Nov 2018 13:18:12 +0100 Subject: [PATCH 18/23] (fluent) Enforce the default variant during parsing --- .../fixtures_structure/select_expressions.ftl | 9 +++ .../select_expressions.json | 72 +++++++++++++++++++ fluent/src/resource.js | 10 ++- .../select_expressions.json | 1 + 4 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 fluent-syntax/test/fixtures_structure/select_expressions.ftl create mode 100644 fluent-syntax/test/fixtures_structure/select_expressions.json create mode 100644 fluent/test/fixtures_structure/select_expressions.json diff --git a/fluent-syntax/test/fixtures_structure/select_expressions.ftl b/fluent-syntax/test/fixtures_structure/select_expressions.ftl new file mode 100644 index 000000000..5a96bf906 --- /dev/null +++ b/fluent-syntax/test/fixtures_structure/select_expressions.ftl @@ -0,0 +1,9 @@ +# ERROR No blanks are allowed between * and [. +err01 = { $sel -> + * [key] Value +} + +# ERROR Missing default variant. +err02 = { $sel -> + [key] Value +} diff --git a/fluent-syntax/test/fixtures_structure/select_expressions.json b/fluent-syntax/test/fixtures_structure/select_expressions.json new file mode 100644 index 000000000..8cee7ae3b --- /dev/null +++ b/fluent-syntax/test/fixtures_structure/select_expressions.json @@ -0,0 +1,72 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Comment", + "content": "ERROR No blanks are allowed between * and [.", + "span": { + "type": "Span", + "start": 0, + "end": 46 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0011", + "args": [], + "message": "Expected at least one variant after \"->\"", + "span": { + "type": "Span", + "start": 69, + "end": 69 + } + } + ], + "content": "err01 = { $sel ->\n * [key] Value\n}\n\n", + "span": { + "type": "Span", + "start": 47, + "end": 87 + } + }, + { + "type": "Comment", + "content": "ERROR Missing default variant.", + "span": { + "type": "Span", + "start": 87, + "end": 119 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0010", + "args": [], + "message": "Expected one of the variants to be marked as default (*)", + "span": { + "type": "Span", + "start": 154, + "end": 154 + } + } + ], + "content": "err02 = { $sel ->\n [key] Value\n}\n", + "span": { + "type": "Span", + "start": 120, + "end": 156 + } + } + ], + "span": { + "type": "Span", + "start": 0, + "end": 156 + } +} diff --git a/fluent/src/resource.js b/fluent/src/resource.js index 1b5aeb215..cc9fe1d7b 100644 --- a/fluent/src/resource.js +++ b/fluent/src/resource.js @@ -367,7 +367,15 @@ export default class FluentResource extends Map { variants[count++] = {key, value}; } - return count > 0 ? {variants, star} : null; + if (count === 0) { + return null; + } + + if (star === undefined) { + throw new FluentError("Expected default variant"); + } + + return {variants, star}; } function parseVariantKey() { diff --git a/fluent/test/fixtures_structure/select_expressions.json b/fluent/test/fixtures_structure/select_expressions.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/fluent/test/fixtures_structure/select_expressions.json @@ -0,0 +1 @@ +{} From d2ce2f7dbd012de0deba5fcc896f4c67ad90cfea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Wed, 28 Nov 2018 10:49:07 +0100 Subject: [PATCH 19/23] Recognize \UHHHHHH as an escape sequence --- fluent-syntax/src/errors.js | 4 +- fluent-syntax/src/parser.js | 53 ++++---- .../fixtures_behavior/escape_sequences.ftl | 2 +- .../fixtures_reference/escaped_characters.ftl | 16 ++- .../escaped_characters.json | 116 +++++++++++++++++- fluent/src/resource.js | 14 +-- .../escaped_characters.json | 16 ++- 7 files changed, 182 insertions(+), 39 deletions(-) diff --git a/fluent-syntax/src/errors.js b/fluent-syntax/src/errors.js index 9b8154fc5..028ee6a4a 100644 --- a/fluent-syntax/src/errors.js +++ b/fluent-syntax/src/errors.js @@ -69,8 +69,8 @@ function getErrorMessage(code, args) { return `Unknown escape sequence: \\${char}.`; } case "E0026": { - const [char] = args; - return `Invalid Unicode escape sequence: \\u${char}.`; + const [sequence] = args; + return `Invalid Unicode escape sequence: ${sequence}.`; } case "E0027": return "Unbalanced closing brace in TextElement."; diff --git a/fluent-syntax/src/parser.js b/fluent-syntax/src/parser.js index 5ad68c96f..37c42affc 100644 --- a/fluent-syntax/src/parser.js +++ b/fluent-syntax/src/parser.js @@ -599,37 +599,44 @@ export default class FluentParser { getEscapeSequence(ps) { const next = ps.currentChar; - if (next === "\\" || next === "\"") { - ps.next(); - return [`\\${next}`, next]; + switch (next) { + case "\\": + case "\"": + ps.next(); + return [`\\${next}`, next]; + case "u": + return this.getUnicodeEscapeSequence(ps, next, 4); + case "U": + return this.getUnicodeEscapeSequence(ps, next, 6); + default: + throw new ParseError("E0025", next); } + } - if (next === "u") { - let sequence = ""; - ps.next(); - - for (let i = 0; i < 4; i++) { - const ch = ps.takeHexDigit(); + getUnicodeEscapeSequence(ps, u, digits) { + ps.expectChar(u); - if (!ch) { - throw new ParseError("E0026", sequence + ps.currentChar); - } + let sequence = ""; + for (let i = 0; i < digits; i++) { + const ch = ps.takeHexDigit(); - sequence += ch; + if (!ch) { + throw new ParseError( + "E0026", `\\${u}${sequence}${ps.currentChar}`); } - const codepoint = parseInt(sequence, 16); - const unescaped = codepoint <= 0xD7FF || 0xE000 <= codepoint - // It's a Unicode scalar value. - ? String.fromCodePoint(codepoint) - // Escape sequences reresenting surrogate code points are well-formed - // but invalid in Fluent. Replace them with U+FFFD REPLACEMENT - // CHARACTER. - : "๏ฟฝ"; - return [`\\u${sequence}`, unescaped]; + sequence += ch; } - throw new ParseError("E0025", next); + const codepoint = parseInt(sequence, 16); + const unescaped = codepoint <= 0xD7FF || 0xE000 <= codepoint + // It's a Unicode scalar value. + ? String.fromCodePoint(codepoint) + // Escape sequences reresenting surrogate code points are well-formed + // but invalid in Fluent. Replace them with U+FFFD REPLACEMENT + // CHARACTER. + : "๏ฟฝ"; + return [`\\${u}${sequence}`, unescaped]; } getPlaceable(ps) { diff --git a/fluent-syntax/test/fixtures_behavior/escape_sequences.ftl b/fluent-syntax/test/fixtures_behavior/escape_sequences.ftl index 801923be2..19ad1329a 100644 --- a/fluent-syntax/test/fixtures_behavior/escape_sequences.ftl +++ b/fluent-syntax/test/fixtures_behavior/escape_sequences.ftl @@ -12,5 +12,5 @@ key08 = {"Escaped \u0041 A"} # ~ERROR E0025, pos 232, args "A" key09 = {"\A"} -# ~ERROR E0026, pos 252, args "000z" +# ~ERROR E0026, pos 252, args "\u000z" key10 = {"\u000z"} diff --git a/fluent-syntax/test/fixtures_reference/escaped_characters.ftl b/fluent-syntax/test/fixtures_reference/escaped_characters.ftl index 5242a4bcb..ec8623202 100644 --- a/fluent-syntax/test/fixtures_reference/escaped_characters.ftl +++ b/fluent-syntax/test/fixtures_reference/escaped_characters.ftl @@ -14,8 +14,20 @@ mismatched-quote = {"\\""} unknown-escape = {"\x"} ## Unicode escapes -string-unicode-sequence = {"\u0041"} -string-escaped-unicode = {"\\u0041"} +string-unicode-4digits = {"\u0041"} +escape-unicode-4digits = {"\\u0041"} +string-unicode-6digits = {"\U01F602"} +escape-unicode-6digits = {"\\U01F602"} + +# OK The trailing "00" is part of the literal value. +string-too-many-4digits = {"\u004100"} +# OK The trailing "00" is part of the literal value. +string-too-many-6digits = {"\U01F60200"} + +# ERROR Too few hex digits after \u. +string-too-few-4digits = {"\u41"} +# ERROR Too few hex digits after \U. +string-too-few-6digits = {"\U1F602"} ## Literal braces brace-open = An opening {"{"} brace. diff --git a/fluent-syntax/test/fixtures_reference/escaped_characters.json b/fluent-syntax/test/fixtures_reference/escaped_characters.json index e05c0efe6..a3220996a 100644 --- a/fluent-syntax/test/fixtures_reference/escaped_characters.json +++ b/fluent-syntax/test/fixtures_reference/escaped_characters.json @@ -179,7 +179,7 @@ "type": "Message", "id": { "type": "Identifier", - "name": "string-unicode-sequence" + "name": "string-unicode-4digits" }, "value": { "type": "Pattern", @@ -201,7 +201,7 @@ "type": "Message", "id": { "type": "Identifier", - "name": "string-escaped-unicode" + "name": "escape-unicode-4digits" }, "value": { "type": "Pattern", @@ -219,6 +219,118 @@ "attributes": [], "comment": null }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "string-unicode-6digits" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "\\U01F602", + "value": "๐Ÿ˜‚" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "escape-unicode-6digits" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "\\\\U01F602", + "value": "\\U01F602" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "string-too-many-4digits" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "\\u004100", + "value": "A00" + } + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "OK The trailing \"00\" is part of the literal value." + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "string-too-many-6digits" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "\\U01F60200", + "value": "๐Ÿ˜‚00" + } + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "OK The trailing \"00\" is part of the literal value." + } + }, + { + "type": "Comment", + "content": "ERROR Too few hex digits after \\u." + }, + { + "type": "Junk", + "annotations": [], + "content": "string-too-few-4digits = {\"\\u41\"}\n" + }, + { + "type": "Comment", + "content": "ERROR Too few hex digits after \\U." + }, + { + "type": "Junk", + "annotations": [], + "content": "string-too-few-6digits = {\"\\U1F602\"}\n\n" + }, { "type": "GroupComment", "content": "Literal braces" diff --git a/fluent/src/resource.js b/fluent/src/resource.js index cc9fe1d7b..ab55cb3a1 100644 --- a/fluent/src/resource.js +++ b/fluent/src/resource.js @@ -23,8 +23,8 @@ const RE_TEXT_RUN = /([^{}\n\r]+)/y; const RE_STRING_RUN = /([^\\"\n\r]*)/y; // Escape sequences. -const RE_UNICODE_ESCAPE = /\\u([a-fA-F0-9]{4})/y; const RE_STRING_ESCAPE = /\\([\\"])/y; +const RE_UNICODE_ESCAPE = /\\u([a-fA-F0-9]{4})|\\U([a-fA-F0-9]{6})/y; // Used for trimming TextElements and indents. const RE_LEADING_NEWLINES = /^\n+/; @@ -425,9 +425,13 @@ export default class FluentResource extends Map { // Unescape known escape sequences. function parseEscapeSequence() { + if (test(RE_STRING_ESCAPE)) { + return match1(RE_STRING_ESCAPE); + } + if (test(RE_UNICODE_ESCAPE)) { - let sequence = match1(RE_UNICODE_ESCAPE); - let codepoint = parseInt(sequence, 16); + let [, codepoint4, codepoint6] = match(RE_UNICODE_ESCAPE); + let codepoint = parseInt(codepoint4 || codepoint6, 16); return codepoint <= 0xD7FF || 0xE000 <= codepoint // It's a Unicode scalar value. ? String.fromCodePoint(codepoint) @@ -436,10 +440,6 @@ export default class FluentResource extends Map { : "๏ฟฝ"; } - if (test(RE_STRING_ESCAPE)) { - return match1(RE_STRING_ESCAPE); - } - throw new FluentError("Unknown escape sequence"); } diff --git a/fluent/test/fixtures_reference/escaped_characters.json b/fluent/test/fixtures_reference/escaped_characters.json index b5046fdb7..374a093c7 100644 --- a/fluent/test/fixtures_reference/escaped_characters.json +++ b/fluent/test/fixtures_reference/escaped_characters.json @@ -18,12 +18,24 @@ "backslash-in-string": [ "\\" ], - "string-unicode-sequence": [ + "string-unicode-4digits": [ "A" ], - "string-escaped-unicode": [ + "escape-unicode-4digits": [ "\\u0041" ], + "string-unicode-6digits": [ + "๐Ÿ˜‚" + ], + "escape-unicode-6digits": [ + "\\U01F602" + ], + "string-too-many-4digits": [ + "A00" + ], + "string-too-many-6digits": [ + "๐Ÿ˜‚00" + ], "brace-open": [ "An opening ", "{", From e8c4d3e59d9a31773bb0d631c42c1afe4e02fc6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Wed, 28 Nov 2018 10:53:05 +0100 Subject: [PATCH 20/23] Comment lines may only end with Fluent EOLs. For compatibility with the reference parser which used to use the /./ regex, the tooling parser used to recognize additional EOL characters in Comments. This has since been fixed in the reference parser: https://github.com/projectfluent/fluent/commit/2d224cbec2b1609e132c0fa3df7677e0d8357916 --- fluent-syntax/src/parser.js | 8 ++------ fluent-syntax/test/fixtures_reference/cr.json | 5 ++--- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/fluent-syntax/src/parser.js b/fluent-syntax/src/parser.js index 37c42affc..7ecf95e33 100644 --- a/fluent-syntax/src/parser.js +++ b/fluent-syntax/src/parser.js @@ -6,10 +6,6 @@ import { ParseError } from "./errors"; const trailingWSRe = /[ \t\n\r]+$/; -// The Fluent Syntax spec uses /.*/ to parse comment lines. It matches all -// characters except the following ones, which are considered line endings by -// the regex engine. -const COMMENT_EOL = ["\n", "\r", "\u2028", "\u2029"]; function withSpan(fn) { @@ -194,10 +190,10 @@ export default class FluentParser { level = i; } - if (!COMMENT_EOL.includes(ps.currentChar)) { + if (ps.currentChar !== EOL) { ps.expectChar(" "); let ch; - while ((ch = ps.takeChar(x => !COMMENT_EOL.includes(x)))) { + while ((ch = ps.takeChar(x => x !== EOL))) { content += ch; } } diff --git a/fluent-syntax/test/fixtures_reference/cr.json b/fluent-syntax/test/fixtures_reference/cr.json index afec8119a..44eab75f3 100644 --- a/fluent-syntax/test/fixtures_reference/cr.json +++ b/fluent-syntax/test/fixtures_reference/cr.json @@ -2,9 +2,8 @@ "type": "Resource", "body": [ { - "type": "Junk", - "annotations": [], - "content": "### This entire file uses CR as EOL.\r\rerr01 = Value 01\rerr02 = Value 02\r\rerr03 =\r\r Value 03\r Continued\r\r .title = Title\r\rerr04 = { \"str\r\rerr05 = { $sel -> }\r" + "type": "ResourceComment", + "content": "This entire file uses CR as EOL.\r\rerr01 = Value 01\rerr02 = Value 02\r\rerr03 =\r\r Value 03\r Continued\r\r .title = Title\r\rerr04 = { \"str\r\rerr05 = { $sel -> }\r" } ] } From c8ecf45c8958fb8825bd436305f45de36fc45954 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Wed, 28 Nov 2018 11:37:59 +0100 Subject: [PATCH 21/23] Forbid nested VariantLists --- fluent-syntax/src/parser.js | 51 +++++++++---------- .../test/fixtures_behavior/variant_lists.ftl | 1 + .../fixtures_reference/select_expressions.ftl | 2 +- .../select_expressions.json | 2 +- .../test/fixtures_reference/variant_lists.ftl | 3 +- .../fixtures_reference/variant_lists.json | 51 +++---------------- 6 files changed, 36 insertions(+), 74 deletions(-) diff --git a/fluent-syntax/src/parser.js b/fluent-syntax/src/parser.js index 7ecf95e33..2fcbd5b29 100644 --- a/fluent-syntax/src/parser.js +++ b/fluent-syntax/src/parser.js @@ -227,7 +227,7 @@ export default class FluentParser { ps.skipBlankInline(); ps.expectChar("="); - const value = this.maybeGetValue(ps, {allowVariantList: false}); + const value = this.maybeGetPattern(ps); const attrs = this.getAttributes(ps); if (value === null && attrs.length === 0) { @@ -244,11 +244,9 @@ export default class FluentParser { ps.skipBlankInline(); ps.expectChar("="); - // XXX Once https://github.com/projectfluent/fluent/pull/220 lands, - // getTerm will be the only place where VariantLists are still legal. Move - // the code from getPatternOrVariantList up to here then, and remove the - // allowVariantList switch. - const value = this.maybeGetValue(ps, {allowVariantList: true}); + // Syntax 0.8 compat: VariantLists are supported but deprecated. They can + // only be found as values of Terms. Nested VariantLists are not allowed. + const value = this.maybeGetVariantList(ps) || this.maybeGetPattern(ps); if (value === null) { throw new ParseError("E0006", id.name); } @@ -265,7 +263,7 @@ export default class FluentParser { ps.skipBlankInline(); ps.expectChar("="); - const value = this.maybeGetValue(ps, {allowVariantList: false}); + const value = this.maybeGetPattern(ps); if (value === null) { throw new ParseError("E0012"); } @@ -312,7 +310,7 @@ export default class FluentParser { return this.getIdentifier(ps); } - getVariant(ps, {hasDefault, allowVariantList}) { + getVariant(ps, {hasDefault}) { let defaultIndex = false; if (ps.currentChar === "*") { @@ -333,9 +331,7 @@ export default class FluentParser { ps.skipBlank(); ps.expectChar("]"); - // XXX We need to pass allowVariantList all the way down to here because - // nested VariantLists in Terms are legal for now. - const value = this.maybeGetValue(ps, {allowVariantList}); + const value = this.maybeGetPattern(ps); if (value === null) { throw new ParseError("E0012"); } @@ -343,13 +339,13 @@ export default class FluentParser { return new AST.Variant(key, value, defaultIndex); } - getVariants(ps, {allowVariantList}) { + getVariants(ps) { const variants = []; let hasDefault = false; ps.skipBlank(); while (ps.isVariantStart()) { - const variant = this.getVariant(ps, {allowVariantList, hasDefault}); + const variant = this.getVariant(ps, {hasDefault}); if (variant.default) { hasDefault = true; @@ -405,34 +401,34 @@ export default class FluentParser { return new AST.NumberLiteral(num); } - // maybeGetValue distinguishes between patterns which start on the same line + // maybeGetPattern distinguishes between patterns which start on the same line // as the identifier (a.k.a. inline signleline patterns and inline multiline // patterns) and patterns which start on a new line (a.k.a. block multiline // patterns). The distinction is important for the dedentation logic: the // indent of the first line of a block pattern must be taken into account when // calculating the maximum common indent. - maybeGetValue(ps, {allowVariantList}) { + maybeGetPattern(ps) { ps.peekBlankInline(); if (ps.isValueStart()) { ps.skipToPeek(); - return this.getPatternOrVariantList( - ps, {isBlock: false, allowVariantList}); + return this.getPattern(ps, {isBlock: false}); } ps.peekBlankBlock(); if (ps.isValueContinuation()) { ps.skipToPeek(); - return this.getPatternOrVariantList( - ps, {isBlock: true, allowVariantList}); + return this.getPattern(ps, {isBlock: true}); } return null; } - // Parse a VariantList (if allowed) or a Pattern. - getPatternOrVariantList(ps, {isBlock, allowVariantList}) { - ps.peekBlankInline(); - if (allowVariantList && ps.currentPeek === "{") { + // Deprecated in Syntax 0.8. VariantLists are only allowed as values of Terms. + // Values of Messages, Attributes and Variants must be Patterns. This method + // is only used in getTerm. + maybeGetVariantList(ps) { + ps.peekBlank(); + if (ps.currentPeek === "{") { const start = ps.peekOffset; ps.peek(); ps.peekBlankInline(); @@ -441,19 +437,18 @@ export default class FluentParser { if (ps.isVariantStart()) { ps.resetPeek(start); ps.skipToPeek(); - return this.getVariantList(ps, {allowVariantList}); + return this.getVariantList(ps); } } } ps.resetPeek(); - const pattern = this.getPattern(ps, {isBlock}); - return pattern; + return null; } getVariantList(ps) { ps.expectChar("{"); - var variants = this.getVariants(ps, {allowVariantList: true}); + var variants = this.getVariants(ps); ps.expectChar("}"); return new AST.VariantList(variants); } @@ -679,7 +674,7 @@ export default class FluentParser { ps.skipBlankInline(); ps.expectLineEnd(); - const variants = this.getVariants(ps, {allowVariantList: false}); + const variants = this.getVariants(ps); return new AST.SelectExpression(selector, variants); } diff --git a/fluent-syntax/test/fixtures_behavior/variant_lists.ftl b/fluent-syntax/test/fixtures_behavior/variant_lists.ftl index 2f3cc5b8d..21ffb8f1d 100644 --- a/fluent-syntax/test/fixtures_behavior/variant_lists.ftl +++ b/fluent-syntax/test/fixtures_behavior/variant_lists.ftl @@ -17,6 +17,7 @@ message2 = *[one] One } +# ~ERROR E0014, pos 211 -term2 = { *[one] { diff --git a/fluent-syntax/test/fixtures_reference/select_expressions.ftl b/fluent-syntax/test/fixtures_reference/select_expressions.ftl index ac888262b..7a1fb820a 100644 --- a/fluent-syntax/test/fixtures_reference/select_expressions.ftl +++ b/fluent-syntax/test/fixtures_reference/select_expressions.ftl @@ -39,7 +39,7 @@ nested-select = } } -# ERROR VariantLists cannot appear in SelectExpressions +# ERROR VariantLists cannot be Variant values. nested-variant-list = { 1 -> *[one] { diff --git a/fluent-syntax/test/fixtures_reference/select_expressions.json b/fluent-syntax/test/fixtures_reference/select_expressions.json index 4e84c0c2a..a3dc5730a 100644 --- a/fluent-syntax/test/fixtures_reference/select_expressions.json +++ b/fluent-syntax/test/fixtures_reference/select_expressions.json @@ -274,7 +274,7 @@ }, { "type": "Comment", - "content": "ERROR VariantLists cannot appear in SelectExpressions" + "content": "ERROR VariantLists cannot be Variant values." }, { "type": "Junk", diff --git a/fluent-syntax/test/fixtures_reference/variant_lists.ftl b/fluent-syntax/test/fixtures_reference/variant_lists.ftl index d9031d91a..e5c61dd80 100644 --- a/fluent-syntax/test/fixtures_reference/variant_lists.ftl +++ b/fluent-syntax/test/fixtures_reference/variant_lists.ftl @@ -23,6 +23,7 @@ variant-list-in-message-attr = Value *[key] Value } +# ERROR VariantLists cannot be Variant values. -nested-variant-list-in-term = { *[one] { @@ -37,7 +38,7 @@ variant-list-in-message-attr = Value } } -# ERROR VariantLists may not appear in SelectExpressions +# ERROR VariantLists cannot be Variant values. nested-select-then-variant-list = { *[one] { 2 -> diff --git a/fluent-syntax/test/fixtures_reference/variant_lists.json b/fluent-syntax/test/fixtures_reference/variant_lists.json index aaac9f7b0..83f166621 100644 --- a/fluent-syntax/test/fixtures_reference/variant_lists.json +++ b/fluent-syntax/test/fixtures_reference/variant_lists.json @@ -94,48 +94,13 @@ "content": " .attr =\n {\n *[key] Value\n }\n\n" }, { - "type": "Term", - "id": { - "type": "Identifier", - "name": "nested-variant-list-in-term" - }, - "value": { - "type": "VariantList", - "variants": [ - { - "type": "Variant", - "key": { - "type": "Identifier", - "name": "one" - }, - "value": { - "type": "VariantList", - "variants": [ - { - "type": "Variant", - "key": { - "type": "Identifier", - "name": "two" - }, - "value": { - "type": "Pattern", - "elements": [ - { - "type": "TextElement", - "value": "Value" - } - ] - }, - "default": true - } - ] - }, - "default": true - } - ] - }, - "attributes": [], - "comment": null + "type": "Comment", + "content": "ERROR VariantLists cannot be Variant values." + }, + { + "type": "Junk", + "annotations": [], + "content": "-nested-variant-list-in-term =\n {\n *[one] {\n *[two] Value\n }\n }\n\n" }, { "type": "Term", @@ -195,7 +160,7 @@ }, { "type": "Comment", - "content": "ERROR VariantLists may not appear in SelectExpressions" + "content": "ERROR VariantLists cannot be Variant values." }, { "type": "Junk", From 1cc091f9440b0ffb0fcf520d8b7a1e8a71b4254e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Tue, 4 Dec 2018 17:05:37 +0100 Subject: [PATCH 22/23] Refactor getInlineExpression (#318) Enumerate all possible simple expressions (as opposed to member expressions) and for each of them, run logic in separate branches of getInlineExpression. --- fluent-syntax/src/errors.js | 2 + fluent-syntax/src/parser.js | 175 ++++++++---------- .../call_expression_with_bad_id.ftl | 2 +- .../call_expression_with_wrong_value_type.ftl | 2 +- .../unclosed_empty_placeable_error.ftl | 2 +- .../variant_expression_as_placeable.ftl | 2 +- .../variant_expression_as_selector.ftl | 2 +- .../test/fixtures_behavior/variant_lists.ftl | 8 +- .../test/fixtures_structure/junk.json | 4 +- 9 files changed, 95 insertions(+), 104 deletions(-) diff --git a/fluent-syntax/src/errors.js b/fluent-syntax/src/errors.js index 028ee6a4a..ee9a1dd00 100644 --- a/fluent-syntax/src/errors.js +++ b/fluent-syntax/src/errors.js @@ -74,6 +74,8 @@ function getErrorMessage(code, args) { } case "E0027": return "Unbalanced closing brace in TextElement."; + case "E0028": + return "Expected an inline expression"; default: return code; } diff --git a/fluent-syntax/src/parser.js b/fluent-syntax/src/parser.js index 2fcbd5b29..0443969e2 100644 --- a/fluent-syntax/src/parser.js +++ b/fluent-syntax/src/parser.js @@ -17,7 +17,7 @@ function withSpan(fn) { const start = ps.index; const node = fn.call(this, ps, ...args); - // Don't re-add the span if the node already has it. This may happen when + // Don't re-add the span if the node already has it. This may happen when // one decorated function calls another decorated function. if (node.span) { return node; @@ -41,7 +41,8 @@ export default class FluentParser { "getComment", "getMessage", "getTerm", "getAttribute", "getIdentifier", "getVariant", "getNumber", "getPattern", "getVariantList", "getTextElement", "getPlaceable", "getExpression", - "getSelectorExpression", "getCallArg", "getString", "getLiteral" + "getInlineExpression", "getCallArgument", "getString", + "getSimpleExpression", "getLiteral" ]; for (const name of methodNames) { this[name] = withSpan(this[name]); @@ -639,11 +640,10 @@ export default class FluentParser { } getExpression(ps) { - const selector = this.getSelectorExpression(ps); + const selector = this.getInlineExpression(ps); ps.skipBlank(); if (ps.currentChar === "-") { - if (ps.peek() !== ">") { ps.resetPeek(); return selector; @@ -691,73 +691,95 @@ export default class FluentParser { return selector; } - getSelectorExpression(ps) { + getInlineExpression(ps) { if (ps.currentChar === "{") { return this.getPlaceable(ps); } - let selector = this.getLiteral(ps); - switch (selector.type) { - case "StringLiteral": + let expr = this.getSimpleExpression(ps); + switch (expr.type) { case "NumberLiteral": + case "StringLiteral": case "VariableReference": - return selector; - } + return expr; + case "MessageReference": { + if (ps.currentChar === ".") { + ps.next(); + const attr = this.getIdentifier(ps); + return new AST.AttributeExpression(expr, attr); + } - if (ps.currentChar === "[") { - ps.next(); + if (ps.currentChar === "(") { + // It's a Function. Ensure it's all upper-case. + if (!/^[A-Z][A-Z_?-]*$/.test(expr.id.name)) { + throw new ParseError("E0008"); + } - if (selector.type === "MessageReference") { - throw new ParseError("E0024"); + const func = new AST.FunctionReference(expr.id); + if (this.withSpans) { + func.addSpan(expr.span.start, expr.span.end); + } + return new AST.CallExpression(func, ...this.getCallArguments(ps)); + } + + return expr; } + case "TermReference": { + if (ps.currentChar === "[") { + ps.next(); + const key = this.getVariantKey(ps); + ps.expectChar("]"); + return new AST.VariantExpression(expr, key); + } - const key = this.getVariantKey(ps); - ps.expectChar("]"); - return new AST.VariantExpression(selector, key); - } + if (ps.currentChar === ".") { + ps.next(); + const attr = this.getIdentifier(ps); + expr = new AST.AttributeExpression(expr, attr); + } - if (ps.currentChar === ".") { - ps.next(); - const attr = this.getIdentifier(ps); - selector = new AST.AttributeExpression(selector, attr); + if (ps.currentChar === "(") { + return new AST.CallExpression(expr, ...this.getCallArguments(ps)); + } + + return expr; + } + default: + throw new ParseError("E0028"); } + } - if (ps.currentChar === "(") { - ps.next(); + getSimpleExpression(ps) { + if (ps.isNumberStart()) { + return this.getNumber(ps); + } - if (selector.type === "MessageReference") { - if (/^[A-Z][A-Z_?-]*$/.test(selector.id.name)) { - // The callee is a Function. - var func = new AST.FunctionReference(selector.id); - if (this.withSpans) { - func.addSpan(selector.span.start, selector.span.end); - } - } else { - // Messages can't be callees. - throw new ParseError("E0008"); - } - } + if (ps.currentChar === '"') { + return this.getString(ps); + } - if (selector.type === "AttributeExpression" - && selector.ref.type === "MessageReference") { - throw new ParseError("E0008"); - } + if (ps.currentChar === "$") { + ps.next(); + const id = this.getIdentifier(ps); + return new AST.VariableReference(id); + } - const args = this.getCallArgs(ps); - ps.expectChar(")"); + if (ps.currentChar === "-") { + ps.next(); + const id = this.getIdentifier(ps); + return new AST.TermReference(id); + } - return new AST.CallExpression( - func || selector, - args.positional, - args.named, - ); + if (ps.isIdentifierStart()) { + const id = this.getIdentifier(ps); + return new AST.MessageReference(id); } - return selector; + throw new ParseError("E0028"); } - getCallArg(ps) { - const exp = this.getSelectorExpression(ps); + getCallArgument(ps) { + const exp = this.getInlineExpression(ps); ps.skipBlank(); @@ -772,16 +794,16 @@ export default class FluentParser { ps.next(); ps.skipBlank(); - const val = this.getArgVal(ps); - - return new AST.NamedArgument(exp.id, val); + const value = this.getLiteral(ps); + return new AST.NamedArgument(exp.id, value); } - getCallArgs(ps) { + getCallArguments(ps) { const positional = []; const named = []; const argumentNames = new Set(); + ps.expectChar("("); ps.skipBlank(); while (true) { @@ -789,7 +811,7 @@ export default class FluentParser { break; } - const arg = this.getCallArg(ps); + const arg = this.getCallArgument(ps); if (arg.type === "NamedArgument") { if (argumentNames.has(arg.name.name)) { throw new ParseError("E0022"); @@ -808,23 +830,13 @@ export default class FluentParser { ps.next(); ps.skipBlank(); continue; - } else { - break; } - } - return { - positional, - named - }; - } - getArgVal(ps) { - if (ps.isNumberStart()) { - return this.getNumber(ps); - } else if (ps.currentChar === '"') { - return this.getString(ps); + break; } - throw new ParseError("E0012"); + + ps.expectChar(")"); + return [positional, named]; } getString(ps) { @@ -855,34 +867,11 @@ export default class FluentParser { } getLiteral(ps) { - const ch = ps.currentChar; - - if (ch === EOF) { - throw new ParseError("E0014"); - } - - if (ch === "$") { - ps.next(); - const id = this.getIdentifier(ps); - return new AST.VariableReference(id); - } - - if (ps.isIdentifierStart()) { - const id = this.getIdentifier(ps); - return new AST.MessageReference(id); - } - if (ps.isNumberStart()) { return this.getNumber(ps); } - if (ch === "-") { - ps.next(); - const id = this.getIdentifier(ps); - return new AST.TermReference(id); - } - - if (ch === '"') { + if (ps.currentChar === '"') { return this.getString(ps); } diff --git a/fluent-syntax/test/fixtures_behavior/call_expression_with_bad_id.ftl b/fluent-syntax/test/fixtures_behavior/call_expression_with_bad_id.ftl index e82146965..9bc2bbbc1 100644 --- a/fluent-syntax/test/fixtures_behavior/call_expression_with_bad_id.ftl +++ b/fluent-syntax/test/fixtures_behavior/call_expression_with_bad_id.ftl @@ -1,3 +1,3 @@ key = { no-caps-name() } -# ~ERROR E0008, pos 21 +# ~ERROR E0008, pos 20 diff --git a/fluent-syntax/test/fixtures_behavior/call_expression_with_wrong_value_type.ftl b/fluent-syntax/test/fixtures_behavior/call_expression_with_wrong_value_type.ftl index 4a8f9604c..b0be5cdeb 100644 --- a/fluent-syntax/test/fixtures_behavior/call_expression_with_wrong_value_type.ftl +++ b/fluent-syntax/test/fixtures_behavior/call_expression_with_wrong_value_type.ftl @@ -1,2 +1,2 @@ key = { BUILTIN(key: foo) } -# ~ERROR E0012, pos 21 +# ~ERROR E0014, pos 21 diff --git a/fluent-syntax/test/fixtures_behavior/unclosed_empty_placeable_error.ftl b/fluent-syntax/test/fixtures_behavior/unclosed_empty_placeable_error.ftl index 5f812a949..e29ced2b5 100644 --- a/fluent-syntax/test/fixtures_behavior/unclosed_empty_placeable_error.ftl +++ b/fluent-syntax/test/fixtures_behavior/unclosed_empty_placeable_error.ftl @@ -1,5 +1,5 @@ # ~ERROR E0003, pos 8, args "}" foo = { bar = Bar -# ~ERROR E0014, pos 26 +# ~ERROR E0028, pos 26 baz = { diff --git a/fluent-syntax/test/fixtures_behavior/variant_expression_as_placeable.ftl b/fluent-syntax/test/fixtures_behavior/variant_expression_as_placeable.ftl index 1866ef64b..6b2796ffa 100644 --- a/fluent-syntax/test/fixtures_behavior/variant_expression_as_placeable.ftl +++ b/fluent-syntax/test/fixtures_behavior/variant_expression_as_placeable.ftl @@ -1,3 +1,3 @@ -# ~ERROR E0024, pos 18 +# ~ERROR E0003, pos 17, args "}" key01 = { message[variant] } key02 = { -term[variant] } diff --git a/fluent-syntax/test/fixtures_behavior/variant_expression_as_selector.ftl b/fluent-syntax/test/fixtures_behavior/variant_expression_as_selector.ftl index 88d6b5a2b..946c1c831 100644 --- a/fluent-syntax/test/fixtures_behavior/variant_expression_as_selector.ftl +++ b/fluent-syntax/test/fixtures_behavior/variant_expression_as_selector.ftl @@ -3,7 +3,7 @@ err1 = *[1] One [2] Two } -# ~ERROR E0024, pos 17 +# ~ERROR E0003, pos 16, args "}" err2 = { -foo[bar] -> diff --git a/fluent-syntax/test/fixtures_behavior/variant_lists.ftl b/fluent-syntax/test/fixtures_behavior/variant_lists.ftl index 21ffb8f1d..681751de1 100644 --- a/fluent-syntax/test/fixtures_behavior/variant_lists.ftl +++ b/fluent-syntax/test/fixtures_behavior/variant_lists.ftl @@ -1,10 +1,10 @@ -# ~ERROR E0014, pos 25 +# ~ERROR E0028, pos 25 message1 = { *[one] One } -# ~ERROR E0014, pos 97 +# ~ERROR E0028, pos 97 message2 = { $sel -> *[one] { @@ -17,7 +17,7 @@ message2 = *[one] One } -# ~ERROR E0014, pos 211 +# ~ERROR E0028, pos 211 -term2 = { *[one] { @@ -25,7 +25,7 @@ message2 = } } -# ~ERROR E0014, pos 292 +# ~ERROR E0028, pos 292 -term3 = { $sel -> *[one] { diff --git a/fluent-syntax/test/fixtures_structure/junk.json b/fluent-syntax/test/fixtures_structure/junk.json index 47d5eb153..df1f862df 100644 --- a/fluent-syntax/test/fixtures_structure/junk.json +++ b/fluent-syntax/test/fixtures_structure/junk.json @@ -212,9 +212,9 @@ "annotations": [ { "type": "Annotation", - "code": "E0014", + "code": "E0028", "args": [], - "message": "Expected literal", + "message": "Expected an inline expression", "span": { "type": "Span", "start": 153, From a1e4230e118f30b099917db01cb98532ce25ebad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Thu, 13 Dec 2018 12:25:50 +0100 Subject: [PATCH 23/23] Remove unecessary hasDefault assignment (#325) --- fluent-syntax/src/parser.js | 1 - 1 file changed, 1 deletion(-) diff --git a/fluent-syntax/src/parser.js b/fluent-syntax/src/parser.js index 0443969e2..1436e7b08 100644 --- a/fluent-syntax/src/parser.js +++ b/fluent-syntax/src/parser.js @@ -320,7 +320,6 @@ export default class FluentParser { } ps.next(); defaultIndex = true; - hasDefault = true; } ps.expectChar("[");