├── .eslintrc ├── .gitattributes ├── .github └── workflows │ ├── deploy.yml │ └── test.yml ├── .gitignore ├── .nojekyll ├── .prettierrc ├── .vscode └── settings.json ├── CODE_OF_CONDUCT.md ├── LICENSE ├── NOTICE ├── README.md ├── bin ├── ebnf.js └── parse.js ├── book.toml ├── guide ├── README.md ├── SUMMARY.md ├── attributes.md ├── builtins.md ├── comments.md ├── css │ └── custom.css ├── functions.md ├── hello.md ├── more.md ├── multiline.md ├── placeables.md ├── references.md ├── selectors.md ├── special.md ├── terms.md ├── text.md └── variables.md ├── lib ├── README.md ├── combinators.js ├── ebnf.js ├── mappers.js ├── parser.js ├── result.js ├── serializer.js ├── stream.js ├── visitor.js └── walker.js ├── package-lock.json ├── package.json ├── spec ├── CHANGELOG.md ├── README.md ├── compatibility.md ├── errors.md ├── fluent.ebnf ├── recommendations.md └── valid.md ├── syntax ├── abstract.js ├── ast.js └── grammar.js └── test ├── bench.js ├── benchmarks └── gecko_strings.ftl ├── ebnf.js ├── fixtures ├── Makefile ├── any_char.ftl ├── any_char.json ├── astral.ftl ├── astral.json ├── call_expressions.ftl ├── call_expressions.json ├── callee_expressions.ftl ├── callee_expressions.json ├── comments.ftl ├── comments.json ├── cr_err_literal.ftl ├── cr_err_literal.json ├── cr_err_selector.ftl ├── cr_err_selector.json ├── cr_multikey.ftl ├── cr_multikey.json ├── cr_multilinevalue.ftl ├── cr_multilinevalue.json ├── crlf.ftl ├── crlf.json ├── eof_comment.ftl ├── eof_comment.json ├── eof_empty.ftl ├── eof_empty.json ├── eof_id.ftl ├── eof_id.json ├── eof_id_equals.ftl ├── eof_id_equals.json ├── eof_junk.ftl ├── eof_junk.json ├── eof_value.ftl ├── eof_value.json ├── escaped_characters.ftl ├── escaped_characters.json ├── junk.ftl ├── junk.json ├── leading_dots.ftl ├── leading_dots.json ├── literal_expressions.ftl ├── literal_expressions.json ├── member_expressions.ftl ├── member_expressions.json ├── messages.ftl ├── messages.json ├── mixed_entries.ftl ├── mixed_entries.json ├── multiline_values.ftl ├── multiline_values.json ├── numbers.ftl ├── numbers.json ├── obsolete.ftl ├── obsolete.json ├── placeables.ftl ├── placeables.json ├── reference_expressions.ftl ├── reference_expressions.json ├── select_expressions.ftl ├── select_expressions.json ├── select_indent.ftl ├── select_indent.json ├── sparse_entries.ftl ├── sparse_entries.json ├── special_chars.ftl ├── special_chars.json ├── tab.ftl ├── tab.json ├── term_parameters.ftl ├── term_parameters.json ├── terms.ftl ├── terms.json ├── variables.ftl ├── variables.json ├── variant_keys.ftl ├── variant_keys.json ├── whitespace_in_value.ftl ├── whitespace_in_value.json ├── zero_length.ftl └── zero_length.json ├── literals.js ├── parser.js ├── suite.js └── validator.js /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "parserOptions": { 3 | "ecmaVersion": 2022, 4 | "sourceType": "module" 5 | }, 6 | "env": { 7 | "es6": true, 8 | "node": true 9 | }, 10 | "rules": { 11 | "brace-style": ["error", "1tbs"], 12 | "eqeqeq": ["error", "always"], 13 | "indent": ["error", 4, { 14 | "CallExpression": {"arguments": 1}, 15 | "MemberExpression": "off", 16 | "SwitchCase": 1 17 | }], 18 | "quotes": ["error", "double"], 19 | "semi": ["error", "always"], 20 | "space-infix-ops": ["error"], 21 | "no-tabs": "error", 22 | "no-undef": "error" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | *.ftl eol=lf 2 | test/fixtures/crlf.ftl eol=crlf 3 | test/fixtures/cr_*.ftl eol=cr 4 | -------------------------------------------------------------------------------- /.github/workflows/deploy.yml: -------------------------------------------------------------------------------- 1 | name: Deploy to GitHub Pages 2 | 3 | on: 4 | # Triggers the workflow on push or pull request events but 5 | # only for the default branch. 6 | push: 7 | branches: 8 | - master 9 | paths: 10 | - '.github/workflows/deploy.yml' 11 | - 'book.toml' 12 | - 'guide/**' 13 | # Allows you to run this workflow manually from the Actions tab 14 | workflow_dispatch: 15 | 16 | jobs: 17 | fetch: 18 | runs-on: ubuntu-latest 19 | env: 20 | MDBOOK_VERSION: v0.4.36 21 | steps: 22 | - name: Clone repository 23 | uses: actions/checkout@v4 24 | - name: Install mdbook 25 | run: | 26 | mkdir installed-bins 27 | curl -Lf "https://github.com/rust-lang/mdBook/releases/download/$MDBOOK_VERSION/mdbook-$MDBOOK_VERSION-x86_64-unknown-linux-gnu.tar.gz" | tar -xz --directory=./installed-bins 28 | echo `pwd`/installed-bins >> $GITHUB_PATH 29 | - name: Install gh-pages 30 | run: | 31 | npm install gh-pages@"~6.1.1" 32 | - name: Build book 33 | run: | 34 | mdbook --version 35 | mdbook build 36 | - name: Set Git config 37 | run: | 38 | git config --global user.email "actions@users.noreply.github.com" 39 | git config --global user.name 'Automation' 40 | git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY 41 | - name: Deploy to GitHub pages 42 | run: | 43 | npx gh-pages --message "Deploy docs" --dist build 44 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Run tests 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | branches: 9 | - master 10 | workflow_dispatch: 11 | 12 | jobs: 13 | test: 14 | name: Run tests 15 | runs-on: ubuntu-latest 16 | steps: 17 | - uses: actions/checkout@v4 18 | - uses: actions/setup-node@v4 19 | with: 20 | node-version: 20 21 | - run: npm ci 22 | - run: npm run lint 23 | - run: npm test 24 | - run: npm run test:ebnf 25 | - run: npm run test:validate 26 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | build 3 | -------------------------------------------------------------------------------- /.nojekyll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/projectfluent/fluent/10a1bc60bee843c14a30216fa4cebdc559bf2076/.nojekyll -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "bracketSpacing": false, 3 | "trailingComma": "es5", 4 | "printWidth": 100, 5 | "tabWidth": 4 6 | } 7 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "editor.formatOnSave": false, 3 | "[javascript]": { 4 | "editor.formatOnSave": false 5 | }, 6 | "[json]": { 7 | "editor.formatOnSave": true 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Community Participation Guidelines 2 | 3 | This repository is governed by Mozilla's code of conduct and etiquette 4 | guidelines. For more details, please read the [Mozilla Community Participation 5 | Guidelines][]. 6 | 7 | 8 | ## How to Report 9 | 10 | For more information on how to report violations of the Community Participation 11 | Guidelines, please read our [How to Report][] page. 12 | 13 | 14 | [Mozilla Community Participation Guidelines]: https://www.mozilla.org/about/governance/policies/participation/ 15 | [How to Report]: https://www.mozilla.org/about/governance/policies/participation/reporting/ 16 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright 2019 Mozilla Foundation and others 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Fluent 2 | 3 | Fluent is a localization system designed to unleash the expressive power of 4 | the natural language. 5 | 6 | This repository contains the specification, the reference implementation of the 7 | parser and the documentation for Fluent. 8 | 9 | ## Fluent Syntax (FTL) 10 | 11 | FTL is the syntax for describing translation resources in Project Fluent. FTL 12 | stands for *Fluent Translation List*. Read the [Fluent Syntax Guide][] to get 13 | started learning Fluent. 14 | 15 | The `syntax/` directory contains the reference implementation of the syntax as 16 | a _LL(infinity)_ parser. 17 | 18 | The `spec/` directory contains the formal EBNF grammar, autogenerated from the 19 | reference implementation. 20 | 21 | ## Development 22 | 23 | While working on the reference parser, use the following commands to test and 24 | validate your work: 25 | 26 | npm test # Test the parser against JSON AST fixtures. 27 | npm run lint # Lint the parser code. 28 | 29 | npm run generate:ebnf # Generate the EBNF from syntax/grammar.js. 30 | npm run generate:fixtures # Generate test fixtures (FTL → JSON AST). 31 | 32 | npm run build:guide # Build the HTML version of the Guide. 33 | 34 | npm run bench # Run the performance benchmark on large FTL. 35 | 36 | ## Other Implementations 37 | 38 | This repository contains the reference implementation of the parser. Other implementations exist which should be preferred for use in production and in tooling. 39 | 40 | - The JavaScript implementation at [`fluent.js`](https://github.com/projectfluent/fluent.js), including the [React bindings](https://github.com/projectfluent/fluent.js/tree/master/fluent-react). 41 | - The Python implementation at [`python-fluent`](https://github.com/projectfluent/python-fluent). 42 | - The Rust implementation at [`fluent-rs`](https://github.com/projectfluent/fluent-rs). 43 | 44 | We also know about the following community-driven implementations: 45 | 46 | - [`Fluent.Net`](https://github.com/blushingpenguin/Fluent.Net) by [@blushingpenguin](https://github.com/blushingpenguin). See [#93](https://github.com/projectfluent/fluent/issues/93) for more info. 47 | - [`Linguini`](https://github.com/Ygg01/Linguini) a .NET library by [@Ygg01](https://github.com/Ygg01) 48 | - A Java/Kotlin implementation has been requested in [#158](https://github.com/projectfluent/fluent/issues/158). 49 | - [`elm-fluent`](https://github.com/elm-fluent/elm-fluent) by [@spookylukey](https://github.com/spookylukey/) 50 | - [`Fluent`](https://github.com/alabamenhu/Fluent) as a Perl 6 module by [@alabamenhu](https://github.com/alabamenhu/) 51 | - [`fluent-dart`](https://github.com/ryanhz/fluent-dart) as a Dart runtime implementation by [@ryanhz](https://github.com/ryanhz). 52 | - [`fluent-compiler`](https://github.com/django-ftl/fluent-compiler) - an alternative Python implementation by [@spookylukey](https://github.com/spookylukey/). 53 | - [`fluent-vue`](https://github.com/demivan/fluent-vue) - Vue.js plugin - integration for `fluent.js` 54 | - A Lua implementation effort is underway at [`fluent-lua`](https://github.com/alerque/fluent-lua) by [@alerque](https://github.com/alerque). 55 | - A D implementation effort is underway at [`fluentd`](https://github.com/SirNickolas/fluentd) by [@SirNickolas](https://github.com/SirNickolas). 56 | - [`fluent.go`](https://github.com/lus/fluent.go) - a Golang implementation by [@lus](https://github.com/lus) 57 | - [`fluent-php`](https://github.com/Ennexa/fluent-php) - a PHP module providing bindings wrapping the Rust library by [@Ennexa](https://github.com/Ennexa) 58 | 59 | ## Learn More and Discuss 60 | 61 | Find out more about Project Fluent at [projectfluent.org][] and discuss the future of Fluent at [Mozilla Discourse][]. 62 | 63 | [Fluent Syntax Guide]: http://projectfluent.org/fluent/guide 64 | [projectfluent.org]: http://projectfluent.org 65 | [Mozilla Discourse]: https://discourse.mozilla.org/c/fluent 66 | -------------------------------------------------------------------------------- /bin/ebnf.js: -------------------------------------------------------------------------------- 1 | import fs from "fs"; 2 | import readline from "readline"; 3 | import parse_args from "minimist"; 4 | import ebnf from "../lib/ebnf.js"; 5 | 6 | const argv = parse_args(process.argv.slice(2), { 7 | boolean: ["help"], 8 | alias: { 9 | help: "h", 10 | }, 11 | }); 12 | 13 | if (argv.help) { 14 | exit_help(0); 15 | } 16 | 17 | const [file_path] = argv._; 18 | 19 | if (file_path === "-") { 20 | from_stdin(); 21 | } else if (file_path) { 22 | from_file(file_path); 23 | } else { 24 | exit_help(1); 25 | } 26 | 27 | function exit_help(exit_code) { 28 | console.log(` 29 | Usage: node ebnf.js [OPTIONS] 30 | 31 | When FILE is "-", read text from stdin. 32 | 33 | Examples: 34 | 35 | node ebnf.js path/to/grammar.js 36 | cat path/to/grammar.js | node ebnf.js - 37 | 38 | Options: 39 | 40 | -h, --help Display help and quit. 41 | `); 42 | process.exit(exit_code); 43 | } 44 | 45 | function from_stdin() { 46 | const rl = readline.createInterface({ 47 | input: process.stdin, 48 | output: process.stdout, 49 | prompt: "fluent>", 50 | }); 51 | 52 | const lines = []; 53 | 54 | rl.on("line", line => lines.push(line)); 55 | rl.on("close", () => 56 | print_ebnf(lines.join("\n") + "\n")); 57 | } 58 | 59 | function from_file(file_path) { 60 | fs.readFile(file_path, "utf8", (err, content) => { 61 | if (err) { 62 | throw err; 63 | } 64 | 65 | print_ebnf(content); 66 | }); 67 | } 68 | 69 | function print_ebnf(source) { 70 | // Each EBNF rule already ends with \n. 71 | process.stdout.write(ebnf(source)); 72 | } 73 | -------------------------------------------------------------------------------- /bin/parse.js: -------------------------------------------------------------------------------- 1 | import fs from "fs"; 2 | import readline from "readline"; 3 | import parse_args from "minimist"; 4 | import {Resource} from "../syntax/grammar.js"; 5 | 6 | const argv = parse_args(process.argv.slice(2), { 7 | boolean: ["help"], 8 | alias: { 9 | help: "h", 10 | }, 11 | }); 12 | 13 | if (argv.help) { 14 | exit_help(0); 15 | } 16 | 17 | const [file_path] = argv._; 18 | 19 | if (file_path === "-") { 20 | parse_stdin(); 21 | } else if (file_path) { 22 | parse_file(file_path); 23 | } else { 24 | exit_help(1); 25 | } 26 | 27 | function exit_help(exit_code) { 28 | console.log(` 29 | Usage: node parse.js [OPTIONS] 30 | 31 | When FILE is "-", read text from stdin. 32 | 33 | Examples: 34 | 35 | node parse.js path/to/file.ftl 36 | cat path/to/file.ftl | node parse.js - 37 | 38 | Options: 39 | 40 | -h, --help Display help and quit. 41 | `); 42 | process.exit(exit_code); 43 | } 44 | 45 | function parse_stdin() { 46 | const rl = readline.createInterface({ 47 | input: process.stdin, 48 | output: process.stdout, 49 | prompt: "fluent>", 50 | }); 51 | 52 | const lines = []; 53 | 54 | rl.on("line", line => lines.push(line)); 55 | rl.on("close", () => 56 | parse(lines.join("\n") + "\n")); 57 | } 58 | 59 | function parse_file(file_path) { 60 | fs.readFile(file_path, "utf8", (err, content) => { 61 | if (err) { 62 | throw err; 63 | } 64 | 65 | parse(content); 66 | }); 67 | } 68 | 69 | 70 | function parse(ftl) { 71 | Resource.run(ftl).fold( 72 | ast => console.log(JSON.stringify(ast, null, 4)), 73 | err => console.error(err)); 74 | } 75 | -------------------------------------------------------------------------------- /book.toml: -------------------------------------------------------------------------------- 1 | [book] 2 | multilingual = false 3 | src = "guide" 4 | 5 | [build] 6 | build-dir = "build/guide" 7 | create-missing = false 8 | 9 | [output.html] 10 | additional-css = ["guide/css/custom.css"] 11 | edit-url-template = "https://github.com/projectfluent/fluent/edit/master/{path}" 12 | git-repository-url = "https://github.com/projectfluent/fluent/" 13 | no-section-label = true 14 | -------------------------------------------------------------------------------- /guide/README.md: -------------------------------------------------------------------------------- 1 | # Fluent Syntax Guide 2 | 3 | Fluent is a localization paradigm designed to unleash the expressive power 4 | of the natural language. The format used to describe translation resources 5 | used by Fluent is called `FTL`. 6 | 7 | FTL is designed to be simple to read, but at the same time allows to represent 8 | complex concepts from natural languages like gender, plurals, conjugations, 9 | and others. 10 | 11 | The following chapters will demonstrate how to use FTL to solve localization 12 | challenges. Each chapter contains a hands-on example of simple FTL concepts. 13 | -------------------------------------------------------------------------------- /guide/SUMMARY.md: -------------------------------------------------------------------------------- 1 | # Summary 2 | 3 | * [Introduction](README.md) 4 | * [Hello, world!](hello.md) 5 | * [Writing Text](text.md) 6 | * [Placeables](placeables.md) 7 | * [Special Characters](special.md) 8 | * [Multiline Text](multiline.md) 9 | * [Variables](variables.md) 10 | * [Referencing Messages](references.md) 11 | * [Selectors](selectors.md) 12 | * [Attributes](attributes.md) 13 | * [Terms](terms.md) 14 | * [Comments](comments.md) 15 | * [Built-in Functions](builtins.md) 16 | * [Functions](functions.md) 17 | * [Learn More](more.md) 18 | -------------------------------------------------------------------------------- /guide/attributes.md: -------------------------------------------------------------------------------- 1 | # Attributes 2 | 3 | ``` 4 | login-input = Predefined value 5 | .placeholder = email@example.com 6 | .aria-label = Login input value 7 | .title = Type your login email 8 | 9 | ``` 10 | 11 | UI elements often contain multiple translatable messages per one widget. For 12 | example, an HTML form input may have a value, but also a `placeholder` 13 | attribute, `aria-label` attribute, and maybe a `title` attribute. 14 | 15 | Another example would be a Web Component confirm window with an `OK` button, 16 | `Cancel` button, and a message. 17 | 18 | In order to prevent having to define multiple separate messages for representing 19 | different strings within a single element, FTL allows you to add attributes to 20 | messages. 21 | 22 | This feature is particularly useful in translating more complex widgets since, 23 | thanks to all attributes being stored on a single unit, it's easier for editors, 24 | comments, and tools to identify and work with the given message. 25 | 26 | Attributes may also be used to define grammatical properties of 27 | [terms](terms.html). Attributes of terms are private and cannot be retrieved 28 | by the localization runtime. They can only be used as 29 | [selectors](selectors.html). 30 | -------------------------------------------------------------------------------- /guide/builtins.md: -------------------------------------------------------------------------------- 1 | # Builtins 2 | 3 | ``` 4 | emails = You have { $unreadEmails } unread emails. 5 | emails2 = You have { NUMBER($unreadEmails) } unread emails. 6 | 7 | last-notice = 8 | Last checked: { DATETIME($lastChecked, day: "numeric", month: "long") }. 9 | ``` 10 | 11 | In most cases, Fluent will automatically select the right formatter for the 12 | argument and format it into a given locale. 13 | 14 | In other cases, the developer will have to annotate the argument with additional 15 | information on how to format it (see [Partial Arguments](functions.html#partial-arguments)) 16 | 17 | But in rare cases there may be a value for a localizer to select some formatting 18 | options that are specific to the given locale. 19 | 20 | Examples include: defining month as `short` or `long` in the `DATE` 21 | formatter (using arguments defined in `Intl.DateTimeFormat`) or whether to use 22 | grouping separator when displaying a large number. 23 | -------------------------------------------------------------------------------- /guide/comments.md: -------------------------------------------------------------------------------- 1 | # Comments 2 | 3 | Comments in Fluent start with `#`, `##`, or `###`, and can be used to 4 | document messages and to define the outline of the file. 5 | 6 | Single-hash comments (`#`) can be standalone or can be bound to messages. If 7 | a comment is located right above a message it is considered part of the 8 | message and localization tools will present the message and the comment 9 | together. Otherwise the comment is standalone (which is useful for commenting 10 | parts of the file out). 11 | 12 | Double-hash comments (`##`) are always standalone. They can be used to divide 13 | files into smaller groups of messages related to each other; they are 14 | group-level comments. Think of them as of headers with a description. 15 | Group-level comments are intended as a hint for localizers and tools about 16 | the layout of the localization resource. The grouping ends with the next 17 | group comment or at the end of the file. 18 | 19 | Triple-hash comments (`###`) are also always standalone and apply to the 20 | entire file; they are file-level comments. They can be used to provide 21 | information about the purpose or the context of the entire file. 22 | 23 | ```properties 24 | # This Source Code Form is subject to the terms of the Mozilla Public 25 | # License, v. 2.0. If a copy of the MPL was not distributed with this 26 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 27 | 28 | ### Localization for Server-side strings of Firefox Screenshots 29 | 30 | ## Global phrases shared across pages 31 | 32 | my-shots = My Shots 33 | home-link = Home 34 | screenshots-description = 35 | Screenshots made simple. Take, save, and 36 | share screenshots without leaving Firefox. 37 | 38 | ## Creating page 39 | 40 | # Note: { $title } is a placeholder for the title of the web page 41 | # captured in the screenshot. The default, for pages without titles, is 42 | # creating-page-title-default. 43 | creating-page-title = Creating { $title } 44 | creating-page-title-default = page 45 | creating-page-wait-message = Saving your shot… 46 | ``` 47 | -------------------------------------------------------------------------------- /guide/css/custom.css: -------------------------------------------------------------------------------- 1 | body { 2 | font-size: 1.8rem; 3 | } 4 | 5 | .chapter li.chapter-item { 6 | line-height: 1.8em; 7 | } 8 | -------------------------------------------------------------------------------- /guide/functions.md: -------------------------------------------------------------------------------- 1 | # Functions in FTL 2 | 3 | Functions provide additional functionality available to the localizers. They 4 | can be either used to format data according to the current language's rules or 5 | can provide additional data that the localizer may use (like, the platform, or 6 | time of the day) to fine tune the translation. 7 | 8 | FTL implementations should ship with a number of built-in functions that can be 9 | used from within localization messages. 10 | 11 | The list of available functions is extensible and environments may want to 12 | introduce additional functions, designed to aid localizers writing translations 13 | targeted for such environments. 14 | 15 | ## Using Functions 16 | 17 | FTL Functions can only be called inside of placeables. Use them to return 18 | a value to be interpolated in the message or as selectors in select 19 | expressions. 20 | 21 | Example: 22 | ``` 23 | today-is = Today is { DATETIME($date) } 24 | ``` 25 | 26 | ## Function parameters 27 | 28 | Functions may accept positional and named arguments. Some named arguments 29 | are only available to developers when they pre-format variables passed as 30 | arguments to translations (see [Partially-formatted 31 | variables](#partially-formatted-variables) below). 32 | 33 | ## Built-in Functions 34 | 35 | Built-in functions are very generic and should be applicable to any translation 36 | environment. 37 | 38 | ### `NUMBER` 39 | 40 | Formats a number to a string in a given locale. 41 | 42 | Example: 43 | ``` 44 | dpi-ratio = Your DPI ratio is { NUMBER($ratio, minimumFractionDigits: 2) } 45 | ``` 46 | 47 | Parameters: 48 | ``` 49 | currencyDisplay 50 | useGrouping 51 | minimumIntegerDigits 52 | minimumFractionDigits 53 | maximumFractionDigits 54 | minimumSignificantDigits 55 | maximumSignificantDigits 56 | ``` 57 | 58 | Developer parameters: 59 | ``` 60 | style 61 | currency 62 | ``` 63 | 64 | See the 65 | [Intl.NumberFormat](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/NumberFormat) 66 | for the description of the parameters. 67 | 68 | ### `DATETIME` 69 | 70 | Formats a date to a string in a given locale. 71 | 72 | Example: 73 | ``` 74 | today-is = Today is { DATETIME($date, month: "long", year: "numeric", day: "numeric") } 75 | ``` 76 | 77 | Parameters: 78 | ``` 79 | hour12 80 | weekday 81 | era 82 | year 83 | month 84 | day 85 | hour 86 | minute 87 | second 88 | timeZoneName 89 | ``` 90 | 91 | Developer parameters: 92 | ``` 93 | timeZone 94 | ``` 95 | 96 | See the [Intl.DateTimeFormat](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/DateTimeFormat) for the description of the parameters. 97 | 98 | ## Implicit use 99 | 100 | In order to simplify most common scenarios, FTL will run some default functions 101 | while resolving placeables. 102 | 103 | ### `NUMBER` 104 | 105 | If the variable passed from the developer is a number and is used in 106 | a placeable, FTL will implicitly call a NUMBER function on it. 107 | 108 | Example: 109 | ``` 110 | emails = Number of unread emails { $unreadEmails } 111 | 112 | emails2 = Number of unread emails { NUMBER($unreadEmails) } 113 | ``` 114 | 115 | Numbers used as selectors in select expressions will match the number exactly 116 | or they will match the current language's [CLDR plural 117 | category](http://www.unicode.org/cldr/charts/30/supplemental/language_plural_rules.html) 118 | for the number. 119 | 120 | The following examples are equivalent and will both work. The second example 121 | may be used to pass additional formatting options to the `NUMBER` formatter for 122 | the purpose of choosing the correct plural category: 123 | 124 | ``` 125 | liked-count = { $num -> 126 | [0] No likes yet. 127 | [one] One person liked your message 128 | *[other] { $num } people liked your message 129 | } 130 | 131 | liked-count2 = { NUMBER($num) -> 132 | [0] No likes yet. 133 | [one] One person liked your message 134 | *[other] { $num } people liked your message 135 | } 136 | ``` 137 | 138 | ### `DATETIME` 139 | 140 | If the variable passed from the developer is a date and is used in a placeable, 141 | FTL will implicitly call a `DATETIME` function on it. 142 | 143 | Example: 144 | ``` 145 | log-time = Entry time: { $date } 146 | 147 | log-time2 = Entry time: { DATETIME($date) } 148 | ``` 149 | 150 | ## Partially-formatted variables 151 | 152 | In most cases localizers don't need to call Functions explicitly, thanks to the 153 | implicit formatting. If the implicit formatting is not sufficient, the Function 154 | can be called explicitly with additional parameters. To ease the burden this 155 | might have on localizers, Fluent implementations may allow developers to set 156 | the default formatting parameters for the variables they pass. 157 | 158 | In other words, developers can provide variables which are already wrapped in 159 | a partial Function call. 160 | 161 | ``` 162 | today = Today is { $day } 163 | ``` 164 | 165 | ```javascript 166 | ctx.format('today', { 167 | day: new FluentDateTime(new Date(), { 168 | weekday: 'long' 169 | }) 170 | }) 171 | ``` 172 | 173 | If the localizer wishes to modify the parameters, for example because the 174 | string doesn't fit in the UI, they can pass the variable to the same 175 | Function and overload the parameters set by the developer. 176 | 177 | ``` 178 | today = Today is { DATETIME($day, weekday: "short") } 179 | ``` 180 | -------------------------------------------------------------------------------- /guide/hello.md: -------------------------------------------------------------------------------- 1 | # Hello, world! 2 | 3 | In Fluent, the basic unit of translation is called a message. Messages are 4 | containers for information. You use messages to identify, store, and recall 5 | translation information to be used in the product. The simplest example of a 6 | message looks like this: 7 | 8 | ``` 9 | hello = Hello, world! 10 | ``` 11 | 12 | Each message has an identifier that allows the developer to bind it to the place 13 | in the software where it will be used. The above message is called `hello`. 14 | 15 | In its simplest form, a message has just a single text value. In the example 16 | above the value is *Hello, world!*. The value begins at the first non-blank 17 | character after the `=` sign, but there are rare exceptions related to 18 | multiline text values. The next chapter ([Writing Text](text.html)) has all 19 | the details. 20 | 21 | The majority of messages in Fluent will look similar to the one above. Fluent 22 | has been designed to keep these simple translations simple. Sometimes, 23 | however, messages need more complexity. Throughout this guide, you'll learn 24 | how to adjust messages to the grammar of your language and the requirements 25 | of the localized product. 26 | 27 | Read on to learn how to read and write Fluent! 28 | -------------------------------------------------------------------------------- /guide/more.md: -------------------------------------------------------------------------------- 1 | # Dive deeper 2 | 3 | You can experiment with the syntax using [Fluent Playground][], an interactive 4 | editor available in the web browser. 5 | 6 | If you are a tool author, you may be interested in the formal description of 7 | the syntax available in the [projectfluent/fluent][] repository. 8 | 9 | [Fluent Playground]: http://projectfluent.org/play 10 | [projectfluent/fluent]: https://github.com/projectfluent/fluent/ 11 | -------------------------------------------------------------------------------- /guide/multiline.md: -------------------------------------------------------------------------------- 1 | # Multiline Text 2 | 3 | Text can span multiple lines as long as it is indented by at least one space. 4 | Only the space character (`U+0020`) can be used for indentation. Fluent 5 | treats tab characters as regular text. 6 | 7 | ``` 8 | single = Text can be written in a single line. 9 | 10 | multi = Text can also span multiple lines 11 | as long as each new line is indented 12 | by at least one space. 13 | 14 | block = 15 | Sometimes it's more readable to format 16 | multiline text as a "block", which means 17 | starting it on a new line. All lines must 18 | be indented by at least one space. 19 | ``` 20 | 21 | In almost all cases, patterns start at the first non-blank character and 22 | end at the last non-blank character. In other words, the leading and 23 | trailing blanks are ignored. There's one exception to this rule, due to 24 | another rule which we'll cover below (in the `multiline2` example). 25 | 26 | ``` 27 | leading-spaces = This message's value starts with the word "This". 28 | leading-lines = 29 | 30 | 31 | This message's value starts with the word "This". 32 | The blank lines under the identifier are ignored. 33 | ``` 34 | 35 | Line breaks and blank lines are preserved as long as they are positioned 36 | inside of multiline text, i.e. there's text before and after them. 37 | 38 | ``` 39 | blank-lines = 40 | 41 | The blank line above this line is ignored. 42 | This is a second line of the value. 43 | 44 | The blank line above this line is preserved. 45 | ``` 46 | 47 | In multiline patterns, all common indent is removed when the text value is 48 | spread across multiple indented lines. 49 | 50 | ``` 51 | multiline1 = 52 | This message has 4 spaces of indent 53 | on the second line of its value. 54 | ``` 55 | 56 | We can visualize this behavior in the following manner and we'll use this 57 | convention in the rest of this chapter: 58 | 59 | ``` 60 | # █ denotes the indent common to all lines (removed from the value). 61 | # · denotes the indent preserved in the final value. 62 | multiline1 = 63 | ████This message has 4 spaces of indent 64 | ████····on the second line of its value. 65 | ``` 66 | 67 | This behavior also applies when the first line of a text block is indented 68 | relative to the following lines. This is the only case where a sequence of 69 | leading blank characters might be preserved as part of the text value, even 70 | if they're technically in the leading position. 71 | 72 | ``` 73 | multiline2 = 74 | ████··This message starts with 2 spaces on the first 75 | ████first line of its value. The first 4 spaces of indent 76 | ████are removed from all lines. 77 | ``` 78 | 79 | Only the indented lines comprising the multiline pattern participate in this 80 | behavior. Specifically, if the text starts on the same line as the message 81 | identifier, then this first line is not considered as indented, and is 82 | excluded from the dedentation behavior. In such cases, the first line (the 83 | unindented one) still has its leading blanks ignored—because patterns start 84 | on the first non-blank character. 85 | 86 | ``` 87 | multiline3 = This message has 4 spaces of indent 88 | ████····on the second line of its value. The first 89 | ████line is not considered indented at all. 90 | 91 | # Same value as multiline3 above. 92 | multiline4 = This message has 4 spaces of indent 93 | ████····on the second line of its value. The first 94 | ████line is not considered indented at all. 95 | ``` 96 | 97 | Note that if a multiline pattern starts on the same line as the identifier 98 | and it only consists of one more line of text below it, then the indent 99 | common to all _indented_ lines is equal to the indent of the second line, 100 | i.e. the only indented line. All indent will be removed in this case. 101 | 102 | ``` 103 | multiline5 = This message ends up having no indent 104 | ████████on the second line of its value. 105 | ``` 106 | -------------------------------------------------------------------------------- /guide/placeables.md: -------------------------------------------------------------------------------- 1 | # Placeables 2 | 3 | Text in Fluent may use special syntax to incorporate small pieces of 4 | programmable interface. Those pieces are denoted with curly braces `{` and 5 | `}` and are called placeables. 6 | 7 | It's common to use placeables to interpolate external 8 | [variables](variables.html) into the translation. Variable values are 9 | provided by the developer and they will be set on runtime. They may also 10 | dynamically change as the user uses the localized product. 11 | 12 | ``` 13 | # $title (String) - The title of the bookmark to remove. 14 | remove-bookmark = Really remove { $title }? 15 | ``` 16 | 17 | It's also possible to [interpolate other messages and terms](references.html) 18 | inside of text values. 19 | 20 | ``` 21 | -brand-name = Firefox 22 | installing = Installing { -brand-name }. 23 | ``` 24 | 25 | Lastly, placeables can be used to insert [special characters](special.html) 26 | into text values. For instance, due to placeables using `{` and `}` as 27 | delimiters, inserting a literal curly brace into the translation requires 28 | special care. Quoted text can be effectively used for the purpose: 29 | 30 | ``` 31 | opening-brace = This message features an opening curly brace: {"{"}. 32 | closing-brace = This message features a closing curly brace: {"}"}. 33 | ``` 34 | -------------------------------------------------------------------------------- /guide/references.md: -------------------------------------------------------------------------------- 1 | # Message References 2 | 3 | Another use-case for placeables is referencing one message in another one. 4 | 5 | ``` 6 | menu-save = Save 7 | help-menu-save = Click { menu-save } to save the file. 8 | ``` 9 | 10 | Referencing other messages generally helps to keep certain translations 11 | consistent across the interface and makes maintenance easier. 12 | 13 | It is also particularly handy for keeping branding separated from the rest of 14 | the translations, so that it can be changed easily when needed, e.g. during 15 | the build process of the application. This use-case is best served by 16 | defining a [term](terms.html) with a leading dash `-`, like `-brand-name` in 17 | the example below. 18 | 19 | ``` 20 | -brand-name = Firefox 21 | installing = Installing { -brand-name }. 22 | ``` 23 | 24 | Using a term here indicates to tools and to the localization runtime that 25 | `-brand-name` is not supposed to be used directly in the product but rather 26 | should be referenced in other messages. 27 | -------------------------------------------------------------------------------- /guide/selectors.md: -------------------------------------------------------------------------------- 1 | # Selectors 2 | 3 | ``` 4 | emails = 5 | { $unreadEmails -> 6 | [one] You have one unread email. 7 | *[other] You have { $unreadEmails } unread emails. 8 | } 9 | ``` 10 | 11 | One of the most common cases when a localizer needs to use a placeable is when 12 | there are multiple variants of the string that depend on some external 13 | variable. In the example above, the `emails` message depends on the value of 14 | the `$unreadEmails` variable. 15 | 16 | FTL has the select expression syntax which allows to define multiple variants 17 | of the translation and choose between them based on the value of the 18 | selector. The `*` indicator identifies the default variant. A default 19 | variant is required. 20 | 21 | The selector may be a string, in which case it will be compared directly to 22 | the keys of variants defined in the select expression. For selectors which 23 | are numbers, the variant keys either match the number exactly or they match 24 | the [CLDR plural category][] for the number. The possible categories are: 25 | `zero`, `one`, `two`, `few`, `many`, and `other`. For instance, English has 26 | two plural categories: `one` and `other`. 27 | 28 | If the translation requires a number to be formatted in a particular 29 | non-default manner, the selector should use the same formatting options. The 30 | formatted number will then be used to choose the correct CLDR plural category 31 | which, for some languages, might be different than the category of the 32 | unformatted number: 33 | 34 | ``` 35 | your-score = 36 | { NUMBER($score, minimumFractionDigits: 1) -> 37 | [0.0] You scored zero points. What happened? 38 | *[other] You scored { NUMBER($score, minimumFractionDigits: 1) } points. 39 | } 40 | ``` 41 | 42 | Using formatting options also allows for selectors using ordinal rather than 43 | cardinal plurals: 44 | 45 | ``` 46 | your-rank = { NUMBER($pos, type: "ordinal") -> 47 | [1] You finished first! 48 | [one] You finished {$pos}st 49 | [two] You finished {$pos}nd 50 | [few] You finished {$pos}rd 51 | *[other] You finished {$pos}th 52 | } 53 | ``` 54 | 55 | [CLDR plural category]: https://www.unicode.org/cldr/charts/latest/supplemental/language_plural_rules.html 56 | -------------------------------------------------------------------------------- /guide/special.md: -------------------------------------------------------------------------------- 1 | # Special Characters 2 | 3 | In Fluent, text is the most important part of the file. As such, it doesn't 4 | have any special syntax: you can just write it without any delimiters (e.g. 5 | quotes) and you can use characters from all Unicode planes. Regular text 6 | should be enough to store the vast majority of translations. In rare cases 7 | when it's not, another type of text can be used: quoted text. 8 | 9 | ## Quoted Text 10 | 11 | Quoted text uses double quotes as delimiters and cannot contain line breaks. 12 | Like other types of expressions, it can be used inside of placeables (e.g. 13 | `{"abc"}`). It's rarely needed but can be used to insert characters which are 14 | otherwise considered special by Fluent in the given context. For instance, 15 | due to placeables using `{` and `}` as delimiters, inserting a literal curly 16 | brace into the translation requires special care. Quoted text can be 17 | effectively used for the purpose: 18 | 19 | ``` 20 | opening-brace = This message features an opening curly brace: {"{"}. 21 | closing-brace = This message features a closing curly brace: {"}"}. 22 | ``` 23 | 24 | In the example above the `{"{"}` syntax can be read as a piece of quoted text 25 | `"{"` being interpolated into regular text via the placeable syntax: `{…}`. 26 | As can be seen from this example, curly braces carry no special meaning in 27 | quoted text. As a result, quoted text cannot feature any interpolations. 28 | 29 | The same strategy as above can be used to ensure blank space is preserved in 30 | the translations: 31 | 32 | ``` 33 | blank-is-removed = This message starts with no blanks. 34 | blank-is-preserved = {" "}This message starts with 4 spaces. 35 | ``` 36 | 37 | In very rare cases, you may need to resort to quoted text to use a literal 38 | dot (`.`), star (`*`) or bracket (`[`) when they are used as the first 39 | character on a new line. Otherwise, they would start a new attribute or a new 40 | variant. 41 | 42 | ``` 43 | leading-bracket = 44 | This message has an opening square bracket 45 | at the beginning of the third line: 46 | {"["}. 47 | ``` 48 | 49 | ``` 50 | attribute-how-to = 51 | To add an attribute to this messages, write 52 | {".attr = Value"} on a new line. 53 | .attr = An actual attribute (not part of the text value above) 54 | ``` 55 | 56 | ## Escape Sequences 57 | 58 | Quoted text supports a small number of escape sequences. The backslash 59 | character (`\`) starts an escape sequence in quoted text. In regular text, 60 | the backslash is just the literal character with no special meaning. 61 | 62 | | Escape sequence | Meaning | 63 | |-----------------|---------| 64 | | `\"` | The literal double quote. | 65 | | `\uHHHH` | A Unicode character in the U+0000 to U+FFFF range. | 66 | | `\UHHHHHH` | Any Unicode character. | 67 | | `\\` | The literal backslash. | 68 | 69 | Escape sequences are rarely needed, but Fluent supports them for the sake of 70 | edge cases. In real life application using the actual character in the 71 | regular text should be preferred. 72 | 73 | ``` 74 | # This is OK, but cryptic and hard to read and edit. 75 | literal-quote1 = Text in {"\""}double quotes{"\""}. 76 | 77 | # This is preferred. Just use the actual double quote character. 78 | literal-quote2 = Text in "double quotes". 79 | ``` 80 | 81 | Quoted text should be used sparingly, most often in scenarios which call for 82 | a special character, or when enclosing characters in `{"` and `"}` makes 83 | them easier to spot. For instance, the non-breaking space character looks 84 | like a regular space in most text editors and it's easy to miss in a 85 | translation. A Unicode escape sequence inside of a quoted text may be used 86 | to make it stand out: 87 | 88 | ``` 89 | privacy-label = Privacy{"\u00A0"}Policy 90 | ``` 91 | 92 | A similar approach will make it clear which dash character is used in the 93 | following example: 94 | 95 | ``` 96 | # The dash character is an EM DASH but depending on the font face, 97 | # it might look like an EN DASH. 98 | which-dash1 = It's a dash—or is it? 99 | 100 | # Using a Unicode escape sequence makes the intent clear. 101 | which-dash2 = It's a dash{"\u2014"}or is it? 102 | ``` 103 | 104 | Any Unicode character can be used in regular text values and in quoted text. 105 | Unless readability calls for using the escape sequence, always prefer the 106 | actual Unicode character. 107 | 108 | ``` 109 | # This will work fine, but the codepoint can be considered 110 | # cryptic by other translators. 111 | tears-of-joy1 = {"\U01F602"} 112 | 113 | # This is preferred. You can instantly see what the Unicode 114 | # character used here is. 115 | tears-of-joy2 = 😂 116 | ``` 117 | 118 | ---- 119 | #### Note for Developers 120 | 121 | If you're writing Fluent inside another programming language that uses 122 | backslash for escaping, you'll need to use _two_ backslashes to start an escape 123 | sequence in Fluent's quoted text. The first backslash is parsed by the host 124 | programming language and makes the second backslash a normal character _in that 125 | language_. The second backslash can then be correctly parsed by Fluent. 126 | 127 | In JavaScript, for instance, the `privacy-label` message from one of the 128 | previous examples could be added programmatically to a bundle by using two 129 | backslashes in the source code: 130 | 131 | ``` 132 | let bundle = new FluentBundle("en"); 133 | bundle.addMessages(` 134 | privacy-label = Privacy{"\\u00A0"}Policy 135 | `); 136 | ``` 137 | -------------------------------------------------------------------------------- /guide/terms.md: -------------------------------------------------------------------------------- 1 | # Terms 2 | 3 | Terms are similar to regular messages but they can only be used as references 4 | in other messages. Their identifiers start with a single dash `-` like in the 5 | example above: `-brand-name`. The runtime cannot retrieve terms directly. 6 | They are best used to define vocabulary and glossary items which can be used 7 | consistently across the localization of the entire product. 8 | 9 | ``` 10 | -brand-name = Firefox 11 | 12 | about = About { -brand-name }. 13 | update-successful = { -brand-name } has been updated. 14 | ``` 15 | 16 | ## Parameterized Terms 17 | 18 | Term values follow the same rules as message values. They can be simple text, 19 | or they can interpolate other expressions, including variables. However, 20 | while messages receive data for variables directly from the app, terms 21 | receive such data from messages in which they are used. Such references take 22 | the form of `-term(…)` where the variables available inside of the term are 23 | defined between the parentheses, e.g. `-term(param: "value")`. 24 | 25 | ``` 26 | # A contrived example to demonstrate how variables 27 | # can be passed to terms. 28 | -https = https://{ $host } 29 | visit = Visit { -https(host: "example.com") } for more information. 30 | ``` 31 | 32 | The above example isn't very useful and a much better approach in this 33 | particular case would be to use the full address directly in the `visit` 34 | message. There is, however, another use-case which takes full advantage of 35 | this feature. By passing variables into the term, you can define select 36 | expressions with multiple variants of the same term value. 37 | 38 | ``` 39 | -brand-name = 40 | { $case -> 41 | *[nominative] Firefox 42 | [locative] Firefoksie 43 | } 44 | 45 | # "About Firefox." 46 | about = Informacje o { -brand-name(case: "locative") }. 47 | ``` 48 | 49 | This pattern can be very useful for defining multiple _facets_ of the term, 50 | which can correspond to grammatical cases or other grammatical or stylistic 51 | properties of the language. In many inflected languages (e.g. German, 52 | Finnish, Hungarian, all Slavic languages), the *about* preposition governs 53 | the grammatical case of the complement. It might be accusative (German), 54 | ablative (Latin), or locative (Slavic languages). The grammatical cases can 55 | be defined as variants of the same term and referred to via parameterization 56 | from other messages. This is what happens in the `about` message above. 57 | 58 | If no parameters are passed into the term, or if the term is referenced 59 | without any parentheses, the default variant will be used. 60 | 61 | ``` 62 | -brand-name = 63 | { $case -> 64 | *[nominative] Firefox 65 | [locative] Firefoksie 66 | } 67 | 68 | # "Firefox has been successfully updated." 69 | update-successful = { -brand-name } został pomyślnie zaktualizowany. 70 | ``` 71 | 72 | ## Terms and Attributes 73 | 74 | Sometimes translations might vary depending on some grammatical trait of a 75 | term references in them. Terms can store this grammatical information about 76 | themselves in [attributes](attributes.html). In the example below the form of 77 | the past tense of _has been updated_ depends on the grammatical gender of 78 | `-brand-name`. 79 | 80 | ``` 81 | -brand-name = Aurora 82 | .gender = feminine 83 | 84 | update-successful = 85 | { -brand-name.gender -> 86 | [masculine] { -brand-name } został zaktualizowany. 87 | [feminine] { -brand-name } została zaktualizowana. 88 | *[other] Program { -brand-name } został zaktualizowany. 89 | } 90 | ``` 91 | 92 | Use attributes to describe grammatical traits and properties. Genders, 93 | animacy, whether the term message starts with a vowel or not etc. Attributes 94 | of terms are private and cannot be retrieved by the localization runtime. 95 | They can only be used as [selectors](selectors.html). If needed, they can 96 | also be parameterized using the `-term.attr(param: "value")` syntax. 97 | -------------------------------------------------------------------------------- /guide/text.md: -------------------------------------------------------------------------------- 1 | # Writing Text 2 | 3 | Fluent is a file format designed for storing translations. In consequence, 4 | text is the most important part of any Fluent file. Messages, terms, variants 5 | and attributes all store their values as text. In Fluent, text values are 6 | referred to as _patterns_. 7 | 8 | ## Placeables 9 | 10 | In Fluent, text can interpolate values of other messages, as well as external 11 | data passed into the translation from the app. Use the curly braces to start 12 | and end interpolating another expression inside of a pattern: 13 | 14 | ``` 15 | # $title (String) - The title of the bookmark to remove. 16 | remove-bookmark = Are you sure you want to remove { $title }? 17 | ``` 18 | 19 | Refer to the chapters about [placeables](placeables.html) for more 20 | information. 21 | 22 | ## Multiline Text 23 | 24 | Text can span multiple lines. In such cases, Fluent will calculate the common 25 | indent of all indented lines and remove it from the final text value. 26 | 27 | ``` 28 | multi = Text can also span multiple lines as long as 29 | each new line is indented by at least one space. 30 | Because all lines in this message are indented 31 | by the same amount, all indentation will be 32 | removed from the final value. 33 | 34 | indents = 35 | Indentation common to all indented lines is removed 36 | from the final text value. 37 | This line has 2 spaces in front of it. 38 | ``` 39 | 40 | Refer to the chapter about [multiline text](multiline.html) for more 41 | information. 42 | -------------------------------------------------------------------------------- /guide/variables.md: -------------------------------------------------------------------------------- 1 | # Variables 2 | 3 | Variables are pieces of data received from the app. They are provided by the 4 | developer of the app and may be interpolated into the translation with 5 | [placeables](placeables.html). Variables can dynamically change as the user 6 | is using the localized product. 7 | 8 | Variables are referred to via the `$variable-name` syntax: 9 | 10 | ``` 11 | welcome = Welcome, { $user }! 12 | unread-emails = { $user } has { $email-count } unread emails. 13 | ``` 14 | 15 | For instance, if the current user's name is _Jane_ and she has 5 unread 16 | emails, the above translations will be displayed as: 17 | 18 | ``` 19 | Welcome, Jane! 20 | Jane has 5 unread emails. 21 | ``` 22 | 23 | There are all kinds of external data that might be useful in providing a good 24 | localization: user names, number of unread messages, battery level, current 25 | time, time left before an alarm goes off, etc. Fluent offers a number of 26 | features designed to make working with variables convenient. 27 | 28 | ## Variants and Selectors 29 | 30 | In some languages, using a number in the middle of a translated sentence will 31 | require proper plural forms of words associated with the number. In Fluent, 32 | you can define multiple [variants](selectors.html) of the translation, each 33 | for a different plural category. 34 | 35 | ## Implicit Formatting 36 | 37 | Numbers and dates are automatically formatted according to your language's 38 | formatting rules. Consider the following translation: 39 | 40 | ``` 41 | # $duration (Number) - The duration in seconds. 42 | time-elapsed = Time elapsed: { $duration }s. 43 | ``` 44 | 45 | For the `$duration` variable value of `12345.678`, the following message will be 46 | displayed in the product, assuming the language is set to British or American 47 | English. Note the use of comma as the thousands separator. 48 | 49 | ``` 50 | Time elapsed: 12,345.678s. 51 | ``` 52 | 53 | In South African English, however, the result would be: 54 | 55 | ``` 56 | Time elapsed: 12 345,678s. 57 | ``` 58 | 59 | ## Explicit Formatting 60 | 61 | In some cases the localizer might want to have a greater control over how a 62 | number or a date is formatted in text. Fluent provides [built-in 63 | functions](builtins.html) for this purpose. 64 | 65 | ``` 66 | # $duration (Number) - The duration in seconds. 67 | time-elapsed = Time elapsed: { NUMBER($duration, maximumFractionDigits: 0) }s. 68 | ``` 69 | 70 | For the same value of `$duration` as above, the result will look like the 71 | following for American and British English: 72 | 73 | ``` 74 | Time elapsed: 12,345s. 75 | ``` 76 | -------------------------------------------------------------------------------- /lib/README.md: -------------------------------------------------------------------------------- 1 | # Fluent Specification Support Code 2 | 3 | This directory contains support code for the parser-combinator underlying the syntax code, as well as the code to transform `grammar.js` to `fluent.ebnf`. The combination allows for a succinct formulation of the syntax in `grammar.js` and a readable representation of that in `fluent.ebnf`. 4 | 5 | ## Parser-Combinator 6 | 7 | **`parser.js`** is the base parser class, **`stream.js`** holds a iterator over the strings to be parsed. 8 | 9 | **`combinators.js`** holds the actual basic grammar productions and logical combinations of grammar productions. 10 | 11 | Both use `Success` and `Failure` from **`result.js`** to pass success and failure conditions forward. 12 | 13 | After the definition of grammar productions, the utilities in **`mappers.js`** are used to concat, flatten, and extract the matched data to prepare it for AST generation. 14 | 15 | ## EBNF Generation 16 | 17 | The `fluent.ebnf` is created by parsing the `grammar.js` and transpilation to an EBNF file. The `babylon` JavaScript parser is used to load a Babel AST. 18 | 19 | **`ebnf.js`** is the top-level entry point used by `bin/ebnf.js`. It uses the iteration logic from **`walker.js`** to go over the Babel AST and to extract the information relevant to the EBNF via the Visitor in **`visitor.js`**. The resulting data is then serialiazed to EBNF via **`serializer.js`**. 20 | -------------------------------------------------------------------------------- /lib/combinators.js: -------------------------------------------------------------------------------- 1 | import Parser from "./parser.js"; 2 | import {Success, Failure} from "./result.js"; 3 | import {join} from "./mappers.js"; 4 | 5 | export function defer(fn) { 6 | // Parsers may be defined as defer(() => parser) to avoid cyclic 7 | // dependecies. 8 | return new Parser(stream => 9 | fn().run(stream)); 10 | } 11 | 12 | export function string(str) { 13 | return new Parser(stream => 14 | stream.head(str.length) === str 15 | ? new Success(str, stream.move(str.length)) 16 | : new Failure(`${str} not found`, stream)); 17 | } 18 | 19 | export function regex(re) { 20 | return new Parser(stream => { 21 | const result = stream.exec(re); 22 | 23 | if (result === null) { 24 | return new Failure("regex did not match", stream); 25 | } 26 | 27 | const [match] = result; 28 | 29 | return new Success(match, stream.move(match.length)); 30 | }); 31 | } 32 | 33 | export function charset(range) { 34 | return regex(`[${range}]`); 35 | } 36 | 37 | export function eof() { 38 | return new Parser(stream => 39 | stream.head() === Symbol.for("eof") 40 | ? new Success(stream.head(), stream.move(1)) 41 | : new Failure("not at EOF", stream)); 42 | } 43 | 44 | export function lookahead(parser) { 45 | return new Parser(stream => 46 | parser 47 | .run(stream) 48 | .fold( 49 | value => new Success(value, stream), 50 | value => new Failure(value, stream))); 51 | } 52 | 53 | export function not(parser) { 54 | return new Parser(stream => 55 | parser 56 | .run(stream) 57 | .fold( 58 | (value, tail) => new Failure("not failed", stream), 59 | (value, tail) => new Success(null, stream))); 60 | } 61 | 62 | export function and(...parsers) { 63 | const final = parsers.pop(); 64 | return sequence(...parsers.map(lookahead), final) 65 | .map(results => results[results.length - 1]); 66 | } 67 | 68 | export function either(...parsers) { 69 | return new Parser(stream => { 70 | for (const parser of parsers) { 71 | const result = parser.run(stream); 72 | if (result instanceof Success) { 73 | return result; 74 | } 75 | } 76 | return new Failure("either failed", stream); 77 | }); 78 | } 79 | 80 | export function always(value) { 81 | return new Parser(stream => new Success(value, stream)); 82 | } 83 | 84 | export function never(value) { 85 | return new Parser(stream => new Failure(value, stream)); 86 | } 87 | 88 | export function maybe(parser) { 89 | return new Parser(stream => 90 | parser 91 | .run(stream) 92 | .fold( 93 | (value, tail) => new Success(value, tail), 94 | (value, tail) => new Success(null, stream))); 95 | } 96 | 97 | export function append(p1, p2) { 98 | return p1.chain(values => 99 | p2.map(value => values.concat([value]))); 100 | } 101 | 102 | export function after(prefix, parser) { 103 | return sequence(prefix, parser) 104 | .map(([left, value]) => value); 105 | } 106 | 107 | export function sequence(...parsers) { 108 | return parsers.reduce( 109 | (acc, parser) => append(acc, parser), 110 | always([])); 111 | } 112 | 113 | // Test combinator to produce an empty list instead of 114 | // repeat or repeat1. 115 | export function repeat0(parser) { 116 | return new Parser(stream => new Success([], stream)); 117 | } 118 | 119 | export function repeat(parser) { 120 | return new Parser(stream => 121 | parser 122 | .run(stream) 123 | .fold( 124 | (value, tail) => 125 | repeat(parser) 126 | .map(rest => [value].concat(rest)) 127 | .run(tail), 128 | (value, tail) => new Success([], stream))); 129 | } 130 | 131 | export function repeat1(parser) { 132 | return new Parser(stream => 133 | parser 134 | .run(stream) 135 | .fold( 136 | (value, tail) => 137 | repeat(parser) 138 | .map(rest => [value].concat(rest)) 139 | .run(tail), 140 | (value, tail) => new Failure("repeat1 failed", stream))); 141 | } 142 | -------------------------------------------------------------------------------- /lib/ebnf.js: -------------------------------------------------------------------------------- 1 | import {parse} from "babylon"; 2 | import walk from "./walker.js"; 3 | import visitor from "./visitor.js"; 4 | import serialize from "./serializer.js"; 5 | 6 | export default 7 | function ebnf(source, min_name_length = 0) { 8 | let grammar_ast = parse(source, {sourceType: "module"}); 9 | let rules = walk(grammar_ast, visitor); 10 | let state = { 11 | max_name_length: Math.max( 12 | min_name_length, 13 | ...rules.map(rule => rule.name.length)), 14 | }; 15 | return rules 16 | .map(rule => serialize(rule, state)) 17 | .join(""); 18 | } 19 | -------------------------------------------------------------------------------- /lib/mappers.js: -------------------------------------------------------------------------------- 1 | import {Abstract} from "./parser.js"; 2 | 3 | // Flatten a list up to a given depth. 4 | // This is useful when a parser uses nested sequences and repeats. 5 | export const flatten = depth => 6 | list => list.reduce( 7 | (acc, cur) => acc.concat( 8 | !Array.isArray(cur) || depth === 1 9 | ? cur 10 | : flatten(depth - 1)(cur)), 11 | []); 12 | 13 | // Mutate an object by merging properties of another object into it. 14 | export const mutate = state => 15 | obj => Object.assign(obj, state); 16 | 17 | // Join the list of parsed values into a string. 18 | export const join = list => 19 | list 20 | .filter(value => value !== Symbol.for("eof")) 21 | .join(""); 22 | 23 | // Prune unmatched maybes from a list. 24 | export const prune = list => 25 | list.filter(value => value !== null); 26 | 27 | // Map a list of {name, value} aliases into an array of values. 28 | export const keep_abstract = list => 29 | list 30 | .filter(value => value instanceof Abstract) 31 | .map(({value}) => value); 32 | 33 | // Map a list to the element at the specified index. Useful for parsers which 34 | // define a prefix or a surrounding delimiter. 35 | export const element_at = index => list => list[index]; 36 | 37 | // Print the parse result of a parser. 38 | export const print = x => 39 | (console.log(JSON.stringify(x, null, 4)), x); 40 | -------------------------------------------------------------------------------- /lib/parser.js: -------------------------------------------------------------------------------- 1 | import Stream from "./stream.js"; 2 | import {Failure} from "./result.js"; 3 | 4 | export class Abstract { 5 | constructor(value) { 6 | this.value = value; 7 | } 8 | } 9 | 10 | export default class Parser { 11 | constructor(parse) { 12 | this.parse = parse; 13 | } 14 | 15 | run(iterable) { 16 | let stream = iterable instanceof Stream 17 | ? iterable 18 | : new Stream(iterable); 19 | return this.parse(stream); 20 | } 21 | 22 | get abstract() { 23 | return this.map(value => new Abstract(value)); 24 | } 25 | 26 | map(f) { 27 | return new Parser(stream => this.run(stream).map(f)); 28 | } 29 | 30 | bimap(s, f) { 31 | return new Parser(stream => this.run(stream).bimap(s, f)); 32 | } 33 | 34 | chain(f) { 35 | return new Parser(stream => 36 | this.run(stream).chain( 37 | (value, tail) => f(value).run(tail))); 38 | } 39 | 40 | fold(s, f) { 41 | return new Parser(stream => this.run(stream).fold(s, f)); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /lib/result.js: -------------------------------------------------------------------------------- 1 | class Result { 2 | constructor(value, rest) { 3 | this.value = value; 4 | this.rest = rest; 5 | } 6 | } 7 | 8 | export class Success extends Result { 9 | map(fn) { 10 | return new Success(fn(this.value), this.rest); 11 | } 12 | bimap(s, f) { 13 | return new Success(s(this.value), this.rest); 14 | } 15 | chain(fn) { 16 | return fn(this.value, this.rest); 17 | } 18 | fold(s, f) { 19 | return s(this.value, this.rest); 20 | } 21 | } 22 | 23 | export class Failure extends Result { 24 | map(fn) { 25 | return this; 26 | } 27 | bimap(s, f) { 28 | return new Failure(f(this.value), this.rest); 29 | } 30 | chain(fn) { 31 | return this; 32 | } 33 | fold(s, f) { 34 | return f(this.value, this.rest); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /lib/serializer.js: -------------------------------------------------------------------------------- 1 | export default 2 | function serialize_rule(rule, state) { 3 | let {name, expression, block_comments} = rule; 4 | let lhs = name.padEnd(state.max_name_length); 5 | let rhs = serialize_expression(expression, state); 6 | let comment = serialize_comments(block_comments); 7 | return `${comment}${lhs} ::= ${rhs}\n`; 8 | } 9 | 10 | function serialize_comments(comments) { 11 | let contents = []; 12 | for (let {value} of comments) { 13 | if (/^[ =*/-]*$/.test(value)) { 14 | contents.push(""); 15 | } else { 16 | contents.push(`/*${value}*/`); 17 | } 18 | } 19 | return contents.length 20 | ? contents.join("\n") + "\n" 21 | : ""; 22 | } 23 | 24 | function serialize_expression(expression, state) { 25 | switch (expression.type) { 26 | case "Symbol": 27 | return expression.name; 28 | case "Terminal": 29 | return expression.value; 30 | case "Operator": 31 | return serialize_operator(expression, state); 32 | } 33 | } 34 | 35 | function serialize_operator({name, args}, state) { 36 | let serialized_args = args.map(arg => 37 | serialize_expression(arg, {...state, parent: name})); 38 | 39 | function ensure_prec(text) { 40 | return state.parent ? `(${text})` : text; 41 | } 42 | 43 | switch (name) { 44 | case "always": { 45 | return null; 46 | } 47 | case "and": { 48 | return ensure_prec( 49 | serialized_args.reverse().join(" ")); 50 | } 51 | case "char": { 52 | return `"${serialized_args[0]}"`; 53 | } 54 | case "charset": { 55 | return `[${serialized_args[0]}]`; 56 | } 57 | case "either": { 58 | if (state.parent) { 59 | return `(${serialized_args.join(" | ")})`; 60 | } 61 | 62 | // Add 5 to align with " ::= ". 63 | let padding = state.max_name_length + 5; 64 | return serialized_args.join("\n" + " | ".padStart(padding)); 65 | } 66 | case "eof": { 67 | return "EOF"; 68 | } 69 | case "maybe": { 70 | return `${serialized_args[0]}?`; 71 | } 72 | case "never": { 73 | return null; 74 | } 75 | case "not": { 76 | return `- ${serialized_args[0]}`; 77 | } 78 | case "regex": { 79 | return `/${serialized_args[0]}/`; 80 | } 81 | case "repeat": { 82 | return `${serialized_args[0]}*`; 83 | } 84 | case "repeat1": { 85 | return `${serialized_args[0]}+`; 86 | } 87 | case "sequence": { 88 | return ensure_prec( 89 | serialized_args.filter(Boolean).join(" ")); 90 | } 91 | case "string": { 92 | return `"${serialized_args[0]}"`; 93 | } 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /lib/stream.js: -------------------------------------------------------------------------------- 1 | export default class Stream { 2 | constructor(iterable, cursor, length) { 3 | this.iterable = iterable; 4 | this.cursor = cursor || 0; 5 | this.length = length === undefined 6 | ? iterable.length - this.cursor 7 | : length; 8 | } 9 | 10 | // Get the element at the cursor. 11 | head(len = 1) { 12 | if (this.length < 0) { 13 | return undefined; 14 | } 15 | 16 | if (this.length === 0) { 17 | return Symbol.for("eof"); 18 | } 19 | 20 | return this.iterable.slice(this.cursor, this.cursor + len); 21 | } 22 | 23 | // Execute a regex on the iterable. 24 | exec(re) { 25 | // The "u" flag is a feature of ES2015 which makes regexes Unicode-aware. 26 | // See https://mathiasbynens.be/notes/es6-unicode-regex. 27 | // The "y" flag makes the regex sticky. The match must start at the 28 | // offset specified by the regex's lastIndex property. 29 | let sticky = new RegExp(re, "uy"); 30 | sticky.lastIndex = this.cursor; 31 | return sticky.exec(this.iterable); 32 | } 33 | 34 | // Consume the stream by moving the cursor. 35 | move(distance) { 36 | return new Stream( 37 | this.iterable, 38 | this.cursor + distance, 39 | this.length - distance); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /lib/visitor.js: -------------------------------------------------------------------------------- 1 | export default { 2 | File(node, state, cont) { 3 | return cont(node.program, state); 4 | }, 5 | Program(node, state, cont) { 6 | return node.body 7 | .map(statement => cont(statement, state)) 8 | .filter(production => production !== undefined); 9 | }, 10 | ExportNamedDeclaration(node, state, cont) { 11 | let {declaration, leadingComments} = node; 12 | let comments = leadingComments && { 13 | block_comments: leadingComments 14 | .filter(comm => comm.type === "CommentBlock") 15 | .map(comm => cont(comm, state)) 16 | }; 17 | return cont(declaration, {...state, ...comments}); 18 | }, 19 | VariableDeclaration(node, state, cont) { 20 | let {declarations, leadingComments} = node; 21 | let [declaration] = declarations; 22 | let comments = leadingComments && { 23 | block_comments: leadingComments 24 | .filter(comm => comm.type === "CommentBlock") 25 | .map(comm => cont(comm, state)) 26 | }; 27 | return cont(declaration, {...state, ...comments}); 28 | }, 29 | VariableDeclarator(node, state, cont) { 30 | let {id: {name}, init} = node; 31 | let {block_comments = []} = state; 32 | let expression = cont(init, state); 33 | return {type: "Rule", name, expression, block_comments}; 34 | }, 35 | CallExpression(node, state, cont) { 36 | let {callee, arguments: args} = node; 37 | switch (callee.type) { 38 | case "MemberExpression": 39 | return cont(callee.object, state); 40 | case "Identifier": { 41 | let {name} = callee; 42 | // defer(() => parser) is used to avoid cyclic dependencies. 43 | if (name === "defer") { 44 | let [arrow_fn] = args; 45 | return cont(arrow_fn.body, state); 46 | } 47 | 48 | // Don't recurse into always() and never(). They don't parse 49 | // the input and are only used for convenient AST building. 50 | if (name === "always" || name === "never") { 51 | return {type: "Operator", name, args: []}; 52 | } 53 | 54 | return { 55 | type: "Operator", 56 | name, 57 | args: args.map(arg => cont(arg, state)), 58 | }; 59 | } 60 | } 61 | }, 62 | MemberExpression(node, state, cont) { 63 | return cont(node.object, state); 64 | }, 65 | Identifier(node, state, cont) { 66 | let {name} = node; 67 | return {type: "Symbol", name}; 68 | }, 69 | StringLiteral({value}, state, cont) { 70 | return {type: "Terminal", value: escape(value)}; 71 | }, 72 | RegExpLiteral({pattern}, state, cont) { 73 | return {type: "Terminal", value: pattern}; 74 | }, 75 | CommentBlock({value}, state, cont) { 76 | return {type: "Comment", value}; 77 | }, 78 | }; 79 | 80 | function escape(str) { 81 | return str 82 | // Escape backslash and double quote, which are special in EBNF. 83 | .replace(/\\/g, "\\\\") 84 | .replace(/"/g, "\\\"") 85 | // Replace all Control and non-Basic Latin characters. 86 | .replace(/([^\u0021-\u007E])/g, unicode_sequence); 87 | } 88 | 89 | function unicode_sequence(char) { 90 | let code_point = char.codePointAt(0).toString(16); 91 | return `\\u${code_point.toUpperCase().padStart(4, "0")}`; 92 | } 93 | -------------------------------------------------------------------------------- /lib/walker.js: -------------------------------------------------------------------------------- 1 | export default 2 | function walk(node, visitor, state) { 3 | function cont(node, state) { 4 | let visit = visitor[node.type]; 5 | if (visit) { 6 | return visit(node, state, cont); 7 | } 8 | } 9 | 10 | return cont(node, state); 11 | } 12 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "fluent-spec", 3 | "description": "Specification and documentation for Fluent", 4 | "version": "1.0.0", 5 | "private": true, 6 | "type": "module", 7 | "scripts": { 8 | "bench": "node ./test/bench.js ./test/benchmarks/gecko_strings.ftl", 9 | "build:guide": "mdbook build", 10 | "clean": "rm -rf build", 11 | "generate:ebnf": "node bin/ebnf.js ./syntax/grammar.js > ./spec/fluent.ebnf", 12 | "generate:fixtures": "make -sC test/fixtures", 13 | "generate": "npm run generate:ebnf && npm run generate:fixtures", 14 | "lint": "eslint **/*.js", 15 | "test:ebnf": "node test/ebnf.js ./syntax/grammar.js ./spec/fluent.ebnf", 16 | "test:fixtures": "node test/parser.js ./test/fixtures", 17 | "test:validate": "node test/validator.js ./test/fixtures", 18 | "test:unit": "node test/literals.js", 19 | "test": "npm run test:fixtures && npm run test:unit" 20 | }, 21 | "homepage": "https://projectfluent.org", 22 | "repository": { 23 | "type": "git", 24 | "url": "git+https://github.com/projectfluent/fluent.git" 25 | }, 26 | "author": "Mozilla ", 27 | "license": "Apache-2.0", 28 | "bugs": { 29 | "url": "https://github.com/projectfluent/fluent/issues" 30 | }, 31 | "engines": { 32 | "node": ">=12.0.0" 33 | }, 34 | "devDependencies": { 35 | "@fluent/bundle": "^0.18.0", 36 | "@fluent/syntax": "^0.19.0", 37 | "babylon": "^6.18.0", 38 | "cli-color": "^2.0.0", 39 | "difflib": "^0.2.4", 40 | "eslint": "^8.57.0", 41 | "json-diff": "^1.0.6" 42 | }, 43 | "dependencies": { 44 | "minimist": "^1.2.3" 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /spec/README.md: -------------------------------------------------------------------------------- 1 | # Fluent Syntax (FTL) 2 | 3 | Specification, design and documentation for FTL, the syntax for describing 4 | translation resources in Project Fluent. FTL stands for *Fluent Translation 5 | List*. 6 | 7 | The grammar is defined in `fluent.ebnf` using [W3C's EBNF][]. 8 | 9 | Read the [Fluent Syntax Guide][]. 10 | 11 | [Fluent Syntax Guide]: http://projectfluent.org/fluent/guide 12 | [W3C's EBNF]: https://www.w3.org/TR/REC-xml/#sec-notation 13 | -------------------------------------------------------------------------------- /spec/compatibility.md: -------------------------------------------------------------------------------- 1 | # Fluent Syntax Compatibility 2 | 3 | The goal of this document is to establish a shared vision for Fluent’s 4 | compatibility strategy after the 1.0 release. The vision will serve two 5 | primary purposes: 6 | 7 | - (External) Manage early-adopters’ expectations of Fluent Syntax and 8 | tooling. 9 | 10 | - (Internal) Contextualize feedback and inform design decisions during the 11 | 1.x lifecycle and beyond. 12 | 13 | 14 | ## Types of Changes 15 | 16 | _Backwards incompatible_ changes make old files not parse in the current 17 | parsers. Removing a feature which was previously available and reporting 18 | a `SyntaxError` instead is a backwards incompatible change. 19 | 20 | _Extensions_ are changes which introduce syntax which was previously forbidden. 21 | Extensions make current files with the extended syntax not parse in old 22 | parsers. Old files continue to parse correctly in the current parsers. 23 | 24 | _Deprecations_ are non-breaking changes meant to encourage the use of more 25 | modern syntax and guarantee a transition period for migrating translations to 26 | the modern syntax. Deprecations herald future removals of syntax features. 27 | 28 | 29 | ## The 1.x 30 | 31 | In the 1.x lifetime, there won't be any breaking changes to Fluent Syntax. 32 | The grammar and the AST must maintain backwards compatibility with 1.0. 33 | 34 | The dot releases may introduce new syntax in a backwards compatible way. New 35 | features are only allowed as extensions to the 1.0 syntax and the AST. 36 | 37 | Extensions may be accompanied by deprecations. Deprecated syntax will continue 38 | to be supported throughout the 1.x lifetime. Parsers will report deprecations 39 | as annotations in the AST. 40 | 41 | Authoring tools should provide per-project settings for specifying the lower 42 | and upper bounds of accepted syntax versions, depending on the currently-used 43 | Fluent implementation in the target project. Syntax newer than the upper bound 44 | must not be allowed in the project's translations; syntax deprecated in 45 | versions older than the lower bound should be reported as warnings, and later 46 | on, as errors—depending on the grace period chosen for the project. 47 | 48 | 49 | ## The 2.0 50 | 51 | In order to clean up the accrued deprecations, a 2.0 version may be published 52 | in the future, breaking the backward compatibility with 1.x. The 2.0 version 53 | will define the compatibility strategy for Fluent 2.x and beyond. 54 | 55 | Removing deprecations will achieve the following goals. 56 | 57 | - Establish a fresh baseline for subsequent Fluent 2.x releases and the 58 | future standard. 59 | 60 | - Improve the learnability of Fluent by reducing the cognitive complexity of 61 | the syntax. 62 | 63 | - Allow implementations to streamline their codebases, which can increase 64 | their quality and give opportunities for new performance optimizations. 65 | 66 | If Fluent succeeds as a future standard, the compatibility strategy will be 67 | redefined by the standards organization publishing it. 68 | -------------------------------------------------------------------------------- /spec/errors.md: -------------------------------------------------------------------------------- 1 | One of the core principles of Fluent is to provide good basis for excellent error fallback and error reporting. To this end, the AST parser needs to return all the essential information about errors which can then be used to create helpful error messages and hints. 2 | 3 | Below is a description of the data model for the AST errors as implemented in https://github.com/projectfluent/fluent.js/pull/7. 4 | 5 | ## API 6 | 7 | The `fluent-syntax` package would expose the `parse` method which takes a source string and returns an instance of the `Resource` node. 8 | 9 | ```javascript 10 | import { parse } from 'fluent-syntax'; 11 | const resource = parse(source_string); 12 | ``` 13 | 14 | ## Resource 15 | 16 | The AST node `Resource` is described as follows: 17 | 18 | ```diff 19 | export class Resource extends Node { 20 | - constructor(body = [], comment = null) { 21 | - super(body, comment); 22 | + constructor(body = [], comment = null, source = '') { 23 | + super(); 24 | this.type = 'Resource'; 25 | this.body = body; 26 | this.comment = comment; 27 | + this.source = source; 28 | } 29 | } 30 | ``` 31 | 32 | ## Entries 33 | 34 | `Entry` instances now have a `span: { from, to }` field. `from` and `to` are positions relative to the start of the file. 35 | 36 | ```json 37 | { 38 | "type": "Message", 39 | "annotations": [], 40 | "id": { 41 | "type": "Identifier", 42 | "name": "hello-world" 43 | }, 44 | "value": { 45 | "type": "Pattern", 46 | "elements": [ 47 | { 48 | "type": "StringExpression", 49 | "value": "Hello, world!" 50 | } 51 | ], 52 | "quoted": false 53 | }, 54 | "attributes": null, 55 | "comment": null, 56 | "span": { 57 | "from": 44, 58 | "to": 71 59 | } 60 | }, 61 | ``` 62 | 63 | ## Annotations 64 | 65 | Parsing errors are now collected into the `annotations` field of `Entry` instances. In the future, we can add warnings and notes there, too. We can also consider adding another field, `labels`, which provide more information about the error: the reason why it happened and a suggestion about how to fix it. 66 | 67 | ```json 68 | { 69 | "type": "Junk", 70 | "annotations": [ 71 | { 72 | "message": "Missing field", 73 | "name": "ParseError", 74 | "pos": 85 75 | } 76 | ], 77 | "content": "hello-world Hello, world!\n\n", 78 | "span": { 79 | "from": 73, 80 | "to": 100 81 | } 82 | }, 83 | ``` 84 | 85 | ## Positions 86 | 87 | `fluent-syntax` exposes two helpers: `lineOffset(source, pos)` and `columnOffset(source, pos)`, both returning 0-based offset numbers. They may be used to place gutter markers in editors or to calculate the number of context lines to show around an error. 88 | 89 | -------------------------------------------------------------------------------- /spec/fluent.ebnf: -------------------------------------------------------------------------------- 1 | 2 | /* An FTL file defines a Resource consisting of Entries. */ 3 | Resource ::= (Entry | blank_block | Junk)* 4 | 5 | /* Entries are the main building blocks of Fluent. They define translations and 6 | * contextual and semantic information about the translations. During the AST 7 | * construction, adjacent comment lines of the same comment type (defined by 8 | * the number of #) are joined together. Single-# comments directly preceding 9 | * Messages and Terms are attached to the Message or Term and are not 10 | * standalone Entries. */ 11 | Entry ::= (Message line_end) 12 | | (Term line_end) 13 | | CommentLine 14 | Message ::= Identifier blank_inline? "=" blank_inline? ((Pattern Attribute*) | (Attribute+)) 15 | Term ::= "-" Identifier blank_inline? "=" blank_inline? Pattern Attribute* 16 | 17 | /* Adjacent comment lines of the same comment type are joined together during 18 | * the AST construction. */ 19 | CommentLine ::= ("###" | "##" | "#") ("\u0020" comment_char*)? line_end 20 | comment_char ::= any_char - line_end 21 | 22 | /* Junk represents unparsed content. 23 | * 24 | * Junk is parsed line-by-line until a line is found which looks like it might 25 | * be a beginning of a new message, term, or a comment. Any whitespace 26 | * following a broken Entry is also considered part of Junk. 27 | */ 28 | Junk ::= junk_line (junk_line - "#" - "-" - [a-zA-Z])* 29 | junk_line ::= /[^\n]*/ ("\u000A" | EOF) 30 | 31 | /* Attributes of Messages and Terms. */ 32 | Attribute ::= line_end blank? "." Identifier blank_inline? "=" blank_inline? Pattern 33 | 34 | /* Patterns are values of Messages, Terms, Attributes and Variants. */ 35 | Pattern ::= PatternElement+ 36 | 37 | /* TextElement and Placeable can occur inline or as block. 38 | * Text needs to be indented and start with a non-special character. 39 | * Placeables can start at the beginning of the line or be indented. 40 | * Adjacent TextElements are joined in AST creation. */ 41 | PatternElement ::= inline_text 42 | | block_text 43 | | inline_placeable 44 | | block_placeable 45 | inline_text ::= text_char+ 46 | block_text ::= blank_block blank_inline indented_char inline_text? 47 | inline_placeable ::= "{" blank? (SelectExpression | InlineExpression) blank? "}" 48 | block_placeable ::= blank_block blank_inline? inline_placeable 49 | 50 | /* Rules for validating expressions in Placeables and as selectors of 51 | * SelectExpressions are documented in spec/valid.md and enforced in 52 | * syntax/abstract.js. */ 53 | InlineExpression ::= StringLiteral 54 | | NumberLiteral 55 | | FunctionReference 56 | | MessageReference 57 | | TermReference 58 | | VariableReference 59 | | inline_placeable 60 | 61 | /* Literals */ 62 | StringLiteral ::= "\"" quoted_char* "\"" 63 | NumberLiteral ::= "-"? digits ("." digits)? 64 | 65 | /* Inline Expressions */ 66 | FunctionReference ::= Identifier CallArguments 67 | MessageReference ::= Identifier AttributeAccessor? 68 | TermReference ::= "-" Identifier AttributeAccessor? CallArguments? 69 | VariableReference ::= "$" Identifier 70 | AttributeAccessor ::= "." Identifier 71 | CallArguments ::= blank? "(" blank? argument_list blank? ")" 72 | argument_list ::= (Argument blank? "," blank?)* Argument? 73 | Argument ::= NamedArgument 74 | | InlineExpression 75 | NamedArgument ::= Identifier blank? ":" blank? (StringLiteral | NumberLiteral) 76 | 77 | /* Block Expressions */ 78 | SelectExpression ::= InlineExpression blank? "->" blank_inline? variant_list 79 | variant_list ::= Variant* DefaultVariant Variant* line_end 80 | Variant ::= line_end blank? VariantKey blank_inline? Pattern 81 | DefaultVariant ::= line_end blank? "*" VariantKey blank_inline? Pattern 82 | VariantKey ::= "[" blank? (NumberLiteral | Identifier) blank? "]" 83 | 84 | /* Identifier */ 85 | Identifier ::= [a-zA-Z] [a-zA-Z0-9_-]* 86 | 87 | /* Content Characters 88 | * 89 | * Translation content can be written using any Unicode characters. However, 90 | * some characters are considered special depending on the type of content 91 | * they're in. See text_char and quoted_char for more information. 92 | * 93 | * Some Unicode characters, even if allowed, should be avoided in Fluent 94 | * resources. See spec/recommendations.md. 95 | */ 96 | any_char ::= [\\u{0}-\\u{10FFFF}] 97 | 98 | /* Text elements 99 | * 100 | * The primary storage for content are text elements. Text elements are not 101 | * delimited with quotes and may span multiple lines as long as all lines are 102 | * indented. The opening brace ({) marks a start of a placeable in the pattern 103 | * and may not be used in text elements verbatim. Due to the indentation 104 | * requirement some text characters may not appear as the first character on a 105 | * new line. 106 | */ 107 | special_text_char ::= "{" 108 | | "}" 109 | text_char ::= any_char - special_text_char - line_end 110 | indented_char ::= text_char - "[" - "*" - "." 111 | 112 | /* String literals 113 | * 114 | * For special-purpose content, quoted string literals can be used where text 115 | * elements are not a good fit. String literals are delimited with double 116 | * quotes and may not contain line breaks. String literals use the backslash 117 | * (\) as the escape character. The literal double quote can be inserted via 118 | * the \" escape sequence. The literal backslash can be inserted with \\. The 119 | * literal opening brace ({) is allowed in string literals because they may not 120 | * comprise placeables. 121 | */ 122 | special_quoted_char ::= "\"" 123 | | "\\" 124 | special_escape ::= "\\" special_quoted_char 125 | unicode_escape ::= ("\\u" /[0-9a-fA-F]{4}/) 126 | | ("\\U" /[0-9a-fA-F]{6}/) 127 | quoted_char ::= (any_char - special_quoted_char - line_end) 128 | | special_escape 129 | | unicode_escape 130 | 131 | /* Numbers */ 132 | digits ::= [0-9]+ 133 | 134 | /* Whitespace */ 135 | blank_inline ::= "\u0020"+ 136 | line_end ::= "\u000D\u000A" 137 | | "\u000A" 138 | | EOF 139 | blank_block ::= (blank_inline? line_end)+ 140 | blank ::= (blank_inline | line_end)+ 141 | -------------------------------------------------------------------------------- /spec/recommendations.md: -------------------------------------------------------------------------------- 1 | # Recommendations for Writing Fluent 2 | 3 | ## Unicode Characters 4 | 5 | Fluent resources can be written using all Unicode characters. The recommended 6 | encoding for Fluent files is UTF-8. 7 | 8 | Translation authors and developers are encouraged to avoid characters defined 9 | in the following code point ranges. They are either control characters or 10 | permanently undefined Unicode characters: 11 | 12 | [U+0000-U+0008], [U+000B-U+000C], [U+000E-U+001F], [U+007F-U+009F], 13 | [U+FDD0-U+FDEF], [U+1FFFE-U+1FFFF], [U+2FFFE-U+2FFFF], [U+3FFFE-U+3FFFF], 14 | [U+4FFFE-U+4FFFF], [U+5FFFE-U+5FFFF], [U+6FFFE-U+6FFFF], [U+7FFFE-U+7FFFF], 15 | [U+8FFFE-U+8FFFF], [U+9FFFE-U+9FFFF], [U+AFFFE-U+AFFFF], [U+BFFFE-U+BFFFF], 16 | [U+CFFFE-U+CFFFF], [U+DFFFE-U+DFFFF], [U+EFFFE-U+EFFFF], [U+FFFFE-U+FFFFF], 17 | [U+10FFFE-U+10FFFF]. 18 | -------------------------------------------------------------------------------- /spec/valid.md: -------------------------------------------------------------------------------- 1 | # Valid Fluent Syntax 2 | 3 | Fluent Syntax distinguishes between well-formed and valid resources. 4 | 5 | - Well-formed Fluent resources conform to the Fluent grammar described by 6 | the Fluent EBNF (`spec/fluent.ebnf`). The EBNF is auto-generated from 7 | `syntax/grammar.js`. 8 | 9 | - Valid Fluent resources must be well-formed and are additionally checked 10 | for semantic correctness. The validation process may reject syntax which is 11 | well-formed. The validation rules are expressed in code in 12 | `syntax/abstract.js`. 13 | 14 | For example, the `message.attr(param: "value")` syntax is _well-formed_. 15 | `message.attr` is an `AttributeExpression` which may be the callee of a 16 | `CallExpression`. However, this syntax is not _valid_. Message attributes, 17 | just like message values, cannot be parameterized the way terms can. 18 | 19 | The distinction between well-formed and valid syntax allows us to keep 20 | Fluent's EBNF simple. More complex rules are enforced in the validation step. 21 | Some implementations might choose to support well-formed Fluent resources and 22 | skip some validation for performance or other reasons. If they do so, it's 23 | recommended that the affected Fluent resources be validated outside of these 24 | implementations, for instance as part of a build-time or compile-time step. 25 | -------------------------------------------------------------------------------- /syntax/abstract.js: -------------------------------------------------------------------------------- 1 | /* 2 | * AST Validation 3 | * 4 | * The parse result of the grammar.js parser is a well-formed AST which is 5 | * validated according to the rules documented in `spec/valid.md`. 6 | */ 7 | 8 | import * as FTL from "./ast.js"; 9 | import {always, never} from "../lib/combinators.js"; 10 | 11 | export function list_into(Type) { 12 | switch (Type) { 13 | case FTL.Comment: 14 | return ([sigil, content = ""]) => { 15 | switch (sigil) { 16 | case "#": 17 | return always(new FTL.Comment(content)); 18 | case "##": 19 | return always(new FTL.GroupComment(content)); 20 | case "###": 21 | return always(new FTL.ResourceComment(content)); 22 | default: 23 | return never(`Unknown comment sigil: ${sigil}.`); 24 | } 25 | }; 26 | case FTL.FunctionReference: 27 | const VALID_FUNCTION_NAME = /^[A-Z][A-Z0-9_-]*$/; 28 | return ([identifier, args]) => { 29 | if (VALID_FUNCTION_NAME.test(identifier.name)) { 30 | return always(new Type(identifier, args)); 31 | } 32 | return never( 33 | `Invalid function name: ${identifier.name}. ` 34 | + "Function names must be all upper-case ASCII letters."); 35 | }; 36 | case FTL.Pattern: 37 | return elements => 38 | always(new FTL.Pattern( 39 | dedent(elements) 40 | .reduce(join_adjacent(FTL.TextElement), []) 41 | .map(trim_text_at_extremes) 42 | .filter(remove_empty_text))); 43 | case FTL.Resource: 44 | return entries => 45 | always(new FTL.Resource( 46 | entries 47 | .reduce(join_adjacent( 48 | FTL.Comment, 49 | FTL.GroupComment, 50 | FTL.ResourceComment), []) 51 | .reduce(attach_comments, []) 52 | .filter(remove_blank_lines))); 53 | case FTL.SelectExpression: 54 | return ([selector, variants]) => { 55 | let selector_is_valid = 56 | selector instanceof FTL.StringLiteral 57 | || selector instanceof FTL.NumberLiteral 58 | || selector instanceof FTL.VariableReference 59 | || selector instanceof FTL.FunctionReference 60 | || (selector instanceof FTL.TermReference 61 | && selector.attribute); 62 | if (!selector_is_valid) { 63 | return never(`Invalid selector type: ${selector.type}.`); 64 | } 65 | 66 | return always(new Type(selector, variants)); 67 | }; 68 | default: 69 | return elements => 70 | always(new Type(...elements)); 71 | } 72 | } 73 | 74 | export function into(Type) { 75 | switch (Type) { 76 | case FTL.CallArguments: 77 | return args => { 78 | let positional = []; 79 | let named = new Map(); 80 | for (let arg of args) { 81 | if (arg instanceof FTL.NamedArgument) { 82 | let name = arg.name.name; 83 | if (named.has(name)) { 84 | return never("Named arguments must be unique."); 85 | } 86 | named.set(name, arg); 87 | } else if (named.size > 0) { 88 | return never("Positional arguments must not follow " 89 | + "named arguments"); 90 | } else { 91 | positional.push(arg); 92 | } 93 | } 94 | return always(new Type( 95 | positional, Array.from(named.values()))); 96 | }; 97 | case FTL.Placeable: 98 | return expression => { 99 | if (expression instanceof FTL.TermReference 100 | && expression.attribute) { 101 | return never( 102 | "Term attributes may not be used as placeables."); 103 | } 104 | return always(new Type(expression)); 105 | }; 106 | default: 107 | return (...args) => 108 | always(new Type(...args)); 109 | } 110 | } 111 | 112 | // Create a reducer suitable for joining adjacent nodes of the same type, if 113 | // type is one of types specified. 114 | function join_adjacent(...types) { 115 | return function(acc, cur) { 116 | let prev = acc[acc.length - 1]; 117 | for (let Type of types) { 118 | if (prev instanceof Type && cur instanceof Type) { 119 | // Replace prev with a new node of the same type whose value is 120 | // the sum of prev and cur, and discard cur. 121 | acc[acc.length - 1] = join_of_type(Type, prev, cur); 122 | return acc; 123 | } 124 | } 125 | return acc.concat(cur); 126 | }; 127 | } 128 | 129 | // Join values of two or more nodes of the same type. Return a new node. 130 | function join_of_type(Type, ...elements) { 131 | // TODO Join annotations and spans. 132 | switch (Type) { 133 | case FTL.TextElement: 134 | return elements.reduce((a, b) => 135 | new Type(a.value + b.value)); 136 | case FTL.Comment: 137 | case FTL.GroupComment: 138 | case FTL.ResourceComment: 139 | return elements.reduce((a, b) => 140 | new Type(a.content + "\n" + b.content)); 141 | } 142 | } 143 | 144 | function attach_comments(acc, cur) { 145 | let prev = acc[acc.length - 1]; 146 | if (prev instanceof FTL.Comment 147 | && (cur instanceof FTL.Message 148 | || cur instanceof FTL.Term)) { 149 | cur.comment = prev; 150 | acc[acc.length - 1] = cur; 151 | return acc; 152 | } else { 153 | return acc.concat(cur); 154 | } 155 | } 156 | 157 | // Remove the largest common indentation from a list of elements of a Pattern. 158 | // The indents are parsed in grammar.js and passed to abstract.js as string 159 | // primitives along with other PatternElements. 160 | function dedent(elements) { 161 | // Calculate the maximum common indent. 162 | let indents = elements.filter(element => typeof element === "string"); 163 | let common = Math.min(...indents.map(indent => indent.length)); 164 | 165 | function trim_indents(element) { 166 | if (typeof element === "string") { 167 | // Trim the indent and convert it to a proper TextElement. 168 | // It will be joined with its adjacents later on. 169 | return new FTL.TextElement(element.slice(common)); 170 | } 171 | return element; 172 | } 173 | 174 | return elements.map(trim_indents); 175 | } 176 | 177 | const LEADING_BLANK_BLOCK = /^\n*/; 178 | const TRAILING_BLANK_INLINE = / *$/; 179 | 180 | function trim_text_at_extremes(element, index, array) { 181 | if (element instanceof FTL.TextElement) { 182 | if (index === 0) { 183 | element.value = element.value.replace( 184 | LEADING_BLANK_BLOCK, ""); 185 | } 186 | if (index === array.length - 1) { 187 | element.value = element.value.replace( 188 | TRAILING_BLANK_INLINE, ""); 189 | } 190 | } 191 | return element; 192 | } 193 | 194 | function remove_empty_text(element) { 195 | return !(element instanceof FTL.TextElement) 196 | || element.value !== ""; 197 | } 198 | 199 | function remove_blank_lines(element) { 200 | return typeof(element) !== "string"; 201 | } 202 | -------------------------------------------------------------------------------- /syntax/ast.js: -------------------------------------------------------------------------------- 1 | // Base class for all Fluent AST nodes. 2 | export class BaseNode { 3 | constructor() {} 4 | } 5 | 6 | // Base class for AST nodes which can have Spans. 7 | export class SyntaxNode extends BaseNode { 8 | addSpan(start, end) { 9 | this.span = new Span(start, end); 10 | } 11 | } 12 | 13 | export class Resource extends SyntaxNode { 14 | constructor(body = []) { 15 | super(); 16 | this.type = "Resource"; 17 | this.body = body; 18 | } 19 | } 20 | 21 | // An abstract base class for useful elements of Resource.body. 22 | export class Entry extends SyntaxNode {} 23 | 24 | export class Message extends Entry { 25 | constructor(id, value = null, attributes = [], comment = null) { 26 | super(); 27 | this.type = "Message"; 28 | this.id = id; 29 | this.value = value; 30 | this.attributes = attributes; 31 | this.comment = comment; 32 | } 33 | } 34 | 35 | export class Term extends Entry { 36 | constructor(id, value, attributes = [], comment = null) { 37 | super(); 38 | this.type = "Term"; 39 | this.id = id; 40 | this.value = value; 41 | this.attributes = attributes; 42 | this.comment = comment; 43 | } 44 | } 45 | 46 | export class Pattern extends SyntaxNode { 47 | constructor(elements) { 48 | super(); 49 | this.type = "Pattern"; 50 | this.elements = elements; 51 | } 52 | } 53 | 54 | // An abstract base class for elements of Patterns. 55 | export class PatternElement extends SyntaxNode {} 56 | 57 | export class TextElement extends PatternElement { 58 | constructor(value) { 59 | super(); 60 | this.type = "TextElement"; 61 | this.value = value; 62 | } 63 | } 64 | 65 | export class Placeable extends PatternElement { 66 | constructor(expression) { 67 | super(); 68 | this.type = "Placeable"; 69 | this.expression = expression; 70 | } 71 | } 72 | 73 | // An abstract base class for Expressions. 74 | export class Expression extends SyntaxNode {} 75 | 76 | // An abstract base class for Literals. 77 | export class Literal extends Expression { 78 | constructor(value) { 79 | super(); 80 | // The "value" field contains the exact contents of the literal, 81 | // character-for-character. 82 | this.value = value; 83 | } 84 | 85 | // Implementations are free to decide how they process the raw value. When 86 | // they do, however, they must comply with the behavior of `Literal.parse`. 87 | parse() { 88 | return {value: this.value}; 89 | } 90 | } 91 | 92 | export class StringLiteral extends Literal { 93 | constructor(value) { 94 | super(value); 95 | this.type = "StringLiteral"; 96 | } 97 | 98 | parse() { 99 | // Backslash backslash, backslash double quote, uHHHH, UHHHHHH. 100 | const KNOWN_ESCAPES = 101 | /(?:\\\\|\\\"|\\u([0-9a-fA-F]{4})|\\U([0-9a-fA-F]{6}))/g; 102 | 103 | function from_escape_sequence(match, codepoint4, codepoint6) { 104 | switch (match) { 105 | case "\\\\": 106 | return "\\"; 107 | case "\\\"": 108 | return "\""; 109 | default: 110 | let codepoint = parseInt(codepoint4 || codepoint6, 16); 111 | if (codepoint <= 0xD7FF || 0xE000 <= codepoint) { 112 | // It's a Unicode scalar value. 113 | return String.fromCodePoint(codepoint); 114 | } 115 | // Escape sequences reresenting surrogate code points are 116 | // well-formed but invalid in Fluent. Replace them with U+FFFD 117 | // REPLACEMENT CHARACTER. 118 | return "�"; 119 | } 120 | } 121 | 122 | let value = this.value.replace(KNOWN_ESCAPES, from_escape_sequence); 123 | return {value}; 124 | } 125 | } 126 | 127 | export class NumberLiteral extends Literal { 128 | constructor(value) { 129 | super(value); 130 | this.type = "NumberLiteral"; 131 | } 132 | 133 | parse() { 134 | let value = parseFloat(this.value); 135 | let decimal_position = this.value.indexOf("."); 136 | let precision = decimal_position > 0 137 | ? this.value.length - decimal_position - 1 138 | : 0; 139 | return {value, precision}; 140 | } 141 | } 142 | 143 | export class MessageReference extends Expression { 144 | constructor(id, attribute) { 145 | super(); 146 | this.type = "MessageReference"; 147 | this.id = id; 148 | this.attribute = attribute; 149 | } 150 | } 151 | 152 | export class TermReference extends Expression { 153 | constructor(id, attribute, args) { 154 | super(); 155 | this.type = "TermReference"; 156 | this.id = id; 157 | this.attribute = attribute; 158 | this.arguments = args; 159 | } 160 | } 161 | 162 | export class VariableReference extends Expression { 163 | constructor(id) { 164 | super(); 165 | this.type = "VariableReference"; 166 | this.id = id; 167 | } 168 | } 169 | 170 | export class FunctionReference extends Expression { 171 | constructor(id, args) { 172 | super(); 173 | this.type = "FunctionReference"; 174 | this.id = id; 175 | this.arguments = args; 176 | } 177 | } 178 | 179 | export class SelectExpression extends Expression { 180 | constructor(selector, variants) { 181 | super(); 182 | this.type = "SelectExpression"; 183 | this.selector = selector; 184 | this.variants = variants; 185 | } 186 | } 187 | 188 | export class Attribute extends SyntaxNode { 189 | constructor(id, value) { 190 | super(); 191 | this.type = "Attribute"; 192 | this.id = id; 193 | this.value = value; 194 | } 195 | } 196 | 197 | export class Variant extends SyntaxNode { 198 | constructor(key, value, def = false) { 199 | super(); 200 | this.type = "Variant"; 201 | this.key = key; 202 | this.value = value; 203 | this.default = def; 204 | } 205 | } 206 | 207 | export class CallArguments extends SyntaxNode { 208 | constructor(positional = [], named = []) { 209 | super(); 210 | this.type = "CallArguments"; 211 | this.positional = positional; 212 | this.named = named; 213 | } 214 | } 215 | 216 | export class NamedArgument extends SyntaxNode { 217 | constructor(name, value) { 218 | super(); 219 | this.type = "NamedArgument"; 220 | this.name = name; 221 | this.value = value; 222 | } 223 | } 224 | 225 | export class Identifier extends SyntaxNode { 226 | constructor(name) { 227 | super(); 228 | this.type = "Identifier"; 229 | this.name = name; 230 | } 231 | } 232 | 233 | export class BaseComment extends Entry { 234 | constructor(content) { 235 | super(); 236 | this.type = "BaseComment"; 237 | this.content = content; 238 | } 239 | } 240 | 241 | export class Comment extends BaseComment { 242 | constructor(content) { 243 | super(content); 244 | this.type = "Comment"; 245 | } 246 | } 247 | 248 | export class GroupComment extends BaseComment { 249 | constructor(content) { 250 | super(content); 251 | this.type = "GroupComment"; 252 | } 253 | } 254 | export class ResourceComment extends BaseComment { 255 | constructor(content) { 256 | super(content); 257 | this.type = "ResourceComment"; 258 | } 259 | } 260 | 261 | export class Junk extends SyntaxNode { 262 | constructor(content) { 263 | super(); 264 | this.type = "Junk"; 265 | this.annotations = []; 266 | this.content = content; 267 | } 268 | 269 | addAnnotation(annot) { 270 | this.annotations.push(annot); 271 | } 272 | } 273 | 274 | export class Span extends BaseNode { 275 | constructor(start, end) { 276 | super(); 277 | this.type = "Span"; 278 | this.start = start; 279 | this.end = end; 280 | } 281 | } 282 | 283 | export class Annotation extends SyntaxNode { 284 | constructor(code, args = [], message) { 285 | super(); 286 | this.type = "Annotation"; 287 | this.code = code; 288 | this.arguments = args; 289 | this.message = message; 290 | } 291 | } 292 | -------------------------------------------------------------------------------- /test/bench.js: -------------------------------------------------------------------------------- 1 | import fs from "fs"; 2 | import {PerformanceObserver, performance} from "perf_hooks"; 3 | 4 | import {parse} from "@fluent/syntax"; 5 | import {FluentResource} from "@fluent/bundle"; 6 | import {Resource} from "../syntax/grammar.js"; 7 | 8 | let args = process.argv.slice(2); 9 | 10 | if (args.length < 1 || 2 < args.length) { 11 | console.error( 12 | "Usage: node bench.js FTL_FILE [SAMPLE SIZE = 30]"); 13 | process.exit(1); 14 | } 15 | 16 | class Subject { 17 | constructor(name, fn, measures = []) { 18 | this.name = name; 19 | this.fn = fn; 20 | this.measures = measures; 21 | } 22 | } 23 | 24 | main(...args); 25 | 26 | function main(ftl_file, sample_size = 30) { 27 | let ftl = fs.readFileSync(ftl_file, "utf8"); 28 | 29 | let subjects = new Map([ 30 | ["Reference", new Subject("Reference", () => Resource.run(ftl))], 31 | ["Tooling", new Subject("Tooling", () => parse(ftl))], 32 | ["Runtime", new Subject("Runtime", () => new FluentResource(ftl))], 33 | ]); 34 | 35 | new PerformanceObserver(items => { 36 | for (const {name, duration} of items.getEntries()) { 37 | subjects.get(name).measures.push(duration); 38 | } 39 | performance.clearMarks(); 40 | 41 | for (let {name, measures} of subjects.values()) { 42 | let m = mean(measures); 43 | let s = stdev(measures, m); 44 | console.log(`${name}: mean ${m}ms, stdev ${s}ms`); 45 | } 46 | }).observe({entryTypes: ["measure"]}); 47 | 48 | for (let i = 0; i < sample_size; i++) { 49 | process.stdout.write("."); 50 | shuffle(...subjects.values()).map(run); 51 | } 52 | process.stdout.write("\n"); 53 | } 54 | 55 | function run({name, fn}) { 56 | performance.mark("start"); 57 | fn(); 58 | performance.mark("end"); 59 | performance.measure(name, "start", "end"); 60 | } 61 | 62 | // Durstenfeld Shuffle 63 | function shuffle(...elements) { 64 | for (let i = elements.length - 1; i > 0; i--) { 65 | const j = Math.floor(Math.random() * (i + 1)); 66 | [elements[i], elements[j]] = [elements[j], elements[i]]; 67 | } 68 | return elements; 69 | } 70 | 71 | function mean(elements) { 72 | let miu = elements.reduce((acc, cur) => acc + cur, 0) 73 | / elements.length; 74 | return +miu.toFixed(2); 75 | } 76 | 77 | function stdev(elements, mean) { 78 | let sigma = elements.reduce((acc, cur) => acc + (cur - mean) ** 2, 0) 79 | / (elements.length - 1); 80 | return +Math.sqrt(sigma).toFixed(2); 81 | } 82 | -------------------------------------------------------------------------------- /test/ebnf.js: -------------------------------------------------------------------------------- 1 | import fs from "fs"; 2 | import color from "cli-color"; 3 | import difflib from "difflib"; 4 | import ebnf from "../lib/ebnf.js"; 5 | import {PASS, FAIL} from "./suite.js"; 6 | 7 | let args = process.argv.slice(2); 8 | 9 | if (args.length !== 2) { 10 | console.error( 11 | "Usage: node ebnf.js " + 12 | "GRAMMAR_FILE EXPECTED_EBNF"); 13 | process.exit(1); 14 | } 15 | 16 | main(...args); 17 | 18 | function main(grammar_mjs, fluent_ebnf) { 19 | let grammar_source = fs.readFileSync(grammar_mjs, "utf8"); 20 | let grammar_ebnf = fs.readFileSync(fluent_ebnf, "utf8"); 21 | 22 | let diffs = difflib.unifiedDiff( 23 | lines(grammar_ebnf), 24 | lines(ebnf(grammar_source)), { 25 | fromfile: "Expected", 26 | tofile: "Actual", 27 | }); 28 | 29 | for (let diff of diffs) { 30 | if (diff.startsWith("+")) { 31 | process.stdout.write(color.green(diff)); 32 | } else if (diff.startsWith("-")) { 33 | process.stdout.write(color.red(diff)); 34 | } else { 35 | process.stdout.write(diff); 36 | } 37 | } 38 | 39 | if (diffs.length === 0) { 40 | console.log(format_summary(PASS)); 41 | process.exit(0); 42 | } else { 43 | console.log(format_summary(FAIL)); 44 | process.exit(1); 45 | } 46 | } 47 | 48 | function lines(text) { 49 | return text.split("\n").map(line => line + "\n"); 50 | } 51 | 52 | function format_summary(result) { 53 | return ` 54 | ======================================================================== 55 | Generated EBNF ${result}. 56 | `; 57 | } 58 | -------------------------------------------------------------------------------- /test/fixtures/Makefile: -------------------------------------------------------------------------------- 1 | FTL_FIXTURES := $(wildcard *.ftl) 2 | AST_FIXTURES := $(FTL_FIXTURES:%.ftl=%.json) 3 | 4 | all: $(AST_FIXTURES) 5 | 6 | .PHONY: $(AST_FIXTURES) 7 | $(AST_FIXTURES): %.json: %.ftl 8 | @node ../../bin/parse.js $< \ 9 | 2> /dev/null \ 10 | 1> $@; 11 | @echo "$< → $@" 12 | -------------------------------------------------------------------------------- /test/fixtures/any_char.ftl: -------------------------------------------------------------------------------- 1 | # ↓ BEL, U+0007 2 | control0 = abcdef 3 | 4 | # ↓ DEL, U+007F 5 | delete = abcdef 6 | 7 | # ↓ BPM, U+0082 8 | control1 = abc‚def 9 | -------------------------------------------------------------------------------- /test/fixtures/any_char.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Resource", 3 | "body": [ 4 | { 5 | "type": "Message", 6 | "id": { 7 | "type": "Identifier", 8 | "name": "control0" 9 | }, 10 | "value": { 11 | "type": "Pattern", 12 | "elements": [ 13 | { 14 | "type": "TextElement", 15 | "value": "abc\u0007def" 16 | } 17 | ] 18 | }, 19 | "attributes": [], 20 | "comment": { 21 | "type": "Comment", 22 | "content": " ↓ BEL, U+0007" 23 | } 24 | }, 25 | { 26 | "type": "Message", 27 | "id": { 28 | "type": "Identifier", 29 | "name": "delete" 30 | }, 31 | "value": { 32 | "type": "Pattern", 33 | "elements": [ 34 | { 35 | "type": "TextElement", 36 | "value": "abcdef" 37 | } 38 | ] 39 | }, 40 | "attributes": [], 41 | "comment": { 42 | "type": "Comment", 43 | "content": " ↓ DEL, U+007F" 44 | } 45 | }, 46 | { 47 | "type": "Message", 48 | "id": { 49 | "type": "Identifier", 50 | "name": "control1" 51 | }, 52 | "value": { 53 | "type": "Pattern", 54 | "elements": [ 55 | { 56 | "type": "TextElement", 57 | "value": "abc‚def" 58 | } 59 | ] 60 | }, 61 | "attributes": [], 62 | "comment": { 63 | "type": "Comment", 64 | "content": " ↓ BPM, U+0082" 65 | } 66 | } 67 | ] 68 | } 69 | -------------------------------------------------------------------------------- /test/fixtures/astral.ftl: -------------------------------------------------------------------------------- 1 | face-with-tears-of-joy = 😂 2 | tetragram-for-centre = 𝌆 3 | 4 | surrogates-in-text = \uD83D\uDE02 5 | surrogates-in-string = {"\uD83D\uDE02"} 6 | surrogates-in-adjacent-strings = {"\uD83D"}{"\uDE02"} 7 | 8 | emoji-in-text = A face 😂 with tears of joy. 9 | emoji-in-string = {"A face 😂 with tears of joy."} 10 | 11 | # ERROR Invalid identifier 12 | err-😂 = Value 13 | 14 | # ERROR Invalid expression 15 | err-invalid-expression = { 😂 } 16 | 17 | # ERROR Invalid variant key 18 | err-invalid-variant-key = { $sel -> 19 | *[😂] Value 20 | } 21 | -------------------------------------------------------------------------------- /test/fixtures/astral.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Resource", 3 | "body": [ 4 | { 5 | "type": "Message", 6 | "id": { 7 | "type": "Identifier", 8 | "name": "face-with-tears-of-joy" 9 | }, 10 | "value": { 11 | "type": "Pattern", 12 | "elements": [ 13 | { 14 | "type": "TextElement", 15 | "value": "😂" 16 | } 17 | ] 18 | }, 19 | "attributes": [], 20 | "comment": null 21 | }, 22 | { 23 | "type": "Message", 24 | "id": { 25 | "type": "Identifier", 26 | "name": "tetragram-for-centre" 27 | }, 28 | "value": { 29 | "type": "Pattern", 30 | "elements": [ 31 | { 32 | "type": "TextElement", 33 | "value": "𝌆" 34 | } 35 | ] 36 | }, 37 | "attributes": [], 38 | "comment": null 39 | }, 40 | { 41 | "type": "Message", 42 | "id": { 43 | "type": "Identifier", 44 | "name": "surrogates-in-text" 45 | }, 46 | "value": { 47 | "type": "Pattern", 48 | "elements": [ 49 | { 50 | "type": "TextElement", 51 | "value": "\\uD83D\\uDE02" 52 | } 53 | ] 54 | }, 55 | "attributes": [], 56 | "comment": null 57 | }, 58 | { 59 | "type": "Message", 60 | "id": { 61 | "type": "Identifier", 62 | "name": "surrogates-in-string" 63 | }, 64 | "value": { 65 | "type": "Pattern", 66 | "elements": [ 67 | { 68 | "type": "Placeable", 69 | "expression": { 70 | "value": "\\uD83D\\uDE02", 71 | "type": "StringLiteral" 72 | } 73 | } 74 | ] 75 | }, 76 | "attributes": [], 77 | "comment": null 78 | }, 79 | { 80 | "type": "Message", 81 | "id": { 82 | "type": "Identifier", 83 | "name": "surrogates-in-adjacent-strings" 84 | }, 85 | "value": { 86 | "type": "Pattern", 87 | "elements": [ 88 | { 89 | "type": "Placeable", 90 | "expression": { 91 | "value": "\\uD83D", 92 | "type": "StringLiteral" 93 | } 94 | }, 95 | { 96 | "type": "Placeable", 97 | "expression": { 98 | "value": "\\uDE02", 99 | "type": "StringLiteral" 100 | } 101 | } 102 | ] 103 | }, 104 | "attributes": [], 105 | "comment": null 106 | }, 107 | { 108 | "type": "Message", 109 | "id": { 110 | "type": "Identifier", 111 | "name": "emoji-in-text" 112 | }, 113 | "value": { 114 | "type": "Pattern", 115 | "elements": [ 116 | { 117 | "type": "TextElement", 118 | "value": "A face 😂 with tears of joy." 119 | } 120 | ] 121 | }, 122 | "attributes": [], 123 | "comment": null 124 | }, 125 | { 126 | "type": "Message", 127 | "id": { 128 | "type": "Identifier", 129 | "name": "emoji-in-string" 130 | }, 131 | "value": { 132 | "type": "Pattern", 133 | "elements": [ 134 | { 135 | "type": "Placeable", 136 | "expression": { 137 | "value": "A face 😂 with tears of joy.", 138 | "type": "StringLiteral" 139 | } 140 | } 141 | ] 142 | }, 143 | "attributes": [], 144 | "comment": null 145 | }, 146 | { 147 | "type": "Comment", 148 | "content": "ERROR Invalid identifier" 149 | }, 150 | { 151 | "type": "Junk", 152 | "annotations": [], 153 | "content": "err-😂 = Value\n\n" 154 | }, 155 | { 156 | "type": "Comment", 157 | "content": "ERROR Invalid expression" 158 | }, 159 | { 160 | "type": "Junk", 161 | "annotations": [], 162 | "content": "err-invalid-expression = { 😂 }\n\n" 163 | }, 164 | { 165 | "type": "Comment", 166 | "content": "ERROR Invalid variant key" 167 | }, 168 | { 169 | "type": "Junk", 170 | "annotations": [], 171 | "content": "err-invalid-variant-key = { $sel ->\n *[😂] Value\n}\n" 172 | } 173 | ] 174 | } 175 | -------------------------------------------------------------------------------- /test/fixtures/call_expressions.ftl: -------------------------------------------------------------------------------- 1 | ## Function names 2 | 3 | valid-func-name-01 = {FUN1()} 4 | valid-func-name-02 = {FUN_FUN()} 5 | valid-func-name-03 = {FUN-FUN()} 6 | 7 | # JUNK 0 is not a valid Identifier start 8 | invalid-func-name-01 = {0FUN()} 9 | # JUNK Function names may not be lowercase 10 | invalid-func-name-02 = {fun()} 11 | # JUNK Function names may not contain lowercase character 12 | invalid-func-name-03 = {Fun()} 13 | # JUNK ? is not a valid Identifier character 14 | invalid-func-name-04 = {FUN?()} 15 | 16 | ## Arguments 17 | 18 | positional-args = {FUN(1, "a", msg)} 19 | named-args = {FUN(x: 1, y: "Y")} 20 | dense-named-args = {FUN(x:1, y:"Y")} 21 | mixed-args = {FUN(1, "a", msg, x: 1, y: "Y")} 22 | 23 | # ERROR Positional arg must not follow keyword args 24 | shuffled-args = {FUN(1, x: 1, "a", y: "Y", msg)} 25 | 26 | # ERROR Named arguments must be unique 27 | duplicate-named-args = {FUN(x: 1, x: "X")} 28 | 29 | 30 | ## Whitespace around arguments 31 | 32 | sparse-inline-call = {FUN ( "a" , msg, x: 1 )} 33 | empty-inline-call = {FUN( )} 34 | multiline-call = {FUN( 35 | "a", 36 | msg, 37 | x: 1 38 | )} 39 | sparse-multiline-call = {FUN 40 | ( 41 | 42 | "a" , 43 | msg 44 | , x: 1 45 | )} 46 | empty-multiline-call = {FUN( 47 | 48 | )} 49 | 50 | 51 | unindented-arg-number = {FUN( 52 | 1)} 53 | 54 | unindented-arg-string = {FUN( 55 | "a")} 56 | 57 | unindented-arg-msg-ref = {FUN( 58 | msg)} 59 | 60 | unindented-arg-term-ref = {FUN( 61 | -msg)} 62 | 63 | unindented-arg-var-ref = {FUN( 64 | $var)} 65 | 66 | unindented-arg-call = {FUN( 67 | OTHER())} 68 | 69 | unindented-named-arg = {FUN( 70 | x:1)} 71 | 72 | unindented-closing-paren = {FUN( 73 | x 74 | )} 75 | 76 | 77 | 78 | ## Optional trailing comma 79 | 80 | one-argument = {FUN(1,)} 81 | many-arguments = {FUN(1, 2, 3,)} 82 | inline-sparse-args = {FUN( 1, 2, 3, )} 83 | multiline-args = {FUN( 84 | 1, 85 | 2, 86 | )} 87 | multiline-sparse-args = {FUN( 88 | 89 | 1 90 | , 91 | 2 92 | , 93 | )} 94 | 95 | 96 | ## Syntax errors for trailing comma 97 | 98 | one-argument = {FUN(1,,)} 99 | missing-arg = {FUN(,)} 100 | missing-sparse-arg = {FUN( , )} 101 | 102 | 103 | ## Whitespace in named arguments 104 | 105 | sparse-named-arg = {FUN( 106 | x : 1, 107 | y : 2, 108 | z 109 | : 110 | 3 111 | )} 112 | 113 | 114 | unindented-colon = {FUN( 115 | x 116 | :1)} 117 | 118 | unindented-value = {FUN( 119 | x: 120 | 1)} 121 | -------------------------------------------------------------------------------- /test/fixtures/callee_expressions.ftl: -------------------------------------------------------------------------------- 1 | ## Callees in placeables. 2 | 3 | function-callee-placeable = {FUNCTION()} 4 | term-callee-placeable = {-term()} 5 | 6 | # ERROR Messages cannot be parameterized. 7 | message-callee-placeable = {message()} 8 | # ERROR Equivalent to a MessageReference callee. 9 | mixed-case-callee-placeable = {Function()} 10 | # ERROR Message attributes cannot be parameterized. 11 | message-attr-callee-placeable = {message.attr()} 12 | # ERROR Term attributes may not be used in Placeables. 13 | term-attr-callee-placeable = {-term.attr()} 14 | # ERROR Variables cannot be parameterized. 15 | variable-callee-placeable = {$variable()} 16 | 17 | 18 | ## Callees in selectors. 19 | 20 | function-callee-selector = {FUNCTION() -> 21 | *[key] Value 22 | } 23 | term-attr-callee-selector = {-term.attr() -> 24 | *[key] Value 25 | } 26 | 27 | # ERROR Messages cannot be parameterized. 28 | message-callee-selector = {message() -> 29 | *[key] Value 30 | } 31 | # ERROR Equivalent to a MessageReference callee. 32 | mixed-case-callee-selector = {Function() -> 33 | *[key] Value 34 | } 35 | # ERROR Message attributes cannot be parameterized. 36 | message-attr-callee-selector = {message.attr() -> 37 | *[key] Value 38 | } 39 | # ERROR Term values may not be used as selectors. 40 | term-callee-selector = {-term() -> 41 | *[key] Value 42 | } 43 | # ERROR Variables cannot be parameterized. 44 | variable-callee-selector = {$variable() -> 45 | *[key] Value 46 | } 47 | -------------------------------------------------------------------------------- /test/fixtures/comments.ftl: -------------------------------------------------------------------------------- 1 | # Standalone Comment 2 | 3 | # Message Comment 4 | foo = Foo 5 | 6 | # Term Comment 7 | # with a blank last line. 8 | # 9 | -term = Term 10 | 11 | # Another standalone 12 | # 13 | # with indent 14 | ## Group Comment 15 | ### Resource Comment 16 | 17 | # Errors 18 | #error 19 | ##error 20 | ###error 21 | -------------------------------------------------------------------------------- /test/fixtures/comments.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Resource", 3 | "body": [ 4 | { 5 | "type": "Comment", 6 | "content": "Standalone Comment" 7 | }, 8 | { 9 | "type": "Message", 10 | "id": { 11 | "type": "Identifier", 12 | "name": "foo" 13 | }, 14 | "value": { 15 | "type": "Pattern", 16 | "elements": [ 17 | { 18 | "type": "TextElement", 19 | "value": "Foo" 20 | } 21 | ] 22 | }, 23 | "attributes": [], 24 | "comment": { 25 | "type": "Comment", 26 | "content": "Message Comment" 27 | } 28 | }, 29 | { 30 | "type": "Term", 31 | "id": { 32 | "type": "Identifier", 33 | "name": "term" 34 | }, 35 | "value": { 36 | "type": "Pattern", 37 | "elements": [ 38 | { 39 | "type": "TextElement", 40 | "value": "Term" 41 | } 42 | ] 43 | }, 44 | "attributes": [], 45 | "comment": { 46 | "type": "Comment", 47 | "content": "Term Comment\nwith a blank last line.\n" 48 | } 49 | }, 50 | { 51 | "type": "Comment", 52 | "content": "Another standalone\n\n with indent" 53 | }, 54 | { 55 | "type": "GroupComment", 56 | "content": "Group Comment" 57 | }, 58 | { 59 | "type": "ResourceComment", 60 | "content": "Resource Comment" 61 | }, 62 | { 63 | "type": "Comment", 64 | "content": "Errors" 65 | }, 66 | { 67 | "type": "Junk", 68 | "annotations": [], 69 | "content": "#error\n" 70 | }, 71 | { 72 | "type": "Junk", 73 | "annotations": [], 74 | "content": "##error\n" 75 | }, 76 | { 77 | "type": "Junk", 78 | "annotations": [], 79 | "content": "###error\n" 80 | } 81 | ] 82 | } 83 | -------------------------------------------------------------------------------- /test/fixtures/cr_err_literal.ftl: -------------------------------------------------------------------------------- 1 | err01 = { "str # ERROR Unclosed StringLiteral r### This entire file uses CR as EOL. -------------------------------------------------------------------------------- /test/fixtures/cr_err_literal.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Resource", 3 | "body": [ 4 | { 5 | "type": "Junk", 6 | "annotations": [], 7 | "content": "err01 = { \"str\r\r# ERROR Unclosed StringLiteral\r\rr### This entire file uses CR as EOL.\r" 8 | } 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /test/fixtures/cr_err_selector.ftl: -------------------------------------------------------------------------------- 1 | err01 = { $sel -> } # ERROR Missing newline after ->. ### This entire file uses CR as EOL. -------------------------------------------------------------------------------- /test/fixtures/cr_err_selector.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Resource", 3 | "body": [ 4 | { 5 | "type": "Junk", 6 | "annotations": [], 7 | "content": "err01 = { $sel -> }\r\r# ERROR Missing newline after ->.\r### This entire file uses CR as EOL.\r" 8 | } 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /test/fixtures/cr_multikey.ftl: -------------------------------------------------------------------------------- 1 | key01 = Value 01 err02 = Value 02 ### This entire file uses CR as EOL. -------------------------------------------------------------------------------- /test/fixtures/cr_multikey.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Resource", 3 | "body": [ 4 | { 5 | "type": "Message", 6 | "id": { 7 | "type": "Identifier", 8 | "name": "key01" 9 | }, 10 | "value": { 11 | "type": "Pattern", 12 | "elements": [ 13 | { 14 | "type": "TextElement", 15 | "value": "Value 01\rerr02 = Value 02\r\r\r### This entire file uses CR as EOL.\r" 16 | } 17 | ] 18 | }, 19 | "attributes": [], 20 | "comment": null 21 | } 22 | ] 23 | } 24 | -------------------------------------------------------------------------------- /test/fixtures/cr_multilinevalue.ftl: -------------------------------------------------------------------------------- 1 | key01 = Value 03 Continued and continued and continued .title = Title ### This entire file uses CR as EOL. -------------------------------------------------------------------------------- /test/fixtures/cr_multilinevalue.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Resource", 3 | "body": [ 4 | { 5 | "type": "Message", 6 | "id": { 7 | "type": "Identifier", 8 | "name": "key01" 9 | }, 10 | "value": { 11 | "type": "Pattern", 12 | "elements": [ 13 | { 14 | "type": "TextElement", 15 | "value": "\r\r Value 03\r Continued\r\r and continued\r \r and continued\r\r .title = Title\r\r\r### This entire file uses CR as EOL.\r" 16 | } 17 | ] 18 | }, 19 | "attributes": [], 20 | "comment": null 21 | } 22 | ] 23 | } 24 | -------------------------------------------------------------------------------- /test/fixtures/crlf.ftl: -------------------------------------------------------------------------------- 1 | 2 | key01 = Value 01 3 | key02 = 4 | 5 | Value 02 6 | Continued 7 | 8 | and continued 9 | 10 | and continued 11 | 12 | .title = Title 13 | 14 | # ERROR Unclosed StringLiteral 15 | err03 = { "str 16 | 17 | # ERROR Missing newline after ->. 18 | err04 = { $sel -> } 19 | -------------------------------------------------------------------------------- /test/fixtures/crlf.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Resource", 3 | "body": [ 4 | { 5 | "type": "Message", 6 | "id": { 7 | "type": "Identifier", 8 | "name": "key01" 9 | }, 10 | "value": { 11 | "type": "Pattern", 12 | "elements": [ 13 | { 14 | "type": "TextElement", 15 | "value": "Value 01" 16 | } 17 | ] 18 | }, 19 | "attributes": [], 20 | "comment": null 21 | }, 22 | { 23 | "type": "Message", 24 | "id": { 25 | "type": "Identifier", 26 | "name": "key02" 27 | }, 28 | "value": { 29 | "type": "Pattern", 30 | "elements": [ 31 | { 32 | "type": "TextElement", 33 | "value": "Value 02\nContinued\n\nand continued\n\nand continued" 34 | } 35 | ] 36 | }, 37 | "attributes": [ 38 | { 39 | "type": "Attribute", 40 | "id": { 41 | "type": "Identifier", 42 | "name": "title" 43 | }, 44 | "value": { 45 | "type": "Pattern", 46 | "elements": [ 47 | { 48 | "type": "TextElement", 49 | "value": "Title" 50 | } 51 | ] 52 | } 53 | } 54 | ], 55 | "comment": null 56 | }, 57 | { 58 | "type": "Comment", 59 | "content": "ERROR Unclosed StringLiteral" 60 | }, 61 | { 62 | "type": "Junk", 63 | "annotations": [], 64 | "content": "err03 = { \"str\r\n\r\n" 65 | }, 66 | { 67 | "type": "Comment", 68 | "content": "ERROR Missing newline after ->." 69 | }, 70 | { 71 | "type": "Junk", 72 | "annotations": [], 73 | "content": "err04 = { $sel -> }\r\n" 74 | } 75 | ] 76 | } 77 | -------------------------------------------------------------------------------- /test/fixtures/eof_comment.ftl: -------------------------------------------------------------------------------- 1 | ### NOTE: Disable final newline insertion when editing this file. 2 | 3 | # No EOL -------------------------------------------------------------------------------- /test/fixtures/eof_comment.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Resource", 3 | "body": [ 4 | { 5 | "type": "ResourceComment", 6 | "content": "NOTE: Disable final newline insertion when editing this file." 7 | }, 8 | { 9 | "type": "Comment", 10 | "content": "No EOL" 11 | } 12 | ] 13 | } 14 | -------------------------------------------------------------------------------- /test/fixtures/eof_empty.ftl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/projectfluent/fluent/10a1bc60bee843c14a30216fa4cebdc559bf2076/test/fixtures/eof_empty.ftl -------------------------------------------------------------------------------- /test/fixtures/eof_empty.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Resource", 3 | "body": [] 4 | } 5 | -------------------------------------------------------------------------------- /test/fixtures/eof_id.ftl: -------------------------------------------------------------------------------- 1 | ### NOTE: Disable final newline insertion when editing this file. 2 | 3 | message-id -------------------------------------------------------------------------------- /test/fixtures/eof_id.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Resource", 3 | "body": [ 4 | { 5 | "type": "ResourceComment", 6 | "content": "NOTE: Disable final newline insertion when editing this file." 7 | }, 8 | { 9 | "type": "Junk", 10 | "annotations": [], 11 | "content": "message-id" 12 | } 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /test/fixtures/eof_id_equals.ftl: -------------------------------------------------------------------------------- 1 | ### NOTE: Disable final newline insertion when editing this file. 2 | 3 | message-id = -------------------------------------------------------------------------------- /test/fixtures/eof_id_equals.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Resource", 3 | "body": [ 4 | { 5 | "type": "ResourceComment", 6 | "content": "NOTE: Disable final newline insertion when editing this file." 7 | }, 8 | { 9 | "type": "Junk", 10 | "annotations": [], 11 | "content": "message-id =" 12 | } 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /test/fixtures/eof_junk.ftl: -------------------------------------------------------------------------------- 1 | ### NOTE: Disable final newline insertion when editing this file. 2 | 3 | 000 -------------------------------------------------------------------------------- /test/fixtures/eof_junk.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Resource", 3 | "body": [ 4 | { 5 | "type": "ResourceComment", 6 | "content": "NOTE: Disable final newline insertion when editing this file." 7 | }, 8 | { 9 | "type": "Junk", 10 | "annotations": [], 11 | "content": "000" 12 | } 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /test/fixtures/eof_value.ftl: -------------------------------------------------------------------------------- 1 | ### NOTE: Disable final newline insertion when editing this file. 2 | 3 | no-eol = No EOL -------------------------------------------------------------------------------- /test/fixtures/eof_value.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Resource", 3 | "body": [ 4 | { 5 | "type": "ResourceComment", 6 | "content": "NOTE: Disable final newline insertion when editing this file." 7 | }, 8 | { 9 | "type": "Message", 10 | "id": { 11 | "type": "Identifier", 12 | "name": "no-eol" 13 | }, 14 | "value": { 15 | "type": "Pattern", 16 | "elements": [ 17 | { 18 | "type": "TextElement", 19 | "value": "No EOL" 20 | } 21 | ] 22 | }, 23 | "attributes": [], 24 | "comment": null 25 | } 26 | ] 27 | } 28 | -------------------------------------------------------------------------------- /test/fixtures/escaped_characters.ftl: -------------------------------------------------------------------------------- 1 | ## Literal text 2 | text-backslash-one = Value with \ a backslash 3 | text-backslash-two = Value with \\ two backslashes 4 | text-backslash-brace = Value with \{placeable} 5 | text-backslash-u = \u0041 6 | text-backslash-backslash-u = \\u0041 7 | 8 | ## String literals 9 | quote-in-string = {"\""} 10 | backslash-in-string = {"\\"} 11 | # ERROR Mismatched quote 12 | mismatched-quote = {"\\""} 13 | # ERROR Unknown escape 14 | unknown-escape = {"\x"} 15 | # ERROR Multiline literal 16 | invalid-multiline-literal = {" 17 | "} 18 | 19 | ## Unicode escapes 20 | string-unicode-4digits = {"\u0041"} 21 | escape-unicode-4digits = {"\\u0041"} 22 | string-unicode-6digits = {"\U01F602"} 23 | escape-unicode-6digits = {"\\U01F602"} 24 | 25 | # OK The trailing "00" is part of the literal value. 26 | string-too-many-4digits = {"\u004100"} 27 | # OK The trailing "00" is part of the literal value. 28 | string-too-many-6digits = {"\U01F60200"} 29 | 30 | # ERROR Too few hex digits after \u. 31 | string-too-few-4digits = {"\u41"} 32 | # ERROR Too few hex digits after \U. 33 | string-too-few-6digits = {"\U1F602"} 34 | 35 | ## Literal braces 36 | brace-open = An opening {"{"} brace. 37 | brace-close = A closing {"}"} brace. 38 | -------------------------------------------------------------------------------- /test/fixtures/junk.ftl: -------------------------------------------------------------------------------- 1 | ## Two adjacent Junks. 2 | err01 = {1x} 3 | err02 = {2x} 4 | 5 | # A single Junk. 6 | err03 = {1x 7 | 2 8 | 9 | # A single Junk. 10 | ą=Invalid identifier 11 | ć=Another one 12 | 13 | # The COMMENT ends this junk. 14 | err04 = { 15 | # COMMENT 16 | 17 | # The COMMENT ends this junk. 18 | # The closing brace is a separate Junk. 19 | err04 = { 20 | # COMMENT 21 | } 22 | -------------------------------------------------------------------------------- /test/fixtures/junk.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Resource", 3 | "body": [ 4 | { 5 | "type": "GroupComment", 6 | "content": "Two adjacent Junks." 7 | }, 8 | { 9 | "type": "Junk", 10 | "annotations": [], 11 | "content": "err01 = {1x}\n" 12 | }, 13 | { 14 | "type": "Junk", 15 | "annotations": [], 16 | "content": "err02 = {2x}\n\n" 17 | }, 18 | { 19 | "type": "Comment", 20 | "content": "A single Junk." 21 | }, 22 | { 23 | "type": "Junk", 24 | "annotations": [], 25 | "content": "err03 = {1x\n2\n\n" 26 | }, 27 | { 28 | "type": "Comment", 29 | "content": "A single Junk." 30 | }, 31 | { 32 | "type": "Junk", 33 | "annotations": [], 34 | "content": "ą=Invalid identifier\nć=Another one\n\n" 35 | }, 36 | { 37 | "type": "Comment", 38 | "content": "The COMMENT ends this junk." 39 | }, 40 | { 41 | "type": "Junk", 42 | "annotations": [], 43 | "content": "err04 = {\n" 44 | }, 45 | { 46 | "type": "Comment", 47 | "content": "COMMENT" 48 | }, 49 | { 50 | "type": "Comment", 51 | "content": "The COMMENT ends this junk.\nThe closing brace is a separate Junk." 52 | }, 53 | { 54 | "type": "Junk", 55 | "annotations": [], 56 | "content": "err04 = {\n" 57 | }, 58 | { 59 | "type": "Comment", 60 | "content": "COMMENT" 61 | }, 62 | { 63 | "type": "Junk", 64 | "annotations": [], 65 | "content": "}\n" 66 | } 67 | ] 68 | } 69 | -------------------------------------------------------------------------------- /test/fixtures/leading_dots.ftl: -------------------------------------------------------------------------------- 1 | key01 = .Value 2 | key02 = …Value 3 | key03 = {"."}Value 4 | key04 = 5 | {"."}Value 6 | 7 | key05 = Value 8 | {"."}Continued 9 | 10 | key06 = .Value 11 | {"."}Continued 12 | 13 | # MESSAGE (value = "Value", attributes = []) 14 | # JUNK (attr .Continued" must have a value) 15 | key07 = Value 16 | .Continued 17 | 18 | # JUNK (attr .Value must have a value) 19 | key08 = 20 | .Value 21 | 22 | # JUNK (attr .Value must have a value) 23 | key09 = 24 | .Value 25 | Continued 26 | 27 | key10 = 28 | .Value = which is an attribute 29 | Continued 30 | 31 | key11 = 32 | {"."}Value = which looks like an attribute 33 | Continued 34 | 35 | key12 = 36 | .accesskey = 37 | A 38 | 39 | key13 = 40 | .attribute = .Value 41 | 42 | key14 = 43 | .attribute = 44 | {"."}Value 45 | 46 | key15 = 47 | { 1 -> 48 | [one] .Value 49 | *[other] 50 | {"."}Value 51 | } 52 | 53 | # JUNK (variant must have a value) 54 | key16 = 55 | { 1 -> 56 | *[one] 57 | .Value 58 | } 59 | 60 | # JUNK (unclosed placeable) 61 | key17 = 62 | { 1 -> 63 | *[one] Value 64 | .Continued 65 | } 66 | 67 | # JUNK (attr .Value must have a value) 68 | key18 = 69 | .Value 70 | 71 | key19 = 72 | .attribute = Value 73 | Continued 74 | 75 | key20 = 76 | {"."}Value 77 | -------------------------------------------------------------------------------- /test/fixtures/literal_expressions.ftl: -------------------------------------------------------------------------------- 1 | string-expression = {"abc"} 2 | number-expression = {123} 3 | number-expression = {-3.14} 4 | -------------------------------------------------------------------------------- /test/fixtures/literal_expressions.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Resource", 3 | "body": [ 4 | { 5 | "type": "Message", 6 | "id": { 7 | "type": "Identifier", 8 | "name": "string-expression" 9 | }, 10 | "value": { 11 | "type": "Pattern", 12 | "elements": [ 13 | { 14 | "type": "Placeable", 15 | "expression": { 16 | "value": "abc", 17 | "type": "StringLiteral" 18 | } 19 | } 20 | ] 21 | }, 22 | "attributes": [], 23 | "comment": null 24 | }, 25 | { 26 | "type": "Message", 27 | "id": { 28 | "type": "Identifier", 29 | "name": "number-expression" 30 | }, 31 | "value": { 32 | "type": "Pattern", 33 | "elements": [ 34 | { 35 | "type": "Placeable", 36 | "expression": { 37 | "value": "123", 38 | "type": "NumberLiteral" 39 | } 40 | } 41 | ] 42 | }, 43 | "attributes": [], 44 | "comment": null 45 | }, 46 | { 47 | "type": "Message", 48 | "id": { 49 | "type": "Identifier", 50 | "name": "number-expression" 51 | }, 52 | "value": { 53 | "type": "Pattern", 54 | "elements": [ 55 | { 56 | "type": "Placeable", 57 | "expression": { 58 | "value": "-3.14", 59 | "type": "NumberLiteral" 60 | } 61 | } 62 | ] 63 | }, 64 | "attributes": [], 65 | "comment": null 66 | } 67 | ] 68 | } 69 | -------------------------------------------------------------------------------- /test/fixtures/member_expressions.ftl: -------------------------------------------------------------------------------- 1 | ## Member expressions in placeables. 2 | 3 | # OK Message attributes may be interpolated in values. 4 | message-attribute-expression-placeable = {msg.attr} 5 | 6 | # ERROR Term attributes may not be used for interpolation. 7 | term-attribute-expression-placeable = {-term.attr} 8 | 9 | 10 | ## Member expressions in selectors. 11 | 12 | # OK Term attributes may be used as selectors. 13 | term-attribute-expression-selector = {-term.attr -> 14 | *[key] Value 15 | } 16 | # ERROR Message attributes may not be used as selectors. 17 | message-attribute-expression-selector = {msg.attr -> 18 | *[key] Value 19 | } 20 | -------------------------------------------------------------------------------- /test/fixtures/member_expressions.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Resource", 3 | "body": [ 4 | { 5 | "type": "GroupComment", 6 | "content": "Member expressions in placeables." 7 | }, 8 | { 9 | "type": "Message", 10 | "id": { 11 | "type": "Identifier", 12 | "name": "message-attribute-expression-placeable" 13 | }, 14 | "value": { 15 | "type": "Pattern", 16 | "elements": [ 17 | { 18 | "type": "Placeable", 19 | "expression": { 20 | "type": "MessageReference", 21 | "id": { 22 | "type": "Identifier", 23 | "name": "msg" 24 | }, 25 | "attribute": { 26 | "type": "Identifier", 27 | "name": "attr" 28 | } 29 | } 30 | } 31 | ] 32 | }, 33 | "attributes": [], 34 | "comment": { 35 | "type": "Comment", 36 | "content": "OK Message attributes may be interpolated in values." 37 | } 38 | }, 39 | { 40 | "type": "Comment", 41 | "content": "ERROR Term attributes may not be used for interpolation." 42 | }, 43 | { 44 | "type": "Junk", 45 | "annotations": [], 46 | "content": "term-attribute-expression-placeable = {-term.attr}\n\n\n" 47 | }, 48 | { 49 | "type": "GroupComment", 50 | "content": "Member expressions in selectors." 51 | }, 52 | { 53 | "type": "Message", 54 | "id": { 55 | "type": "Identifier", 56 | "name": "term-attribute-expression-selector" 57 | }, 58 | "value": { 59 | "type": "Pattern", 60 | "elements": [ 61 | { 62 | "type": "Placeable", 63 | "expression": { 64 | "type": "SelectExpression", 65 | "selector": { 66 | "type": "TermReference", 67 | "id": { 68 | "type": "Identifier", 69 | "name": "term" 70 | }, 71 | "attribute": { 72 | "type": "Identifier", 73 | "name": "attr" 74 | }, 75 | "arguments": null 76 | }, 77 | "variants": [ 78 | { 79 | "type": "Variant", 80 | "key": { 81 | "type": "Identifier", 82 | "name": "key" 83 | }, 84 | "value": { 85 | "type": "Pattern", 86 | "elements": [ 87 | { 88 | "type": "TextElement", 89 | "value": "Value" 90 | } 91 | ] 92 | }, 93 | "default": true 94 | } 95 | ] 96 | } 97 | } 98 | ] 99 | }, 100 | "attributes": [], 101 | "comment": { 102 | "type": "Comment", 103 | "content": "OK Term attributes may be used as selectors." 104 | } 105 | }, 106 | { 107 | "type": "Comment", 108 | "content": "ERROR Message attributes may not be used as selectors." 109 | }, 110 | { 111 | "type": "Junk", 112 | "annotations": [], 113 | "content": "message-attribute-expression-selector = {msg.attr ->\n *[key] Value\n}\n" 114 | } 115 | ] 116 | } 117 | -------------------------------------------------------------------------------- /test/fixtures/messages.ftl: -------------------------------------------------------------------------------- 1 | key01 = Value 2 | 3 | key02 = Value 4 | .attr = Attribute 5 | 6 | key02 = Value 7 | .attr1 = Attribute 1 8 | .attr2 = Attribute 2 9 | 10 | key03 = 11 | .attr = Attribute 12 | 13 | key04 = 14 | .attr1 = Attribute 1 15 | .attr2 = Attribute 2 16 | 17 | # < whitespace > 18 | key05 = 19 | .attr1 = Attribute 1 20 | 21 | no-whitespace=Value 22 | .attr1=Attribute 1 23 | 24 | extra-whitespace = Value 25 | .attr1 = Attribute 1 26 | 27 | key06 = {""} 28 | 29 | # JUNK Missing value 30 | key07 = 31 | 32 | # JUNK Missing = 33 | key08 34 | 35 | KEY09 = Value 09 36 | 37 | key-10 = Value 10 38 | key_11 = Value 11 39 | key-12- = Value 12 40 | key_13_ = Value 13 41 | 42 | # JUNK Invalid id 43 | 0err-14 = Value 14 44 | 45 | # JUNK Invalid id 46 | err-15? = Value 15 47 | 48 | # JUNK Invalid id 49 | err-ąę-16 = Value 16 50 | -------------------------------------------------------------------------------- /test/fixtures/mixed_entries.ftl: -------------------------------------------------------------------------------- 1 | # License Comment 2 | 3 | ### Resource Comment 4 | 5 | -brand-name = Aurora 6 | 7 | ## Group Comment 8 | 9 | key01 = 10 | .attr = Attribute 11 | 12 | ą=Invalid identifier 13 | ć=Another one 14 | 15 | # Message Comment 16 | key02 = Value 17 | 18 | # Standalone Comment 19 | .attr = Dangling attribute 20 | 21 | # There are 5 spaces on the line between key03 and key04. 22 | key03 = Value 03 23 | 24 | key04 = Value 04 25 | -------------------------------------------------------------------------------- /test/fixtures/mixed_entries.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Resource", 3 | "body": [ 4 | { 5 | "type": "Comment", 6 | "content": "License Comment" 7 | }, 8 | { 9 | "type": "ResourceComment", 10 | "content": "Resource Comment" 11 | }, 12 | { 13 | "type": "Term", 14 | "id": { 15 | "type": "Identifier", 16 | "name": "brand-name" 17 | }, 18 | "value": { 19 | "type": "Pattern", 20 | "elements": [ 21 | { 22 | "type": "TextElement", 23 | "value": "Aurora" 24 | } 25 | ] 26 | }, 27 | "attributes": [], 28 | "comment": null 29 | }, 30 | { 31 | "type": "GroupComment", 32 | "content": "Group Comment" 33 | }, 34 | { 35 | "type": "Message", 36 | "id": { 37 | "type": "Identifier", 38 | "name": "key01" 39 | }, 40 | "value": null, 41 | "attributes": [ 42 | { 43 | "type": "Attribute", 44 | "id": { 45 | "type": "Identifier", 46 | "name": "attr" 47 | }, 48 | "value": { 49 | "type": "Pattern", 50 | "elements": [ 51 | { 52 | "type": "TextElement", 53 | "value": "Attribute" 54 | } 55 | ] 56 | } 57 | } 58 | ], 59 | "comment": null 60 | }, 61 | { 62 | "type": "Junk", 63 | "annotations": [], 64 | "content": "ą=Invalid identifier\nć=Another one\n\n" 65 | }, 66 | { 67 | "type": "Message", 68 | "id": { 69 | "type": "Identifier", 70 | "name": "key02" 71 | }, 72 | "value": { 73 | "type": "Pattern", 74 | "elements": [ 75 | { 76 | "type": "TextElement", 77 | "value": "Value" 78 | } 79 | ] 80 | }, 81 | "attributes": [], 82 | "comment": { 83 | "type": "Comment", 84 | "content": "Message Comment" 85 | } 86 | }, 87 | { 88 | "type": "Comment", 89 | "content": "Standalone Comment" 90 | }, 91 | { 92 | "type": "Junk", 93 | "annotations": [], 94 | "content": " .attr = Dangling attribute\n\n" 95 | }, 96 | { 97 | "type": "Message", 98 | "id": { 99 | "type": "Identifier", 100 | "name": "key03" 101 | }, 102 | "value": { 103 | "type": "Pattern", 104 | "elements": [ 105 | { 106 | "type": "TextElement", 107 | "value": "Value 03" 108 | } 109 | ] 110 | }, 111 | "attributes": [], 112 | "comment": { 113 | "type": "Comment", 114 | "content": "There are 5 spaces on the line between key03 and key04." 115 | } 116 | }, 117 | { 118 | "type": "Message", 119 | "id": { 120 | "type": "Identifier", 121 | "name": "key04" 122 | }, 123 | "value": { 124 | "type": "Pattern", 125 | "elements": [ 126 | { 127 | "type": "TextElement", 128 | "value": "Value 04" 129 | } 130 | ] 131 | }, 132 | "attributes": [], 133 | "comment": null 134 | } 135 | ] 136 | } 137 | -------------------------------------------------------------------------------- /test/fixtures/multiline_values.ftl: -------------------------------------------------------------------------------- 1 | key01 = A multiline value 2 | continued on the next line 3 | 4 | and also down here. 5 | 6 | key02 = 7 | A multiline value starting 8 | on a new line. 9 | 10 | key03 = 11 | .attr = A multiline attribute value 12 | continued on the next line 13 | 14 | and also down here. 15 | 16 | key04 = 17 | .attr = 18 | A multiline attribute value 19 | staring on a new line 20 | 21 | key05 = 22 | 23 | A multiline value with non-standard 24 | 25 | indentation. 26 | 27 | key06 = 28 | A multiline value with {"placeables"} 29 | {"at"} the beginning and the end 30 | {"of lines"}{"."} 31 | 32 | key07 = 33 | {"A multiline value"} starting and ending {"with a placeable"} 34 | 35 | key08 = Leading and trailing whitespace. 36 | 37 | key09 = zero 38 | three 39 | two 40 | one 41 | zero 42 | 43 | key10 = 44 | two 45 | zero 46 | four 47 | 48 | key11 = 49 | 50 | 51 | two 52 | zero 53 | 54 | key12 = 55 | {"."} 56 | four 57 | 58 | key13 = 59 | four 60 | {"."} 61 | -------------------------------------------------------------------------------- /test/fixtures/numbers.ftl: -------------------------------------------------------------------------------- 1 | int-zero = {0} 2 | int-positive = {1} 3 | int-negative = {-1} 4 | int-negative-zero = {-0} 5 | 6 | int-positive-padded = {01} 7 | int-negative-padded = {-01} 8 | int-zero-padded = {00} 9 | int-negative-zero-padded = {-00} 10 | 11 | float-zero = {0.0} 12 | float-positive = {0.01} 13 | float-positive-one = {1.03} 14 | float-positive-without-fraction = {1.000} 15 | 16 | float-negative = {-0.01} 17 | float-negative-one = {-1.03} 18 | float-negative-zero = {-0.0} 19 | float-negative-without-fraction = {-1.000} 20 | 21 | float-positive-padded-left = {01.03} 22 | float-positive-padded-right = {1.0300} 23 | float-positive-padded-both = {01.0300} 24 | 25 | float-negative-padded-left = {-01.03} 26 | float-negative-padded-right = {-1.0300} 27 | float-negative-padded-both = {-01.0300} 28 | 29 | 30 | ## ERRORS 31 | 32 | err01 = {1.} 33 | err02 = {.02} 34 | err03 = {1.02.03} 35 | err04 = {1. 02} 36 | err05 = {1 .02} 37 | err06 = {- 1} 38 | err07 = {1,02} 39 | -------------------------------------------------------------------------------- /test/fixtures/obsolete.ftl: -------------------------------------------------------------------------------- 1 | ### The syntax in this file has been discontinued. It is no longer part of the 2 | ### Fluent specification and should not be implemented nor used. We're keeping 3 | ### these fixtures around to protect against accidental syntax reuse. 4 | 5 | 6 | ## Variant lists. 7 | 8 | message-variant-list = 9 | { 10 | *[key] Value 11 | } 12 | 13 | -term-variant-list = 14 | { 15 | *[key] Value 16 | } 17 | 18 | 19 | ## Variant expressions. 20 | 21 | message-variant-expression-placeable = {msg[case]} 22 | message-variant-expression-selector = {msg[case] -> 23 | *[key] Value 24 | } 25 | 26 | term-variant-expression-placeable = {-term[case]} 27 | term-variant-expression-selector = {-term[case] -> 28 | *[key] Value 29 | } 30 | -------------------------------------------------------------------------------- /test/fixtures/obsolete.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Resource", 3 | "body": [ 4 | { 5 | "type": "ResourceComment", 6 | "content": "The syntax in this file has been discontinued. It is no longer part of the\nFluent specification and should not be implemented nor used. We're keeping\nthese fixtures around to protect against accidental syntax reuse." 7 | }, 8 | { 9 | "type": "GroupComment", 10 | "content": "Variant lists." 11 | }, 12 | { 13 | "type": "Junk", 14 | "annotations": [], 15 | "content": "message-variant-list =\n {\n *[key] Value\n }\n\n" 16 | }, 17 | { 18 | "type": "Junk", 19 | "annotations": [], 20 | "content": "-term-variant-list =\n {\n *[key] Value\n }\n\n\n" 21 | }, 22 | { 23 | "type": "GroupComment", 24 | "content": "Variant expressions." 25 | }, 26 | { 27 | "type": "Junk", 28 | "annotations": [], 29 | "content": "message-variant-expression-placeable = {msg[case]}\n" 30 | }, 31 | { 32 | "type": "Junk", 33 | "annotations": [], 34 | "content": "message-variant-expression-selector = {msg[case] ->\n *[key] Value\n}\n\n" 35 | }, 36 | { 37 | "type": "Junk", 38 | "annotations": [], 39 | "content": "term-variant-expression-placeable = {-term[case]}\n" 40 | }, 41 | { 42 | "type": "Junk", 43 | "annotations": [], 44 | "content": "term-variant-expression-selector = {-term[case] ->\n *[key] Value\n}\n" 45 | } 46 | ] 47 | } 48 | -------------------------------------------------------------------------------- /test/fixtures/placeables.ftl: -------------------------------------------------------------------------------- 1 | nested-placeable = {{{1}}} 2 | padded-placeable = { 1 } 3 | sparse-placeable = { { 1 } } 4 | 5 | # ERROR Unmatched opening brace 6 | unmatched-open1 = { 1 7 | 8 | # ERROR Unmatched opening brace 9 | unmatched-open2 = {{ 1 } 10 | 11 | # ERROR Unmatched closing brace 12 | unmatched-close1 = 1 } 13 | 14 | # ERROR Unmatched closing brace 15 | unmatched-close2 = { 1 }} 16 | -------------------------------------------------------------------------------- /test/fixtures/placeables.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Resource", 3 | "body": [ 4 | { 5 | "type": "Message", 6 | "id": { 7 | "type": "Identifier", 8 | "name": "nested-placeable" 9 | }, 10 | "value": { 11 | "type": "Pattern", 12 | "elements": [ 13 | { 14 | "type": "Placeable", 15 | "expression": { 16 | "type": "Placeable", 17 | "expression": { 18 | "type": "Placeable", 19 | "expression": { 20 | "value": "1", 21 | "type": "NumberLiteral" 22 | } 23 | } 24 | } 25 | } 26 | ] 27 | }, 28 | "attributes": [], 29 | "comment": null 30 | }, 31 | { 32 | "type": "Message", 33 | "id": { 34 | "type": "Identifier", 35 | "name": "padded-placeable" 36 | }, 37 | "value": { 38 | "type": "Pattern", 39 | "elements": [ 40 | { 41 | "type": "Placeable", 42 | "expression": { 43 | "value": "1", 44 | "type": "NumberLiteral" 45 | } 46 | } 47 | ] 48 | }, 49 | "attributes": [], 50 | "comment": null 51 | }, 52 | { 53 | "type": "Message", 54 | "id": { 55 | "type": "Identifier", 56 | "name": "sparse-placeable" 57 | }, 58 | "value": { 59 | "type": "Pattern", 60 | "elements": [ 61 | { 62 | "type": "Placeable", 63 | "expression": { 64 | "type": "Placeable", 65 | "expression": { 66 | "value": "1", 67 | "type": "NumberLiteral" 68 | } 69 | } 70 | } 71 | ] 72 | }, 73 | "attributes": [], 74 | "comment": null 75 | }, 76 | { 77 | "type": "Comment", 78 | "content": "ERROR Unmatched opening brace" 79 | }, 80 | { 81 | "type": "Junk", 82 | "annotations": [], 83 | "content": "unmatched-open1 = { 1\n\n" 84 | }, 85 | { 86 | "type": "Comment", 87 | "content": "ERROR Unmatched opening brace" 88 | }, 89 | { 90 | "type": "Junk", 91 | "annotations": [], 92 | "content": "unmatched-open2 = {{ 1 }\n\n" 93 | }, 94 | { 95 | "type": "Comment", 96 | "content": "ERROR Unmatched closing brace" 97 | }, 98 | { 99 | "type": "Junk", 100 | "annotations": [], 101 | "content": "unmatched-close1 = 1 }\n\n" 102 | }, 103 | { 104 | "type": "Comment", 105 | "content": "ERROR Unmatched closing brace" 106 | }, 107 | { 108 | "type": "Junk", 109 | "annotations": [], 110 | "content": "unmatched-close2 = { 1 }}\n" 111 | } 112 | ] 113 | } 114 | -------------------------------------------------------------------------------- /test/fixtures/reference_expressions.ftl: -------------------------------------------------------------------------------- 1 | ## Reference expressions in placeables. 2 | 3 | message-reference-placeable = {msg} 4 | term-reference-placeable = {-term} 5 | variable-reference-placeable = {$var} 6 | 7 | # Function references are invalid outside of call expressions. 8 | # This parses as a valid MessageReference. 9 | function-reference-placeable = {FUN} 10 | 11 | 12 | ## Reference expressions in selectors. 13 | 14 | variable-reference-selector = {$var -> 15 | *[key] Value 16 | } 17 | 18 | # ERROR Message values may not be used as selectors. 19 | message-reference-selector = {msg -> 20 | *[key] Value 21 | } 22 | # ERROR Term values may not be used as selectors. 23 | term-reference-selector = {-term -> 24 | *[key] Value 25 | } 26 | # ERROR Function references are invalid outside of call expressions, and this 27 | # parses as a MessageReference which isn't a valid selector. 28 | function-expression-selector = {FUN -> 29 | *[key] Value 30 | } 31 | -------------------------------------------------------------------------------- /test/fixtures/reference_expressions.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Resource", 3 | "body": [ 4 | { 5 | "type": "GroupComment", 6 | "content": "Reference expressions in placeables." 7 | }, 8 | { 9 | "type": "Message", 10 | "id": { 11 | "type": "Identifier", 12 | "name": "message-reference-placeable" 13 | }, 14 | "value": { 15 | "type": "Pattern", 16 | "elements": [ 17 | { 18 | "type": "Placeable", 19 | "expression": { 20 | "type": "MessageReference", 21 | "id": { 22 | "type": "Identifier", 23 | "name": "msg" 24 | }, 25 | "attribute": null 26 | } 27 | } 28 | ] 29 | }, 30 | "attributes": [], 31 | "comment": null 32 | }, 33 | { 34 | "type": "Message", 35 | "id": { 36 | "type": "Identifier", 37 | "name": "term-reference-placeable" 38 | }, 39 | "value": { 40 | "type": "Pattern", 41 | "elements": [ 42 | { 43 | "type": "Placeable", 44 | "expression": { 45 | "type": "TermReference", 46 | "id": { 47 | "type": "Identifier", 48 | "name": "term" 49 | }, 50 | "attribute": null, 51 | "arguments": null 52 | } 53 | } 54 | ] 55 | }, 56 | "attributes": [], 57 | "comment": null 58 | }, 59 | { 60 | "type": "Message", 61 | "id": { 62 | "type": "Identifier", 63 | "name": "variable-reference-placeable" 64 | }, 65 | "value": { 66 | "type": "Pattern", 67 | "elements": [ 68 | { 69 | "type": "Placeable", 70 | "expression": { 71 | "type": "VariableReference", 72 | "id": { 73 | "type": "Identifier", 74 | "name": "var" 75 | } 76 | } 77 | } 78 | ] 79 | }, 80 | "attributes": [], 81 | "comment": null 82 | }, 83 | { 84 | "type": "Message", 85 | "id": { 86 | "type": "Identifier", 87 | "name": "function-reference-placeable" 88 | }, 89 | "value": { 90 | "type": "Pattern", 91 | "elements": [ 92 | { 93 | "type": "Placeable", 94 | "expression": { 95 | "type": "MessageReference", 96 | "id": { 97 | "type": "Identifier", 98 | "name": "FUN" 99 | }, 100 | "attribute": null 101 | } 102 | } 103 | ] 104 | }, 105 | "attributes": [], 106 | "comment": { 107 | "type": "Comment", 108 | "content": "Function references are invalid outside of call expressions.\nThis parses as a valid MessageReference." 109 | } 110 | }, 111 | { 112 | "type": "GroupComment", 113 | "content": "Reference expressions in selectors." 114 | }, 115 | { 116 | "type": "Message", 117 | "id": { 118 | "type": "Identifier", 119 | "name": "variable-reference-selector" 120 | }, 121 | "value": { 122 | "type": "Pattern", 123 | "elements": [ 124 | { 125 | "type": "Placeable", 126 | "expression": { 127 | "type": "SelectExpression", 128 | "selector": { 129 | "type": "VariableReference", 130 | "id": { 131 | "type": "Identifier", 132 | "name": "var" 133 | } 134 | }, 135 | "variants": [ 136 | { 137 | "type": "Variant", 138 | "key": { 139 | "type": "Identifier", 140 | "name": "key" 141 | }, 142 | "value": { 143 | "type": "Pattern", 144 | "elements": [ 145 | { 146 | "type": "TextElement", 147 | "value": "Value" 148 | } 149 | ] 150 | }, 151 | "default": true 152 | } 153 | ] 154 | } 155 | } 156 | ] 157 | }, 158 | "attributes": [], 159 | "comment": null 160 | }, 161 | { 162 | "type": "Comment", 163 | "content": "ERROR Message values may not be used as selectors." 164 | }, 165 | { 166 | "type": "Junk", 167 | "annotations": [], 168 | "content": "message-reference-selector = {msg ->\n *[key] Value\n}\n" 169 | }, 170 | { 171 | "type": "Comment", 172 | "content": "ERROR Term values may not be used as selectors." 173 | }, 174 | { 175 | "type": "Junk", 176 | "annotations": [], 177 | "content": "term-reference-selector = {-term ->\n *[key] Value\n}\n" 178 | }, 179 | { 180 | "type": "Comment", 181 | "content": "ERROR Function references are invalid outside of call expressions, and this\nparses as a MessageReference which isn't a valid selector." 182 | }, 183 | { 184 | "type": "Junk", 185 | "annotations": [], 186 | "content": "function-expression-selector = {FUN ->\n *[key] Value\n}\n" 187 | } 188 | ] 189 | } 190 | -------------------------------------------------------------------------------- /test/fixtures/select_expressions.ftl: -------------------------------------------------------------------------------- 1 | new-messages = 2 | { BUILTIN() -> 3 | [0] Zero 4 | *[other] {""}Other 5 | } 6 | 7 | valid-selector-term-attribute = 8 | { -term.case -> 9 | *[key] value 10 | } 11 | 12 | # ERROR Term values are not valid selectors 13 | invalid-selector-term-value = 14 | { -term -> 15 | *[key] value 16 | } 17 | 18 | # ERROR CallExpressions on Terms are similar to TermReferences 19 | invalid-selector-term-variant = 20 | { -term(case: "nominative") -> 21 | *[key] value 22 | } 23 | 24 | # ERROR Nested expressions are not valid selectors 25 | invalid-selector-nested-expression = 26 | { { 3 } -> 27 | *[key] default 28 | } 29 | 30 | # ERROR Select expressions are not valid selectors 31 | invalid-selector-select-expression = 32 | { { $sel -> 33 | *[key] value 34 | } -> 35 | *[key] default 36 | } 37 | 38 | empty-variant = 39 | { $sel -> 40 | *[key] {""} 41 | } 42 | 43 | reduced-whitespace = 44 | {FOO()-> 45 | *[key] {""} 46 | } 47 | 48 | nested-select = 49 | { $sel -> 50 | *[one] { $sel -> 51 | *[two] Value 52 | } 53 | } 54 | 55 | # ERROR Missing selector 56 | missing-selector = 57 | { 58 | *[key] Value 59 | } 60 | 61 | # ERROR Missing line end after variant list 62 | missing-line-end = 63 | { $sel -> 64 | *[key] Value} 65 | -------------------------------------------------------------------------------- /test/fixtures/select_indent.ftl: -------------------------------------------------------------------------------- 1 | select-1tbs-inline = { $selector -> 2 | *[key] Value 3 | } 4 | 5 | select-1tbs-newline = { 6 | $selector -> 7 | *[key] Value 8 | } 9 | 10 | select-1tbs-indent = { 11 | $selector -> 12 | *[key] Value 13 | } 14 | 15 | select-allman-inline = 16 | { $selector -> 17 | *[key] Value 18 | [other] Other 19 | } 20 | 21 | select-allman-newline = 22 | { 23 | $selector -> 24 | *[key] Value 25 | } 26 | 27 | select-allman-indent = 28 | { 29 | $selector -> 30 | *[key] Value 31 | } 32 | 33 | select-gnu-inline = 34 | { $selector -> 35 | *[key] Value 36 | } 37 | 38 | select-gnu-newline = 39 | { 40 | $selector -> 41 | *[key] Value 42 | } 43 | 44 | select-gnu-indent = 45 | { 46 | $selector -> 47 | *[key] Value 48 | } 49 | 50 | select-no-indent = 51 | { 52 | $selector -> 53 | *[key] Value 54 | [other] Other 55 | } 56 | 57 | select-no-indent-multiline = 58 | { 59 | $selector -> 60 | *[key] Value 61 | Continued 62 | [other] 63 | Other 64 | Multiline 65 | } 66 | 67 | # ERROR (Multiline text must be indented) 68 | select-no-indent-multiline = { $selector -> 69 | *[key] Value 70 | Continued without indent. 71 | } 72 | 73 | select-flat = 74 | { 75 | $selector 76 | -> 77 | *[ 78 | key 79 | ] Value 80 | [ 81 | other 82 | ] Other 83 | } 84 | 85 | # Each line ends with 5 spaces. 86 | select-flat-with-trailing-spaces = 87 | { 88 | $selector 89 | -> 90 | *[ 91 | key 92 | ] Value 93 | [ 94 | other 95 | ] Other 96 | } 97 | -------------------------------------------------------------------------------- /test/fixtures/sparse_entries.ftl: -------------------------------------------------------------------------------- 1 | key01 = 2 | 3 | 4 | Value 5 | 6 | key02 = 7 | 8 | 9 | .attr = Attribute 10 | 11 | 12 | key03 = 13 | Value 14 | Continued 15 | 16 | 17 | Over multiple 18 | Lines 19 | 20 | 21 | 22 | .attr = Attribute 23 | 24 | 25 | key05 = Value 26 | 27 | key06 = { 1 -> 28 | 29 | 30 | [one] One 31 | 32 | 33 | 34 | 35 | *[two] Two 36 | 37 | 38 | 39 | } 40 | -------------------------------------------------------------------------------- /test/fixtures/sparse_entries.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Resource", 3 | "body": [ 4 | { 5 | "type": "Message", 6 | "id": { 7 | "type": "Identifier", 8 | "name": "key01" 9 | }, 10 | "value": { 11 | "type": "Pattern", 12 | "elements": [ 13 | { 14 | "type": "TextElement", 15 | "value": "Value" 16 | } 17 | ] 18 | }, 19 | "attributes": [], 20 | "comment": null 21 | }, 22 | { 23 | "type": "Message", 24 | "id": { 25 | "type": "Identifier", 26 | "name": "key02" 27 | }, 28 | "value": null, 29 | "attributes": [ 30 | { 31 | "type": "Attribute", 32 | "id": { 33 | "type": "Identifier", 34 | "name": "attr" 35 | }, 36 | "value": { 37 | "type": "Pattern", 38 | "elements": [ 39 | { 40 | "type": "TextElement", 41 | "value": "Attribute" 42 | } 43 | ] 44 | } 45 | } 46 | ], 47 | "comment": null 48 | }, 49 | { 50 | "type": "Message", 51 | "id": { 52 | "type": "Identifier", 53 | "name": "key03" 54 | }, 55 | "value": { 56 | "type": "Pattern", 57 | "elements": [ 58 | { 59 | "type": "TextElement", 60 | "value": "Value\nContinued\n\n\nOver multiple\nLines" 61 | } 62 | ] 63 | }, 64 | "attributes": [ 65 | { 66 | "type": "Attribute", 67 | "id": { 68 | "type": "Identifier", 69 | "name": "attr" 70 | }, 71 | "value": { 72 | "type": "Pattern", 73 | "elements": [ 74 | { 75 | "type": "TextElement", 76 | "value": "Attribute" 77 | } 78 | ] 79 | } 80 | } 81 | ], 82 | "comment": null 83 | }, 84 | { 85 | "type": "Message", 86 | "id": { 87 | "type": "Identifier", 88 | "name": "key05" 89 | }, 90 | "value": { 91 | "type": "Pattern", 92 | "elements": [ 93 | { 94 | "type": "TextElement", 95 | "value": "Value" 96 | } 97 | ] 98 | }, 99 | "attributes": [], 100 | "comment": null 101 | }, 102 | { 103 | "type": "Message", 104 | "id": { 105 | "type": "Identifier", 106 | "name": "key06" 107 | }, 108 | "value": { 109 | "type": "Pattern", 110 | "elements": [ 111 | { 112 | "type": "Placeable", 113 | "expression": { 114 | "type": "SelectExpression", 115 | "selector": { 116 | "value": "1", 117 | "type": "NumberLiteral" 118 | }, 119 | "variants": [ 120 | { 121 | "type": "Variant", 122 | "key": { 123 | "type": "Identifier", 124 | "name": "one" 125 | }, 126 | "value": { 127 | "type": "Pattern", 128 | "elements": [ 129 | { 130 | "type": "TextElement", 131 | "value": "One" 132 | } 133 | ] 134 | }, 135 | "default": false 136 | }, 137 | { 138 | "type": "Variant", 139 | "key": { 140 | "type": "Identifier", 141 | "name": "two" 142 | }, 143 | "value": { 144 | "type": "Pattern", 145 | "elements": [ 146 | { 147 | "type": "TextElement", 148 | "value": "Two" 149 | } 150 | ] 151 | }, 152 | "default": true 153 | } 154 | ] 155 | } 156 | } 157 | ] 158 | }, 159 | "attributes": [], 160 | "comment": null 161 | } 162 | ] 163 | } 164 | -------------------------------------------------------------------------------- /test/fixtures/special_chars.ftl: -------------------------------------------------------------------------------- 1 | ## OK 2 | 3 | bracket-inline = [Value] 4 | dot-inline = .Value 5 | star-inline = *Value 6 | 7 | ## ERRORS 8 | 9 | bracket-newline = 10 | [Value] 11 | dot-newline = 12 | .Value 13 | star-newline = 14 | *Value 15 | -------------------------------------------------------------------------------- /test/fixtures/special_chars.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Resource", 3 | "body": [ 4 | { 5 | "type": "GroupComment", 6 | "content": "OK" 7 | }, 8 | { 9 | "type": "Message", 10 | "id": { 11 | "type": "Identifier", 12 | "name": "bracket-inline" 13 | }, 14 | "value": { 15 | "type": "Pattern", 16 | "elements": [ 17 | { 18 | "type": "TextElement", 19 | "value": "[Value]" 20 | } 21 | ] 22 | }, 23 | "attributes": [], 24 | "comment": null 25 | }, 26 | { 27 | "type": "Message", 28 | "id": { 29 | "type": "Identifier", 30 | "name": "dot-inline" 31 | }, 32 | "value": { 33 | "type": "Pattern", 34 | "elements": [ 35 | { 36 | "type": "TextElement", 37 | "value": ".Value" 38 | } 39 | ] 40 | }, 41 | "attributes": [], 42 | "comment": null 43 | }, 44 | { 45 | "type": "Message", 46 | "id": { 47 | "type": "Identifier", 48 | "name": "star-inline" 49 | }, 50 | "value": { 51 | "type": "Pattern", 52 | "elements": [ 53 | { 54 | "type": "TextElement", 55 | "value": "*Value" 56 | } 57 | ] 58 | }, 59 | "attributes": [], 60 | "comment": null 61 | }, 62 | { 63 | "type": "GroupComment", 64 | "content": "ERRORS" 65 | }, 66 | { 67 | "type": "Junk", 68 | "annotations": [], 69 | "content": "bracket-newline =\n [Value]\n" 70 | }, 71 | { 72 | "type": "Junk", 73 | "annotations": [], 74 | "content": "dot-newline =\n .Value\n" 75 | }, 76 | { 77 | "type": "Junk", 78 | "annotations": [], 79 | "content": "star-newline =\n *Value\n" 80 | } 81 | ] 82 | } 83 | -------------------------------------------------------------------------------- /test/fixtures/tab.ftl: -------------------------------------------------------------------------------- 1 | # OK (tab after = is part of the value) 2 | key01 = Value 01 3 | 4 | # Error (tab before =) 5 | key02 = Value 02 6 | 7 | # Error (tab is not a valid indent) 8 | key03 = 9 | This line isn't properly indented. 10 | 11 | # Partial Error (tab is not a valid indent) 12 | key04 = 13 | This line is indented by 4 spaces, 14 | whereas this line by 1 tab. 15 | -------------------------------------------------------------------------------- /test/fixtures/tab.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Resource", 3 | "body": [ 4 | { 5 | "type": "Message", 6 | "id": { 7 | "type": "Identifier", 8 | "name": "key01" 9 | }, 10 | "value": { 11 | "type": "Pattern", 12 | "elements": [ 13 | { 14 | "type": "TextElement", 15 | "value": "\tValue 01" 16 | } 17 | ] 18 | }, 19 | "attributes": [], 20 | "comment": { 21 | "type": "Comment", 22 | "content": "OK (tab after = is part of the value)" 23 | } 24 | }, 25 | { 26 | "type": "Comment", 27 | "content": "Error (tab before =)" 28 | }, 29 | { 30 | "type": "Junk", 31 | "annotations": [], 32 | "content": "key02\t= Value 02\n\n" 33 | }, 34 | { 35 | "type": "Comment", 36 | "content": "Error (tab is not a valid indent)" 37 | }, 38 | { 39 | "type": "Junk", 40 | "annotations": [], 41 | "content": "key03 =\n\tThis line isn't properly indented.\n\n" 42 | }, 43 | { 44 | "type": "Message", 45 | "id": { 46 | "type": "Identifier", 47 | "name": "key04" 48 | }, 49 | "value": { 50 | "type": "Pattern", 51 | "elements": [ 52 | { 53 | "type": "TextElement", 54 | "value": "This line is indented by 4 spaces," 55 | } 56 | ] 57 | }, 58 | "attributes": [], 59 | "comment": { 60 | "type": "Comment", 61 | "content": "Partial Error (tab is not a valid indent)" 62 | } 63 | }, 64 | { 65 | "type": "Junk", 66 | "annotations": [], 67 | "content": "\twhereas this line by 1 tab.\n" 68 | } 69 | ] 70 | } 71 | -------------------------------------------------------------------------------- /test/fixtures/term_parameters.ftl: -------------------------------------------------------------------------------- 1 | -term = { $arg -> 2 | *[key] Value 3 | } 4 | 5 | key01 = { -term } 6 | key02 = { -term () } 7 | key03 = { -term(arg: 1) } 8 | key04 = { -term("positional", narg1: 1, narg2: 2) } 9 | -------------------------------------------------------------------------------- /test/fixtures/term_parameters.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Resource", 3 | "body": [ 4 | { 5 | "type": "Term", 6 | "id": { 7 | "type": "Identifier", 8 | "name": "term" 9 | }, 10 | "value": { 11 | "type": "Pattern", 12 | "elements": [ 13 | { 14 | "type": "Placeable", 15 | "expression": { 16 | "type": "SelectExpression", 17 | "selector": { 18 | "type": "VariableReference", 19 | "id": { 20 | "type": "Identifier", 21 | "name": "arg" 22 | } 23 | }, 24 | "variants": [ 25 | { 26 | "type": "Variant", 27 | "key": { 28 | "type": "Identifier", 29 | "name": "key" 30 | }, 31 | "value": { 32 | "type": "Pattern", 33 | "elements": [ 34 | { 35 | "type": "TextElement", 36 | "value": "Value" 37 | } 38 | ] 39 | }, 40 | "default": true 41 | } 42 | ] 43 | } 44 | } 45 | ] 46 | }, 47 | "attributes": [], 48 | "comment": null 49 | }, 50 | { 51 | "type": "Message", 52 | "id": { 53 | "type": "Identifier", 54 | "name": "key01" 55 | }, 56 | "value": { 57 | "type": "Pattern", 58 | "elements": [ 59 | { 60 | "type": "Placeable", 61 | "expression": { 62 | "type": "TermReference", 63 | "id": { 64 | "type": "Identifier", 65 | "name": "term" 66 | }, 67 | "attribute": null, 68 | "arguments": null 69 | } 70 | } 71 | ] 72 | }, 73 | "attributes": [], 74 | "comment": null 75 | }, 76 | { 77 | "type": "Message", 78 | "id": { 79 | "type": "Identifier", 80 | "name": "key02" 81 | }, 82 | "value": { 83 | "type": "Pattern", 84 | "elements": [ 85 | { 86 | "type": "Placeable", 87 | "expression": { 88 | "type": "TermReference", 89 | "id": { 90 | "type": "Identifier", 91 | "name": "term" 92 | }, 93 | "attribute": null, 94 | "arguments": { 95 | "type": "CallArguments", 96 | "positional": [], 97 | "named": [] 98 | } 99 | } 100 | } 101 | ] 102 | }, 103 | "attributes": [], 104 | "comment": null 105 | }, 106 | { 107 | "type": "Message", 108 | "id": { 109 | "type": "Identifier", 110 | "name": "key03" 111 | }, 112 | "value": { 113 | "type": "Pattern", 114 | "elements": [ 115 | { 116 | "type": "Placeable", 117 | "expression": { 118 | "type": "TermReference", 119 | "id": { 120 | "type": "Identifier", 121 | "name": "term" 122 | }, 123 | "attribute": null, 124 | "arguments": { 125 | "type": "CallArguments", 126 | "positional": [], 127 | "named": [ 128 | { 129 | "type": "NamedArgument", 130 | "name": { 131 | "type": "Identifier", 132 | "name": "arg" 133 | }, 134 | "value": { 135 | "value": "1", 136 | "type": "NumberLiteral" 137 | } 138 | } 139 | ] 140 | } 141 | } 142 | } 143 | ] 144 | }, 145 | "attributes": [], 146 | "comment": null 147 | }, 148 | { 149 | "type": "Message", 150 | "id": { 151 | "type": "Identifier", 152 | "name": "key04" 153 | }, 154 | "value": { 155 | "type": "Pattern", 156 | "elements": [ 157 | { 158 | "type": "Placeable", 159 | "expression": { 160 | "type": "TermReference", 161 | "id": { 162 | "type": "Identifier", 163 | "name": "term" 164 | }, 165 | "attribute": null, 166 | "arguments": { 167 | "type": "CallArguments", 168 | "positional": [ 169 | { 170 | "value": "positional", 171 | "type": "StringLiteral" 172 | } 173 | ], 174 | "named": [ 175 | { 176 | "type": "NamedArgument", 177 | "name": { 178 | "type": "Identifier", 179 | "name": "narg1" 180 | }, 181 | "value": { 182 | "value": "1", 183 | "type": "NumberLiteral" 184 | } 185 | }, 186 | { 187 | "type": "NamedArgument", 188 | "name": { 189 | "type": "Identifier", 190 | "name": "narg2" 191 | }, 192 | "value": { 193 | "value": "2", 194 | "type": "NumberLiteral" 195 | } 196 | } 197 | ] 198 | } 199 | } 200 | } 201 | ] 202 | }, 203 | "attributes": [], 204 | "comment": null 205 | } 206 | ] 207 | } 208 | -------------------------------------------------------------------------------- /test/fixtures/terms.ftl: -------------------------------------------------------------------------------- 1 | -term01 = Value 2 | .attr = Attribute 3 | 4 | -term02 = {""} 5 | 6 | # JUNK Missing value 7 | -term03 = 8 | .attr = Attribute 9 | 10 | # JUNK Missing value 11 | # < whitespace > 12 | -term04 = 13 | .attr1 = Attribute 1 14 | 15 | # JUNK Missing value 16 | -term05 = 17 | 18 | # JUNK Missing value 19 | # < whitespace > 20 | -term06 = 21 | 22 | # JUNK Missing = 23 | -term07 24 | 25 | -term08=Value 26 | .attr=Attribute 27 | 28 | -term09 = Value 29 | .attr = Attribute 30 | -------------------------------------------------------------------------------- /test/fixtures/terms.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Resource", 3 | "body": [ 4 | { 5 | "type": "Term", 6 | "id": { 7 | "type": "Identifier", 8 | "name": "term01" 9 | }, 10 | "value": { 11 | "type": "Pattern", 12 | "elements": [ 13 | { 14 | "type": "TextElement", 15 | "value": "Value" 16 | } 17 | ] 18 | }, 19 | "attributes": [ 20 | { 21 | "type": "Attribute", 22 | "id": { 23 | "type": "Identifier", 24 | "name": "attr" 25 | }, 26 | "value": { 27 | "type": "Pattern", 28 | "elements": [ 29 | { 30 | "type": "TextElement", 31 | "value": "Attribute" 32 | } 33 | ] 34 | } 35 | } 36 | ], 37 | "comment": null 38 | }, 39 | { 40 | "type": "Term", 41 | "id": { 42 | "type": "Identifier", 43 | "name": "term02" 44 | }, 45 | "value": { 46 | "type": "Pattern", 47 | "elements": [ 48 | { 49 | "type": "Placeable", 50 | "expression": { 51 | "value": "", 52 | "type": "StringLiteral" 53 | } 54 | } 55 | ] 56 | }, 57 | "attributes": [], 58 | "comment": null 59 | }, 60 | { 61 | "type": "Comment", 62 | "content": "JUNK Missing value" 63 | }, 64 | { 65 | "type": "Junk", 66 | "annotations": [], 67 | "content": "-term03 =\n .attr = Attribute\n\n" 68 | }, 69 | { 70 | "type": "Comment", 71 | "content": "JUNK Missing value\n < whitespace >" 72 | }, 73 | { 74 | "type": "Junk", 75 | "annotations": [], 76 | "content": "-term04 = \n .attr1 = Attribute 1\n\n" 77 | }, 78 | { 79 | "type": "Comment", 80 | "content": "JUNK Missing value" 81 | }, 82 | { 83 | "type": "Junk", 84 | "annotations": [], 85 | "content": "-term05 =\n\n" 86 | }, 87 | { 88 | "type": "Comment", 89 | "content": "JUNK Missing value\n < whitespace >" 90 | }, 91 | { 92 | "type": "Junk", 93 | "annotations": [], 94 | "content": "-term06 = \n\n" 95 | }, 96 | { 97 | "type": "Comment", 98 | "content": "JUNK Missing =" 99 | }, 100 | { 101 | "type": "Junk", 102 | "annotations": [], 103 | "content": "-term07\n\n" 104 | }, 105 | { 106 | "type": "Term", 107 | "id": { 108 | "type": "Identifier", 109 | "name": "term08" 110 | }, 111 | "value": { 112 | "type": "Pattern", 113 | "elements": [ 114 | { 115 | "type": "TextElement", 116 | "value": "Value" 117 | } 118 | ] 119 | }, 120 | "attributes": [ 121 | { 122 | "type": "Attribute", 123 | "id": { 124 | "type": "Identifier", 125 | "name": "attr" 126 | }, 127 | "value": { 128 | "type": "Pattern", 129 | "elements": [ 130 | { 131 | "type": "TextElement", 132 | "value": "Attribute" 133 | } 134 | ] 135 | } 136 | } 137 | ], 138 | "comment": null 139 | }, 140 | { 141 | "type": "Term", 142 | "id": { 143 | "type": "Identifier", 144 | "name": "term09" 145 | }, 146 | "value": { 147 | "type": "Pattern", 148 | "elements": [ 149 | { 150 | "type": "TextElement", 151 | "value": "Value" 152 | } 153 | ] 154 | }, 155 | "attributes": [ 156 | { 157 | "type": "Attribute", 158 | "id": { 159 | "type": "Identifier", 160 | "name": "attr" 161 | }, 162 | "value": { 163 | "type": "Pattern", 164 | "elements": [ 165 | { 166 | "type": "TextElement", 167 | "value": "Attribute" 168 | } 169 | ] 170 | } 171 | } 172 | ], 173 | "comment": null 174 | } 175 | ] 176 | } 177 | -------------------------------------------------------------------------------- /test/fixtures/variables.ftl: -------------------------------------------------------------------------------- 1 | key01 = {$var} 2 | key02 = { $var } 3 | key03 = { 4 | $var 5 | } 6 | key04 = { 7 | $var} 8 | 9 | 10 | ## Errors 11 | 12 | # ERROR Missing variable identifier 13 | err01 = {$} 14 | # ERROR Double $$ 15 | err02 = {$$var} 16 | # ERROR Invalid first char of the identifier 17 | err03 = {$-var} 18 | -------------------------------------------------------------------------------- /test/fixtures/variables.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Resource", 3 | "body": [ 4 | { 5 | "type": "Message", 6 | "id": { 7 | "type": "Identifier", 8 | "name": "key01" 9 | }, 10 | "value": { 11 | "type": "Pattern", 12 | "elements": [ 13 | { 14 | "type": "Placeable", 15 | "expression": { 16 | "type": "VariableReference", 17 | "id": { 18 | "type": "Identifier", 19 | "name": "var" 20 | } 21 | } 22 | } 23 | ] 24 | }, 25 | "attributes": [], 26 | "comment": null 27 | }, 28 | { 29 | "type": "Message", 30 | "id": { 31 | "type": "Identifier", 32 | "name": "key02" 33 | }, 34 | "value": { 35 | "type": "Pattern", 36 | "elements": [ 37 | { 38 | "type": "Placeable", 39 | "expression": { 40 | "type": "VariableReference", 41 | "id": { 42 | "type": "Identifier", 43 | "name": "var" 44 | } 45 | } 46 | } 47 | ] 48 | }, 49 | "attributes": [], 50 | "comment": null 51 | }, 52 | { 53 | "type": "Message", 54 | "id": { 55 | "type": "Identifier", 56 | "name": "key03" 57 | }, 58 | "value": { 59 | "type": "Pattern", 60 | "elements": [ 61 | { 62 | "type": "Placeable", 63 | "expression": { 64 | "type": "VariableReference", 65 | "id": { 66 | "type": "Identifier", 67 | "name": "var" 68 | } 69 | } 70 | } 71 | ] 72 | }, 73 | "attributes": [], 74 | "comment": null 75 | }, 76 | { 77 | "type": "Message", 78 | "id": { 79 | "type": "Identifier", 80 | "name": "key04" 81 | }, 82 | "value": { 83 | "type": "Pattern", 84 | "elements": [ 85 | { 86 | "type": "Placeable", 87 | "expression": { 88 | "type": "VariableReference", 89 | "id": { 90 | "type": "Identifier", 91 | "name": "var" 92 | } 93 | } 94 | } 95 | ] 96 | }, 97 | "attributes": [], 98 | "comment": null 99 | }, 100 | { 101 | "type": "GroupComment", 102 | "content": "Errors" 103 | }, 104 | { 105 | "type": "Comment", 106 | "content": "ERROR Missing variable identifier" 107 | }, 108 | { 109 | "type": "Junk", 110 | "annotations": [], 111 | "content": "err01 = {$}\n" 112 | }, 113 | { 114 | "type": "Comment", 115 | "content": "ERROR Double $$" 116 | }, 117 | { 118 | "type": "Junk", 119 | "annotations": [], 120 | "content": "err02 = {$$var}\n" 121 | }, 122 | { 123 | "type": "Comment", 124 | "content": "ERROR Invalid first char of the identifier" 125 | }, 126 | { 127 | "type": "Junk", 128 | "annotations": [], 129 | "content": "err03 = {$-var}\n" 130 | } 131 | ] 132 | } 133 | -------------------------------------------------------------------------------- /test/fixtures/variant_keys.ftl: -------------------------------------------------------------------------------- 1 | simple-identifier = 2 | { $sel -> 3 | *[key] value 4 | } 5 | 6 | identifier-surrounded-by-whitespace = 7 | { $sel -> 8 | *[ key ] value 9 | } 10 | 11 | int-number = 12 | { $sel -> 13 | *[1] value 14 | } 15 | 16 | float-number = 17 | { $sel -> 18 | *[3.14] value 19 | } 20 | 21 | # ERROR 22 | invalid-identifier = 23 | { $sel -> 24 | *[two words] value 25 | } 26 | 27 | # ERROR 28 | invalid-int = 29 | { $sel -> 30 | *[1 apple] value 31 | } 32 | 33 | # ERROR 34 | invalid-int = 35 | { $sel -> 36 | *[3.14 apples] value 37 | } 38 | -------------------------------------------------------------------------------- /test/fixtures/whitespace_in_value.ftl: -------------------------------------------------------------------------------- 1 | # Caution, lines 6 and 7 contain white-space-only lines 2 | key = 3 | first line 4 | 5 | 6 | 7 | 8 | 9 | 10 | last line 11 | -------------------------------------------------------------------------------- /test/fixtures/whitespace_in_value.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Resource", 3 | "body": [ 4 | { 5 | "type": "Message", 6 | "id": { 7 | "type": "Identifier", 8 | "name": "key" 9 | }, 10 | "value": { 11 | "type": "Pattern", 12 | "elements": [ 13 | { 14 | "type": "TextElement", 15 | "value": "first line\n\n\n\n\n\n\nlast line" 16 | } 17 | ] 18 | }, 19 | "attributes": [], 20 | "comment": { 21 | "type": "Comment", 22 | "content": "Caution, lines 6 and 7 contain white-space-only lines" 23 | } 24 | } 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /test/fixtures/zero_length.ftl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/projectfluent/fluent/10a1bc60bee843c14a30216fa4cebdc559bf2076/test/fixtures/zero_length.ftl -------------------------------------------------------------------------------- /test/fixtures/zero_length.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Resource", 3 | "body": [] 4 | } 5 | -------------------------------------------------------------------------------- /test/literals.js: -------------------------------------------------------------------------------- 1 | /* eslint quotes: "off" */ 2 | import suite from "./suite.js"; 3 | import {StringLiteral, NumberLiteral} from "../syntax/grammar.js"; 4 | 5 | if (process.argv.length > 2) { 6 | console.error("Usage: node literals.js"); 7 | process.exit(1); 8 | } 9 | 10 | suite(tester => { 11 | let title = node => `${node.type} {value: "${node.value}"}`; 12 | let test = (result, expected) => 13 | result.fold( 14 | node => tester.deep_equal(title(node), node.parse(), expected), 15 | tester.fail); 16 | 17 | // Unescape raw value of StringLiterals. 18 | { 19 | test(StringLiteral.run(`"abc"`), {value: "abc"}); 20 | test(StringLiteral.run(`"\\""`), {value: "\""}); 21 | test(StringLiteral.run(`"\\\\"`), {value: "\\"}); 22 | 23 | // Unicode escapes. 24 | test(StringLiteral.run(`"\\u0041"`), {value: "A"}); 25 | test(StringLiteral.run(`"\\\\u0041"`), {value: "\\u0041"}); 26 | test(StringLiteral.run(`"\\U01F602"`), {value: "😂"}); 27 | test(StringLiteral.run(`"\\\\U01F602"`), {value: "\\U01F602"}); 28 | 29 | // Trailing "00" are part of the literal values. 30 | test(StringLiteral.run(`"\\u004100"`), {value: "A00"}); 31 | test(StringLiteral.run(`"\\U01F60200"`), {value: "😂00"}); 32 | 33 | // Literal braces. 34 | test(StringLiteral.run(`"{"`), {value: "{"}); 35 | test(StringLiteral.run(`"}"`), {value: "}"}); 36 | }; 37 | 38 | // Parse float value and precision of NumberLiterals. 39 | { 40 | 41 | // Integers. 42 | test(NumberLiteral.run("0"), {value: 0, precision: 0}); 43 | test(NumberLiteral.run("1"), {value: 1, precision: 0}); 44 | test(NumberLiteral.run("-1"), {value: -1, precision: 0}); 45 | test(NumberLiteral.run("-0"), {value: 0, precision: 0}); 46 | 47 | // Padded integers. 48 | test(NumberLiteral.run("01"), {value: 1, precision: 0}); 49 | test(NumberLiteral.run("-01"), {value: -1, precision: 0}); 50 | test(NumberLiteral.run("00"), {value: 0, precision: 0}); 51 | test(NumberLiteral.run("-00"), {value: 0, precision: 0}); 52 | 53 | // Positive floats. 54 | test(NumberLiteral.run("0.0"), {value: 0, precision: 1}); 55 | test(NumberLiteral.run("0.01"), {value: 0.01, precision: 2}); 56 | test(NumberLiteral.run("1.03"), {value: 1.03, precision: 2}); 57 | test(NumberLiteral.run("1.000"), {value: 1, precision: 3}); 58 | 59 | // Negative floats. 60 | test(NumberLiteral.run("-0.01"), {value: -0.01, precision: 2}); 61 | test(NumberLiteral.run("-1.03"), {value: -1.03, precision: 2}); 62 | test(NumberLiteral.run("-0.0"), {value: 0, precision: 1}); 63 | test(NumberLiteral.run("-1.000"), {value: -1, precision: 3}); 64 | 65 | // Padded floats. 66 | test(NumberLiteral.run("01.03"), {value: 1.03, precision: 2}); 67 | test(NumberLiteral.run("1.0300"), {value: 1.03, precision: 4}); 68 | test(NumberLiteral.run("01.0300"), {value: 1.03, precision: 4}); 69 | test(NumberLiteral.run("-01.03"), {value: -1.03, precision: 2}); 70 | test(NumberLiteral.run("-1.0300"), {value: -1.03, precision: 4}); 71 | test(NumberLiteral.run("-01.0300"), {value: -1.03, precision: 4}); 72 | }; 73 | }); 74 | -------------------------------------------------------------------------------- /test/parser.js: -------------------------------------------------------------------------------- 1 | import assert from "assert"; 2 | import path from "path"; 3 | import fs from "fs"; 4 | import {Resource} from "../syntax/grammar.js"; 5 | import {print_generic_error, print_assert_error, exit_summary, PASS, FAIL} 6 | from "./suite.js"; 7 | 8 | const bail = process.argv[2] === "--bail"; 9 | const fixtures_dir = process.argv[bail ? 3 : 2]; 10 | 11 | if (!fixtures_dir) { 12 | console.error( 13 | "Usage: node parser.js [--bail] FIXTURE"); 14 | process.exit(1); 15 | } 16 | 17 | main(fixtures_dir); 18 | 19 | function main(fixtures_dir) { 20 | if (fixtures_dir.endsWith(".ftl")) { 21 | // Actually, this is a filepath, split the path and the dir. 22 | var ftls = [path.basename(fixtures_dir)]; 23 | fixtures_dir = path.dirname(fixtures_dir); 24 | } else { 25 | let files = fs.readdirSync(fixtures_dir); 26 | var ftls = files.filter( 27 | filename => filename.endsWith(".ftl")); 28 | } 29 | 30 | // Collect all AssertionErrors. 31 | const errors = new Map(); 32 | 33 | // Parse each FTL fixture and compare against the expected AST. 34 | for (const file_name of ftls) { 35 | const ftl_path = path.join(fixtures_dir, file_name); 36 | const ast_path = ftl_path.replace(/ftl$/, "json"); 37 | 38 | process.stdout.write(`${ftl_path} `); 39 | 40 | try { 41 | var ftl_source = fs.readFileSync(ftl_path, "utf8"); 42 | var expected_ast = fs.readFileSync(ast_path, "utf8"); 43 | } catch (err) { 44 | errors.set(ftl_path, err); 45 | console.log(FAIL); 46 | continue; 47 | } 48 | 49 | Resource.run(ftl_source).fold( 50 | assert_equal, 51 | err => assert.fail(err)); 52 | 53 | function assert_equal(ast) { 54 | try { 55 | validate(ast, expected_ast); 56 | console.log(PASS); 57 | } catch (err) { 58 | errors.set(ftl_path, err); 59 | console.log(FAIL); 60 | } 61 | } 62 | } 63 | 64 | // Print all errors. 65 | for (const [ftl_path, err] of errors) { 66 | if (err instanceof assert.AssertionError) { 67 | print_assert_error(ftl_path, err); 68 | } else { 69 | print_generic_error(ftl_path, err); 70 | } 71 | } 72 | 73 | exit_summary(errors.size); 74 | } 75 | 76 | function validate(actual_ast, expected_serialized) { 77 | const actual_json = JSON.parse(JSON.stringify(actual_ast)); 78 | const expected_json = JSON.parse(expected_serialized); 79 | assert.deepEqual(actual_json, expected_json); 80 | } 81 | -------------------------------------------------------------------------------- /test/suite.js: -------------------------------------------------------------------------------- 1 | import assert from "assert"; 2 | import {diffString} from "json-diff"; 3 | import color from "cli-color"; 4 | 5 | export const PASS = color.green("PASS"); 6 | export const FAIL = color.red("FAIL"); 7 | 8 | export default function suite(fn) { 9 | let errors = new Map(); 10 | fn(create_tester(errors)); 11 | 12 | if (errors.size > 0) { 13 | for (let [err, title] of errors) { 14 | print_assert_error(title, err); 15 | } 16 | } 17 | 18 | exit_summary(errors.size); 19 | } 20 | 21 | function create_tester(errors) { 22 | return { 23 | ...assert, 24 | deep_equal(title, actual, expected) { 25 | try { 26 | assert.deepEqual(actual, expected); 27 | console.log(`${title} ${PASS}`); 28 | } catch (err) { 29 | if (err instanceof assert.AssertionError) { 30 | console.log(`${title} ${FAIL}`); 31 | errors.set(err, title); 32 | } else { 33 | throw err; 34 | } 35 | } 36 | }, 37 | }; 38 | } 39 | 40 | export function print_generic_error(ftl_path, err) { 41 | console.log(` 42 | ======================================================================== 43 | ${FAIL} ${ftl_path} 44 | ------------------------------------------------------------------------ 45 | ${err.message} 46 | `); 47 | } 48 | 49 | export function print_assert_error(title, err) { 50 | console.log(` 51 | ======================================================================== 52 | ${FAIL} ${title} 53 | ------------------------------------------------------------------------ 54 | ${diffString(err.expected, err.actual)} 55 | `); 56 | } 57 | 58 | export function exit_summary(error_count) { 59 | const message = error_count 60 | ? `Tests ${FAIL}: ${error_count}.` 61 | : `All tests ${PASS}.`; 62 | console.log(` 63 | ======================================================================== 64 | ${message} 65 | `); 66 | process.exit(Number(error_count > 0)); 67 | } 68 | -------------------------------------------------------------------------------- /test/validator.js: -------------------------------------------------------------------------------- 1 | import fs from "fs"; 2 | import assert from "assert"; 3 | import {execSync} from "child_process"; 4 | import {resolve} from "path"; 5 | import * as FTL from "../syntax/ast.js"; 6 | 7 | const ABSTRACT = resolve("syntax/abstract.js"); 8 | const GRAMMAR = resolve("syntax/grammar.js"); 9 | const FIXTURES = resolve("test/fixtures"); 10 | 11 | main(); 12 | 13 | function main() { 14 | let changed = modify_grammar(); 15 | changed |= validate_selector(); 16 | if (changed) { 17 | execSync("git diff syntax", {stdio: "inherit"}); 18 | } 19 | process.exit(Number(changed)); 20 | } 21 | 22 | /** 23 | * Modify grammar.js 24 | * 25 | * The combinator parsers can cover multiple variants. 26 | * For example, `repeat()` is the same as `maybe(repeat1)`. 27 | * And `maybe()` is `not() or once()`. 28 | * Now, when you replace several options with just one, you should 29 | * get test failures. 30 | * In the same way, if you add options to a more restricted production, 31 | * you should also get test failures. 32 | */ 33 | function modify_grammar() { 34 | let changed = false; 35 | let grammar = fs.readFileSync(GRAMMAR, "utf8"); 36 | let to_replace = /(not|maybe|repeat1?)\(/g; 37 | let m, 38 | iteration = 0; 39 | while ((m = to_replace.exec(grammar))) { 40 | let replacement, replacements; 41 | switch (m[1]) { 42 | case "not": 43 | replacements = ["maybe"]; 44 | break; 45 | case "maybe": 46 | replacements = ["not", "sequence"]; 47 | break; 48 | case "repeat": 49 | if (iteration === 0) { 50 | // skip Resource 51 | replacements = []; 52 | } else { 53 | replacements = ["repeat0", "repeat1"]; 54 | } 55 | break; 56 | case "repeat1": 57 | let further = /repeat1/g; 58 | further.lastIndex = m.index + "repeat1".length; 59 | if (further.test(grammar)) { 60 | replacements = ["repeat"]; 61 | } else { 62 | // ignore blank, which is the last production 63 | replacements = []; 64 | } 65 | break; 66 | } 67 | for (replacement of replacements) { 68 | let new_grammar = grammar.slice(0, m.index); 69 | new_grammar += replacement; 70 | new_grammar += grammar.slice(m.index + m[1].length); 71 | fs.writeFileSync(GRAMMAR, new_grammar); 72 | console.log(`Grammar validation iteration ${++iteration}`); 73 | const keep_change = verify_fixtures(); 74 | if (keep_change) { 75 | grammar = new_grammar; 76 | changed = true; 77 | console.log("new grammar"); 78 | break; 79 | } 80 | } 81 | } 82 | fs.writeFileSync(GRAMMAR, grammar); 83 | return changed; 84 | } 85 | 86 | /** 87 | * Modify abstract.js 88 | * 89 | * Test if the constraints to selector expressions are all tested. 90 | * Invert existing instanceof checks, and allow more AST nodes 91 | * that can be returned by InlineExpression. 92 | */ 93 | function validate_selector() { 94 | let changed = false; 95 | let abstract = fs.readFileSync(ABSTRACT, "utf8"); 96 | let expressions = Object.values(FTL) 97 | .filter((node) => FTL.Expression.isPrototypeOf(node)) 98 | .map((node) => node.name); 99 | // This is a bit hacky, a Placeable can be a PatternElement and 100 | // an expression, and is only typed to be the former. 101 | expressions.push("Placeable"); 102 | let chunks = abstract.split(/(^.*?selector_is_valid.*$)/m); 103 | assert.equal(chunks.length, 5); 104 | let head = chunks.slice(0, 2).join(""); 105 | let valid_selector = chunks[2]; 106 | let tail = chunks.slice(3).join(""); 107 | assert.strictEqual([head, valid_selector, tail].join(""), abstract); 108 | // Literals are abstract, and SelectExpression can only appear 109 | // inside a Placeable. 110 | let m, 111 | iteration = 0, 112 | checked_types = ["Literal", "SelectExpression"]; 113 | const instances = /selector instanceof FTL.([a-zA-Z]+)/g; 114 | while ((m = instances.exec(valid_selector))) { 115 | checked_types.push(m[1]); 116 | let new_valid = valid_selector.replace(m[0], `!(selector instanceof FTL.${m[1]})`); 117 | fs.writeFileSync(ABSTRACT, [head, new_valid, tail].join("")); 118 | console.log(`Abstract validation iteration ${++iteration}`); 119 | const keep_change = verify_fixtures(); 120 | if (keep_change) { 121 | valid_selector = new_valid; 122 | changed = true; 123 | console.log("new abstract"); 124 | } 125 | } 126 | for (const node of expressions) { 127 | if (checked_types.includes(node)) { 128 | continue; 129 | } 130 | let new_valid = ` 131 | selector instanceof FTL.${node} ||${valid_selector}`; 132 | fs.writeFileSync(ABSTRACT, [head, new_valid, tail].join("")); 133 | console.log(`Abstract validation iteration ${++iteration}`); 134 | const keep_change = verify_fixtures(); 135 | if (keep_change) { 136 | valid_selector = new_valid; 137 | changed = true; 138 | console.log("new abstract"); 139 | } 140 | } 141 | fs.writeFileSync(ABSTRACT, [head, valid_selector, tail].join("")); 142 | return changed; 143 | } 144 | 145 | function verify_fixtures() { 146 | try { 147 | execSync("node test/parser.js --bail " + FIXTURES); 148 | return true; 149 | } catch (e) { 150 | return false; 151 | } 152 | } 153 | --------------------------------------------------------------------------------