├── tsfmt.json ├── .eslintrc.js ├── .npmignore ├── src ├── ast.ts ├── fileUtils.ts ├── prettyPrinting.ts ├── browserIndex.js ├── index.d.ts ├── evalApi.ts ├── bml.ts ├── evalBlockValidator.ts ├── tokenType.ts ├── userDefs.ts ├── token.ts ├── reference.ts ├── stringUtils.ts ├── weightedChoice.ts ├── choiceFork.ts ├── settings.ts ├── rand.ts ├── evalBlock.ts ├── errors.ts ├── postprocessing.ts ├── analysis.ts ├── interactive.ts ├── renderer.ts ├── cli.ts ├── lexer.ts └── parsers.ts ├── .gitignore ├── .swcrc ├── tsconfig.json ├── test ├── token.ts ├── testPrettyPrinting.ts ├── lao_tzu_36_expected_output_seed_1234.bml ├── testSettings.ts ├── testStringUtils.ts ├── manual │ └── check-analysis.js ├── testUserDefs.ts ├── testWeightedChoice.ts ├── testChoiceFork.ts ├── manualBrowserTest.html ├── releaseTest.js ├── testAnalysis.ts ├── testCli.ts ├── testRand.ts ├── lao_tzu_36.bml ├── testPostprocessing.ts ├── testLexer.ts ├── testRenderer.ts └── testParsers.ts ├── README.md ├── man └── bml.1 ├── .github └── workflows │ └── main.yml ├── LICENSE ├── package.json ├── jest.config.js └── CHANGELOG.md /tsfmt.json: -------------------------------------------------------------------------------- 1 | { 2 | "indentSize": 2 3 | } 4 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | "extends": "google", 3 | "parser": "babel-eslint" 4 | }; 5 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .\#* 3 | npm-debug.log 4 | coverage/ 5 | .nyc_output/ 6 | todo.org 7 | 8 | **/.venv/ 9 | **/build/ 10 | **/.parcel-cache/ -------------------------------------------------------------------------------- /src/ast.ts: -------------------------------------------------------------------------------- 1 | import { ChoiceFork } from './choiceFork'; 2 | import { Reference } from './reference' 3 | 4 | export type AstNode = string | ChoiceFork | Reference 5 | -------------------------------------------------------------------------------- /src/fileUtils.ts: -------------------------------------------------------------------------------- 1 | import fs from 'fs'; 2 | 3 | 4 | export function readStdin(): string { 5 | return fs.readFileSync(0, 'utf8'); // STDIN_FILENO = 0 6 | } 7 | 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .\#* 3 | npm-debug.log 4 | coverage/ 5 | .nyc_output/ 6 | todo.org 7 | 8 | **/.venv/ 9 | **/build/ 10 | **/.parcel-cache/ 11 | 12 | dist -------------------------------------------------------------------------------- /src/prettyPrinting.ts: -------------------------------------------------------------------------------- 1 | export function prettyPrintArray(array: any[]) { 2 | if (array.length == 0) { 3 | return '[]'; 4 | } 5 | return `[${array.join(', ')}]`; 6 | } 7 | 8 | -------------------------------------------------------------------------------- /src/browserIndex.js: -------------------------------------------------------------------------------- 1 | /* @license BML - BSD 3 Clause License - Source at github.com/ajyoon/bml - Docs at bml-lang.org */ 2 | import { render } from './renderer'; 3 | import { analyze } from './analysis'; 4 | 5 | window['bml'] = render; 6 | window.bml.analyze = analyze; 7 | -------------------------------------------------------------------------------- /.swcrc: -------------------------------------------------------------------------------- 1 | { 2 | "jsc": { 3 | "parser": { 4 | "syntax": "typescript", 5 | "tsx": false, 6 | "decorators": false, 7 | "dynamicImport": false 8 | } 9 | }, 10 | "module": { 11 | "type": "umd" 12 | }, 13 | "sourceMaps": true 14 | } 15 | -------------------------------------------------------------------------------- /src/index.d.ts: -------------------------------------------------------------------------------- 1 | // This file contains type declarations for dependencies 2 | 3 | declare module 'seedrandom' { 4 | // The actual seedrandom declaration is much more complicated than 5 | // this - but this simplified interface is all we use. 6 | type RNGFunc = () => number; 7 | function seedrandom(seed?: number): RNGFunc; 8 | export = seedrandom; 9 | } 10 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2015", 4 | "module": "commonjs", 5 | "strict": true, 6 | "esModuleInterop": true, 7 | "skipLibCheck": true, 8 | "forceConsistentCasingInFileNames": true 9 | }, 10 | "$schema": "https://json.schemastore.org/tsconfig", 11 | "display": "Recommended", 12 | "include": ["src/**/*", "test/**/*"] 13 | } 14 | -------------------------------------------------------------------------------- /src/evalApi.ts: -------------------------------------------------------------------------------- 1 | import * as rand from './rand'; 2 | import { WeightedChoice } from './weightedChoice'; 3 | 4 | /** 5 | * This module is exposed to BML script `eval` blocks in a `bml` object namespace. 6 | */ 7 | 8 | export const api = { 9 | WeightedChoice: WeightedChoice, 10 | weightedChoose: rand.weightedChoose, 11 | randomInt: rand.randomInt, 12 | randomFloat: rand.randomFloat 13 | } 14 | -------------------------------------------------------------------------------- /test/token.ts: -------------------------------------------------------------------------------- 1 | import expect from 'expect'; 2 | 3 | const TokenType = require('../src/tokenType.ts').TokenType; 4 | const Token = require('../src/token.ts').Token; 5 | 6 | describe('Token', function() { 7 | it('has a useful toString', function() { 8 | let token = new Token(TokenType.TEXT, 0, 4, 'test'); 9 | expect(token.toString()).toBe('Token{tokenType: TEXT, index: 0, endIndex: 4, string: \'test\'}'); 10 | }); 11 | }); 12 | -------------------------------------------------------------------------------- /test/testPrettyPrinting.ts: -------------------------------------------------------------------------------- 1 | import expect from 'expect'; 2 | 3 | import * as prettyPrinting from '../src/prettyPrinting'; 4 | 5 | 6 | describe('prettyPrintArray', function() { 7 | it('prints an empty array as "[]"', function() { 8 | expect(prettyPrinting.prettyPrintArray([])).toBe('[]'); 9 | }); 10 | 11 | it('prints a populated array with brackets and spaces', function() { 12 | expect(prettyPrinting.prettyPrintArray([1, 2])).toBe('[1, 2]'); 13 | }); 14 | }); 15 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # [bml](https://bml-lang.org) 2 | 3 | A lightweight programming language for writing text that changes 4 | 5 | - [Read the docs](https://bml-lang.org) 6 | - [Try it online!](https://bml-lang.org/sandbox) 7 | - [Install the VS Code extension](https://marketplace.visualstudio.com/items?itemName=bml-lang.bml-vscode) 8 | 9 | ``` 10 | $ echo 'Hello {(blurry), (chancy)} world!' | npx bml 11 | Hello blurry world! 12 | ``` 13 | 14 | --- 15 | 16 | ![Build](https://github.com/ajyoon/bml/actions/workflows/main.yml/badge.svg) 17 | -------------------------------------------------------------------------------- /test/lao_tzu_36_expected_output_seed_1234.bml: -------------------------------------------------------------------------------- 1 | If you wish to shrink it, 2 | you must certainly grow it. 3 | 4 | If you wish to weaken it, 5 | you must certainly reinforce it. 6 | 7 | If you wish to destroy it, 8 | you must certainly build it up. 9 | 10 | If you wish to take it, 11 | one must certainly give it. 12 | 13 | Some call this delicate insight. 14 | The tender, the yielding prevail 15 | over the steadfast, the firm. 16 | 17 | Function called 18 | 19 | a fork directly inside a fork branch 20 | 21 | it 22 | 23 | {(foo)} 24 | -------------------------------------------------------------------------------- /test/testSettings.ts: -------------------------------------------------------------------------------- 1 | import expect from 'expect'; 2 | 3 | import { mergeSettings } from '../src/settings'; 4 | 5 | 6 | describe('mergeSettings', function() { 7 | it('should handle replacing all fields', function() { 8 | interface TestObj { 9 | foo?: boolean; 10 | bar?: number; 11 | }; 12 | let originalSettings: TestObj = { 13 | foo: true, 14 | bar: 10, 15 | }; 16 | let merged = mergeSettings( 17 | originalSettings, 18 | { 19 | foo: false 20 | }); 21 | expect(merged.foo).toBe(false); 22 | expect(merged.bar).toBe(10); 23 | }); 24 | }); 25 | -------------------------------------------------------------------------------- /src/bml.ts: -------------------------------------------------------------------------------- 1 | /* @license BML - BSD 3 Clause License - Source at github.com/ajyoon/bml - Docs at bml-lang.org */ 2 | import { render } from './renderer'; 3 | import { RenderSettings } from './settings'; 4 | import { analyze } from './analysis'; 5 | 6 | // Wrap the main entrypoint function so we can attach further API parts to it 7 | export function entryFunc(bmlDocumentString: string, 8 | renderSettings: RenderSettings | null): string { 9 | return render(bmlDocumentString, renderSettings); 10 | } 11 | 12 | entryFunc.analyze = analyze; 13 | 14 | // satisfy the module gods? 15 | module.exports = entryFunc; 16 | export default entryFunc; 17 | -------------------------------------------------------------------------------- /src/evalBlockValidator.ts: -------------------------------------------------------------------------------- 1 | import { EvalBlock } from './evalBlock'; 2 | 3 | const MATH_RANDOM_RE = /\bMath\.random\(\)/; 4 | 5 | function checkForMathRandom(code: string) { 6 | if (MATH_RANDOM_RE.test(code)) { 7 | console.warn('Eval block appears to use Math.random(); ' 8 | + 'use bml.randomInt or bml.randomFloat instead. ' 9 | + 'See https://bml-lang.org/docs/the-language/eval-api/') 10 | } 11 | } 12 | 13 | /** 14 | * Run basic sanity checks on an eval block. 15 | * 16 | * Currently validates that blocks: 17 | * - Do not access Math.random 18 | * 19 | * Validation failures result in warnings logged to console; 20 | * errors are not thrown. 21 | */ 22 | export function validateEvalBlock(evalBlock: EvalBlock) { 23 | checkForMathRandom(evalBlock.contents); 24 | } 25 | 26 | -------------------------------------------------------------------------------- /test/testStringUtils.ts: -------------------------------------------------------------------------------- 1 | import expect from 'expect'; 2 | 3 | import { lineAndColumnOf } from '../src/stringUtils'; 4 | 5 | 6 | describe('lineAndColumnOf', function() { 7 | it('should work at the first character', function() { 8 | expect(lineAndColumnOf('a', 0)).toEqual({ line: 1, column: 0 }); 9 | }); 10 | 11 | it('should count newline characters as the ending of a line', function() { 12 | expect(lineAndColumnOf('a\nb', 1)).toEqual({ line: 1, column: 1 }); 13 | }); 14 | 15 | it('should work at the start of a new line', function() { 16 | expect(lineAndColumnOf('a\nb', 2)).toEqual({ line: 2, column: 0 }); 17 | }); 18 | 19 | it('should work at the end of the string', function() { 20 | expect(lineAndColumnOf('a\nba', 3)).toEqual({ line: 2, column: 1 }); 21 | }); 22 | }); 23 | -------------------------------------------------------------------------------- /src/tokenType.ts: -------------------------------------------------------------------------------- 1 | export enum TokenType { 2 | WHITESPACE = 'WHITESPACE', 3 | NEW_LINE = 'NEW_LINE', 4 | VISUAL_NEW_LINE = 'VISUAL_NEW_LINE', 5 | COMMENT = 'COMMENT', 6 | OPEN_BLOCK_COMMENT = 'OPEN_BLOCK_COMMENT', 7 | CLOSE_BLOCK_COMMENT = 'CLOSE_BLOCK_COMMENT', 8 | SLASH = 'SLASH', 9 | SINGLE_QUOTE = 'SINGLE_QUOTE', 10 | DOUBLE_QUOTE = 'DOUBLE_QUOTE', 11 | BACKTICK = 'BACKTICK', 12 | OPEN_PAREN = 'OPEN_PAREN', 13 | CLOSE_PAREN = 'CLOSE_PAREN', 14 | OPEN_BRACE = 'OPEN_BRACE', 15 | CLOSE_BRACE = 'CLOSE_BRACE', 16 | OPEN_BRACKET = 'OPEN_BRACKET', 17 | CLOSE_BRACKET = 'CLOSE_BRACKET', 18 | COMMA = 'COMMA', 19 | COLON = 'COLON', 20 | AT = 'AT', 21 | HASH = 'HASH', 22 | BANG = 'BANG', 23 | DOLLAR = 'DOLLAR', 24 | ARROW = 'ARROW', 25 | NUMBER = 'NUMBER', 26 | TEXT = 'TEXT', 27 | } 28 | -------------------------------------------------------------------------------- /test/manual/check-analysis.js: -------------------------------------------------------------------------------- 1 | const bml = require('bml'); 2 | 3 | /* 4 | * A script for comparing branch counts from static analysis against 5 | * real bml outputs. This necessarily is not effective for outputs 6 | * with high branch counts. 7 | */ 8 | 9 | const assert = require('assert'); 10 | 11 | let bmlScript = ` 12 | // {#foo: (x), (y)} 13 | // {#bar: (a), (b), (c)} 14 | // {{@foo}, {@bar}} 15 | // {{@foo}} 16 | 17 | {{foo: (x), (y)}, {(a), (b), (c)}} 18 | {@foo} 19 | `; 20 | 21 | const ITERS = 1000; 22 | let acc = new Set(); 23 | for (let i = 0; i < ITERS; i++) { 24 | acc.add(bml(bmlScript)); 25 | } 26 | console.log(`After ${ITERS} iterations, I found ${acc.size} unique outputs`); 27 | // debugging 28 | let fixedOutputs = []; 29 | for (let output of acc) { 30 | fixedOutputs.push(output.replaceAll('\n', ' ').toUpperCase()); 31 | } 32 | fixedOutputs.sort(); 33 | fixedOutputs.forEach(o => console.log(o)); 34 | -------------------------------------------------------------------------------- /src/userDefs.ts: -------------------------------------------------------------------------------- 1 | import { DocumentSettings } from './settings'; 2 | import { EvalBoundSettingsError } from './errors'; 3 | 4 | 5 | function nullOrUndefined(object: T | undefined | null): object is T { 6 | return object === undefined || object === null; 7 | } 8 | 9 | export type UserDefs = { [index: string]: any }; 10 | 11 | function validateSettingField(settings: UserDefs, field: string, expectedType: string) { 12 | const value = settings[field]; 13 | if (!nullOrUndefined(value) && typeof value !== expectedType) { 14 | throw new EvalBoundSettingsError('setting.' + field, value); 15 | } 16 | } 17 | 18 | export function validateUserDefs(userDefs: UserDefs) { 19 | let settings = userDefs['settings']; 20 | if (settings) { 21 | validateSettingField(settings, 'whitespaceCleanup', 'boolean'); 22 | validateSettingField(settings, 'punctuationCleanup', 'boolean'); 23 | validateSettingField(settings, 'capitalizationCleanup', 'boolean'); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /man/bml.1: -------------------------------------------------------------------------------- 1 | .TH bml 1 "2019" "bml" 2 | 3 | .SH NAME 4 | bml \-\- blur markup language command line utility 5 | 6 | .SH SYNOPSIS 7 | bml [options] [path] 8 | 9 | .SH DESCRIPTION 10 | Render a bml document read from stdin or a file, if given. 11 | 12 | Prints result to STDOUT. 13 | 14 | If any error occurs during processing, exit code 1 is returned. 15 | 16 | .SH OPTIONS 17 | .TP 18 | .BR \-h ", " \-\-h ", " \-help ", " \-\-help 19 | Print help and quit 20 | 21 | .TP 22 | .BR \-v ", " \-\-version 23 | Print version information and quit 24 | 25 | .TP 26 | .BR \-\-seed " INTEGER" 27 | Set the random seed to be used for the render run 28 | 29 | .TP 30 | .BR \-\-no\-eval 31 | Disable Javascript evaluation 32 | 33 | .TP 34 | .BR \-\-analyze 35 | Analyze the document instead of executing 36 | 37 | .SH BUG REPORTS 38 | Please report bugs at https://github.com/ajyoon/bml/issues 39 | 40 | .SH AUTHOR 41 | Andrew Yoon (andrew@nothing-to-say.org) 42 | 43 | .SH LICENSE 44 | BSD 3-Clause, 2017 45 | -------------------------------------------------------------------------------- /src/token.ts: -------------------------------------------------------------------------------- 1 | import { TokenType } from './tokenType'; 2 | 3 | export class Token { 4 | 5 | tokenType: TokenType; 6 | /** Index of the first character of this token in the string */ 7 | index: number; 8 | /** Index after the last character of this token in the string (exclusive bound) */ 9 | endIndex: number; 10 | /** 11 | * Output string for the token. 12 | * 13 | * This may not always be the same as the input text consumed by the token. 14 | * For example, escape sequences like `\{` will have `token.str === '{'`. 15 | */ 16 | str: string; 17 | 18 | constructor(tokenType: TokenType, index: number, endIndex: number, str: string) { 19 | this.tokenType = tokenType; 20 | this.index = index; 21 | this.endIndex = endIndex; 22 | this.str = str; 23 | } 24 | 25 | toString(): string { 26 | return `Token{tokenType: ${this.tokenType}, index: ${this.index}, ` 27 | + `endIndex: ${this.endIndex}, string: '${this.str}'}`; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/reference.ts: -------------------------------------------------------------------------------- 1 | import { Choice, WeightedChoice } from './weightedChoice'; 2 | import { ChoiceFork } from './choiceFork'; 3 | 4 | export type ReferenceMap = Map; 5 | 6 | export class Reference { 7 | 8 | id: string; 9 | referenceMap: ReferenceMap; 10 | fallbackChoiceFork: ChoiceFork | null; 11 | reExecute: boolean; 12 | 13 | constructor(id: string, choiceMap: ReferenceMap, fallbackChocies: WeightedChoice[], reExecute: boolean) { 14 | this.id = id; 15 | this.referenceMap = choiceMap; 16 | if (fallbackChocies.length) { 17 | this.fallbackChoiceFork = new ChoiceFork(fallbackChocies, null, false, false); 18 | } else { 19 | this.fallbackChoiceFork = null; 20 | } 21 | this.reExecute = reExecute; 22 | if (reExecute) { 23 | if (choiceMap.size || fallbackChocies.length) { 24 | throw new Error('Got reExecute=true but mappings were provided. ' + 25 | 'This error should be caught earlier in the parser.') 26 | } 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/stringUtils.ts: -------------------------------------------------------------------------------- 1 | export function lineAndColumnOf(str: string, index: number): { line: number, column: number } { 2 | if (index > str.length) { 3 | throw new Error('charIndex > string.length'); 4 | } 5 | let line = 1; 6 | let column = -1; 7 | let newLine = false; 8 | for (let i = 0; i <= index; i++) { 9 | if (newLine) { 10 | line++; 11 | column = 0; 12 | newLine = false; 13 | } else { 14 | column++; 15 | } 16 | if (str[i] === '\n') { 17 | newLine = true; 18 | } 19 | } 20 | return { line: line, column: column }; 21 | } 22 | 23 | export function lineColumnString(str: string, index: number): string { 24 | let lineAndColumn = lineAndColumnOf(str, index); 25 | return 'line: ' + lineAndColumn.line + ', column: ' + lineAndColumn.column; 26 | } 27 | 28 | export function isWhitespace(str: string): boolean { 29 | return str.trim() === ''; 30 | } 31 | 32 | export function isStr(obj: any): obj is string { 33 | return obj instanceof String || typeof obj === "string"; 34 | } 35 | -------------------------------------------------------------------------------- /src/weightedChoice.ts: -------------------------------------------------------------------------------- 1 | import { EvalBlock } from './evalBlock'; 2 | import { AstNode } from './ast'; 3 | 4 | 5 | export type Choice = EvalBlock | AstNode[]; 6 | 7 | /** 8 | * An outcome with a weight. 9 | */ 10 | export class WeightedChoice { 11 | choice: Choice; 12 | weight: number | null; 13 | 14 | constructor(choice: Choice, weight: number | null) { 15 | this.choice = choice; 16 | this.weight = weight; 17 | } 18 | 19 | toString(): string { 20 | return `WeightedChoice{choice: ${String(this.choice)}, weight: ${this.weight}}`; 21 | } 22 | 23 | /* Create a new WeightedChoice object with the same properties as this one. */ 24 | clone(): WeightedChoice { 25 | return new WeightedChoice(this.choice, this.weight); 26 | } 27 | } 28 | 29 | export function sumWeights(weights: WeightedChoice[]) { 30 | // Note that if weights have been normalized, as they are in `ChoiceFork`s, 31 | // `wc.weight` will always be non-null here so the default should never occur. 32 | return weights.reduce((acc, val) => acc + (val.weight ?? 0), 0); 33 | } 34 | -------------------------------------------------------------------------------- /test/testUserDefs.ts: -------------------------------------------------------------------------------- 1 | import expect from 'expect'; 2 | import { defaultBMLSettings } from '../src/settings'; 3 | import { validateUserDefs } from '../src/userDefs'; 4 | import { EvalBoundSettingsError } from '../src/errors'; 5 | 6 | describe('validateUserDefs', function() { 7 | it('doesnt error on well-formed settings and defs', function() { 8 | validateUserDefs({ 9 | settings: defaultBMLSettings, 10 | someFunc: () => { } 11 | }); 12 | }); 13 | 14 | it('errors on malformed whitespaceCleanup', function() { 15 | expect(() => validateUserDefs({ 16 | settings: { 17 | whitespaceCleanup: 0 18 | } 19 | })).toThrow(EvalBoundSettingsError); 20 | }); 21 | 22 | it('errors on malformed punctuationCleanup', function() { 23 | expect(() => validateUserDefs({ 24 | settings: { 25 | punctuationCleanup: 123 26 | } 27 | })).toThrow(EvalBoundSettingsError); 28 | }); 29 | 30 | it('errors on malformed capitalizationCleanup', function() { 31 | expect(() => validateUserDefs({ 32 | settings: { 33 | capitalizationCleanup: {} 34 | } 35 | })).toThrow(EvalBoundSettingsError); 36 | }); 37 | }); 38 | -------------------------------------------------------------------------------- /test/testWeightedChoice.ts: -------------------------------------------------------------------------------- 1 | import expect from 'expect'; 2 | 3 | import { WeightedChoice, sumWeights } from '../src/weightedChoice'; 4 | import { EvalBlock } from '../src/evalBlock'; 5 | 6 | describe('WeightedChoice', function() { 7 | it('has a useful toString for all choice types', function() { 8 | expect(new WeightedChoice(['foo'], 1).toString()) 9 | .toBe('WeightedChoice{choice: foo, weight: 1}'); 10 | expect(new WeightedChoice(new EvalBlock('foo'), 1).toString()) 11 | .toBe('WeightedChoice{choice: EvalBlock(\'foo\'), weight: 1}'); 12 | }); 13 | }); 14 | 15 | describe('sumWeights', function() { 16 | it('Converts null weights to 0', function() { 17 | let weights = [ 18 | new WeightedChoice(['foo'], null), 19 | new WeightedChoice(['bar'], 5), 20 | ]; 21 | expect(sumWeights(weights)).toEqual(5); 22 | }); 23 | 24 | it('Accepts empty arrays', function() { 25 | expect(sumWeights([])).toEqual(0); 26 | }); 27 | 28 | it('Works on simple cases', function() { 29 | let weights = [ 30 | new WeightedChoice(['foo'], 2), 31 | new WeightedChoice(['bar'], 5), 32 | new WeightedChoice(['biz'], 6), 33 | ]; 34 | expect(sumWeights(weights)).toEqual(13); 35 | }); 36 | 37 | }); 38 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | # This is a basic workflow to help you get started with Actions 2 | 3 | name: Build 4 | 5 | # Controls when the workflow will run 6 | on: 7 | # Triggers the workflow on push or pull request events but only for the master branch 8 | push: 9 | branches: [ master ] 10 | pull_request: 11 | branches: [ master ] 12 | 13 | # Allows you to run this workflow manually from the Actions tab 14 | workflow_dispatch: 15 | 16 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel 17 | jobs: 18 | # This workflow contains a single job called "build" 19 | build: 20 | # The type of runner that the job will run on 21 | runs-on: ubuntu-latest 22 | 23 | # Steps represent a sequence of tasks that will be executed as part of the job 24 | steps: 25 | # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it 26 | - uses: actions/checkout@v4 27 | 28 | - name: Setup Node.js environment 29 | uses: actions/setup-node@v4 30 | 31 | # Runs a set of commands using the runners shell 32 | - name: Run a multi-line script 33 | run: | 34 | npm install 35 | npm run test 36 | npm run build 37 | -------------------------------------------------------------------------------- /test/testChoiceFork.ts: -------------------------------------------------------------------------------- 1 | import expect from 'expect'; 2 | import * as rand from '../src/rand'; 3 | import { WeightedChoice } from '../src/weightedChoice'; 4 | import { ChoiceFork } from '../src/choiceFork'; 5 | 6 | describe('ChoiceFork', function() { 7 | 8 | beforeEach(function() { 9 | rand.setRandomSeed(0); // pin seed for reproducibility 10 | }); 11 | 12 | it('on call returns well-formed object', function() { 13 | let weights = [ 14 | new WeightedChoice(['foo'], 40), 15 | new WeightedChoice(['bar'], 60), 16 | ]; 17 | let choiceFork = new ChoiceFork(weights, null, false, false); 18 | let result = choiceFork.call(); 19 | expect(result.replacement).toStrictEqual(['foo']); 20 | expect(result.choiceIndex).toBe(0); 21 | }); 22 | 23 | it('has a useful toString', function() { 24 | let weights = [ 25 | new WeightedChoice(['foo'], 40), 26 | new WeightedChoice(['bar'], 60), 27 | ]; 28 | let choiceFork = new ChoiceFork(weights, 'identifier', true, false); 29 | expect(choiceFork.toString()).toBe( 30 | 'ChoiceFork{weights: WeightedChoice{choice: foo, weight: 40},' 31 | + 'WeightedChoice{choice: bar, weight: 60}, identifier: identifier, ' 32 | + 'isSilent: true, isSet: false}'); 33 | }); 34 | }); 35 | -------------------------------------------------------------------------------- /test/manualBrowserTest.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | Test 11 | 12 | 13 | 14 |
15 | 16 | 17 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2017 Andrew Yoon 2 | 3 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 4 | 5 | 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 6 | 7 | 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 8 | 9 | 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 10 | 11 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /test/releaseTest.js: -------------------------------------------------------------------------------- 1 | // Pre-release smoke test run against build artifacts in bml/dist 2 | 3 | const { execSync } = require('child_process'); 4 | const path = require('path'); 5 | const assert = require('assert'); 6 | const fs = require('fs'); 7 | const expect = require('expect'); 8 | 9 | const bmlScriptPath = path.resolve(__dirname, 'lao_tzu_36.bml'); 10 | const expectedOutputPath = path.resolve( 11 | __dirname, 'lao_tzu_36_expected_output_seed_1234.bml'); 12 | const distDir = path.resolve(__dirname, '../dist'); 13 | const cliPath = path.resolve(distDir, 'cli.js'); 14 | const webBundlePath = path.resolve(distDir, 'bml.bundle.js'); 15 | const libraryPath = path.resolve(distDir, 'index.js'); 16 | 17 | 18 | if (!(fs.existsSync(cliPath) 19 | && fs.existsSync(webBundlePath) 20 | && fs.existsSync(libraryPath))) { 21 | console.error( 22 | 'Dist artifacts not found. This release test ' 23 | + 'runs against build artifacts generated by `npm run build`.'); 24 | process.exit(1); 25 | } 26 | 27 | const bmlScript = fs.readFileSync(bmlScriptPath).toString(); 28 | const seed = 1234; 29 | const expectedOutput = fs.readFileSync(expectedOutputPath).toString(); 30 | 31 | function assertResult(result) { 32 | if (result !== expectedOutput) { 33 | console.error( 34 | 'Rendered output did not match expected. Expected:\n\n\n', expectedOutput, 35 | '\n\n\n---------------------------\n\n\n', 'Got:\n', result); 36 | throw new Error(); 37 | } 38 | } 39 | 40 | console.log('Checking CLI'); 41 | let cliResult = execSync(`node ${cliPath} ${bmlScriptPath} --seed ${seed}`).toString(); 42 | assertResult(cliResult); 43 | 44 | console.log('Checking node library'); 45 | const bmlLib = require(libraryPath); 46 | let libResult = bmlLib(bmlScript, { randomSeed: seed }); 47 | assertResult(libResult); 48 | 49 | // Check that the analysis function is accessible to the public API. 50 | bmlLib.analyze(bmlScript); 51 | 52 | console.log('All smoke tests passed'); 53 | -------------------------------------------------------------------------------- /src/choiceFork.ts: -------------------------------------------------------------------------------- 1 | import { WeightedChoice, Choice, sumWeights } from './weightedChoice' 2 | import { 3 | normalizeWeights, 4 | weightedChoose 5 | } from './rand'; 6 | import { NoPossibleChoiceError, InvalidForkWeightsError } from './errors'; 7 | 8 | export type ChoiceForkCallResult = { 9 | replacement: Choice, 10 | choiceIndex: number 11 | } 12 | 13 | export class ChoiceFork { 14 | weights: WeightedChoice[]; 15 | initWeights: number[]; 16 | identifier: string | null; 17 | isSilent: boolean; 18 | isSet: boolean; 19 | 20 | constructor(weights: WeightedChoice[], identifier: string | null, isSilent: boolean, isSet: boolean) { 21 | this.weights = normalizeWeights(weights); 22 | this.validateWeights(); 23 | this.initWeights = this.weights.map((w) => (w.weight!)); 24 | this.identifier = identifier; 25 | this.isSilent = isSilent; 26 | this.isSet = isSet; 27 | } 28 | 29 | /** 30 | * returns an object of the form {replacement: String, choiceIndex: Int} 31 | */ 32 | call(): ChoiceForkCallResult { 33 | let result; 34 | try { 35 | result = weightedChoose(this.weights); 36 | } catch (error) { 37 | if (error instanceof NoPossibleChoiceError && this.isSet) { 38 | console.warn(`Set '${this.identifier}' is exhausted; resetting weights.`) 39 | this.resetWeights(); 40 | return this.call(); 41 | } else { 42 | throw error; 43 | } 44 | } 45 | if (this.isSet) { 46 | this.weights[result.choiceIndex].weight = 0; 47 | } 48 | return { replacement: result.choice, choiceIndex: result.choiceIndex }; 49 | } 50 | 51 | private resetWeights() { 52 | for (let [idx, val] of this.weights.entries()) { 53 | val.weight = this.initWeights[idx]; 54 | } 55 | } 56 | 57 | private validateWeights() { 58 | if (sumWeights(this.weights) === 0) { 59 | throw new InvalidForkWeightsError(); 60 | } 61 | } 62 | 63 | toString(): string { 64 | return `ChoiceFork{weights: ${this.weights}, ` 65 | + `identifier: ${this.identifier}, isSilent: ${this.isSilent}, isSet: ${this.isSet}}`; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bml", 3 | "version": "0.2.0", 4 | "description": "A stochastic markup language", 5 | "author": { 6 | "name": "Andrew Yoon", 7 | "email": "andrew@nothing-to-say.org", 8 | "url": "https://andrewyoon.art" 9 | }, 10 | "man": "./man/bml.1", 11 | "repository": { 12 | "type": "git", 13 | "url": "https://github.com/ajyoon/bml" 14 | }, 15 | "bugs": { 16 | "url": "https://github.com/ajyoon/bml/issues" 17 | }, 18 | "source": "src/bml.ts", 19 | "main": "dist/index.js", 20 | "bundle": "dist/bml.bundle.js", 21 | "cli": "dist/cli.js", 22 | "targets": { 23 | "main": { 24 | "optimize": true 25 | }, 26 | "bundle": { 27 | "source": "src/browserIndex.js", 28 | "optimize": true, 29 | "outputFormat": "global", 30 | "context": "browser", 31 | "isLibrary": false 32 | }, 33 | "cli": { 34 | "source": "src/cli.ts", 35 | "optimize": true, 36 | "sourceMap": false, 37 | "context": "node", 38 | "isLibrary": false 39 | } 40 | }, 41 | "bin": { 42 | "bml": "dist/cli.js" 43 | }, 44 | "scripts": { 45 | "test": "jest", 46 | "releaseTest": "node test/releaseTest.js", 47 | "checkTypes": "./node_modules/.bin/tsc --noEmit", 48 | "buildOnly": "./node_modules/.bin/parcel build", 49 | "buildDebug": "./node_modules/.bin/parcel build --no-optimize", 50 | "build": "npm run checkTypes && npm run buildOnly && npm run releaseTest", 51 | "preversion": "npm test", 52 | "version": "npm run build" 53 | }, 54 | "devDependencies": { 55 | "@parcel/transformer-typescript-types": "^2.0.1", 56 | "@swc/core": "^1.2.119", 57 | "@swc/jest": "^0.2.14", 58 | "@types/blessed": "^0.1.19", 59 | "@types/jest": "^27.0.3", 60 | "@types/node": "^18.11.18", 61 | "@types/sha.js": "^2.4.0", 62 | "@types/tmp": "^0.2.3", 63 | "@types/indefinite": "2.3.1", 64 | "jest": "^27.4.4", 65 | "parcel": "^2.0.1", 66 | "path-browserify": "^1.0.1", 67 | "process": "^0.11.10", 68 | "sha.js": "2.4.11", 69 | "tmp": "^0.2.1", 70 | "typescript": "^4.5.4" 71 | }, 72 | "dependencies": { 73 | "blessed": "^0.1.81", 74 | "clipboardy": "^2.3.0", 75 | "seedrandom": "3.0.5", 76 | "indefinite": "2.4.2" 77 | }, 78 | "license": "BSD-3-Clause" 79 | } 80 | -------------------------------------------------------------------------------- /src/settings.ts: -------------------------------------------------------------------------------- 1 | export interface DocumentSettings { 2 | /** 3 | * Whether to perform a post-processing step cleaning up whitespace. 4 | */ 5 | whitespaceCleanup?: boolean | null; 6 | 7 | /** 8 | * Whether to perform a post-processing step repositioning 9 | * punctuation marks according to *some* English grammar rules. 10 | */ 11 | punctuationCleanup?: boolean | null; 12 | 13 | /** 14 | * Whether to perform basic capitalization correction for words 15 | * following sentence-ending punctuation. 16 | */ 17 | capitalizationCleanup?: boolean | null; 18 | 19 | /** 20 | * Whether to correct English indefinite articles (a / an) 21 | */ 22 | indefiniteArticleCleanup?: boolean | null; 23 | } 24 | 25 | export interface RenderSettings { 26 | /** 27 | * The random seed to use for this render. 28 | * 29 | * Can be any type, as this is fed directly to the `seedrandom` 30 | * library, which converts the object to a string and uses that as 31 | * the actual seed. 32 | */ 33 | randomSeed?: number | null; 34 | 35 | /** 36 | * Whether to disable `eval` blocks in the document. 37 | * 38 | * This can be useful for security purposes. 39 | */ 40 | allowEval?: boolean | null; 41 | 42 | /** 43 | * The working directory used for imports in this render. 44 | * 45 | * If the document uses imports, this should generally be set to the 46 | * absolute path of the document file's directory. 47 | */ 48 | workingDir?: string | null; 49 | } 50 | 51 | /** 52 | * Default settings. These are passed in to the main bml rendering function. 53 | */ 54 | export const defaultBMLSettings: DocumentSettings = { 55 | whitespaceCleanup: true, 56 | punctuationCleanup: true, 57 | capitalizationCleanup: true, 58 | indefiniteArticleCleanup: true, 59 | }; 60 | 61 | export const defaultRenderSettings: RenderSettings = { 62 | randomSeed: null, 63 | allowEval: true, 64 | workingDir: null, 65 | }; 66 | 67 | /** 68 | * Return a new settings object with all the properties defined in newSettings, 69 | * defaulting to those in originalSettings where absent. 70 | * 71 | * If `newSettings` is falsy, return `originalSettings` unmodified. 72 | */ 73 | export function mergeSettings(originalSettings: T, newSettings: T | null | undefined): T { 74 | return Object.assign({}, originalSettings, newSettings); 75 | } 76 | -------------------------------------------------------------------------------- /src/rand.ts: -------------------------------------------------------------------------------- 1 | import { WeightedChoice, Choice, sumWeights } from "./weightedChoice"; 2 | import { NoPossibleChoiceError } from "./errors"; 3 | import seedrandom from "seedrandom"; 4 | 5 | // A module-local seedable random number generator 6 | // The selected seed will be random unless `setRandomSeed()` is called. 7 | // @ts-ignore 8 | let rng = seedrandom.xor4096(null, { state: true }); 9 | 10 | export function saveRngState(): Object { 11 | // @ts-ignore 12 | return rng.state(); 13 | } 14 | 15 | export function restoreRngState(state: Object) { 16 | // @ts-ignore 17 | rng = seedrandom.xor4096(null, { state: state }); 18 | } 19 | 20 | export function setRandomSeed(seed: number) { 21 | // @ts-ignore 22 | rng = seedrandom.xor4096(seed, { state: true }); 23 | } 24 | 25 | export function normalizeWeights( 26 | weightedChoices: WeightedChoice[] 27 | ): WeightedChoice[] { 28 | let normalized = []; 29 | let sum = 0; 30 | let nullWeightCount = 0; 31 | for (let w = 0; w < weightedChoices.length; w++) { 32 | let weightedChoice = weightedChoices[w]; 33 | normalized.push(weightedChoice.clone()); 34 | if (weightedChoice.weight === null) { 35 | nullWeightCount++; 36 | } else { 37 | sum += weightedChoice.weight; 38 | } 39 | } 40 | let nullWeight = (100 - sum) / nullWeightCount; 41 | for (let n = 0; n < normalized.length; n++) { 42 | if (normalized[n].weight === null) { 43 | normalized[n].weight = nullWeight; 44 | } 45 | } 46 | return normalized; 47 | } 48 | 49 | export function randomFloat(min: number, max: number): number { 50 | return rng() * (max - min) + min; 51 | } 52 | 53 | export function randomInt(min: number, max: number): number { 54 | min = Math.ceil(min); 55 | max = Math.floor(max); 56 | return Math.floor(rng() * (max - min)) + min; 57 | } 58 | 59 | /** 60 | * Randomly choose from an array of weighted choices. 61 | * 62 | * The probability of any given `WeightedChoice` being 63 | * chosen is its weight divided by the sum of all given 64 | * choices. 65 | * 66 | * Returns an object of the form {choice, choiceIndex} 67 | */ 68 | export function weightedChoose(weights: WeightedChoice[]): { 69 | choice: Choice; 70 | choiceIndex: number; 71 | } { 72 | let sum = sumWeights(weights); 73 | if (sum === 0) { 74 | throw new NoPossibleChoiceError(); 75 | } 76 | let progress = 0; 77 | let pickedValue = randomFloat(0, sum); 78 | for (let i = 0; i < weights.length; i++) { 79 | let wc = weights[i]; 80 | progress += wc.weight ?? 0; 81 | if (progress >= pickedValue) { 82 | return { choice: wc.choice, choiceIndex: i }; 83 | } 84 | } 85 | // If we're still here, something went wrong. 86 | // Log a warning but try to return a random value anyways. 87 | console.warn("Unable to pick weighted choice for weights: " + weights); 88 | let fallbackIndex = randomInt(0, weights.length); 89 | return { choice: weights[fallbackIndex].choice, choiceIndex: fallbackIndex }; 90 | } 91 | -------------------------------------------------------------------------------- /src/evalBlock.ts: -------------------------------------------------------------------------------- 1 | import * as evalApi from './evalApi'; 2 | import { UserDefs, validateUserDefs } from './userDefs'; 3 | import { EvalBindingError } from './errors'; 4 | import type { Renderer } from './renderer'; 5 | 6 | const evalFuncTemplate = ` 7 | const bml = this; 8 | 9 | const __new_bindings = {}; 10 | 11 | function bind(obj) { 12 | for (let key in obj) { 13 | __new_bindings[key] = obj[key]; 14 | } 15 | } 16 | 17 | // Note ref lookups are currently unstable 18 | bml.ref = function(id) { 19 | let lookupResult = __context.renderer.executedForkMap.get(id); 20 | if (lookupResult === undefined) { 21 | throw new Error('Ref lookup inside eval failed for: ' + id); 22 | } 23 | return lookupResult.renderedOutput; 24 | } 25 | bml.refDetail = function(id) { 26 | let lookupResult = __context.renderer.executedForkMap.get(id); 27 | if (lookupResult === undefined) { 28 | throw new Error('Ref lookup inside eval failed for: ' + id); 29 | } 30 | return lookupResult; 31 | } 32 | 33 | ***USER DEFS BINDING SLOT*** 34 | 35 | 36 | function insert(str) { 37 | __context.output += str; 38 | } 39 | 40 | function include(includePath) { 41 | let result = __context.renderer.renderInclude(includePath); 42 | insert(result); 43 | } 44 | 45 | ////////// start user code 46 | 47 | ***USER CODE SLOT*** 48 | 49 | ////////// end user code 50 | 51 | ***SAVE USER MUTATIONS SLOT*** 52 | 53 | return __new_bindings; 54 | `; 55 | 56 | 57 | export type EvalContext = { 58 | bindings: UserDefs; 59 | renderer: Renderer; 60 | output: string; 61 | } 62 | 63 | 64 | export class EvalBlock { 65 | contents: string; 66 | 67 | constructor(contents: string) { 68 | this.contents = contents; 69 | } 70 | 71 | toString(): string { 72 | return `EvalBlock('${this.contents}')`; 73 | } 74 | 75 | generateBindingCode(userDefs: UserDefs): string { 76 | let lines = []; 77 | if (userDefs.settings) { 78 | lines.push('let settings = __context.bindings.settings;'); 79 | } 80 | for (let key in userDefs) { 81 | lines.push(`let ${key} = __context.bindings.${key}`) 82 | } 83 | return lines.join('\n'); 84 | } 85 | 86 | generateSaveUserMutationsCode(userDefs: UserDefs): string { 87 | let lines = []; 88 | if (userDefs.settings) { 89 | lines.push('__context.bindings.settings = settings;'); 90 | } 91 | for (let key in userDefs) { 92 | lines.push(`__context.bindings.${key} = ${key};`) 93 | } 94 | return lines.join('\n'); 95 | } 96 | 97 | toFunc(userDefs: UserDefs): Function { 98 | let funcSrc = evalFuncTemplate.replace('***USER CODE SLOT***', this.contents); 99 | funcSrc = funcSrc.replace('***USER DEFS BINDING SLOT***', 100 | this.generateBindingCode(userDefs)); 101 | funcSrc = funcSrc.replace('***SAVE USER MUTATIONS SLOT***', 102 | this.generateSaveUserMutationsCode(userDefs)); 103 | let funcContext = Object.assign({}, evalApi.api); 104 | return new Function('__context', funcSrc).bind(funcContext); 105 | } 106 | 107 | /* 108 | * Execution results are stored in the passed-in context 109 | */ 110 | execute(context: EvalContext) { 111 | let newBindings = this.toFunc(context.bindings)(context); 112 | validateUserDefs(newBindings); 113 | for (let [key, value] of Object.entries(newBindings)) { 114 | if (context.bindings.hasOwnProperty(key)) { 115 | throw new EvalBindingError(key); 116 | } 117 | context.bindings[key] = value; 118 | } 119 | } 120 | } 121 | 122 | -------------------------------------------------------------------------------- /src/errors.ts: -------------------------------------------------------------------------------- 1 | import * as stringUtils from './stringUtils'; 2 | 3 | export class IllegalStateError extends Error { 4 | constructor(message: string) { 5 | super(message + ' This is a bug. Please report at https://github.com/ajyoon/bml/issues'); 6 | this.name = 'IllegalStateError'; 7 | Object.setPrototypeOf(this, IllegalStateError.prototype); 8 | } 9 | } 10 | 11 | export class JavascriptSyntaxError extends Error { 12 | constructor(bmlDoc: string, errorIndex: number) { 13 | let message = 'Syntax error found while parsing bml javascript at ' 14 | + stringUtils.lineColumnString(bmlDoc, errorIndex); 15 | super(message); 16 | this.name = 'JavascriptSyntaxError'; 17 | Object.setPrototypeOf(this, JavascriptSyntaxError.prototype); 18 | } 19 | } 20 | 21 | export class BMLSyntaxError extends Error { 22 | constructor(message: string | null, bmlDoc: string, errorIndex: number, hint?: string) { 23 | let resolvedMsg = (message || 'Syntax error found while parsing bml') + 24 | ' at ' + stringUtils.lineColumnString(bmlDoc, errorIndex); 25 | if (hint) { 26 | resolvedMsg += '\n' + hint; 27 | } 28 | super(resolvedMsg); 29 | this.name = 'BMLSyntaxError'; 30 | Object.setPrototypeOf(this, BMLSyntaxError.prototype); 31 | } 32 | } 33 | 34 | export class BMLDuplicatedRefIndexError extends Error { 35 | constructor(refIdentifier: string, choiceIndex: number, bmlDoc: string, errorIndex: number) { 36 | let msg = `Duplicated reference index ${choiceIndex} for reference ${refIdentifier} ` 37 | + `at ${stringUtils.lineColumnString(bmlDoc, errorIndex)}`; 38 | super(msg); 39 | this.name = 'BMLDuplicatedRefIndexError'; 40 | Object.setPrototypeOf(this, BMLDuplicatedRefIndexError.prototype); 41 | } 42 | } 43 | 44 | export class BMLDuplicatedRefError extends Error { 45 | constructor(refIdentifier: string, bmlDoc: string, errorIndex: number) { 46 | let msg = `Duplicated reference ${refIdentifier} ` 47 | + `at ${stringUtils.lineColumnString(bmlDoc, errorIndex)}`; 48 | super(msg); 49 | this.name = 'BMLDuplicatedRefError'; 50 | Object.setPrototypeOf(this, BMLDuplicatedRefError.prototype); 51 | } 52 | } 53 | 54 | export class EvalBoundSettingsError extends Error { 55 | constructor(field: string, value: any) { 56 | super(`Eval binding of settings field '${field}' of '${value}' is invalid`); 57 | this.name = 'EvalBoundSettingsError'; 58 | Object.setPrototypeOf(this, EvalBoundSettingsError.prototype); 59 | } 60 | } 61 | 62 | export class EvalBindingError extends Error { 63 | constructor(key: string) { 64 | super(`Eval binding ${key} was bound multiple times.`); 65 | this.name = 'EvalBindingError'; 66 | Object.setPrototypeOf(this, EvalBindingError.prototype); 67 | } 68 | } 69 | 70 | export class EvalDisabledError extends Error { 71 | constructor() { 72 | super(`This document includes eval blocks and cannot be rendered with allowEval=false.`); 73 | this.name = 'EvalDisabledError'; 74 | Object.setPrototypeOf(this, EvalDisabledError.prototype); 75 | } 76 | } 77 | 78 | export class IncludeError extends Error { 79 | constructor(includePath: string, message: string) { 80 | super(`Include failed for '${includePath}': ${message}`) 81 | this.name = 'IncludeError'; 82 | Object.setPrototypeOf(this, IncludeError.prototype); 83 | } 84 | } 85 | 86 | export class InvalidForkWeightsError extends Error { 87 | constructor() { 88 | super('Fork has invalid weights'); 89 | this.name = 'InvalidForkWeightsError'; 90 | Object.setPrototypeOf(this, InvalidForkWeightsError.prototype); 91 | } 92 | } 93 | 94 | export class NoPossibleChoiceError extends Error { 95 | constructor() { 96 | super('Cannot perform weighted choice where the given weights have a sum of zero'); 97 | this.name = 'NoPossibleWeightsError'; 98 | Object.setPrototypeOf(this, NoPossibleChoiceError.prototype); 99 | } 100 | } 101 | 102 | 103 | -------------------------------------------------------------------------------- /test/testAnalysis.ts: -------------------------------------------------------------------------------- 1 | import expect from 'expect'; 2 | 3 | import { analyze } from '../src/analysis'; 4 | 5 | 6 | /* 7 | 8 | A way to sanity check these results is by actually executing snippets 9 | repeatedly and counting the unique results. Of course this is only 10 | useful with small branch counts! 11 | 12 | let bmlScript = ` 13 | {c: (foo), (bar), (biz)} 14 | {(x), (y)} 15 | {@c: 0, 1 -> {(a), (b), (c)}, 2 -> ()} 16 | `; 17 | 18 | const ITERS = 1000; 19 | let acc = new Set(); 20 | for (let i = 0; i < ITERS; i++) { 21 | acc.add(bml(bmlScript)); 22 | } 23 | console.log(`After ${ITERS} iterations, I found ${acc.size} unique outputs`); 24 | */ 25 | 26 | 27 | describe('analyze', function() { 28 | it('counts an empty document', function() { 29 | let testString = ``; 30 | expect(analyze(testString).possibleOutcomes) 31 | .toEqual(BigInt(1)); 32 | }); 33 | 34 | it('counts a document with no forks', function() { 35 | let testString = `test`; 36 | expect(analyze(testString).possibleOutcomes) 37 | .toEqual(BigInt(1)); 38 | }); 39 | 40 | it('counts a document with a simple fork', function() { 41 | let testString = `test {(a), (b)}`; 42 | expect(analyze(testString).possibleOutcomes) 43 | .toEqual(BigInt(2)); 44 | }); 45 | 46 | it('counts a document with multiple top-level forks', function() { 47 | let testString = `test {(a), (b)} {(1), (2), (3)}`; 48 | expect(analyze(testString).possibleOutcomes) 49 | .toEqual(BigInt(6)); 50 | }); 51 | 52 | it('counts a document with a nested fork', function() { 53 | let testString = `test {(a), (b), {(c), (d), (e)}}`; 54 | expect(analyze(testString).possibleOutcomes) 55 | .toEqual(BigInt(5)); 56 | }); 57 | 58 | it('counts a document with a top-level fork containing an eval', function() { 59 | let testString = `test {(a), [insert('foo')]}`; 60 | expect(analyze(testString).possibleOutcomes) 61 | .toEqual(BigInt(2)); 62 | }); 63 | 64 | it('counts a document with a ref adding no branches', function() { 65 | let testString = ` 66 | test {id: (a), (b)} 67 | {@id: 0 -> (foo), 1 -> (bar)} 68 | `; 69 | expect(analyze(testString).possibleOutcomes) 70 | .toEqual(BigInt(2)); 71 | }); 72 | 73 | it('counts a document with a ref with a forking value', function() { 74 | let testString = ` 75 | test {id: (a), (b)} 76 | {@id: 0 -> {(foo), (bar)}, 1 -> (bar)} 77 | `; 78 | expect(analyze(testString).possibleOutcomes) 79 | .toEqual(BigInt(3)); 80 | }); 81 | 82 | it('counts a document with a ref with multiple forking values', function() { 83 | let testString = ` 84 | test {id: (a), (b)} 85 | {@id: 0 -> {(foo), (bar), (biz)}, 1 -> {(a), (b)}} 86 | `; 87 | expect(analyze(testString).possibleOutcomes) 88 | .toEqual(BigInt(2 + 2 + 1)); 89 | }); 90 | 91 | it('counts a document with a ref with fallback branches', function() { 92 | let testString = ` 93 | test {id: (a), (b), (c)} 94 | {@id: 0 -> {(foo), (bar), (biz), (baz)}, {(1), (2), (3)}} 95 | `; 96 | expect(analyze(testString).possibleOutcomes) 97 | .toEqual(BigInt(3 + 3 + (2 * 2))); 98 | }); 99 | 100 | it('counts a document with a ref with explicit many-to-one mappings', function() { 101 | let testString = ` 102 | test {id: (a), (b), (c)} 103 | {@id: 0, 1 -> {(foo), (bar), (biz)}, (fallback)} 104 | `; 105 | expect(analyze(testString).possibleOutcomes) 106 | .toEqual(BigInt(3 + 2 + 2)); 107 | }); 108 | 109 | it('counts a bare ref as adding no branches', function() { 110 | let testString = `{id: (a), (b)} {@id}`; 111 | expect(analyze(testString).possibleOutcomes) 112 | .toEqual(BigInt(2)); 113 | }); 114 | 115 | xit('counts silent forks only where referenced', function() { 116 | let testString = '{#foo: (a), (b)} {#bar: (x), (y), (z)} {{@foo}, {@bar}}'; 117 | expect(analyze(testString).possibleOutcomes) 118 | .toEqual(BigInt(5)); 119 | }) 120 | 121 | it('Handles unresolved refs gracefully', function() { 122 | let testString = `{@id}`; 123 | expect(analyze(testString).possibleOutcomes) 124 | .toEqual(BigInt(1)); 125 | }) 126 | 127 | }); 128 | -------------------------------------------------------------------------------- /test/testCli.ts: -------------------------------------------------------------------------------- 1 | import expect from 'expect'; 2 | import * as cli from '../src/cli'; 3 | import { defaultRenderSettings } from '../src/settings'; 4 | 5 | 6 | describe('cli', function() { 7 | 8 | beforeAll(() => { 9 | // Many of these tests trigger console errors which spam the Jest 10 | // test results because for some reason Jest doesn't support 11 | // silencing logs on passing tests... 12 | console.error = jest.fn(); 13 | }); 14 | 15 | it('prints help when given any help switch', function() { 16 | for (let arg of cli.HELP_SWITCHES) { 17 | let action = cli.determineAction([arg]); 18 | expect(action.function).toBe(cli.printHelp); 19 | } 20 | }); 21 | 22 | it('prints version info when given any version switch', function() { 23 | for (let arg of cli.VERSION_SWITCHES) { 24 | let action = cli.determineAction([arg]); 25 | expect(action.function).toBe(cli.printVersionInfo); 26 | } 27 | }); 28 | 29 | it('reads from a path when given an argument', function() { 30 | let path = 'some path'; 31 | let action = cli.determineAction([path]); 32 | expect(action.function).toBe(cli.executeFromPath); 33 | let expectedSettings = { 34 | ...defaultRenderSettings, 35 | workingDir: ".", 36 | } 37 | expect(action.args).toEqual([path, expectedSettings]); 38 | }); 39 | 40 | it('reads from stdin when not given any arguments', function() { 41 | let action = cli.determineAction([]); 42 | expect(action.function).toBe(cli.executeFromStdin); 43 | expect(action.args).toEqual([defaultRenderSettings]); 44 | }); 45 | 46 | it('strips away only the first arg when `node` is not the first', function() { 47 | let stripped = cli.stripArgs(['first', 'second']); 48 | expect(stripped).toEqual(['second']); 49 | }); 50 | 51 | it('strips away the first two args when `node` is the first', function() { 52 | let stripped = cli.stripArgs(['/usr/bin/node', 'second', 'third']); 53 | expect(stripped).toEqual(['third']); 54 | }); 55 | 56 | it('fails when seed flag is used but no seed is provided', function() { 57 | let action = cli.determineAction(['--seed']); 58 | expect(action.function).toBe(cli.printHelpForError); 59 | }); 60 | 61 | it('fails when seed is invalid', function() { 62 | let badSeeds = [ 63 | '123.4', 64 | 'foo', 65 | '--foo' 66 | ]; 67 | for (let seed of badSeeds) { 68 | let action = cli.determineAction(['--seed', seed]); 69 | expect(action.function).toBe(cli.printHelpForError); 70 | } 71 | }); 72 | 73 | it('supports negative seeds', function() { 74 | let action = cli.determineAction(['--seed', '-123']); 75 | expect(action.function).toBe(cli.executeFromStdin); 76 | }); 77 | 78 | it('fails on unknown flags', function() { 79 | let action = cli.determineAction(['--foo']); 80 | expect(action.function).toBe(cli.printHelpForError); 81 | }); 82 | 83 | it('fails when more than one path is provided', function() { 84 | let action = cli.determineAction(['1.bml', '2.bml']); 85 | expect(action.function).toBe(cli.printHelpForError); 86 | }); 87 | 88 | it('supports analysis mode from stdin', function() { 89 | let action = cli.determineAction(['--analyze']); 90 | expect(action.function).toBe(cli.analyzeFromStdin); 91 | expect(action.args).toEqual([]); 92 | }); 93 | 94 | it('supports analysis mode from path', function() { 95 | let action = cli.determineAction(['--analyze', '1.bml']); 96 | expect(action.function).toBe(cli.analyzeFromPath); 97 | expect(action.args).toEqual(['1.bml']); 98 | }); 99 | 100 | it('does not support interactive mode from stdin', function() { 101 | let action = cli.determineAction(['--interactive']); 102 | expect(action.function).toBe(cli.printHelpForError); 103 | action = cli.determineAction(['-i']); 104 | expect(action.function).toBe(cli.printHelpForError); 105 | }); 106 | 107 | it('supports interactive mode from path', function() { 108 | let action = cli.determineAction(['--interactive', '1.bml']); 109 | expect(action.function).toBe(cli.executeInteractively); 110 | let expectedSettings = { 111 | ...defaultRenderSettings, 112 | workingDir: ".", 113 | } 114 | expect(action.args).toEqual(['1.bml', expectedSettings]); 115 | action = cli.determineAction(['-i', '1.bml']); 116 | expect(action.function).toBe(cli.executeInteractively); 117 | expect(action.args).toEqual(['1.bml', expectedSettings]); 118 | }); 119 | 120 | it('supports all settings', function() { 121 | let path = 'foo.bml'; 122 | let action = cli.determineAction([ 123 | '--seed', '123', '--no-eval', path]); 124 | let expectedSettings = { 125 | randomSeed: 123, 126 | allowEval: false, 127 | workingDir: ".", 128 | }; 129 | expect(action.function).toBe(cli.executeFromPath); 130 | expect(action.args).toEqual([path, expectedSettings]); 131 | }); 132 | }); 133 | -------------------------------------------------------------------------------- /src/postprocessing.ts: -------------------------------------------------------------------------------- 1 | const indefinite = require('indefinite'); 2 | 3 | const BLANK_LINE_RE = /^\s*$/; 4 | const TRAILING_WHITESPACE_RE = /\s+$/; 5 | 6 | /** 7 | * Cleans whitespace in the given text by: 8 | * 1. Removing all trailing whitespace in every line 9 | * 2. Collapsing any runs of over 1 blank line 10 | * 3. Collapsing any runs of over 1 space in the middle of a line. 11 | * 4. Removing blank lines at the start of the text 12 | * 5. Ensuring the text ends with a single line break 13 | */ 14 | export function whitespaceCleanup(text: string): string { 15 | let out = ''; 16 | let atDocStart = true; 17 | let lastLineWasBlank = false; 18 | for (let line of text.split('\n')) { 19 | let isBlank = BLANK_LINE_RE.test(line); 20 | 21 | if (atDocStart) { 22 | if (isBlank) { 23 | // Skip blank lines at start of document 24 | continue; 25 | } else { 26 | atDocStart = false; 27 | } 28 | } 29 | 30 | if (isBlank) { 31 | if (lastLineWasBlank) { 32 | // Skip runs of blank lines 33 | continue; 34 | } 35 | // Lines consisting of only whitespace should 36 | // become simply blank lines 37 | line = ''; 38 | } else { 39 | // intra-line cleanups 40 | line = line.replace(TRAILING_WHITESPACE_RE, ''); 41 | let rewrittenLine = ''; 42 | let atLineStart = true; 43 | let lastCharWasSpace = false; 44 | for (let char of line) { 45 | let charIsSpace = char === ' '; 46 | if (!atLineStart && lastCharWasSpace && charIsSpace) { 47 | continue; 48 | } else { 49 | if (!charIsSpace) { 50 | atLineStart = false; 51 | } 52 | rewrittenLine += char; 53 | lastCharWasSpace = charIsSpace; 54 | } 55 | } 56 | line = rewrittenLine; 57 | } 58 | 59 | lastLineWasBlank = isBlank; 60 | 61 | out += line + '\n'; 62 | } 63 | 64 | // Edge case: if input ended with a line break already, above code 65 | // will result in \n\n ending the output. Correct this so output 66 | // always terminates with a single \n 67 | if (out.endsWith('\n\n')) { 68 | out = out.substring(0, out.length - 1); 69 | } 70 | 71 | return out; 72 | } 73 | 74 | // Note the 3 dashes here are the different kinds, not the same character 75 | const MISPLACED_WORD_ENDING_PUNC_RE = /([a-zA-Z0-9\xA0-\uFFFF"'_])(\s+)([.,:;!?\-\–\—]+)/g; 76 | 77 | /** 78 | * Performs simple English-like correction of whitespace around 79 | * punctuation marks. 80 | * 81 | * Snap [, . : ; ! ?] to the end of preceding words, quotes, or underscores, 82 | * when separated by whitespace (including line breaks.) 83 | * 84 | * Note that this will shift punctuation to the *right* of quotes ([" ' _]), 85 | * but will not shift punctuation to the inside of quotes, as this is often 86 | * dependent on style and context. 87 | */ 88 | export function punctuationCleanup(text: string): string { 89 | return text.replace(MISPLACED_WORD_ENDING_PUNC_RE, '$1$3$2'); 90 | } 91 | 92 | 93 | // \p{Ll} matches unicode lowercase letters which have uppercase variants. 94 | const INCORRECT_CAPS_RE = /([.!?]["'_\]\)\}\s]*?\s+|^\s*)(\p{Ll})/gu; 95 | 96 | 97 | /** 98 | * Tries to correct capitalization of the first words of sentences. 99 | * 100 | * This automatically capitalizes the first letter of the first word 101 | * following a sentence-ending punctuation mark. 102 | */ 103 | export function capitalizationCleanup(text: string): string { 104 | // Conforms to `text.replace` replacer function interface 105 | function correctCaps(_match: string, p1: string, p2: string) { 106 | return p1 + p2.toUpperCase(); 107 | } 108 | 109 | return text.replace(INCORRECT_CAPS_RE, correctCaps); 110 | } 111 | 112 | const VISUAL_LINE_BREAK_RE = /\\ *(\r?\n|\r)[ \t]*/g 113 | 114 | export function replaceVisualLineBreaks(text: string): string { 115 | return text.replace(VISUAL_LINE_BREAK_RE, ' '); 116 | } 117 | 118 | const INDEFINITE_ARTICLE_RE = /\b(a|an) ([\p{L}0-9]+)\b/igu 119 | 120 | /** 121 | * Attempt to correct English indefinite articles (a / an) 122 | */ 123 | export function correctIndefiniteArticles(text: string) { 124 | function upcaseFirstLetter(s: string): string { 125 | if (s.length === 0) { 126 | return s; 127 | } else if (s.length === 1) { 128 | return s.toUpperCase(); 129 | } else { 130 | return s[0].toUpperCase() + s.slice(1); 131 | } 132 | } 133 | // Conforms to `text.replace` replacer function interface 134 | function correctArticle(_match: string, originalArticle: string, word: string) { 135 | let article = indefinite(word, { articleOnly: true }); 136 | if (originalArticle === 'a' || originalArticle === 'an') { 137 | return article + ' ' + word; 138 | } else if (originalArticle === 'A' || originalArticle === 'An') { 139 | return upcaseFirstLetter(article) + ' ' + word; 140 | } else { 141 | // All caps 142 | return article.toUpperCase() + ' ' + word; 143 | } 144 | } 145 | 146 | return text.replace(INDEFINITE_ARTICLE_RE, correctArticle); 147 | } 148 | -------------------------------------------------------------------------------- /src/analysis.ts: -------------------------------------------------------------------------------- 1 | import { parseDocument } from './parsers'; 2 | import { Lexer } from './lexer'; 3 | import { AstNode } from './ast'; 4 | import { ChoiceFork } from './choiceFork'; 5 | import { Reference } from './reference' 6 | import { EvalBlock } from './evalBlock'; 7 | 8 | export type AnalysisResult = { 9 | possibleOutcomes: bigint 10 | }; 11 | 12 | type ForkIdMap = Map; 13 | 14 | abstract class AnalysisNode { 15 | } 16 | 17 | 18 | class ForkNode extends AnalysisNode { 19 | id: string | null; 20 | branches: AnalysisTree[]; 21 | referencedBy: RefNode[] = []; 22 | 23 | constructor(id: string | null = null, branches: AnalysisTree[] = []) { 24 | super(); 25 | this.id = id; 26 | this.branches = branches; 27 | } 28 | 29 | countOutcomes(): bigint { 30 | let outcomes = BigInt(0); 31 | for (let branch of this.branches) { 32 | outcomes += countOutcomesForAnalysisTree(branch); 33 | } 34 | for (let ref of this.referencedBy) { 35 | outcomes += ref.countOutcomesAddedToReferredNode(); 36 | } 37 | return outcomes; 38 | } 39 | } 40 | 41 | class RefNode extends AnalysisNode { 42 | forkNodeReferredTo: ForkNode; 43 | referenceMap: Map; 44 | 45 | constructor(forkNodeReferredTo: ForkNode, referenceMap: Map) { 46 | super(); 47 | this.forkNodeReferredTo = forkNodeReferredTo; 48 | this.referenceMap = referenceMap; 49 | } 50 | 51 | countOutcomesAddedToReferredNode(): bigint { 52 | let outcomes = BigInt(0); 53 | for (let [_, value] of this.referenceMap.entries()) { 54 | // Mapped choices contribute 1 less than plain choices 55 | // because 1 has already been accounted for in the referenced fork 56 | outcomes += countOutcomesForAnalysisTree(value) - BigInt(1); 57 | } 58 | return outcomes; 59 | } 60 | } 61 | 62 | 63 | 64 | type AnalysisTree = AnalysisNode[] 65 | 66 | 67 | function countOutcomesForAnalysisTree(tree: AnalysisTree): bigint { 68 | let outcomes = BigInt(1); // Base case is 1 for an empty tree 69 | for (let node of tree) { 70 | if (node instanceof ForkNode) { 71 | outcomes *= node.countOutcomes(); 72 | } 73 | // branches added by ref nodes are resolved by their referred forks, so skip them. 74 | } 75 | return outcomes; 76 | } 77 | 78 | 79 | function deriveForkNode(choiceFork: ChoiceFork, forkIdMap: ForkIdMap): ForkNode { 80 | let forkNode = new ForkNode(choiceFork.identifier, []); 81 | if (forkNode.id) { 82 | forkIdMap.set(forkNode.id, [forkNode, choiceFork]); 83 | } 84 | for (let weight of choiceFork.weights) { 85 | let choice = weight.choice; 86 | if (choice instanceof EvalBlock) { 87 | forkNode.branches.push([]) 88 | } else { 89 | let subTree = deriveAnalysisTree(choice, forkIdMap); 90 | forkNode.branches.push(subTree) 91 | } 92 | 93 | } 94 | return forkNode; 95 | } 96 | 97 | function deriveRefNode(ref: Reference, forkIdMap: ForkIdMap): RefNode { 98 | // TODO handle re-executing refs 99 | let forkMapLookupResult = forkIdMap.get(ref.id); 100 | if (!forkMapLookupResult) { 101 | // Handle unmapped refs gracefully - this is expected 102 | // for refs pointing to declarations in included files 103 | return new RefNode(new ForkNode(), new Map()); 104 | } 105 | let forkNodeReferredTo = forkIdMap.get(ref.id)![0]; 106 | let refNode = new RefNode(forkNodeReferredTo, new Map()); 107 | forkNodeReferredTo.referencedBy.push(refNode); 108 | for (let [key, value] of ref.referenceMap.entries()) { 109 | let mappedValue: AnalysisTree; 110 | if (value instanceof EvalBlock) { 111 | mappedValue = []; 112 | } else { 113 | let subTree = deriveAnalysisTree(value, forkIdMap); 114 | mappedValue = subTree; 115 | } 116 | refNode.referenceMap.set(key, mappedValue); 117 | } 118 | // To make later analysis simple, create the implied mappings for the fallback fork 119 | if (ref.fallbackChoiceFork) { 120 | let fallbackForkNode = deriveForkNode(ref.fallbackChoiceFork, forkIdMap); 121 | let referredFork = forkIdMap.get(ref.id)![1]; 122 | for (let i = 0; i < referredFork.weights.length; i++) { 123 | if (!refNode.referenceMap.has(i)) { 124 | refNode.referenceMap.set(i, [fallbackForkNode]); 125 | } 126 | } 127 | } 128 | return refNode; 129 | } 130 | 131 | function deriveAnalysisTree(ast: AstNode[], forkIdMap: ForkIdMap): AnalysisTree { 132 | let analysisTree: AnalysisTree = []; 133 | for (let node of ast) { 134 | if (node instanceof ChoiceFork) { 135 | analysisTree.push(deriveForkNode(node, forkIdMap)); 136 | } else if (node instanceof Reference) { 137 | analysisTree.push(deriveRefNode(node, forkIdMap)); 138 | } 139 | // string nodes add no branches, so they are removed from the analysis tree 140 | } 141 | return analysisTree; 142 | } 143 | 144 | 145 | 146 | 147 | /** 148 | * Analyze a BML document without executing it 149 | * 150 | * This does a rough back-of-the-envelope approximation of the number of possible 151 | * branches through the document. It has several shortcomings, especially when dealing 152 | * with refs and silent forks. 153 | * 154 | * Note that if your document contains cyclical reference loops, this will hang. 155 | */ 156 | export function analyze(bmlDocument: string): AnalysisResult { 157 | let lexer = new Lexer(bmlDocument); 158 | let ast = parseDocument(lexer, true); 159 | let analysisTree = deriveAnalysisTree(ast, new Map()); 160 | let possibleOutcomes = countOutcomesForAnalysisTree(analysisTree); 161 | return { 162 | possibleOutcomes 163 | }; 164 | } 165 | -------------------------------------------------------------------------------- /test/testRand.ts: -------------------------------------------------------------------------------- 1 | import expect from "expect"; 2 | import sha from "sha.js"; 3 | import { Buffer } from "buffer"; 4 | 5 | import { WeightedChoice } from "../src/weightedChoice"; 6 | import { NoPossibleChoiceError } from "../src/errors"; 7 | let rand = require("../src/rand.ts"); 8 | 9 | type RandomFunction = (a: number, b: number) => number; 10 | type ValidateNumberKind = (a: number) => boolean; 11 | 12 | function testGeneratorFunctionTypeAndRange( 13 | randomFunction: RandomFunction, 14 | typeValidationFunction: ValidateNumberKind, 15 | min: number, 16 | max: number 17 | ) { 18 | for (let i = 0; i < 100; i++) { 19 | let value = randomFunction(min, max); 20 | expect(typeValidationFunction(value)).toBe(true); 21 | expect(value).toBeGreaterThanOrEqual(min); 22 | expect(value).toBeLessThanOrEqual(max); 23 | } 24 | } 25 | 26 | function testGeneratorFunctionOutputMean( 27 | randomFunction: RandomFunction, 28 | min: number, 29 | max: number 30 | ) { 31 | let sum = 0; 32 | for (let i = 0; i < 1000; i++) { 33 | sum += randomFunction(min, max); 34 | } 35 | let mean = sum / 1000; 36 | let diff = Math.abs(mean - (min + max) / 2); 37 | expect(diff).toBeLessThan((min + max) / 4); 38 | } 39 | 40 | describe("randomFloat", function () { 41 | function isFloat(n: number): boolean { 42 | return n % 1 !== 0; 43 | } 44 | 45 | it("generates floats within the given range", function () { 46 | testGeneratorFunctionTypeAndRange(rand.randomFloat, isFloat, 0, 10); 47 | }); 48 | 49 | it("generates values whose mean is near bounds midpoint", function () { 50 | testGeneratorFunctionOutputMean(rand.randomFloat, 0, 10); 51 | }); 52 | }); 53 | 54 | describe("randomInt", function () { 55 | function isInt(n: number) { 56 | return n % 1 === 0; 57 | } 58 | 59 | it("generates ints within the given range", function () { 60 | testGeneratorFunctionTypeAndRange(rand.randomInt, isInt, 0, 10); 61 | }); 62 | 63 | it("generates values whose mean is near bounds midpoint", function () { 64 | testGeneratorFunctionOutputMean(rand.randomInt, 0, 10); 65 | }); 66 | }); 67 | 68 | describe("normalizeWeights", function () { 69 | it("should do nothing when all weights sum to 100", function () { 70 | let weights = [ 71 | new WeightedChoice(["a"], 40), 72 | new WeightedChoice(["b"], 60), 73 | ]; 74 | expect(rand.normalizeWeights(weights)).toEqual(weights); 75 | }); 76 | }); 77 | 78 | describe("setRandomSeed", function () { 79 | it("should make the output of randomFloat predictable", function () { 80 | jest.isolateModules(() => (rand = require("../src/rand.ts"))); 81 | rand.setRandomSeed(1234); 82 | let firstResult = rand.randomFloat(0, 1); 83 | rand.setRandomSeed(1234); 84 | expect(rand.randomFloat(0, 1)).toBeCloseTo(firstResult); 85 | }); 86 | 87 | it("should make the output of randomInt predictable", function () { 88 | jest.isolateModules(() => (rand = require("../src/rand.ts"))); 89 | rand.setRandomSeed(1234); 90 | let firstResult = rand.randomInt(0, 1000); 91 | rand.setRandomSeed(1234); 92 | expect(rand.randomInt(0, 1000)).toBe(firstResult); 93 | }); 94 | 95 | it("when not called, should produce different outputs on program runs", function () { 96 | jest.isolateModules(() => (rand = require("../src/rand.ts"))); 97 | let firstResult = rand.randomFloat(0, 1); 98 | expect(rand.randomFloat(0, 1)).not.toBeCloseTo(firstResult, 10); 99 | }); 100 | 101 | it("produces stable results forever", function () { 102 | jest.isolateModules(() => (rand = require("../src/rand.ts"))); 103 | rand.setRandomSeed(1234); 104 | let iters = 100000; 105 | let results = Buffer.alloc(iters * 8); 106 | for (let i = 0; i < iters; i++) { 107 | results.writeFloatBE(rand.randomInt(), i * 8); 108 | } 109 | let hash = sha("sha256").update(results).digest("hex"); 110 | expect(hash).toBe( 111 | "f2f381924630531a5e188f5cdbd110b90f90b796f7daf0dcf402070e8d46ae80" 112 | ); 113 | }); 114 | }); 115 | 116 | describe("weightedChoose", function () { 117 | beforeEach(function () { 118 | rand.setRandomSeed(0); // pin seed for reproducibility 119 | }); 120 | 121 | it("behaves on well-formed weights", function () { 122 | let weights = [ 123 | new WeightedChoice(["foo"], 40), 124 | new WeightedChoice(["bar"], 60), 125 | ]; 126 | let result = rand.weightedChoose(weights); 127 | expect(result.choice).toStrictEqual(["foo"]); 128 | expect(result.choiceIndex).toBe(0); 129 | 130 | result = rand.weightedChoose(weights); 131 | expect(result.choice).toStrictEqual(["bar"]); 132 | expect(result.choiceIndex).toBe(1); 133 | 134 | result = rand.weightedChoose(weights); 135 | expect(result.choice).toStrictEqual(["foo"]); 136 | expect(result.choiceIndex).toBe(0); 137 | }); 138 | 139 | it("errors when given no weights", function () { 140 | expect(() => rand.weightedChoose([])).toThrow( 141 | "Cannot perform weighted choice where the given weights have a sum of zero" 142 | ); 143 | // For reasons beyond me, toThrow here fails when I reference the actual error type 144 | }); 145 | 146 | it("errors when given all-zero weights", function () { 147 | expect(() => { 148 | let weights = [ 149 | new WeightedChoice(["foo"], 0), 150 | new WeightedChoice(["bar"], 0), 151 | ]; 152 | rand.weightedChoose(weights); 153 | }).toThrow( 154 | "Cannot perform weighted choice where the given weights have a sum of zero" 155 | ); 156 | // For reasons beyond me, toThrow here fails when I reference the actual error type 157 | }); 158 | }); 159 | -------------------------------------------------------------------------------- /test/lao_tzu_36.bml: -------------------------------------------------------------------------------- 1 | {#form: 2 | (That which [] must first []), 3 | (If you would [] it, you must first []), 4 | (If you want to [] something, you must first []), 5 | (For something to [], it must first []), 6 | (If you wish to [] it, you must certainly [] it), 7 | (what seeks [] must first []) 8 | } 9 | {#endPunc: 10 | ({(.) 70, (;)} {(\n), ()}) 11 | } 12 | {#cherishIt: (hold it dear), (cherish it), (hold it close), (exalt it), (keep it close)} 13 | {#beRaised: (be raised high), (have been raised)} 14 | {#weiMing: 15 | (the {(small), (little)} dark light), 16 | ({(subtle), (quiet), (mysterious)} { 17 | (knowledge), (understanding), (insight), (discernment), (vision)}) 20, 18 | (formless {(light), (sight)}), 19 | (delicate {(mystery), (understanding), (insight)}), 20 | (the mystery of discernment), 21 | ({(perceiving the imperceptible), (discerning the indiscernable)}), 22 | (the cause of cause), 23 | (the illuminating shadow), 24 | (the mist of reality) 25 | } 26 | {#sheng: 27 | (overcomes), 28 | (prevails over), 29 | (conquers), 30 | (subdues), 31 | (defeats), 32 | (passes by), 33 | (leaves behind) 34 | } 35 | 36 | {@form: 37 | 0 -> (That which shrinks\nmust first {(grow), (have grown), (expand)}), 38 | 1 -> (If you would shrink it,\nyou must first grow it), 39 | 2 -> (If you want to shrink something,\nyou must first allow it to grow), 40 | 3 -> (For something to shrink,\nit must first {(have grown), (expand)}), 41 | 4 -> (If you wish to shrink it,\nyou must certainly grow it), 42 | 5 -> (What seeks to shrink\nmust first {(grow), (have grown), (expand)}) 43 | }{@endPunc} 44 | {@form: 45 | 0 -> (That which weakens\nmust first {(be strong), (have strength)}), 46 | 1 -> (If you would weaken it,\nyou must first \ 47 | {(let it grow strong), (give it strength)}), 48 | 2 -> (If you want to weaken something,\nyou must first\ 49 | {(allow), (invite), (nurture)} its strength), 50 | 3 -> (For something to grow weak,\nit must first\ 51 | {(have grown strong), (have strength)}), 52 | 4 -> (If you wish to weaken it,\nyou must certainly\ 53 | {(strengthen), (reinforce)} it), 54 | 5 -> (What seeks weakness\nmust first { 55 | (have strength), (have resolve), (have conviction), 56 | (be strong), (be firm)}) 57 | }{@endPunc} 58 | {@form: 59 | 0 -> (That which {(collapses), (falls)},\ 60 | \nmust first {@beRaised}), 61 | 1 -> (If you would {(abandon), (spurn), (discard), (forget)} it,\ 62 | \nyou must first {@cherishIt}), 63 | 2 -> (If you want to { 64 | (abandon a thing), (cast a thing away), (leave a thing behind), (spurn a thing) 65 | },\nyou must first {@cherishIt}), 66 | 3 -> (For something to {(collapse), (fall)},\nit must first {@beRaised}), 67 | 4 -> (If you wish to {(destroy), (crush), (wreck), (raze)} it, \ 68 | \nyou must certainly {(build it up), (construct it)}), 69 | 5 -> (What seeks {(oblivion), (obscurity), (to be forgotten)}, \ 70 | \nmust first { 71 | (be far exalted), (be told in legend), 72 | (be marked in history), (be widely regarded) 73 | }) 74 | }{@endPunc} 75 | {@form: 76 | 0 -> (That which is {(seized), (taken)}\nmust first be {(given), (granted)}), 77 | 1 -> (If you would {(seize), (take)} it,\nyou must first {(give), (grant)} it), 78 | 2 -> (What is stolen\nwas surely given), 79 | 3 -> ({(With), (Before)} receiving\nthere must be giving), 80 | 4 -> (If you wish to take it,\none must certainly give it), 81 | 5 -> (What seeks to take\nhas {(surely), (without doubt)} given) 82 | }. 83 | 84 | {desc: 85 | (This is known as ), (Some call this ), (This is what is meant by ") 86 | }{@weiMing}{@desc: 2 -> (."), (.)} 87 | { 88 | (The {(soft), (tender)}, the {(weak), (gentle), (yielding)} prevail\ 89 | \nover the {(hard), (steadfast), (rigid)}, the {(strong), (firm)}), 90 | (The {(submissive), (weak), (gentle), (yielding)} {@sheng} \ 91 | the {(strong), (firm), (unyielding)}), 92 | ({(Thus), (It is thus that)} the {(soft {@sheng} the hard), (weak {@sheng} the strong)}) 93 | }. 94 | 95 | {#yuan: (deep waters), (the depths), (the abyss)} 96 | {#liChi: (sharp tools), (sharp weapons), (powerful tools), (exacting tools), (fine instruments), (coercive methods), (tools of enforcement), (methods of force)} 97 | {#jen: (its people), (others), (the people), (the outside)} 98 | {#kuo: (a country), (a nation), (an empire), (a ruler)} 99 | {#kuoPronoun: ({@kuo: 3 -> (their), (its)})} 100 | 101 | {() 30, /* Sometimes omit last section following Le Guin */ 102 | ( 103 | {simile: (As), (So as), (And so as), ()} { 104 | (fish cannot leave {@yuan}), 105 | (fish must not be taken from {@yuan}), 106 | (fish must not leave {@yuan})}{@simile: 3 -> ({@endPunc}), (,)} 107 | { 108 | ({@kuo}'s {@liChi} must remain hidden from {@kuoPronoun} people), 109 | ({@kuo}'s {@liChi} {(cannot be revealed), (must remain hidden), (must remain in the dark)}), 110 | ({@kuo}'s {@liChi} cannot be revealed to others) 111 | }. 112 | )} 113 | 114 | // And for test completeness, include features not demonstrated above: 115 | 116 | // A function bound in one eval block then used and inserted in another 117 | {[ 118 | bind({ 119 | foo: () => { 120 | return 'function called'; 121 | } 122 | }); 123 | ]} 124 | {[insert(foo())]} 125 | 126 | // A fork directly placed inside a fork branch 127 | {{(a fork directly inside a fork branch)}} 128 | 129 | // Multiple fallback options in a backref 130 | {@kuo: 0 -> (it), (a), (b)} 131 | 132 | // A literal block containing escaped code 133 | [[ 134 | {(foo)} 135 | ]] 136 | -------------------------------------------------------------------------------- /src/interactive.ts: -------------------------------------------------------------------------------- 1 | import { render } from './renderer'; 2 | import { analyze } from './analysis'; 3 | import * as blessed from 'blessed'; 4 | import { RenderSettings } from './settings'; 5 | // Old-school require is needed for some deps to prevent weird build breakage 6 | const fs = require('fs'); 7 | const process = require('process'); 8 | const clipboard = require('clipboardy'); 9 | const path = require('path'); 10 | 11 | type InteractiveState = { 12 | refreshTimeoutId: NodeJS.Timeout | null, 13 | refreshIntervalSecs: number, 14 | scriptLastModTime: Date, 15 | currentRender: string, 16 | capturedErr: string, 17 | }; 18 | 19 | export function launchInteractive(scriptPath: string, settings: RenderSettings) { 20 | let state: InteractiveState = { 21 | refreshTimeoutId: null, 22 | refreshIntervalSecs: 10, 23 | scriptLastModTime: new Date(), 24 | currentRender: '', 25 | capturedErr: '', 26 | }; 27 | 28 | const realConsoleError = console.error; 29 | const realConsoleWarn = console.warn; 30 | console.error = (data: any) => { 31 | state.capturedErr += data; 32 | }; 33 | console.warn = console.error; 34 | // Ideally, these console overrides should be cleaned up afterward, 35 | // but the interactive view runs async so it'd have to be attached 36 | // to a shutdown callback of some kind. 37 | runInternal(scriptPath, settings, state) 38 | } 39 | 40 | export function runInternal(scriptPath: string, settings: RenderSettings, state: InteractiveState) { 41 | settings.workingDir = path.dirname(scriptPath); 42 | 43 | const screen = blessed.screen({ 44 | smartCSR: true, 45 | fullUnicode: true 46 | }); 47 | 48 | const infoBox = blessed.box({ 49 | width: '100%', 50 | height: 'shrink', 51 | border: { 52 | type: 'line', 53 | }, 54 | content: 'test', 55 | }); 56 | 57 | screen.append(infoBox); 58 | 59 | const renderBox = blessed.box({ 60 | top: 4, 61 | bottom: 3, 62 | border: { 63 | type: 'line', 64 | }, 65 | content: 'output goes here', 66 | scrollable: true, 67 | scrollback: 100, 68 | alwaysScroll: true, 69 | scrollbar: { 70 | ch: ' ', 71 | style: { 72 | inverse: true, 73 | }, 74 | }, 75 | mouse: true, 76 | // TODO scrolling with keys doesn't seem to work. 77 | // I think the screen is grabbing all the keypresses 78 | // or something like that. 79 | key: true, 80 | vi: true, 81 | }); 82 | 83 | screen.append(renderBox); 84 | 85 | const analysisBox = blessed.box({ 86 | top: '100%-3', 87 | bottom: 0, 88 | border: { 89 | type: 'line', 90 | }, 91 | content: 'analysis goes here', 92 | }); 93 | 94 | screen.append(analysisBox); 95 | 96 | 97 | const alertPopup = blessed.message({ 98 | left: 'center', 99 | top: 'center', 100 | width: 'shrink', 101 | height: 'shrink', 102 | border: { 103 | type: 'line', 104 | }, 105 | hidden: true, 106 | }); 107 | 108 | screen.append(alertPopup); 109 | 110 | function formatInfoBoxText() { 111 | return `Source: ${scriptPath} | Refresh delay: ${state.refreshIntervalSecs}s\n` 112 | + `R/Spc: Refresh | C: Copy | Ctrl-Up/Dwn: Change refresh delay` 113 | 114 | } 115 | 116 | function updateAnalysis() { 117 | let bmlSource = '' + fs.readFileSync(scriptPath); 118 | let text; 119 | try { 120 | let { possibleOutcomes } = analyze(bmlSource); 121 | text = `Possible outcomes: ${possibleOutcomes.toLocaleString()}` 122 | } catch (e) { 123 | text = '[error]' 124 | } 125 | analysisBox.setContent(text); 126 | // Don't trigger screen render, let the re-render do it 127 | } 128 | 129 | function interruptingRefresh() { 130 | clearTimeout(state.refreshTimeoutId!); 131 | refresh() 132 | } 133 | 134 | function refresh() { 135 | state.capturedErr = ''; // Clear stderr output 136 | let statResult = fs.statSync(scriptPath); 137 | state.scriptLastModTime = statResult.mtime; 138 | let bmlSource = '' + fs.readFileSync(scriptPath); 139 | let result = ''; 140 | try { 141 | result = render(bmlSource, settings); 142 | } catch (e: any) { 143 | // Also need to capture warnings somehow and print them out. 144 | // Currently when a missing ref is found it gets printed weirdly over the TUI 145 | result = e.stack.toString(); 146 | } 147 | if (state.capturedErr) { 148 | result += '\n////////////////////\n\nWarning:\n' + state.capturedErr; 149 | } 150 | renderBox.setContent(result); 151 | infoBox.setContent(formatInfoBoxText()); 152 | screen.render(); 153 | state.refreshTimeoutId = setTimeout( 154 | refresh, state.refreshIntervalSecs * 1000); 155 | state.currentRender = result; 156 | } 157 | 158 | // If the file changes, trigger a refresh 159 | setInterval(() => { 160 | let statResult = fs.statSync(scriptPath); 161 | if (statResult.mtime.getTime() !== state.scriptLastModTime.getTime()) { 162 | updateAnalysis(); 163 | interruptingRefresh(); 164 | } 165 | }, 500); 166 | 167 | // TODO 168 | // - Support pausing the refresher 169 | 170 | // Attach key listener for force-refresh 171 | screen.key(['r', 'S-r', 'space'], interruptingRefresh); 172 | 173 | // Attach key listener for copying render output 174 | screen.key(['c', 'S-c'], function(ch, key) { 175 | clipboard.writeSync(state.currentRender); 176 | alertPopup.display('Copied to clipboard', 1, () => { }); 177 | }); 178 | 179 | // Attach key listeners for changing refresh interval 180 | screen.key(['C-up'], function(ch, key) { 181 | state.refreshIntervalSecs += 1; 182 | interruptingRefresh(); 183 | }); 184 | 185 | screen.key(['C-down'], function(ch, key) { 186 | state.refreshIntervalSecs -= 1; 187 | interruptingRefresh(); 188 | }); 189 | 190 | // Quit on Escape, q, or Control-C. 191 | screen.key(['escape', 'q', 'C-c'], function(ch, key) { 192 | process.exit(0); 193 | }); 194 | 195 | // Give render box focus so text navigation goes to it 196 | renderBox.enableKeys(); 197 | renderBox.enableInput(); 198 | renderBox.focus(); 199 | 200 | updateAnalysis(); 201 | refresh(); 202 | } 203 | 204 | 205 | -------------------------------------------------------------------------------- /test/testPostprocessing.ts: -------------------------------------------------------------------------------- 1 | import expect from 'expect'; 2 | import * as postprocessing from '../src/postprocessing'; 3 | 4 | 5 | describe('replaceVisualLineBreaks', function() { 6 | it('works in a basic case', function() { 7 | expect(postprocessing.replaceVisualLineBreaks('foo\\\nbar')).toBe('foo bar'); 8 | }); 9 | 10 | it('works backslash is followed by whitespace before line break', function() { 11 | expect(postprocessing.replaceVisualLineBreaks('foo\\ \nbar')).toBe('foo bar'); 12 | }); 13 | }); 14 | 15 | describe('whitespaceCleanup', function() { 16 | it('removes blank lines at start and end of string', function() { 17 | expect(postprocessing.whitespaceCleanup('\n\n\n\nfoo\n \n')).toBe('foo\n'); 18 | }); 19 | 20 | it('collapses runs of more than 1 blank line into 1', function() { 21 | let input = 'foo\n\nbar\n\n\n\n\nbiz'; 22 | let expectedOutput = 'foo\n\nbar\n\nbiz\n'; 23 | expect(postprocessing.whitespaceCleanup(input)).toBe(expectedOutput); 24 | 25 | }); 26 | 27 | it('removes trailing whitespace on every line', function() { 28 | expect(postprocessing.whitespaceCleanup('foo\n bar \n \n')).toBe('foo\n bar\n'); 29 | }); 30 | 31 | it('automatically inserts an EOF line break', function() { 32 | expect(postprocessing.whitespaceCleanup('foo')).toBe('foo\n'); 33 | }); 34 | 35 | it('doesnt insert a redundant EOF line break when one already exists', function() { 36 | expect(postprocessing.whitespaceCleanup('foo\n')).toBe('foo\n'); 37 | }); 38 | 39 | it('preserves leading whitespace on every line', function() { 40 | expect(postprocessing.whitespaceCleanup(' foo')).toBe(' foo\n'); 41 | }); 42 | 43 | it('collapses runs of more than 1 whitespace in the middle of a line', function() { 44 | expect(postprocessing.whitespaceCleanup(' foo bar')).toBe(' foo bar\n'); 45 | }); 46 | }); 47 | 48 | describe('punctuationCleanup', function() { 49 | it('snaps punctuation left', function() { 50 | expect(postprocessing.punctuationCleanup('test . ')).toBe('test. '); 51 | expect(postprocessing.punctuationCleanup('test , ')).toBe('test, '); 52 | expect(postprocessing.punctuationCleanup('test : ')).toBe('test: '); 53 | expect(postprocessing.punctuationCleanup('test ; ')).toBe('test; '); 54 | expect(postprocessing.punctuationCleanup('test ! ')).toBe('test! '); 55 | expect(postprocessing.punctuationCleanup('test ? ')).toBe('test? '); 56 | // Hyphen and multiple hyphens 57 | expect(postprocessing.punctuationCleanup('test - ')).toBe('test- '); 58 | expect(postprocessing.punctuationCleanup('test --- ')).toBe('test--- '); 59 | // En dash 60 | expect(postprocessing.punctuationCleanup('test – ')).toBe('test– '); 61 | // Em dash 62 | expect(postprocessing.punctuationCleanup('test — ')).toBe('test— '); 63 | }); 64 | 65 | it('snaps punctuation left with Chinese characters', function() { 66 | expect(postprocessing.punctuationCleanup('道 . ')).toBe('道. '); 67 | }); 68 | 69 | it('snaps groups of punctuation left together', function() { 70 | expect(postprocessing.punctuationCleanup('test ?! ')).toBe('test?! '); 71 | }); 72 | 73 | it('preserves whatever whitespace comes before', function() { 74 | expect(postprocessing.punctuationCleanup('test \t?! ')).toBe('test?! \t '); 75 | }); 76 | 77 | it('corrects across newlines too', function() { 78 | expect(postprocessing.punctuationCleanup('test \n\n. ')).toBe('test. \n\n '); 79 | }); 80 | 81 | it('corrects after quotes', function() { 82 | expect(postprocessing.punctuationCleanup('"test" .')).toBe('"test". '); 83 | expect(postprocessing.punctuationCleanup('\'test\' .')).toBe('\'test\'. '); 84 | expect(postprocessing.punctuationCleanup('_test_ .')).toBe('_test_. '); 85 | }); 86 | 87 | it('does nothing on correctly written text', function() { 88 | let src = 'test, test: test; test! test? '; 89 | expect(postprocessing.punctuationCleanup(src)).toBe(src); 90 | }); 91 | }); 92 | 93 | describe('capitalizationCleanup', function() { 94 | it('Does nothing on well-capitalized text', function() { 95 | let src = 'Test. Test 2! 123 test? Test'; 96 | expect(postprocessing.capitalizationCleanup(src)).toBe(src); 97 | }); 98 | 99 | it('Capitalizes plain ASCII characters', function() { 100 | let src = 'test. test.'; 101 | expect(postprocessing.capitalizationCleanup(src)).toBe('Test. Test.'); 102 | }); 103 | 104 | it('Capitalizes extended latin characters', function() { 105 | let src = 'test! ä'; 106 | expect(postprocessing.capitalizationCleanup(src)).toBe('Test! Ä'); 107 | }); 108 | 109 | it('Works across line breaks', function() { 110 | let src = 'test. \ntest.'; 111 | expect(postprocessing.capitalizationCleanup(src)).toBe('Test. \nTest.'); 112 | }); 113 | 114 | it('Works across quotation marks', function() { 115 | let src = '"Test." test.'; 116 | expect(postprocessing.capitalizationCleanup(src)).toBe('"Test." Test.'); 117 | }); 118 | 119 | it('Works across close brackets', function() { 120 | let src = '[test.] test'; 121 | expect(postprocessing.capitalizationCleanup(src)).toBe('[test.] Test'); 122 | }); 123 | 124 | it('Works across close parens', function() { 125 | let src = '(test.) test'; 126 | expect(postprocessing.capitalizationCleanup(src)).toBe('(test.) Test'); 127 | }); 128 | 129 | 130 | it('Works across close braces', function() { 131 | let src = '{test.} test'; 132 | expect(postprocessing.capitalizationCleanup(src)).toBe('{test.} Test'); 133 | }); 134 | 135 | it('Works across separators with spaces between', function() { 136 | let src = '{test. \n} test'; 137 | expect(postprocessing.capitalizationCleanup(src)).toBe('{test. \n} Test'); 138 | }); 139 | }); 140 | 141 | describe('correctIndefiniteArticles', function() { 142 | function testCase(input: string, output: string) { 143 | expect(postprocessing.correctIndefiniteArticles(input)).toBe(output); 144 | } 145 | 146 | it('Leaves correct cases intact', function() { 147 | testCase('a dog', 'a dog'); 148 | testCase('an apple', 'an apple'); 149 | testCase('a union', 'a union'); 150 | testCase('a 10', 'a 10'); 151 | testCase('an 8', 'an 8'); 152 | testCase('a UFO', 'a UFO'); 153 | }); 154 | 155 | it('Corrects incorrect cases', function() { 156 | testCase('an dog', 'a dog'); 157 | testCase('a apple', 'an apple'); 158 | testCase('an union', 'a union'); 159 | testCase('an 10', 'a 10'); 160 | testCase('a 8', 'an 8'); 161 | testCase('an UFO', 'a UFO'); 162 | }); 163 | 164 | it('Corrects multiple cases in a string', function() { 165 | testCase('an dog\nand a apple', 'a dog\nand an apple'); 166 | }); 167 | 168 | it('Preserves capitalization schemes', function() { 169 | testCase('An dog', 'A dog'); 170 | testCase('AN dog', 'A dog'); 171 | testCase('A apple', 'An apple'); 172 | testCase('a apple', 'an apple'); 173 | testCase('AN apple', 'AN apple'); 174 | }); 175 | 176 | it('Works on words with diacritics', function() { 177 | testCase('an jalapeño', 'a jalapeño'); 178 | }) 179 | 180 | it('Doesnt act on words spelled with article-like endings', function() { 181 | // Regression test 182 | testCase('can dog', 'can dog'); 183 | }); 184 | }); 185 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | /* 2 | * For a detailed explanation regarding each configuration property, visit: 3 | * https://jestjs.io/docs/configuration 4 | */ 5 | 6 | module.exports = { 7 | // All imported modules in your tests should be mocked automatically 8 | // automock: false, 9 | 10 | // Stop running tests after `n` failures 11 | // bail: 0, 12 | 13 | // The directory where Jest should store its cached dependency information 14 | // cacheDirectory: "/tmp/jest_rs", 15 | 16 | // Automatically clear mock calls, instances and results before every test 17 | clearMocks: true, 18 | 19 | // Indicates whether the coverage information should be collected while executing the test 20 | // collectCoverage: false, 21 | 22 | // An array of glob patterns indicating a set of files for which coverage information should be collected 23 | // collectCoverageFrom: undefined, 24 | 25 | // The directory where Jest should output its coverage files 26 | // coverageDirectory: undefined, 27 | 28 | // An array of regexp pattern strings used to skip coverage collection 29 | // coveragePathIgnorePatterns: [ 30 | // "/node_modules/" 31 | // ], 32 | 33 | // Indicates which provider should be used to instrument code for coverage 34 | coverageProvider: "v8", 35 | 36 | // A list of reporter names that Jest uses when writing coverage reports 37 | // coverageReporters: [ 38 | // "json", 39 | // "text", 40 | // "lcov", 41 | // "clover" 42 | // ], 43 | 44 | // An object that configures minimum threshold enforcement for coverage results 45 | // coverageThreshold: undefined, 46 | 47 | // A path to a custom dependency extractor 48 | // dependencyExtractor: undefined, 49 | 50 | // Make calling deprecated APIs throw helpful error messages 51 | // errorOnDeprecated: false, 52 | 53 | // Force coverage collection from ignored files using an array of glob patterns 54 | // forceCoverageMatch: [], 55 | 56 | // A path to a module which exports an async function that is triggered once before all test suites 57 | // globalSetup: undefined, 58 | 59 | // A path to a module which exports an async function that is triggered once after all test suites 60 | // globalTeardown: undefined, 61 | 62 | // A set of global variables that need to be available in all test environments 63 | // globals: {}, 64 | 65 | // The maximum amount of workers used to run your tests. Can be specified as % or a number. E.g. maxWorkers: 10% will use 10% of your CPU amount + 1 as the maximum worker number. maxWorkers: 2 will use a maximum of 2 workers. 66 | // maxWorkers: "50%", 67 | 68 | // An array of directory names to be searched recursively up from the requiring module's location 69 | // moduleDirectories: [ 70 | // "node_modules" 71 | // ], 72 | 73 | // An array of file extensions your modules use 74 | // moduleFileExtensions: [ 75 | // "js", 76 | // "jsx", 77 | // "ts", 78 | // "tsx", 79 | // "json", 80 | // "node" 81 | // ], 82 | 83 | // A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module 84 | // moduleNameMapper: {}, 85 | 86 | // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader 87 | // modulePathIgnorePatterns: [], 88 | 89 | // Activates notifications for test results 90 | // notify: false, 91 | 92 | // An enum that specifies notification mode. Requires { notify: true } 93 | // notifyMode: "failure-change", 94 | 95 | // A preset that is used as a base for Jest's configuration 96 | // preset: undefined, 97 | 98 | // Run tests from one or more projects 99 | // projects: undefined, 100 | 101 | // Use this configuration option to add custom reporters to Jest 102 | // reporters: undefined, 103 | 104 | // Automatically reset mock state before every test 105 | // resetMocks: false, 106 | 107 | // Reset the module registry before running each individual test 108 | // resetModules: false, 109 | 110 | // A path to a custom resolver 111 | // resolver: undefined, 112 | 113 | // Automatically restore mock state and implementation before every test 114 | // restoreMocks: false, 115 | 116 | // The root directory that Jest should scan for tests and modules within 117 | // rootDir: undefined, 118 | 119 | // A list of paths to directories that Jest should use to search for files in 120 | // roots: [ 121 | // "" 122 | // ], 123 | 124 | // Allows you to use a custom runner instead of Jest's default test runner 125 | // runner: "jest-runner", 126 | 127 | // The paths to modules that run some code to configure or set up the testing environment before each test 128 | // setupFiles: [], 129 | 130 | // A list of paths to modules that run some code to configure or set up the testing framework before each test 131 | // setupFilesAfterEnv: [], 132 | 133 | // The number of seconds after which a test is considered as slow and reported as such in the results. 134 | // slowTestThreshold: 5, 135 | 136 | // A list of paths to snapshot serializer modules Jest should use for snapshot testing 137 | // snapshotSerializers: [], 138 | 139 | // The test environment that will be used for testing 140 | // testEnvironment: "jest-environment-node", 141 | 142 | // Options that will be passed to the testEnvironment 143 | // testEnvironmentOptions: {}, 144 | 145 | // Adds a location field to test results 146 | // testLocationInResults: false, 147 | 148 | // The glob patterns Jest uses to detect test files 149 | testMatch: [ 150 | '/test/**', 151 | ], 152 | 153 | // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped 154 | testPathIgnorePatterns: [ 155 | "releaseTest.js", 156 | "utils.ts", 157 | "test/manual/" 158 | ], 159 | 160 | // The regexp pattern or array of patterns that Jest uses to detect test files 161 | // testRegex: [], 162 | 163 | // This option allows the use of a custom results processor 164 | // testResultsProcessor: undefined, 165 | 166 | // This option allows use of a custom test runner 167 | // testRunner: "jest-circus/runner", 168 | 169 | // This option sets the URL for the jsdom environment. It is reflected in properties such as location.href 170 | // testURL: "http://localhost", 171 | 172 | // Setting this value to "fake" allows the use of fake timers for functions such as "setTimeout" 173 | // timers: "real", 174 | 175 | // A map from regular expressions to paths to transformers 176 | transform: { 177 | "^.+\\.(t|j)sx?$": ["@swc/jest"], 178 | }, 179 | 180 | // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation 181 | // transformIgnorePatterns: [ 182 | // "/node_modules/", 183 | // "\\.pnp\\.[^\\/]+$" 184 | // ], 185 | 186 | // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them 187 | // unmockedModulePathPatterns: undefined, 188 | 189 | // Indicates whether each individual test should be reported during the run 190 | // verbose: undefined, 191 | 192 | // An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode 193 | // watchPathIgnorePatterns: [], 194 | 195 | // Whether to use watchman for file crawling 196 | // watchman: true, 197 | }; 198 | -------------------------------------------------------------------------------- /src/renderer.ts: -------------------------------------------------------------------------------- 1 | import path from 'path'; 2 | import process from 'process'; 3 | import fs from 'fs'; 4 | import * as rand from './rand'; 5 | import * as postprocessing from './postprocessing'; 6 | import { defaultBMLSettings, defaultRenderSettings, mergeSettings, RenderSettings } from './settings'; 7 | import { Reference } from './reference'; 8 | import { parseDocument } from './parsers'; 9 | import { AstNode } from './ast'; 10 | import { Lexer } from './lexer'; 11 | import { ChoiceFork } from './choiceFork'; 12 | import { Choice } from './weightedChoice'; 13 | import { isStr } from './stringUtils'; 14 | import { EvalDisabledError, IncludeError } from './errors'; 15 | import { EvalContext } from './evalBlock'; 16 | 17 | 18 | // If the referred fork is a silent set fork that has not yet been executed, 19 | // choiceIndex will be -1 and renderedOutput will be ''. 20 | export type ExecutedFork = { choiceFork: ChoiceFork, choiceIndex: number, renderedOutput: string } 21 | export type ExecutedForkMap = Map 22 | 23 | /** 24 | * A helper class for rendering an AST. 25 | * 26 | * This is meant to be used only once per render. 27 | */ 28 | export class Renderer { 29 | 30 | settings: RenderSettings; 31 | executedForkMap: ExecutedForkMap; 32 | evalContext: EvalContext; 33 | 34 | constructor(settings: RenderSettings) { 35 | this.settings = settings; 36 | this.executedForkMap = new Map(); 37 | this.evalContext = { bindings: {}, output: '', renderer: this }; 38 | } 39 | 40 | resolveReference(reference: Reference): string { 41 | let referredExecutedFork = this.executedForkMap.get(reference.id); 42 | if (referredExecutedFork) { 43 | if (reference.reExecute) { 44 | // this is a special "re-execute" reference 45 | let { replacement, choiceIndex } = referredExecutedFork.choiceFork.call(); 46 | let renderedOutput = this.renderChoice(replacement); 47 | referredExecutedFork.choiceIndex = choiceIndex; 48 | referredExecutedFork.renderedOutput = renderedOutput; 49 | return renderedOutput; 50 | } 51 | if (!reference.referenceMap.size && !reference.fallbackChoiceFork) { 52 | // this is a special "copy" reference 53 | return this.renderChoice([referredExecutedFork.renderedOutput]); 54 | } 55 | let matchedReferenceResult = reference.referenceMap.get(referredExecutedFork.choiceIndex); 56 | if (matchedReferenceResult !== undefined) { 57 | return this.renderChoice(matchedReferenceResult); 58 | } 59 | } 60 | if (!reference.fallbackChoiceFork) { 61 | console.warn(`No matching reference or fallback found for ${reference.id}`); 62 | return this.renderChoice(['']); 63 | } 64 | return this.renderChoice(reference.fallbackChoiceFork.call().replacement); 65 | } 66 | 67 | renderChoice(choice: Choice): string { 68 | if (choice instanceof Array) { 69 | return this.renderAst(choice); 70 | } else { 71 | if (!this.settings.allowEval) { 72 | throw new EvalDisabledError(); 73 | } 74 | choice.execute(this.evalContext); 75 | let output = this.evalContext.output; 76 | this.evalContext.output = ''; 77 | return output; 78 | } 79 | } 80 | 81 | renderAst(ast: AstNode[]): string { 82 | let output = ''; 83 | for (let i = 0; i < ast.length; i++) { 84 | let node = ast[i]; 85 | if (isStr(node)) { 86 | output += node; 87 | } else if (node instanceof ChoiceFork) { 88 | if (node.isSet && node.isSilent) { 89 | // Silent set declarations are *not* immediately executed. 90 | this.executedForkMap.set(node.identifier!, 91 | { choiceFork: node, choiceIndex: -1, renderedOutput: '' }); 92 | continue; 93 | } 94 | let { replacement, choiceIndex } = node.call(); 95 | let renderedOutput = this.renderChoice(replacement); 96 | if (node.isSilent) { 97 | if (output.length && output[output.length - 1] == '\n') { 98 | // This silent fork started a new line. 99 | // If the next character is also a newline, skip it. 100 | let nextNode = i < ast.length - 1 ? (ast[i + 1]) : null; 101 | if (isStr(nextNode)) { 102 | if (nextNode.startsWith('\n')) { 103 | ast[i + 1] = nextNode.slice(1); 104 | } else if (nextNode.startsWith('\r\n')) { 105 | ast[i + 1] = nextNode.slice(2); 106 | } 107 | } 108 | } 109 | } else { 110 | output += renderedOutput; 111 | } 112 | if (node.identifier) { 113 | this.executedForkMap.set(node.identifier, 114 | { choiceFork: node, choiceIndex, renderedOutput }) 115 | } 116 | } else { 117 | output += this.resolveReference(node) 118 | } 119 | } 120 | return output; 121 | } 122 | 123 | postprocess(text: string): string { 124 | let documentSettings = mergeSettings( 125 | defaultBMLSettings, this.evalContext.bindings.settings); 126 | let output = text; 127 | output = postprocessing.replaceVisualLineBreaks(output); 128 | if (documentSettings.punctuationCleanup) { 129 | output = postprocessing.punctuationCleanup(output); 130 | } 131 | if (documentSettings.capitalizationCleanup) { 132 | output = postprocessing.capitalizationCleanup(output); 133 | } 134 | if (documentSettings.whitespaceCleanup) { 135 | output = postprocessing.whitespaceCleanup(output); 136 | } 137 | if (documentSettings.indefiniteArticleCleanup) { 138 | output = postprocessing.correctIndefiniteArticles(output); 139 | } 140 | return output; 141 | } 142 | 143 | renderWithoutPostProcess(ast: AstNode[]): string { 144 | if (this.settings.randomSeed) { 145 | rand.setRandomSeed(this.settings.randomSeed); 146 | } 147 | return this.renderAst(ast); 148 | } 149 | 150 | renderAndPostProcess(ast: AstNode[]): string { 151 | return this.postprocess(this.renderWithoutPostProcess(ast)); 152 | } 153 | 154 | /* Load and render a given BML file path, 155 | * merging its eval context and fork map this renderer's. 156 | */ 157 | renderInclude(includePath: string): string { 158 | let rngState = rand.saveRngState(); 159 | let bmlDocumentString; 160 | let resolvedWorkingDir = this.settings.workingDir || process.cwd(); 161 | let resolvedIncludePath = path.resolve(resolvedWorkingDir, includePath); 162 | try { 163 | bmlDocumentString = '' + fs.readFileSync(resolvedIncludePath); 164 | } catch (e) { 165 | if (typeof window !== 'undefined') { 166 | throw new IncludeError(includePath, `Includes can't be used in browsers`); 167 | } 168 | if (e instanceof Error && e.message && e.message.includes('no such file or directory')) { 169 | throw new IncludeError(includePath, `File not found`); 170 | } 171 | throw e; 172 | } 173 | 174 | let lexer = new Lexer(bmlDocumentString); 175 | let ast = parseDocument(lexer, true); 176 | let subWorkingDir = path.dirname(resolvedIncludePath) 177 | let subSettings = { 178 | ...this.settings, 179 | workingDir: subWorkingDir, 180 | } 181 | let subRenderer = new Renderer(subSettings); 182 | let result = subRenderer.renderWithoutPostProcess(ast); 183 | // Merge state from subrenderer into this renderer 184 | // Any redefined references are silently overwritten; this is needed to support 185 | // repeated includes (caused by diamond-like include graphs) without using namespacing. 186 | for (let [key, value] of Object.entries(subRenderer.evalContext.bindings)) { 187 | this.evalContext.bindings[key] = value; 188 | } 189 | for (let [key, value] of subRenderer.executedForkMap) { 190 | this.executedForkMap.set(key, value); 191 | } 192 | rand.restoreRngState(rngState); 193 | return result; 194 | } 195 | } 196 | 197 | export function render(bmlDocumentString: string, 198 | renderSettings: RenderSettings | null): string { 199 | renderSettings = mergeSettings(defaultRenderSettings, renderSettings); 200 | let lexer = new Lexer(bmlDocumentString); 201 | let ast = parseDocument(lexer, true); 202 | return new Renderer(renderSettings).renderAndPostProcess(ast); 203 | } 204 | 205 | -------------------------------------------------------------------------------- /src/cli.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import process from 'process'; 3 | import path from 'path'; 4 | import fs from 'fs'; 5 | import { RenderSettings } from './settings'; 6 | import { analyze } from './analysis'; 7 | import { launchInteractive } from './interactive'; 8 | import * as fileUtils from './fileUtils'; 9 | 10 | const packageJson = require('../package.json'); 11 | // Seems this needs to use `require` to bundle correctly. No idea why. 12 | const bml = require('./bml.ts'); 13 | 14 | const SEED_RE = /^-?\d+$/; 15 | 16 | export const HELP_SWITCHES = ['-h', '--h', '-help', '--help']; 17 | export const VERSION_SWITCHES = ['-v', '--version']; 18 | export const SEED_SWITCHES = ['--seed']; 19 | export const NO_EVAL_SWITCHES = ['--no-eval']; 20 | export const ANALYZE_SWITCHES = ['--analyze']; 21 | export const INTERACTIVE_SWITCHES = ['-i', '--interactive']; 22 | export const ALL_SWITCHES = ([] as string[]).concat( 23 | HELP_SWITCHES, VERSION_SWITCHES, 24 | SEED_SWITCHES, NO_EVAL_SWITCHES, 25 | ANALYZE_SWITCHES, INTERACTIVE_SWITCHES); 26 | 27 | export type BMLArgs = { 28 | bmlSource: string, 29 | settings: RenderSettings, 30 | }; 31 | export type Action = { function: Function, args: any[] }; 32 | 33 | function readFile(path: string): string { 34 | try { 35 | return '' + fs.readFileSync(path); 36 | } catch { 37 | handleNonexistingPath(path); 38 | process.exit(1); 39 | } 40 | } 41 | 42 | export function executeFromStdin(settings: RenderSettings): BMLArgs { 43 | return { 44 | bmlSource: fileUtils.readStdin(), 45 | settings: settings, 46 | }; 47 | } 48 | 49 | export function executeFromPath(scriptPath: string, settings: RenderSettings): BMLArgs { 50 | return { 51 | bmlSource: readFile(scriptPath), 52 | settings: settings 53 | } 54 | } 55 | 56 | export function analyzeFromStdin(): string { 57 | return fileUtils.readStdin() 58 | } 59 | 60 | export function analyzeFromPath(path: string): string { 61 | return readFile(path); 62 | } 63 | 64 | export function handleNonexistingPath(path: string) { 65 | console.log(`Could not read from ${path}`); 66 | printHelp(); 67 | } 68 | 69 | 70 | export function printHelp() { 71 | console.log( 72 | ` 73 | Usage: bml [options] [path] 74 | 75 | Render a bml document read from stdin or a file, if given. 76 | Prints result to STDOUT. 77 | 78 | Options: 79 | 80 | Options which, if present, should be the only options given: 81 | 82 | ${HELP_SWITCHES} print this help and quit 83 | ${VERSION_SWITCHES} print the BML version number and quiet 84 | 85 | Other options: 86 | 87 | ${SEED_SWITCHES} INTEGER set the random seed for the bml render 88 | ${NO_EVAL_SWITCHES} disable Javascript evaluation 89 | ${ANALYZE_SWITCHES} analyze the document instead of executing 90 | ${INTERACTIVE_SWITCHES} run BML interactively 91 | 92 | Source Code at https://github.com/ajyoon/bml 93 | Report Bugs at https://github.com/ajyoon/bml/issues 94 | Copyright (c) 2017 Andrew Yoon, under the BSD 3-Clause license 95 | ` 96 | ); 97 | } 98 | 99 | // The way this function is just a passthrough really illustrates 100 | // why the higher-level-function approach of this module is awkward 101 | export function printHelpForError() { 102 | printHelp(); 103 | } 104 | 105 | export function printVersionInfo() { 106 | process.stdout.write(packageJson.version + '\n'); 107 | } 108 | 109 | export function argsContainAnyUnknownSwitches(args: string[]): boolean { 110 | let unknown_arg = args.find( 111 | (arg) => !SEED_RE.test(arg) && arg.startsWith('-') && !ALL_SWITCHES.includes(arg)); 112 | if (unknown_arg) { 113 | console.error(`Unknown argument ${unknown_arg}`); 114 | return true; 115 | } 116 | return false; 117 | } 118 | 119 | /** 120 | * Parse the given command line arguments and determine the action needed. 121 | * 122 | * @param {String[]} args - command line arguments, stripped of `node`, 123 | * if present, and the script name 124 | * @return {Object} of the form {function: Function, args: ...Any} 125 | */ 126 | export function determineAction(args: string[]): Action { 127 | let errorAction = { 128 | function: printHelpForError, 129 | args: [] 130 | }; 131 | 132 | if (argsContainAnyUnknownSwitches(args)) { 133 | return errorAction; 134 | } 135 | 136 | let expectSeed = false; 137 | 138 | let file = null; 139 | let noEval = false; 140 | let seed = null; 141 | let analyze = false; 142 | let interactive = false; 143 | 144 | for (let arg of args) { 145 | if (expectSeed) { 146 | if (SEED_RE.test(arg)) { 147 | seed = Number(arg); 148 | expectSeed = false; 149 | } else { 150 | console.error('Invalid seed: ' + arg); 151 | return errorAction; 152 | } 153 | } else if (HELP_SWITCHES.includes(arg)) { 154 | return { 155 | function: printHelp, 156 | args: [] 157 | }; 158 | } else if (VERSION_SWITCHES.includes(arg)) { 159 | return { 160 | function: printVersionInfo, 161 | args: [], 162 | }; 163 | } else if (NO_EVAL_SWITCHES.includes(arg)) { 164 | noEval = true; 165 | } else if (SEED_SWITCHES.includes(arg)) { 166 | expectSeed = true; 167 | } else if (ANALYZE_SWITCHES.includes(arg)) { 168 | analyze = true; 169 | } else if (INTERACTIVE_SWITCHES.includes(arg)) { 170 | interactive = true; 171 | } else { 172 | if (file !== null) { 173 | console.error('More than one path provided.'); 174 | return errorAction; 175 | } 176 | file = arg; 177 | } 178 | } 179 | 180 | if (expectSeed) { 181 | console.error('No seed provided.') 182 | return errorAction; 183 | } 184 | 185 | let settings = { 186 | randomSeed: seed, 187 | allowEval: !noEval, 188 | workingDir: file ? path.dirname(file) : null, 189 | }; 190 | 191 | if (interactive) { 192 | if (file === null) { 193 | console.error('Interactive mode requires a file name') 194 | return errorAction; 195 | } 196 | return { 197 | function: executeInteractively, 198 | args: [file, settings] 199 | }; 200 | } 201 | 202 | if (file === null) { 203 | if (analyze) { 204 | return { 205 | function: analyzeFromStdin, 206 | args: [] 207 | } 208 | } else { 209 | return { 210 | function: executeFromStdin, 211 | args: [settings] 212 | } 213 | } 214 | } else { 215 | if (analyze) { 216 | return { 217 | function: analyzeFromPath, 218 | args: [file] 219 | } 220 | } else { 221 | return { 222 | function: executeFromPath, 223 | args: [file, settings] 224 | } 225 | } 226 | } 227 | } 228 | 229 | 230 | export function stripArgs(argv: string[]): string[] { 231 | let sliceFrom = argv[0].indexOf('node') !== -1 ? 2 : 1; 232 | return argv.slice(sliceFrom); 233 | } 234 | 235 | 236 | export function runBmlWithErrorCheck(bmlSource: string, settings: RenderSettings): string { 237 | try { 238 | return bml(bmlSource, settings); 239 | } catch (e: any) { 240 | console.error('BML rendering failed with error:\n' 241 | + e.stack + '\n\n' 242 | + 'If you think this is a bug, please file one at ' + packageJson.bugs.url); 243 | process.exit(1); 244 | } 245 | } 246 | 247 | 248 | export function executeInteractively(path: string, settings: RenderSettings) { 249 | launchInteractive(path, settings); 250 | } 251 | 252 | function main() { 253 | let strippedArgs = stripArgs(process.argv); 254 | let action = determineAction(strippedArgs); 255 | 256 | if (action.function === printHelp || action.function === printVersionInfo) { 257 | action.function(); 258 | process.exit(0); 259 | } else if (action.function == printHelpForError) { 260 | action.function(); 261 | process.exit(1); 262 | } else if (action.function == executeInteractively) { 263 | action.function(...action.args); 264 | } else if (action.function == analyzeFromPath || action.function == analyzeFromStdin) { 265 | let bmlSource = action.function(...action.args); 266 | let analysisResult = analyze(bmlSource); 267 | let formattedCount = analysisResult.possibleOutcomes.toLocaleString(); 268 | process.stdout.write(`Approx possible branches: ${formattedCount}\n`); 269 | process.exit(0); 270 | } else { 271 | let { bmlSource, settings } = action.function(...action.args); 272 | let renderedContent = runBmlWithErrorCheck(bmlSource, settings); 273 | process.stdout.write(renderedContent); 274 | } 275 | } 276 | 277 | // Execute when run as main module 278 | if (!module.parent) { 279 | main(); 280 | } 281 | -------------------------------------------------------------------------------- /src/lexer.ts: -------------------------------------------------------------------------------- 1 | import { Token } from './token'; 2 | import { TokenType } from './tokenType'; 3 | 4 | const ANY_WHITESPACE_RE = /\s/; 5 | 6 | export class Lexer { 7 | 8 | str: string; 9 | index: number; 10 | private _cachedNext: Token | null = null; 11 | private _newLineRe: RegExp = /\r?\n/y; 12 | private _whitespaceRe: RegExp = /[^\S\r\n]+/y; // non-newline whitespace 13 | private _numberRe: RegExp = /(\d+(\.\d+)?)|(\.\d+)/y; 14 | 15 | constructor(str: string) { 16 | this.str = str; 17 | this.index = 0; 18 | } 19 | 20 | /** 21 | * Set this.index and invalidate the next-token cache 22 | */ 23 | overrideIndex(newIndex: number) { 24 | this._cachedNext = null; 25 | this.index = newIndex; 26 | } 27 | 28 | /** 29 | * Determine the next item in the token stream 30 | */ 31 | _determineNextRaw(): Token | null { 32 | if (this.index >= this.str.length) { 33 | return null; 34 | } 35 | let tokenType; 36 | let tokenIndex = this.index; 37 | let tokenEndIndex = null; 38 | let tokenString; 39 | this._newLineRe.lastIndex = this.index; 40 | this._whitespaceRe.lastIndex = this.index; 41 | this._numberRe.lastIndex = this.index; 42 | let newLineMatch = this._newLineRe.exec(this.str); 43 | let whitespaceMatch = this._whitespaceRe.exec(this.str); 44 | let numberMatch = this._numberRe.exec(this.str); 45 | if (newLineMatch !== null) { 46 | tokenType = TokenType.NEW_LINE; 47 | tokenString = newLineMatch[0]; 48 | } else if (whitespaceMatch !== null) { 49 | tokenType = TokenType.WHITESPACE; 50 | tokenString = whitespaceMatch[0]; 51 | } else if (numberMatch !== null) { 52 | tokenType = TokenType.NUMBER; 53 | tokenString = numberMatch[0]; 54 | } else if (this.str.slice(this.index, this.index + 2) === '//') { 55 | tokenType = TokenType.COMMENT; 56 | tokenString = '//'; 57 | } else if (this.str.slice(this.index, this.index + 2) === '/*') { 58 | tokenType = TokenType.OPEN_BLOCK_COMMENT; 59 | tokenString = '/*'; 60 | } else if (this.str.slice(this.index, this.index + 2) === '*/') { 61 | tokenType = TokenType.CLOSE_BLOCK_COMMENT; 62 | tokenString = '*/'; 63 | } else if (this.str[this.index] === '/') { 64 | tokenType = TokenType.SLASH; 65 | tokenString = '/'; 66 | } else if (this.str[this.index] === '\'') { 67 | tokenType = TokenType.SINGLE_QUOTE; 68 | tokenString = '\''; 69 | } else if (this.str[this.index] === '"') { 70 | tokenType = TokenType.DOUBLE_QUOTE; 71 | tokenString = '"'; 72 | } else if (this.str[this.index] === '`') { 73 | tokenType = TokenType.BACKTICK; 74 | tokenString = '`'; 75 | } else if (this.str[this.index] === '(') { 76 | tokenType = TokenType.OPEN_PAREN; 77 | tokenString = '('; 78 | } else if (this.str[this.index] === ')') { 79 | tokenType = TokenType.CLOSE_PAREN; 80 | tokenString = ')'; 81 | } else if (this.str[this.index] === '{') { 82 | tokenType = TokenType.OPEN_BRACE; 83 | tokenString = '{'; 84 | } else if (this.str[this.index] === '}') { 85 | tokenType = TokenType.CLOSE_BRACE; 86 | tokenString = '}'; 87 | } else if (this.str[this.index] === ',') { 88 | tokenType = TokenType.COMMA; 89 | tokenString = ','; 90 | } else if (this.str[this.index] === ':') { 91 | tokenType = TokenType.COLON; 92 | tokenString = ':'; 93 | } else if (this.str[this.index] === '@') { 94 | tokenType = TokenType.AT; 95 | tokenString = '@'; 96 | } else if (this.str[this.index] === '#') { 97 | tokenType = TokenType.HASH; 98 | tokenString = '#'; 99 | } else if (this.str[this.index] === '!') { 100 | tokenType = TokenType.BANG; 101 | tokenString = '!'; 102 | } else if (this.str[this.index] === '$') { 103 | tokenType = TokenType.DOLLAR; 104 | tokenString = '$'; 105 | } else if (this.str[this.index] === '[') { 106 | tokenType = TokenType.OPEN_BRACKET; 107 | tokenString = '['; 108 | } else if (this.str[this.index] === ']') { 109 | tokenType = TokenType.CLOSE_BRACKET; 110 | tokenString = ']'; 111 | } else if (this.str.slice(this.index, this.index + 2) === '->') { 112 | tokenType = TokenType.ARROW; 113 | tokenString = '->'; 114 | } else { 115 | tokenType = TokenType.TEXT; 116 | if (this.str[this.index] === '\\') { 117 | let nextChar = this.str[this.index + 1]; 118 | if ('\\/[{])'.includes(nextChar)) { 119 | tokenEndIndex = this.index + 2; 120 | tokenString = nextChar; 121 | } else if (nextChar === 'n') { 122 | tokenEndIndex = this.index + 2; 123 | tokenString = '\n'; 124 | } else if (nextChar === 't') { 125 | tokenEndIndex = this.index + 2; 126 | tokenString = '\t'; 127 | } else if (nextChar === 'r') { 128 | tokenEndIndex = this.index + 2; 129 | tokenString = '\r'; 130 | } else { 131 | tokenString = '\\'; 132 | } 133 | } else { 134 | tokenString = this.str[this.index]; 135 | } 136 | } 137 | 138 | if (tokenEndIndex === null) { 139 | tokenEndIndex = tokenIndex + tokenString.length; 140 | } 141 | let token = new Token(tokenType, tokenIndex, tokenEndIndex, tokenString); 142 | return token; 143 | } 144 | 145 | _determineNextReal(): Token | null { 146 | let inLineComment = false; 147 | let inBlockComment = false; 148 | let token; 149 | let startIndex = this.index; 150 | 151 | while ((token = this._determineNextRaw()) !== null) { 152 | if (inLineComment) { 153 | if (token.tokenType === TokenType.NEW_LINE) { 154 | inLineComment = false; 155 | return new Token(TokenType.NEW_LINE, token.index, token.endIndex, token.str); 156 | } 157 | } else if (inBlockComment) { 158 | if (token.tokenType === TokenType.CLOSE_BLOCK_COMMENT) { 159 | // Block comments output a single whitespace positioned at 160 | // the closing slash of the `*/` 161 | // TODO why???? isn't it more intuitive that they should emit nothing??? 162 | let virtualSpaceIdx = token.index + 1; 163 | return new Token(TokenType.WHITESPACE, virtualSpaceIdx, virtualSpaceIdx + 1, ' '); 164 | } 165 | } else { 166 | if (token.tokenType === TokenType.COMMENT) { 167 | // Use some hacky checks to work around lack of lookbehind 168 | // and elegant lookahead. 169 | let commentFollowedByWhitespace = token.index >= this.str.length 170 | || ANY_WHITESPACE_RE.test(this.str[token.endIndex]); 171 | let commentPrecededByWhitespace = token.index === 0 172 | || ANY_WHITESPACE_RE.test(this.str[token.index - 1]); 173 | if (commentPrecededByWhitespace || commentFollowedByWhitespace) { 174 | inLineComment = true; 175 | } else { 176 | // If line comment isn't preceded or followed by whitespace, 177 | // emit a TEXT token for it instead. 178 | return new Token(TokenType.TEXT, token.index, token.endIndex, token.str); 179 | } 180 | } else if (token.tokenType === TokenType.OPEN_BLOCK_COMMENT) { 181 | inBlockComment = true; 182 | } else { 183 | this.index = startIndex; 184 | return token; 185 | } 186 | } 187 | // Determining the next real token currently requires 188 | // fake-consuming tokens until a real one is found. It's a bad 189 | // hack, but `this.index` should be reset to the initial 190 | // position before this function exits. 191 | this.index = token.endIndex; 192 | } 193 | this.index = startIndex; 194 | return null; 195 | } 196 | 197 | next(): Token | null { 198 | let token; 199 | if (this._cachedNext !== null) { 200 | token = this._cachedNext; 201 | this._cachedNext = null; 202 | } else { 203 | token = this._determineNextReal(); 204 | } 205 | if (token !== null) { 206 | this.index = token.endIndex; 207 | } else { 208 | this.index = this.str.length; 209 | } 210 | return token; 211 | } 212 | 213 | peek(): Token | null { 214 | if (this._cachedNext !== null) { 215 | return this._cachedNext; 216 | } 217 | let token = this._determineNextReal(); 218 | this._cachedNext = token; 219 | return token; 220 | } 221 | 222 | nextSatisfying(predicate: (a: Token) => boolean): Token | null { 223 | let token; 224 | while ((token = this.next()) !== null) { 225 | if (predicate(token)) { 226 | return token; 227 | } 228 | } 229 | return null; 230 | } 231 | 232 | nextNonWhitespace(): Token | null { 233 | return this.nextSatisfying((t) => 234 | (t.tokenType !== TokenType.WHITESPACE && t.tokenType !== TokenType.NEW_LINE)); 235 | } 236 | } 237 | -------------------------------------------------------------------------------- /test/testLexer.ts: -------------------------------------------------------------------------------- 1 | import expect from 'expect'; 2 | 3 | import { Lexer } from '../src/lexer'; 4 | import { Token } from '../src/token'; 5 | import { TokenType } from '../src/tokenType'; 6 | 7 | 8 | describe('Lexer', function() { 9 | it('doesn\'t explode with an empty string', function() { 10 | let lexer = new Lexer(''); 11 | expect(lexer.next()).toBeNull(); 12 | }); 13 | 14 | it('tokenizes new lines', function() { 15 | let lexer = new Lexer('\n'); 16 | expect(lexer.next()).toEqual(new Token(TokenType.NEW_LINE, 0, 1, '\n')); 17 | expect(lexer.next()).toBeNull(); 18 | }); 19 | 20 | it('tokenizes CLRF new lines', function() { 21 | let lexer = new Lexer('\r\n'); 22 | expect(lexer.next()).toEqual(new Token(TokenType.NEW_LINE, 0, 2, '\r\n')); 23 | expect(lexer.next()).toBeNull(); 24 | }); 25 | 26 | it('tokenizes spaces and tabs as WHITESPACE', function() { 27 | let lexer = new Lexer(' '); 28 | expect(lexer.next()).toEqual(new Token(TokenType.WHITESPACE, 0, 1, ' ')); 29 | expect(lexer.next()).toBeNull(); 30 | 31 | lexer = new Lexer('\t'); 32 | expect(lexer.next()).toEqual(new Token(TokenType.WHITESPACE, 0, 1, '\t')); 33 | expect(lexer.next()).toBeNull(); 34 | }); 35 | 36 | it('tokenizes comments', function() { 37 | let lexer = new Lexer('//'); 38 | expect(lexer._determineNextRaw()).toEqual(new Token(TokenType.COMMENT, 0, 2, '//')); 39 | }); 40 | 41 | it('tokenizes block comment openings', function() { 42 | let lexer = new Lexer('/*'); 43 | expect(lexer._determineNextRaw()).toEqual(new Token(TokenType.OPEN_BLOCK_COMMENT, 0, 2, '/*')); 44 | }); 45 | 46 | it('tokenizes block comment closings', function() { 47 | let lexer = new Lexer('*/'); 48 | expect(lexer.next()).toEqual(new Token(TokenType.CLOSE_BLOCK_COMMENT, 0, 2, '*/')); 49 | expect(lexer.next()).toBeNull(); 50 | }); 51 | 52 | it('tokenizes slashes', function() { 53 | let lexer = new Lexer('/'); 54 | expect(lexer.next()).toEqual(new Token(TokenType.SLASH, 0, 1, '/')); 55 | expect(lexer.next()).toBeNull(); 56 | }); 57 | 58 | it('tokenizes single quotes', function() { 59 | let lexer = new Lexer('\''); 60 | expect(lexer.next()).toEqual(new Token(TokenType.SINGLE_QUOTE, 0, 1, '\'')); 61 | expect(lexer.next()).toBeNull(); 62 | }); 63 | 64 | it('tokenizes double quotes', function() { 65 | let lexer = new Lexer('"'); 66 | expect(lexer.next()).toEqual(new Token(TokenType.DOUBLE_QUOTE, 0, 1, '"')); 67 | expect(lexer.next()).toBeNull(); 68 | }); 69 | 70 | it('tokenizes backticks', function() { 71 | let lexer = new Lexer('`'); 72 | expect(lexer.next()).toEqual(new Token(TokenType.BACKTICK, 0, 1, '`')); 73 | expect(lexer.next()).toBeNull(); 74 | }); 75 | 76 | it('tokenizes open paren', function() { 77 | let lexer = new Lexer('('); 78 | expect(lexer.next()).toEqual(new Token(TokenType.OPEN_PAREN, 0, 1, '(')); 79 | expect(lexer.next()).toBeNull(); 80 | }); 81 | 82 | it('tokenizes close paren', function() { 83 | let lexer = new Lexer(')'); 84 | expect(lexer.next()).toEqual(new Token(TokenType.CLOSE_PAREN, 0, 1, ')')); 85 | expect(lexer.next()).toBeNull(); 86 | }); 87 | 88 | it('tokenizes open brace', function() { 89 | let lexer = new Lexer('{'); 90 | expect(lexer.next()).toEqual(new Token(TokenType.OPEN_BRACE, 0, 1, '{')); 91 | expect(lexer.next()).toBeNull(); 92 | }); 93 | 94 | it('tokenizes close brace', function() { 95 | let lexer = new Lexer('}'); 96 | expect(lexer.next()).toEqual(new Token(TokenType.CLOSE_BRACE, 0, 1, '}')); 97 | expect(lexer.next()).toBeNull(); 98 | }); 99 | 100 | it('tokenizes commas', function() { 101 | let lexer = new Lexer(','); 102 | expect(lexer.next()).toEqual(new Token(TokenType.COMMA, 0, 1, ',')); 103 | expect(lexer.next()).toBeNull(); 104 | }); 105 | 106 | it('tokenizes colons', function() { 107 | let lexer = new Lexer(':'); 108 | expect(lexer.next()).toEqual(new Token(TokenType.COLON, 0, 1, ':')); 109 | expect(lexer.next()).toBeNull(); 110 | }); 111 | 112 | it('tokenizes at', function() { 113 | let lexer = new Lexer('@'); 114 | expect(lexer.next()).toEqual(new Token(TokenType.AT, 0, 1, '@')); 115 | expect(lexer.next()).toBeNull(); 116 | }); 117 | 118 | it('tokenizes hash', function() { 119 | let lexer = new Lexer('#'); 120 | expect(lexer.next()).toEqual(new Token(TokenType.HASH, 0, 1, '#')); 121 | expect(lexer.next()).toBeNull(); 122 | }); 123 | 124 | it('tokenizes bang', function() { 125 | let lexer = new Lexer('!'); 126 | expect(lexer.next()).toEqual(new Token(TokenType.BANG, 0, 1, '!')); 127 | expect(lexer.next()).toBeNull(); 128 | }); 129 | 130 | it('tokenizes dollar', function() { 131 | let lexer = new Lexer('$'); 132 | expect(lexer.next()).toEqual(new Token(TokenType.DOLLAR, 0, 1, '$')); 133 | expect(lexer.next()).toBeNull(); 134 | }); 135 | 136 | it('tokenizes arrow', function() { 137 | let lexer = new Lexer('->'); 138 | expect(lexer.next()).toEqual(new Token(TokenType.ARROW, 0, 2, '->')); 139 | expect(lexer.next()).toBeNull(); 140 | }); 141 | 142 | it('tokenizes open bracket', function() { 143 | let lexer = new Lexer('['); 144 | expect(lexer.next()).toEqual(new Token(TokenType.OPEN_BRACKET, 0, 1, '[')); 145 | expect(lexer.next()).toBeNull(); 146 | }); 147 | 148 | it('tokenizes close bracket', function() { 149 | let lexer = new Lexer(']'); 150 | expect(lexer.next()).toEqual(new Token(TokenType.CLOSE_BRACKET, 0, 1, ']')); 151 | expect(lexer.next()).toBeNull(); 152 | }); 153 | 154 | it('tokenizes misc characters as individual text tokens', function() { 155 | let lexer = new Lexer('ab%'); 156 | expect(lexer.next()).toEqual(new Token(TokenType.TEXT, 0, 1, 'a')); 157 | expect(lexer.next()).toEqual(new Token(TokenType.TEXT, 1, 2, 'b')); 158 | expect(lexer.next()).toEqual(new Token(TokenType.TEXT, 2, 3, '%')); 159 | expect(lexer.next()).toBeNull(); 160 | }); 161 | 162 | it('tokenizes numbers', function() { 163 | let lexer = new Lexer('12345'); 164 | expect(lexer.next()).toEqual(new Token(TokenType.NUMBER, 0, 5, '12345')); 165 | expect(lexer.next()).toBeNull(); 166 | }); 167 | 168 | it('tokenizes numbers with decimal places', function() { 169 | let lexer = new Lexer('12345.67'); 170 | expect(lexer.next()).toEqual(new Token(TokenType.NUMBER, 0, 8, '12345.67')); 171 | expect(lexer.next()).toBeNull(); 172 | }); 173 | 174 | it('tokenizes numbers with a leading decimal', function() { 175 | let lexer = new Lexer('.67'); 176 | expect(lexer.next()).toEqual(new Token(TokenType.NUMBER, 0, 3, '.67')); 177 | expect(lexer.next()).toBeNull(); 178 | }); 179 | 180 | it('tokenizes escape sequences without consuming the following character', function() { 181 | let lexer = new Lexer('\\nP'); 182 | expect(lexer.next()).toEqual(new Token(TokenType.TEXT, 0, 2, '\n')); 183 | expect(lexer.next()).toEqual(new Token(TokenType.TEXT, 2, 3, 'P')); 184 | expect(lexer.next()).toBeNull(); 185 | }); 186 | 187 | it('tokenizes known escape sequences', function() { 188 | let lexer; 189 | 190 | lexer = new Lexer('\\\\'); 191 | expect(lexer.next()).toEqual(new Token(TokenType.TEXT, 0, 2, '\\')); 192 | expect(lexer.next()).toBeNull(); 193 | 194 | lexer = new Lexer('\\/'); 195 | expect(lexer.next()).toEqual(new Token(TokenType.TEXT, 0, 2, '/')); 196 | expect(lexer.next()).toBeNull(); 197 | 198 | lexer = new Lexer('\\['); 199 | expect(lexer.next()).toEqual(new Token(TokenType.TEXT, 0, 2, '[')); 200 | expect(lexer.next()).toBeNull(); 201 | 202 | lexer = new Lexer('\\{'); 203 | expect(lexer.next()).toEqual(new Token(TokenType.TEXT, 0, 2, '{')); 204 | expect(lexer.next()).toBeNull(); 205 | 206 | lexer = new Lexer('\\]'); 207 | expect(lexer.next()).toEqual(new Token(TokenType.TEXT, 0, 2, ']')); 208 | expect(lexer.next()).toBeNull(); 209 | 210 | lexer = new Lexer('\\)'); 211 | expect(lexer.next()).toEqual(new Token(TokenType.TEXT, 0, 2, ')')); 212 | expect(lexer.next()).toBeNull(); 213 | 214 | lexer = new Lexer('\\n'); 215 | expect(lexer.next()).toEqual(new Token(TokenType.TEXT, 0, 2, '\n')); 216 | expect(lexer.next()).toBeNull(); 217 | 218 | lexer = new Lexer('\\t'); 219 | expect(lexer.next()).toEqual(new Token(TokenType.TEXT, 0, 2, '\t')); 220 | expect(lexer.next()).toBeNull(); 221 | 222 | lexer = new Lexer('\\r'); 223 | expect(lexer.next()).toEqual(new Token(TokenType.TEXT, 0, 2, '\r')); 224 | expect(lexer.next()).toBeNull(); 225 | }); 226 | 227 | it('treats backslashes before unknown escape sequences as literal', function() { 228 | let lexer = new Lexer('\\f'); 229 | expect(lexer.next()).toEqual(new Token(TokenType.TEXT, 0, 1, '\\')); 230 | expect(lexer.next()).toEqual(new Token(TokenType.TEXT, 1, 2, 'f')); 231 | expect(lexer.next()).toBeNull(); 232 | }); 233 | 234 | it('can peek at the next token without consuming it', function() { 235 | let lexer = new Lexer('ab'); 236 | expect(lexer.peek()).toEqual(new Token(TokenType.TEXT, 0, 1, 'a')); 237 | expect(lexer.next()).toEqual(new Token(TokenType.TEXT, 0, 1, 'a')); 238 | expect(lexer.peek()).toEqual(new Token(TokenType.TEXT, 1, 2, 'b')); 239 | expect(lexer.next()).toEqual(new Token(TokenType.TEXT, 1, 2, 'b')); 240 | expect(lexer.peek()).toBeNull(); 241 | expect(lexer.next()).toBeNull(); 242 | }); 243 | 244 | it('line comments emit NEW_LINE', function() { 245 | let lexer = new Lexer('//foo\ntest'); 246 | expect(lexer.peek()).toEqual(new Token(TokenType.NEW_LINE, 5, 6, '\n')); 247 | expect(lexer.next()).toEqual(new Token(TokenType.NEW_LINE, 5, 6, '\n')); 248 | expect(lexer.peek()).toEqual(new Token(TokenType.TEXT, 6, 7, 't')); 249 | expect(lexer.next()).toEqual(new Token(TokenType.TEXT, 6, 7, 't')); 250 | }); 251 | 252 | it('converts block comments into a single white space', function() { 253 | let lexer = new Lexer('/* foo \n\n bar */test'); 254 | expect(lexer.peek()).toEqual(new Token(TokenType.WHITESPACE, 19, 20, ' ')); 255 | expect(lexer.next()).toEqual(new Token(TokenType.WHITESPACE, 19, 20, ' ')); 256 | expect(lexer.peek()).toEqual(new Token(TokenType.TEXT, 20, 21, 't')); 257 | expect(lexer.next()).toEqual(new Token(TokenType.TEXT, 20, 21, 't')); 258 | }); 259 | 260 | it('can skip to the next token satisfying a predicate', function() { 261 | let lexer = new Lexer('abc'); 262 | expect(lexer.nextSatisfying((t) => t.str === 'b')).toEqual(new Token(TokenType.TEXT, 1, 2, 'b')); 263 | }); 264 | 265 | it('can skip to the next non-whitespace (or comment) token', function() { 266 | let lexer = new Lexer(' \n\n test'); 267 | expect(lexer.nextNonWhitespace()).toEqual(new Token(TokenType.TEXT, 6, 7, 't')); 268 | }); 269 | }); 270 | -------------------------------------------------------------------------------- /test/testRenderer.ts: -------------------------------------------------------------------------------- 1 | import expect from "expect"; 2 | import * as tmp from "tmp"; 3 | import fs from "fs"; 4 | import path from "path"; 5 | 6 | import * as rand from "../src/rand"; 7 | import { EvalBindingError, EvalDisabledError } from "../src/errors"; 8 | import { render } from "../src/renderer"; 9 | 10 | describe("render", function () { 11 | let consoleWarnMock: any; 12 | 13 | beforeEach(function () { 14 | rand.setRandomSeed(0); // pin seed for reproducibility 15 | consoleWarnMock = jest.spyOn(console, "warn").mockImplementation(); 16 | }); 17 | 18 | afterEach(function () { 19 | consoleWarnMock.mockRestore(); 20 | }); 21 | 22 | it("executes simple forks", function () { 23 | let testString = "foo {(bar), (biz)}"; 24 | expect(render(testString, null)).toEqual("Foo bar\n"); 25 | }); 26 | 27 | it("executes forks with eval blocks", function () { 28 | let testString = 'foo {[insert("eval")], (biz)}'; 29 | expect(render(testString, null)).toEqual("Foo eval\n"); 30 | }); 31 | 32 | it("executes forks with weights", function () { 33 | let testString = 'foo {[insert("eval")], (biz) 99}'; 34 | expect(render(testString, null)).toEqual("Foo biz\n"); 35 | }); 36 | 37 | it("executes nested forks", function () { 38 | let testString = 39 | "foo {(bar {(nested branch), (other nested branch)}), (biz)}"; 40 | expect(render(testString, null)).toEqual("Foo bar other nested branch\n"); 41 | }); 42 | 43 | it("supports bare references", function () { 44 | let testString = "foo {id: (bar), (biz)} {@id}"; 45 | expect(render(testString, null)).toEqual("Foo bar bar\n"); 46 | }); 47 | 48 | it("supports silent fork references", function () { 49 | let testString = "foo {#id: (bar), (biz)} {@id}"; 50 | expect(render(testString, null)).toEqual("Foo bar\n"); 51 | }); 52 | 53 | it("emits nothing for lines containing only silent forks", function () { 54 | let testString = ` 55 | {[bind({settings: {whitespaceCleanup: false}})]} 56 | foo 57 | {#id: (bar), (biz)} 58 | {#id2: 59 | (baz), (buz) 60 | } 61 | biz 62 | `; 63 | expect(render(testString, null)).toEqual("\n\nFoo\nbiz\n"); 64 | }); 65 | 66 | it("supports mapped references", function () { 67 | let testString = "foo {id: (bar), (biz)} {@id: 0 -> (buzz), (bazz)}"; 68 | expect(render(testString, null)).toEqual("Foo bar buzz\n"); 69 | }); 70 | 71 | it("supports re-executing references", function () { 72 | let testString = 73 | "foo {id: (bar), (biz)} {@!id} {@!id} {@!id} {@!id} {@!id}"; 74 | expect(render(testString, null)).toEqual("Foo bar biz bar biz biz biz\n"); 75 | }); 76 | 77 | it("updates the executed fork map on re-executing references", function () { 78 | let testString = ` 79 | {id: (foo), (bar)} {@id: 0 -> (fff), 1 -> (bbb)} 80 | {@!id} {@id: 0 -> (fff), 1 -> (bbb)} 81 | {@!id} {@id: 0 -> (fff), 1 -> (bbb)} 82 | {@!id} {@id: 0 -> (fff), 1 -> (bbb)} 83 | `; 84 | expect(render(testString, null)).toEqual( 85 | "Foo fff\nbar bbb\nfoo fff\nbar bbb\n" 86 | ); 87 | }); 88 | 89 | it("supports initial set fork declarations", function () { 90 | let testString = "foo {$id: (bar), (biz)}"; 91 | expect(render(testString, null)).toEqual("Foo bar\n"); 92 | }); 93 | 94 | it("supports initial silent set fork declarations", function () { 95 | let testString = "foo {#$id: (bar), (biz)}"; 96 | expect(render(testString, null)).toEqual("Foo\n"); 97 | }); 98 | 99 | it("supports re-executing set forks", function () { 100 | let testString = "{$id: (A), (B), (C), (D)} {@!id} {@!id} {@!id}"; 101 | expect(render(testString, null)).toEqual("B D A C\n"); 102 | }); 103 | 104 | it("supports re-executing silent set forks", function () { 105 | // Note that silent set forks are *not* immediately executed, 106 | // so the initial declaration does not cause a set member to be exhausted 107 | let testString = "{#$id: (A), (B), (C), (D)} {@!id} {@!id} {@!id} {@!id}"; 108 | expect(render(testString, null)).toEqual(" B D A C\n"); 109 | }); 110 | 111 | it("resets weights on exhausted sets", function () { 112 | let testString = 113 | "{#$id: (A), (B), (C), (D)} {@!id} {@!id} {@!id} {@!id} {@!id}"; 114 | expect(render(testString, null)).toEqual(" B D A C C\n"); 115 | expect(consoleWarnMock).toBeCalled(); 116 | }); 117 | 118 | it("allows mapping on set forks", function () { 119 | let testString = "{#$id: (A), (B)} {@!id} {@!id} {@id: 0 -> (a), 1 -> (b)}"; 120 | expect(render(testString, null)).toEqual(" A B b\n"); 121 | }); 122 | 123 | it("gracefully warns when trying to map unexecuted silent set forks", function () { 124 | let testString = "{#$id: (A), (B), (C), (D)} {@id: 0 -> (foo)}"; 125 | expect(render(testString, null)).toEqual(""); 126 | expect(consoleWarnMock).toBeCalled(); 127 | }); 128 | 129 | it("preserves plaintext parentheses", function () { 130 | let testString = "foo (bar)"; 131 | expect(render(testString, null)).toEqual("Foo (bar)\n"); 132 | }); 133 | 134 | it("preserves plaintext parentheses in fork text", function () { 135 | let testString = "foo (bar {((biz))})"; 136 | expect(render(testString, null)).toEqual("Foo (bar (biz))\n"); 137 | }); 138 | 139 | it("skips line break after silent fork on its own line", function () { 140 | let testString = "foo\n{#id: (bar)}\nbiz"; 141 | expect(render(testString, null)).toEqual("Foo\nbiz\n"); 142 | }); 143 | 144 | it("errors on duplicate eval bindings", function () { 145 | let testString = ` 146 | {[bind({foo: 123})]} 147 | {[bind({foo: 456})]} 148 | `; 149 | expect(() => render(testString, null)).toThrow(EvalBindingError); 150 | }); 151 | 152 | it("Allows a value bound in one eval block to be accessed in others", function () { 153 | let testString = ` 154 | {[bind({foo: 123})]} 155 | {[insert(foo)]} 156 | `; 157 | expect(render(testString, null)).toEqual("123\n"); 158 | }); 159 | 160 | it("Allows a value bound in one eval block to be mutated in others", function () { 161 | let testString = ` 162 | {[bind({foo: 123})]} 163 | {[foo = 456;]} 164 | {[insert(foo)]} 165 | `; 166 | expect(render(testString, null)).toEqual("456\n"); 167 | }); 168 | 169 | it("Allows calling insert within bound functions", function () { 170 | let testString = ` 171 | {[ 172 | bind({ 173 | someValue: 'bar', 174 | myFunc: (value) => { 175 | insert(value); 176 | } 177 | }); 178 | ]} 179 | {[myFunc('foo')]} 180 | {[myFunc(someValue)]} 181 | `; 182 | expect(render(testString, null)).toEqual("Foo\nbar\n"); 183 | }); 184 | 185 | // Note that in-eval ref lookups are currently unstable 186 | it("Allows performing detailed ref lookups in eval blocks", function () { 187 | let testString = ` 188 | {foo: (bar), (biz)} 189 | {[ 190 | let forkResult = bml.refDetail('foo'); 191 | insert('foo got index ' + forkResult.choiceIndex + ': ' + forkResult.renderedOutput); 192 | ]} 193 | `; 194 | expect(render(testString, null)).toEqual("Bar\nfoo got index 0: bar\n"); 195 | }); 196 | 197 | it("Allows performing simple ref lookups in eval blocks", function () { 198 | let testString = ` 199 | {foo: (bar), (biz)} 200 | {[ 201 | let forkResult = bml.ref('foo'); 202 | insert('foo got ' + forkResult); 203 | ]} 204 | `; 205 | expect(render(testString, null)).toEqual("Bar\nfoo got bar\n"); 206 | }); 207 | 208 | it("Dynamically loads fork map in evals", function () { 209 | let testString = ` 210 | {#foo: (x)} 211 | {[ 212 | bind({ 213 | test: (id) => { 214 | insert(bml.ref(id)); 215 | } 216 | }); 217 | ]} 218 | {#foo: (a)} 219 | {[test('foo')]} 220 | `; 221 | expect(render(testString, null)).toEqual("A\n"); 222 | }); 223 | 224 | it("Allows disabling eval execution", function () { 225 | let testString = `{[insert('foo')]}`; 226 | expect(() => render(testString, { allowEval: false })).toThrow( 227 | EvalDisabledError 228 | ); 229 | }); 230 | 231 | function createTmpFile(contents: string): string { 232 | let path = tmp.fileSync().name; 233 | fs.writeFileSync(path, contents); 234 | return path; 235 | } 236 | 237 | it("Supports including a simple BML script", function () { 238 | let tmpScript = createTmpFile(`foo`); 239 | let tmpDir = path.dirname(tmpScript); 240 | let tmpFilename = path.basename(tmpScript); 241 | let testString = ` 242 | {[include('${tmpFilename}')]} 243 | `; 244 | 245 | expect(render(testString, { workingDir: tmpDir })).toEqual("Foo\n"); 246 | }); 247 | 248 | it("Retains eval bindings in includes", function () { 249 | let tmpScript = createTmpFile(` 250 | {[ 251 | bind({ 252 | test: 'foo' 253 | }); 254 | ]} 255 | `); 256 | // Note that included bindings are NOT available within the same eval block. 257 | // To access the included binding 'test', a new eval block must be opened. 258 | // This is an internal limitation which could be fixed if needed. 259 | let testString = ` 260 | {[include('${tmpScript}')]} 261 | {[insert(test)]} 262 | `; 263 | expect(render(testString, null)).toEqual("Foo\n"); 264 | }); 265 | 266 | // This documents a known issue: functions bound inside included evals 267 | // don't share the same eval context as the outer BML context, and so 268 | // things like calling `insert` inside a bound function or accessing the 269 | // ref map don't work as expected. I'm unsure what's the best way to handle this, 270 | // so for now I'm leaving the behavior in place and pinning it with this test. 271 | it("Does not support inserts in include-bound functions", function () { 272 | let tmpScript = createTmpFile(` 273 | {[ 274 | bind({ 275 | test: () => { insert('foo') } 276 | }); 277 | ]} 278 | `); 279 | // Note that included bindings are NOT available within the same eval block. 280 | // To access the included binding 'test', a new eval block must be opened. 281 | // This is an internal limitation which could be fixed if needed. 282 | let testString = ` 283 | {[include('${tmpScript}')]} 284 | {[test()]} 285 | `; 286 | expect(render(testString, null)).toEqual(""); 287 | }); 288 | 289 | it("Retains choice references in includes", function () { 290 | let tmpScript = createTmpFile(` 291 | {#foo: (x), (y)} 292 | `); 293 | let testString = ` 294 | {[include('${tmpScript}')]} 295 | {@foo} 296 | {@foo: 0 -> (bar), 1 -> (biz)} 297 | `; 298 | expect(render(testString, null)).toEqual("Y\nbiz\n"); 299 | }); 300 | 301 | it("Allows repeated includes", function () { 302 | let tmpScript = createTmpFile(` 303 | {#foo: (x), (y)} 304 | {[ 305 | bind({ 306 | test: () => { return 'bar'; } 307 | }); 308 | ]} 309 | `); 310 | let testString = ` 311 | {[include('${tmpScript}')]} 312 | {[include('${tmpScript}')]} 313 | `; 314 | expect(render(testString, null)).toEqual(""); 315 | }); 316 | 317 | it("Preserves RNG state around includes", function () { 318 | // I suspect this doesn't actually test RNG state restoration 319 | // but at least it pins some stability around the RNG behavior 320 | // before/during/after includes which contain rng use. 321 | let tmpScript = createTmpFile(`{foo: (x), (y)}`); 322 | let testString = ` 323 | {(a), (b), (c), (d), (e), (f), (g), (h), (i), (j), (k), (l), (m), (n), (o), (p)} 324 | {[include('${tmpScript}')]} 325 | {(a), (b), (c), (d), (e), (f), (g), (h), (i), (j), (k), (l), (m), (n), (o), (p)} 326 | `; 327 | expect(render(testString, null)).toEqual("E\nx\nd\n"); 328 | }); 329 | }); 330 | -------------------------------------------------------------------------------- /src/parsers.ts: -------------------------------------------------------------------------------- 1 | import { EvalBlock } from './evalBlock'; 2 | import { WeightedChoice } from './weightedChoice'; 3 | import { Lexer } from './lexer'; 4 | import { TokenType } from './tokenType'; 5 | import { ChoiceFork } from './choiceFork'; 6 | import { Reference } from './reference'; 7 | import { 8 | IllegalStateError, 9 | JavascriptSyntaxError, 10 | BMLSyntaxError, 11 | BMLDuplicatedRefIndexError, 12 | } from './errors'; 13 | import { AstNode } from './ast'; 14 | import { isStr } from './stringUtils'; 15 | 16 | 17 | /** 18 | * Parse an `eval` block 19 | * 20 | * @param lexer - A lexer whose next token is OPEN_BRACKET. This will be 21 | * mutated in place such that when the method returns, the lexer's 22 | * next token will be after the closing bracket of the block. 23 | * 24 | * @return The string of Javascript code extracted from the eval block 25 | * 26 | * @throws {JavascriptSyntaxError} If the javascript snippet inside the eval 27 | * block contains a syntax error which makes parsing it impossible. 28 | */ 29 | export function parseEval(lexer: Lexer): EvalBlock { 30 | if (lexer.next()?.tokenType !== TokenType.OPEN_BRACKET) { 31 | throw new IllegalStateError('parseEval started with non-OPEN_BRACKET'); 32 | } 33 | 34 | let state = 'code'; 35 | let index = lexer.index; 36 | let startIndex = index; 37 | let openBracketCount = 1; 38 | let token; 39 | while ((token = lexer.next()) !== null) { 40 | switch (state) { 41 | case 'block comment': 42 | if (token.tokenType === TokenType.CLOSE_BLOCK_COMMENT) { 43 | state = 'code'; 44 | } 45 | break; 46 | case 'inline comment': 47 | if (token.tokenType === TokenType.NEW_LINE) { 48 | state = 'code'; 49 | } 50 | break; 51 | case 'template literal': 52 | if (token.tokenType === TokenType.BACKTICK) { 53 | state = 'code'; 54 | } 55 | break; 56 | case 'single-quote string': 57 | if (token.tokenType === TokenType.SINGLE_QUOTE) { 58 | state = 'code'; 59 | } else if (token.tokenType === TokenType.NEW_LINE) { 60 | throw new JavascriptSyntaxError(lexer.str, lexer.index); 61 | } 62 | break; 63 | case 'double-quote string': 64 | if (token.tokenType === TokenType.DOUBLE_QUOTE) { 65 | state = 'code'; 66 | } else if (token.tokenType === TokenType.NEW_LINE) { 67 | throw new JavascriptSyntaxError(lexer.str, lexer.index); 68 | } 69 | break; 70 | case 'regexp literal': 71 | if (token.tokenType === TokenType.SLASH) { 72 | state = 'code'; 73 | } 74 | break; 75 | case 'code': 76 | switch (token.tokenType) { 77 | case TokenType.OPEN_BRACKET: 78 | openBracketCount++; 79 | break; 80 | case TokenType.CLOSE_BRACKET: 81 | openBracketCount--; 82 | if (openBracketCount < 1) { 83 | let source = lexer.str.slice(startIndex, lexer.index - 1); 84 | return new EvalBlock(source); 85 | } 86 | break; 87 | case TokenType.COMMENT: 88 | state = 'inline comment'; 89 | break; 90 | case TokenType.OPEN_BLOCK_COMMENT: 91 | state = 'block comment'; 92 | break; 93 | case TokenType.BACKTICK: 94 | state = 'template literal'; 95 | break; 96 | case TokenType.SINGLE_QUOTE: 97 | state = 'single-quote string'; 98 | break; 99 | case TokenType.DOUBLE_QUOTE: 100 | state = 'double-quote string'; 101 | break; 102 | case TokenType.SLASH: 103 | state = 'regexp literal'; 104 | break; 105 | default: 106 | // pass over.. 107 | } 108 | break; 109 | default: 110 | throw new Error(`Invalid state: ${state}`); 111 | } 112 | } 113 | throw new JavascriptSyntaxError('could not find end of javascript code block', 114 | startIndex); 115 | } 116 | 117 | /** 118 | * The main function for parsing {} blocks. 119 | * 120 | * Expects the lexer's previous token to be the opening curly brace, 121 | * and the next token whatever comes next. 122 | */ 123 | export function parseFork(lexer: Lexer): ChoiceFork | Reference { 124 | let startIndex = lexer.index; 125 | 126 | let mappedChoices = new Map(); 127 | let unmappedChoices: WeightedChoice[] = []; 128 | 129 | // Big blob in 2nd capture is for identifiers inclusive of non-ascii chars 130 | // It's an approximation of JS identifiers. 131 | let idRe = /(@|#|@!|\$|#\$|)([_a-zA-Z\xA0-\uFFFF][_a-zA-Z0-9\xA0-\uFFFF]*)(:?)/y; 132 | 133 | let id = null; 134 | let isReference = false; 135 | let isSilent = false; 136 | let isReExecuting = false; 137 | let isSet = false; 138 | 139 | let acceptId = true; 140 | let acceptWeight = false; 141 | let acceptChoiceIndex = false; 142 | let acceptArrow = false; 143 | let acceptReplacement = true; 144 | let acceptComma = false; 145 | let acceptBlockEnd = false; 146 | 147 | let currentChoiceIndexes = []; 148 | let currentReplacement = null; 149 | let token; 150 | 151 | while ((token = lexer.peek()) !== null) { 152 | switch (token.tokenType) { 153 | case TokenType.WHITESPACE: 154 | case TokenType.NEW_LINE: 155 | break; 156 | case TokenType.NUMBER: 157 | if (acceptChoiceIndex) { 158 | acceptChoiceIndex = false; 159 | acceptArrow = true; 160 | acceptComma = true; 161 | acceptBlockEnd = false; 162 | currentChoiceIndexes.push(Number(token.str)); 163 | } else if (acceptWeight) { 164 | acceptWeight = false; 165 | acceptComma = true; 166 | acceptBlockEnd = true; 167 | unmappedChoices[unmappedChoices.length - 1].weight = Number(token.str); 168 | } else { 169 | throw new BMLSyntaxError('Unexpected number in fork', 170 | lexer.str, token.index); 171 | } 172 | break; 173 | case TokenType.ARROW: 174 | if (acceptArrow) { 175 | acceptArrow = false; 176 | acceptReplacement = true; 177 | acceptComma = false; 178 | } else { 179 | throw new BMLSyntaxError('Unexpected arrow in fork', 180 | lexer.str, token.index); 181 | } 182 | break; 183 | case TokenType.OPEN_PAREN: 184 | case TokenType.OPEN_BRACKET: 185 | case TokenType.OPEN_BRACE: 186 | if (acceptReplacement) { 187 | acceptChoiceIndex = false; 188 | if (token.tokenType === TokenType.OPEN_PAREN) { 189 | lexer.next(); 190 | currentReplacement = parseDocument(lexer, false); 191 | } else if (token.tokenType === TokenType.OPEN_BRACKET) { 192 | currentReplacement = parseEval(lexer); 193 | } else { 194 | lexer.next(); 195 | currentReplacement = [parseFork(lexer)]; 196 | } 197 | if (currentChoiceIndexes.length) { 198 | for (let choiceIndex of currentChoiceIndexes) { 199 | if (mappedChoices.has(choiceIndex)) { 200 | // it's not ideal to validate this here, but with the way it's currently 201 | // built, if we don't it will just silently overwrite the key 202 | throw new BMLDuplicatedRefIndexError( 203 | id!, choiceIndex, lexer.str, token.index); 204 | } 205 | mappedChoices.set(choiceIndex, currentReplacement); 206 | } 207 | // Reset state for next choice 208 | acceptReplacement = false; 209 | acceptComma = true; 210 | acceptBlockEnd = true; 211 | currentChoiceIndexes = []; 212 | currentReplacement = null; 213 | } else { 214 | // Since there is no current choice index, this must be an unmapped choice 215 | unmappedChoices.push(new WeightedChoice(currentReplacement, null)); 216 | acceptReplacement = false; 217 | acceptComma = true; 218 | acceptWeight = true; 219 | acceptBlockEnd = true; 220 | } 221 | } else { 222 | throw new BMLSyntaxError('Unexpected replacement in fork', 223 | lexer.str, token.index); 224 | } 225 | continue; 226 | case TokenType.COMMA: 227 | if (acceptComma) { 228 | acceptComma = false; 229 | acceptChoiceIndex = true; 230 | if (!acceptArrow) { 231 | acceptReplacement = true; 232 | } 233 | } else { 234 | throw new BMLSyntaxError('Unexpected comma in fork', 235 | lexer.str, token.index); 236 | } 237 | break; 238 | case TokenType.CLOSE_BRACE: 239 | if (acceptBlockEnd) { 240 | lexer.next(); // consume close brace 241 | if (isReference) { 242 | return new Reference(id!, mappedChoices, unmappedChoices, isReExecuting); 243 | } else { 244 | return new ChoiceFork(unmappedChoices, id, isSilent, isSet) 245 | } 246 | } else { 247 | if (!mappedChoices.size && !unmappedChoices.length) { 248 | // Special case for a common mistake 249 | if (id) { 250 | throw new BMLSyntaxError(`Fork '${id}' has no branches`, 251 | lexer.str, token.index, `Did you mean '{@${id}}'?`); 252 | } else { 253 | throw new BMLSyntaxError('Fork has no branches', 254 | lexer.str, token.index); 255 | } 256 | } 257 | throw new BMLSyntaxError('Unexpected close brace in fork', 258 | lexer.str, token.index); 259 | } 260 | case TokenType.AT: 261 | case TokenType.HASH: 262 | case TokenType.DOLLAR: 263 | case TokenType.TEXT: 264 | if (acceptId) { 265 | idRe.lastIndex = lexer.index; 266 | let idMatch = idRe.exec(lexer.str); 267 | if (!idMatch) { 268 | throw new BMLSyntaxError(`Unexpected token ${token}`, 269 | lexer.str, token.index); 270 | } 271 | let typeSlug = idMatch[1]; 272 | id = idMatch[2]; 273 | let includesColon = !!idMatch[3]; 274 | if (typeSlug == '@') { 275 | isReference = true; 276 | if (includesColon) { 277 | acceptChoiceIndex = true; 278 | } else { 279 | acceptBlockEnd = true; 280 | } 281 | } else if (typeSlug == '#') { 282 | isSilent = true; 283 | } else if (typeSlug == '@!') { 284 | isReference = true; 285 | isReExecuting = true; 286 | if (includesColon) { 287 | throw new BMLSyntaxError(`Re-executing reference '${id}' should not have a colon, ` 288 | + 'since re-executing references cannot have mappings.', 289 | lexer.str, token.index, `Did you mean '{@!${id}}'?`) 290 | } else { 291 | acceptBlockEnd = true; 292 | } 293 | } else if (typeSlug == '$') { 294 | isSet = true; 295 | } else if (typeSlug == '#$') { 296 | isSet = true; 297 | isSilent = true; 298 | } 299 | lexer.overrideIndex(lexer.index + idMatch[0].length); 300 | acceptId = false; 301 | continue; 302 | } else { 303 | throw new BMLSyntaxError(`Unexpected token ${token}`, 304 | lexer.str, token.index); 305 | } 306 | default: 307 | throw new BMLSyntaxError(`Unexpected token ${token}`, 308 | lexer.str, token.index); 309 | } 310 | // If we haven't broken out or thrown an error by now, consume this token. 311 | lexer.next(); 312 | } 313 | throw new BMLSyntaxError('Could not find end of fork.', 314 | lexer.str, startIndex); 315 | } 316 | 317 | 318 | /** 319 | * Parse a literal block expressed with double-brackets 320 | * 321 | * Expects the lexer's next token to be the second opening bracket. 322 | * Upon returning, the lexer's next token is the one right after the final closing bracket. 323 | */ 324 | export function parseLiteralBlock(lexer: Lexer): string { 325 | let blockStartIndex = lexer.index - 1; 326 | if (lexer.next()?.tokenType !== TokenType.OPEN_BRACKET) { 327 | throw new IllegalStateError('parseLiteralBlock started with non-OPEN_BRACKET'); 328 | } 329 | let token; 330 | let result = ''; 331 | while ((token = lexer.next()) !== null) { 332 | switch (token.tokenType) { 333 | case TokenType.CLOSE_BRACKET: 334 | if (lexer.peek()?.tokenType == TokenType.CLOSE_BRACKET) { 335 | lexer.next(); 336 | return result; 337 | } 338 | default: 339 | result += token.str; 340 | } 341 | } 342 | throw new BMLSyntaxError('Could not find end of literal block', lexer.str, blockStartIndex); 343 | } 344 | 345 | /** 346 | * The top-level (or recursively called) parsing function. Returns an AST. 347 | * 348 | * If being recursively called, isTopLevel should be false and the 349 | * lexer's previous token should be an OPEN_PAREN. 350 | */ 351 | export function parseDocument(lexer: Lexer, isTopLevel: boolean): AstNode[] { 352 | let startIndex = lexer.index; 353 | let token; 354 | let openParenCount = 1; 355 | let astNodes: AstNode[] = []; 356 | 357 | function pushString(str: string) { 358 | // To keep the AST more compact, sequential string nodes are joined together. 359 | if (!astNodes.length) { 360 | astNodes.push(str); 361 | return; 362 | } 363 | let lastNode = astNodes[astNodes.length - 1]; 364 | if (isStr(lastNode)) { 365 | astNodes[astNodes.length - 1] = lastNode.concat(str); 366 | } else { 367 | astNodes.push(str); 368 | } 369 | } 370 | 371 | while ((token = lexer.next()) !== null) { 372 | switch (token.tokenType) { 373 | case TokenType.OPEN_PAREN: 374 | openParenCount++; 375 | pushString(token.str); 376 | break; 377 | case TokenType.CLOSE_PAREN: 378 | openParenCount--; 379 | if (openParenCount < 1) { 380 | return astNodes; 381 | } else { 382 | pushString(token.str); 383 | } 384 | break; 385 | case TokenType.OPEN_BRACKET: 386 | if (lexer.peek()?.tokenType == TokenType.OPEN_BRACKET) { 387 | pushString(parseLiteralBlock(lexer)); 388 | } else { 389 | pushString(token.str); 390 | } 391 | break; 392 | case TokenType.OPEN_BRACE: 393 | let fork = parseFork(lexer); 394 | astNodes = astNodes.concat(fork); 395 | break; 396 | default: 397 | // Any other input is treated as a string 398 | pushString(token.str); 399 | } 400 | } 401 | if (!isTopLevel) { 402 | throw new BMLSyntaxError(`Reached end of document while parsing string.`, 403 | lexer.str, startIndex) 404 | } 405 | return astNodes; 406 | } 407 | -------------------------------------------------------------------------------- /test/testParsers.ts: -------------------------------------------------------------------------------- 1 | import expect from 'expect'; 2 | import fs from 'fs'; 3 | import path from 'path' 4 | 5 | import { ChoiceFork } from '../src/choiceFork'; 6 | import { Lexer } from '../src/lexer'; 7 | import { WeightedChoice } from '../src/weightedChoice'; 8 | import { Reference } from '../src/reference'; 9 | import { AstNode } from '../src/ast'; 10 | 11 | import { 12 | JavascriptSyntaxError, 13 | BMLSyntaxError, 14 | BMLDuplicatedRefIndexError, 15 | } from '../src/errors'; 16 | 17 | import { 18 | parseEval, 19 | parseFork, 20 | parseLiteralBlock, 21 | parseDocument, 22 | } from '../src/parsers'; 23 | import { EvalBlock } from '../src/evalBlock'; 24 | 25 | 26 | function assertParseForkGivesSyntaxError(forkSrc: string) { 27 | expect(() => { 28 | parseFork(new Lexer(forkSrc)); 29 | }).toThrowError(BMLSyntaxError); 30 | } 31 | 32 | 33 | describe('parseEval', function() { 34 | it('can parse an empty block', function() { 35 | let testString = '[]'; 36 | let lexer = new Lexer(testString); 37 | let block = parseEval(lexer); 38 | expect(block.contents).toBe(''); 39 | expect(lexer.index).toBe(testString.length); 40 | }); 41 | 42 | it('should ignore brackets in line comments', function() { 43 | let testString = '[ //]\n]'; 44 | let lexer = new Lexer(testString); 45 | let block = parseEval(lexer); 46 | expect(block.contents).toBe(' //]\n'); 47 | expect(lexer.index).toBe(testString.length); 48 | }); 49 | 50 | it('should ignore brackets in block comments', function() { 51 | let testString = '[4/*\n]*/]'; 52 | let lexer = new Lexer(testString); 53 | let block = parseEval(lexer); 54 | expect(block.contents).toBe('4/*\n]*/'); 55 | expect(lexer.index).toBe(testString.length); 56 | }); 57 | 58 | it('should ignore brackets in single-quote string literals', function() { 59 | let testString = '[\']\']'; 60 | let lexer = new Lexer(testString); 61 | let block = parseEval(lexer); 62 | expect(block.contents).toBe('\']\''); 63 | expect(lexer.index).toBe(testString.length); 64 | }); 65 | 66 | it('should ignore brackets in double-quote string literals', function() { 67 | let testString = '["]"]'; 68 | let lexer = new Lexer(testString); 69 | let block = parseEval(lexer); 70 | expect(block.contents).toBe('"]"'); 71 | expect(lexer.index).toBe(testString.length); 72 | }); 73 | 74 | it('should error on newline before matching single-quote', function() { 75 | let testString = '[\'\n'; 76 | let lexer = new Lexer(testString); 77 | expect(() => parseEval(lexer)).toThrowError(JavascriptSyntaxError); 78 | }); 79 | 80 | it('should error on newline before matching double-quote', function() { 81 | let testString = '["\n'; 82 | let lexer = new Lexer(testString); 83 | expect(() => parseEval(lexer)).toThrowError(JavascriptSyntaxError); 84 | }); 85 | 86 | it('should ignore brackets in backtick string literals', function() { 87 | let testString = '[`\n]`]'; 88 | let lexer = new Lexer(testString); 89 | let block = parseEval(lexer); 90 | expect(block.contents).toBe('`\n]`'); 91 | expect(lexer.index).toBe(testString.length); 92 | }); 93 | 94 | it('should handle matching brackets in javascript', function() { 95 | let testString = '[[[]]]'; 96 | let lexer = new Lexer(testString); 97 | let block = parseEval(lexer); 98 | expect(block.contents).toBe('[[]]'); 99 | expect(lexer.index).toBe(testString.length); 100 | }); 101 | 102 | it('should be able to read itself (very meta)', function() { 103 | // Test the parser with a lot of real javascript (typescript, close enough) 104 | // by reading this file 105 | const parsersFilePath = require.resolve('../src/parsers.ts'); 106 | let parsersFileContents = ('' + fs.readFileSync(parsersFilePath)); 107 | let testString = '[' + parsersFileContents + ']'; 108 | let lexer = new Lexer(testString); 109 | let block = parseEval(lexer); 110 | expect(block.contents).toBe(parsersFileContents); 111 | expect(lexer.index).toBe(testString.length); 112 | }); 113 | }); 114 | 115 | 116 | describe('parseFork', function() { 117 | it('allows a single unweighted item', function() { 118 | let lexer = new Lexer('(test)}'); 119 | let result = parseFork(lexer); 120 | expect(result).toBeInstanceOf(ChoiceFork); 121 | }); 122 | 123 | it('allows a single weighted item', function() { 124 | let lexer = new Lexer('(test) 100}'); 125 | let result = parseFork(lexer); 126 | expect(result).toBeInstanceOf(ChoiceFork); 127 | }); 128 | 129 | it('allows a single unweighted eval block item', function() { 130 | let lexer = new Lexer('[some js]}'); 131 | let result = parseFork(lexer); 132 | expect(result).toBeInstanceOf(ChoiceFork); 133 | }); 134 | 135 | it('allows a single weighted eval block item', function() { 136 | let lexer = new Lexer('[some js] 100}'); 137 | let result = parseFork(lexer); 138 | expect(result).toBeInstanceOf(ChoiceFork); 139 | }); 140 | 141 | it('allows a comma separated mix of literals and eval blocks', function() { 142 | let lexer = new Lexer('(test) 50, [some js] 40}'); 143 | let result = parseFork(lexer); 144 | expect(result).toBeInstanceOf(ChoiceFork); 145 | }); 146 | 147 | it('allows trailing commas', function() { 148 | let lexer = new Lexer('(foo), (bar),}'); 149 | let result = parseFork(lexer); 150 | expect(result).toBeInstanceOf(ChoiceFork); 151 | }); 152 | 153 | it('allows identifiers for use in references', function() { 154 | let lexer = new Lexer('TestChoice: (test) 50, (test 2) 40}'); 155 | let result = parseFork(lexer); 156 | expect(result).toBeInstanceOf(ChoiceFork); 157 | expect((result as ChoiceFork).identifier).toBe('TestChoice'); 158 | }); 159 | 160 | it('allows identifiers using non-ascii characters', function() { 161 | // Some characters from Tao Te Ching 15, where I ran into this bug 162 | let lexer = new Lexer('微: (foo)}'); 163 | let result = parseFork(lexer); 164 | expect(result).toBeInstanceOf(ChoiceFork); 165 | expect((result as ChoiceFork).identifier).toBe('微'); 166 | lexer = new Lexer('微妙玄通: (foo)}'); 167 | result = parseFork(lexer); 168 | expect(result).toBeInstanceOf(ChoiceFork); 169 | expect((result as ChoiceFork).identifier).toBe('微妙玄通'); 170 | }); 171 | 172 | it('allows single-character identifiers', function() { 173 | let lexer = new Lexer('t: (test), (test 2)}'); 174 | let result = parseFork(lexer); 175 | expect(result).toBeInstanceOf(ChoiceFork); 176 | expect((result as ChoiceFork).identifier).toBe('t'); 177 | }); 178 | 179 | it('errors on a bare id with no branches', function() { 180 | assertParseForkGivesSyntaxError('TestChoice:}'); 181 | assertParseForkGivesSyntaxError('TestChoice}'); 182 | }); 183 | 184 | it('allows blocks with identifiers to be marked silent with # prefix', function() { 185 | let lexer = new Lexer('#TestChoice: (test)}'); 186 | let result = parseFork(lexer); 187 | expect(result).toBeInstanceOf(ChoiceFork); 188 | expect((result as ChoiceFork).identifier).toBe('TestChoice'); 189 | expect((result as ChoiceFork).isSilent).toBe(true); 190 | expect((result as ChoiceFork).isSet).toBe(false); 191 | }); 192 | 193 | it('allows non-silent sets', function() { 194 | let lexer = new Lexer('$TestChoice: (test)}'); 195 | let result = parseFork(lexer); 196 | expect(result).toBeInstanceOf(ChoiceFork); 197 | expect((result as ChoiceFork).identifier).toBe('TestChoice'); 198 | expect((result as ChoiceFork).isSilent).toBe(false); 199 | expect((result as ChoiceFork).isSet).toBe(true); 200 | }) 201 | 202 | it('allows silent sets', function() { 203 | let lexer = new Lexer('#$TestChoice: (test)}'); 204 | let result = parseFork(lexer); 205 | expect(result).toBeInstanceOf(ChoiceFork); 206 | expect((result as ChoiceFork).identifier).toBe('TestChoice'); 207 | expect((result as ChoiceFork).isSilent).toBe(true); 208 | expect((result as ChoiceFork).isSet).toBe(true); 209 | }) 210 | 211 | it('allows references', function() { 212 | let lexer = new Lexer('@TestChoice: 0 -> (foo)}'); 213 | let result = parseFork(lexer); 214 | let expectedChoiceMap = new Map(); 215 | expectedChoiceMap.set(0, ['foo']); 216 | expect(result).toBeInstanceOf(Reference); 217 | expect(result).toEqual(new Reference('TestChoice', expectedChoiceMap, [], false)); 218 | }); 219 | 220 | it('allows grouped mappings in references', function() { 221 | let lexer = new Lexer('@TestChoice: 0, 1 -> (foo), 2, 3 -> (bar), (baz)}'); 222 | let result = parseFork(lexer); 223 | let expectedChoiceMap = new Map(); 224 | expectedChoiceMap.set(0, ['foo']); 225 | expectedChoiceMap.set(1, ['foo']); 226 | expectedChoiceMap.set(2, ['bar']); 227 | expectedChoiceMap.set(3, ['bar']); 228 | expect(result).toBeInstanceOf(Reference); 229 | expect(result).toEqual(new Reference( 230 | 'TestChoice', expectedChoiceMap, [new WeightedChoice(['baz'], 100)], false)); 231 | }); 232 | 233 | it('allows multiple fallback branches in references', function() { 234 | let lexer = new Lexer('@TestChoice: 0 -> (foo), (bar) 60, (baz)}'); 235 | let result = parseFork(lexer); 236 | let expectedChoiceMap = new Map(); 237 | expectedChoiceMap.set(0, ['foo']); 238 | expect(result).toBeInstanceOf(Reference); 239 | let expectedWeights = [ 240 | new WeightedChoice(['bar'], 60), 241 | new WeightedChoice(['baz'], 40), 242 | ]; 243 | expect(result).toEqual(new Reference( 244 | 'TestChoice', expectedChoiceMap, expectedWeights, false)); 245 | }) 246 | 247 | it('allows forks to be used directly as branches', function() { 248 | let lexer = new Lexer('{(foo)} 60, (bar)}'); 249 | let result = parseFork(lexer) as ChoiceFork; 250 | let expectedResult = new ChoiceFork([ 251 | new WeightedChoice([ 252 | // Nested fork 253 | new ChoiceFork([ 254 | new WeightedChoice(['foo'], 100) 255 | ], null, false, false) 256 | ], 60), 257 | // Alternate branch in outer fork 258 | new WeightedChoice(['bar'], 40) 259 | ], null, false, false); 260 | expect(result).toEqual(expectedResult); 261 | }); 262 | }); 263 | 264 | describe('parseLiteralBlock', function() { 265 | it('parses a simple literal block', function() { 266 | let lexer = new Lexer('[some literal text]]'); 267 | let result = parseLiteralBlock(lexer); 268 | expect(result).toBe('some literal text'); 269 | }) 270 | 271 | it('parses a literal block containing a would-be fork', function() { 272 | let lexer = new Lexer('[some {(would-be fork)}]]'); 273 | let result = parseLiteralBlock(lexer); 274 | expect(result).toBe('some {(would-be fork)}'); 275 | }) 276 | 277 | it('allows escaping square brackets', function() { 278 | let lexer = new Lexer('[escaped \\]] brackets]]'); 279 | let result = parseLiteralBlock(lexer); 280 | expect(result).toBe('escaped ]] brackets'); 281 | }) 282 | }) 283 | 284 | 285 | describe('parseFork', function() { 286 | it('parses a choice fork with a single text branch', function() { 287 | let testString = '(test)}'; 288 | let lexer = new Lexer(testString); 289 | let result = parseFork(lexer) as ChoiceFork; 290 | expect(result.weights.length).toBe(1); 291 | expect(result.weights[0]).toBeInstanceOf(WeightedChoice); 292 | expect(result.weights[0].choice).toStrictEqual(['test']); 293 | expect(result.weights[0].weight).toBe(100); 294 | expect(lexer.index).toBe(testString.length); 295 | }); 296 | 297 | it('parses an eval block branch', function() { 298 | let testString = '[some js]}'; 299 | let lexer = new Lexer(testString); 300 | let result = parseFork(lexer) as ChoiceFork; 301 | expect(result.weights.length).toBe(1); 302 | expect(result.weights[0]).toBeInstanceOf(WeightedChoice); 303 | expect(result.weights[0].weight).toBe(100); 304 | expect(result.weights[0].choice).toBeInstanceOf(EvalBlock); 305 | let evalBlock = result.weights[0].choice as EvalBlock; 306 | expect(evalBlock.contents).toBe('some js'); 307 | }); 308 | 309 | it('parses strings with weights', function() { 310 | let testString = '(test) 5}'; 311 | let lexer = new Lexer(testString); 312 | let result = parseFork(lexer) as ChoiceFork; 313 | expect(result.weights.length).toBe(1); 314 | expect(result.weights[0]).toBeInstanceOf(WeightedChoice); 315 | expect(result.weights[0].choice).toStrictEqual(['test']); 316 | expect(result.weights[0].weight).toBe(5); 317 | expect(lexer.index).toBe(testString.length); 318 | }); 319 | 320 | it('parses eval block branch with weight', function() { 321 | let testString = '[some js] 5}'; 322 | let lexer = new Lexer(testString); 323 | let result = parseFork(lexer) as ChoiceFork; 324 | expect(result.weights.length).toBe(1); 325 | expect(result.weights[0]).toBeInstanceOf(WeightedChoice); 326 | expect(result.weights[0].weight).toBe(5); 327 | expect(result.weights[0].choice).toBeInstanceOf(EvalBlock); 328 | let evalBlock = result.weights[0].choice as EvalBlock; 329 | expect(evalBlock.contents).toBe('some js'); 330 | }); 331 | 332 | it('parses many branches with and without weights', function() { 333 | let testString = '[some js] 5, (test2), (test3) 3}'; 334 | let lexer = new Lexer(testString); 335 | let result = parseFork(lexer) as ChoiceFork; 336 | expect(result.weights.length).toBe(3); 337 | 338 | expect(result.weights[0]).toBeInstanceOf(WeightedChoice); 339 | expect(result.weights[0].weight).toBe(5); 340 | expect(result.weights[0].choice).toBeInstanceOf(EvalBlock); 341 | let evalBlock = result.weights[0].choice as EvalBlock; 342 | expect(evalBlock.contents).toBe('some js'); 343 | 344 | expect(result.weights[0]).toBeInstanceOf(WeightedChoice); 345 | expect(result.weights[0].weight).toBe(5); 346 | expect(result.weights[0].choice).toBeInstanceOf(EvalBlock); 347 | 348 | expect(result.weights[1]).toBeInstanceOf(WeightedChoice); 349 | expect(result.weights[1].choice).toStrictEqual(['test2']); 350 | expect(result.weights[1].weight).toBe(92); 351 | 352 | expect(result.weights[2]).toBeInstanceOf(WeightedChoice); 353 | expect(result.weights[2].choice).toStrictEqual(['test3']); 354 | expect(result.weights[2].weight).toBe(3); 355 | 356 | expect(lexer.index).toBe(testString.length); 357 | }); 358 | 359 | it('fails when two choices are not separated by a comma', function() { 360 | assertParseForkGivesSyntaxError('(test) (test 2)}'); 361 | }); 362 | 363 | xit('fails when an ID is not followed by a colon', function() { 364 | assertParseForkGivesSyntaxError('foo (test), (test 2)}'); 365 | }); 366 | }); 367 | 368 | describe('parseFork', function() { 369 | it('parses a simple case with a single string branch and no fallback', function() { 370 | let testString = '@TestRef: 0 -> (foo)}'; 371 | let lexer = new Lexer(testString); 372 | let result = parseFork(lexer)! as Reference; 373 | expect(result.id).toBe('TestRef'); 374 | expect(result.referenceMap.size).toBe(1); 375 | expect(result.referenceMap.get(0)).toStrictEqual(['foo']); 376 | }); 377 | 378 | it('parses a simple case with a single eval block branch and no fallback', function() { 379 | let testString = '@TestRef: 0 -> [some js]}'; 380 | let result = parseFork(new Lexer(testString)) as Reference; 381 | expect(result.id).toBe('TestRef'); 382 | expect(result.referenceMap.size).toBe(1); 383 | expect(result.referenceMap.get(0)).toBeInstanceOf(EvalBlock); 384 | expect((result.referenceMap.get(0) as EvalBlock).contents).toBe('some js'); 385 | expect(result.reExecute).toBeFalsy() 386 | }); 387 | 388 | it('allows a single branch with a fallback', function() { 389 | let testString = '@TestRef: 0 -> [some js], (fallback)}'; 390 | let result = parseFork(new Lexer(testString))! as Reference; 391 | expect(result.id).toBe('TestRef'); 392 | expect(result.referenceMap.size).toBe(1); 393 | expect(result.referenceMap.get(0)).toBeInstanceOf(EvalBlock); 394 | expect((result.referenceMap.get(0) as EvalBlock).contents).toBe('some js'); 395 | expect(result.fallbackChoiceFork!).not.toBeNull(); 396 | expect(result.fallbackChoiceFork!.weights).toHaveLength(1); 397 | let fallbackChoice = result.fallbackChoiceFork!.weights[0].choice as AstNode[]; 398 | expect(fallbackChoice).toStrictEqual(['fallback']); 399 | expect(result.reExecute).toBeFalsy() 400 | }); 401 | 402 | it('parses multiple branches of all types with fallback', function() { 403 | let testString = '@TestRef: 0 -> (foo), 1 -> [some js], 2 -> (bar), [some more js]}'; 404 | let result = parseFork(new Lexer(testString)) as Reference; 405 | 406 | expect(result.id).toBe('TestRef'); 407 | expect(result.referenceMap.size).toBe(3); 408 | expect(result.referenceMap.get(0)).toStrictEqual(['foo']); 409 | 410 | expect(result.referenceMap.get(1)).toBeInstanceOf(EvalBlock); 411 | expect((result.referenceMap.get(1) as EvalBlock).contents).toBe('some js'); 412 | 413 | expect(result.referenceMap.get(2)).toStrictEqual(['bar']); 414 | 415 | expect(result.fallbackChoiceFork!).not.toBeNull(); 416 | expect(result.fallbackChoiceFork!.weights).toHaveLength(1); 417 | let fallbackChoice = result.fallbackChoiceFork!.weights[0].choice as EvalBlock; 418 | expect(fallbackChoice).toBeInstanceOf(EvalBlock); 419 | expect(fallbackChoice.contents).toBe('some more js'); 420 | expect(result.reExecute).toBeFalsy() 421 | }); 422 | 423 | it('parses copy refs', function() { 424 | let testString = '@TestRef}'; 425 | let lexer = new Lexer(testString); 426 | let result = parseFork(lexer)! as Reference; 427 | expect(result).toBeInstanceOf(Reference); 428 | expect(result.id).toBe('TestRef'); 429 | expect(result.referenceMap.size).toBe(0); 430 | expect(result.fallbackChoiceFork).toBeNull(); 431 | expect(result.reExecute).toBeFalsy() 432 | }); 433 | 434 | it('parses re-executing refs', function() { 435 | let testString = '@!TestRef}'; 436 | let lexer = new Lexer(testString); 437 | let result = parseFork(lexer)! as Reference; 438 | expect(result).toBeInstanceOf(Reference); 439 | expect(result.id).toBe('TestRef'); 440 | expect(result.referenceMap.size).toBe(0); 441 | expect(result.fallbackChoiceFork).toBeNull(); 442 | expect(result.reExecute).toBeTruthy(); 443 | }); 444 | 445 | it('throws an error when invalid syntax is used', function() { 446 | assertParseForkGivesSyntaxError('@TestRef:}'); 447 | assertParseForkGivesSyntaxError('@TestRef: aaskfj}'); 448 | assertParseForkGivesSyntaxError('@TestRef: 0 - > (foo)}'); 449 | assertParseForkGivesSyntaxError('@TestRef: (foo) -> {foo}}'); 450 | assertParseForkGivesSyntaxError('@TestRef: 0 -> (foo),, 1 -> (bar)}'); 451 | assertParseForkGivesSyntaxError('@TestRef: 0,,2 -> (foo), 1 -> (bar)}'); 452 | assertParseForkGivesSyntaxError('@TestRef: 0, 1}'); 453 | assertParseForkGivesSyntaxError('@TestRef: 0 -> (foo), [some js], @TestRef2: 0 -> (foo)}'); 454 | assertParseForkGivesSyntaxError('@TestRef: (foo) 5, 2}'); 455 | assertParseForkGivesSyntaxError('@!TestRef:'); 456 | assertParseForkGivesSyntaxError('!@TestRef'); 457 | }); 458 | 459 | it('errors on repeated indexes', function() { 460 | expect(() => { 461 | parseFork(new Lexer('@TestRef: 0 -> (foo), 1 -> (bar), 0 -> (biz)')); 462 | }).toThrowError(BMLDuplicatedRefIndexError); 463 | }); 464 | }); 465 | 466 | describe('parseDocument', function() { 467 | it('can parse the kitchen sink test script', function() { 468 | const bmlScriptPath = path.resolve(__dirname, 'lao_tzu_36.bml'); 469 | const bmlScript = fs.readFileSync(bmlScriptPath).toString(); 470 | let lexer = new Lexer(bmlScript); 471 | parseDocument(lexer, true); 472 | }); 473 | 474 | it('includes whitespace in plain text', function() { 475 | let testString = 'testing 1 2\n3'; 476 | let lexer = new Lexer(testString); 477 | let result = parseDocument(lexer, true); 478 | expect(result).toHaveLength(1); 479 | expect(result[0]).toBe('testing 1 2\n3') 480 | }); 481 | 482 | it('preserves whitespace around forks', function() { 483 | let testString = 'foo {(bar)} biz'; 484 | let lexer = new Lexer(testString); 485 | let result = parseDocument(lexer, true); 486 | expect(result).toHaveLength(3); 487 | expect(result[0]).toBe('foo '); 488 | expect(result[1]).toBeInstanceOf(ChoiceFork); 489 | expect(result[2]).toBe(' biz'); 490 | }); 491 | 492 | it('provides literal blocks as plain strings', function() { 493 | let testString = 'foo [[{(bar)}]]'; 494 | let lexer = new Lexer(testString); 495 | let result = parseDocument(lexer, true); 496 | expect(result).toHaveLength(1); 497 | expect(result[0]).toBe('foo {(bar)}') 498 | }); 499 | }); 500 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ### 0.2.0 4 | 5 | - **BREAKING CHANGE**: The underlying pseudorandom number generator has been 6 | changed from the `seedrandom` default of `ARC4` to `xor4096`. The new 7 | generator is faster and has a significantly longer period of 2^4096-2^32. 8 | This is a breaking change because BML documents with fixed seeds will have 9 | different outputs with 0.2.0 as compared to earlier versions. No other 10 | changes are included in this release. 11 | 12 | ### 0.1.10 13 | 14 | - Improve interactive view error/warning capture 15 | - Update capitalization cleanup postprocessing to support sentences 16 | ending before quotation marks and other matching delimiters like brackets. 17 | For instance, `"Test." test` will now correct to `"Test." Test` where 18 | previously it would not. 19 | - Fix visual line breaks when backslash is followed by whitespace 20 | - Update punctuation cleanup postprocessing to shift punctuation to the 21 | right of quotes (single, double, or underscores). For example `"test" .` 22 | will now correct to `"test".`. Note that this does not shift punctuation 23 | to the inside of quotes, as this is context and style dependent. 24 | - Move working directory from top-level call argument to render settings. 25 | 26 | ### 0.1.9 27 | 28 | - Update experimental in-eval ref lookup. Now exposed by two 29 | functions, `bml.ref(id)` (returns rendered output of ref) and 30 | `bml.refDetail(id)` (returns ref output, index, and original fork 31 | object). The new functions should be lazily evaluated so they act on 32 | the fork map state at the time of function use, not function 33 | declaration. 34 | - In interactive mode, make spacebar also force refresh 35 | - Update punctuation cleanup to work around more script types (like Chinese) 36 | - Make interactive view support unicode 37 | - Fix interactive view confusingly suppressing error messages, for 38 | example when trying to read a non-existent file. 39 | - Provide better error messages when including non-existent files. 40 | - Fix bug where files included from included files were being done so 41 | relative to the working directory of the top-level script. Includes 42 | should now always be relative to the containing file. 43 | - Change include behavior so duplicate references and eval bindings 44 | are now allowed, and silently overwrite previous bindings. This is 45 | needed to support repeated includes (caused by diamond-like include 46 | graphs) without using namespacing. 47 | 48 | ### 0.1.8 49 | 50 | - Support set forks with `{$id: (foo), (bar)}` syntax. Silent set 51 | forks are also supported. 52 | 53 | ### 0.1.7 54 | 55 | - Fix bug introduced in 0.1.6 article correction 56 | - Fix bug in relative path includes on non-interactive execution. 57 | (Yet another bug caused by the very confusing way cli.ts is written.) 58 | - Make punctuation cleanup include dashes (hyphens, en-dash, and em-dash) 59 | - Support re-executing forks with the syntax `{@!foo}`, which is the 60 | equivalent of rewriting the original `{@foo}` fork: re-executing it, 61 | inserting its result, and updating the executed fork map 62 | accordingly. 63 | 64 | ### 0.1.6 65 | 66 | - Add and enable by default correction of English indefinite articles (a / an). 67 | This can be disabled using the new document setting `indefiniteArticleCleanup: false`. 68 | - Make static analysis handle unmapped refs more gracefully, since these are expected 69 | while includes are not analyzed. 70 | 71 | ### 0.1.5 72 | 73 | - Make `include` command paths relative to parent script's location 74 | - Fix bug with plaintext parentheses 75 | - _Experimentally_ expose fork results in eval blocks 76 | 77 | ### 0.1.4 78 | 79 | - Support including other BML scripts using the function `include` 80 | exposed in eval blocks. 81 | 82 | ### 0.1.3 83 | 84 | - Interactive view improvements 85 | - Support copying last render to clipboard 86 | - Display analysis statistics 87 | - Support non-ascii characters in fork ids 88 | 89 | ### 0.1.2 90 | 91 | - Add an interactive mode in the CLI with `--interactive`. 92 | 93 | ### 0.1.1 94 | 95 | - Add support for static branch-counting analysis. This can be done 96 | with the cli using `--analyze` and through the library with `bml.analyze` 97 | 98 | ### 0.1.0: MAJOR BREAKING CHANGE 99 | 100 | In this release, the language and parser have been largely rewritten, 101 | with substantial improvements and feature changes. The most commonly 102 | used features are unchanged, and many old programs should continue to 103 | work in this version. 104 | 105 | - Support for replacer rules and modes has been removed. This includes 106 | things like the `mode` and `use` keywords. Users who want this 107 | functionality should instead do this with a custom post-processor. 108 | - The document prelude section has been removed. 109 | - The language nomenclature has been changed to improve clarity: 110 | - "Fork" refers to any curly-braces `{}` block, including common 111 | replacers and references (formerly called "back references"). 112 | - "Branch" refers to any possible execution path a fork can go 113 | down. This includes text blocks, eval blocks, and nested forks. 114 | - "Reference" and "Ref" refer to what used to be called "back 115 | reference" blocks, e.g. `{@foo: 0 -> (bar)}` and `{@foo}`. 116 | - The labels that identify forks are now called "Ids" or "ForkIds". 117 | - Eval blocks have been completely overhauled 118 | - The `eval` block has been changed to an eval directive usable only 119 | as branches in forks, marked with single square brackets, for 120 | example `{[js code]}` 121 | - The `provide()` function has been replaced with a `bind()` 122 | function that accepts any object where its keys are valid 123 | javascript identifiers. Bound values are made available as local 124 | variables in the scope of subsequent eval blocks, and mutations to 125 | these values are persisted in the bound context. 126 | - Document settings are now set through this `bind()` function using 127 | the reserved key name `settings`. This can be bound anywhere, 128 | though it is recommended to do so at the top of the document in a 129 | bare choice block, e.g. `{[bind(settings: {...})]}` 130 | - The `call` keyword syntax has been replaced with eval blocks. To 131 | insert text from an eval block, use the new `insert()` function, 132 | e.g. `{[insert('foo')]}`. 133 | - Attempting to bind the same field more than once will result in an 134 | error. 135 | - Reference blocks can now include multiple fallback branches, 136 | including those with weights, which are grouped together and used to 137 | make a fallback fork. 138 | - For more convenient nesting, forks can be used directly themselves 139 | as branches. What used to be `{@foo: 0 -> ({(bar), (biz)})}` can now 140 | be written `{@foo: 0 -> {(bar), (biz)}}`, omitting the branch parentheses. 141 | - The interpreter has been divided cleanly into a separate parser and 142 | renderer, allowing proper static analysis. This will allow fairly 143 | straightforward branch counting in the future. 144 | - Comments should now properly be well-supported in all contexts 145 | 146 | ### 0.0.35: BREAKING CHANGE 147 | 148 | - Make line comments emit a single newline. This fixes the behavior of 149 | lines which end in line comments. like `foo // comment\n` 150 | - Require line comments to be preceded or followed by a whitespace or 151 | beginning/end of input. This is needed to allow writing things like 152 | URLs which use the `//` sequence. 153 | 154 | ### 0.0.34: BREAKING CHANGE 155 | 156 | - Remove built-in markdown support. This has long been an outlier 157 | feature that needlessly bloats the library bundle size for users 158 | which don't need it. Users needing markdown should now pull in a 159 | markdown rendering library themselves and manually run it on BML's 160 | output. This results in a web bundle size reduction of ~16kb, nearly 161 | a third. 162 | - To exactly replicate old behavior, use `marked@0.3.19` and 163 | manually plug in any markdown settings needed. 164 | - The CLI `--render-markdown` flag has been removed 165 | - The BML document setting `markdownSettings` has been removed 166 | - Change the signature for custom JS functions. The new signature is: 167 | 168 | ```ts 169 | (match: RegExpMatchArray | null, 170 | inlineCall: { input: string, index: number } | null) -> string 171 | ``` 172 | 173 | This corrects old awkwardness in the signatures by making it unclear 174 | from which context functions were being called. It also corrects an 175 | old redundancy where the `match` object was always a regexp match 176 | array, which already includes the input and match 177 | index. Furthermore, this provides a natural location for potential 178 | future arguments that could be applied to inline calls. 179 | 180 | - Make mode changes inside recursively rendered text bubble up. 181 | - Allow deactivating the active mode using `{use none}`. The mode name 182 | `none` is now reserved and BML will throw a `ModeNameError` if a 183 | document tries to shadow it. 184 | - Fix bug breaking regexp matchers ending with asterisks. 185 | - Fix bug where line comments ending with whitespace didn't terminate 186 | - Fix bugs with line comments ending with backslashes, escaped 187 | (literal) and not (visual line breaks.) 188 | 189 | ### 0.0.33: BREAKING CHANGE 190 | 191 | - Change the `as` keyword used in mode rules to the arrow `->` used in 192 | reference mappings. 193 | 194 | ### 0.0.32: 195 | 196 | - Add validations to eval-proided fields. 197 | - Fix several bugs around escape sequences, including escaped braces 198 | and square brackets. 199 | 200 | ### 0.0.31: 201 | 202 | - Improve whitespace cleanup by making it collapse runs of spaces in 203 | the middle of lines, for example `·foo···bar·` is cleaned to 204 | `·foo·bar\n` 205 | - Remove support for the long-deprecated `using` alias of the `use` 206 | keyword. 207 | - Move `whitespaceCleanup` setting from cli and `renderSettings` to 208 | document-defined `settings` provided through `eval`. 209 | - Add new post-processing step for correcting position of some 210 | punctuation marks according to English grammar rules. This is 211 | enabled by default and can be disabled using the document-defined 212 | setting `punctuationCleanup: false`. 213 | - Add a new post-processing step for correcting capitalization of the 214 | first words of sentences. This is enabled by default and can be 215 | disabled using the document-defined setting `capitalizationCleanup: 216 | false`. 217 | 218 | ### 0.0.30: 219 | 220 | - _Internal change_: the repo has been migrated to Typescript. All 221 | commands like `npm run build` and `npm run test` should still work 222 | just as before. 223 | - Fix bug where `UnknownModeError` incorrectly called itself a 224 | `JavascriptSyntaxError`. 225 | - Fix bug causing literal blocks to not be properly treated literally 226 | - Fix several small parser bugs unearthed by Typescript migration. 227 | - Add basic safety checks to eval blocks - logging warnings when 228 | `Math.random()` is used and when `provide()` is omitted. 229 | 230 | ### 0.0.28, 0.0.29: 231 | 232 | _skipped due to bad release from Typescript build headaches_ 233 | 234 | ### 0.0.27: 235 | 236 | - Fix bug causing comments to not be stripped out in many situations 237 | 238 | ### 0.0.26: BREAKING CHANGE 239 | 240 | - Line and block comments are now supported in plain body text, 241 | including text in choice branches. Line comments emit no output 242 | (including their terminating newlines), while block comments emit a 243 | single whitespace. 244 | 245 | ### 0.0.25: BREAKING CHANGE 246 | 247 | - Rule replacers must now be surrounded by curly braces 248 | 249 | Rule replacers are now defined using the same syntax as anonymous 250 | inline choices, harmonizing the syntax. 251 | 252 | ``` 253 | mode example { 254 | (foo) as (bar), (biz) 255 | // is now 256 | (foo) as {(bar), (biz)} 257 | } 258 | ``` 259 | 260 | - Rules no longer automatically insert an implicit no-op replacement 261 | branch. Users must now specify no-op replacement branches explicitly 262 | using the new `match` keyword. 263 | 264 | Where `(foo) as {(bar)}` used to be interpreted as "`foo` 50% of the 265 | time and `bar` 50% of the time," this code is now interpreted as 266 | "`bar` 100% of the time." To replicate the old behavior, use the new 267 | `match` explicitly like so: `(foo) as {(bar), match}`. 268 | 269 | ### 0.0.24: 270 | 271 | - No real library or language changes. This is a stub release to start 272 | uploading bundles to jsdelivr. 273 | 274 | ### 0.0.23: 275 | 276 | - Make all render settings available to the CLI 277 | - Make CLI errors from invalid arguments log more useful messages to 278 | stderr and give exit code 1. 279 | - Support trailing commas in inline choices 280 | - Support visual line breaks, marked by ending a line with a backslash. 281 | - Support grouping backrefs using `{@ref: 0, 1 -> (foo)}` syntax 282 | 283 | ### 0.0.22: 284 | 285 | - Remove accidentally-left-in debug log on regex matcher parsing 286 | - Fix bug preventing backref mappings pointing to empty strings from 287 | being matched. 288 | 289 | ### 0.0.21: BREAKING CHANGE 290 | 291 | - Fix bug introduced in 0.0.20 which prevented user-defined markdown 292 | settings from being passed to the markdown processor. 293 | - Change the syntax for regex matchers from `r(foo)` to `/foo/`. This 294 | is necessary because the old syntax used parens for its delimiter, 295 | which is a special character in regex, meaning it was impossible to 296 | match a regex like `/\)/`. This change also makes syntax 297 | highlighting simpler. 298 | 299 | ### 0.0.20: BREAKING CHANGE 300 | 301 | - Overhaul the `eval` system: 302 | - `eval` blocks are now executed with `new Function(...)` instead of 303 | raw (evil) eval. 304 | - User-defined functions and settings are now explicitly passed to 305 | the BML interpreter using the `provide` function. 306 | - The provided eval API (micro-stdlib) is now scoped to a `bml` 307 | namespace available in `eval` blocks. 308 | - Set a render recursion sanity limit of 1000 309 | 310 | ### 0.0.19: BREAKING CHANGE 311 | 312 | - Move `renderMarkdown` setting from BML document settings (defined in 313 | document `eval` blocks) to the `renderSettings` passed into the BML 314 | render call. To set this going forward, use `bml(src, 315 | {renderMarkdown: true, ...})`. 316 | - Support a new additional `renderSettings` field, 317 | `whitespaceCleanup`, which performs typically desirable whitespace 318 | cleanup after rendering. This new field is enabled by default. 319 | - Remove the top-level API field `defaultDocumentSettings` 320 | argument. This behavior is no longer supported. 321 | 322 | ### 0.0.18: BREAKING CHANGE 323 | 324 | - Fix markdown rendering bug by only rendering markdown at the topmost 325 | render pass. 326 | 327 | ### 0.0.17: BREAKING CHANGE 328 | 329 | - Fix bug causing the active mode to not be passed down into 330 | recursively rendered text. 331 | - Fix bug causing named choices executed inside recursively rendered 332 | text to not propagate upward. 333 | - Fix a CLI bug where passed-in seeds were interpreted as strings, not 334 | integers. This caused discrepancies between generated text with the 335 | same fixed seed when BML was invoked from the CLI vs the API. The 336 | CLI has been updated to require that seeds are integers and it now 337 | casts to integers as expected, aligning it with expected outputs as 338 | seen in the wild. 339 | 340 | This change breaks breaks fixed-seed reproducibility on the CLI. 341 | Texts generated with fixed seeds on the CLI prior to `0.0.17` will 342 | differ from newly reproducibly outputs. 343 | 344 | ### 0.0.16 345 | 346 | - Support copy back-references 347 | ```bml 348 | {Name: (Alice), (Bob)} {@Name} 349 | // results in "Alice Alice" or "Bob Bob" 350 | ``` 351 | - Support silent named choices 352 | ```bml 353 | silent {#Name: (Alice), (Bob)} then referenced {@Name} 354 | // results in "silent then referenced Alice" or "silent then referenced Bob" 355 | ``` 356 | - No longer log warnings when no bml version is present in settings. 357 | While this is probably a good idea, in practice it's pretty 358 | annoying. 359 | 360 | ### 0.0.15 361 | 362 | - Add experimental support for references and back-references 363 | ```bml 364 | {Name: (Alice), (Bob)} went to the store. 365 | {@Name: 0 -> (She), 1 -> (He)} bought some tofu. 366 | ``` 367 | 368 | ### 0.0.14: MAJOR BREAKING CHANGES 369 | 370 | - Replaces double-brace syntax with single braces. This affects inline 371 | choice blocks and mode switch blocks. Also replaces single/double 372 | quoted string syntax with parentheses so that all string literals 373 | outside `eval` blocks are now simply surrounded with 374 | parentheses. This helps simplify natural language (where quotation 375 | marks are commonly used) and allows the syntax for nested 376 | replacements to be much more elegant. | before | after | 377 | |------------------------------|------------------------------| | 378 | `{{'a' 10, 'b'}}` | `{(a) 10, (b)}` | | `{{use anotherMode}}` | 379 | `{use anotherMode}` | | `'foo' as 'bar' 5, call foo` | `(foo) as 380 | (bar) 5, call foo` | 381 | 382 | For migrating existing BML text, the following emacs regexps (in 383 | this order) have proven helpful: 384 | 385 | 1. `{{ -> {` 386 | 2. `}} -> }` 387 | 3. `'\([^{"\\]*?\)' -> (\1)` 388 | 4. `"\([^{\\]*?\)" -> (\1)` 389 | 390 | - Remove the `begin` statement; instead the first non-prelude-y text 391 | will cause the prelude to end. To start with an active mode, simply 392 | call the `{use someMode}` command. 393 | - Remove the `using` variant of the `use` keyword 394 | - Support recursive rendering within replacements, both in inline 395 | choices and in rule replacements. For instance: 396 | ```bml 397 | mode main { 398 | (recurse!) as (just kidding), (outer {(inner 1), (inner 2)}) 399 | } 400 | {use main} 401 | recurse! 402 | a {(simple inline), ({(complex), (fancy)} recursive)} inline choice 403 | ``` 404 | - Add new render setting (passed in `bml()` call) for `allowEval` 405 | which defualts to `true` and allows ignoring `eval` blocks, mostly 406 | for security purposes. 407 | - Add new `defaultDocumentSettings` argument to main `bml()` function 408 | which overrides the global default document settings before applying 409 | any settings defined in the document itself. 410 | 411 | ### 0.0.13 412 | 413 | - Expose `randomInt` and `randomFloat` to eval blocks. 414 | 415 | ### 0.0.12 416 | 417 | - Fix bug where random float generation was rounding results to integers, 418 | causing incorrect behavior when using floating-point or small values 419 | in replacement choice weights. 420 | - Add full support for random seed pinning for fully reproducible bml 421 | render artifacts 422 | - Document some of the API provided to `eval` blocks 423 | 424 | ### 0.0.11 425 | 426 | - Added experimental support for built-in javascript utils, 427 | starting with exposing `weightedChoose()` and `WeightedChoice`. 428 | - Fixed a bug causing version checks to emit a warning when 429 | a correct version was provided. 430 | 431 | ### 0.0.10 432 | 433 | - Changed `evaluate` keyword to `eval` 434 | - Added experimental support for syntax highlighting in browsers 435 | using a custom language definition in highlightjs 436 | 437 | ### 0.0.9 438 | 439 | - Added a command line interface and man page 440 | (requires a global install with `npm install -g bml`) 441 | 442 | ### 0.0.8 443 | 444 | - Support double quotes in inline replacement options. 445 | `hello {{"double" 60, 'single'}} quoted world!` 446 | - Support bml documents which do not have preludes. 447 | Note that this changes the default behavior around malformed preludes; 448 | while previously a missing prelude or a prelude whose ending cannot be 449 | found would trigger a `BMLSyntaxError`, the behavior now is to consider 450 | it to not be a prelude at all, but normal bml text. 451 | - Add `settings.version`: an optional setting to specify a bml version 452 | for a document. If the specified setting does not match the running 453 | bml version, a warning is logged. If no version number is specified, 454 | a warning is logged informing that unexpected behavior may occur. 455 | 456 | ### 0.0.7 457 | 458 | - Fix regression breaking regex matchers 459 | 460 | ### 0.0.6 461 | 462 | - Support double-quotes in addition to single quotes 463 | for all string literals 464 | - support escaping special tokens in regex matchers without 465 | needing a double-backslash. e.g. `r'\\\\s+' -> r'\\s+'` 466 | - a major internal refactor of all parsers increases long-term 467 | stability and flexibility of the language implementation. 468 | 469 | ### 0.0.5 470 | 471 | - Fix silly bug causing no-op options to never occur 472 | 473 | ### 0.0.4 474 | 475 | - Remove the explicit bml.renderBML function - to render a string of bml, 476 | simply call the package as a function. 477 | - Implement automatic no-op options in choice rules. 478 | Rules now have a default chance to not do anything. 479 | A no-op option is automatically inserted for all choice rules 480 | with a weight of `null`, to share an equal probability as all 481 | other options without explicit weights. 482 | - Fix bug in renderer causing halt after first match found 483 | - Add settings.markdownSettings. Allows users to specify settings 484 | to pass to marked.js at render time. 485 | 486 | ### 0.0.3 487 | 488 | - Add WeightedChoice class and use it in place of weight objects. 489 | `{option: ___, weight: ___} -> new WeightedChoice(choice, weight)` 490 | The new class has a `choice` property in place of an `option` one. 491 | - rename `rand.weightedChoice() -> rand.weightedChoose()` 492 | - Implement toString() methods in all classes 493 | - Fix rand.normalizeWeights and rand.weightedChoose not correctly calculating values. 494 | - Regex matchers can be specified with the character `r` immediately before 495 | the opening quote of a matcher string. Internally, strings not prepended with an `r` 496 | are still stored as regexps, but they are fully escaped. Regex flags cannot 497 | be set by users - they will always be the single sticky-y flag. 498 | For example, `r'.*'` gives the RegExp `/.*/y`, while `'.*'` gives 499 | the RegExp `/\.\*/y`. 500 | - Transform/replacer functions now take a RegExp match array. 501 | 502 | ### 0.0.2 503 | 504 | - [[Double square brackets]] are now used for marking blocks of literal text 505 | in order to prevent collisions with HTML in the previous << double angle bracket >> 506 | marker 507 | - Fixed a bug with backslash escapes inside literal blocks so literal double square 508 | brackets can be used [[inside blocks like this \\]] ]] 509 | --------------------------------------------------------------------------------