├── .gitignore ├── .gitmodules ├── .npmignore ├── .travis.yml ├── LICENSE ├── README.md ├── appveyor.yml ├── binding.gyp ├── cli.js ├── lib ├── api │ ├── binding.js │ ├── dsl.js │ ├── index.js │ └── properties.js └── cli │ ├── generate.js │ ├── helpers │ └── profile-command.js │ ├── parse.js │ ├── templates.js │ ├── templates │ ├── binding.cc.ejs │ ├── binding.gyp.ejs │ └── index.js.ejs │ └── test.js ├── package.json ├── src ├── binding.cc ├── generate.cc ├── generate.h ├── language.cc ├── language.h ├── rule_builder.cc └── rule_builder.h └── test ├── fixtures ├── arithmetic_language.js ├── external_scan.c ├── import.css └── schema.json ├── grammar_test.js ├── node_test.js ├── parser_test.js ├── properties_test.js └── tree_test.js /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.log 3 | lib-cov 4 | coverage 5 | .grunt 6 | build 7 | gyp-mac-tool 8 | tree_sitter 9 | *.xcodeproj 10 | node_modules 11 | .vagrant 12 | package-lock.json 13 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "vendor/tree-sitter"] 2 | path = vendor/tree-sitter 3 | url = http://github.com/maxbrunsfeld/tree-sitter.git 4 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | test 2 | 3 | out 4 | build 5 | *.xcodeproj 6 | 7 | vendor/tree-sitter/**/* 8 | !vendor/tree-sitter/src/compiler/**/* 9 | !vendor/tree-sitter/include/**/* 10 | !vendor/tree-sitter/externals/utf8proc/* 11 | !vendor/tree-sitter/externals/json-parser/* 12 | !vendor/tree-sitter/project.gyp 13 | 14 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: node_js 3 | compiler: clang 4 | node_js: 5 | - "8" 6 | 7 | before_install: 8 | - git submodule update --init --recursive 9 | 10 | env: 11 | - CXX=clang++ 12 | 13 | branches: 14 | only: 15 | - master 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 maxbrunsfeld 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # tree-sitter-cli 2 | 3 | Incremental parsers for node 4 | 5 | [![Build Status](https://travis-ci.org/tree-sitter/tree-sitter-cli.svg?branch=master)](https://travis-ci.org/tree-sitter/tree-sitter-cli) 6 | [![Build status](https://ci.appveyor.com/api/projects/status/t9775gnhcnsb5na1/branch/master?svg=true)](https://ci.appveyor.com/project/maxbrunsfeld/tree-sitter-cli/branch/master) 7 | 8 | --- 9 | 10 | :warning: This repository is deprecated. :warning: 11 | 12 | The source code for the Tree-sitter CLI is now part of [the main Tree-sitter repository](https://github.com/tree-sitter/tree-sitter). 13 | 14 | --- 15 | 16 | ### Installation 17 | 18 | ``` 19 | npm install tree-sitter-cli 20 | ``` 21 | 22 | ### Creating a language 23 | 24 | Create a `grammar.js` in the root directory of your module. This file 25 | should create and export a grammar object using tree-sitter's helper functions: 26 | 27 | ```js 28 | module.exports = grammar({ 29 | name: "arithmetic", 30 | 31 | extras: $ => [$.comment, /\s/], 32 | 33 | rules: { 34 | program: $ => repeat(choice( 35 | $.assignment_statement, 36 | $.expression_statement 37 | )), 38 | 39 | assignment_statement: $ => seq( 40 | $.variable, "=", $.expression, ";" 41 | ), 42 | 43 | expression_statement: $ => seq( 44 | $.expression, ";" 45 | ), 46 | 47 | expression: $ => choice( 48 | $.variable, 49 | $.number, 50 | prec.left(1, seq($.expression, "+", $.expression)), 51 | prec.left(1, seq($.expression, "-", $.expression)), 52 | prec.left(2, seq($.expression, "*", $.expression)), 53 | prec.left(2, seq($.expression, "/", $.expression)), 54 | prec.left(3, seq($.expression, "^", $.expression)) 55 | ), 56 | 57 | variable: $ => /\a\w*/, 58 | 59 | number: $ => /\d+/, 60 | 61 | comment: $ => /#.*/ 62 | } 63 | }); 64 | ``` 65 | 66 | Run `tree-sitter generate`. This will generate a C function for parsing your 67 | language, a C++ function that exposes the parser to javascript, and a 68 | `binding.gyp` file for compiling these sources into a native node module. 69 | 70 | #### Grammar syntax 71 | 72 | The `grammar` function takes an object with the following keys: 73 | 74 | * `name` - the name of the language. 75 | * `rules` - an object whose keys are rule names and whose values are Grammar Rules. The first key in the map will be the start symbol. See the 'Rules' section for how to construct grammar rules. 76 | * `extras` - an array of Grammar Rules which may appear anywhere in a document. This construct is used to useful for things like whitespace and comments in programming languages. 77 | * `conflicts` - an array of arrays of Grammar Rules which are known to conflict with each other in [an LR(1) parser](https://en.wikipedia.org/wiki/Canonical_LR_parser). You'll need to use this if writing a grammar that is not LR(1). 78 | 79 | #### Rules 80 | 81 | * `$.property` references - match another rule with the given name. 82 | * `String` literals - match exact strings. 83 | * `RegExp` literals - match strings according to ECMAScript regexp syntax. 84 | Assertions (e.g. `^`, `$`) are not yet supported. 85 | * `choice(rule...)` - matches any one of the given rules. 86 | * `repeat(rule)` - matches any number of repetitions of the given rule. 87 | * `seq(rule...)` - matches each of the given rules in sequence. 88 | * `blank()` - matches the empty string. 89 | * `optional(rule)` - matches the given rule or the empty string. 90 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | environment: 2 | nodejs_version: "8" 3 | 4 | platform: 5 | - x64 6 | - x86 7 | 8 | install: 9 | - call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x86 10 | - ps: Install-Product node $env:nodejs_version 11 | - git submodule update --init --recursive 12 | - npm install 13 | 14 | test_script: 15 | - npm test 16 | 17 | build: off 18 | 19 | branches: 20 | only: 21 | - master 22 | -------------------------------------------------------------------------------- /binding.gyp: -------------------------------------------------------------------------------- 1 | { 2 | "targets": [ 3 | { 4 | "target_name": "tree_sitter_cli_binding", 5 | "dependencies": [ 6 | "vendor/tree-sitter/project.gyp:compiler", 7 | ], 8 | "include_dirs": [ 9 | " { 52 | const nodeGyp = require.resolve('node-gyp/bin/node-gyp.js'); 53 | if (!fs.existsSync(path.join('build', 'config.gypi'))) { 54 | execFileSync(nodeGyp, ['configure'], {stdio: 'inherit'}); 55 | } 56 | execFileSync(nodeGyp, ['build', ...args], {stdio: 'inherit'}); 57 | process.exit(0) 58 | }); 59 | break; 60 | 61 | case "test": 62 | if (needsHelp) 63 | usage("test", [ 64 | "Run the tests for the parser in the current working directory.", 65 | "", 66 | "Arguments", 67 | " --focus, -f - Only run tests whose name contain the given string", 68 | " --debug, -d - Output debugging information during parsing" 69 | ]); 70 | 71 | require("./lib/cli/test")({ 72 | focus: argv.focus || argv.f, 73 | debug: argv.debug || argv.d, 74 | debugGraph: argv['debug-graph'] || argv.D 75 | }, process.exit); 76 | break; 77 | 78 | case "parse": 79 | const codePaths = argv._; 80 | if (needsHelp || codePaths.length === 0) 81 | usage("parse ", [ 82 | "Parse the given file using the parser in the current working directory and print the syntax tree.", 83 | "", 84 | "Arguments", 85 | " code-path - The file to parse", 86 | " --quiet, -q - Parse, but don't print any output", 87 | " --time, -t - Print the time it took to parse", 88 | " --debug, -d - Print a log of parse actions", 89 | " --debug-graph, -D - Render a sequence of diagrams showing the changing parse stack", 90 | " --profile, -P - Render a flame graph of the parse performance (requires sudo)", 91 | " --heap, -H - Report heap allocation breakdown (requires google perf tools)", 92 | " --repeat - Parse the file the given number of times (useful for profiling)", 93 | " --edit - Reparse the file after performing the given edit.", 94 | " For example, pass '5,3,\"x\"' to delete three characters and", 95 | " insert an 'x' at index 5. You can repeat this flag multiple times", 96 | " to perform a series of edits." 97 | ]); 98 | 99 | require("./lib/cli/parse")({ 100 | codePaths: codePaths, 101 | debugGraph: argv['debug-graph'] || argv.D, 102 | debug: argv.debug || argv.d, 103 | quiet: argv.quiet || argv.q, 104 | time: argv.time || argv.t, 105 | profile: argv.profile || argv.P, 106 | heapProfile: argv.heap || argv.H, 107 | repeat: argv.repeat, 108 | edits: argv.edit, 109 | properties: argv.properties 110 | }, process.exit); 111 | break; 112 | 113 | default: 114 | usage(" [flags]", [ 115 | "Run `tree-sitter --help` for more information about a particular command." 116 | ]) 117 | break; 118 | } 119 | 120 | function usage(command, lines) { 121 | console.log("Usage: tree-sitter " + command + "\n"); 122 | if (lines) 123 | console.log(lines.join('\n') + '\n') 124 | process.exit(0); 125 | } 126 | -------------------------------------------------------------------------------- /lib/api/binding.js: -------------------------------------------------------------------------------- 1 | module.exports = require("../../build/Release/tree_sitter_cli_binding"); 2 | -------------------------------------------------------------------------------- /lib/api/dsl.js: -------------------------------------------------------------------------------- 1 | const {RuleBuilder} = require("./binding"); 2 | const UNICODE_ESCAPE_PATTERN = /\\u([0-9a-f]{4})/gi; 3 | 4 | function blank() { 5 | return { 6 | type: "BLANK" 7 | }; 8 | } 9 | 10 | function choice(...elements) { 11 | return { 12 | type: "CHOICE", 13 | members: elements.map(normalize) 14 | }; 15 | } 16 | 17 | function err(content) { 18 | return { 19 | type: "ERROR", 20 | content: content 21 | }; 22 | } 23 | 24 | function pattern(value) { 25 | return { 26 | type: "PATTERN", 27 | value: value 28 | }; 29 | } 30 | 31 | function prec(number, rule) { 32 | if (rule == null) { 33 | rule = number; 34 | number = 0; 35 | } 36 | 37 | return { 38 | type: "PREC", 39 | value: number, 40 | content: normalize(rule) 41 | }; 42 | } 43 | 44 | prec.left = function(number, rule) { 45 | if (rule == null) { 46 | rule = number; 47 | number = 0; 48 | } 49 | 50 | return { 51 | type: "PREC_LEFT", 52 | value: number, 53 | content: normalize(rule) 54 | }; 55 | } 56 | 57 | prec.right = function(number, rule) { 58 | if (rule == null) { 59 | rule = number; 60 | number = 0; 61 | } 62 | 63 | return { 64 | type: "PREC_RIGHT", 65 | value: number, 66 | content: normalize(rule) 67 | }; 68 | } 69 | 70 | prec.dynamic = function(number, rule) { 71 | return { 72 | type: "PREC_DYNAMIC", 73 | value: number, 74 | content: normalize(rule) 75 | }; 76 | } 77 | 78 | function alias(rule, value) { 79 | const result = { 80 | type: "ALIAS", 81 | content: normalize(rule) 82 | }; 83 | 84 | switch (value.constructor) { 85 | case String: 86 | result.named = false; 87 | result.value = value; 88 | return result; 89 | case ReferenceError: 90 | result.named = true; 91 | result.value = value.symbol.name; 92 | return result; 93 | case Object: 94 | if (typeof value.type === 'string' && value.type === 'SYMBOL') { 95 | result.named = true; 96 | result.value = value.name; 97 | return result; 98 | } 99 | } 100 | 101 | throw new Error('Invalid alias value ' + value); 102 | } 103 | 104 | function repeat(rule) { 105 | return { 106 | type: "REPEAT", 107 | content: normalize(rule) 108 | }; 109 | } 110 | 111 | function repeat1(rule) { 112 | return { 113 | type: "REPEAT1", 114 | content: normalize(rule) 115 | }; 116 | } 117 | 118 | function seq(...elements) { 119 | return { 120 | type: "SEQ", 121 | members: elements.map(normalize) 122 | }; 123 | } 124 | 125 | function string(value) { 126 | return { 127 | type: "STRING", 128 | value: value 129 | }; 130 | } 131 | 132 | function sym(name) { 133 | return { 134 | type: "SYMBOL", 135 | name: name 136 | }; 137 | } 138 | 139 | function token(value) { 140 | return { 141 | type: "TOKEN", 142 | content: normalize(value) 143 | }; 144 | } 145 | 146 | token.immediate = function(value) { 147 | return { 148 | type: "IMMEDIATE_TOKEN", 149 | content: normalize(value) 150 | }; 151 | } 152 | 153 | function optional(value) { 154 | return choice(value, blank()); 155 | } 156 | 157 | function normalize(value) { 158 | if (typeof value == "undefined") 159 | throw new Error("Undefined symbol"); 160 | 161 | switch (value.constructor) { 162 | case String: 163 | return string(value); 164 | case RegExp: 165 | return pattern(value.source.replace( 166 | UNICODE_ESCAPE_PATTERN, 167 | function(match, group) { 168 | return String.fromCharCode(parseInt(group, 16)); 169 | } 170 | )); 171 | case ReferenceError: 172 | throw value 173 | default: 174 | if (typeof value.type === 'string') { 175 | return value; 176 | } else { 177 | throw new TypeError("Invalid rule: " + value.toString()); 178 | } 179 | } 180 | } 181 | 182 | function grammar(baseGrammar, options) { 183 | if (!options) { 184 | options = baseGrammar; 185 | baseGrammar = { 186 | name: null, 187 | rules: {}, 188 | extras: [normalize(/\s/)], 189 | conflicts: [], 190 | externals: [], 191 | inline: [] 192 | }; 193 | } 194 | 195 | let externals = baseGrammar.externals; 196 | if (options.externals) { 197 | if (typeof options.externals !== "function") { 198 | throw new Error("Grammar's 'externals' property must be a function."); 199 | } 200 | 201 | const externalsRuleBuilder = new RuleBuilder() 202 | const externalRules = options.externals.call(externalsRuleBuilder, externalsRuleBuilder, baseGrammar.externals); 203 | 204 | if (!Array.isArray(externalRules)) { 205 | throw new Error("Grammar's externals must be an array of rules."); 206 | } 207 | 208 | externals = externalRules.map(normalize); 209 | } 210 | 211 | const rulesSet = {}; 212 | for (const key in options.rules) { 213 | rulesSet[key] = true; 214 | } 215 | for (const key in baseGrammar.rules) { 216 | rulesSet[key] = true; 217 | } 218 | for (const external of externals) { 219 | if (typeof external.name === 'string') { 220 | rulesSet[external.name] = true; 221 | } 222 | } 223 | 224 | const ruleBuilder = new RuleBuilder(rulesSet); 225 | 226 | const name = options.name; 227 | if (typeof name !== "string") { 228 | throw new Error("Grammar's 'name' property must be a string."); 229 | } 230 | 231 | if (!/^[a-zA-Z_]\w*$/.test(name)) { 232 | throw new Error("Grammar's 'name' property must not start with a digit and cannot contain non-word characters."); 233 | } 234 | 235 | let rules = Object.assign({}, baseGrammar.rules); 236 | if (options.rules) { 237 | if (typeof options.rules !== "object") { 238 | throw new Error("Grammar's 'rules' property must be an object."); 239 | } 240 | 241 | for (const ruleName in options.rules) { 242 | const ruleFn = options.rules[ruleName]; 243 | if (typeof ruleFn !== "function") { 244 | throw new Error("Grammar rules must all be functions. '" + ruleName + "' rule is not."); 245 | } 246 | rules[ruleName] = normalize(ruleFn.call(ruleBuilder, ruleBuilder, baseGrammar.rules[ruleName])); 247 | } 248 | } 249 | 250 | let extras = baseGrammar.extras.slice(); 251 | if (options.extras) { 252 | if (typeof options.extras !== "function") { 253 | throw new Error("Grammar's 'extras' property must be a function."); 254 | } 255 | 256 | extras = options.extras 257 | .call(ruleBuilder, ruleBuilder, baseGrammar.extras) 258 | .map(normalize); 259 | } 260 | 261 | let word = baseGrammar.word; 262 | if (options.word) { 263 | word = options.word.call(ruleBuilder, ruleBuilder).name; 264 | if (typeof word != 'string') { 265 | throw new Error("Grammar's 'word' property must be a named rule."); 266 | } 267 | } 268 | 269 | let conflicts = baseGrammar.conflicts; 270 | if (options.conflicts) { 271 | if (typeof options.conflicts !== "function") { 272 | throw new Error("Grammar's 'conflicts' property must be a function."); 273 | } 274 | 275 | const baseConflictRules = baseGrammar.conflicts.map(conflict => conflict.map(sym)); 276 | const conflictRules = options.conflicts.call(ruleBuilder, ruleBuilder, baseConflictRules); 277 | 278 | if (!Array.isArray(conflictRules)) { 279 | throw new Error("Grammar's conflicts must be an array of arrays of rules."); 280 | } 281 | 282 | conflicts = conflictRules.map(conflictSet => { 283 | if (!Array.isArray(conflictSet)) { 284 | throw new Error("Grammar's conflicts must be an array of arrays of rules."); 285 | } 286 | 287 | return conflictSet.map(symbol => symbol.name); 288 | }); 289 | } 290 | 291 | let inline = baseGrammar.inline; 292 | if (options.inline) { 293 | if (typeof options.inline !== "function") { 294 | throw new Error("Grammar's 'inline' property must be a function."); 295 | } 296 | 297 | const baseInlineRules = baseGrammar.inline.map(sym); 298 | const inlineRules = options.inline.call(ruleBuilder, ruleBuilder, baseInlineRules); 299 | 300 | if (!Array.isArray(inlineRules)) { 301 | throw new Error("Grammar's inline must be an array of rules."); 302 | } 303 | 304 | inline = inlineRules.map(symbol => symbol.name); 305 | } 306 | 307 | if (Object.keys(rules).length == 0) { 308 | throw new Error("Grammar must have at least one rule."); 309 | } 310 | 311 | return {name, word, rules, extras, conflicts, externals, inline}; 312 | } 313 | 314 | module.exports = { 315 | alias: alias, 316 | grammar: grammar, 317 | blank: blank, 318 | choice: choice, 319 | err: err, 320 | optional: optional, 321 | prec: prec, 322 | repeat: repeat, 323 | repeat1: repeat1, 324 | seq: seq, 325 | sym: sym, 326 | token: token 327 | }; 328 | -------------------------------------------------------------------------------- /lib/api/index.js: -------------------------------------------------------------------------------- 1 | const fs = require("fs"); 2 | const path = require("path"); 3 | const tmp = require("temp"); 4 | const {spawnSync} = require("child_process"); 5 | const binding = require("./binding"); 6 | const properties = require('./properties'); 7 | const dsl = require("./dsl"); 8 | 9 | const INCLUDE_PATH = path.join(__dirname, '..', '..', 'vendor', 'tree-sitter', 'include') 10 | 11 | function loadLanguage(code, otherSourceFiles = []) { 12 | const {path: srcPath, fd} = tmp.openSync({suffix: '.c'}); 13 | fs.closeSync(fd); 14 | fs.writeFileSync(srcPath, code); 15 | 16 | let compileResult, libPath 17 | if (process.platform === 'win32') { 18 | libPath = srcPath + ".dll"; 19 | compileResult = spawnSync('cl.exe', [ 20 | "/nologo", 21 | "/LD", 22 | "/I", INCLUDE_PATH, 23 | "/Od", 24 | srcPath, 25 | ...otherSourceFiles, 26 | "/link", 27 | "/out:" + libPath 28 | ], { 29 | encoding: 'utf8' 30 | }); 31 | } else { 32 | libPath = srcPath + ".so"; 33 | compileResult = spawnSync(process.env['CC'] || 'gcc', [ 34 | "-shared", 35 | "-x", "c", 36 | "-fPIC", 37 | "-g", 38 | "-I", INCLUDE_PATH, 39 | "-o", libPath, 40 | srcPath, 41 | ...otherSourceFiles 42 | ], { 43 | encoding: 'utf8' 44 | }); 45 | } 46 | 47 | if (compileResult.error) { 48 | throw compileResult.error; 49 | } 50 | 51 | if (compileResult.status !== 0) { 52 | throw new Error("Compiling parser failed:\n" + compileResult.stdout + "\n" + compileResult.stderr); 53 | } 54 | 55 | const languageFunctionName = code.match(/(tree_sitter_\w+)\(\) {/)[1]; 56 | return require("./binding").loadLanguage(libPath, languageFunctionName); 57 | } 58 | 59 | function generate(grammar, logToStderr) { 60 | return binding.generateParserCode(JSON.stringify(grammar), logToStderr); 61 | } 62 | 63 | function generatePropertyJSON(css, baseDirectory, logToStderr) { 64 | const parsedProperties = properties.parseProperties(css, baseDirectory); 65 | for (const rule of parsedProperties) { 66 | for (const key in rule.properties) { 67 | rule.properties[key] = JSON.stringify(rule.properties[key]); 68 | } 69 | } 70 | const stateMachineJSON = binding.generatePropertyJSON(JSON.stringify(parsedProperties), logToStderr); 71 | const stateMachine = JSON.parse(stateMachineJSON); 72 | for (const propertySet of stateMachine.property_sets) { 73 | for (const key in propertySet) { 74 | propertySet[key] = JSON.parse(propertySet[key]); 75 | } 76 | } 77 | return JSON.stringify(stateMachine, null, 2); 78 | } 79 | 80 | module.exports = { 81 | generate, 82 | generatePropertyJSON, 83 | loadLanguage, 84 | dsl, 85 | parseProperties: properties.parseProperties, 86 | queryProperties: properties.queryProperties 87 | }; 88 | -------------------------------------------------------------------------------- /lib/api/properties.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const fs = require('fs'); 3 | const path = require('path'); 4 | const parseCSS = require('scss-parser').parse; 5 | const {Validator} = require('jsonschema'); 6 | const Module = require('module') 7 | 8 | // Test a property state machine against a node structure 9 | // specified as an array of node types. 10 | function queryProperties(props, nodeTypeStack, childIndexStack = [], text) { 11 | let stateId = 0 12 | for (let i = 0, {length} = nodeTypeStack; i < length; i++) { 13 | const nodeType = nodeTypeStack[i]; 14 | const state = props.states[stateId]; 15 | let found_transition = false; 16 | for (const transition of state.transitions) { 17 | if ( 18 | (transition.type === nodeType) && 19 | (transition.index == null || transition.index === childIndexStack[i]) && 20 | (transition.text == null || new RegExp(transition.text).test(text)) 21 | ) { 22 | stateId = transition.state_id; 23 | found_transition = true; 24 | break; 25 | } 26 | } 27 | if (!found_transition) { 28 | stateId = state.default_next_state_id 29 | } 30 | // console.log( 31 | // nodeType, '->', stateId, 32 | // props.property_sets[props.states[stateId].property_set_id] 33 | // ); 34 | } 35 | return props.property_sets[props.states[stateId].property_set_id] 36 | } 37 | 38 | // Parse a property sheet written in CSS into the intermediate 39 | // object structure that is passed to the `ts_compile_property_sheet` 40 | // function. 41 | function parseProperties(source, baseDirectory) { 42 | // Get the raw SCSS concrete syntax tree. 43 | const rawTree = parseCSS(source); 44 | removeWhitespace(rawTree); 45 | 46 | // Convert the concrete syntax tree into a more convenient AST. 47 | let schema 48 | const rootRules = [] 49 | for (const rule of rawTree.value) { 50 | if (rule.type === 'atrule') { 51 | removeWhitespace(rule); 52 | const [keyword, argument] = rule.value; 53 | if (keyword.value === 'schema') { 54 | schema = visitSchema(argument, baseDirectory) 55 | } else if (keyword.value === 'import') { 56 | rootRules.push(...visitImport(argument, baseDirectory)) 57 | } 58 | } else { 59 | rootRules.push(visitRule(rule, false)) 60 | } 61 | } 62 | 63 | // Flatten out any nested selectors. 64 | const rules = [] 65 | for (const rootRule of rootRules) { 66 | flattenRule(rootRule, rules, [[]]); 67 | } 68 | 69 | const validator = new Validator(); 70 | 71 | for (const rule of rules) { 72 | // Validate each rule's properties against the schema. 73 | for (const property in rule.properties) { 74 | if (schema) { 75 | const propertySchema = schema[property] 76 | const value = rule.properties[property] 77 | if (!propertySchema) { 78 | throw new Error(`Property '${property}' is not present in the schema`); 79 | } 80 | const {errors} = validator.validate(value, propertySchema); 81 | if (errors.length > 0) { 82 | throw new Error(`Invalid value '${value}' for property '${property}'`) 83 | } 84 | } 85 | } 86 | 87 | rule.selectors = rule.selectors.map(applyPseudoClasses) 88 | } 89 | 90 | return rules; 91 | } 92 | 93 | function applyPseudoClasses(rule) { 94 | const result = [] 95 | for (const entry of rule) { 96 | if (entry.pseudo) { 97 | result[result.length - 1] = Object.assign({}, result[result.length - 1], entry.pseudo) 98 | } else { 99 | result.push(entry) 100 | } 101 | } 102 | return result 103 | } 104 | 105 | function flattenRule(node, flatRules, prefixes) { 106 | const newSelectors = [] 107 | 108 | for (const prefix of prefixes) { 109 | for (const selector of node.selectors) { 110 | newSelectors.push(prefix.concat(selector)) 111 | } 112 | } 113 | 114 | flatRules.push({ 115 | selectors: newSelectors, 116 | properties: node.properties 117 | }) 118 | 119 | for (const childNode of node.rules || []) { 120 | flattenRule(childNode, flatRules, newSelectors); 121 | } 122 | } 123 | 124 | function visitSchema(argument, baseDirectory) { 125 | return require(resolvePath(argument, baseDirectory)); 126 | } 127 | 128 | function visitImport(argument, baseDirectory) { 129 | const importPath = resolvePath(argument, baseDirectory); 130 | const source = fs.readFileSync(importPath, 'utf8'); 131 | return parseProperties(source, path.dirname(importPath)); 132 | } 133 | 134 | function resolvePath(argument, baseDirectory) { 135 | if (!['string_double', 'string_single'].includes(argument.type)) { 136 | throw new Error(`Unexpected path argument type ${argument.type}`); 137 | } 138 | if (argument.value.startsWith('.')) { 139 | return path.resolve(baseDirectory, argument.value); 140 | } else { 141 | return require.resolve(argument.value, { 142 | paths: Module._nodeModulePaths(baseDirectory) 143 | }) 144 | } 145 | } 146 | 147 | function visitRule(tree, nested) { 148 | assert.equal(tree.type, 'rule'); 149 | removeWhitespace(tree); 150 | assert.equal(tree.value.length, 2); 151 | return { 152 | selectors: visitSelectors(tree.value[0], nested), 153 | ...visitBlock(tree.value[1]) 154 | }; 155 | } 156 | 157 | function visitSelectors(tree, nested) { 158 | assert.equal(tree.type, 'selector'); 159 | const selectors = []; 160 | let selector = []; 161 | let expectAmpersand = nested; 162 | for (const entry of tree.value) { 163 | if (entry.type === 'punctuation' && entry.value === ',') { 164 | assert.notEqual(selector.length, 0); 165 | selectors.push(visitSelector(selector, nested)); 166 | selector = []; 167 | expectAmpersand = nested 168 | } else if (expectAmpersand) { 169 | if (!isWhitespace(entry)) { 170 | assert.equal(entry.value, '&') 171 | expectAmpersand = false 172 | } 173 | } else { 174 | selector.push(entry); 175 | } 176 | } 177 | assert.notEqual(selector.length, 0); 178 | selectors.push(visitSelector(selector, nested)); 179 | return selectors; 180 | } 181 | 182 | function visitSelector(selector, nested) { 183 | const result = []; 184 | let immediate = false; 185 | let previousEntryWasNodeType = false; 186 | for (let i = 0, {length} = selector; i < length; i++) { 187 | const entry = selector[i]; 188 | if (entry.type === 'operator' && entry.value === '>') { 189 | immediate = true 190 | previousEntryWasNodeType = false; 191 | } else if ( 192 | entry.type === 'identifier' || 193 | entry.type === 'string_double' || 194 | entry.type === 'string_single' 195 | ) { 196 | let named = entry.type === 'identifier'; 197 | result.push({ 198 | type: entry.value, 199 | named, 200 | immediate 201 | }); 202 | immediate = false 203 | previousEntryWasNodeType = true; 204 | } else if (entry.type === 'function') { 205 | const pseudo = entry.value[0].value[0].value; 206 | 207 | let hasPrecedingNodeType = (i === 0) 208 | ? nested 209 | : previousEntryWasNodeType 210 | 211 | if (!previousEntryWasNodeType && !(nested && i === 0)) { 212 | throw new Error(`Pseudo class ':${pseudo}' must be used together with a node type`) 213 | } 214 | 215 | if (pseudo === 'nth-child') { 216 | const args = entry.value[1]; 217 | assert.equal(args.type, 'arguments'); 218 | result.push({pseudo: {index: parseInt(args.value[0].value, 10)}}) 219 | } else if (pseudo === 'text') { 220 | const args = entry.value[1]; 221 | assert.equal(args.type, 'arguments'); 222 | assert.equal(args.value.length, 1) 223 | result.push({pseudo: {text: args.value[0].value}}) 224 | } 225 | previousEntryWasNodeType = false; 226 | } else { 227 | previousEntryWasNodeType = false; 228 | } 229 | } 230 | return result; 231 | } 232 | 233 | function visitBlock(tree) { 234 | assert.equal(tree.type, 'block'); 235 | removeWhitespace(tree); 236 | const properties = {}; 237 | const rules = []; 238 | for (const entry of tree.value) { 239 | if (entry.type === 'declaration') { 240 | const {key, value} = visitDeclaration(entry); 241 | if (properties[key] != null) { 242 | if (Array.isArray(properties[key])) { 243 | properties[key].push(value); 244 | } else { 245 | properties[key] = [properties[key], value]; 246 | } 247 | } else { 248 | properties[key] = value; 249 | } 250 | } else { 251 | rules.push(visitRule(entry, true)); 252 | } 253 | } 254 | return {properties, rules}; 255 | } 256 | 257 | function visitDeclaration(tree) { 258 | assert.equal(tree.type, 'declaration'); 259 | removeWhitespace(tree); 260 | const propertyTree = tree.value[0]; 261 | const valueTree = tree.value[2]; 262 | removeWhitespace(propertyTree); 263 | removeWhitespace(valueTree); 264 | assert.equal(propertyTree.type, 'property') 265 | assert.equal(valueTree.type, 'value') 266 | 267 | return { 268 | key: propertyTree.value[0].value, 269 | value: visitPropertyValue(valueTree.value[0]) 270 | }; 271 | } 272 | 273 | function visitPropertyValue(tree) { 274 | if (tree.type === 'function') { 275 | return visitFunctionCall(tree); 276 | } else if (tree.type === 'number') { 277 | return parseInt(tree.value); 278 | } else { 279 | return tree.value; 280 | } 281 | } 282 | 283 | function visitFunctionCall(tree) { 284 | assert.equal(tree.value[0].type, 'identifier'); 285 | const name = tree.value[0].value; 286 | assert.equal(tree.value[1].type, 'arguments'); 287 | const argumentsTree = tree.value[1]; 288 | removeWhitespace(argumentsTree); 289 | 290 | const args = [] 291 | for (const argumentTree of argumentsTree.value) { 292 | if (argumentTree.type === 'punctuation') continue; 293 | args.push(visitPropertyValue(argumentTree)); 294 | } 295 | 296 | return {name, args} 297 | } 298 | 299 | // Get rid of useless nodes and properties. 300 | function removeWhitespace(node) { 301 | node.value = node.value.filter(node => !isWhitespace(node)) 302 | } 303 | 304 | function isWhitespace(node) { 305 | return ( 306 | node.type === 'space' || 307 | node.type === 'comment_multiline' || 308 | node.type === 'comment_singleline' 309 | ) 310 | } 311 | 312 | module.exports = {parseProperties, queryProperties} 313 | -------------------------------------------------------------------------------- /lib/cli/generate.js: -------------------------------------------------------------------------------- 1 | module.exports = function generate(options, callback) { 2 | const api = require("../api"); 3 | const fs = require("fs-extra"); 4 | const path = require("path"); 5 | const mkdirp = require("mkdirp"); 6 | const templates = require("./templates"); 7 | const cwd = process.cwd(); 8 | const profileCommand = require('./helpers/profile-command'); 9 | 10 | if (options.profile) { 11 | options.profile = false; 12 | profileCommand(invokeSelfCommand(options).join(' '), 'ts_compile_grammar', callback); 13 | return; 14 | } 15 | 16 | for (const key in api.dsl) { 17 | global[key] = api.dsl[key]; 18 | } 19 | 20 | let grammar 21 | if (fs.existsSync(path.join(cwd, 'grammar.json'))) { 22 | const grammarString = fs.readFileSync(path.join(cwd, 'grammar.json'), 'utf8') 23 | grammar = eval('(' + grammarString + ')') 24 | } else { 25 | grammar = require(path.join(cwd, "grammar")); 26 | } 27 | 28 | const srcPath = path.join(cwd, 'src'); 29 | const grammarJSONPath = path.join(srcPath, 'grammar.json'); 30 | const parserPath = path.join(srcPath, 'parser.c'); 31 | const bindingCCPath = path.join(srcPath, 'binding.cc'); 32 | const bindingGypPath = path.join(cwd, 'binding.gyp'); 33 | const indexJSPath = path.join(cwd, 'index.js'); 34 | 35 | mkdirp.sync(srcPath); 36 | mkdirp.sync(path.join(srcPath, "tree_sitter")); 37 | 38 | let code; 39 | if (!options.properties) { 40 | try { 41 | code = api.generate(grammar, options.debug); 42 | } catch (e) { 43 | if (e.isGrammarError) { 44 | console.warn("Error: " + e.message); 45 | return callback(1); 46 | } else { 47 | throw e; 48 | } 49 | } 50 | } 51 | 52 | const propertiesPath = path.join(cwd, 'properties') 53 | let propertiesFilenames = [] 54 | try { 55 | propertiesFilenames = fs.readdirSync(propertiesPath).filter(p => p.endsWith('.css')) 56 | } catch (_) {} 57 | 58 | for (const propertiesFilename of propertiesFilenames) { 59 | const propertiesCSS = fs.readFileSync(path.join(propertiesPath, propertiesFilename), 'utf8'); 60 | const propertiesJSON = api.generatePropertyJSON(propertiesCSS, propertiesPath); 61 | fs.writeFileSync( 62 | path.join(srcPath, propertiesFilename.replace(/\.css$/, '.json')), 63 | JSON.stringify(JSON.parse(propertiesJSON), null, 2) 64 | ) 65 | } 66 | 67 | const headerPath = path.join(__dirname, "..", "..", "vendor", "tree-sitter", "include", "tree_sitter"); 68 | fs.copySync(path.join(headerPath, "parser.h"), path.join(srcPath, "tree_sitter", "parser.h")); 69 | 70 | fs.writeFileSync(grammarJSONPath, JSON.stringify(grammar, null, 2)); 71 | if (code) { 72 | fs.writeFileSync(parserPath, code); 73 | } 74 | 75 | if (!fs.existsSync(bindingCCPath)) { 76 | fs.writeFileSync(bindingCCPath, templates.bindingCC(grammar.name)); 77 | } 78 | 79 | if (!fs.existsSync(bindingGypPath)) { 80 | fs.writeFileSync(bindingGypPath, templates.bindingGyp(grammar.name)); 81 | } 82 | 83 | if (!fs.existsSync(indexJSPath)) { 84 | fs.writeFileSync(indexJSPath, templates.indexJS(grammar.name)); 85 | } 86 | 87 | callback(0); 88 | }; 89 | 90 | function invokeSelfCommand(options) { 91 | return [ 92 | process.argv[0], 93 | "-e", 94 | "require('" + __filename + "')(" + JSON.stringify(options) + ",process.exit);" 95 | ]; 96 | } 97 | -------------------------------------------------------------------------------- /lib/cli/helpers/profile-command.js: -------------------------------------------------------------------------------- 1 | const fs = require("fs") 2 | const temp = require("temp") 3 | const {spawn} = require('child_process') 4 | const {generateFlameGraphForCommand} = require('@maxbrunsfeld/flame-graph') 5 | 6 | module.exports = 7 | async function profileCommand (command, containingFunctionName, callback) { 8 | const html = await generateFlameGraphForCommand(command, { 9 | functionNames: containingFunctionName, 10 | fullPage: true 11 | }) 12 | 13 | const flamegraphFile = temp.openSync({prefix: 'flamegraph', suffix: '.html'}) 14 | fs.chmodSync(flamegraphFile.path, '755') 15 | fs.writeSync(flamegraphFile.fd, html, 'utf8') 16 | 17 | spawn('open', [flamegraphFile.path]) 18 | callback() 19 | } 20 | -------------------------------------------------------------------------------- /lib/cli/parse.js: -------------------------------------------------------------------------------- 1 | const {queryProperties} = require('../api/properties'); 2 | 3 | module.exports = function parse(options, callback) { 4 | const fs = require("fs"); 5 | const path = require("path"); 6 | const {spawn, execSync, spawnSync} = require("child_process"); 7 | const temp = require("temp"); 8 | const Parser = require("tree-sitter"); 9 | const profileCommand = require('./helpers/profile-command'); 10 | 11 | const cwd = process.cwd(); 12 | const language = require(cwd); 13 | const parser = new Parser().setLanguage(language); 14 | 15 | if (options.repeat) { 16 | console.log("Benchmarking with %d repetitions", options.repeat); 17 | } else { 18 | options.repeat = 1; 19 | } 20 | 21 | if (options.heapProfile) { 22 | const command = invokeSelfCommand(options, { 23 | heapProfile: false, 24 | repeat: null, 25 | quiet: true 26 | }); 27 | 28 | const parseProcess = spawn(command[0], command.slice(1), { 29 | env: { 30 | DYLD_INSERT_LIBRARIES: '/usr/local/lib/libtcmalloc.dylib', 31 | HEAPPROFILE: 'heap' 32 | }, 33 | stdio: ['ignore', 'inherit', 'pipe'] 34 | }); 35 | 36 | let heapProfileOutput = '' 37 | parseProcess.stderr.on('data', chunk => { 38 | heapProfileOutput += chunk.toString('utf8') 39 | }); 40 | 41 | parseProcess.on('close', code => { 42 | const heapProfilePaths = heapProfileOutput.match(/\S+\.heap/g) 43 | const heapProfilePath = heapProfilePaths.pop() 44 | const pprofProcess = spawn('pprof', [ 45 | '--text', 46 | '--focus=parse', 47 | process.execPath, 48 | heapProfilePath 49 | ], {stdio: ['ignore', 'inherit', 'inherit']}); 50 | pprofProcess.on('close', callback); 51 | }); 52 | 53 | return; 54 | } else if (options.debugGraph) { 55 | const dotOutputFile = temp.openSync({prefix: 'stack-debug', suffix: '.dot'}); 56 | const htmlOutputFile = temp.openSync({prefix: 'stack-debug', suffix: '.html'}); 57 | 58 | const command = invokeSelfCommand(options, { 59 | debugGraph: false, 60 | isDebuggingGraph: true 61 | }); 62 | const parseProcess = spawn(command[0], command.slice(1), { 63 | stdio: ['ignore', 'ignore', dotOutputFile.fd] 64 | }); 65 | 66 | process.on('SIGINT', () => parseProcess.kill()) 67 | 68 | parseProcess.on('close', code => { 69 | 70 | // If the parse process was killed before completing, it may have written a 71 | // partial graph, which would cause an error when running `dot`. Find the 72 | // last empty line and truncate the file to that line. 73 | const trimmedDotFilePath = dotOutputFile.path + '.trimmed'; 74 | const trimmedDotFilePid = fs.openSync(trimmedDotFilePath, 'w+'); 75 | const lastBlankLineNumber = execSync(`grep -n '^$' ${dotOutputFile.path} | tail -n1`) 76 | .toString('utf8') 77 | .split(':')[0]; 78 | spawnSync('head', ['-n', lastBlankLineNumber, dotOutputFile.path], { 79 | stdio: ['ignore', trimmedDotFilePid] 80 | }); 81 | fs.renameSync(trimmedDotFilePath, dotOutputFile.path); 82 | 83 | // Create an HTML file with some basic styling for SVG graphs. 84 | fs.writeSync(htmlOutputFile.fd, [ 85 | "", 86 | "" 87 | ].join('\n')); 88 | 89 | // Write the graphs to the HTML file using `dot`. 90 | const dotProcess = spawn("dot", [ 91 | "-Tsvg", 92 | dotOutputFile.path 93 | ], { 94 | stdio: ['ignore', htmlOutputFile.fd, process.stderr] 95 | }); 96 | 97 | dotProcess.on('close', function (code) { 98 | if (code !== 0) { 99 | console.log("dot failed", code); 100 | return; 101 | } 102 | 103 | console.log('Opening', htmlOutputFile.path); 104 | const openCommand = process.platform === 'linux' 105 | ? 'xdg-open' 106 | : 'open'; 107 | spawn(openCommand, [htmlOutputFile.path]); 108 | }); 109 | }); 110 | 111 | return 112 | } else if (options.isDebuggingGraph) { 113 | parser.printDotGraphs(true); 114 | console.log = function() {} 115 | } else if (options.debug) { 116 | parser.setLogger(function(topic, params, type) { 117 | switch (type) { 118 | case 'parse': 119 | console.log(topic, params) 120 | break; 121 | case 'lex': 122 | console.log(" ", topic, params); 123 | } 124 | }); 125 | } 126 | 127 | if (options.profile) { 128 | profileCommand(invokeSelfCommand(options, {profile: false}).join(' '), 'tree_sitter', callback) 129 | return 130 | } 131 | 132 | let foundError = false 133 | 134 | for (let i = 0; i < options.codePaths.length; i++) { 135 | const codePath = options.codePaths[i]; 136 | if (codePath !== '-' && fs.statSync(codePath).isDirectory()) { 137 | const childPaths = []; 138 | for (const directoryEntry of fs.readdirSync(codePath)) { 139 | if (!directoryEntry.startsWith('.')) { 140 | childPaths.push(path.join(codePath, directoryEntry)); 141 | } 142 | } 143 | options.codePaths.splice(i, 1, ...childPaths); 144 | i--; 145 | continue; 146 | } 147 | } 148 | 149 | const maxLength = Math.max(...options.codePaths.map(path => path.length)); 150 | 151 | for (const codePath of options.codePaths) { 152 | let sourceCode = fs.readFileSync(codePath === '-' ? 0 : codePath, 'utf8') 153 | 154 | let tree; 155 | const t0 = process.hrtime() 156 | for (let i = 0; i < options.repeat; i++) { 157 | tree = parser.parse(sourceCode); 158 | if (options.edits) { 159 | if (!Array.isArray(options.edits)) options.edits = [options.edits] 160 | for (const edit of options.edits) { 161 | let [startIndex, lengthRemoved] = edit.split(',', 2) 162 | const newText = JSON.parse(edit.slice(startIndex.length + lengthRemoved.length + 2)) 163 | startIndex = parseInt(startIndex); 164 | lengthRemoved = parseInt(lengthRemoved); 165 | const prefix = sourceCode.slice(0, startIndex); 166 | const deletedText = sourceCode.slice(startIndex, startIndex + lengthRemoved); 167 | const suffix = sourceCode.slice(startIndex + lengthRemoved); 168 | const startPosition = getExtent(prefix); 169 | sourceCode = prefix + newText + suffix; 170 | tree.edit({ 171 | startIndex, 172 | oldEndIndex: startIndex + lengthRemoved, 173 | newEndIndex: startIndex + newText.length, 174 | startPosition, 175 | oldEndPosition: addPoint(startPosition, getExtent(deletedText)), 176 | newEndPosition: addPoint(startPosition, getExtent(newText)) 177 | }) 178 | tree = parser.parse(sourceCode, tree) 179 | } 180 | } 181 | } 182 | const [seconds, nanoseconds] = process.hrtime(t0) 183 | 184 | if (!options.quiet && !options.debug && !options.debugGraph) { 185 | let propertiesJSON 186 | if (options.properties) { 187 | propertiesJSON = require(path.join(cwd, 'src', options.properties + '.json')) 188 | } 189 | printNode(tree.rootNode, [], propertiesJSON); 190 | process.stdout.write("\n"); 191 | } 192 | 193 | const duration = Math.round((seconds * 1000 + nanoseconds / 1000000) / options.repeat); 194 | if (tree.rootNode.hasError()) { 195 | foundError = true; 196 | console.log("%s\t%d ms\t%s", pad(codePath, maxLength), duration, firstErrorDescription(tree.rootNode)); 197 | } else if (options.time) { 198 | console.log("%s\t%d ms", pad(codePath, maxLength), duration); 199 | } 200 | } 201 | 202 | callback(foundError ? 1 : 0); 203 | }; 204 | 205 | function firstErrorDescription(node) { 206 | if (!node.hasError) return null; 207 | 208 | if (node.type === 'ERROR') { 209 | return "ERROR " + pointString(node.startPosition) + " - " + pointString(node.endPosition); 210 | } 211 | 212 | if (node.isMissing()) { 213 | return "MISSING " + node.type + " " + pointString(node.startPosition) + " - " + pointString(node.endPosition); 214 | } 215 | 216 | const {children} = node; 217 | for (let i = 0, length = children.length; i < length; i++) { 218 | const description = firstErrorDescription(children[i]); 219 | if (description) return description; 220 | } 221 | } 222 | 223 | function printNode(node, stack, propertiesJSON) { 224 | const indentLevel = stack.length; 225 | process.stdout.write(' '.repeat(2 * indentLevel)); 226 | process.stdout.write("("); 227 | process.stdout.write(node.isMissing() ? 'MISSING' : node.type); 228 | process.stdout.write(" "); 229 | process.stdout.write(pointString(node.startPosition)); 230 | process.stdout.write(" - "); 231 | process.stdout.write(pointString(node.endPosition)); 232 | 233 | stack.push(node.type) 234 | 235 | if (propertiesJSON) { 236 | const props = queryProperties(propertiesJSON, stack, [], node.text) 237 | for (const key in props) { 238 | process.stdout.write(`, ${key}: ${props[key]}`) 239 | } 240 | } 241 | 242 | const {namedChildren} = node; 243 | for (let i = 0, length = namedChildren.length; i < length; i++) { 244 | process.stdout.write("\n"); 245 | printNode(namedChildren[i], stack, propertiesJSON); 246 | } 247 | 248 | stack.pop() 249 | 250 | process.stdout.write(")"); 251 | } 252 | 253 | function pointString(point) { 254 | return "[" + point.row + ", " + point.column + "]"; 255 | } 256 | 257 | function invokeSelfCommand(options, overrides) { 258 | return [ 259 | process.argv[0], 260 | "-e", 261 | "require('" + __filename + "')(" + 262 | JSON.stringify(Object.assign({}, options, overrides)) + 263 | ",process.exit);" 264 | ]; 265 | } 266 | 267 | function pad(string, length) { 268 | return string + ' '.repeat(length - string.length) 269 | } 270 | 271 | function addPoint(a, b) { 272 | if (b.row > 0) { 273 | return {row: a.row + b.row, column: b.column}; 274 | } else { 275 | return {row: a.row, column: a.column + b.column}; 276 | } 277 | } 278 | 279 | 280 | function getExtent (string) { 281 | const result = {row: 0, column: 0} 282 | for (const character of string) { 283 | if (character === '\n') { 284 | result.row++ 285 | result.column = 0 286 | } else { 287 | result.column++ 288 | } 289 | } 290 | return result 291 | } 292 | -------------------------------------------------------------------------------- /lib/cli/templates.js: -------------------------------------------------------------------------------- 1 | var fs = require("fs"), 2 | path = require("path"), 3 | ejs = require("ejs"); 4 | 5 | var bindingGypTemplate = fs.readFileSync( 6 | path.join(__dirname, "templates", "binding.gyp.ejs"), 7 | "utf8"); 8 | 9 | var bindingCCTemplate = fs.readFileSync( 10 | path.join(__dirname, "templates", "binding.cc.ejs"), 11 | "utf8"); 12 | 13 | var indexJSTemplate = fs.readFileSync( 14 | path.join(__dirname, "templates", "index.js.ejs"), 15 | "utf8"); 16 | 17 | exports.bindingGyp = function(parserName) { 18 | return ejs.render(bindingGypTemplate, { parserName: parserName }); 19 | }; 20 | 21 | exports.bindingCC = function(parserName) { 22 | return ejs.render(bindingCCTemplate, { 23 | parserName: parserName, 24 | camelizedParserName: camelize(parserName) 25 | }); 26 | }; 27 | 28 | exports.indexJS = function(parserName) { 29 | return ejs.render(indexJSTemplate, { parserName: parserName }); 30 | }; 31 | 32 | function camelize(str) { 33 | return (str[0].toUpperCase() + str.slice(1)).replace(/_(\w)/, function(_, match) { 34 | return match.toUpperCase(); 35 | }); 36 | } 37 | -------------------------------------------------------------------------------- /lib/cli/templates/binding.cc.ejs: -------------------------------------------------------------------------------- 1 | #include "tree_sitter/parser.h" 2 | #include 3 | #include "nan.h" 4 | 5 | using namespace v8; 6 | 7 | extern "C" TSLanguage * tree_sitter_<%= parserName %>(); 8 | 9 | namespace { 10 | 11 | NAN_METHOD(New) {} 12 | 13 | void Init(Handle exports, Handle module) { 14 | Local tpl = Nan::New(New); 15 | tpl->SetClassName(Nan::New("Language").ToLocalChecked()); 16 | tpl->InstanceTemplate()->SetInternalFieldCount(1); 17 | 18 | Local constructor = tpl->GetFunction(); 19 | Local instance = constructor->NewInstance(Nan::GetCurrentContext()).ToLocalChecked(); 20 | Nan::SetInternalFieldPointer(instance, 0, tree_sitter_<%= parserName %>()); 21 | 22 | instance->Set(Nan::New("name").ToLocalChecked(), Nan::New("<%= parserName %>").ToLocalChecked()); 23 | module->Set(Nan::New("exports").ToLocalChecked(), instance); 24 | } 25 | 26 | NODE_MODULE(tree_sitter_<%= parserName %>_binding, Init) 27 | 28 | } // namespace 29 | -------------------------------------------------------------------------------- /lib/cli/templates/binding.gyp.ejs: -------------------------------------------------------------------------------- 1 | { 2 | "targets": [ 3 | { 4 | "target_name": "tree_sitter_<%= parserName %>_binding", 5 | "include_dirs": [ 6 | "_binding"); 3 | } catch (error) { 4 | try { 5 | module.exports = require("./build/Debug/tree_sitter_<%= parserName %>_binding"); 6 | } catch (_) { 7 | throw error 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /lib/cli/test.js: -------------------------------------------------------------------------------- 1 | module.exports = function test(options, callback) { 2 | const fs = require("fs"); 3 | const vows = require("vows"); 4 | const specReporter = require("vows/lib/vows/reporters/spec"); 5 | const path = require("path"); 6 | const assert = require("assert"); 7 | const Parser = require('tree-sitter'); 8 | 9 | const cwd = process.cwd(); 10 | const language = require(cwd) 11 | const parser = new Parser().setLanguage(language); 12 | 13 | let testDir = path.join(cwd, 'corpus'); 14 | if (!fs.existsSync(testDir)) { 15 | testDir = path.join(cwd, 'grammar_test') 16 | if (!fs.existsSync(testDir)) { 17 | console.error("Couldn't find a `corpus` or `grammar_test` directory in the current working directory"); 18 | callback(1) 19 | return 20 | } 21 | } 22 | 23 | if (options.debug) 24 | parser.setLogger(function(topic, params, type) { 25 | switch (type) { 26 | case 'parse': 27 | console.log(topic, params) 28 | break; 29 | case 'lex': 30 | console.log(" ", topic, params); 31 | } 32 | }); 33 | 34 | vows 35 | .describe("The " + language.name + " language") 36 | .addBatch(suiteForPath(testDir, parser)) 37 | .run( 38 | { 39 | reporter: specReporter, 40 | matcher: options.focus && new RegExp(options.focus), 41 | }, 42 | (result) => callback(result.broken) 43 | ); 44 | 45 | function suiteForPath(filepath, parser) { 46 | const result = {}; 47 | 48 | if (fs.statSync(filepath).isDirectory()) { 49 | fs.readdirSync(filepath).forEach(function(name) { 50 | const description = name.split('.')[0]; 51 | result[description] = suiteForPath(path.join(filepath, name), parser); 52 | }); 53 | } else { 54 | let content = fs.readFileSync(filepath, "utf8"); 55 | 56 | for (;;) { 57 | const headerMatch = content.match(/===+\r?\n([^\r\n=]+)\r?\n===+\r?\n/); 58 | const dividerMatch = content.match(/\n(---+\r?\n)/); 59 | 60 | if (!headerMatch || !dividerMatch) break; 61 | 62 | const testName = headerMatch[1]; 63 | const inputStart = headerMatch[0].length; 64 | const inputEnd = dividerMatch.index; 65 | const outputStart = dividerMatch.index + dividerMatch[1].length; 66 | const nextTestStart = content.slice(outputStart).search(/\n===/); 67 | const outputEnd = (nextTestStart > 0) ? (nextTestStart + outputStart) : content.length; 68 | const input = content.slice(inputStart, inputEnd); 69 | const output = content.slice(outputStart, outputEnd); 70 | 71 | ((input, output) => { 72 | result[testName] = () => { 73 | const tree = parser.parse(input); 74 | assert.equal(normalizeWhitespace(output), tree.rootNode.toString()); 75 | }; 76 | })(input, output); 77 | 78 | content = content.slice(outputEnd + 1); 79 | } 80 | } 81 | 82 | return result; 83 | } 84 | 85 | function normalizeWhitespace(str) { 86 | return str 87 | .replace(/\s+/g, " ") 88 | .replace(/ \)/g, ')') 89 | .trim() 90 | } 91 | }; 92 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "tree-sitter-cli", 3 | "author": "Max Brunsfeld", 4 | "license": "MIT", 5 | "repository": { 6 | "type": "git", 7 | "url": "http://github.com/tree-sitter/tree-sitter-cli.git" 8 | }, 9 | "description": "Incremental parsers for node", 10 | "keywords": [ 11 | "parser", 12 | "lexer" 13 | ], 14 | "main": "lib/api/index.js", 15 | "scripts": { 16 | "test": "mocha" 17 | }, 18 | "bin": { 19 | "tree-sitter": "cli.js" 20 | }, 21 | "version": "0.13.16", 22 | "dependencies": { 23 | "@maxbrunsfeld/flame-graph": "^0.1.9", 24 | "ejs": "^2.5.7", 25 | "fs-extra": "^0.30.0", 26 | "jsonschema": "^1.2.4", 27 | "minimist": "^1.2.0", 28 | "mkdirp": "^0.5.1", 29 | "nan": "^2.10.0", 30 | "node-gyp": "^3.8.0", 31 | "prettier": "^1.12.1", 32 | "scss-parser": "^1.0.0", 33 | "temp": "0.8.x", 34 | "tree-sitter": "^0.13.22", 35 | "vows": "0.7.x" 36 | }, 37 | "devDependencies": { 38 | "chai": "3.5.x", 39 | "mocha": "^5.2.0", 40 | "superstring": "^2.3.1-0" 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/binding.cc: -------------------------------------------------------------------------------- 1 | #include "./generate.h" 2 | #include "./language.h" 3 | #include "./rule_builder.h" 4 | #include 5 | #include 6 | #include "nan.h" 7 | 8 | namespace node_tree_sitter_cli { 9 | 10 | using namespace v8; 11 | 12 | void InitAll(Handle exports) { 13 | node_tree_sitter_cli::InitLanguage(exports); 14 | exports->Set( 15 | Nan::New("generateParserCode").ToLocalChecked(), 16 | Nan::New(GenerateParserCode)->GetFunction()); 17 | exports->Set( 18 | Nan::New("generatePropertyJSON").ToLocalChecked(), 19 | Nan::New(GeneratePropertyJSON)->GetFunction()); 20 | exports->Set( 21 | Nan::New("loadLanguage").ToLocalChecked(), 22 | Nan::New(LoadLanguage)->GetFunction()); 23 | rule_builder::Init(exports); 24 | } 25 | 26 | NODE_MODULE(tree_sitter_cli_binding, InitAll) 27 | 28 | } // namespace node_tree_sitter_cli 29 | -------------------------------------------------------------------------------- /src/generate.cc: -------------------------------------------------------------------------------- 1 | #include "./generate.h" 2 | #include 3 | #include "tree_sitter/compiler.h" 4 | 5 | namespace node_tree_sitter_cli { 6 | 7 | using namespace v8; 8 | 9 | void GenerateParserCode(const Nan::FunctionCallbackInfo &info) { 10 | String::Utf8Value grammar_json(info[0]); 11 | 12 | FILE *log_file; 13 | if (info.Length() > 1 && info[1]->BooleanValue()) { 14 | log_file = stderr; 15 | } else { 16 | log_file = nullptr; 17 | } 18 | 19 | TSCompileResult result = ts_compile_grammar(*grammar_json, log_file); 20 | 21 | if (result.error_type != TSCompileErrorTypeNone) { 22 | Local error = Nan::Error(result.error_message); 23 | Local::Cast(error)->Set(Nan::New("isGrammarError").ToLocalChecked(), Nan::True()); 24 | Nan::ThrowError(error); 25 | } else { 26 | info.GetReturnValue().Set(Nan::New(result.code).ToLocalChecked()); 27 | } 28 | } 29 | 30 | void GeneratePropertyJSON(const Nan::FunctionCallbackInfo &info) { 31 | String::Utf8Value property_sheet_json(info[0]); 32 | 33 | FILE *log_file; 34 | if (info.Length() > 1 && info[1]->BooleanValue()) { 35 | log_file = stderr; 36 | } else { 37 | log_file = nullptr; 38 | } 39 | 40 | TSCompileResult result = ts_compile_property_sheet(*property_sheet_json, log_file); 41 | 42 | if (result.error_type != TSCompileErrorTypeNone) { 43 | Local error = Nan::Error(result.error_message); 44 | Local::Cast(error)->Set(Nan::New("isGrammarError").ToLocalChecked(), Nan::True()); 45 | Nan::ThrowError(error); 46 | } else { 47 | info.GetReturnValue().Set(Nan::New(result.code).ToLocalChecked()); 48 | } 49 | } 50 | 51 | } // namespace node_tree_sitter_cli 52 | -------------------------------------------------------------------------------- /src/generate.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_COMPILE_H 2 | #define TREE_SITTER_COMPILE_H 3 | 4 | #include 5 | #include "nan.h" 6 | 7 | namespace node_tree_sitter_cli { 8 | 9 | void GenerateParserCode(const Nan::FunctionCallbackInfo &); 10 | void GeneratePropertyJSON(const Nan::FunctionCallbackInfo &); 11 | 12 | } // namespace node_tree_sitter_cli 13 | 14 | #endif // TREE_SITTER_COMPILE_H 15 | -------------------------------------------------------------------------------- /src/language.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "tree_sitter/runtime.h" 5 | #include "nan.h" 6 | 7 | using namespace v8; 8 | 9 | namespace node_tree_sitter_cli { 10 | 11 | static Nan::Persistent constructor; 12 | 13 | void LoadLanguage(const Nan::FunctionCallbackInfo &info) { 14 | Handle js_lib_file_name = Handle::Cast(info[0]); 15 | Handle js_language_function_name = Handle::Cast(info[1]); 16 | std::string language_function_name(*String::Utf8Value(js_language_function_name)); 17 | std::string lib_file_name(*String::Utf8Value(js_lib_file_name)); 18 | 19 | uv_lib_t parser_lib; 20 | int error_code = uv_dlopen(lib_file_name.c_str(), &parser_lib); 21 | if (error_code) { 22 | std::string message(uv_dlerror(&parser_lib)); 23 | Nan::ThrowError(("Couldn't open language file - " + message).c_str()); 24 | return; 25 | } 26 | 27 | const TSLanguage * (* language_fn)() = NULL; 28 | error_code = uv_dlsym(&parser_lib, language_function_name.c_str(), (void **)&language_fn); 29 | if (error_code) { 30 | std::string message(uv_dlerror(&parser_lib)); 31 | Nan::ThrowError(("Couldn't load language function - " + message).c_str()); 32 | return; 33 | } 34 | 35 | if (!language_fn) { 36 | Nan::ThrowError("Could not load language"); 37 | return; 38 | } 39 | 40 | Local instance = Nan::New(constructor)->NewInstance(Nan::GetCurrentContext()).ToLocalChecked(); 41 | Nan::SetInternalFieldPointer(instance, 0, (void *)language_fn()); 42 | info.GetReturnValue().Set(instance); 43 | } 44 | 45 | NAN_METHOD(NewLanguage) { 46 | info.GetReturnValue().Set(Nan::Null()); 47 | } 48 | 49 | void InitLanguage(v8::Handle exports) { 50 | Local tpl = Nan::New(NewLanguage); 51 | tpl->SetClassName(Nan::New("DynamicallyLoadedLanguage").ToLocalChecked()); 52 | tpl->InstanceTemplate()->SetInternalFieldCount(1); 53 | constructor.Reset(tpl->GetFunction()); 54 | } 55 | 56 | } // namespace node_tree_sitter_cli 57 | -------------------------------------------------------------------------------- /src/language.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_LOAD_LANGUAGE_H_ 2 | #define TREE_SITTER_LOAD_LANGUAGE_H_ 3 | 4 | #include 5 | #include "nan.h" 6 | 7 | namespace node_tree_sitter_cli { 8 | 9 | void LoadLanguage(const Nan::FunctionCallbackInfo &); 10 | void InitLanguage(v8::Handle exports); 11 | 12 | } // namespace node_tree_sitter_cli 13 | 14 | #endif // TREE_SITTER_LOAD_LANGUAGE_H_ 15 | -------------------------------------------------------------------------------- /src/rule_builder.cc: -------------------------------------------------------------------------------- 1 | #include "./rule_builder.h" 2 | #include 3 | #include 4 | #include "nan.h" 5 | 6 | namespace node_tree_sitter_cli { 7 | namespace rule_builder { 8 | 9 | using namespace v8; 10 | 11 | Nan::Persistent constructor; 12 | 13 | Local build_symbol(Local name) { 14 | auto result = Nan::New(); 15 | result->Set(Nan::New("type").ToLocalChecked(), Nan::New("SYMBOL").ToLocalChecked()); 16 | result->Set(Nan::New("name").ToLocalChecked(), name); 17 | return result; 18 | } 19 | 20 | static void GetProperty(Local property, const Nan::PropertyCallbackInfo &info) { 21 | Local rules = info.This()->GetInternalField(0); 22 | Local symbol = build_symbol(property); 23 | 24 | if (rules->IsObject()) { 25 | Local rules_object = Local::Cast(rules); 26 | if (!rules_object->HasRealNamedProperty(property)) { 27 | Nan::Utf8String property_name(property); 28 | std::string error_message = std::string("Undefined rule '") + *property_name + "'"; 29 | Local error; 30 | if (Nan::To(Nan::ReferenceError(error_message.c_str())).ToLocal(&error)) { 31 | error->Set(Nan::New("symbol").ToLocalChecked(), symbol); 32 | info.GetReturnValue().Set(error); 33 | } 34 | return; 35 | } 36 | } 37 | 38 | info.GetReturnValue().Set(symbol); 39 | } 40 | 41 | static void construct(const Nan::FunctionCallbackInfo &info) { 42 | Local data = Nan::Null(); 43 | if (info.Length() == 1 && info[0]->IsObject()) { 44 | data = info[0]; 45 | } 46 | info.This()->SetInternalField(0, data); 47 | } 48 | 49 | void Init(Handle exports) { 50 | Local tpl = Nan::New(construct); 51 | tpl->SetClassName(Nan::New("RuleBuilder").ToLocalChecked()); 52 | Nan::SetNamedPropertyHandler(tpl->InstanceTemplate(), GetProperty); 53 | tpl->InstanceTemplate()->SetInternalFieldCount(1); 54 | constructor.Reset(tpl->GetFunction()); 55 | exports->Set(Nan::New("RuleBuilder").ToLocalChecked(), Nan::New(constructor)); 56 | } 57 | 58 | } // namespace rule_builder 59 | } // namespace node_tree_sitter_cli 60 | -------------------------------------------------------------------------------- /src/rule_builder.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_RULE_BUILDER_H 2 | #define TREE_SITTER_RULE_BUILDER_H 3 | 4 | #include 5 | #include "nan.h" 6 | 7 | namespace node_tree_sitter_cli { 8 | namespace rule_builder { 9 | 10 | void Init(v8::Handle exports); 11 | 12 | } // namespace rule_builder 13 | } // namespace node_tree_sitter_cli 14 | 15 | #endif // TREE_SITTER_RULE_BUILDER_H 16 | -------------------------------------------------------------------------------- /test/fixtures/arithmetic_language.js: -------------------------------------------------------------------------------- 1 | const { dsl, generate, loadLanguage } = require("../.."); 2 | const { grammar, seq, choice, prec, repeat, token } = dsl; 3 | 4 | module.exports = loadLanguage(generate(grammar({ 5 | name: "arithmetic", 6 | 7 | rules: { 8 | program: $ => $._expression, 9 | 10 | _expression: $ => 11 | choice( 12 | $.sum, 13 | $.difference, 14 | $.product, 15 | $.quotient, 16 | $.number, 17 | $.variable 18 | ), 19 | 20 | sum: $ => prec.left(0, seq($._expression, "+", $._expression)), 21 | 22 | difference: $ => prec.left(0, seq($._expression, "-", $._expression)), 23 | 24 | product: $ => prec.left(1, seq($._expression, "*", $._expression)), 25 | 26 | quotient: $ => prec.left(1, seq($._expression, "/", $._expression)), 27 | 28 | number: $ => /\d+/, 29 | 30 | variable: $ => token(seq( 31 | choice(/[a-zα-ω]/, '👍', '👎'), 32 | repeat(choice(/[a-zα-ω0-9]/, '👍', '👎')), 33 | )) 34 | } 35 | }))); 36 | -------------------------------------------------------------------------------- /test/fixtures/external_scan.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | enum { 4 | EXTERNAL_A, 5 | EXTERNAL_B 6 | }; 7 | 8 | void *tree_sitter_test_grammar_external_scanner_create() { 9 | return NULL; 10 | } 11 | 12 | void tree_sitter_test_grammar_external_scanner_destroy(void *payload) { 13 | } 14 | 15 | bool tree_sitter_test_grammar_external_scanner_scan(void *payload, TSLexer *lexer, const bool *valid_symbols) { 16 | while (lexer->lookahead == ' ') { 17 | lexer->advance(lexer, true); 18 | } 19 | 20 | if (lexer->lookahead == 'a') { 21 | lexer->advance(lexer, false); 22 | lexer->result_symbol = EXTERNAL_A; 23 | return true; 24 | } 25 | 26 | if (lexer->lookahead == 'b') { 27 | lexer->advance(lexer, false); 28 | lexer->result_symbol = EXTERNAL_B; 29 | return true; 30 | } 31 | 32 | return false; 33 | } 34 | 35 | void tree_sitter_test_grammar_external_scanner_reset(void *payload) { 36 | } 37 | 38 | unsigned tree_sitter_test_grammar_external_scanner_serialize( 39 | void *payload, 40 | char *buffer 41 | ) { return 0; } 42 | 43 | void tree_sitter_test_grammar_external_scanner_deserialize( 44 | void *payload, 45 | const char *buffer, 46 | unsigned length) {} 47 | -------------------------------------------------------------------------------- /test/fixtures/import.css: -------------------------------------------------------------------------------- 1 | @schema "./schema.json"; 2 | 3 | foo { number: two; } 4 | -------------------------------------------------------------------------------- /test/fixtures/schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "number": { 3 | "type": "string", 4 | "enum": [ 5 | "one", 6 | "two" 7 | ] 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /test/grammar_test.js: -------------------------------------------------------------------------------- 1 | const Parser = require("tree-sitter"); 2 | const path = require("path"); 3 | const { assert } = require("chai"); 4 | const { dsl, generate, loadLanguage } = require(".."); 5 | const { alias, blank, choice, prec, repeat, seq, sym, grammar } = dsl; 6 | const jsonSchema = require("jsonschema"); 7 | const GRAMMAR_SCHEMA = require("../vendor/tree-sitter/src/compiler/grammar-schema"); 8 | 9 | const schemaValidator = new jsonSchema.Validator(); 10 | 11 | describe("Writing a grammar", () => { 12 | let parser, tree; 13 | 14 | beforeEach(() => { 15 | parser = new Parser(); 16 | }); 17 | 18 | describe("rules", () => { 19 | describe("blank", () => { 20 | it("matches the empty string", () => { 21 | const language = generateAndLoadLanguage( 22 | grammar({ 23 | name: "test_grammar", 24 | rules: { 25 | the_rule: $ => blank() 26 | } 27 | }) 28 | ); 29 | 30 | parser.setLanguage(language); 31 | 32 | tree = parser.parse(""); 33 | assert.equal(tree.rootNode.toString(), "(the_rule)"); 34 | 35 | tree = parser.parse("not-blank"); 36 | assert.equal( 37 | tree.rootNode.toString(), 38 | "(the_rule (ERROR (UNEXPECTED 'n')))" 39 | ); 40 | }); 41 | }); 42 | 43 | describe("string", () => { 44 | it("matches one particular string", () => { 45 | const language = generateAndLoadLanguage( 46 | grammar({ 47 | name: "test_grammar", 48 | rules: { 49 | the_rule: $ => "the-string" 50 | } 51 | }) 52 | ); 53 | 54 | parser.setLanguage(language); 55 | 56 | tree = parser.parse("the-string"); 57 | assert.equal(tree.rootNode.toString(), "(the_rule)"); 58 | 59 | tree = parser.parse("another-string"); 60 | assert.equal(tree.rootNode.toString(), "(ERROR (UNEXPECTED 'a'))"); 61 | }); 62 | }); 63 | 64 | describe("regex", () => { 65 | it("matches according to a regular expression", () => { 66 | const language = generateAndLoadLanguage( 67 | grammar({ 68 | name: "test_grammar", 69 | rules: { 70 | the_rule: $ => /[a-c]+/ 71 | } 72 | }) 73 | ); 74 | 75 | parser.setLanguage(language); 76 | 77 | tree = parser.parse("abcba"); 78 | assert.equal(tree.rootNode.toString(), "(the_rule)"); 79 | 80 | tree = parser.parse("def"); 81 | assert.equal(tree.rootNode.toString(), "(ERROR (UNEXPECTED 'd'))"); 82 | }); 83 | 84 | it("handles unicode escape sequences in regular expressions", () => { 85 | const language = generateAndLoadLanguage( 86 | grammar({ 87 | name: "test_grammar", 88 | rules: { 89 | the_rule: $ => choice(/\u09afb/, /\u19afc/) 90 | } 91 | }) 92 | ); 93 | 94 | parser.setLanguage(language); 95 | 96 | tree = parser.parse("\u09af" + "b"); 97 | assert.equal(tree.rootNode.toString(), "(the_rule)"); 98 | 99 | tree = parser.parse("\u09af" + "c"); 100 | assert.equal(tree.rootNode.toString(), "(ERROR (UNEXPECTED 'c'))"); 101 | 102 | tree = parser.parse("\u19af" + "c"); 103 | assert.equal(tree.rootNode.toString(), "(the_rule)"); 104 | }); 105 | }); 106 | 107 | describe("repeat", () => { 108 | it("applies the given rule any number of times", () => { 109 | const language = generateAndLoadLanguage( 110 | grammar({ 111 | name: "test_grammar", 112 | rules: { 113 | the_rule: $ => repeat("o") 114 | } 115 | }) 116 | ); 117 | 118 | parser.setLanguage(language); 119 | 120 | tree = parser.parse(""); 121 | assert.equal(tree.rootNode.toString(), "(the_rule)"); 122 | 123 | tree = parser.parse("o"); 124 | assert.equal(tree.rootNode.toString(), "(the_rule)"); 125 | 126 | tree = parser.parse("ooo"); 127 | assert.equal(tree.rootNode.toString(), "(the_rule)"); 128 | }); 129 | }); 130 | 131 | describe("sequence", () => { 132 | it("applies a list of other rules in sequence", () => { 133 | const language = generateAndLoadLanguage( 134 | grammar({ 135 | name: "test_grammar", 136 | rules: { 137 | the_rule: $ => seq("1", "2", "3") 138 | } 139 | }) 140 | ); 141 | 142 | parser.setLanguage(language); 143 | 144 | tree = parser.parse("123"); 145 | assert.equal(tree.rootNode.toString(), "(the_rule)"); 146 | 147 | tree = parser.parse("12"); 148 | assert.equal(tree.rootNode.toString(), "(the_rule (MISSING))"); 149 | 150 | tree = parser.parse("1234"); 151 | assert.equal( 152 | tree.rootNode.toString(), 153 | "(the_rule (ERROR (UNEXPECTED '4')))" 154 | ); 155 | }); 156 | }); 157 | 158 | describe("choice", () => { 159 | it("applies any of a list of rules", () => { 160 | const language = generateAndLoadLanguage( 161 | grammar({ 162 | name: "test_grammar", 163 | rules: { 164 | the_rule: $ => choice("1", "2", "3") 165 | } 166 | }) 167 | ); 168 | 169 | parser.setLanguage(language); 170 | 171 | tree = parser.parse("1"); 172 | assert.equal(tree.rootNode.toString(), "(the_rule)"); 173 | 174 | tree = parser.parse("4"); 175 | assert.equal(tree.rootNode.toString(), "(ERROR (UNEXPECTED '4'))"); 176 | }); 177 | }); 178 | 179 | describe("symbol", () => { 180 | it("applies another rule in the grammar by name", () => { 181 | const language = generateAndLoadLanguage( 182 | grammar({ 183 | name: "test_grammar", 184 | rules: { 185 | the_rule: $ => seq($.second_rule, "-", $.third_rule), 186 | second_rule: $ => "one", 187 | third_rule: $ => "two" 188 | } 189 | }) 190 | ); 191 | 192 | parser.setLanguage(language); 193 | 194 | tree = parser.parse("one-two"); 195 | assert.equal( 196 | tree.rootNode.toString(), 197 | "(the_rule (second_rule) (third_rule))" 198 | ); 199 | }); 200 | }); 201 | 202 | describe("prec", () => { 203 | it("alters the precedence and associativity of the given rule", () => { 204 | const language = generateAndLoadLanguage( 205 | grammar({ 206 | name: "test_grammar", 207 | rules: { 208 | expression: $ => $._expression, 209 | _expression: $ => 210 | choice($.sum, $.product, $.equation, $.variable), 211 | product: $ => 212 | prec.left(2, seq($._expression, "*", $._expression)), 213 | sum: $ => prec.left(1, seq($._expression, "+", $._expression)), 214 | equation: $ => 215 | prec.right(0, seq($._expression, "=", $._expression)), 216 | variable: $ => /[a-z]+/ 217 | } 218 | }) 219 | ); 220 | 221 | parser.setLanguage(language); 222 | 223 | // product has higher precedence than sum 224 | tree = parser.parse("a + b * c"); 225 | assert.equal( 226 | tree.rootNode.toString(), 227 | "(expression (sum (variable) (product (variable) (variable))))" 228 | ); 229 | tree = parser.parse("a * b + c"); 230 | assert.equal( 231 | tree.rootNode.toString(), 232 | "(expression (sum (product (variable) (variable)) (variable)))" 233 | ); 234 | 235 | // product and sum are left-associative 236 | tree = parser.parse("a * b * c"); 237 | assert.equal( 238 | tree.rootNode.toString(), 239 | "(expression (product (product (variable) (variable)) (variable)))" 240 | ); 241 | tree = parser.parse("a + b + c"); 242 | assert.equal( 243 | tree.rootNode.toString(), 244 | "(expression (sum (sum (variable) (variable)) (variable)))" 245 | ); 246 | 247 | // equation is right-associative 248 | tree = parser.parse("a = b = c"); 249 | assert.equal( 250 | tree.rootNode.toString(), 251 | "(expression (equation (variable) (equation (variable) (variable))))" 252 | ); 253 | }); 254 | }); 255 | 256 | describe("alias", () => { 257 | it("assigns syntax nodes matched by the given rule an alternative name", () => { 258 | const language = generateAndLoadLanguage( 259 | grammar({ 260 | name: "test_grammar", 261 | rules: { 262 | rule_1: $ => seq(alias($.rule_2, $.rule_2000), "\n", $.rule_3), 263 | rule_2: $ => "#two", 264 | rule_3: $ => alias(/#[ \t]*three/, "#three") 265 | } 266 | }) 267 | ); 268 | 269 | parser.setLanguage(language); 270 | tree = parser.parse("#two\n# three"); 271 | assert.equal(tree.rootNode.toString(), "(rule_1 (rule_2000) (rule_3))"); 272 | 273 | const rule2Node = tree.rootNode.namedChildren[0]; 274 | assert.equal(rule2Node.namedChildren.length, 0); 275 | assert.equal(rule2Node.children.length, 0); 276 | 277 | const rule3Node = tree.rootNode.namedChildren[1]; 278 | assert.equal(rule3Node.namedChildren.length, 0); 279 | assert.deepEqual(rule3Node.children.map(node => node.type), ["#three"]); 280 | assert.deepEqual(rule3Node.children.map(node => node.isNamed), [false]); 281 | }); 282 | }); 283 | }); 284 | 285 | describe("inlining rules", () => { 286 | it("duplicates the content of the specified rules at all of their usage sites", () => { 287 | const language = generateAndLoadLanguage( 288 | grammar({ 289 | name: "test_grammar", 290 | inline: $ => [$.rule_c], 291 | rules: { 292 | rule_a: $ => seq($.rule_b, $.rule_c), 293 | rule_b: $ => "b", 294 | rule_c: $ => seq($.rule_d, $.rule_e), 295 | rule_d: $ => "d", 296 | rule_e: $ => "e" 297 | } 298 | }) 299 | ); 300 | 301 | parser.setLanguage(language); 302 | 303 | tree = parser.parse("b d e"); 304 | assert.equal( 305 | tree.rootNode.toString(), 306 | "(rule_a (rule_b) (rule_d) (rule_e))" 307 | ); 308 | }); 309 | }); 310 | 311 | describe("extra tokens", () => { 312 | it("allows the given tokens to appear anywhere in the input", () => { 313 | const language = generateAndLoadLanguage( 314 | grammar({ 315 | name: "test_grammar", 316 | extras: $ => [$.ellipsis, " "], 317 | rules: { 318 | the_rule: $ => repeat($.word), 319 | word: $ => /\w+/, 320 | ellipsis: $ => "..." 321 | } 322 | }) 323 | ); 324 | 325 | parser.setLanguage(language); 326 | 327 | tree = parser.parse("one two ... three ... four"); 328 | 329 | assert.equal( 330 | tree.rootNode.toString(), 331 | "(the_rule (word) (word) (ellipsis) (word) (ellipsis) (word))" 332 | ); 333 | }); 334 | 335 | it("allows anonymous rules to be provided", () => { 336 | const language = generateAndLoadLanguage( 337 | grammar({ 338 | name: "test_grammar", 339 | extras: $ => ["...", "---"], 340 | rules: { 341 | the_rule: $ => repeat($.word), 342 | word: $ => "hello" 343 | } 344 | }) 345 | ); 346 | 347 | parser.setLanguage(language); 348 | 349 | tree = parser.parse("hello...hello---hello"); 350 | assert.equal(tree.rootNode.toString(), "(the_rule (word) (word) (word))"); 351 | 352 | tree = parser.parse("hello hello"); 353 | assert.equal( 354 | tree.rootNode.toString(), 355 | "(the_rule (word) (ERROR (UNEXPECTED ' ')) (word))" 356 | ); 357 | }); 358 | 359 | it("defaults to whitespace characters", () => { 360 | const language = generateAndLoadLanguage( 361 | grammar({ 362 | name: "test_grammar", 363 | rules: { 364 | the_rule: $ => repeat($.word), 365 | word: $ => "hello" 366 | } 367 | }) 368 | ); 369 | 370 | parser.setLanguage(language); 371 | 372 | tree = parser.parse("hello hello\thello\nhello\rhello"); 373 | assert.equal( 374 | tree.rootNode.toString(), 375 | "(the_rule (word) (word) (word) (word) (word))" 376 | ); 377 | 378 | tree = parser.parse("hello.hello"); 379 | assert.equal( 380 | tree.rootNode.toString(), 381 | "(the_rule (word) (ERROR (UNEXPECTED '.')) (word))" 382 | ); 383 | }); 384 | }); 385 | 386 | describe("expected conflicts", () => { 387 | it("causes the grammar to work even with LR(1) conflicts", () => { 388 | let grammarOptions = { 389 | name: "test_grammar", 390 | rules: { 391 | sentence: $ => 392 | choice(seq($.first_rule, "c", "d"), seq($.second_rule, "c", "e")), 393 | 394 | first_rule: $ => seq("a", "b"), 395 | 396 | second_rule: $ => seq("a", "b") 397 | } 398 | }; 399 | 400 | let threw = false; 401 | 402 | try { 403 | generate(grammar(grammarOptions)); 404 | } catch (e) { 405 | assert.match(e.message, /Unresolved conflict/); 406 | threw = true; 407 | } 408 | 409 | assert.ok(threw, "Expected a conflict exception"); 410 | 411 | grammarOptions.conflicts = $ => [[$.first_rule, $.second_rule]]; 412 | 413 | const language = generateAndLoadLanguage(grammar(grammarOptions)); 414 | parser.setLanguage(language); 415 | 416 | tree = parser.parse("a b c d"); 417 | assert.equal("(sentence (first_rule))", tree.rootNode.toString()); 418 | 419 | tree = parser.parse("a b c e"); 420 | assert.equal("(sentence (second_rule))", tree.rootNode.toString()); 421 | }); 422 | 423 | it("allows ambiguities to be resolved via dynamic precedence", () => { 424 | let callPrecedence = -1; 425 | 426 | const grammarOptions = { 427 | name: "test_grammar", 428 | conflicts: $ => [[$.expression, $.command], [$.call, $.command]], 429 | rules: { 430 | expression: $ => 431 | choice($.call, $.command, $.parenthesized, $.identifier), 432 | 433 | call: $ => 434 | prec.dynamic( 435 | callPrecedence, 436 | seq( 437 | $.expression, 438 | "(", 439 | $.expression, 440 | repeat(seq(",", $.expression)), 441 | ")" 442 | ) 443 | ), 444 | 445 | command: $ => seq($.identifier, $.expression), 446 | 447 | parenthesized: $ => seq("(", $.expression, ")"), 448 | 449 | identifier: $ => /[a-z]+/ 450 | } 451 | }; 452 | 453 | parser.setLanguage(generateAndLoadLanguage(grammar(grammarOptions))); 454 | tree = parser.parse("a(b)"); 455 | assert.equal( 456 | tree.rootNode.toString(), 457 | "(expression (command (identifier) (expression (parenthesized (expression (identifier))))))" 458 | ); 459 | 460 | callPrecedence = 1; 461 | parser.setLanguage(generateAndLoadLanguage(grammar(grammarOptions))); 462 | tree = parser.parse("a(b)"); 463 | assert.equal( 464 | tree.rootNode.toString(), 465 | "(expression (call (expression (identifier)) (expression (identifier))))" 466 | ); 467 | }); 468 | }); 469 | 470 | describe("external tokens", function() { 471 | it("causes the grammar to work even with LR(1) conflicts", () => { 472 | const language = generateAndLoadLanguage( 473 | grammar({ 474 | name: "test_grammar", 475 | externals: $ => [$.external_a, $.external_b], 476 | rules: { 477 | program: $ => seq($.external_a, $.external_b) 478 | } 479 | }), 480 | [path.join(__dirname, "fixtures", "external_scan.c")] 481 | ); 482 | 483 | parser.setLanguage(language); 484 | tree = parser.parse("a b"); 485 | assert.equal( 486 | tree.rootNode.toString(), 487 | "(program (external_a) (external_b))" 488 | ); 489 | }); 490 | }); 491 | 492 | describe("extending another grammar", () => { 493 | it("allows rules, extras, and conflicts to be added", () => { 494 | let grammar1 = grammar({ 495 | name: "grammar1", 496 | extras: $ => [" "], 497 | rules: { 498 | thing: $ => repeat($.triple), 499 | triple: $ => seq($.word, $.word, $.word), 500 | word: $ => /\w+/ 501 | } 502 | }); 503 | 504 | parser.setLanguage(generateAndLoadLanguage(grammar1)); 505 | tree = parser.parse("one two three"); 506 | assert.equal( 507 | tree.rootNode.toString(), 508 | "(thing (triple (word) (word) (word)))" 509 | ); 510 | 511 | tree = parser.parse("one two ... three"); 512 | assert.equal( 513 | tree.rootNode.toString(), 514 | "(thing (triple (word) (word) (ERROR (UNEXPECTED '.')) (word)))" 515 | ); 516 | 517 | let grammar2 = grammar(grammar1, { 518 | name: "grammar2", 519 | extras: ($, original) => original.concat([$.ellipsis]), 520 | rules: { 521 | ellipsis: $ => "..." 522 | } 523 | }); 524 | 525 | parser.setLanguage(generateAndLoadLanguage(grammar2)); 526 | tree = parser.parse("one two ... three"); 527 | assert.equal( 528 | tree.rootNode.toString(), 529 | "(thing (triple (word) (word) (ellipsis) (word)))" 530 | ); 531 | 532 | tree = parser.parse("one two ... three ... four"); 533 | assert.equal( 534 | tree.rootNode.toString(), 535 | "(thing (triple (word) (word) (ellipsis) (word)) (ellipsis) (ERROR (word)))" 536 | ); 537 | 538 | let grammar3 = grammar(grammar2, { 539 | name: "grammar3", 540 | conflicts: $ => [[$.triple, $.double]], 541 | rules: { 542 | thing: ($, original) => choice(original, repeat($.double)), 543 | double: $ => seq($.word, $.word) 544 | } 545 | }); 546 | 547 | parser.setLanguage(generateAndLoadLanguage(grammar3)); 548 | tree = parser.parse("one two ... three ... four"); 549 | assert.equal( 550 | tree.rootNode.toString(), 551 | "(thing (double (word) (word)) (ellipsis) (double (word) (ellipsis) (word)))" 552 | ); 553 | }); 554 | 555 | it("allows inlines to be added", () => { 556 | const grammar1 = grammar({ 557 | name: "grammar1", 558 | 559 | inline: $ => [$.something], 560 | 561 | rules: { 562 | statement: $ => seq($.something, ";"), 563 | something: $ => $.expression, 564 | expression: $ => choice($.property, $.call, $.identifier), 565 | property: $ => seq($.expression, ".", $.identifier), 566 | call: $ => seq($.expression, "(", $.expression, ")"), 567 | identifier: $ => /[a-z]+/ 568 | } 569 | }); 570 | 571 | parser.setLanguage(generateAndLoadLanguage(grammar1)); 572 | tree = parser.parse("a.b(c);"); 573 | assert.equal( 574 | tree.rootNode.toString(), 575 | "(statement (expression (call (expression (property (expression (identifier)) (identifier))) (expression (identifier)))))" 576 | ); 577 | 578 | const grammar2 = grammar(grammar1, { 579 | name: "grammar2", 580 | 581 | inline: ($, original) => original.concat([$.expression]) 582 | }); 583 | 584 | parser.setLanguage(generateAndLoadLanguage(grammar2)); 585 | tree = parser.parse("a.b(c);"); 586 | assert.equal( 587 | tree.rootNode.toString(), 588 | "(statement (call (property (identifier) (identifier)) (identifier)))" 589 | ); 590 | }); 591 | }); 592 | 593 | describe("error handling", () => { 594 | describe("when the grammar has conflicts", () => { 595 | it("raises an error describing the conflict", () => { 596 | let threw = false; 597 | 598 | try { 599 | generate( 600 | grammar({ 601 | name: "test_grammar", 602 | rules: { 603 | sentence: $ => choice($.first_rule, $.second_rule), 604 | first_rule: $ => seq("things", "stuff"), 605 | second_rule: $ => seq("things", "stuff") 606 | } 607 | }) 608 | ); 609 | } catch (e) { 610 | assert.match(e.message, /Unresolved conflict /); 611 | assert.match(e.message, /first_rule/); 612 | assert.match(e.message, /second_rule/); 613 | assert.property(e, "isGrammarError"); 614 | threw = true; 615 | } 616 | 617 | assert.ok(threw, "Expected an exception!"); 618 | }); 619 | }); 620 | 621 | describe("when the grammar has no name", () => { 622 | it("raises an error", () => { 623 | assert.throws( 624 | () => 625 | grammar({ 626 | rules: { 627 | the_rule: $ => blank() 628 | } 629 | }), 630 | /Grammar.*name.*string/ 631 | ); 632 | 633 | assert.throws( 634 | () => 635 | grammar({ 636 | name: {}, 637 | rules: { 638 | the_rule: $ => blank() 639 | } 640 | }), 641 | /Grammar.*name.*string/ 642 | ); 643 | }); 644 | }); 645 | 646 | describe("when the grammar has an invalid name", () => { 647 | it("raises an error describing valid names", () => { 648 | assert.throws( 649 | () => 650 | grammar({ 651 | name: "hyphen-ated", 652 | rules: { 653 | the_rule: $ => blank() 654 | } 655 | }), 656 | /Grammar.*name.*word/ 657 | ); 658 | 659 | assert.throws( 660 | () => 661 | grammar({ 662 | name: "space ", 663 | rules: { 664 | the_rule: $ => blank() 665 | } 666 | }), 667 | /Grammar.*name.*word/ 668 | ); 669 | 670 | assert.doesNotThrow( 671 | () => 672 | grammar({ 673 | name: "a_A_8", 674 | rules: { 675 | the_rule: $ => blank() 676 | } 677 | }) 678 | ); 679 | }); 680 | }); 681 | 682 | describe("when the grammar has no rules", () => { 683 | it("raises an error", () => { 684 | assert.throws( 685 | () => 686 | grammar({ 687 | name: "test_grammar" 688 | }), 689 | /Grammar.*must have.*rule/ 690 | ); 691 | }); 692 | }); 693 | 694 | describe("when the grammar contains a reference to an undefined rule", () => { 695 | it("throws an error with the rule name", () => { 696 | assert.throws( 697 | () => 698 | grammar({ 699 | name: "test_grammar", 700 | rules: { 701 | something: $ => seq("(", $.something_else, ")") 702 | } 703 | }), 704 | /Undefined.*rule.*something_else/ 705 | ); 706 | }); 707 | }); 708 | 709 | describe("when one of the grammar rules is not a function", () => { 710 | it("raises an error", () => { 711 | assert.throws( 712 | () => 713 | grammar({ 714 | name: "test_grammar", 715 | rules: { 716 | the_rule: blank() 717 | } 718 | }), 719 | /Grammar.*rule.*function.*the_rule/ 720 | ); 721 | }); 722 | }); 723 | 724 | describe("when the grammar's extras value is not a function", () => { 725 | it("raises an error", () => { 726 | assert.throws( 727 | () => 728 | grammar({ 729 | extras: [], 730 | name: "test_grammar", 731 | rules: { 732 | the_rule: $ => blank() 733 | } 734 | }), 735 | /Grammar.*extras.*function/ 736 | ); 737 | }); 738 | }); 739 | 740 | describe("when one of the grammar's extras tokens is not a token", () => { 741 | it("raises an error", () => { 742 | let threw = false; 743 | 744 | try { 745 | generate( 746 | grammar({ 747 | name: "test_grammar", 748 | extras: $ => [$.yyy], 749 | rules: { 750 | xxx: $ => seq($.yyy, $.yyy), 751 | yyy: $ => seq($.zzz, $.zzz), 752 | zzz: $ => "zzz" 753 | } 754 | }) 755 | ); 756 | } catch (e) { 757 | assert.match(e.message, /Non-token symbol yyy/); 758 | assert.property(e, "isGrammarError"); 759 | threw = true; 760 | } 761 | 762 | assert.ok(threw, "Expected an exception!"); 763 | }); 764 | }); 765 | 766 | describe("when a symbol references an undefined rule", () => { 767 | it("raises an error", () => { 768 | assert.throws( 769 | () => 770 | generate( 771 | grammar({ 772 | name: "test_grammar", 773 | rules: { 774 | xxx: $ => sym("yyy") 775 | } 776 | }) 777 | ), 778 | /Undefined.*rule.*yyy/ 779 | ); 780 | }); 781 | }); 782 | }); 783 | }); 784 | 785 | function generateAndLoadLanguage(grammar, ...args) { 786 | var validation = schemaValidator.validate(grammar, GRAMMAR_SCHEMA); 787 | if (!validation.valid) throw new Error(validation.errors[0]); 788 | const parserCode = generate(grammar); 789 | return loadLanguage(parserCode, ...args); 790 | } 791 | -------------------------------------------------------------------------------- /test/node_test.js: -------------------------------------------------------------------------------- 1 | const Parser = require("tree-sitter"); 2 | const { TextBuffer } = require("superstring"); 3 | const { assert } = require("chai"); 4 | const ARITHMETIC = require('./fixtures/arithmetic_language'); 5 | 6 | describe("Node", () => { 7 | let parser; 8 | 9 | beforeEach(() => { 10 | parser = new Parser().setLanguage(ARITHMETIC); 11 | }); 12 | 13 | describe(".children", () => { 14 | it("returns an array of child nodes", () => { 15 | const tree = parser.parse("x10 + 1000"); 16 | const sumNode = tree.rootNode.firstChild; 17 | assert.equal(1, tree.rootNode.children.length); 18 | assert.deepEqual( 19 | sumNode.children.map(child => child.type), 20 | ["variable", "+", "number"] 21 | ); 22 | }); 23 | }); 24 | 25 | describe(".namedChildren", () => { 26 | it("returns an array of named child nodes", () => { 27 | const tree = parser.parse("x10 + 1000"); 28 | const sumNode = tree.rootNode.firstChild; 29 | assert.equal(1, tree.rootNode.namedChildren.length); 30 | assert.deepEqual( 31 | ["variable", "number"], 32 | sumNode.namedChildren.map(child => child.type) 33 | ); 34 | }); 35 | }); 36 | 37 | describe(".startIndex and .endIndex", () => { 38 | it("returns the character index where the node starts/ends in the text", () => { 39 | const tree = parser.parse("a👍👎1 / b👎c👎"); 40 | const quotientNode = tree.rootNode.firstChild; 41 | 42 | assert.equal(0, quotientNode.startIndex); 43 | assert.equal(15, quotientNode.endIndex); 44 | assert.deepEqual( 45 | [0, 7, 9], 46 | quotientNode.children.map(child => child.startIndex) 47 | ); 48 | assert.deepEqual( 49 | [6, 8, 15], 50 | quotientNode.children.map(child => child.endIndex) 51 | ); 52 | }); 53 | }); 54 | 55 | describe(".startPosition and .endPosition", () => { 56 | it("returns the row and column where the node starts/ends in the text", () => { 57 | const tree = parser.parse("x10 + 1000"); 58 | const sumNode = tree.rootNode.firstChild; 59 | assert.equal("sum", sumNode.type); 60 | 61 | assert.deepEqual({ row: 0, column: 0 }, sumNode.startPosition); 62 | assert.deepEqual({ row: 0, column: 10 }, sumNode.endPosition); 63 | assert.deepEqual( 64 | [{ row: 0, column: 0 }, { row: 0, column: 4 }, { row: 0, column: 6 }], 65 | sumNode.children.map(child => child.startPosition) 66 | ); 67 | assert.deepEqual( 68 | [{ row: 0, column: 3 }, { row: 0, column: 5 }, { row: 0, column: 10 }], 69 | sumNode.children.map(child => child.endPosition) 70 | ); 71 | }); 72 | 73 | it("handles characters that occupy two UTF16 code units", () => { 74 | const tree = parser.parse("a👍👎1 /\n b👎c👎"); 75 | const sumNode = tree.rootNode.firstChild; 76 | assert.deepEqual( 77 | [ 78 | [{ row: 0, column: 0 }, { row: 0, column: 6 }], 79 | [{ row: 0, column: 7 }, { row: 0, column: 8 }], 80 | [{ row: 1, column: 1 }, { row: 1, column: 7 }] 81 | ], 82 | sumNode.children.map(child => [child.startPosition, child.endPosition]) 83 | ); 84 | }); 85 | }); 86 | 87 | describe(".parent", () => { 88 | it("returns the node's parent", () => { 89 | const tree = parser.parse("x10 + 1000"); 90 | const sumNode = tree.rootNode.firstChild; 91 | const variableNode = sumNode.firstChild; 92 | assert.equal(sumNode, variableNode.parent); 93 | assert.equal(tree.rootNode, sumNode.parent); 94 | }); 95 | }); 96 | 97 | describe('.child(), .firstChild, .lastChild', () => { 98 | it('returns null when the node has no children', () => { 99 | const tree = parser.parse("x10 + 1000"); 100 | const sumNode = tree.rootNode.firstChild; 101 | const variableNode = sumNode.firstChild; 102 | assert.equal(variableNode.firstChild, null); 103 | assert.equal(variableNode.lastChild, null); 104 | assert.equal(variableNode.firstNamedChild, null); 105 | assert.equal(variableNode.lastNamedChild, null); 106 | assert.equal(variableNode.child(1), null); 107 | }) 108 | }); 109 | 110 | describe(".nextSibling and .previousSibling", () => { 111 | it("returns the node's next and previous sibling", () => { 112 | const tree = parser.parse("x10 + 1000"); 113 | const sumNode = tree.rootNode.firstChild; 114 | assert.equal(sumNode.children[1], sumNode.children[0].nextSibling); 115 | assert.equal(sumNode.children[2], sumNode.children[1].nextSibling); 116 | assert.equal( 117 | sumNode.children[0], 118 | sumNode.children[1].previousSibling 119 | ); 120 | assert.equal( 121 | sumNode.children[1], 122 | sumNode.children[2].previousSibling 123 | ); 124 | }); 125 | }); 126 | 127 | describe(".nextNamedSibling and .previousNamedSibling", () => { 128 | it("returns the node's next and previous named sibling", () => { 129 | const tree = parser.parse("x10 + 1000"); 130 | const sumNode = tree.rootNode.firstChild; 131 | assert.equal( 132 | sumNode.namedChildren[1], 133 | sumNode.namedChildren[0].nextNamedSibling 134 | ); 135 | assert.equal( 136 | sumNode.namedChildren[0], 137 | sumNode.namedChildren[1].previousNamedSibling 138 | ); 139 | }); 140 | }); 141 | 142 | describe(".descendantForIndex(min, max)", () => { 143 | it("returns the smallest node that spans the given range", () => { 144 | const tree = parser.parse("x10 + 1000"); 145 | const sumNode = tree.rootNode.firstChild; 146 | assert.equal("variable", sumNode.descendantForIndex(1, 2).type); 147 | assert.equal("+", sumNode.descendantForIndex(4, 4).type); 148 | 149 | assert.throws(() => { 150 | sumNode.descendantForIndex(1, {}); 151 | }, /Character index must be a number/); 152 | 153 | assert.throws(() => { 154 | sumNode.descendantForIndex(); 155 | }, /Character index must be a number/); 156 | }); 157 | }); 158 | 159 | describe(".namedDescendantForIndex", () => { 160 | it("returns the smallest node that spans the given range", () => { 161 | const tree = parser.parse("x10 + 1000"); 162 | const sumNode = tree.rootNode.firstChild; 163 | assert.equal("variable", sumNode.descendantForIndex(1, 2).type); 164 | assert.equal("+", sumNode.descendantForIndex(4, 4).type); 165 | }); 166 | }); 167 | 168 | describe(".descendantForPosition(min, max)", () => { 169 | it("returns the smallest node that spans the given range", () => { 170 | const tree = parser.parse("x10 + 1000"); 171 | const sumNode = tree.rootNode.firstChild; 172 | 173 | assert.equal( 174 | "variable", 175 | sumNode.descendantForPosition( 176 | { row: 0, column: 1 }, 177 | { row: 0, column: 2 } 178 | ).type 179 | ); 180 | 181 | assert.equal( 182 | "+", 183 | sumNode.descendantForPosition({ row: 0, column: 4 }).type 184 | ); 185 | 186 | assert.throws(() => { 187 | sumNode.descendantForPosition(1, {}); 188 | }, /Point.row must be a number/); 189 | 190 | assert.throws(() => { 191 | sumNode.descendantForPosition(); 192 | }, /Point must be a .* object/); 193 | }); 194 | }); 195 | 196 | describe(".namedDescendantForPosition(min, max)", () => { 197 | it("returns the smallest named node that spans the given range", () => { 198 | const tree = parser.parse("x10 + 1000"); 199 | const sumNode = tree.rootNode.firstChild; 200 | 201 | assert.equal( 202 | "variable", 203 | sumNode.namedDescendantForPosition( 204 | { row: 0, column: 1 }, 205 | { row: 0, column: 2 } 206 | ).type 207 | ); 208 | 209 | assert.equal( 210 | "sum", 211 | sumNode.namedDescendantForPosition({ row: 0, column: 4 }).type 212 | ); 213 | }); 214 | }); 215 | 216 | describe('.descendantsOfType(type, min, max)', () => { 217 | it('finds all of the descendants of the given type in the given range', () => { 218 | const tree = parser.parse("a + 1 * b * 2 + c + 3"); 219 | const outerSum = tree.rootNode.firstChild; 220 | let descendants = outerSum.descendantsOfType('number', {row: 0, column: 2}, {row: 0, column: 15}) 221 | assert.deepEqual( 222 | descendants.map(node => node.startIndex), 223 | [4, 12] 224 | ); 225 | 226 | descendants = outerSum.descendantsOfType('variable', {row: 0, column: 2}, {row: 0, column: 15}) 227 | assert.deepEqual( 228 | descendants.map(node => node.startIndex), 229 | [8] 230 | ); 231 | 232 | descendants = outerSum.descendantsOfType('variable', {row: 0, column: 0}, {row: 0, column: 30}) 233 | assert.deepEqual( 234 | descendants.map(node => node.startIndex), 235 | [0, 8, 16] 236 | ); 237 | 238 | descendants = outerSum.descendantsOfType('number', {row: 0, column: 0}, {row: 0, column: 30}) 239 | assert.deepEqual( 240 | descendants.map(node => node.startIndex), 241 | [4, 12, 20] 242 | ); 243 | 244 | descendants = outerSum.descendantsOfType( 245 | ['variable', 'number'], 246 | {row: 0, column: 0}, 247 | {row: 0, column: 30} 248 | ) 249 | assert.deepEqual( 250 | descendants.map(node => node.startIndex), 251 | [0, 4, 8, 12, 16, 20] 252 | ); 253 | 254 | descendants = outerSum.descendantsOfType('number') 255 | assert.deepEqual( 256 | descendants.map(node => node.startIndex), 257 | [4, 12, 20] 258 | ); 259 | 260 | descendants = outerSum.firstChild.descendantsOfType('number', {row: 0, column: 0}, {row: 0, column: 30}) 261 | assert.deepEqual( 262 | descendants.map(node => node.startIndex), 263 | [4, 12] 264 | ); 265 | }) 266 | }); 267 | 268 | describe('.closest(type)', () => { 269 | it('returns the closest ancestor of the given type', () => { 270 | const tree = parser.parse("a + 1 * b * 2 + c + 3"); 271 | const number = tree.rootNode.descendantForIndex(4) 272 | 273 | const product = number.closest('product') 274 | assert.equal(product.type, 'product') 275 | assert.equal(product.startIndex, 4) 276 | assert.equal(product.endIndex, 9) 277 | 278 | const sum = number.closest(['sum', 'variable']) 279 | assert.equal(sum.type, 'sum') 280 | assert.equal(sum.startIndex, 0) 281 | assert.equal(sum.endIndex, 13) 282 | }); 283 | 284 | it('throws an exception when an invalid argument is given', () => { 285 | const tree = parser.parse("a + 1 * b * 2 + c + 3"); 286 | const number = tree.rootNode.descendantForIndex(4) 287 | 288 | assert.throws(() => number.closest({a: 1}), /Argument must be a string or array of strings/) 289 | }); 290 | }); 291 | 292 | describe(".firstChildForIndex(index)", () => { 293 | it("returns the first child that extends beyond the given index", () => { 294 | const tree = parser.parse("x10 + 1000"); 295 | const sumNode = tree.rootNode.firstChild; 296 | 297 | assert.equal("variable", sumNode.firstChildForIndex(0).type); 298 | assert.equal("variable", sumNode.firstChildForIndex(1).type); 299 | assert.equal("+", sumNode.firstChildForIndex(3).type); 300 | assert.equal("number", sumNode.firstChildForIndex(5).type); 301 | }); 302 | }); 303 | 304 | describe(".firstNamedChildForIndex(index)", () => { 305 | it("returns the first child that extends beyond the given index", () => { 306 | const tree = parser.parse("x10 + 1000"); 307 | const sumNode = tree.rootNode.firstChild; 308 | 309 | assert.equal("variable", sumNode.firstNamedChildForIndex(0).type); 310 | assert.equal("variable", sumNode.firstNamedChildForIndex(1).type); 311 | assert.equal("number", sumNode.firstNamedChildForIndex(3).type); 312 | }); 313 | }); 314 | 315 | describe(".hasError()", () => { 316 | it("returns true if the node contains an error", () => { 317 | const tree = parser.parse("1 + 2 * * 3"); 318 | const node = tree.rootNode; 319 | assert.equal( 320 | node.toString(), 321 | "(program (sum (number) (product (number) (ERROR) (number))))" 322 | ); 323 | 324 | const sum = node.firstChild; 325 | assert(sum.hasError()); 326 | assert(!sum.children[0].hasError()); 327 | assert(!sum.children[1].hasError()); 328 | assert(sum.children[2].hasError()); 329 | }); 330 | }); 331 | 332 | describe(".isMissing()", () => { 333 | it("returns true if the node is missing from the source and was inserted via error recovery", () => { 334 | const tree = parser.parse("2 +"); 335 | const node = tree.rootNode; 336 | assert.equal(node.toString(), "(program (sum (number) (MISSING)))"); 337 | 338 | const sum = node.firstChild; 339 | assert(sum.hasError()); 340 | assert(!sum.children[0].isMissing()); 341 | assert(!sum.children[1].isMissing()); 342 | assert(sum.children[2].isMissing()); 343 | }); 344 | }); 345 | 346 | describe(".text", () => { 347 | Object.entries({ 348 | '.parse(String)': (parser, src) => parser.parse(src), 349 | '.parse(Function)': (parser, src) => 350 | parser.parse(offset => src.substr(offset, 4)), 351 | '.parseTextBuffer': (parser, src) => 352 | parser.parseTextBuffer(new TextBuffer(src)), 353 | '.parseTextBufferSync': (parser, src) => 354 | parser.parseTextBufferSync(new TextBuffer(src)) 355 | }).forEach(([method, parse]) => 356 | it(`returns the text of a node generated by ${method}`, async () => { 357 | const src = "α0 / b👎c👎" 358 | const [numeratorSrc, denominatorSrc] = src.split(/\s*\/\s+/) 359 | const tree = await parse(parser, src) 360 | const quotientNode = tree.rootNode.firstChild; 361 | const [numerator, slash, denominator] = quotientNode.children; 362 | 363 | assert.equal(src, tree.rootNode.text, 'root node text'); 364 | assert.equal(denominatorSrc, denominator.text, 'denominator text'); 365 | assert.equal(src, quotientNode.text, 'quotient text'); 366 | assert.equal(numeratorSrc, numerator.text, 'numerator text'); 367 | assert.equal('/', slash.text, '"/" text'); 368 | }) 369 | ) 370 | }) 371 | }); 372 | -------------------------------------------------------------------------------- /test/parser_test.js: -------------------------------------------------------------------------------- 1 | const Parser = require("tree-sitter"); 2 | const { assert } = require("chai"); 3 | const { dsl, generate, loadLanguage } = require(".."); 4 | const { choice, prec, repeat, seq, grammar } = dsl; 5 | const {TextBuffer} = require('superstring'); 6 | const ARITHMETIC = require('./fixtures/arithmetic_language'); 7 | 8 | describe("Parser", () => { 9 | let parser, language; 10 | 11 | before(() => { 12 | language = loadLanguage( 13 | generate( 14 | grammar({ 15 | name: "test", 16 | rules: { 17 | sentence: $ => repeat(choice($.word1, $.word2, $.word3, $.word4)), 18 | word1: $ => "first-word", 19 | word2: $ => "second-word", 20 | word3: $ => "αβ", 21 | word4: $ => "αβδ" 22 | } 23 | }) 24 | ) 25 | ); 26 | }); 27 | 28 | beforeEach(() => { 29 | parser = new Parser(); 30 | }); 31 | 32 | describe(".setLanguage", () => { 33 | describe("when the supplied object is not a tree-sitter language", () => { 34 | it("throws an exception", () => { 35 | assert.throws(() => parser.setLanguage({}), /Invalid language/); 36 | 37 | assert.throws(() => parser.setLanguage(undefined), /Invalid language/); 38 | }); 39 | }); 40 | 41 | describe("when the input has not yet been set", () => { 42 | it("doesn't try to parse", () => { 43 | parser.setLanguage(language); 44 | assert.equal(null, parser.children); 45 | }); 46 | }); 47 | }); 48 | 49 | describe(".setLogger", () => { 50 | let debugMessages; 51 | 52 | beforeEach(() => { 53 | debugMessages = []; 54 | parser.setLanguage(language); 55 | parser.setLogger((message) => debugMessages.push(message)); 56 | }); 57 | 58 | it("calls the given callback for each parse event", () => { 59 | parser.parse("first-word second-word"); 60 | assert.includeMembers(debugMessages, ["reduce", "accept", "shift"]); 61 | }); 62 | 63 | it("allows the callback to be retrieved later", () => { 64 | let callback = () => null; 65 | 66 | parser.setLogger(callback); 67 | assert.equal(callback, parser.getLogger()); 68 | 69 | parser.setLogger(false); 70 | assert.equal(null, parser.getLogger()); 71 | }); 72 | 73 | describe("when given a falsy value", () => { 74 | beforeEach(() => { 75 | parser.setLogger(false); 76 | }); 77 | 78 | it("disables debugging", () => { 79 | parser.parse("first-word second-word"); 80 | assert.equal(0, debugMessages.length); 81 | }); 82 | }); 83 | 84 | describe("when given a truthy value that isn't a function", () => { 85 | it("raises an exception", () => { 86 | assert.throws( 87 | () => parser.setLogger("5"), 88 | /Logger callback must .* function .* falsy/ 89 | ); 90 | }); 91 | }); 92 | 93 | describe("when the given callback throws an exception", () => { 94 | let errorMessages, originalConsoleError, thrownError; 95 | 96 | beforeEach(() => { 97 | errorMessages = []; 98 | thrownError = new Error("dang."); 99 | 100 | originalConsoleError = console.error; 101 | console.error = (message, error) => { 102 | errorMessages.push([message, error]); 103 | }; 104 | 105 | parser.setLogger((msg, params) => { 106 | throw thrownError; 107 | }); 108 | }); 109 | 110 | afterEach(() => { 111 | console.error = originalConsoleError; 112 | }); 113 | 114 | it("logs the error to the console", () => { 115 | parser.parse("first-word"); 116 | 117 | assert.deepEqual(errorMessages[0], [ 118 | "Error in debug callback:", 119 | thrownError 120 | ]); 121 | }); 122 | }); 123 | }); 124 | 125 | describe(".parse", () => { 126 | beforeEach(() => { 127 | parser.setLanguage(language); 128 | }); 129 | 130 | it("reads from the given input", () => { 131 | parser.setLanguage(language); 132 | 133 | const parts = ["first", "-", "word", " ", "second", "-", "word", ""]; 134 | const tree = parser.parse(() => parts.shift()); 135 | 136 | assert.equal("(sentence (word1) (word2))", tree.rootNode.toString()); 137 | }); 138 | 139 | describe("when the input callback returns something other than a string", () => { 140 | it("stops reading", () => { 141 | parser.setLanguage(language); 142 | 143 | const parts = ["first", "-", "word", {}, "second-word", " "]; 144 | const tree = parser.parse(() => parts.shift()); 145 | 146 | assert.equal("(sentence (word1))", tree.rootNode.toString()); 147 | assert.equal(parts.length, 2); 148 | }); 149 | }); 150 | 151 | describe("when the given input is not a function", () => { 152 | it("throws an exception", () => { 153 | assert.throws(() => parser.parse(null), /Input.*function/); 154 | assert.throws(() => parser.parse(5), /Input.*function/); 155 | assert.throws(() => parser.parse({}), /Input.*function/); 156 | }); 157 | }); 158 | 159 | it("handles long input strings", () => { 160 | const repeatCount = 10000; 161 | const wordCount = 4 * repeatCount; 162 | const inputString = "first-word second-word αβ αβδ".repeat(repeatCount); 163 | 164 | const tree = parser.parse(inputString); 165 | assert.equal(tree.rootNode.type, "sentence"); 166 | assert.equal(tree.rootNode.childCount, wordCount); 167 | }); 168 | 169 | describe('when the `includedRanges` option is given', () => { 170 | it('parses the text within those ranges of the string', () => { 171 | const sourceCode = "const expression = `1 + a${c}b * 4`"; 172 | const exprStart = sourceCode.indexOf('1'); 173 | const interpStart = sourceCode.indexOf('${'); 174 | const interpEnd = sourceCode.indexOf('}') + 1; 175 | const exprEnd = sourceCode.lastIndexOf('`'); 176 | 177 | parser.setLanguage(ARITHMETIC); 178 | 179 | const tree = parser.parse(sourceCode, null, { 180 | includedRanges: [ 181 | { 182 | startIndex: exprStart, 183 | endIndex: interpStart, 184 | startPosition: {row: 0, column: exprStart}, 185 | endPosition: {row: 0, column: interpStart} 186 | }, 187 | { 188 | startIndex: interpEnd, 189 | endIndex: exprEnd, 190 | startPosition: {row: 0, column: interpEnd}, 191 | endPosition: {row: 0, column: exprEnd} 192 | }, 193 | ] 194 | }); 195 | 196 | assert.equal(tree.rootNode.toString(), '(program (sum (number) (product (variable) (number))))'); 197 | }) 198 | }) 199 | }); 200 | 201 | describe('.parseTextBuffer', () => { 202 | beforeEach(() => { 203 | parser.setLanguage(language); 204 | }); 205 | 206 | it('parses the contents of the given text buffer asynchronously', async () => { 207 | const repeatCount = 4; 208 | const wordCount = 4 * repeatCount; 209 | const repeatedString = "first-word second-word αβ αβδ "; 210 | const buffer = new TextBuffer(repeatedString.repeat(repeatCount)) 211 | 212 | const tree = await parser.parseTextBuffer(buffer); 213 | assert.equal(tree.rootNode.type, "sentence"); 214 | assert.equal(tree.rootNode.children.length, wordCount); 215 | 216 | const editIndex = repeatedString.length * 2; 217 | buffer.setTextInRange( 218 | {start: {row: 0, column: editIndex}, end: {row: 0, column: editIndex}}, 219 | 'αβδ ' 220 | ); 221 | tree.edit({ 222 | startIndex: editIndex, 223 | oldEndIndex: editIndex, 224 | newEndIndex: editIndex + 4, 225 | startPosition: {row: 0, column: editIndex}, 226 | oldEndPosition: {row: 0, column: editIndex}, 227 | newEndPosition: {row: 0, column: editIndex + 4} 228 | }); 229 | 230 | const newTree = await parser.parseTextBuffer(buffer, tree); 231 | assert.equal(newTree.rootNode.type, "sentence"); 232 | assert.equal(newTree.rootNode.children.length, wordCount + 1); 233 | }); 234 | 235 | it('does not allow the parser to be mutated while parsing', async () => { 236 | const buffer = new TextBuffer('first-word second-word first-word second-word'); 237 | const treePromise = parser.parseTextBuffer(buffer); 238 | 239 | assert.throws(() => { 240 | parser.parse('first-word'); 241 | }, /Parser is in use/); 242 | 243 | assert.throws(() => { 244 | parser.setLanguage(language); 245 | }, /Parser is in use/); 246 | 247 | assert.throws(() => { 248 | parser.printDotGraphs(true); 249 | }, /Parser is in use/); 250 | 251 | const tree = await treePromise; 252 | assert.equal(tree.rootNode.type, "sentence"); 253 | assert.equal(tree.rootNode.children.length, 4); 254 | 255 | parser.parse('first-word'); 256 | parser.setLanguage(language); 257 | parser.printDotGraphs(true); 258 | }); 259 | 260 | it('throws an error if the given object is not a TextBuffer', () => { 261 | assert.throws(() => { 262 | parser.parseTextBuffer({}); 263 | }); 264 | }); 265 | 266 | it('does not try to call JS logger functions when parsing asynchronously', async () => { 267 | const messages = []; 268 | parser.setLogger(message => messages.push(message)); 269 | 270 | const tree1 = parser.parse('first-word second-word'); 271 | assert(messages.length > 0); 272 | messages.length = 0; 273 | 274 | const buffer = new TextBuffer('first-word second-word'); 275 | const tree2 = await parser.parseTextBuffer(buffer); 276 | assert(messages.length === 0); 277 | 278 | const tree3 = parser.parseTextBufferSync(buffer); 279 | assert(messages.length > 0); 280 | 281 | assert.equal(tree2.rootNode.toString(), tree1.rootNode.toString()) 282 | assert.equal(tree3.rootNode.toString(), tree1.rootNode.toString()) 283 | }) 284 | 285 | describe('when the `includedRanges` option is given', () => { 286 | it('parses the text within those ranges of the string', async () => { 287 | const sourceCode = "const expression = `1 + a${c}b * 4`"; 288 | const exprStart = sourceCode.indexOf('1'); 289 | const interpStart = sourceCode.indexOf('${'); 290 | const interpEnd = sourceCode.indexOf('}') + 1; 291 | const exprEnd = sourceCode.lastIndexOf('`'); 292 | 293 | parser.setLanguage(ARITHMETIC); 294 | 295 | const tree = await parser.parseTextBuffer(new TextBuffer(sourceCode), null, { 296 | includedRanges: [ 297 | { 298 | startIndex: exprStart, 299 | endIndex: interpStart, 300 | startPosition: {row: 0, column: exprStart}, 301 | endPosition: {row: 0, column: interpStart} 302 | }, 303 | { 304 | startIndex: interpEnd, 305 | endIndex: exprEnd, 306 | startPosition: {row: 0, column: interpEnd}, 307 | endPosition: {row: 0, column: exprEnd} 308 | }, 309 | ] 310 | }); 311 | 312 | assert.equal(tree.rootNode.toString(), '(program (sum (number) (product (variable) (number))))'); 313 | }) 314 | }) 315 | }); 316 | 317 | describe('.parseTextBufferSync', () => { 318 | it('parses the contents of the given text buffer synchronously', () => { 319 | parser.setLanguage(language); 320 | const buffer = new TextBuffer('αβ αβδ') 321 | const tree = parser.parseTextBufferSync(buffer); 322 | assert.equal(tree.rootNode.type, "sentence"); 323 | assert.equal(tree.rootNode.children.length, 2); 324 | }); 325 | 326 | it('returns null if no language has been set', () => { 327 | const buffer = new TextBuffer('αβ αβδ') 328 | const tree = parser.parseTextBufferSync(buffer); 329 | assert.equal(tree, null); 330 | }) 331 | }); 332 | }); 333 | -------------------------------------------------------------------------------- /test/properties_test.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const {parseProperties, generatePropertyJSON, queryProperties} = require('..'); 3 | 4 | describe('grammar properties', () => { 5 | describe(".parseProperties(css)", () => { 6 | it('parses pseudo classes', () => { 7 | const css = ` 8 | arrow_function > identifier:nth-child(0) { 9 | define: local; 10 | } 11 | `; 12 | 13 | assert.deepEqual(parseProperties(css), [ 14 | { 15 | selectors: [ 16 | [ 17 | {type: 'arrow_function', immediate: false, named: true}, 18 | {type: 'identifier', immediate: true, named: true, index: 0} 19 | ] 20 | ], 21 | properties: {define: 'local'} 22 | } 23 | ]) 24 | }); 25 | 26 | it('parses nested selectors', () => { 27 | const css = ` 28 | a, b { 29 | p: q; 30 | 31 | & > c > d, & e { 32 | p: w; 33 | } 34 | } 35 | `; 36 | 37 | assert.deepEqual(parseProperties(css), [ 38 | { 39 | selectors: [ 40 | [ 41 | {type: 'a', immediate: false, named: true}, 42 | ], 43 | [ 44 | {type: 'b', immediate: false, named: true}, 45 | ] 46 | ], 47 | properties: {p: 'q'} 48 | }, 49 | { 50 | selectors: [ 51 | [ 52 | {type: 'a', immediate: false, named: true}, 53 | {type: 'c', named: true, immediate: true}, 54 | {type: 'd', named: true, immediate: true} 55 | ], 56 | [ 57 | {type: 'a', immediate: false, named: true}, 58 | {type: 'e', named: true, immediate: false} 59 | ], 60 | [ 61 | {type: 'b', immediate: false, named: true}, 62 | {type: 'c', named: true, immediate: true}, 63 | {type: 'd', named: true, immediate: true}, 64 | ], 65 | [ 66 | {type: 'b', immediate: false, named: true}, 67 | {type: 'e', named: true, immediate: false} 68 | ] 69 | ], 70 | properties: {p: 'w'} 71 | } 72 | ]) 73 | }); 74 | 75 | it('parses function calls as property values', () => { 76 | const css = ` 77 | a { 78 | b: f(); 79 | c: f(g(h), i, "j", 10); 80 | } 81 | `; 82 | 83 | assert.deepEqual(parseProperties(css), [ 84 | { 85 | selectors: [ 86 | [ 87 | {type: 'a', immediate: false, named: true}, 88 | ], 89 | ], 90 | properties: { 91 | b: { 92 | name: 'f', 93 | args: [] 94 | }, 95 | c: { 96 | name: 'f', 97 | args: [ 98 | { 99 | name: 'g', 100 | args: [ 101 | 'h' 102 | ], 103 | }, 104 | 'i', 105 | 'j', 106 | 10 107 | ] 108 | }, 109 | } 110 | } 111 | ]); 112 | }); 113 | 114 | it('creates arrays when properties are listed multiple times in a block', () => { 115 | const css = ` 116 | a { 117 | b: 'foo'; 118 | b: 'bar'; 119 | b: 'baz'; 120 | c: f(g()); 121 | c: h(); 122 | } 123 | `; 124 | 125 | assert.deepEqual(parseProperties(css), [ 126 | { 127 | selectors: [ 128 | [ 129 | {type: 'a', immediate: false, named: true}, 130 | ], 131 | ], 132 | properties: { 133 | b: ['foo', 'bar', 'baz'], 134 | c: [ 135 | {name: 'f', args: [{name: 'g', args: []}]}, 136 | {name: 'h', args: []} 137 | ] 138 | } 139 | } 140 | ]); 141 | }); 142 | 143 | it('validates schemas', () => { 144 | assert.throws(() => 145 | parseProperties( 146 | ` 147 | @schema "./fixtures/schema.json"; 148 | a { number: three; } 149 | `, 150 | __dirname 151 | ) 152 | , /Invalid value 'three' for property 'number'/) 153 | 154 | // The value 'two' is in the schema. 155 | parseProperties( 156 | ` 157 | @schema "./fixtures/schema.json"; 158 | a { number: two; } 159 | `, 160 | __dirname 161 | ) 162 | }); 163 | 164 | it('processes imports', () => { 165 | const properties = parseProperties( 166 | ` 167 | a { b: c; } 168 | @import "./fixtures/import.css"; 169 | d { e: f; } 170 | `, 171 | __dirname 172 | ) 173 | 174 | assert.deepEqual(properties, [ 175 | { 176 | selectors: [[{immediate: false, named: true, type: 'a'}]], 177 | properties: {b: 'c'} 178 | }, 179 | { 180 | selectors: [[{immediate: false, named: true, type: 'foo'}]], 181 | properties: {number: 'two'} 182 | }, 183 | { 184 | selectors: [[{immediate: false, named: true, type: 'd'}]], 185 | properties: {e: 'f'} 186 | } 187 | ]); 188 | }); 189 | }); 190 | 191 | describe('.generatePropertyJSON and .queryProperties', () => { 192 | it('handles immediate child selectors', () => { 193 | const css = ` 194 | f1 { 195 | color: red; 196 | 197 | & > f2 { 198 | color: green; 199 | } 200 | 201 | & f3 { 202 | color: blue; 203 | } 204 | } 205 | 206 | f2 { 207 | color: indigo; 208 | height: 2; 209 | } 210 | 211 | f3 { 212 | color: violet; 213 | height: 3; 214 | } 215 | `; 216 | 217 | const properties = JSON.parse(generatePropertyJSON(css)); 218 | 219 | // f1 single-element selector 220 | assert.deepEqual(queryProperties(properties, ['f1']), {color: 'red'}) 221 | assert.deepEqual(queryProperties(properties, ['f2', 'f1']), {color: 'red'}) 222 | assert.deepEqual(queryProperties(properties, ['f2', 'f3', 'f1']), {color: 'red'}) 223 | 224 | // f2 single-element selector 225 | assert.deepEqual(queryProperties(properties, ['f2']), {color: 'indigo', height: '2'}) 226 | assert.deepEqual(queryProperties(properties, ['f2', 'f2']), {color: 'indigo', height: '2'}) 227 | assert.deepEqual(queryProperties(properties, ['f1', 'f3', 'f2']), {color: 'indigo', height: '2'}) 228 | assert.deepEqual(queryProperties(properties, ['f1', 'f6', 'f2']), {color: 'indigo', height: '2'}) 229 | 230 | // f3 single-element selector 231 | assert.deepEqual(queryProperties(properties, ['f3']), {color: 'violet', height: '3'}) 232 | assert.deepEqual(queryProperties(properties, ['f2', 'f3']), {color: 'violet', height: '3'}) 233 | 234 | // f2 child selector 235 | assert.deepEqual(queryProperties(properties, ['f1', 'f2']), {color: 'green', height: '2'}) 236 | assert.deepEqual(queryProperties(properties, ['f2', 'f1', 'f2']), {color: 'green', height: '2'}) 237 | assert.deepEqual(queryProperties(properties, ['f3', 'f1', 'f2']), {color: 'green', height: '2'}) 238 | 239 | // f3 descendant selector 240 | assert.deepEqual(queryProperties(properties, ['f1', 'f3']), {color: 'blue', height: '3'}) 241 | assert.deepEqual(queryProperties(properties, ['f1', 'f2', 'f3']), {color: 'blue', height: '3'}) 242 | assert.deepEqual(queryProperties(properties, ['f1', 'f6', 'f7', 'f8', 'f3']), {color: 'blue', height: '3'}) 243 | 244 | // no match 245 | assert.deepEqual(queryProperties(properties, ['f1', 'f3', 'f4']), {}) 246 | assert.deepEqual(queryProperties(properties, ['f1', 'f2', 'f5']), {}) 247 | }); 248 | 249 | it('handles the :text pseudo class', () => { 250 | const css = ` 251 | f1 { 252 | color: red; 253 | 254 | &:text('^[A-Z]') { 255 | color: green; 256 | } 257 | } 258 | 259 | f2:text('^[A-Z_]+$') { 260 | color: purple; 261 | } 262 | `; 263 | 264 | const properties = JSON.parse(generatePropertyJSON(css)); 265 | assert.deepEqual(queryProperties(properties, ['f1'], [0], 'abc'), {color: 'red'}); 266 | assert.deepEqual(queryProperties(properties, ['f1'], [0], 'Abc'), {color: 'green'}); 267 | assert.deepEqual(queryProperties(properties, ['f2'], [0], 'abc'), {}); 268 | assert.deepEqual(queryProperties(properties, ['f2'], [0], 'ABC'), {color: 'purple'}); 269 | }); 270 | 271 | it('does not allow pseudo classes to be used without a node type', () => { 272 | assert.throws(() => { 273 | generatePropertyJSON(`:text('a') {}`) 274 | }, /Pseudo class ':text' must be used together with a node type/) 275 | 276 | assert.throws(() => { 277 | generatePropertyJSON(`a :text('a') {}`) 278 | }, /Pseudo class ':text' must be used together with a node type/) 279 | 280 | assert.throws(() => { 281 | generatePropertyJSON(`a > :nth-child(0) {}`) 282 | }, /Pseudo class ':nth-child' must be used together with a node type/) 283 | }); 284 | 285 | it('breaks specificity ties using the order in the cascade', () => { 286 | const css = ` 287 | f1 f2:nth-child(1) { color: red; } 288 | f1:nth-child(1) f2 { color: green; } 289 | f1 f2:text('a') { color: blue; } 290 | f1 f2:text('b') { color: violet; } 291 | `; 292 | 293 | const properties = JSON.parse(generatePropertyJSON(css)); 294 | assert.deepEqual(queryProperties(properties, ['f1', 'f2'], [0, 0], 'x'), {}); 295 | assert.deepEqual(queryProperties(properties, ['f1', 'f2'], [0, 1], 'x'), {color: 'red'}); 296 | assert.deepEqual(queryProperties(properties, ['f1', 'f2'], [1, 1], 'x'), {color: 'green'}); 297 | assert.deepEqual(queryProperties(properties, ['f1', 'f2'], [1, 1], 'a'), {color: 'blue'}); 298 | assert.deepEqual(queryProperties(properties, ['f1', 'f2'], [1, 1], 'ab'), {color: 'violet'}); 299 | }) 300 | }); 301 | }); 302 | -------------------------------------------------------------------------------- /test/tree_test.js: -------------------------------------------------------------------------------- 1 | const Parser = require("tree-sitter"); 2 | const { assert } = require("chai"); 3 | const { dsl, generate, loadLanguage } = require(".."); 4 | const { choice, prec, repeat, seq, grammar } = dsl; 5 | const ARITHMETIC = require('./fixtures/arithmetic_language'); 6 | 7 | describe("Tree", () => { 8 | let parser; 9 | 10 | beforeEach(() => { 11 | parser = new Parser(); 12 | parser.setLanguage(ARITHMETIC) 13 | }); 14 | 15 | describe('.edit', () => { 16 | let input, edit 17 | 18 | it('updates the positions of existing nodes', () => { 19 | input = 'abc + cde'; 20 | 21 | tree = parser.parse(input); 22 | assert.equal( 23 | tree.rootNode.toString(), 24 | "(program (sum (variable) (variable)))" 25 | ); 26 | 27 | let variableNode1 = tree.rootNode.firstChild.firstChild; 28 | let variableNode2 = tree.rootNode.firstChild.lastChild; 29 | assert.equal(variableNode1.startIndex, 0); 30 | assert.equal(variableNode1.endIndex, 3); 31 | assert.equal(variableNode2.startIndex, 6); 32 | assert.equal(variableNode2.endIndex, 9); 33 | 34 | ([input, edit] = spliceInput(input, input.indexOf('bc'), 0, ' * ')); 35 | assert.equal(input, 'a * bc + cde'); 36 | 37 | tree.edit(edit); 38 | assert.equal(variableNode1.startIndex, 0); 39 | assert.equal(variableNode1.endIndex, 6); 40 | assert.equal(variableNode2.startIndex, 9); 41 | assert.equal(variableNode2.endIndex, 12); 42 | 43 | tree = parser.parse(input, tree); 44 | assert.equal( 45 | tree.rootNode.toString(), 46 | "(program (sum (product (variable) (variable)) (variable)))" 47 | ); 48 | }); 49 | 50 | it("handles non-ascii characters", () => { 51 | input = 'αβδ + cde'; 52 | 53 | tree = parser.parse(input); 54 | assert.equal( 55 | tree.rootNode.toString(), 56 | "(program (sum (variable) (variable)))" 57 | ); 58 | 59 | const variableNode = tree.rootNode.firstChild.lastChild; 60 | 61 | ([input, edit] = spliceInput(input, input.indexOf('δ'), 0, '👍 * ')); 62 | assert.equal(input, 'αβ👍 * δ + cde'); 63 | 64 | tree.edit(edit); 65 | assert.equal(variableNode.startIndex, input.indexOf('cde')); 66 | 67 | tree = parser.parse(input, tree); 68 | assert.equal( 69 | tree.rootNode.toString(), 70 | "(program (sum (product (variable) (variable)) (variable)))" 71 | ); 72 | }); 73 | }); 74 | 75 | describe('.getEditedRange()', () => { 76 | it('returns the range of tokens that have been edited', () => { 77 | const inputString = 'abc + def + ghi + jkl + mno'; 78 | const tree = parser.parse(inputString); 79 | 80 | assert.equal(tree.getEditedRange(), null) 81 | 82 | tree.edit({ 83 | startIndex: 7, 84 | oldEndIndex: 7, 85 | newEndIndex: 8, 86 | startPosition: { row: 0, column: 7 }, 87 | oldEndPosition: { row: 0, column: 7 }, 88 | newEndPosition: { row: 0, column: 8 } 89 | }); 90 | 91 | tree.edit({ 92 | startIndex: 21, 93 | oldEndIndex: 21, 94 | newEndIndex: 22, 95 | startPosition: { row: 0, column: 21 }, 96 | oldEndPosition: { row: 0, column: 21 }, 97 | newEndPosition: { row: 0, column: 22 } 98 | }); 99 | 100 | assert.deepEqual(tree.getEditedRange(), { 101 | startIndex: 6, 102 | endIndex: 23, 103 | startPosition: {row: 0, column: 6}, 104 | endPosition: {row: 0, column: 23}, 105 | }); 106 | }) 107 | }); 108 | 109 | describe(".getChangedRanges()", () => { 110 | let language 111 | 112 | before(() => { 113 | language = loadLanguage( 114 | generate( 115 | grammar({ 116 | name: "test2", 117 | rules: { 118 | expression: $ => 119 | choice( 120 | prec.left(seq($.expression, "+", $.expression)), 121 | $.variable 122 | ), 123 | 124 | variable: $ => /\w+/ 125 | } 126 | }) 127 | ) 128 | ); 129 | }); 130 | 131 | it("reports the ranges of text whose syntactic meaning has changed", () => { 132 | parser.setLanguage(language); 133 | 134 | let sourceCode = "abcdefg + hij"; 135 | const tree1 = parser.parse(sourceCode); 136 | 137 | assert.equal( 138 | tree1.rootNode.toString(), 139 | "(expression (expression (variable)) (expression (variable)))" 140 | ); 141 | 142 | sourceCode = "abc + defg + hij"; 143 | tree1.edit({ 144 | startIndex: 2, 145 | oldEndIndex: 2, 146 | newEndIndex: 5, 147 | startPosition: { row: 0, column: 2 }, 148 | oldEndPosition: { row: 0, column: 2 }, 149 | newEndPosition: { row: 0, column: 5 } 150 | }); 151 | 152 | const tree2 = parser.parse(sourceCode, tree1); 153 | assert.equal( 154 | tree2.rootNode.toString(), 155 | "(expression (expression (expression (variable)) (expression (variable))) (expression (variable)))" 156 | ); 157 | 158 | const ranges = tree1.getChangedRanges(tree2); 159 | assert.deepEqual(ranges, [ 160 | { 161 | startIndex: 0, 162 | endIndex: "abc + defg".length, 163 | startPosition: { row: 0, column: 0 }, 164 | endPosition: { row: 0, column: "abc + defg".length } 165 | } 166 | ]); 167 | }); 168 | 169 | it('throws an exception if the argument is not a tree', () => { 170 | parser.setLanguage(language); 171 | const tree1 = parser.parse("abcdefg + hij"); 172 | 173 | assert.throws(() => { 174 | tree1.getChangedRanges({}); 175 | }, /Argument must be a tree/); 176 | }) 177 | }); 178 | 179 | describe(".walk()", () => { 180 | it('returns a cursor that can be used to walk the tree', () => { 181 | const tree = parser.parse('a * b + c / d'); 182 | 183 | const cursor = tree.walk(); 184 | assertCursorState(cursor, { 185 | nodeType: 'program', 186 | nodeIsNamed: true, 187 | startPosition: {row: 0, column: 0}, 188 | endPosition: {row: 0, column: 13}, 189 | startIndex: 0, 190 | endIndex: 13 191 | }); 192 | 193 | assert(cursor.gotoFirstChild()); 194 | assertCursorState(cursor, { 195 | nodeType: 'sum', 196 | nodeIsNamed: true, 197 | startPosition: {row: 0, column: 0}, 198 | endPosition: {row: 0, column: 13}, 199 | startIndex: 0, 200 | endIndex: 13 201 | }); 202 | 203 | assert(cursor.gotoFirstChild()); 204 | assertCursorState(cursor, { 205 | nodeType: 'product', 206 | nodeIsNamed: true, 207 | startPosition: {row: 0, column: 0}, 208 | endPosition: {row: 0, column: 5}, 209 | startIndex: 0, 210 | endIndex: 5 211 | }); 212 | 213 | assert(cursor.gotoFirstChild()); 214 | assertCursorState(cursor, { 215 | nodeType: 'variable', 216 | nodeIsNamed: true, 217 | startPosition: {row: 0, column: 0}, 218 | endPosition: {row: 0, column: 1}, 219 | startIndex: 0, 220 | endIndex: 1 221 | }); 222 | 223 | assert(!cursor.gotoFirstChild()) 224 | assert(cursor.gotoNextSibling()); 225 | assertCursorState(cursor, { 226 | nodeType: '*', 227 | nodeIsNamed: false, 228 | startPosition: {row: 0, column: 2}, 229 | endPosition: {row: 0, column: 3}, 230 | startIndex: 2, 231 | endIndex: 3 232 | }); 233 | 234 | assert(cursor.gotoNextSibling()); 235 | assertCursorState(cursor, { 236 | nodeType: 'variable', 237 | nodeIsNamed: true, 238 | startPosition: {row: 0, column: 4}, 239 | endPosition: {row: 0, column: 5}, 240 | startIndex: 4, 241 | endIndex: 5 242 | }); 243 | 244 | assert(!cursor.gotoNextSibling()); 245 | assert(cursor.gotoParent()); 246 | assertCursorState(cursor, { 247 | nodeType: 'product', 248 | nodeIsNamed: true, 249 | startPosition: {row: 0, column: 0}, 250 | endPosition: {row: 0, column: 5}, 251 | startIndex: 0, 252 | endIndex: 5 253 | }); 254 | 255 | assert(cursor.gotoNextSibling()); 256 | assertCursorState(cursor, { 257 | nodeType: '+', 258 | nodeIsNamed: false, 259 | startPosition: {row: 0, column: 6}, 260 | endPosition: {row: 0, column: 7}, 261 | startIndex: 6, 262 | endIndex: 7 263 | }); 264 | 265 | assert(cursor.gotoNextSibling()); 266 | assertCursorState(cursor, { 267 | nodeType: 'quotient', 268 | nodeIsNamed: true, 269 | startPosition: {row: 0, column: 8}, 270 | endPosition: {row: 0, column: 13}, 271 | startIndex: 8, 272 | endIndex: 13 273 | }); 274 | 275 | const childIndex = cursor.gotoFirstChildForIndex(12); 276 | assertCursorState(cursor, { 277 | nodeType: 'variable', 278 | nodeIsNamed: true, 279 | startPosition: {row: 0, column: 12}, 280 | endPosition: {row: 0, column: 13}, 281 | startIndex: 12, 282 | endIndex: 13 283 | }); 284 | assert.equal(childIndex, 2); 285 | 286 | assert(!cursor.gotoNextSibling()); 287 | assert(cursor.gotoParent()); 288 | assert(cursor.gotoParent()); 289 | assert(cursor.gotoParent()); 290 | assert(!cursor.gotoParent()); 291 | }); 292 | 293 | it('returns a cursor that can be reset anywhere in the tree', () => { 294 | const tree = parser.parse('a * b + c / d'); 295 | const cursor = tree.walk(); 296 | const root = tree.rootNode; 297 | 298 | cursor.reset(root.children[0].children[0]); 299 | assertCursorState(cursor, { 300 | nodeType: 'product', 301 | nodeIsNamed: true, 302 | startPosition: {row: 0, column: 0}, 303 | endPosition: {row: 0, column: 5}, 304 | startIndex: 0, 305 | endIndex: 5 306 | }); 307 | 308 | cursor.gotoFirstChild() 309 | assertCursorState(cursor, { 310 | nodeType: 'variable', 311 | nodeIsNamed: true, 312 | startPosition: {row: 0, column: 0}, 313 | endPosition: {row: 0, column: 1}, 314 | startIndex: 0, 315 | endIndex: 1 316 | }); 317 | 318 | cursor.reset(root); 319 | assert(!cursor.gotoParent()); 320 | }) 321 | }); 322 | }); 323 | 324 | function assertCursorState(cursor, params) { 325 | assert.equal(cursor.nodeType, params.nodeType); 326 | assert.equal(cursor.nodeIsNamed, params.nodeIsNamed); 327 | assert.deepEqual(cursor.startPosition, params.startPosition); 328 | assert.deepEqual(cursor.endPosition, params.endPosition); 329 | assert.deepEqual(cursor.startIndex, params.startIndex); 330 | assert.deepEqual(cursor.endIndex, params.endIndex); 331 | 332 | const node = cursor.currentNode 333 | assert.equal(node.type, params.nodeType); 334 | assert.equal(node.isNamed, params.nodeIsNamed); 335 | assert.deepEqual(node.startPosition, params.startPosition); 336 | assert.deepEqual(node.endPosition, params.endPosition); 337 | assert.deepEqual(node.startIndex, params.startIndex); 338 | assert.deepEqual(node.endIndex, params.endIndex); 339 | } 340 | 341 | function spliceInput(input, startIndex, lengthRemoved, newText) { 342 | const oldEndIndex = startIndex + lengthRemoved; 343 | const newEndIndex = startIndex + newText.length; 344 | const startPosition = getExtent(input.slice(0, startIndex)); 345 | const oldEndPosition = getExtent(input.slice(0, oldEndIndex)); 346 | input = input.slice(0, startIndex) + newText + input.slice(oldEndIndex); 347 | const newEndPosition = getExtent(input.slice(0, newEndIndex)); 348 | return [ 349 | input, 350 | { 351 | startIndex, startPosition, 352 | oldEndIndex, oldEndPosition, 353 | newEndIndex, newEndPosition 354 | } 355 | ]; 356 | } 357 | 358 | function getExtent(text) { 359 | let row = 0 360 | let index; 361 | for (index = 0; index != -1; index = text.indexOf('\n', index)) { 362 | index++ 363 | row++; 364 | } 365 | return {row, column: text.length - index}; 366 | } 367 | --------------------------------------------------------------------------------