├── .gitignore ├── .mocharc.cjs ├── .npmignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── build.js ├── package.json ├── src ├── README.md ├── index.ts ├── mix.ts ├── parse.ts └── tree.ts ├── test ├── test-tree.ts └── tsconfig.json └── tsconfig.json /.gitignore: -------------------------------------------------------------------------------- 1 | /node_modules/ 2 | /dist/* 3 | /test/*.js 4 | .tern-* 5 | .rpt2_cache -------------------------------------------------------------------------------- /.mocharc.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | extension: ["ts"], 3 | spec: ["test/test-*.ts"], 4 | loader: "ts-node/esm/transpile-only" 5 | } 6 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | /node_modules 2 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## 1.2.3 (2024-10-16) 2 | 3 | ### Bug fixes 4 | 5 | Fix an issue in `TreeCursor.iterate` that made it iterate through siblings of the current node. 6 | 7 | ## 1.2.2 (2024-10-02) 8 | 9 | ### Bug fixes 10 | 11 | Fix a bug in `TreeCursor.matchContext` where, if the context fell partially in a buffer node and partially in tree nodes, it could return incorrect results. 12 | 13 | ## 1.2.1 (2024-01-16) 14 | 15 | ### Bug fixes 16 | 17 | Fix a bug where `getChild` and `getChildren` would, if the first and second arguments matched the same node, return incorrect results. 18 | 19 | ## 1.2.0 (2023-12-28) 20 | 21 | ### New features 22 | 23 | The new `NodeProp.isolate` can be used by parser to signal that a given node should be treated as isolated when it comes to bidirectional text rendering. 24 | 25 | ## 1.1.2 (2023-12-07) 26 | 27 | ### Bug fixes 28 | 29 | Fix a crash that could happen in mixed-language parsing when mounting a tree to a zero-length node. 30 | 31 | ## 1.1.1 (2023-11-10) 32 | 33 | ### Bug fixes 34 | 35 | Fix a bug where `resolveStack` could sometimes yield the same node multiple times. 36 | 37 | Allow mixed-parsing trees to be mounted for zero-length nodes. 38 | 39 | Fix a bug in mixed-language parsing that could some parts of an inner language to be missed when incrementally reparsing from a stopped parse. 40 | 41 | In `Tree.build`, when given an extremely deeply nested tree structure, flatten it instead of overflowing the stack. 42 | 43 | ## 1.1.0 (2023-09-19) 44 | 45 | ### New features 46 | 47 | The new `Tree.resolveStack` method returns an iterator through the nodes covering a position, including those from overlays not active at that point. 48 | 49 | ## 1.0.4 (2023-08-17) 50 | 51 | ### Bug fixes 52 | 53 | Make the package usable in TypeScript with node16/nodenext resolution. 54 | 55 | ## 1.0.3 (2023-06-02) 56 | 57 | ### Bug fixes 58 | 59 | `Tree.iterate` now properly includes anonymous nodes when `IterMode.IncludeAnonymous` is enabled. 60 | 61 | ## 1.0.2 (2022-11-23) 62 | 63 | ### Bug fixes 64 | 65 | Fix a bug in mixed parsing that would sometimes produce invalid trees. 66 | 67 | ## 1.0.1 (2022-08-30) 68 | 69 | ### Bug fixes 70 | 71 | Fix a bug that could cause incremental parsing to incorrectly reuse nodes inside nested mixed parses. 72 | 73 | ## 1.0.0 (2022-06-06) 74 | 75 | ### New features 76 | 77 | First stable version. 78 | 79 | ## 0.16.1 (2022-06-01) 80 | 81 | ### Bug fixes 82 | 83 | Declare `matchContext` as a property of `SyntaxNodeRef`. 84 | 85 | ## 0.16.0 (2022-04-20) 86 | 87 | ### Breaking changes 88 | 89 | The mixed parsing interface now passes `SyntaxNodeRef` values instead of `TreeCursor` instances around. 90 | 91 | Creating a cursor at a given position is now done with `Tree.cursorAt`, not `cursor`. 92 | 93 | Getting a cursor from a syntax node is now done with a method (`cursor`) rather than a getter. 94 | 95 | `Tree.fullCursor` was removed and replaced by the `IncludeAnonymous` iteration mode. 96 | 97 | The optional arguments to `enter` tree traversal methods were replaced with a single `mode` argument. 98 | 99 | `Tree.iterate` now passes node objects, rather than type/from/to as separate arguments, to its callback functions. 100 | 101 | ### New features 102 | 103 | The new `SyntaxNodeRef` type provides an interface shared by `SyntaxNode` and `TreeCursor`. 104 | 105 | `TreeCursor` instances now how have an `iterate` method that calls a function for each descendant of the current node. 106 | 107 | The new `matchContext` method on `SyntaxNode` and `TreeCursor` provides a convenient way to match the names of direct parent nodes. 108 | 109 | Cursors can now be passed flags to control their behavior. 110 | 111 | ## 0.15.12 (2022-03-18) 112 | 113 | ### Bug fixes 114 | 115 | Work around a TypeScript issue that caused it to infer return type `any` for `resolve` and `resolveInner`. 116 | 117 | Fix a bug in incremental mixed-language parsing where it could incorrectly add a parse range twice, causing a crash in the inner parser. 118 | 119 | ## 0.15.11 (2021-12-16) 120 | 121 | ### Bug fixes 122 | 123 | Fix a bug where nested parsing would sometimes corrupt the length of parent nodes around the nesting point. 124 | 125 | ## 0.15.10 (2021-11-30) 126 | 127 | ### New features 128 | 129 | `SyntaxNode` now has a `resolveInner` method (analogous to `Tree.resolveInner`). 130 | 131 | ## 0.15.9 (2021-11-24) 132 | 133 | ### Bug fixes 134 | 135 | Full tree cursors no longer automatically enter mounted subtrees. 136 | 137 | Fix a bug where a nested parser would not re-parse inner sections when given fragments produced by a parse that finished the outer tree but was stopped before the inner trees were done. 138 | 139 | ## 0.15.8 (2021-11-10) 140 | 141 | ### Bug fixes 142 | 143 | Fix a bug that could cause incorrectly structured nodes to be created for repeat rules, breaking incremental parsing using the resulting tree. 144 | 145 | ## 0.15.7 (2021-10-05) 146 | 147 | ### Bug fixes 148 | 149 | Fix an issue in `parseMixed` where parses nested two or more levels deep would use the wrong document offsets. 150 | 151 | ## 0.15.6 (2021-09-30) 152 | 153 | ### Bug fixes 154 | 155 | Fix a null-dereference crash in mixed-language parsing. 156 | 157 | ## 0.15.5 (2021-09-09) 158 | 159 | ### New features 160 | 161 | Syntax node objects now have a method `enterUnfinishedNodesBefore` to scan down the tree for nodes that were broken off directly in front of a given position (which can provide a more accurate context that just resolving the position). 162 | 163 | ## 0.15.4 (2021-08-31) 164 | 165 | ### Bug fixes 166 | 167 | `parseMixed` will now scan children not covered by the ranges of an eagerly computed overlay for further nesting. 168 | 169 | ## 0.15.3 (2021-08-12) 170 | 171 | ### Bug fixes 172 | 173 | Fix an issue where `parseMixed` could create overlay mounts with zero ranges, which were useless and confused CodeMirror's highlighter. 174 | 175 | ## 0.15.2 (2021-08-12) 176 | 177 | ### Bug fixes 178 | 179 | Fix a bug that would cause `enter` to return incorrect results when called entering children in a buffer with . 180 | 181 | ## 0.15.1 (2021-08-12) 182 | 183 | ### Bug fixes 184 | 185 | Fix a bug where `parseMixed` could crash by dereferencing null. 186 | 187 | ## 0.15.0 (2021-08-11) 188 | 189 | ### Breaking changes 190 | 191 | The module name has changed from `lezer-tree` to `@lezer/common`. 192 | 193 | `TreeBuffer`s no longer accept a node type. 194 | 195 | `Tree.balance` no longer takes a buffer size as argument. 196 | 197 | `NodeProp.string`, `NodeProp.number`, and `NodeProp.flag` have been removed (the thing they provided is trivial to write by hand). 198 | 199 | A node's context hash is now stored in the `NodeProp.contextHash` prop. 200 | 201 | Reused nodes passed to `Tree.build` must now be `Tree` instances (not tree buffers). 202 | 203 | `Parser` is now an abstract class that all parser implementations must extend, implementing the `createParse` method. 204 | 205 | The `PartialParse` interface changed to make multi-pass parsers possible. 206 | 207 | `Parser.startParse` now takes different arguments `(input, fragments, ranges)` instead of `(input, startPos, context)`. 208 | 209 | The `Input` interface has changed (to become chunk-based and more low-level). A single `Input` object is now shared between outer and inner parses. 210 | 211 | `stringInput` is no longer exported (`Parser` methods will automatically wrap strings when appropriate). 212 | 213 | ### Bug fixes 214 | 215 | Fix a bug in `TreeFragment.applyChanges` that prevented some valid reuse of syntax nodes. 216 | 217 | Fix a bug where reused nodes could incorrectly be dropped by `Tree.build`. 218 | 219 | ### New features 220 | 221 | Node props can now be per-node, in which case they are stored on `Tree` instances rather than `NodeType`s. 222 | 223 | Tree nodes can now be replaced with other trees through `NodeProp.mountedTree`. 224 | 225 | `Tree.resolveInner` can now be used to resolve into overlay trees. 226 | 227 | `SyntaxNode` objects now have a `toTree` method to convert them to a stand-alone tree. 228 | 229 | `Tree.balance` now accepts a helper function to create the inner nodes. 230 | 231 | Tree cursors' `next`/`prev` methods now take an `enter` argument to control whether they enter the current node. 232 | 233 | `SyntaxNode` and `TreeCursor` now have an `enter` method to directly enter the child at the given position (if any). 234 | 235 | `Tree.iterate` callbacks now get an extra argument that allows them to create a `SyntaxNode` for the current node. 236 | 237 | The parsing interface now supports parsing specific, non-contiguous ranges of the input in a single parse. 238 | 239 | The module now exports a `parseMixed` helper function for creating mixed-language parsers. 240 | 241 | ## 0.13.2 (2021-02-17) 242 | 243 | ### New features 244 | 245 | Add support for context tracking. 246 | 247 | ## 0.13.1 (2021-02-11) 248 | 249 | ### Bug fixes 250 | 251 | Fix a bug where building a tree from a buffer would go wrong for repeat nodes whose children were all repeat nodes of the same type. 252 | 253 | ## 0.13.0 (2020-12-04) 254 | 255 | ### Breaking changes 256 | 257 | `NodeType.isRepeated` is now called `isAnonymous`, which more accurately describes what it means. 258 | 259 | `NodeGroup` has been renamed to `NodeSet` to avoid confusion with `NodeProp.group`. 260 | 261 | The `applyChanges` method on trees is no longer supported (`TreeFragment` is now used to track reusable content). 262 | 263 | Trees no longer have `cut` and `append` methods. 264 | 265 | ### New features 266 | 267 | It is now possible to pass a node ID to `SyntaxNode.getChild`/`getChildren` and `NodeType.is`. Allow specifying a tree length in Tree.build 268 | 269 | `Tree.build` now allows you to specify the length of the resulting tree. 270 | 271 | `Tree.fullCursor()` can now be used to get a cursor that includes anonymous nodes, rather than skipping them. 272 | 273 | Introduces `NodeType.define` to define node types. 274 | 275 | The new `TreeFragment` type is used to manage reusable subtrees for incremental parsing. 276 | 277 | `Tree.build` now accepts a `start` option indicating the start offset of the tree. 278 | 279 | The `Input` type, which used to be `InputStream` in the lezer package, is now exported from this package. 280 | 281 | This package now exports a `PartialParse` interface, which describes the interface used, for example, as return type from `Parser.startParse`. 282 | 283 | ## 0.12.3 (2020-11-02) 284 | 285 | ### New features 286 | 287 | Make `NodePropSource` a function type. 288 | 289 | ## 0.12.2 (2020-10-28) 290 | 291 | ### Bug fixes 292 | 293 | Fix a bug that made `SyntaxNode.prevSibling` fail in most cases when the node is part of a buffer. 294 | 295 | ## 0.12.1 (2020-10-26) 296 | 297 | ### Bug fixes 298 | 299 | Fix issue where using `Tree.append` with an empty tree as argument would return a tree with a nonsensical `length` property. 300 | 301 | ## 0.12.0 (2020-10-23) 302 | 303 | ### Breaking changes 304 | 305 | `Tree.iterate` no longer allows returning from inside the iteration (use cursors directly for that kind of use cases). 306 | 307 | `Subtree` has been renamed to `SyntaxNode` and narrowed in scope a little. 308 | 309 | The `top`, `skipped`, and `error` node props no longer exist. 310 | 311 | ### New features 312 | 313 | The package now offers a `TreeCursor` abstraction, which can be used for both regular iteration and for custom traversal of a tree. 314 | 315 | `SyntaxNode` instances have `nextSibling`/`prevSibling` getters that allow more direct navigation through the tree. 316 | 317 | Node types now expose `isTop`, `isSkipped`, `isError`, and `isRepeated` properties that indicate special status. 318 | 319 | Adds `NodeProp.group` to assign group names to node types. 320 | 321 | Syntax nodes now have helper functions `getChild` and `getChildren` to retrieve direct child nodes by type or group. 322 | 323 | `NodeType.match` (and thus `NodeProp.add`) now allows types to be targeted by group name. 324 | 325 | Node types have a new `is` method for checking whether their name or one of their groups matches a given string. 326 | 327 | ## 0.11.1 (2020-09-26) 328 | 329 | ### Bug fixes 330 | 331 | Fix lezer dependency versions 332 | 333 | ## 0.11.0 (2020-09-26) 334 | 335 | ### Breaking changes 336 | 337 | Adjust to new output format of repeat rules. 338 | 339 | ## 0.10.0 (2020-08-07) 340 | 341 | ### Breaking changes 342 | 343 | No longer list internal properties in the type definitions. 344 | 345 | ## 0.9.0 (2020-06-08) 346 | 347 | ### Breaking changes 348 | 349 | Drop `NodeProp.delim` in favor of `NodeProp.openedBy`/`closedBy`. 350 | 351 | ## 0.8.4 (2020-04-01) 352 | 353 | ### Bug fixes 354 | 355 | Make the package load as an ES module on node 356 | 357 | ## 0.8.3 (2020-02-28) 358 | 359 | ### New features 360 | 361 | The package now provides an ES6 module. 362 | 363 | ## 0.8.2 (2020-02-26) 364 | 365 | ### Bug fixes 366 | 367 | Fix a bug that caused `applyChanges` to include parts of the old tree that weren't safe to reuse. 368 | 369 | ## 0.8.1 (2020-02-14) 370 | 371 | ### Bug fixes 372 | 373 | Fix bug that would cause tree balancing of deep trees to produce corrupt output. 374 | 375 | ## 0.8.0 (2020-02-03) 376 | 377 | ### New features 378 | 379 | Bump version along with the rest of the lezer packages. 380 | 381 | ## 0.7.1 (2020-01-23) 382 | 383 | ### Bug fixes 384 | 385 | In `applyChanges`, make sure the tree is collapsed all the way to the 386 | nearest non-error node next to the change. 387 | 388 | ## 0.7.0 (2020-01-20) 389 | 390 | ### Bug fixes 391 | 392 | Fix a bug that prevented balancing of repeat nodes when there were skipped nodes present between the repeated elements (which ruined the efficiency of incremental parses). 393 | 394 | ### New features 395 | 396 | `TreeBuffer` objects now have an `iterate` function. 397 | 398 | Buffers can optionally be tagged with an (unnamed) node type to allow reusing them in an incremental parse without wrapping them in a tree. 399 | 400 | ### Breaking changes 401 | 402 | `Tree.build` now takes its arguments wrapped in an object. It also expects the buffer content to conform to from lezer 0.7.0's representation of repeated productions. 403 | 404 | The `repeated` node prop was removed (the parser generator now encodes repetition in the type ids). 405 | 406 | ## 0.5.1 (2019-10-22) 407 | 408 | ### New features 409 | 410 | `NodeProp.add` now also allows a selector object to be passed. 411 | 412 | ## 0.5.0 (2019-10-22) 413 | 414 | ### New features 415 | 416 | Adds `NodeProp.top`, which flags a grammar's outer node type. 417 | 418 | ### Breaking changes 419 | 420 | Drops the `NodeProp.lang` prop (superseded by `top`). 421 | 422 | ## 0.4.0 (2019-09-10) 423 | 424 | ### Bug fixes 425 | 426 | Export `BufferCursor` again, which was accidentally removed from the exports in 0.3.0. 427 | 428 | ### Breaking changes 429 | 430 | The `iterate` method now takes an object instead of separate parameters. 431 | 432 | ## 0.3.0 (2019-08-22) 433 | 434 | ### New features 435 | 436 | Introduces node props. 437 | 438 | Node types are now objects holding a name, id, and set of props. 439 | 440 | ### Breaking changes 441 | 442 | Tags are gone again, nodes have plain string names. 443 | 444 | ## 0.2.0 (2019-08-02) 445 | 446 | ### Bug fixes 447 | 448 | Fix incorrect node length calculation in `Tree.build`. 449 | 450 | ### New features 451 | 452 | Tree nodes are now identified with tags. 453 | 454 | New `Tag` data structure to represent node tags. 455 | 456 | ### Breaking changes 457 | 458 | Drop support for grammar ids and node types. 459 | 460 | ## 0.1.1 (2019-07-09) 461 | 462 | ### Bug Fixes 463 | 464 | Actually include the .d.ts file in the published package. 465 | 466 | ## 0.1.0 (2019-07-09) 467 | 468 | ### New Features 469 | 470 | First documented release. 471 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (C) 2018 by Marijn Haverbeke and others 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # @lezer/common 2 | 3 | [ [**WEBSITE**](http://lezer.codemirror.net) | [**ISSUES**](https://github.com/lezer-parser/lezer/issues) | [**FORUM**](https://discuss.codemirror.net/c/lezer) | [**CHANGELOG**](https://github.com/lezer-parser/common/blob/master/CHANGELOG.md) ] 4 | 5 | [Lezer](https://lezer.codemirror.net/) is an incremental parser system 6 | intended for use in an editor or similar system. 7 | 8 | @lezer/common provides the syntax tree data structure and parser 9 | abstractions for Lezer parsers. 10 | 11 | Its programming interface is documented on [the 12 | website](https://lezer.codemirror.net/docs/ref/#common). 13 | 14 | This code is licensed under an MIT license. 15 | -------------------------------------------------------------------------------- /build.js: -------------------------------------------------------------------------------- 1 | import {build, watch} from "@marijn/buildtool" 2 | import {fileURLToPath} from "url" 3 | import {dirname, join} from "path" 4 | 5 | let tsOptions = { 6 | lib: ["es5", "es6"], 7 | target: "es6" 8 | } 9 | 10 | let main = join(dirname(fileURLToPath(import.meta.url)), "src", "index.ts") 11 | 12 | if (process.argv.includes("--watch")) { 13 | watch([main], [], {tsOptions}) 14 | } else { 15 | build(main, {tsOptions}) 16 | } 17 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@lezer/common", 3 | "version": "1.2.3", 4 | "description": "Syntax tree data structure and parser interfaces for the lezer parser", 5 | "main": "dist/index.cjs", 6 | "type": "module", 7 | "exports": { 8 | "import": "./dist/index.js", 9 | "require": "./dist/index.cjs" 10 | }, 11 | "module": "dist/index.js", 12 | "types": "dist/index.d.ts", 13 | "author": "Marijn Haverbeke ", 14 | "license": "MIT", 15 | "devDependencies": { 16 | "ist": "^1.1.1", 17 | "@marijn/buildtool": "^0.1.5", 18 | "@types/mocha": "^5.2.6", 19 | "mocha": "^10.2.0", 20 | "ts-node": "^10.9.2" 21 | }, 22 | "files": ["dist"], 23 | "repository": { 24 | "type" : "git", 25 | "url" : "https://github.com/lezer-parser/common.git" 26 | }, 27 | "scripts": { 28 | "watch": "node build.js --watch", 29 | "prepare": "node build.js", 30 | "test": "mocha" 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/README.md: -------------------------------------------------------------------------------- 1 | This package provides common data structures used by all Lezer-related 2 | parsing—those related to syntax trees and the generic interface of 3 | parsers. Their main use is the [LR](#lr) parsers generated by the 4 | [parser generator](#generator), but for example the [Markdown 5 | parser](https://github.com/lezer-parser/markdown) implements a 6 | different parsing algorithm using the same interfaces. 7 | 8 | ### Trees 9 | 10 | Lezer syntax trees are _not_ abstract, they just tell you which nodes 11 | were parsed where, without providing additional information about 12 | their role or relation (beyond parent-child relations). This makes 13 | them rather unsuited for some purposes, but quick to construct and 14 | cheap to store. 15 | 16 | @Tree 17 | 18 | @SyntaxNodeRef 19 | 20 | @SyntaxNode 21 | 22 | @NodeIterator 23 | 24 | @TreeCursor 25 | 26 | @IterMode 27 | 28 | @NodeWeakMap 29 | 30 | #### Node types 31 | 32 | @NodeType 33 | 34 | @NodeSet 35 | 36 | @NodeProp 37 | 38 | @NodePropSource 39 | 40 | #### Buffers 41 | 42 | Buffers are an optimization in the way Lezer trees are stored. 43 | 44 | @TreeBuffer 45 | 46 | @DefaultBufferLength 47 | 48 | @BufferCursor 49 | 50 | ### Parsing 51 | 52 | @Parser 53 | 54 | @Input 55 | 56 | @PartialParse 57 | 58 | @ParseWrapper 59 | 60 | ### Incremental Parsing 61 | 62 | Efficient reparsing happens by reusing parts of the original parsed 63 | structure. 64 | 65 | @TreeFragment 66 | 67 | @ChangedRange 68 | 69 | ### Mixed Parsing 70 | 71 | @parseMixed 72 | 73 | @NestedParse 74 | 75 | @MountedTree 76 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | export {DefaultBufferLength, NodeProp, MountedTree, NodePropSource, NodeType, NodeSet, Tree, 2 | TreeBuffer, SyntaxNode, SyntaxNodeRef, TreeCursor, BufferCursor, NodeWeakMap, IterMode, NodeIterator} from "./tree" 3 | export {ChangedRange, TreeFragment, PartialParse, Parser, Input, ParseWrapper} from "./parse" 4 | export {NestedParse, parseMixed} from "./mix" 5 | -------------------------------------------------------------------------------- /src/mix.ts: -------------------------------------------------------------------------------- 1 | import {Tree, TreeBuffer, NodeType, SyntaxNodeRef, SyntaxNode, NodeProp, 2 | TreeCursor, MountedTree, Range, IterMode, TreeNode, BufferNode} from "./tree" 3 | import {Input, Parser, PartialParse, TreeFragment, ParseWrapper} from "./parse" 4 | 5 | /// Objects returned by the function passed to 6 | /// [`parseMixed`](#common.parseMixed) should conform to this 7 | /// interface. 8 | export interface NestedParse { 9 | /// The parser to use for the inner region. 10 | parser: Parser 11 | 12 | /// When this property is not given, the entire node is parsed with 13 | /// this parser, and it is [mounted](#common.NodeProp^mounted) as a 14 | /// non-overlay node, replacing its host node in tree iteration. 15 | /// 16 | /// When an array of ranges is given, only those ranges are parsed, 17 | /// and the tree is mounted as an 18 | /// [overlay](#common.MountedTree.overlay). 19 | /// 20 | /// When a function is given, that function will be called for 21 | /// descendant nodes of the target node, not including child nodes 22 | /// that are covered by another nested parse, to determine the 23 | /// overlay ranges. When it returns true, the entire descendant is 24 | /// included, otherwise just the range given. The mixed parser will 25 | /// optimize range-finding in reused nodes, which means it's a good 26 | /// idea to use a function here when the target node is expected to 27 | /// have a large, deep structure. 28 | overlay?: readonly {from: number, to: number}[] | ((node: SyntaxNodeRef) => {from: number, to: number} | boolean) 29 | } 30 | 31 | /// Create a parse wrapper that, after the inner parse completes, 32 | /// scans its tree for mixed language regions with the `nest` 33 | /// function, runs the resulting [inner parses](#common.NestedParse), 34 | /// and then [mounts](#common.NodeProp^mounted) their results onto the 35 | /// tree. 36 | export function parseMixed(nest: (node: SyntaxNodeRef, input: Input) => NestedParse | null): ParseWrapper { 37 | return (parse, input, fragments, ranges): PartialParse => new MixedParse(parse, nest, input, fragments, ranges) 38 | } 39 | 40 | class InnerParse { 41 | constructor( 42 | readonly parser: Parser, 43 | readonly parse: PartialParse, 44 | readonly overlay: readonly {from: number, to: number}[] | null, 45 | readonly target: Tree, 46 | readonly from: number 47 | ) {} 48 | } 49 | 50 | function checkRanges(ranges: readonly {from: number, to: number}[]) { 51 | if (!ranges.length || ranges.some(r => r.from >= r.to)) 52 | throw new RangeError("Invalid inner parse ranges given: " + JSON.stringify(ranges)) 53 | } 54 | 55 | class ActiveOverlay { 56 | depth = 0 57 | readonly ranges: {from: number, to: number}[] = [] 58 | 59 | constructor( 60 | readonly parser: Parser, 61 | readonly predicate: (node: SyntaxNodeRef) => {from: number, to: number} | boolean, 62 | readonly mounts: readonly ReusableMount[], 63 | readonly index: number, 64 | readonly start: number, 65 | readonly target: Tree, 66 | readonly prev: ActiveOverlay | null, 67 | ) {} 68 | } 69 | 70 | type CoverInfo = null | {ranges: readonly {from: number, to: number}[], depth: number, prev: CoverInfo} 71 | 72 | const stoppedInner = new NodeProp({perNode: true}) 73 | 74 | class MixedParse implements PartialParse { 75 | baseParse: PartialParse | null 76 | inner: InnerParse[] = [] 77 | innerDone = 0 78 | baseTree: Tree | null = null 79 | stoppedAt: number | null = null 80 | 81 | constructor( 82 | base: PartialParse, 83 | readonly nest: (node: SyntaxNodeRef, input: Input) => NestedParse | null, 84 | readonly input: Input, 85 | readonly fragments: readonly TreeFragment[], 86 | readonly ranges: readonly {from: number, to: number}[] 87 | ) { 88 | this.baseParse = base 89 | } 90 | 91 | advance() { 92 | if (this.baseParse) { 93 | let done = this.baseParse.advance() 94 | if (!done) return null 95 | this.baseParse = null 96 | this.baseTree = done 97 | this.startInner() 98 | if (this.stoppedAt != null) for (let inner of this.inner) inner.parse.stopAt(this.stoppedAt) 99 | } 100 | if (this.innerDone == this.inner.length) { 101 | let result = this.baseTree! 102 | if (this.stoppedAt != null) 103 | result = new Tree(result.type, result.children, result.positions, result.length, 104 | result.propValues.concat([[stoppedInner, this.stoppedAt]])) 105 | return result 106 | } 107 | let inner = this.inner[this.innerDone], done = inner.parse.advance() 108 | if (done) { 109 | this.innerDone++ 110 | // This is a somewhat dodgy but super helpful hack where we 111 | // patch up nodes created by the inner parse (and thus 112 | // presumably not aliased anywhere else) to hold the information 113 | // about the inner parse. 114 | let props = Object.assign(Object.create(null), inner.target.props) 115 | props[NodeProp.mounted.id] = new MountedTree(done, inner.overlay, inner.parser) 116 | ;(inner.target as any).props = props 117 | } 118 | return null 119 | } 120 | 121 | get parsedPos() { 122 | if (this.baseParse) return 0 123 | let pos = this.input.length 124 | for (let i = this.innerDone; i < this.inner.length; i++) { 125 | if (this.inner[i].from < pos) 126 | pos = Math.min(pos, this.inner[i].parse.parsedPos) 127 | } 128 | return pos 129 | } 130 | 131 | stopAt(pos: number) { 132 | this.stoppedAt = pos 133 | if (this.baseParse) this.baseParse.stopAt(pos) 134 | else for (let i = this.innerDone; i < this.inner.length; i++) this.inner[i].parse.stopAt(pos) 135 | } 136 | 137 | startInner() { 138 | let fragmentCursor = new FragmentCursor(this.fragments) 139 | let overlay: ActiveOverlay | null = null 140 | let covered: CoverInfo = null 141 | let cursor = new TreeCursor(new TreeNode(this.baseTree!, this.ranges[0].from, 0, null), 142 | IterMode.IncludeAnonymous | IterMode.IgnoreMounts) 143 | scan: for (let nest, isCovered;;) { 144 | let enter = true, range 145 | if (this.stoppedAt != null && cursor.from >= this.stoppedAt) { 146 | enter = false 147 | } else if (fragmentCursor.hasNode(cursor)) { 148 | if (overlay) { 149 | let match = overlay.mounts.find(m => m.frag.from <= cursor.from && m.frag.to >= cursor.to && m.mount.overlay) 150 | if (match) for (let r of match.mount.overlay!) { 151 | let from = r.from + match.pos, to = r.to + match.pos 152 | if (from >= cursor.from && to <= cursor.to && !overlay.ranges.some(r => r.from < to && r.to > from)) 153 | overlay.ranges.push({from, to}) 154 | } 155 | } 156 | enter = false 157 | } else if (covered && (isCovered = checkCover(covered.ranges, cursor.from, cursor.to))) { 158 | enter = isCovered != Cover.Full 159 | } else if (!cursor.type.isAnonymous && (nest = this.nest(cursor, this.input)) && 160 | (cursor.from < cursor.to || !nest.overlay)) { 161 | if (!cursor.tree) { 162 | materialize(cursor); 163 | // materialize create one more level of nesting 164 | // we need to add depth to active overlay for going backwards 165 | if (overlay) 166 | overlay.depth++; 167 | if (covered) 168 | covered.depth++; 169 | } 170 | let oldMounts = fragmentCursor.findMounts(cursor.from, nest.parser) 171 | if (typeof nest.overlay == "function") { 172 | overlay = new ActiveOverlay(nest.parser, nest.overlay, oldMounts, this.inner.length, 173 | cursor.from, cursor.tree!, overlay) 174 | } else { 175 | let ranges = punchRanges(this.ranges, nest.overlay || 176 | (cursor.from < cursor.to ? [new Range(cursor.from, cursor.to)] : [])) 177 | if (ranges.length) checkRanges(ranges) 178 | if (ranges.length || !nest.overlay) this.inner.push(new InnerParse( 179 | nest.parser, 180 | ranges.length ? nest.parser.startParse(this.input, enterFragments(oldMounts, ranges), ranges) 181 | : nest.parser.startParse(""), 182 | nest.overlay ? nest.overlay.map(r => new Range(r.from - cursor.from, r.to - cursor.from)) : null, 183 | cursor.tree!, 184 | ranges.length ? ranges[0].from : cursor.from, 185 | )) 186 | if (!nest.overlay) enter = false 187 | else if (ranges.length) covered = {ranges, depth: 0, prev: covered} 188 | } 189 | } else if (overlay && (range = overlay.predicate(cursor))) { 190 | if (range === true) range = new Range(cursor.from, cursor.to) 191 | if (range.from < range.to) { 192 | let last = overlay.ranges.length - 1 193 | if (last >= 0 && overlay.ranges[last].to == range.from) 194 | overlay.ranges[last] = {from: overlay.ranges[last].from, to: range.to} 195 | else 196 | overlay.ranges.push(range) 197 | } 198 | } 199 | if (enter && cursor.firstChild()) { 200 | if (overlay) overlay.depth++ 201 | if (covered) covered.depth++ 202 | } else { 203 | for (;;) { 204 | if (cursor.nextSibling()) break 205 | if (!cursor.parent()) break scan 206 | if (overlay && !--overlay.depth) { 207 | let ranges = punchRanges(this.ranges, overlay.ranges) 208 | if (ranges.length) { 209 | checkRanges(ranges) 210 | this.inner.splice(overlay.index, 0, new InnerParse( 211 | overlay.parser, 212 | overlay.parser.startParse(this.input, enterFragments(overlay.mounts, ranges), ranges), 213 | overlay.ranges.map(r => new Range(r.from - overlay!.start, r.to - overlay!.start)), 214 | overlay.target, 215 | ranges[0].from 216 | )) 217 | } 218 | overlay = overlay.prev 219 | } 220 | if (covered && !--covered.depth) covered = covered.prev 221 | } 222 | } 223 | } 224 | } 225 | } 226 | 227 | const enum Cover { None = 0, Partial = 1, Full = 2 } 228 | 229 | function checkCover(covered: readonly {from: number, to: number}[], from: number, to: number) { 230 | for (let range of covered) { 231 | if (range.from >= to) break 232 | if (range.to > from) return range.from <= from && range.to >= to ? Cover.Full : Cover.Partial 233 | } 234 | return Cover.None 235 | } 236 | 237 | // Take a piece of buffer and convert it into a stand-alone 238 | // TreeBuffer. 239 | function sliceBuf(buf: TreeBuffer, startI: number, endI: number, nodes: (Tree | TreeBuffer)[], positions: number[], off: number) { 240 | if (startI < endI) { 241 | let from = buf.buffer[startI + 1] 242 | nodes.push(buf.slice(startI, endI, from)) 243 | positions.push(from - off) 244 | } 245 | } 246 | 247 | // This function takes a node that's in a buffer, and converts it, and 248 | // its parent buffer nodes, into a Tree. This is again acting on the 249 | // assumption that the trees and buffers have been constructed by the 250 | // parse that was ran via the mix parser, and thus aren't shared with 251 | // any other code, making violations of the immutability safe. 252 | function materialize(cursor: TreeCursor) { 253 | let {node} = cursor, stack: number[] = [] 254 | let buffer = (node as BufferNode).context.buffer 255 | // Scan up to the nearest tree 256 | do { stack.push(cursor.index); cursor.parent() } while (!cursor.tree) 257 | // Find the index of the buffer in that tree 258 | let base = cursor.tree!, i = base.children.indexOf(buffer) 259 | let buf = base.children[i] as TreeBuffer, b = buf.buffer, newStack: number[] = [i] 260 | // Split a level in the buffer, putting the nodes before and after 261 | // the child that contains `node` into new buffers. 262 | function split(startI: number, endI: number, type: NodeType, innerOffset: number, length: number, stackPos: number): Tree { 263 | let targetI = stack[stackPos] 264 | let children: (Tree | TreeBuffer)[] = [], positions: number[] = [] 265 | sliceBuf(buf, startI, targetI, children, positions, innerOffset) 266 | let from = b[targetI + 1], to = b[targetI + 2] 267 | newStack.push(children.length) 268 | let child = stackPos 269 | ? split(targetI + 4, b[targetI + 3], buf.set.types[b[targetI]], from, to - from, stackPos - 1) 270 | : node.toTree() 271 | children.push(child) 272 | positions.push(from - innerOffset) 273 | sliceBuf(buf, b[targetI + 3], endI, children, positions, innerOffset) 274 | return new Tree(type, children, positions, length) 275 | } 276 | // Overwrite (!) the child at the buffer's index with the split-up tree 277 | ;(base.children as any)[i] = split(0, b.length, NodeType.none, 0, buf.length, stack.length - 1) 278 | // Move the cursor back to the target node 279 | for (let index of newStack) { 280 | let tree = cursor.tree.children[index] as Tree, pos = cursor.tree.positions[index] 281 | cursor.yield(new TreeNode(tree, pos + cursor.from, index, cursor._tree)) 282 | } 283 | } 284 | 285 | class StructureCursor { 286 | cursor: TreeCursor 287 | done = false 288 | 289 | constructor( 290 | root: Tree, 291 | private offset: number 292 | ) { 293 | this.cursor = root.cursor(IterMode.IncludeAnonymous | IterMode.IgnoreMounts) 294 | } 295 | 296 | // Move to the first node (in pre-order) that starts at or after `pos`. 297 | moveTo(pos: number) { 298 | let {cursor} = this, p = pos - this.offset 299 | while (!this.done && cursor.from < p) { 300 | if (cursor.to >= pos && cursor.enter(p, 1, IterMode.IgnoreOverlays | IterMode.ExcludeBuffers)) {} 301 | else if (!cursor.next(false)) this.done = true 302 | } 303 | } 304 | 305 | hasNode(cursor: TreeCursor) { 306 | this.moveTo(cursor.from) 307 | if (!this.done && this.cursor.from + this.offset == cursor.from && this.cursor.tree) { 308 | for (let tree = this.cursor.tree!;;) { 309 | if (tree == cursor.tree) return true 310 | if (tree.children.length && tree.positions[0] == 0 && tree.children[0] instanceof Tree) tree = tree.children[0] 311 | else break 312 | } 313 | } 314 | return false 315 | } 316 | } 317 | 318 | class FragmentCursor { 319 | curFrag: TreeFragment | null 320 | curTo = 0 321 | fragI = 0 322 | inner: StructureCursor | null 323 | 324 | constructor(readonly fragments: readonly TreeFragment[]) { 325 | if (fragments.length) { 326 | let first = this.curFrag = fragments[0] 327 | this.curTo = first.tree.prop(stoppedInner) ?? first.to 328 | this.inner = new StructureCursor(first.tree, -first.offset) 329 | } else { 330 | this.curFrag = this.inner = null 331 | } 332 | } 333 | 334 | hasNode(node: TreeCursor) { 335 | while (this.curFrag && node.from >= this.curTo) this.nextFrag() 336 | return this.curFrag && this.curFrag.from <= node.from && this.curTo >= node.to && this.inner!.hasNode(node) 337 | } 338 | 339 | nextFrag() { 340 | this.fragI++ 341 | if (this.fragI == this.fragments.length) { 342 | this.curFrag = this.inner = null 343 | } else { 344 | let frag = this.curFrag = this.fragments[this.fragI] 345 | this.curTo = frag.tree.prop(stoppedInner) ?? frag.to 346 | this.inner = new StructureCursor(frag.tree, -frag.offset) 347 | } 348 | } 349 | 350 | findMounts(pos: number, parser: Parser) { 351 | let result: ReusableMount[] = [] 352 | if (this.inner) { 353 | this.inner.cursor.moveTo(pos, 1) 354 | for (let pos: SyntaxNode | null = this.inner.cursor.node; pos; pos = pos.parent) { 355 | let mount = pos.tree?.prop(NodeProp.mounted) 356 | if (mount && mount.parser == parser) { 357 | for (let i = this.fragI; i < this.fragments.length; i++) { 358 | let frag = this.fragments[i] 359 | if (frag.from >= pos.to) break 360 | if (frag.tree == this.curFrag!.tree) result.push({ 361 | frag, 362 | pos: pos.from - frag.offset, 363 | mount 364 | }) 365 | } 366 | } 367 | } 368 | } 369 | return result 370 | } 371 | } 372 | 373 | function punchRanges(outer: readonly {from: number, to: number}[], ranges: readonly {from: number, to: number}[]) { 374 | let copy: {from: number, to: number}[] | null = null, current = ranges 375 | for (let i = 1, j = 0; i < outer.length; i++) { 376 | let gapFrom = outer[i - 1].to, gapTo = outer[i].from 377 | for (; j < current.length; j++) { 378 | let r = current[j] 379 | if (r.from >= gapTo) break 380 | if (r.to <= gapFrom) continue 381 | if (!copy) current = copy = ranges.slice() 382 | if (r.from < gapFrom) { 383 | copy[j] = new Range(r.from, gapFrom) 384 | if (r.to > gapTo) copy.splice(j + 1, 0, new Range(gapTo, r.to)) 385 | } else if (r.to > gapTo) { 386 | copy[j--] = new Range(gapTo, r.to) 387 | } else { 388 | copy.splice(j--, 1) 389 | } 390 | } 391 | } 392 | return current 393 | } 394 | 395 | type ReusableMount = { 396 | frag: TreeFragment, 397 | mount: MountedTree, 398 | pos: number 399 | } 400 | 401 | function findCoverChanges(a: readonly {from: number, to: number}[], 402 | b: readonly {from: number, to: number}[], 403 | from: number, to: number) { 404 | let iA = 0, iB = 0, inA = false, inB = false, pos = -1e9 405 | let result = [] 406 | for (;;) { 407 | let nextA = iA == a.length ? 1e9 : inA ? a[iA].to : a[iA].from 408 | let nextB = iB == b.length ? 1e9 : inB ? b[iB].to : b[iB].from 409 | if (inA != inB) { 410 | let start = Math.max(pos, from), end = Math.min(nextA, nextB, to) 411 | if (start < end) result.push(new Range(start, end)) 412 | } 413 | pos = Math.min(nextA, nextB) 414 | if (pos == 1e9) break 415 | if (nextA == pos) { 416 | if (!inA) inA = true 417 | else { inA = false; iA++ } 418 | } 419 | if (nextB == pos) { 420 | if (!inB) inB = true 421 | else { inB = false; iB++ } 422 | } 423 | } 424 | return result 425 | } 426 | 427 | // Given a number of fragments for the outer tree, and a set of ranges 428 | // to parse, find fragments for inner trees mounted around those 429 | // ranges, if any. 430 | function enterFragments(mounts: readonly ReusableMount[], ranges: readonly {from: number, to: number}[]) { 431 | let result: TreeFragment[] = [] 432 | for (let {pos, mount, frag} of mounts) { 433 | let startPos = pos + (mount.overlay ? mount.overlay[0].from : 0), endPos = startPos + mount.tree.length 434 | let from = Math.max(frag.from, startPos), to = Math.min(frag.to, endPos) 435 | if (mount.overlay) { 436 | let overlay = mount.overlay.map(r => new Range(r.from + pos, r.to + pos)) 437 | let changes = findCoverChanges(ranges, overlay, from, to) 438 | for (let i = 0, pos = from;; i++) { 439 | let last = i == changes.length, end = last ? to : changes[i].from 440 | if (end > pos) 441 | result.push(new TreeFragment(pos, end, mount.tree, -startPos, 442 | frag.from >= pos || frag.openStart, frag.to <= end || frag.openEnd)) 443 | if (last) break 444 | pos = changes[i].to 445 | } 446 | } else { 447 | result.push(new TreeFragment(from, to, mount.tree, -startPos, 448 | frag.from >= startPos || frag.openStart, frag.to <= endPos || frag.openEnd)) 449 | } 450 | } 451 | return result 452 | } 453 | -------------------------------------------------------------------------------- /src/parse.ts: -------------------------------------------------------------------------------- 1 | import {Tree, Range} from "./tree" 2 | 3 | /// The [`TreeFragment.applyChanges`](#common.TreeFragment^applyChanges) 4 | /// method expects changed ranges in this format. 5 | export interface ChangedRange { 6 | /// The start of the change in the start document 7 | fromA: number 8 | /// The end of the change in the start document 9 | toA: number 10 | /// The start of the replacement in the new document 11 | fromB: number 12 | /// The end of the replacement in the new document 13 | toB: number 14 | } 15 | 16 | const enum Open { Start = 1, End = 2 } 17 | 18 | /// Tree fragments are used during [incremental 19 | /// parsing](#common.Parser.startParse) to track parts of old trees 20 | /// that can be reused in a new parse. An array of fragments is used 21 | /// to track regions of an old tree whose nodes might be reused in new 22 | /// parses. Use the static 23 | /// [`applyChanges`](#common.TreeFragment^applyChanges) method to 24 | /// update fragments for document changes. 25 | export class TreeFragment { 26 | /// @internal 27 | open: Open 28 | 29 | /// Construct a tree fragment. You'll usually want to use 30 | /// [`addTree`](#common.TreeFragment^addTree) and 31 | /// [`applyChanges`](#common.TreeFragment^applyChanges) instead of 32 | /// calling this directly. 33 | constructor( 34 | /// The start of the unchanged range pointed to by this fragment. 35 | /// This refers to an offset in the _updated_ document (as opposed 36 | /// to the original tree). 37 | readonly from: number, 38 | /// The end of the unchanged range. 39 | readonly to: number, 40 | /// The tree that this fragment is based on. 41 | readonly tree: Tree, 42 | /// The offset between the fragment's tree and the document that 43 | /// this fragment can be used against. Add this when going from 44 | /// document to tree positions, subtract it to go from tree to 45 | /// document positions. 46 | readonly offset: number, 47 | openStart: boolean = false, 48 | openEnd: boolean = false 49 | ) { 50 | this.open = (openStart ? Open.Start : 0) | (openEnd ? Open.End : 0) 51 | } 52 | 53 | /// Whether the start of the fragment represents the start of a 54 | /// parse, or the end of a change. (In the second case, it may not 55 | /// be safe to reuse some nodes at the start, depending on the 56 | /// parsing algorithm.) 57 | get openStart() { return (this.open & Open.Start) > 0 } 58 | 59 | /// Whether the end of the fragment represents the end of a 60 | /// full-document parse, or the start of a change. 61 | get openEnd() { return (this.open & Open.End) > 0 } 62 | 63 | /// Create a set of fragments from a freshly parsed tree, or update 64 | /// an existing set of fragments by replacing the ones that overlap 65 | /// with a tree with content from the new tree. When `partial` is 66 | /// true, the parse is treated as incomplete, and the resulting 67 | /// fragment has [`openEnd`](#common.TreeFragment.openEnd) set to 68 | /// true. 69 | static addTree(tree: Tree, fragments: readonly TreeFragment[] = [], partial = false): readonly TreeFragment[] { 70 | let result = [new TreeFragment(0, tree.length, tree, 0, false, partial)] 71 | for (let f of fragments) if (f.to > tree.length) result.push(f) 72 | return result 73 | } 74 | 75 | /// Apply a set of edits to an array of fragments, removing or 76 | /// splitting fragments as necessary to remove edited ranges, and 77 | /// adjusting offsets for fragments that moved. 78 | static applyChanges(fragments: readonly TreeFragment[], changes: readonly ChangedRange[], minGap = 128) { 79 | if (!changes.length) return fragments 80 | let result: TreeFragment[] = [] 81 | let fI = 1, nextF = fragments.length ? fragments[0] : null 82 | for (let cI = 0, pos = 0, off = 0;; cI++) { 83 | let nextC = cI < changes.length ? changes[cI] : null 84 | let nextPos = nextC ? nextC.fromA : 1e9 85 | if (nextPos - pos >= minGap) while (nextF && nextF.from < nextPos) { 86 | let cut: TreeFragment | null = nextF 87 | if (pos >= cut.from || nextPos <= cut.to || off) { 88 | let fFrom = Math.max(cut.from, pos) - off, fTo = Math.min(cut.to, nextPos) - off 89 | cut = fFrom >= fTo ? null : new TreeFragment(fFrom, fTo, cut.tree, cut.offset + off, cI > 0, !!nextC) 90 | } 91 | if (cut) result.push(cut) 92 | if (nextF.to > nextPos) break 93 | nextF = fI < fragments.length ? fragments[fI++] : null 94 | } 95 | if (!nextC) break 96 | pos = nextC.toA 97 | off = nextC.toA - nextC.toB 98 | } 99 | return result 100 | } 101 | } 102 | 103 | /// Interface used to represent an in-progress parse, which can be 104 | /// moved forward piece-by-piece. 105 | export interface PartialParse { 106 | /// Advance the parse state by some amount. Will return the finished 107 | /// syntax tree when the parse completes. 108 | advance(): Tree | null 109 | 110 | /// The position up to which the document has been parsed. Note 111 | /// that, in multi-pass parsers, this will stay back until the last 112 | /// pass has moved past a given position. 113 | readonly parsedPos: number 114 | 115 | /// Tell the parse to not advance beyond the given position. 116 | /// `advance` will return a tree when the parse has reached the 117 | /// position. Note that, depending on the parser algorithm and the 118 | /// state of the parse when `stopAt` was called, that tree may 119 | /// contain nodes beyond the position. It is an error to call 120 | /// `stopAt` with a higher position than it's [current 121 | /// value](#common.PartialParse.stoppedAt). 122 | stopAt(pos: number): void 123 | 124 | /// Reports whether `stopAt` has been called on this parse. 125 | readonly stoppedAt: number | null 126 | } 127 | 128 | /// A superclass that parsers should extend. 129 | export abstract class Parser { 130 | /// Start a parse for a single tree. This is the method concrete 131 | /// parser implementations must implement. Called by `startParse`, 132 | /// with the optional arguments resolved. 133 | abstract createParse( 134 | input: Input, 135 | fragments: readonly TreeFragment[], 136 | ranges: readonly {from: number, to: number}[] 137 | ): PartialParse 138 | 139 | /// Start a parse, returning a [partial parse](#common.PartialParse) 140 | /// object. [`fragments`](#common.TreeFragment) can be passed in to 141 | /// make the parse incremental. 142 | /// 143 | /// By default, the entire input is parsed. You can pass `ranges`, 144 | /// which should be a sorted array of non-empty, non-overlapping 145 | /// ranges, to parse only those ranges. The tree returned in that 146 | /// case will start at `ranges[0].from`. 147 | startParse( 148 | input: Input | string, 149 | fragments?: readonly TreeFragment[], 150 | ranges?: readonly {from: number, to: number}[] 151 | ): PartialParse { 152 | if (typeof input == "string") input = new StringInput(input) 153 | ranges = !ranges ? [new Range(0, input.length)] : ranges.length ? ranges.map(r => new Range(r.from, r.to)) : [new Range(0, 0)] 154 | return this.createParse(input, fragments || [], ranges) 155 | } 156 | 157 | /// Run a full parse, returning the resulting tree. 158 | parse( 159 | input: Input | string, 160 | fragments?: readonly TreeFragment[], 161 | ranges?: readonly {from: number, to: number}[] 162 | ) { 163 | let parse = this.startParse(input, fragments, ranges) 164 | for (;;) { 165 | let done = parse.advance() 166 | if (done) return done 167 | } 168 | } 169 | } 170 | 171 | /// This is the interface parsers use to access the document. To run 172 | /// Lezer directly on your own document data structure, you have to 173 | /// write an implementation of it. 174 | export interface Input { 175 | /// The length of the document. 176 | readonly length: number 177 | /// Get the chunk after the given position. The returned string 178 | /// should start at `from` and, if that isn't the end of the 179 | /// document, may be of any length greater than zero. 180 | chunk(from: number): string 181 | /// Indicates whether the chunks already end at line breaks, so that 182 | /// client code that wants to work by-line can avoid re-scanning 183 | /// them for line breaks. When this is true, the result of `chunk()` 184 | /// should either be a single line break, or the content between 185 | /// `from` and the next line break. 186 | readonly lineChunks: boolean 187 | /// Read the part of the document between the given positions. 188 | read(from: number, to: number): string 189 | } 190 | 191 | class StringInput implements Input { 192 | constructor(readonly string: string) {} 193 | 194 | get length() { return this.string.length } 195 | 196 | chunk(from: number) { return this.string.slice(from) } 197 | 198 | get lineChunks() { return false } 199 | 200 | read(from: number, to: number) { return this.string.slice(from, to) } 201 | } 202 | 203 | /// Parse wrapper functions are supported by some parsers to inject 204 | /// additional parsing logic. 205 | export type ParseWrapper = ( 206 | inner: PartialParse, 207 | input: Input, 208 | fragments: readonly TreeFragment[], 209 | ranges: readonly {from: number, to: number}[] 210 | ) => PartialParse 211 | -------------------------------------------------------------------------------- /src/tree.ts: -------------------------------------------------------------------------------- 1 | import {Parser} from "./parse" 2 | 3 | /// The default maximum length of a `TreeBuffer` node. 4 | export const DefaultBufferLength = 1024 5 | 6 | let nextPropID = 0 7 | 8 | export class Range { 9 | constructor(readonly from: number, readonly to: number) {} 10 | } 11 | 12 | /// Each [node type](#common.NodeType) or [individual tree](#common.Tree) 13 | /// can have metadata associated with it in props. Instances of this 14 | /// class represent prop names. 15 | export class NodeProp { 16 | /// @internal 17 | id: number 18 | 19 | /// Indicates whether this prop is stored per [node 20 | /// type](#common.NodeType) or per [tree node](#common.Tree). 21 | perNode: boolean 22 | 23 | /// A method that deserializes a value of this prop from a string. 24 | /// Can be used to allow a prop to be directly written in a grammar 25 | /// file. 26 | deserialize: (str: string) => T 27 | 28 | /// Create a new node prop type. 29 | constructor(config: { 30 | /// The [deserialize](#common.NodeProp.deserialize) function to 31 | /// use for this prop, used for example when directly providing 32 | /// the prop from a grammar file. Defaults to a function that 33 | /// raises an error. 34 | deserialize?: (str: string) => T, 35 | /// By default, node props are stored in the [node 36 | /// type](#common.NodeType). It can sometimes be useful to directly 37 | /// store information (usually related to the parsing algorithm) 38 | /// in [nodes](#common.Tree) themselves. Set this to true to enable 39 | /// that for this prop. 40 | perNode?: boolean 41 | } = {}) { 42 | this.id = nextPropID++ 43 | this.perNode = !!config.perNode 44 | this.deserialize = config.deserialize || (() => { 45 | throw new Error("This node type doesn't define a deserialize function") 46 | }) 47 | } 48 | 49 | /// This is meant to be used with 50 | /// [`NodeSet.extend`](#common.NodeSet.extend) or 51 | /// [`LRParser.configure`](#lr.ParserConfig.props) to compute 52 | /// prop values for each node type in the set. Takes a [match 53 | /// object](#common.NodeType^match) or function that returns undefined 54 | /// if the node type doesn't get this prop, and the prop's value if 55 | /// it does. 56 | add(match: {[selector: string]: T} | ((type: NodeType) => T | undefined)): NodePropSource { 57 | if (this.perNode) throw new RangeError("Can't add per-node props to node types") 58 | if (typeof match != "function") match = NodeType.match(match) 59 | return (type) => { 60 | let result = (match as (type: NodeType) => T | undefined)(type) 61 | return result === undefined ? null : [this, result] 62 | } 63 | } 64 | 65 | /// Prop that is used to describe matching delimiters. For opening 66 | /// delimiters, this holds an array of node names (written as a 67 | /// space-separated string when declaring this prop in a grammar) 68 | /// for the node types of closing delimiters that match it. 69 | static closedBy = new NodeProp({deserialize: str => str.split(" ")}) 70 | 71 | /// The inverse of [`closedBy`](#common.NodeProp^closedBy). This is 72 | /// attached to closing delimiters, holding an array of node names 73 | /// of types of matching opening delimiters. 74 | static openedBy = new NodeProp({deserialize: str => str.split(" ")}) 75 | 76 | /// Used to assign node types to groups (for example, all node 77 | /// types that represent an expression could be tagged with an 78 | /// `"Expression"` group). 79 | static group = new NodeProp({deserialize: str => str.split(" ")}) 80 | 81 | /// Attached to nodes to indicate these should be 82 | /// [displayed](https://codemirror.net/docs/ref/#language.syntaxTree) 83 | /// in a bidirectional text isolate, so that direction-neutral 84 | /// characters on their sides don't incorrectly get associated with 85 | /// surrounding text. You'll generally want to set this for nodes 86 | /// that contain arbitrary text, like strings and comments, and for 87 | /// nodes that appear _inside_ arbitrary text, like HTML tags. When 88 | /// not given a value, in a grammar declaration, defaults to 89 | /// `"auto"`. 90 | static isolate = new NodeProp<"rtl" | "ltr" | "auto">({deserialize: value => { 91 | if (value && value != "rtl" && value != "ltr" && value != "auto") 92 | throw new RangeError("Invalid value for isolate: " + value) 93 | return (value as any) || "auto" 94 | }}) 95 | 96 | /// The hash of the [context](#lr.ContextTracker.constructor) 97 | /// that the node was parsed in, if any. Used to limit reuse of 98 | /// contextual nodes. 99 | static contextHash = new NodeProp({perNode: true}) 100 | 101 | /// The distance beyond the end of the node that the tokenizer 102 | /// looked ahead for any of the tokens inside the node. (The LR 103 | /// parser only stores this when it is larger than 25, for 104 | /// efficiency reasons.) 105 | static lookAhead = new NodeProp({perNode: true}) 106 | 107 | /// This per-node prop is used to replace a given node, or part of a 108 | /// node, with another tree. This is useful to include trees from 109 | /// different languages in mixed-language parsers. 110 | static mounted = new NodeProp({perNode: true}) 111 | } 112 | 113 | /// A mounted tree, which can be [stored](#common.NodeProp^mounted) on 114 | /// a tree node to indicate that parts of its content are 115 | /// represented by another tree. 116 | export class MountedTree { 117 | constructor( 118 | /// The inner tree. 119 | readonly tree: Tree, 120 | /// If this is null, this tree replaces the entire node (it will 121 | /// be included in the regular iteration instead of its host 122 | /// node). If not, only the given ranges are considered to be 123 | /// covered by this tree. This is used for trees that are mixed in 124 | /// a way that isn't strictly hierarchical. Such mounted trees are 125 | /// only entered by [`resolveInner`](#common.Tree.resolveInner) 126 | /// and [`enter`](#common.SyntaxNode.enter). 127 | readonly overlay: readonly {from: number, to: number}[] | null, 128 | /// The parser used to create this subtree. 129 | readonly parser: Parser 130 | ) {} 131 | 132 | /// @internal 133 | static get(tree: Tree | null): MountedTree | null { 134 | return tree && tree.props && tree.props[NodeProp.mounted.id] 135 | } 136 | } 137 | 138 | /// Type returned by [`NodeProp.add`](#common.NodeProp.add). Describes 139 | /// whether a prop should be added to a given node type in a node set, 140 | /// and what value it should have. 141 | export type NodePropSource = (type: NodeType) => null | [NodeProp, any] 142 | 143 | // Note: this is duplicated in lr/src/constants.ts 144 | const enum NodeFlag { 145 | Top = 1, 146 | Skipped = 2, 147 | Error = 4, 148 | Anonymous = 8 149 | } 150 | 151 | const noProps: {[propID: number]: any} = Object.create(null) 152 | 153 | /// Each node in a syntax tree has a node type associated with it. 154 | export class NodeType { 155 | /// @internal 156 | constructor( 157 | /// The name of the node type. Not necessarily unique, but if the 158 | /// grammar was written properly, different node types with the 159 | /// same name within a node set should play the same semantic 160 | /// role. 161 | readonly name: string, 162 | /// @internal 163 | readonly props: {readonly [prop: number]: any}, 164 | /// The id of this node in its set. Corresponds to the term ids 165 | /// used in the parser. 166 | readonly id: number, 167 | /// @internal 168 | readonly flags: number = 0) {} 169 | 170 | /// Define a node type. 171 | static define(spec: { 172 | /// The ID of the node type. When this type is used in a 173 | /// [set](#common.NodeSet), the ID must correspond to its index in 174 | /// the type array. 175 | id: number, 176 | /// The name of the node type. Leave empty to define an anonymous 177 | /// node. 178 | name?: string, 179 | /// [Node props](#common.NodeProp) to assign to the type. The value 180 | /// given for any given prop should correspond to the prop's type. 181 | props?: readonly ([NodeProp, any] | NodePropSource)[], 182 | /// Whether this is a [top node](#common.NodeType.isTop). 183 | top?: boolean, 184 | /// Whether this node counts as an [error 185 | /// node](#common.NodeType.isError). 186 | error?: boolean, 187 | /// Whether this node is a [skipped](#common.NodeType.isSkipped) 188 | /// node. 189 | skipped?: boolean 190 | }) { 191 | let props = spec.props && spec.props.length ? Object.create(null) : noProps 192 | let flags = (spec.top ? NodeFlag.Top : 0) | (spec.skipped ? NodeFlag.Skipped : 0) | 193 | (spec.error ? NodeFlag.Error : 0) | (spec.name == null ? NodeFlag.Anonymous : 0) 194 | let type = new NodeType(spec.name || "", props, spec.id, flags) 195 | if (spec.props) for (let src of spec.props) { 196 | if (!Array.isArray(src)) src = src(type)! 197 | if (src) { 198 | if (src[0].perNode) throw new RangeError("Can't store a per-node prop on a node type") 199 | props[src[0].id] = src[1] 200 | } 201 | } 202 | return type 203 | } 204 | 205 | /// Retrieves a node prop for this type. Will return `undefined` if 206 | /// the prop isn't present on this node. 207 | prop(prop: NodeProp): T | undefined { return this.props[prop.id] } 208 | 209 | /// True when this is the top node of a grammar. 210 | get isTop() { return (this.flags & NodeFlag.Top) > 0 } 211 | 212 | /// True when this node is produced by a skip rule. 213 | get isSkipped() { return (this.flags & NodeFlag.Skipped) > 0 } 214 | 215 | /// Indicates whether this is an error node. 216 | get isError() { return (this.flags & NodeFlag.Error) > 0 } 217 | 218 | /// When true, this node type doesn't correspond to a user-declared 219 | /// named node, for example because it is used to cache repetition. 220 | get isAnonymous() { return (this.flags & NodeFlag.Anonymous) > 0 } 221 | 222 | /// Returns true when this node's name or one of its 223 | /// [groups](#common.NodeProp^group) matches the given string. 224 | is(name: string | number) { 225 | if (typeof name == 'string') { 226 | if (this.name == name) return true 227 | let group = this.prop(NodeProp.group) 228 | return group ? group.indexOf(name) > -1 : false 229 | } 230 | return this.id == name 231 | } 232 | 233 | /// An empty dummy node type to use when no actual type is available. 234 | static none: NodeType = new NodeType("", Object.create(null), 0, NodeFlag.Anonymous) 235 | 236 | /// Create a function from node types to arbitrary values by 237 | /// specifying an object whose property names are node or 238 | /// [group](#common.NodeProp^group) names. Often useful with 239 | /// [`NodeProp.add`](#common.NodeProp.add). You can put multiple 240 | /// names, separated by spaces, in a single property name to map 241 | /// multiple node names to a single value. 242 | static match(map: {[selector: string]: T}): (node: NodeType) => T | undefined { 243 | let direct = Object.create(null) 244 | for (let prop in map) 245 | for (let name of prop.split(" ")) direct[name] = map[prop] 246 | return (node: NodeType) => { 247 | for (let groups = node.prop(NodeProp.group), i = -1; i < (groups ? groups.length : 0); i++) { 248 | let found = direct[i < 0 ? node.name : groups![i]] 249 | if (found) return found 250 | } 251 | } 252 | } 253 | } 254 | 255 | /// A node set holds a collection of node types. It is used to 256 | /// compactly represent trees by storing their type ids, rather than a 257 | /// full pointer to the type object, in a numeric array. Each parser 258 | /// [has](#lr.LRParser.nodeSet) a node set, and [tree 259 | /// buffers](#common.TreeBuffer) can only store collections of nodes 260 | /// from the same set. A set can have a maximum of 2**16 (65536) node 261 | /// types in it, so that the ids fit into 16-bit typed array slots. 262 | export class NodeSet { 263 | /// Create a set with the given types. The `id` property of each 264 | /// type should correspond to its position within the array. 265 | constructor( 266 | /// The node types in this set, by id. 267 | readonly types: readonly NodeType[] 268 | ) { 269 | for (let i = 0; i < types.length; i++) if (types[i].id != i) 270 | throw new RangeError("Node type ids should correspond to array positions when creating a node set") 271 | } 272 | 273 | /// Create a copy of this set with some node properties added. The 274 | /// arguments to this method can be created with 275 | /// [`NodeProp.add`](#common.NodeProp.add). 276 | extend(...props: NodePropSource[]): NodeSet { 277 | let newTypes: NodeType[] = [] 278 | for (let type of this.types) { 279 | let newProps: null | {[id: number]: any} = null 280 | for (let source of props) { 281 | let add = source(type) 282 | if (add) { 283 | if (!newProps) newProps = Object.assign({}, type.props) 284 | newProps[add[0].id] = add[1] 285 | } 286 | } 287 | newTypes.push(newProps ? new NodeType(type.name, newProps, type.id, type.flags) : type) 288 | } 289 | return new NodeSet(newTypes) 290 | } 291 | } 292 | 293 | const CachedNode = new WeakMap(), CachedInnerNode = new WeakMap() 294 | 295 | /// Options that control iteration. Can be combined with the `|` 296 | /// operator to enable multiple ones. 297 | export enum IterMode { 298 | /// When enabled, iteration will only visit [`Tree`](#common.Tree) 299 | /// objects, not nodes packed into 300 | /// [`TreeBuffer`](#common.TreeBuffer)s. 301 | ExcludeBuffers = 1, 302 | /// Enable this to make iteration include anonymous nodes (such as 303 | /// the nodes that wrap repeated grammar constructs into a balanced 304 | /// tree). 305 | IncludeAnonymous = 2, 306 | /// By default, regular [mounted](#common.NodeProp^mounted) nodes 307 | /// replace their base node in iteration. Enable this to ignore them 308 | /// instead. 309 | IgnoreMounts = 4, 310 | /// This option only applies in 311 | /// [`enter`](#common.SyntaxNode.enter)-style methods. It tells the 312 | /// library to not enter mounted overlays if one covers the given 313 | /// position. 314 | IgnoreOverlays = 8, 315 | } 316 | 317 | /// A piece of syntax tree. There are two ways to approach these 318 | /// trees: the way they are actually stored in memory, and the 319 | /// convenient way. 320 | /// 321 | /// Syntax trees are stored as a tree of `Tree` and `TreeBuffer` 322 | /// objects. By packing detail information into `TreeBuffer` leaf 323 | /// nodes, the representation is made a lot more memory-efficient. 324 | /// 325 | /// However, when you want to actually work with tree nodes, this 326 | /// representation is very awkward, so most client code will want to 327 | /// use the [`TreeCursor`](#common.TreeCursor) or 328 | /// [`SyntaxNode`](#common.SyntaxNode) interface instead, which provides 329 | /// a view on some part of this data structure, and can be used to 330 | /// move around to adjacent nodes. 331 | export class Tree { 332 | /// @internal 333 | props: null | {[id: number]: any} = null 334 | 335 | /// Construct a new tree. See also [`Tree.build`](#common.Tree^build). 336 | constructor( 337 | /// The type of the top node. 338 | readonly type: NodeType, 339 | /// This node's child nodes. 340 | readonly children: readonly (Tree | TreeBuffer)[], 341 | /// The positions (offsets relative to the start of this tree) of 342 | /// the children. 343 | readonly positions: readonly number[], 344 | /// The total length of this tree 345 | readonly length: number, 346 | /// Per-node [node props](#common.NodeProp) to associate with this node. 347 | props?: readonly [NodeProp | number, any][] 348 | ) { 349 | if (props && props.length) { 350 | this.props = Object.create(null) 351 | for (let [prop, value] of props) this.props![typeof prop == "number" ? prop : prop.id] = value 352 | } 353 | } 354 | 355 | /// @internal 356 | toString(): string { 357 | let mounted = MountedTree.get(this) 358 | if (mounted && !mounted.overlay) return mounted.tree.toString() 359 | let children = "" 360 | for (let ch of this.children) { 361 | let str = ch.toString() 362 | if (str) { 363 | if (children) children += "," 364 | children += str 365 | } 366 | } 367 | return !this.type.name ? children : 368 | (/\W/.test(this.type.name) && !this.type.isError ? JSON.stringify(this.type.name) : this.type.name) + 369 | (children.length ? "(" + children + ")" : "") 370 | } 371 | 372 | /// The empty tree 373 | static empty = new Tree(NodeType.none, [], [], 0) 374 | 375 | /// Get a [tree cursor](#common.TreeCursor) positioned at the top of 376 | /// the tree. Mode can be used to [control](#common.IterMode) which 377 | /// nodes the cursor visits. 378 | cursor(mode: IterMode = 0 as IterMode) { 379 | return new TreeCursor(this.topNode as TreeNode, mode) 380 | } 381 | 382 | /// Get a [tree cursor](#common.TreeCursor) pointing into this tree 383 | /// at the given position and side (see 384 | /// [`moveTo`](#common.TreeCursor.moveTo). 385 | cursorAt(pos: number, side: -1 | 0 | 1 = 0, mode: IterMode = 0 as IterMode): TreeCursor { 386 | let scope = CachedNode.get(this) || this.topNode 387 | let cursor = new TreeCursor(scope as TreeNode | BufferNode) 388 | cursor.moveTo(pos, side) 389 | CachedNode.set(this, cursor._tree) 390 | return cursor 391 | } 392 | 393 | /// Get a [syntax node](#common.SyntaxNode) object for the top of the 394 | /// tree. 395 | get topNode(): SyntaxNode { 396 | return new TreeNode(this, 0, 0, null) 397 | } 398 | 399 | /// Get the [syntax node](#common.SyntaxNode) at the given position. 400 | /// If `side` is -1, this will move into nodes that end at the 401 | /// position. If 1, it'll move into nodes that start at the 402 | /// position. With 0, it'll only enter nodes that cover the position 403 | /// from both sides. 404 | /// 405 | /// Note that this will not enter 406 | /// [overlays](#common.MountedTree.overlay), and you often want 407 | /// [`resolveInner`](#common.Tree.resolveInner) instead. 408 | resolve(pos: number, side: -1 | 0 | 1 = 0) { 409 | let node = resolveNode(CachedNode.get(this) || this.topNode, pos, side, false) 410 | CachedNode.set(this, node) 411 | return node 412 | } 413 | 414 | /// Like [`resolve`](#common.Tree.resolve), but will enter 415 | /// [overlaid](#common.MountedTree.overlay) nodes, producing a syntax node 416 | /// pointing into the innermost overlaid tree at the given position 417 | /// (with parent links going through all parent structure, including 418 | /// the host trees). 419 | resolveInner(pos: number, side: -1 | 0 | 1 = 0) { 420 | let node = resolveNode(CachedInnerNode.get(this) || this.topNode, pos, side, true) 421 | CachedInnerNode.set(this, node) 422 | return node 423 | } 424 | 425 | /// In some situations, it can be useful to iterate through all 426 | /// nodes around a position, including those in overlays that don't 427 | /// directly cover the position. This method gives you an iterator 428 | /// that will produce all nodes, from small to big, around the given 429 | /// position. 430 | resolveStack(pos: number, side: -1 | 0 | 1 = 0): NodeIterator { 431 | return stackIterator(this, pos, side) 432 | } 433 | 434 | /// Iterate over the tree and its children, calling `enter` for any 435 | /// node that touches the `from`/`to` region (if given) before 436 | /// running over such a node's children, and `leave` (if given) when 437 | /// leaving the node. When `enter` returns `false`, that node will 438 | /// not have its children iterated over (or `leave` called). 439 | iterate(spec: { 440 | enter(node: SyntaxNodeRef): boolean | void, 441 | leave?(node: SyntaxNodeRef): void, 442 | from?: number, 443 | to?: number, 444 | mode?: IterMode 445 | }) { 446 | let {enter, leave, from = 0, to = this.length} = spec 447 | let mode = spec.mode || 0, anon = (mode & IterMode.IncludeAnonymous) > 0 448 | for (let c = this.cursor(mode | IterMode.IncludeAnonymous);;) { 449 | let entered = false 450 | if (c.from <= to && c.to >= from && (!anon && c.type.isAnonymous || enter(c) !== false)) { 451 | if (c.firstChild()) continue 452 | entered = true 453 | } 454 | for (;;) { 455 | if (entered && leave && (anon || !c.type.isAnonymous)) leave(c) 456 | if (c.nextSibling()) break 457 | if (!c.parent()) return 458 | entered = true 459 | } 460 | } 461 | } 462 | 463 | /// Get the value of the given [node prop](#common.NodeProp) for this 464 | /// node. Works with both per-node and per-type props. 465 | prop(prop: NodeProp): T | undefined { 466 | return !prop.perNode ? this.type.prop(prop) : this.props ? this.props[prop.id] : undefined 467 | } 468 | 469 | /// Returns the node's [per-node props](#common.NodeProp.perNode) in a 470 | /// format that can be passed to the [`Tree`](#common.Tree) 471 | /// constructor. 472 | get propValues(): readonly [NodeProp | number, any][] { 473 | let result: [NodeProp | number, any][] = [] 474 | if (this.props) for (let id in this.props) result.push([+id, this.props[id]]) 475 | return result 476 | } 477 | 478 | /// Balance the direct children of this tree, producing a copy of 479 | /// which may have children grouped into subtrees with type 480 | /// [`NodeType.none`](#common.NodeType^none). 481 | balance(config: { 482 | /// Function to create the newly balanced subtrees. 483 | makeTree?: (children: readonly (Tree | TreeBuffer)[], positions: readonly number[], length: number) => Tree 484 | } = {}) { 485 | return this.children.length <= Balance.BranchFactor ? this : 486 | balanceRange(NodeType.none, this.children, this.positions, 0, this.children.length, 0, this.length, 487 | (children, positions, length) => new Tree(this.type, children, positions, length, this.propValues), 488 | config.makeTree || ((children, positions, length) => new Tree(NodeType.none, children, positions, length))) 489 | } 490 | 491 | /// Build a tree from a postfix-ordered buffer of node information, 492 | /// or a cursor over such a buffer. 493 | static build(data: BuildData) { return buildTree(data) } 494 | } 495 | 496 | /// Represents a sequence of nodes. 497 | export type NodeIterator = {node: SyntaxNode, next: NodeIterator | null} 498 | 499 | type BuildData = { 500 | /// The buffer or buffer cursor to read the node data from. 501 | /// 502 | /// When this is an array, it should contain four values for every 503 | /// node in the tree. 504 | /// 505 | /// - The first holds the node's type, as a node ID pointing into 506 | /// the given `NodeSet`. 507 | /// - The second holds the node's start offset. 508 | /// - The third the end offset. 509 | /// - The fourth the amount of space taken up in the array by this 510 | /// node and its children. Since there's four values per node, 511 | /// this is the total number of nodes inside this node (children 512 | /// and transitive children) plus one for the node itself, times 513 | /// four. 514 | /// 515 | /// Parent nodes should appear _after_ child nodes in the array. As 516 | /// an example, a node of type 10 spanning positions 0 to 4, with 517 | /// two children, of type 11 and 12, might look like this: 518 | /// 519 | /// [11, 0, 1, 4, 12, 2, 4, 4, 10, 0, 4, 12] 520 | buffer: BufferCursor | readonly number[], 521 | /// The node types to use. 522 | nodeSet: NodeSet, 523 | /// The id of the top node type. 524 | topID: number, 525 | /// The position the tree should start at. Defaults to 0. 526 | start?: number, 527 | /// The position in the buffer where the function should stop 528 | /// reading. Defaults to 0. 529 | bufferStart?: number, 530 | /// The length of the wrapping node. The end offset of the last 531 | /// child is used when not provided. 532 | length?: number, 533 | /// The maximum buffer length to use. Defaults to 534 | /// [`DefaultBufferLength`](#common.DefaultBufferLength). 535 | maxBufferLength?: number, 536 | /// An optional array holding reused nodes that the buffer can refer 537 | /// to. 538 | reused?: readonly Tree[], 539 | /// The first node type that indicates repeat constructs in this 540 | /// grammar. 541 | minRepeatType?: number 542 | } 543 | 544 | /// This is used by `Tree.build` as an abstraction for iterating over 545 | /// a tree buffer. A cursor initially points at the very last element 546 | /// in the buffer. Every time `next()` is called it moves on to the 547 | /// previous one. 548 | export interface BufferCursor { 549 | /// The current buffer position (four times the number of nodes 550 | /// remaining). 551 | pos: number 552 | /// The node ID of the next node in the buffer. 553 | id: number 554 | /// The start position of the next node in the buffer. 555 | start: number 556 | /// The end position of the next node. 557 | end: number 558 | /// The size of the next node (the number of nodes inside, counting 559 | /// the node itself, times 4). 560 | size: number 561 | /// Moves `this.pos` down by 4. 562 | next(): void 563 | /// Create a copy of this cursor. 564 | fork(): BufferCursor 565 | } 566 | 567 | class FlatBufferCursor implements BufferCursor { 568 | constructor(readonly buffer: readonly number[], public index: number) {} 569 | 570 | get id() { return this.buffer[this.index - 4] } 571 | get start() { return this.buffer[this.index - 3] } 572 | get end() { return this.buffer[this.index - 2] } 573 | get size() { return this.buffer[this.index - 1] } 574 | 575 | get pos() { return this.index } 576 | 577 | next() { this.index -= 4 } 578 | 579 | fork() { return new FlatBufferCursor(this.buffer, this.index) } 580 | } 581 | 582 | /// Tree buffers contain (type, start, end, endIndex) quads for each 583 | /// node. In such a buffer, nodes are stored in prefix order (parents 584 | /// before children, with the endIndex of the parent indicating which 585 | /// children belong to it). 586 | export class TreeBuffer { 587 | /// Create a tree buffer. 588 | constructor( 589 | /// The buffer's content. 590 | readonly buffer: Uint16Array, 591 | /// The total length of the group of nodes in the buffer. 592 | readonly length: number, 593 | /// The node set used in this buffer. 594 | readonly set: NodeSet 595 | ) {} 596 | 597 | /// @internal 598 | get type() { return NodeType.none } 599 | 600 | /// @internal 601 | toString() { 602 | let result: string[] = [] 603 | for (let index = 0; index < this.buffer.length;) { 604 | result.push(this.childString(index)) 605 | index = this.buffer[index + 3] 606 | } 607 | return result.join(",") 608 | } 609 | 610 | /// @internal 611 | childString(index: number): string { 612 | let id = this.buffer[index], endIndex = this.buffer[index + 3] 613 | let type = this.set.types[id], result = type.name 614 | if (/\W/.test(result) && !type.isError) result = JSON.stringify(result) 615 | index += 4 616 | if (endIndex == index) return result 617 | let children: string[] = [] 618 | while (index < endIndex) { 619 | children.push(this.childString(index)) 620 | index = this.buffer[index + 3] 621 | } 622 | return result + "(" + children.join(",") + ")" 623 | } 624 | 625 | /// @internal 626 | findChild(startIndex: number, endIndex: number, dir: 1 | -1, pos: number, side: Side) { 627 | let {buffer} = this, pick = -1 628 | for (let i = startIndex; i != endIndex; i = buffer[i + 3]) { 629 | if (checkSide(side, pos, buffer[i + 1], buffer[i + 2])) { 630 | pick = i 631 | if (dir > 0) break 632 | } 633 | } 634 | return pick 635 | } 636 | 637 | /// @internal 638 | slice(startI: number, endI: number, from: number) { 639 | let b = this.buffer 640 | let copy = new Uint16Array(endI - startI), len = 0 641 | for (let i = startI, j = 0; i < endI;) { 642 | copy[j++] = b[i++] 643 | copy[j++] = b[i++] - from 644 | let to = copy[j++] = b[i++] - from 645 | copy[j++] = b[i++] - startI 646 | len = Math.max(len, to) 647 | } 648 | return new TreeBuffer(copy, len, this.set) 649 | } 650 | } 651 | 652 | /// The set of properties provided by both [`SyntaxNode`](#common.SyntaxNode) 653 | /// and [`TreeCursor`](#common.TreeCursor). Note that, if you need 654 | /// an object that is guaranteed to stay stable in the future, you 655 | /// need to use the [`node`](#common.SyntaxNodeRef.node) accessor. 656 | export interface SyntaxNodeRef { 657 | /// The start position of the node. 658 | readonly from: number 659 | /// The end position of the node. 660 | readonly to: number 661 | /// The type of the node. 662 | readonly type: NodeType 663 | /// The name of the node (`.type.name`). 664 | readonly name: string 665 | /// Get the [tree](#common.Tree) that represents the current node, 666 | /// if any. Will return null when the node is in a [tree 667 | /// buffer](#common.TreeBuffer). 668 | readonly tree: Tree | null 669 | /// Retrieve a stable [syntax node](#common.SyntaxNode) at this 670 | /// position. 671 | readonly node: SyntaxNode 672 | /// Test whether the node matches a given context—a sequence of 673 | /// direct parent nodes. Empty strings in the context array act as 674 | /// wildcards, other strings must match the ancestor node's name. 675 | matchContext(context: readonly string[]): boolean 676 | } 677 | 678 | /// A syntax node provides an immutable pointer to a given node in a 679 | /// tree. When iterating over large amounts of nodes, you may want to 680 | /// use a mutable [cursor](#common.TreeCursor) instead, which is more 681 | /// efficient. 682 | export interface SyntaxNode extends SyntaxNodeRef { 683 | /// The node's parent node, if any. 684 | parent: SyntaxNode | null 685 | /// The first child, if the node has children. 686 | firstChild: SyntaxNode | null 687 | /// The node's last child, if available. 688 | lastChild: SyntaxNode | null 689 | /// The first child that ends after `pos`. 690 | childAfter(pos: number): SyntaxNode | null 691 | /// The last child that starts before `pos`. 692 | childBefore(pos: number): SyntaxNode | null 693 | /// Enter the child at the given position. If side is -1 the child 694 | /// may end at that position, when 1 it may start there. 695 | /// 696 | /// This will by default enter 697 | /// [overlaid](#common.MountedTree.overlay) 698 | /// [mounted](#common.NodeProp^mounted) trees. You can set 699 | /// `overlays` to false to disable that. 700 | /// 701 | /// Similarly, when `buffers` is false this will not enter 702 | /// [buffers](#common.TreeBuffer), only [nodes](#common.Tree) (which 703 | /// is mostly useful when looking for props, which cannot exist on 704 | /// buffer-allocated nodes). 705 | enter(pos: number, side: -1 | 0 | 1, mode?: IterMode): SyntaxNode | null 706 | /// This node's next sibling, if any. 707 | nextSibling: SyntaxNode | null 708 | /// This node's previous sibling. 709 | prevSibling: SyntaxNode | null 710 | /// A [tree cursor](#common.TreeCursor) starting at this node. 711 | cursor(mode?: IterMode): TreeCursor 712 | /// Find the node around, before (if `side` is -1), or after (`side` 713 | /// is 1) the given position. Will look in parent nodes if the 714 | /// position is outside this node. 715 | resolve(pos: number, side?: -1 | 0 | 1): SyntaxNode 716 | /// Similar to `resolve`, but enter 717 | /// [overlaid](#common.MountedTree.overlay) nodes. 718 | resolveInner(pos: number, side?: -1 | 0 | 1): SyntaxNode 719 | /// Move the position to the innermost node before `pos` that looks 720 | /// like it is unfinished (meaning it ends in an error node or has a 721 | /// child ending in an error node right at its end). 722 | enterUnfinishedNodesBefore(pos: number): SyntaxNode 723 | /// Get a [tree](#common.Tree) for this node. Will allocate one if it 724 | /// points into a buffer. 725 | toTree(): Tree 726 | 727 | /// Get the first child of the given type (which may be a [node 728 | /// name](#common.NodeType.name) or a [group 729 | /// name](#common.NodeProp^group)). If `before` is non-null, only 730 | /// return children that occur somewhere after a node with that name 731 | /// or group. If `after` is non-null, only return children that 732 | /// occur somewhere before a node with that name or group. 733 | getChild(type: string | number, before?: string | number | null, after?: string | number | null): SyntaxNode | null 734 | 735 | /// Like [`getChild`](#common.SyntaxNode.getChild), but return all 736 | /// matching children, not just the first. 737 | getChildren(type: string | number, before?: string | number | null, after?: string | number | null): SyntaxNode[] 738 | } 739 | 740 | const enum Side { 741 | Before = -2, 742 | AtOrBefore = -1, 743 | Around = 0, 744 | AtOrAfter = 1, 745 | After = 2, 746 | DontCare = 4 747 | } 748 | 749 | function checkSide(side: Side, pos: number, from: number, to: number) { 750 | switch (side) { 751 | case Side.Before: return from < pos 752 | case Side.AtOrBefore: return to >= pos && from < pos 753 | case Side.Around: return from < pos && to > pos 754 | case Side.AtOrAfter: return from <= pos && to > pos 755 | case Side.After: return to > pos 756 | case Side.DontCare: return true 757 | } 758 | } 759 | 760 | function resolveNode(node: SyntaxNode, pos: number, side: -1 | 0 | 1, overlays: boolean): SyntaxNode { 761 | // Move up to a node that actually holds the position, if possible 762 | while (node.from == node.to || 763 | (side < 1 ? node.from >= pos : node.from > pos) || 764 | (side > -1 ? node.to <= pos : node.to < pos)) { 765 | let parent = !overlays && node instanceof TreeNode && node.index < 0 ? null : node.parent 766 | if (!parent) return node 767 | node = parent 768 | } 769 | let mode = overlays ? 0 : IterMode.IgnoreOverlays 770 | // Must go up out of overlays when those do not overlap with pos 771 | if (overlays) for (let scan: SyntaxNode | null = node, parent = scan.parent; parent; scan = parent, parent = scan.parent) { 772 | if (scan instanceof TreeNode && scan.index < 0 && parent.enter(pos, side, mode)?.from != scan.from) 773 | node = parent 774 | } 775 | for (;;) { 776 | let inner = node.enter(pos, side, mode) 777 | if (!inner) return node 778 | node = inner 779 | } 780 | } 781 | 782 | abstract class BaseNode implements SyntaxNode { 783 | abstract from: number 784 | abstract to: number 785 | abstract type: NodeType 786 | abstract name: string 787 | abstract tree: Tree | null 788 | abstract parent: SyntaxNode | null 789 | abstract firstChild: SyntaxNode | null 790 | abstract lastChild: SyntaxNode | null 791 | abstract childAfter(pos: number): SyntaxNode | null 792 | abstract childBefore(pos: number): SyntaxNode | null 793 | abstract enter(pos: number, side: -1 | 0 | 1, mode?: IterMode): SyntaxNode | null 794 | abstract nextSibling: SyntaxNode | null 795 | abstract prevSibling: SyntaxNode | null 796 | abstract toTree(): Tree 797 | 798 | cursor(mode: IterMode = 0 as IterMode) { return new TreeCursor(this as any, mode) } 799 | 800 | getChild(type: string | number, before: string | number | null = null, after: string | number | null = null) { 801 | let r = getChildren(this, type, before, after) 802 | return r.length ? r[0] : null 803 | } 804 | 805 | getChildren(type: string | number, before: string | number | null = null, after: string | number | null = null): SyntaxNode[] { 806 | return getChildren(this, type, before, after) 807 | } 808 | 809 | resolve(pos: number, side: -1 | 0 | 1 = 0): SyntaxNode { 810 | return resolveNode(this, pos, side, false) 811 | } 812 | 813 | resolveInner(pos: number, side: -1 | 0 | 1 = 0): SyntaxNode { 814 | return resolveNode(this, pos, side, true) 815 | } 816 | 817 | matchContext(context: readonly string[]): boolean { 818 | return matchNodeContext(this.parent, context) 819 | } 820 | 821 | enterUnfinishedNodesBefore(pos: number) { 822 | let scan = this.childBefore(pos), node: SyntaxNode = this 823 | while (scan) { 824 | let last = scan.lastChild 825 | if (!last || last.to != scan.to) break 826 | if (last.type.isError && last.from == last.to) { 827 | node = scan 828 | scan = last.prevSibling 829 | } else { 830 | scan = last 831 | } 832 | } 833 | return node 834 | } 835 | 836 | get node() { return this } 837 | 838 | get next() { return this.parent } 839 | } 840 | 841 | export class TreeNode extends BaseNode implements SyntaxNode { 842 | constructor(readonly _tree: Tree, 843 | readonly from: number, 844 | // Index in parent node, set to -1 if the node is not a direct child of _parent.node (overlay) 845 | readonly index: number, 846 | readonly _parent: TreeNode | null) { super() } 847 | 848 | get type() { return this._tree.type } 849 | 850 | get name() { return this._tree.type.name } 851 | 852 | get to() { return this.from + this._tree.length } 853 | 854 | nextChild(i: number, dir: 1 | -1, pos: number, side: Side, mode: IterMode = 0 as IterMode): TreeNode | BufferNode | null { 855 | for (let parent: TreeNode = this;;) { 856 | for (let {children, positions} = parent._tree, e = dir > 0 ? children.length : -1; i != e; i += dir) { 857 | let next = children[i], start = positions[i] + parent.from 858 | if (!checkSide(side, pos, start, start + next.length)) continue 859 | if (next instanceof TreeBuffer) { 860 | if (mode & IterMode.ExcludeBuffers) continue 861 | let index = next.findChild(0, next.buffer.length, dir, pos - start, side) 862 | if (index > -1) return new BufferNode(new BufferContext(parent, next, i, start), null, index) 863 | } else if ((mode & IterMode.IncludeAnonymous) || (!next.type.isAnonymous || hasChild(next))) { 864 | let mounted 865 | if (!(mode & IterMode.IgnoreMounts) && (mounted = MountedTree.get(next)) && !mounted.overlay) 866 | return new TreeNode(mounted.tree, start, i, parent) 867 | let inner = new TreeNode(next, start, i, parent) 868 | return (mode & IterMode.IncludeAnonymous) || !inner.type.isAnonymous ? inner 869 | : inner.nextChild(dir < 0 ? next.children.length - 1 : 0, dir, pos, side) 870 | } 871 | } 872 | if ((mode & IterMode.IncludeAnonymous) || !parent.type.isAnonymous) return null 873 | if (parent.index >= 0) i = parent.index + dir 874 | else i = dir < 0 ? -1 : parent._parent!._tree.children.length 875 | parent = parent._parent! 876 | if (!parent) return null 877 | } 878 | } 879 | 880 | get firstChild() { return this.nextChild(0, 1, 0, Side.DontCare) } 881 | get lastChild() { return this.nextChild(this._tree.children.length - 1, -1, 0, Side.DontCare) } 882 | 883 | childAfter(pos: number) { return this.nextChild(0, 1, pos, Side.After) } 884 | childBefore(pos: number) { return this.nextChild(this._tree.children.length - 1, -1, pos, Side.Before) } 885 | 886 | enter(pos: number, side: -1 | 0 | 1, mode = 0) { 887 | let mounted 888 | if (!(mode & IterMode.IgnoreOverlays) && (mounted = MountedTree.get(this._tree)) && mounted.overlay) { 889 | let rPos = pos - this.from 890 | for (let {from, to} of mounted.overlay) { 891 | if ((side > 0 ? from <= rPos : from < rPos) && 892 | (side < 0 ? to >= rPos : to > rPos)) 893 | return new TreeNode(mounted.tree, mounted.overlay[0].from + this.from, -1, this) 894 | } 895 | } 896 | return this.nextChild(0, 1, pos, side, mode) 897 | } 898 | 899 | nextSignificantParent() { 900 | let val: TreeNode = this 901 | while (val.type.isAnonymous && val._parent) val = val._parent 902 | return val 903 | } 904 | 905 | get parent(): TreeNode | null { 906 | return this._parent ? this._parent.nextSignificantParent() : null 907 | } 908 | 909 | get nextSibling(): SyntaxNode | null { 910 | return this._parent && this.index >= 0 ? this._parent.nextChild(this.index + 1, 1, 0, Side.DontCare) : null 911 | } 912 | get prevSibling(): SyntaxNode | null { 913 | return this._parent && this.index >= 0 ? this._parent.nextChild(this.index - 1, -1, 0, Side.DontCare) : null 914 | } 915 | 916 | get tree() { return this._tree } 917 | 918 | toTree() { return this._tree } 919 | 920 | /// @internal 921 | toString() { return this._tree.toString() } 922 | } 923 | 924 | function getChildren(node: SyntaxNode, type: string | number, before: string | number | null, after: string | number | null): SyntaxNode[] { 925 | let cur = node.cursor(), result: SyntaxNode[] = [] 926 | if (!cur.firstChild()) return result 927 | if (before != null) for (let found = false; !found;) { 928 | found = cur.type.is(before) 929 | if (!cur.nextSibling()) return result 930 | } 931 | for (;;) { 932 | if (after != null && cur.type.is(after)) return result 933 | if (cur.type.is(type)) result.push(cur.node) 934 | if (!cur.nextSibling()) return after == null ? result : [] 935 | } 936 | } 937 | 938 | function matchNodeContext(node: SyntaxNode | null, context: readonly string[], i = context.length - 1): boolean { 939 | for (let p = node; i >= 0; p = p.parent) { 940 | if (!p) return false 941 | if (!p.type.isAnonymous) { 942 | if (context[i] && context[i] != p.name) return false 943 | i-- 944 | } 945 | } 946 | return true 947 | } 948 | 949 | class BufferContext { 950 | constructor(readonly parent: TreeNode, 951 | readonly buffer: TreeBuffer, 952 | readonly index: number, 953 | readonly start: number) {} 954 | } 955 | 956 | export class BufferNode extends BaseNode { 957 | type: NodeType 958 | 959 | get name() { return this.type.name } 960 | 961 | get from() { return this.context.start + this.context.buffer.buffer[this.index + 1] } 962 | 963 | get to() { return this.context.start + this.context.buffer.buffer[this.index + 2] } 964 | 965 | constructor(readonly context: BufferContext, 966 | readonly _parent: BufferNode | null, 967 | readonly index: number) { 968 | super() 969 | this.type = context.buffer.set.types[context.buffer.buffer[index]] 970 | } 971 | 972 | child(dir: 1 | -1, pos: number, side: Side): BufferNode | null { 973 | let {buffer} = this.context 974 | let index = buffer.findChild(this.index + 4, buffer.buffer[this.index + 3], dir, pos - this.context.start, side) 975 | return index < 0 ? null : new BufferNode(this.context, this, index) 976 | } 977 | 978 | get firstChild() { return this.child(1, 0, Side.DontCare) } 979 | get lastChild() { return this.child(-1, 0, Side.DontCare) } 980 | 981 | childAfter(pos: number) { return this.child(1, pos, Side.After) } 982 | childBefore(pos: number) { return this.child(-1, pos, Side.Before) } 983 | 984 | enter(pos: number, side: -1 | 0 | 1, mode: IterMode = 0 as IterMode) { 985 | if (mode & IterMode.ExcludeBuffers) return null 986 | let {buffer} = this.context 987 | let index = buffer.findChild(this.index + 4, buffer.buffer[this.index + 3], side > 0 ? 1 : -1, pos - this.context.start, side) 988 | return index < 0 ? null : new BufferNode(this.context, this, index) 989 | } 990 | 991 | get parent(): SyntaxNode | null { 992 | return this._parent || this.context.parent.nextSignificantParent() 993 | } 994 | 995 | externalSibling(dir: 1 | -1) { 996 | return this._parent ? null : this.context.parent.nextChild(this.context.index + dir, dir, 0, Side.DontCare) 997 | } 998 | 999 | get nextSibling(): SyntaxNode | null { 1000 | let {buffer} = this.context 1001 | let after = buffer.buffer[this.index + 3] 1002 | if (after < (this._parent ? buffer.buffer[this._parent.index + 3] : buffer.buffer.length)) 1003 | return new BufferNode(this.context, this._parent, after) 1004 | return this.externalSibling(1) 1005 | } 1006 | 1007 | get prevSibling(): SyntaxNode | null { 1008 | let {buffer} = this.context 1009 | let parentStart = this._parent ? this._parent.index + 4 : 0 1010 | if (this.index == parentStart) return this.externalSibling(-1) 1011 | return new BufferNode(this.context, this._parent, buffer.findChild(parentStart, this.index, -1, 0, Side.DontCare)) 1012 | } 1013 | 1014 | get tree() { return null } 1015 | 1016 | toTree() { 1017 | let children = [], positions = [] 1018 | let {buffer} = this.context 1019 | let startI = this.index + 4, endI = buffer.buffer[this.index + 3] 1020 | if (endI > startI) { 1021 | let from = buffer.buffer[this.index + 1] 1022 | children.push(buffer.slice(startI, endI, from)) 1023 | positions.push(0) 1024 | } 1025 | return new Tree(this.type, children, positions, this.to - this.from) 1026 | } 1027 | 1028 | /// @internal 1029 | toString() { return this.context.buffer.childString(this.index) } 1030 | } 1031 | 1032 | function iterStack(heads: readonly SyntaxNode[]): NodeIterator | null { 1033 | if (!heads.length) return null 1034 | let pick = 0, picked = heads[0] 1035 | for (let i = 1; i < heads.length; i++) { 1036 | let node = heads[i] 1037 | if (node.from > picked.from || node.to < picked.to) { picked = node; pick = i } 1038 | } 1039 | let next = picked instanceof TreeNode && picked.index < 0 ? null : picked.parent 1040 | let newHeads = heads.slice() 1041 | if (next) newHeads[pick] = next 1042 | else newHeads.splice(pick, 1) 1043 | return new StackIterator(newHeads, picked) 1044 | } 1045 | 1046 | class StackIterator implements NodeIterator { 1047 | constructor(readonly heads: readonly SyntaxNode[], 1048 | readonly node: SyntaxNode) {} 1049 | get next() { return iterStack(this.heads) } 1050 | } 1051 | 1052 | function stackIterator(tree: Tree, pos: number, side: -1 | 0 | 1): NodeIterator { 1053 | let inner = tree.resolveInner(pos, side), layers: SyntaxNode[] | null = null 1054 | for (let scan: TreeNode | null = inner instanceof TreeNode ? inner : (inner as BufferNode).context.parent; 1055 | scan; scan = scan.parent) { 1056 | if (scan.index < 0) { // This is an overlay root 1057 | let parent: TreeNode | null = scan.parent! 1058 | ;(layers || (layers = [inner])).push(parent.resolve(pos, side)) 1059 | scan = parent 1060 | } else { 1061 | let mount = MountedTree.get(scan.tree) 1062 | // Relevant overlay branching off 1063 | if (mount && mount.overlay && mount.overlay[0].from <= pos && mount.overlay[mount.overlay.length - 1].to >= pos) { 1064 | let root = new TreeNode(mount.tree, mount.overlay[0].from + scan.from, -1, scan) 1065 | ;(layers || (layers = [inner])).push(resolveNode(root, pos, side, false)) 1066 | } 1067 | } 1068 | } 1069 | return layers ? iterStack(layers) : inner as any 1070 | } 1071 | 1072 | /// A tree cursor object focuses on a given node in a syntax tree, and 1073 | /// allows you to move to adjacent nodes. 1074 | export class TreeCursor implements SyntaxNodeRef { 1075 | /// The node's type. 1076 | type!: NodeType 1077 | 1078 | /// Shorthand for `.type.name`. 1079 | get name() { return this.type.name } 1080 | 1081 | /// The start source offset of this node. 1082 | from!: number 1083 | 1084 | /// The end source offset. 1085 | to!: number 1086 | 1087 | /// @internal 1088 | _tree!: TreeNode 1089 | /// @internal 1090 | buffer: BufferContext | null = null 1091 | private stack: number[] = [] 1092 | /// @internal 1093 | index: number = 0 1094 | private bufferNode: BufferNode | null = null 1095 | 1096 | /// @internal 1097 | constructor( 1098 | node: TreeNode | BufferNode, 1099 | /// @internal 1100 | readonly mode = 0 1101 | ) { 1102 | if (node instanceof TreeNode) { 1103 | this.yieldNode(node) 1104 | } else { 1105 | this._tree = node.context.parent 1106 | this.buffer = node.context 1107 | for (let n: BufferNode | null = node._parent; n; n = n._parent) this.stack.unshift(n.index) 1108 | this.bufferNode = node 1109 | this.yieldBuf(node.index) 1110 | } 1111 | } 1112 | 1113 | private yieldNode(node: TreeNode | null) { 1114 | if (!node) return false 1115 | this._tree = node 1116 | this.type = node.type 1117 | this.from = node.from 1118 | this.to = node.to 1119 | return true 1120 | } 1121 | 1122 | private yieldBuf(index: number, type?: NodeType) { 1123 | this.index = index 1124 | let {start, buffer} = this.buffer! 1125 | this.type = type || buffer.set.types[buffer.buffer[index]] 1126 | this.from = start + buffer.buffer[index + 1] 1127 | this.to = start + buffer.buffer[index + 2] 1128 | return true 1129 | } 1130 | 1131 | /// @internal 1132 | yield(node: TreeNode | BufferNode | null) { 1133 | if (!node) return false 1134 | if (node instanceof TreeNode) { 1135 | this.buffer = null 1136 | return this.yieldNode(node) 1137 | } 1138 | this.buffer = node.context 1139 | return this.yieldBuf(node.index, node.type) 1140 | } 1141 | 1142 | /// @internal 1143 | toString() { 1144 | return this.buffer ? this.buffer.buffer.childString(this.index) : this._tree.toString() 1145 | } 1146 | 1147 | /// @internal 1148 | enterChild(dir: 1 | -1, pos: number, side: Side) { 1149 | if (!this.buffer) 1150 | return this.yield(this._tree.nextChild(dir < 0 ? this._tree._tree.children.length - 1 : 0, dir, pos, side, this.mode)) 1151 | 1152 | let {buffer} = this.buffer 1153 | let index = buffer.findChild(this.index + 4, buffer.buffer[this.index + 3], dir, pos - this.buffer.start, side) 1154 | if (index < 0) return false 1155 | this.stack.push(this.index) 1156 | return this.yieldBuf(index) 1157 | } 1158 | 1159 | /// Move the cursor to this node's first child. When this returns 1160 | /// false, the node has no child, and the cursor has not been moved. 1161 | firstChild() { return this.enterChild(1, 0, Side.DontCare) } 1162 | 1163 | /// Move the cursor to this node's last child. 1164 | lastChild() { return this.enterChild(-1, 0, Side.DontCare) } 1165 | 1166 | /// Move the cursor to the first child that ends after `pos`. 1167 | childAfter(pos: number) { return this.enterChild(1, pos, Side.After) } 1168 | 1169 | /// Move to the last child that starts before `pos`. 1170 | childBefore(pos: number) { return this.enterChild(-1, pos, Side.Before) } 1171 | 1172 | /// Move the cursor to the child around `pos`. If side is -1 the 1173 | /// child may end at that position, when 1 it may start there. This 1174 | /// will also enter [overlaid](#common.MountedTree.overlay) 1175 | /// [mounted](#common.NodeProp^mounted) trees unless `overlays` is 1176 | /// set to false. 1177 | enter(pos: number, side: -1 | 0 | 1, mode: IterMode = this.mode) { 1178 | if (!this.buffer) 1179 | return this.yield(this._tree.enter(pos, side, mode)) 1180 | return mode & IterMode.ExcludeBuffers ? false : this.enterChild(1, pos, side) 1181 | } 1182 | 1183 | /// Move to the node's parent node, if this isn't the top node. 1184 | parent() { 1185 | if (!this.buffer) return this.yieldNode((this.mode & IterMode.IncludeAnonymous) ? this._tree._parent : this._tree.parent) 1186 | if (this.stack.length) return this.yieldBuf(this.stack.pop()!) 1187 | let parent = (this.mode & IterMode.IncludeAnonymous) ? this.buffer.parent : this.buffer.parent.nextSignificantParent() 1188 | this.buffer = null 1189 | return this.yieldNode(parent) 1190 | } 1191 | 1192 | /// @internal 1193 | sibling(dir: 1 | -1) { 1194 | if (!this.buffer) 1195 | return !this._tree._parent ? false 1196 | : this.yield(this._tree.index < 0 ? null 1197 | : this._tree._parent.nextChild(this._tree.index + dir, dir, 0, Side.DontCare, this.mode)) 1198 | 1199 | let {buffer} = this.buffer, d = this.stack.length - 1 1200 | if (dir < 0) { 1201 | let parentStart = d < 0 ? 0 : this.stack[d] + 4 1202 | if (this.index != parentStart) 1203 | return this.yieldBuf(buffer.findChild(parentStart, this.index, -1, 0, Side.DontCare)) 1204 | } else { 1205 | let after = buffer.buffer[this.index + 3] 1206 | if (after < (d < 0 ? buffer.buffer.length : buffer.buffer[this.stack[d] + 3])) 1207 | return this.yieldBuf(after) 1208 | } 1209 | return d < 0 ? this.yield(this.buffer.parent.nextChild(this.buffer.index + dir, dir, 0, Side.DontCare, this.mode)) : false 1210 | } 1211 | 1212 | /// Move to this node's next sibling, if any. 1213 | nextSibling() { return this.sibling(1) } 1214 | 1215 | /// Move to this node's previous sibling, if any. 1216 | prevSibling() { return this.sibling(-1) } 1217 | 1218 | private atLastNode(dir: 1 | -1) { 1219 | let index, parent: TreeNode | null, {buffer} = this 1220 | if (buffer) { 1221 | if (dir > 0) { 1222 | if (this.index < buffer.buffer.buffer.length) return false 1223 | } else { 1224 | for (let i = 0; i < this.index; i++) if (buffer.buffer.buffer[i + 3] < this.index) return false 1225 | } 1226 | ;({index, parent} = buffer) 1227 | } else { 1228 | ({index, _parent: parent} = this._tree) 1229 | } 1230 | for (; parent; {index, _parent: parent} = parent) { 1231 | if (index > -1) for (let i = index + dir, e = dir < 0 ? -1 : parent._tree.children.length; i != e; i += dir) { 1232 | let child = parent._tree.children[i] 1233 | if ((this.mode & IterMode.IncludeAnonymous) || 1234 | child instanceof TreeBuffer || 1235 | !child.type.isAnonymous || 1236 | hasChild(child)) return false 1237 | } 1238 | } 1239 | return true 1240 | } 1241 | 1242 | private move(dir: 1 | -1, enter: boolean) { 1243 | if (enter && this.enterChild(dir, 0, Side.DontCare)) return true 1244 | for (;;) { 1245 | if (this.sibling(dir)) return true 1246 | if (this.atLastNode(dir) || !this.parent()) return false 1247 | } 1248 | } 1249 | 1250 | /// Move to the next node in a 1251 | /// [pre-order](https://en.wikipedia.org/wiki/Tree_traversal#Pre-order,_NLR) 1252 | /// traversal, going from a node to its first child or, if the 1253 | /// current node is empty or `enter` is false, its next sibling or 1254 | /// the next sibling of the first parent node that has one. 1255 | next(enter = true) { return this.move(1, enter) } 1256 | 1257 | /// Move to the next node in a last-to-first pre-order traversal. A 1258 | /// node is followed by its last child or, if it has none, its 1259 | /// previous sibling or the previous sibling of the first parent 1260 | /// node that has one. 1261 | prev(enter = true) { return this.move(-1, enter) } 1262 | 1263 | /// Move the cursor to the innermost node that covers `pos`. If 1264 | /// `side` is -1, it will enter nodes that end at `pos`. If it is 1, 1265 | /// it will enter nodes that start at `pos`. 1266 | moveTo(pos: number, side: -1 | 0 | 1 = 0) { 1267 | // Move up to a node that actually holds the position, if possible 1268 | while (this.from == this.to || 1269 | (side < 1 ? this.from >= pos : this.from > pos) || 1270 | (side > -1 ? this.to <= pos : this.to < pos)) 1271 | if (!this.parent()) break 1272 | 1273 | // Then scan down into child nodes as far as possible 1274 | while (this.enterChild(1, pos, side)) {} 1275 | return this 1276 | } 1277 | 1278 | /// Get a [syntax node](#common.SyntaxNode) at the cursor's current 1279 | /// position. 1280 | get node(): SyntaxNode { 1281 | if (!this.buffer) return this._tree 1282 | 1283 | let cache = this.bufferNode, result: BufferNode | null = null, depth = 0 1284 | if (cache && cache.context == this.buffer) { 1285 | scan: for (let index = this.index, d = this.stack.length; d >= 0;) { 1286 | for (let c: BufferNode | null = cache; c; c = c._parent) if (c.index == index) { 1287 | if (index == this.index) return c 1288 | result = c 1289 | depth = d + 1 1290 | break scan 1291 | } 1292 | index = this.stack[--d] 1293 | } 1294 | } 1295 | for (let i = depth; i < this.stack.length; i++) result = new BufferNode(this.buffer, result, this.stack[i]) 1296 | return this.bufferNode = new BufferNode(this.buffer, result, this.index) 1297 | } 1298 | 1299 | /// Get the [tree](#common.Tree) that represents the current node, if 1300 | /// any. Will return null when the node is in a [tree 1301 | /// buffer](#common.TreeBuffer). 1302 | get tree(): Tree | null { 1303 | return this.buffer ? null : this._tree._tree 1304 | } 1305 | 1306 | /// Iterate over the current node and all its descendants, calling 1307 | /// `enter` when entering a node and `leave`, if given, when leaving 1308 | /// one. When `enter` returns `false`, any children of that node are 1309 | /// skipped, and `leave` isn't called for it. 1310 | iterate(enter: (node: SyntaxNodeRef) => boolean | void, 1311 | leave?: (node: SyntaxNodeRef) => void) { 1312 | for (let depth = 0;;) { 1313 | let mustLeave = false 1314 | if (this.type.isAnonymous || enter(this) !== false) { 1315 | if (this.firstChild()) { depth++; continue } 1316 | if (!this.type.isAnonymous) mustLeave = true 1317 | } 1318 | for (;;) { 1319 | if (mustLeave && leave) leave(this) 1320 | mustLeave = this.type.isAnonymous 1321 | if (!depth) return 1322 | if (this.nextSibling()) break 1323 | this.parent() 1324 | depth-- 1325 | mustLeave = true 1326 | } 1327 | } 1328 | } 1329 | 1330 | /// Test whether the current node matches a given context—a sequence 1331 | /// of direct parent node names. Empty strings in the context array 1332 | /// are treated as wildcards. 1333 | matchContext(context: readonly string[]): boolean { 1334 | if (!this.buffer) return matchNodeContext(this.node.parent, context) 1335 | let {buffer} = this.buffer, {types} = buffer.set 1336 | for (let i = context.length - 1, d = this.stack.length - 1; i >= 0; d--) { 1337 | if (d < 0) return matchNodeContext(this._tree, context, i) 1338 | let type = types[buffer.buffer[this.stack[d]]] 1339 | if (!type.isAnonymous) { 1340 | if (context[i] && context[i] != type.name) return false 1341 | i-- 1342 | } 1343 | } 1344 | return true 1345 | } 1346 | } 1347 | 1348 | function hasChild(tree: Tree): boolean { 1349 | return tree.children.some(ch => ch instanceof TreeBuffer || !ch.type.isAnonymous || hasChild(ch)) 1350 | } 1351 | 1352 | const enum Balance { BranchFactor = 8 } 1353 | const enum CutOff { Depth = 2500 } 1354 | 1355 | const enum SpecialRecord { 1356 | Reuse = -1, 1357 | ContextChange = -3, 1358 | LookAhead = -4 1359 | } 1360 | 1361 | function buildTree(data: BuildData) { 1362 | let {buffer, nodeSet, 1363 | maxBufferLength = DefaultBufferLength, 1364 | reused = [], 1365 | minRepeatType = nodeSet.types.length} = data 1366 | let cursor = Array.isArray(buffer) ? new FlatBufferCursor(buffer, buffer.length) : buffer as BufferCursor 1367 | let types = nodeSet.types 1368 | 1369 | let contextHash = 0, lookAhead = 0 1370 | 1371 | function takeNode(parentStart: number, minPos: number, 1372 | children: (Tree | TreeBuffer)[], positions: number[], 1373 | inRepeat: number, depth: number) { 1374 | let {id, start, end, size} = cursor 1375 | let lookAheadAtStart = lookAhead, contextAtStart = contextHash 1376 | while (size < 0) { 1377 | cursor.next() 1378 | if (size == SpecialRecord.Reuse) { 1379 | let node = reused[id] 1380 | children.push(node) 1381 | positions.push(start - parentStart) 1382 | return 1383 | } else if (size == SpecialRecord.ContextChange) { // Context change 1384 | contextHash = id 1385 | return 1386 | } else if (size == SpecialRecord.LookAhead) { 1387 | lookAhead = id 1388 | return 1389 | } else { 1390 | throw new RangeError(`Unrecognized record size: ${size}`) 1391 | } 1392 | ;({id, start, end, size} = cursor) 1393 | } 1394 | 1395 | let type = types[id], node, buffer: {size: number, start: number, skip: number} | undefined 1396 | let startPos = start - parentStart 1397 | if (end - start <= maxBufferLength && (buffer = findBufferSize(cursor.pos - minPos, inRepeat))) { 1398 | // Small enough for a buffer, and no reused nodes inside 1399 | let data = new Uint16Array(buffer.size - buffer.skip) 1400 | let endPos = cursor.pos - buffer.size, index = data.length 1401 | while (cursor.pos > endPos) 1402 | index = copyToBuffer(buffer.start, data, index) 1403 | node = new TreeBuffer(data, end - buffer.start, nodeSet) 1404 | startPos = buffer.start - parentStart 1405 | } else { // Make it a node 1406 | let endPos = cursor.pos - size 1407 | cursor.next() 1408 | let localChildren: (Tree | TreeBuffer)[] = [], localPositions: number[] = [] 1409 | let localInRepeat = id >= minRepeatType ? id : -1 1410 | let lastGroup = 0, lastEnd = end 1411 | while (cursor.pos > endPos) { 1412 | if (localInRepeat >= 0 && cursor.id == localInRepeat && cursor.size >= 0) { 1413 | if (cursor.end <= lastEnd - maxBufferLength) { 1414 | makeRepeatLeaf(localChildren, localPositions, start, lastGroup, cursor.end, lastEnd, 1415 | localInRepeat, lookAheadAtStart, contextAtStart) 1416 | lastGroup = localChildren.length 1417 | lastEnd = cursor.end 1418 | } 1419 | cursor.next() 1420 | } else if (depth > CutOff.Depth) { 1421 | takeFlatNode(start, endPos, localChildren, localPositions) 1422 | } else { 1423 | takeNode(start, endPos, localChildren, localPositions, localInRepeat, depth + 1) 1424 | } 1425 | } 1426 | if (localInRepeat >= 0 && lastGroup > 0 && lastGroup < localChildren.length) 1427 | makeRepeatLeaf(localChildren, localPositions, start, lastGroup, start, lastEnd, localInRepeat, 1428 | lookAheadAtStart, contextAtStart) 1429 | localChildren.reverse(); localPositions.reverse() 1430 | 1431 | if (localInRepeat > -1 && lastGroup > 0) { 1432 | let make = makeBalanced(type, contextAtStart) 1433 | node = balanceRange(type, localChildren, localPositions, 0, localChildren.length, 0, end - start, make, make) 1434 | } else { 1435 | node = makeTree(type, localChildren, localPositions, end - start, lookAheadAtStart - end, contextAtStart) 1436 | } 1437 | } 1438 | 1439 | children.push(node) 1440 | positions.push(startPos) 1441 | } 1442 | 1443 | function takeFlatNode(parentStart: number, minPos: number, 1444 | children: (Tree | TreeBuffer)[], positions: number[]) { 1445 | let nodes = [] // Temporary, inverted array of leaf nodes found, with absolute positions 1446 | let nodeCount = 0, stopAt = -1 1447 | while (cursor.pos > minPos) { 1448 | let {id, start, end, size} = cursor 1449 | if (size > 4) { // Not a leaf 1450 | cursor.next() 1451 | } else if (stopAt > -1 && start < stopAt) { 1452 | break 1453 | } else { 1454 | if (stopAt < 0) stopAt = end - maxBufferLength 1455 | nodes.push(id, start, end) 1456 | nodeCount++ 1457 | cursor.next() 1458 | } 1459 | } 1460 | if (nodeCount) { 1461 | let buffer = new Uint16Array(nodeCount * 4) 1462 | let start = nodes[nodes.length - 2] 1463 | for (let i = nodes.length - 3, j = 0; i >= 0; i -= 3) { 1464 | buffer[j++] = nodes[i] 1465 | buffer[j++] = nodes[i + 1] - start 1466 | buffer[j++] = nodes[i + 2] - start 1467 | buffer[j++] = j 1468 | } 1469 | children.push(new TreeBuffer(buffer, nodes[2] - start, nodeSet)) 1470 | positions.push(start - parentStart) 1471 | } 1472 | } 1473 | 1474 | function makeBalanced(type: NodeType, contextHash: number) { 1475 | return (children: readonly (Tree | TreeBuffer)[], positions: readonly number[], length: number) => { 1476 | let lookAhead = 0, lastI = children.length - 1, last, lookAheadProp 1477 | if (lastI >= 0 && (last = children[lastI]) instanceof Tree) { 1478 | if (!lastI && last.type == type && last.length == length) return last 1479 | if (lookAheadProp = last.prop(NodeProp.lookAhead)) 1480 | lookAhead = positions[lastI] + last.length + lookAheadProp 1481 | } 1482 | return makeTree(type, children, positions, length, lookAhead, contextHash) 1483 | } 1484 | } 1485 | 1486 | function makeRepeatLeaf(children: (Tree | TreeBuffer)[], positions: number[], base: number, i: number, 1487 | from: number, to: number, type: number, lookAhead: number, contextHash: number) { 1488 | let localChildren = [], localPositions = [] 1489 | while (children.length > i) { localChildren.push(children.pop()!); localPositions.push(positions.pop()! + base - from) } 1490 | children.push(makeTree(nodeSet.types[type], localChildren, localPositions, to - from, lookAhead - to, contextHash)) 1491 | positions.push(from - base) 1492 | } 1493 | 1494 | function makeTree(type: NodeType, 1495 | children: readonly (Tree | TreeBuffer)[], 1496 | positions: readonly number[], length: number, 1497 | lookAhead: number, 1498 | contextHash: number, 1499 | props?: readonly [number | NodeProp, any][]) { 1500 | if (contextHash) { 1501 | let pair: [number | NodeProp, any] = [NodeProp.contextHash, contextHash] 1502 | props = props ? [pair].concat(props) : [pair] 1503 | } 1504 | if (lookAhead > 25) { 1505 | let pair: [number | NodeProp, any] = [NodeProp.lookAhead, lookAhead] 1506 | props = props ? [pair].concat(props) : [pair] 1507 | } 1508 | return new Tree(type, children, positions, length, props) 1509 | } 1510 | 1511 | function findBufferSize(maxSize: number, inRepeat: number) { 1512 | // Scan through the buffer to find previous siblings that fit 1513 | // together in a TreeBuffer, and don't contain any reused nodes 1514 | // (which can't be stored in a buffer). 1515 | // If `inRepeat` is > -1, ignore node boundaries of that type for 1516 | // nesting, but make sure the end falls either at the start 1517 | // (`maxSize`) or before such a node. 1518 | let fork = cursor.fork() 1519 | let size = 0, start = 0, skip = 0, minStart = fork.end - maxBufferLength 1520 | let result = {size: 0, start: 0, skip: 0} 1521 | scan: for (let minPos = fork.pos - maxSize; fork.pos > minPos;) { 1522 | let nodeSize = fork.size 1523 | // Pretend nested repeat nodes of the same type don't exist 1524 | if (fork.id == inRepeat && nodeSize >= 0) { 1525 | // Except that we store the current state as a valid return 1526 | // value. 1527 | result.size = size; result.start = start; result.skip = skip 1528 | skip += 4; size += 4 1529 | fork.next() 1530 | continue 1531 | } 1532 | let startPos = fork.pos - nodeSize 1533 | if (nodeSize < 0 || startPos < minPos || fork.start < minStart) break 1534 | let localSkipped = fork.id >= minRepeatType ? 4 : 0 1535 | let nodeStart = fork.start 1536 | fork.next() 1537 | while (fork.pos > startPos) { 1538 | if (fork.size < 0) { 1539 | if (fork.size == SpecialRecord.ContextChange) localSkipped += 4 1540 | else break scan 1541 | } else if (fork.id >= minRepeatType) { 1542 | localSkipped += 4 1543 | } 1544 | fork.next() 1545 | } 1546 | start = nodeStart 1547 | size += nodeSize 1548 | skip += localSkipped 1549 | } 1550 | if (inRepeat < 0 || size == maxSize) { 1551 | result.size = size; result.start = start; result.skip = skip 1552 | } 1553 | return result.size > 4 ? result : undefined 1554 | } 1555 | 1556 | function copyToBuffer(bufferStart: number, buffer: Uint16Array, index: number): number { 1557 | let {id, start, end, size} = cursor 1558 | cursor.next() 1559 | if (size >= 0 && id < minRepeatType) { 1560 | let startIndex = index 1561 | if (size > 4) { 1562 | let endPos = cursor.pos - (size - 4) 1563 | while (cursor.pos > endPos) 1564 | index = copyToBuffer(bufferStart, buffer, index) 1565 | } 1566 | buffer[--index] = startIndex 1567 | buffer[--index] = end - bufferStart 1568 | buffer[--index] = start - bufferStart 1569 | buffer[--index] = id 1570 | } else if (size == SpecialRecord.ContextChange) { 1571 | contextHash = id 1572 | } else if (size == SpecialRecord.LookAhead) { 1573 | lookAhead = id 1574 | } 1575 | return index 1576 | } 1577 | 1578 | let children: (Tree | TreeBuffer)[] = [], positions: number[] = [] 1579 | while (cursor.pos > 0) takeNode(data.start || 0, data.bufferStart || 0, children, positions, -1, 0) 1580 | let length = data.length ?? (children.length ? positions[0] + children[0].length : 0) 1581 | return new Tree(types[data.topID], children.reverse(), positions.reverse(), length) 1582 | } 1583 | 1584 | const nodeSizeCache: WeakMap = new WeakMap 1585 | function nodeSize(balanceType: NodeType, node: Tree | TreeBuffer): number { 1586 | if (!balanceType.isAnonymous || node instanceof TreeBuffer || node.type != balanceType) return 1 1587 | let size = nodeSizeCache.get(node) 1588 | if (size == null) { 1589 | size = 1 1590 | for (let child of node.children) { 1591 | if (child.type != balanceType || !(child instanceof Tree)) { 1592 | size = 1 1593 | break 1594 | } 1595 | size += nodeSize(balanceType, child) 1596 | } 1597 | nodeSizeCache.set(node, size) 1598 | } 1599 | return size 1600 | } 1601 | 1602 | function balanceRange( 1603 | // The type the balanced tree's inner nodes. 1604 | balanceType: NodeType, 1605 | // The direct children and their positions 1606 | children: readonly (Tree | TreeBuffer)[], 1607 | positions: readonly number[], 1608 | // The index range in children/positions to use 1609 | from: number, to: number, 1610 | // The start position of the nodes, relative to their parent. 1611 | start: number, 1612 | // Length of the outer node 1613 | length: number, 1614 | // Function to build the top node of the balanced tree 1615 | mkTop: ((children: readonly (Tree | TreeBuffer)[], positions: readonly number[], length: number) => Tree) | null, 1616 | // Function to build internal nodes for the balanced tree 1617 | mkTree: (children: readonly (Tree | TreeBuffer)[], positions: readonly number[], length: number) => Tree 1618 | ): Tree { 1619 | let total = 0 1620 | for (let i = from; i < to; i++) total += nodeSize(balanceType, children[i]) 1621 | 1622 | let maxChild = Math.ceil((total * 1.5) / Balance.BranchFactor) 1623 | let localChildren: (Tree | TreeBuffer)[] = [], localPositions: number[] = [] 1624 | function divide(children: readonly (Tree | TreeBuffer)[], positions: readonly number[], 1625 | from: number, to: number, offset: number) { 1626 | for (let i = from; i < to;) { 1627 | let groupFrom = i, groupStart = positions[i], groupSize = nodeSize(balanceType, children[i]) 1628 | i++ 1629 | for (; i < to; i++) { 1630 | let nextSize = nodeSize(balanceType, children[i]) 1631 | if (groupSize + nextSize >= maxChild) break 1632 | groupSize += nextSize 1633 | } 1634 | if (i == groupFrom + 1) { 1635 | if (groupSize > maxChild) { 1636 | let only = children[groupFrom] as Tree // Only trees can have a size > 1 1637 | divide(only.children, only.positions, 0, only.children.length, positions[groupFrom] + offset) 1638 | continue 1639 | } 1640 | localChildren.push(children[groupFrom]) 1641 | } else { 1642 | let length = positions[i - 1] + children[i - 1].length - groupStart 1643 | localChildren.push(balanceRange(balanceType, children, positions, groupFrom, i, groupStart, length, null, mkTree)) 1644 | } 1645 | localPositions.push(groupStart + offset - start) 1646 | } 1647 | } 1648 | divide(children, positions, from, to, 0) 1649 | return (mkTop || mkTree)(localChildren, localPositions, length) 1650 | } 1651 | 1652 | /// Provides a way to associate values with pieces of trees. As long 1653 | /// as that part of the tree is reused, the associated values can be 1654 | /// retrieved from an updated tree. 1655 | export class NodeWeakMap { 1656 | private map = new WeakMap>() 1657 | 1658 | private setBuffer(buffer: TreeBuffer, index: number, value: T) { 1659 | let inner = this.map.get(buffer) as Map | undefined 1660 | if (!inner) this.map.set(buffer, inner = new Map) 1661 | inner.set(index, value) 1662 | } 1663 | 1664 | private getBuffer(buffer: TreeBuffer, index: number): T | undefined { 1665 | let inner = this.map.get(buffer) as Map | undefined 1666 | return inner && inner.get(index) 1667 | } 1668 | 1669 | /// Set the value for this syntax node. 1670 | set(node: SyntaxNode, value: T) { 1671 | if (node instanceof BufferNode) this.setBuffer(node.context.buffer, node.index, value) 1672 | else if (node instanceof TreeNode) this.map.set(node.tree, value) 1673 | } 1674 | 1675 | /// Retrieve value for this syntax node, if it exists in the map. 1676 | get(node: SyntaxNode): T | undefined { 1677 | return node instanceof BufferNode ? this.getBuffer(node.context.buffer, node.index) 1678 | : node instanceof TreeNode ? this.map.get(node.tree) as T | undefined : undefined 1679 | } 1680 | 1681 | /// Set the value for the node that a cursor currently points to. 1682 | cursorSet(cursor: TreeCursor, value: T) { 1683 | if (cursor.buffer) this.setBuffer(cursor.buffer.buffer, cursor.index, value) 1684 | else this.map.set(cursor.tree!, value) 1685 | } 1686 | 1687 | /// Retrieve the value for the node that a cursor currently points 1688 | /// to. 1689 | cursorGet(cursor: TreeCursor): T | undefined { 1690 | return cursor.buffer ? this.getBuffer(cursor.buffer.buffer, cursor.index) : this.map.get(cursor.tree!) as T | undefined 1691 | } 1692 | } 1693 | -------------------------------------------------------------------------------- /test/test-tree.ts: -------------------------------------------------------------------------------- 1 | import {Tree, NodeSet, NodeType, SyntaxNode, NodeProp, IterMode} from "../dist/index.js" 2 | import ist from "ist" 3 | 4 | let types = "T a b c Pa Br".split(" ").map((s, i) => NodeType.define({ 5 | id: i, 6 | name: s, 7 | props: /^[abc]$/.test(s) ? [[NodeProp.group, ["atom"]]] : [] 8 | })) 9 | let repeat = NodeType.define({id: types.length}) 10 | types.push(repeat) 11 | let nodeSet = new NodeSet(types) 12 | 13 | function id(n: string) { return types.find(x => x.name == n)!.id } 14 | 15 | function mk(spec: string) { 16 | let starts: number[] = [], buffer: number[] = [] 17 | for (let pos = 0; pos < spec.length;) { 18 | let [m, letters, open, close] = /^(?:([abc])|([\[\(])|([\]\)]))/.exec(spec.slice(pos))! 19 | if (letters) { 20 | let bufStart = buffer.length 21 | for (let i = 0; i < letters.length; i++) { 22 | buffer.push(id(letters[i]), pos + i, pos + i + 1, 4) 23 | if (i) buffer.push(repeat.id, pos, pos + i + 1, (buffer.length + 4) - bufStart) 24 | } 25 | } else if (open) { 26 | starts.push(buffer.length, pos) 27 | } else { 28 | let start = starts.pop()!, startOff = starts.pop()! 29 | buffer.push(id(close == ")" ? "Pa" : "Br"), start, pos + 1, (buffer.length + 4) - startOff) 30 | } 31 | pos += m.length 32 | } 33 | return Tree.build({buffer, nodeSet, topID: 0, maxBufferLength: 10, minRepeatType: repeat.id}) 34 | } 35 | 36 | let _recur: Tree | null = null 37 | function recur() { 38 | return _recur || (_recur = mk(function build(depth: number): string { 39 | if (depth) { 40 | let inner = build(depth - 1) 41 | return "(" + inner + ")[" + inner + "]" 42 | } else { 43 | let result = "" 44 | for (let i = 0; i < 20; i++) result += "abc"[i % 3] 45 | return result 46 | } 47 | }(6))) 48 | } 49 | 50 | let _simple: Tree | null = null 51 | function simple() { 52 | return _simple || (_simple = mk("aaaa(bbb[ccc][aaa][()])")) 53 | } 54 | 55 | const anonTree = new Tree(nodeSet.types[0], [ 56 | new Tree(NodeType.none, [ 57 | new Tree(nodeSet.types[1], [], [], 1), 58 | new Tree(nodeSet.types[2], [], [], 1) 59 | ], [0, 1], 2), 60 | ], [0], 2) 61 | 62 | describe("SyntaxNode", () => { 63 | it("can resolve at the top level", () => { 64 | let c = simple().resolve(2, -1) 65 | ist(c.from, 1) 66 | ist(c.to, 2) 67 | ist(c.name, "a") 68 | ist(c.parent!.name, "T") 69 | ist(!c.parent!.parent) 70 | c = simple().resolve(2, 1) 71 | ist(c.from, 2) 72 | ist(c.to, 3) 73 | c = simple().resolve(2) 74 | ist(c.name, "T") 75 | ist(c.from, 0) 76 | ist(c.to, 23) 77 | }) 78 | 79 | it("can resolve deeper", () => { 80 | let c = simple().resolve(10, 1) 81 | ist(c.name, "c") 82 | ist(c.from, 10) 83 | ist(c.parent!.name, "Br") 84 | ist(c.parent!.parent!.name, "Pa") 85 | ist(c.parent!.parent!.parent!.name, "T") 86 | }) 87 | 88 | it("can resolve in a large tree", () => { 89 | let c: SyntaxNode | null = recur().resolve(10, 1), depth = 1 90 | while (c = c && c.parent) depth++ 91 | ist(depth, 8) 92 | }) 93 | 94 | it("caches resolved parents", () => { 95 | let a = recur().resolve(3, 1), b = recur().resolve(3, 1) 96 | ist(a, b) 97 | }) 98 | 99 | describe("getChild", () => { 100 | function flat(children: readonly SyntaxNode[]) { 101 | return children.map(c => c.name).join(",") 102 | } 103 | 104 | it("can get children by group", () => { 105 | let tree = mk("aa(bb)[aabbcc]").topNode 106 | ist(flat(tree.getChildren("atom")), "a,a") 107 | ist(flat(tree.firstChild!.getChildren("atom")), "") 108 | ist(flat(tree.lastChild!.getChildren("atom")), "a,a,b,b,c,c") 109 | }) 110 | 111 | it("can get single children", () => { 112 | let tree = mk("abc()").topNode 113 | ist(tree.getChild("Br"), null) 114 | ist(tree.getChild("Pa")?.name, "Pa") 115 | }) 116 | 117 | it("can get children between others", () => { 118 | let tree = mk("aa(bb)[aabbcc]").topNode 119 | ist(tree.getChild("Pa", "atom", "Br")) 120 | ist(!tree.getChild("Pa", "atom", "atom")) 121 | let last = tree.lastChild! 122 | ist(flat(last.getChildren("b", "a", "c")), "b,b") 123 | ist(flat(last.getChildren("a", null, "c")), "a,a") 124 | ist(flat(last.getChildren("c", "b", null)), "c,c") 125 | ist(flat(last.getChildren("b", "c")), "") 126 | }) 127 | }) 128 | 129 | it("skips anonymous nodes", () => { 130 | ist(anonTree + "", "T(a,b)") 131 | ist(anonTree.resolve(1).name, "T") 132 | ist(anonTree.topNode.lastChild!.name, "b") 133 | ist(anonTree.topNode.firstChild!.name, "a") 134 | ist(anonTree.topNode.childAfter(1)!.name, "b") 135 | }) 136 | 137 | it("allows access to the underlying tree", () => { 138 | let tree = mk("aaa[bbbbb(bb)bbbbbbb]aaa") 139 | let node = tree.topNode.firstChild! 140 | while (node.name != "Br") node = node.nextSibling! 141 | ist(node.tree instanceof Tree) 142 | ist(node.tree!.type.name, "Br") 143 | node = node.firstChild! 144 | while (node.name != "Pa") node = node.nextSibling! 145 | ist(!node.tree) 146 | ist(node.toTree().toString(), "Pa(b,b)") 147 | node = node.firstChild! 148 | ist(node.name, "b") 149 | ist(node.toTree().toString(), "b") 150 | ist(node.toTree().children.length, 0) 151 | }) 152 | }) 153 | 154 | describe("TreeCursor", () => { 155 | const simpleCount: Record = {a: 7, b: 3, c: 3, Br: 3, Pa: 2, T: 1} 156 | 157 | it("iterates over all nodes", () => { 158 | let count: Record = Object.create(null) 159 | let pos = 0, cur = simple().cursor() 160 | do { 161 | ist(cur.from, pos, ">=") 162 | pos = cur.from 163 | count[cur.name] = (count[cur.name] || 0) + 1 164 | } while (cur.next()) 165 | for (let k of Object.keys(simpleCount)) ist(count[k], simpleCount[k]) 166 | }) 167 | 168 | it("iterates over all nodes in reverse", () => { 169 | let count: Record = Object.create(null) 170 | let pos = 100, cur = simple().cursor() 171 | do { 172 | ist(cur.to, pos, "<=") 173 | pos = cur.to 174 | count[cur.name] = (count[cur.name] || 0) + 1 175 | } while (cur.prev()) 176 | for (let k of Object.keys(simpleCount)) ist(count[k], simpleCount[k]) 177 | }) 178 | 179 | it("works with internal iteration", () => { 180 | let openCount: Record = Object.create(null) 181 | let closeCount: Record = Object.create(null) 182 | simple().iterate({ 183 | enter(t) { openCount[t.name] = (openCount[t.name] || 0) + 1 }, 184 | leave(t) { closeCount[t.name] = (closeCount[t.name] || 0) + 1 } 185 | }) 186 | for (let k of Object.keys(simpleCount)) { 187 | ist(openCount[k], simpleCount[k]) 188 | ist(closeCount[k], simpleCount[k]) 189 | } 190 | }) 191 | 192 | it("handles iterating out of bounds", () => { 193 | let hit = 0 194 | Tree.empty.iterate({ 195 | from: 0, 196 | to: 200, 197 | enter() { hit++ }, 198 | leave() { hit++ } 199 | }) 200 | ist(hit, 0) 201 | }) 202 | 203 | it("internal iteration can be limited to a range", () => { 204 | let seen: string[] = [] 205 | simple().iterate({ 206 | enter(t) { seen.push(t.name); return t.name == "Br" ? false : undefined }, 207 | from: 3, 208 | to: 14 209 | }) 210 | ist(seen.join(","), "T,a,a,Pa,b,b,b,Br,Br") 211 | }) 212 | 213 | it("can leave nodes", () => { 214 | let cur = simple().cursor() 215 | ist(!cur.parent()) 216 | cur.next(); cur.next() 217 | ist(cur.from, 1) 218 | ist(cur.parent()) 219 | ist(cur.from, 0) 220 | for (let j = 0; j < 6; j++) cur.next() 221 | ist(cur.from, 5) 222 | ist(cur.parent()) 223 | ist(cur.from, 4) 224 | ist(cur.parent()) 225 | ist(cur.from, 0) 226 | ist(!cur.parent()) 227 | }) 228 | 229 | it("can move to a given position", () => { 230 | let tree = recur(), start = tree.length >> 1, cursor = tree.cursorAt(start, 1) 231 | do { ist(cursor.from, start, ">=") } 232 | while (cursor.next()) 233 | }) 234 | 235 | it("can move into a parent node", () => { 236 | let c = simple().cursorAt(10).moveTo(2) 237 | ist(c.name, "T") 238 | }) 239 | 240 | it("can move to a specific sibling", () => { 241 | let cursor = simple().cursor() 242 | ist(cursor.childAfter(2)) 243 | ist(cursor.to, 3) 244 | cursor.parent() 245 | ist(cursor.childBefore(5)) 246 | ist(cursor.from, 4) 247 | ist(cursor.childAfter(11)) 248 | ist(cursor.from, 8) 249 | ist(cursor.childBefore(10)) 250 | ist(cursor.from, 9) 251 | ist(!simple().cursor().childBefore(0)) 252 | ist(!simple().cursor().childAfter(100)) 253 | }) 254 | 255 | it("isn't slow", () => { 256 | let tree = recur(), t0 = Date.now(), count = 0 257 | for (let i = 0; i < 2000; i++) { 258 | let cur = tree.cursor() 259 | do { 260 | if (cur.from < 0 || !cur.name) throw new Error("BAD") 261 | count++ 262 | } while (cur.next()) 263 | } 264 | let perMS = count / (Date.now() - t0) 265 | ist(perMS, 10000, ">") 266 | }) 267 | 268 | it("can produce nodes", () => { 269 | let node = simple().cursorAt(8, 1).node 270 | ist(node.name, "Br") 271 | ist(node.from, 8) 272 | ist(node.parent!.name, "Pa") 273 | ist(node.parent!.from, 4) 274 | ist(node.parent!.parent!.name, "T") 275 | ist(node.parent!.parent!.from, 0) 276 | ist(node.parent!.parent!.parent, null) 277 | }) 278 | 279 | it("can produce node from cursors created from nodes", () => { 280 | let cur = simple().topNode.lastChild!.childAfter(8)!.childAfter(10)!.cursor() 281 | ist(cur.name, "c") 282 | ist(cur.from, 10) 283 | ist(cur.parent()) 284 | let node = cur.node 285 | ist(node.name, "Br") 286 | ist(node.from, 8) 287 | ist(node.parent!.name, "Pa") 288 | ist(node.parent!.from, 4) 289 | ist(node.parent!.parent!.name, "T") 290 | ist(node.parent!.parent!.parent, null) 291 | }) 292 | 293 | it("reuses nodes in buffers", () => { 294 | let cur = simple().cursorAt(10, 1) 295 | let n10 = cur.node 296 | ist(n10.name, "c") 297 | ist(n10.from, 10) 298 | ist(cur.node, n10) 299 | cur.nextSibling() 300 | ist(cur.node.parent, n10.parent) 301 | cur.parent() 302 | ist(cur.node, n10.parent) 303 | }) 304 | 305 | it("skips anonymous nodes", () => { 306 | let c = anonTree.cursor() 307 | c.moveTo(1) 308 | ist(c.name, "T") 309 | c.firstChild() 310 | ist(c.name, "a") 311 | c.nextSibling() 312 | ist(c.name, "b") 313 | ist(!c.next()) 314 | }) 315 | 316 | it("stops at anonymous nodes when configured as full", () => { 317 | let c = anonTree.cursor(IterMode.IncludeAnonymous) 318 | c.moveTo(1) 319 | ist(c.type, NodeType.none) 320 | ist(c.tree!.length, 2) 321 | c.firstChild() 322 | ist(c.name, "a") 323 | c.parent() 324 | ist(c.type, NodeType.none) 325 | }) 326 | }) 327 | 328 | describe("matchContext", () => { 329 | it("can match on nodes", () => { 330 | ist(simple().resolve(10, 1).matchContext(["T", "Pa", "Br"])) 331 | }) 332 | 333 | it("can match wildcards", () => { 334 | ist(simple().resolve(10, 1).matchContext(["T", "", "Br"])) 335 | }) 336 | 337 | it("can mismatch on nodes", () => { 338 | ist(!simple().resolve(10, 1).matchContext(["Q", "Br"])) 339 | }) 340 | 341 | it("can match on cursor", () => { 342 | let c = simple().cursor() 343 | for (let i = 0; i < 3; i++) c.enter(15, -1) 344 | ist(c.matchContext(["T", "Pa", "Br"])) 345 | }) 346 | }) 347 | -------------------------------------------------------------------------------- /test/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "lib": ["es2017"], 4 | "noImplicitReturns": true, 5 | "noUnusedLocals": true, 6 | "sourceMap": true, 7 | "strict": true, 8 | "target": "es5", 9 | "newLine": "lf", 10 | "stripInternal": true 11 | }, 12 | "include": ["*.ts"] 13 | } 14 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "lib": ["es2017"], 4 | "noImplicitReturns": true, 5 | "noUnusedLocals": true, 6 | "strict": true, 7 | "target": "es2015", 8 | "module": "es2015", 9 | "newLine": "lf", 10 | "stripInternal": true, 11 | "moduleResolution": "node" 12 | }, 13 | "include": ["src/*.ts"] 14 | } 15 | --------------------------------------------------------------------------------