├── .github └── workflows │ └── dispatch.yml ├── .gitignore ├── .npmignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── package.json ├── src ├── README.md ├── fold.ts ├── highlight.ts ├── indent.ts ├── index.ts ├── isolate.ts ├── language.ts ├── matchbrackets.ts ├── stream-parser.ts └── stringstream.ts └── test ├── test-fold.ts ├── test-stream-parser.ts └── test-syntax.ts /.github/workflows/dispatch.yml: -------------------------------------------------------------------------------- 1 | name: Trigger CI 2 | on: push 3 | 4 | jobs: 5 | build: 6 | name: Dispatch to main repo 7 | runs-on: ubuntu-latest 8 | steps: 9 | - name: Emit repository_dispatch 10 | uses: mvasigh/dispatch-action@main 11 | with: 12 | # You should create a personal access token and store it in your repository 13 | token: ${{ secrets.DISPATCH_AUTH }} 14 | repo: dev 15 | owner: codemirror 16 | event_type: push 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /node_modules 2 | package-lock.json 3 | /dist 4 | /test/*.js 5 | /test/*.d.ts 6 | /test/*.d.ts.map 7 | .tern-* 8 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | /src 2 | /test 3 | /node_modules 4 | .tern-* 5 | rollup.config.js 6 | tsconfig.json 7 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## 6.11.1 (2025-06-02) 2 | 3 | ### Bug fixes 4 | 5 | Fix an issue where indentation would sometimes miss nodes in mixed-language situations. 6 | 7 | ## 6.11.0 (2025-03-13) 8 | 9 | ### New features 10 | 11 | Stream parsers now support a `mergeTokens` option that can be used to turn off automatic merging of adjacent tokens. 12 | 13 | ## 6.10.8 (2024-12-23) 14 | 15 | ### Bug fixes 16 | 17 | Fix a regression introduced 6.10.7 that caused indention to sometimes crash on nested language boundaries. 18 | 19 | ## 6.10.7 (2024-12-17) 20 | 21 | ### Bug fixes 22 | 23 | Fix an issue where indentation for a stream language would fail to work when the parse covered only part of the document, far from the start. 24 | 25 | Make sure the inner mode gets a chance to indent when indenting right at the end of a nested language section. 26 | 27 | ## 6.10.6 (2024-11-29) 28 | 29 | ### Bug fixes 30 | 31 | Fix a crash in `StreamLanguage` when the input range is entirely before the editor viewport. 32 | 33 | ## 6.10.5 (2024-11-27) 34 | 35 | ### Bug fixes 36 | 37 | Fix an issue where a `StreamLanguage` could get confused when trying to reuse existing parse data when the parsed range changed. 38 | 39 | ## 6.10.4 (2024-11-24) 40 | 41 | ### Bug fixes 42 | 43 | Join adjacent tokens of the same type into a single token in . 44 | 45 | Call stream language indent functions even when the language is used as a nested parser. 46 | 47 | Fix a crash in `StreamParser` when a parse was resumed with different input ranges. 48 | 49 | ## 6.10.3 (2024-09-19) 50 | 51 | ### Bug fixes 52 | 53 | Fix a TypeScript error when using `HighlightStyle` with the `exactOptionalPropertyTypes` typechecking option enabled. 54 | 55 | Make `delimitedIndent` align to spaces after the opening token. 56 | 57 | ## 6.10.2 (2024-06-03) 58 | 59 | ### Bug fixes 60 | 61 | Fix an infinite loop that could occur when enabling `bidiIsolates` in documents with both bidirectional text and very long lines. 62 | 63 | ## 6.10.1 (2024-02-02) 64 | 65 | ### Bug fixes 66 | 67 | Fix an issue where, when a lot of code is visible in the initial editor, the bottom bit of code is shown without highlighting for one frame. 68 | 69 | ## 6.10.0 (2023-12-28) 70 | 71 | ### New features 72 | 73 | The new `bidiIsolates` extension can be used to wrap syntactic elements where this is appropriate in an element that isolates their text direction, avoiding weird ordering of neutral characters on direction boundaries. 74 | 75 | ## 6.9.3 (2023-11-27) 76 | 77 | ### Bug fixes 78 | 79 | Fix an issue in `StreamLanguage` where it ran out of node type ids if you repeatedly redefined a language with the same token table. 80 | 81 | ## 6.9.2 (2023-10-24) 82 | 83 | ### Bug fixes 84 | 85 | Allow `StreamParser` tokens get multiple highlighting tags. 86 | 87 | ## 6.9.1 (2023-09-20) 88 | 89 | ### Bug fixes 90 | 91 | Indentation now works a lot better in mixed-language documents that interleave the languages in a complex way. 92 | 93 | Code folding is now able to pick the right foldable syntax node when the line end falls in a mixed-parsing language that doesn't match the target node. 94 | 95 | ## 6.9.0 (2023-08-16) 96 | 97 | ### Bug fixes 98 | 99 | Make `getIndentation` return null, rather than 0, when there is no syntax tree available. 100 | 101 | ### New features 102 | 103 | The new `preparePlaceholder` option to `codeFolding` makes it possible to display contextual information in a folded range placeholder widget. 104 | 105 | ## 6.8.0 (2023-06-12) 106 | 107 | ### New features 108 | 109 | The new `baseIndentFor` method in `TreeIndentContext` can be used to find the base indentation for an arbitrary node. 110 | 111 | ## 6.7.0 (2023-05-19) 112 | 113 | ### New features 114 | 115 | Export `DocInput` class for feeding editor documents to a Lezer parser. 116 | 117 | ## 6.6.0 (2023-02-13) 118 | 119 | ### New features 120 | 121 | Syntax-driven language data queries now support sublanguages, which make it possible to return different data for specific parts of the tree produced by a single language. 122 | 123 | ## 6.5.0 (2023-02-07) 124 | 125 | ### Bug fixes 126 | 127 | Make indentation for stream languages more reliable by having `StringStream.indentation` return overridden indentations from the indent context. 128 | 129 | ### New features 130 | 131 | The `toggleFold` command folds or unfolds depending on whether there's an existing folded range on the current line. 132 | 133 | `indentUnit` now accepts any (repeated) whitespace character, not just spaces and tabs. 134 | 135 | ## 6.4.0 (2023-01-12) 136 | 137 | ### New features 138 | 139 | The `bracketMatchingHandle` node prop can now be used to limit bracket matching behavior for larger nodes to a single subnode (for example the tag name of an HTML tag). 140 | 141 | ## 6.3.2 (2022-12-16) 142 | 143 | ### Bug fixes 144 | 145 | Fix a bug that caused `ensureSyntaxTree` to return incomplete trees when using a viewport-aware parser like `StreamLanguage`. 146 | 147 | ## 6.3.1 (2022-11-14) 148 | 149 | ### Bug fixes 150 | 151 | Make syntax-based folding include syntax nodes that start right at the end of a line as potential fold targets. 152 | 153 | Fix the `indentService` protocol to allow a distinction between declining to handle the indentation and returning null to indicate the line has no definite indentation. 154 | 155 | ## 6.3.0 (2022-10-24) 156 | 157 | ### New features 158 | 159 | `HighlightStyle` objects now have a `specs` property holding the tag styles that were used to define them. 160 | 161 | `Language` objects now have a `name` field holding the language name. 162 | 163 | ## 6.2.1 (2022-07-21) 164 | 165 | ### Bug fixes 166 | 167 | Fix a bug where `bracketMatching` would incorrectly match nested brackets in syntax trees that put multiple pairs of brackets in the same parent node. 168 | 169 | Fix a bug that could cause `indentRange` to loop infinitely. 170 | 171 | ## 6.2.0 (2022-06-30) 172 | 173 | ### Bug fixes 174 | 175 | Fix a bug that prevented bracket matching to recognize plain brackets inside a language parsed as an overlay. 176 | 177 | ### New features 178 | 179 | The `indentRange` function provides an easy way to programatically auto-indent a range of the document. 180 | 181 | ## 6.1.0 (2022-06-20) 182 | 183 | ### New features 184 | 185 | The `foldState` field is now public, and can be used to serialize and deserialize the fold state. 186 | 187 | ## 6.0.0 (2022-06-08) 188 | 189 | ### New features 190 | 191 | The `foldingChanged` option to `foldGutter` can now be used to trigger a recomputation of the fold markers. 192 | 193 | ## 0.20.2 (2022-05-20) 194 | 195 | ### Bug fixes 196 | 197 | List style-mod as a dependency. 198 | 199 | ## 0.20.1 (2022-05-18) 200 | 201 | ### Bug fixes 202 | 203 | Make sure `all` styles in the CSS generated for a `HighlightStyle` have a lower precedence than the other rules defined for the style. Use a shorthand property 204 | 205 | ## 0.20.0 (2022-04-20) 206 | 207 | ### Breaking changes 208 | 209 | `HighlightStyle.get` is now called `highlightingFor`. 210 | 211 | `HighlightStyles` no longer function as extensions (to improve tree shaking), and must be wrapped with `syntaxHighlighting` to add to an editor configuration. 212 | 213 | `Language` objects no longer have a `topNode` property. 214 | 215 | ### New features 216 | 217 | `HighlightStyle` and `defaultHighlightStyle` from the now-removed @codemirror/highlight package now live in this package. 218 | 219 | The new `forceParsing` function can be used to run the parser forward on an editor view. 220 | 221 | The exports that used to live in @codemirror/matchbrackets are now exported from this package. 222 | 223 | The @codemirror/fold package has been merged into this one. 224 | 225 | The exports from the old @codemirror/stream-parser package now live in this package. 226 | 227 | ## 0.19.10 (2022-03-31) 228 | 229 | ### Bug fixes 230 | 231 | Autocompletion may now also trigger automatic indentation on input. 232 | 233 | ## 0.19.9 (2022-03-30) 234 | 235 | ### Bug fixes 236 | 237 | Make sure nodes that end at the end of a partial parse aren't treated as valid fold targets. 238 | 239 | Fix an issue where the parser sometimes wouldn't reuse parsing work done in the background on transactions. 240 | 241 | ## 0.19.8 (2022-03-03) 242 | 243 | ### Bug fixes 244 | 245 | Fix an issue that could cause indentation logic to use the wrong line content when indenting multiple lines at once. 246 | 247 | ## 0.19.7 (2021-12-02) 248 | 249 | ### Bug fixes 250 | 251 | Fix an issue where the parse worker could incorrectly stop working when the parse tree has skipped gaps in it. 252 | 253 | ## 0.19.6 (2021-11-26) 254 | 255 | ### Bug fixes 256 | 257 | Fixes an issue where the background parse work would be scheduled too aggressively, degrading responsiveness on a newly-created editor with a large document. 258 | 259 | Improve initial highlight for mixed-language editors and limit the amount of parsing done on state creation for faster startup. 260 | 261 | ## 0.19.5 (2021-11-17) 262 | 263 | ### New features 264 | 265 | The new function `syntaxTreeAvailable` can be used to check if a fully-parsed syntax tree is available up to a given document position. 266 | 267 | The module now exports `syntaxParserRunning`, which tells you whether the background parser is still planning to do more work for a given editor view. 268 | 269 | ## 0.19.4 (2021-11-13) 270 | 271 | ### New features 272 | 273 | `LanguageDescription.of` now takes an optional already-loaded extension. 274 | 275 | ## 0.19.3 (2021-09-13) 276 | 277 | ### Bug fixes 278 | 279 | Fix an issue where a parse that skipped content with `skipUntilInView` would in some cases not be restarted when the range came into view. 280 | 281 | ## 0.19.2 (2021-08-11) 282 | 283 | ### Bug fixes 284 | 285 | Fix a bug that caused `indentOnInput` to fire for the wrong kinds of transactions. 286 | 287 | Fix a bug that could cause `indentOnInput` to apply its changes incorrectly. 288 | 289 | ## 0.19.1 (2021-08-11) 290 | 291 | ### Bug fixes 292 | 293 | Fix incorrect versions for @lezer dependencies. 294 | 295 | ## 0.19.0 (2021-08-11) 296 | 297 | ### Breaking changes 298 | 299 | CodeMirror now uses lezer 0.15, which means different package names (scoped with @lezer) and some breaking changes in the library. 300 | 301 | `EditorParseContext` is now called `ParseContext`. It is no longer passed to parsers, but must be retrieved with `ParseContext.get`. 302 | 303 | `IndentContext.lineIndent` now takes a position, not a `Line` object, as argument. 304 | 305 | `LezerLanguage` was renamed to `LRLanguage` (because all languages must emit Lezer-style trees, the name was misleading). 306 | 307 | `Language.parseString` no longer exists. You can just call `.parser.parse(...)` instead. 308 | 309 | ### New features 310 | 311 | New `IndentContext.lineAt` method to access lines in a way that is aware of simulated line breaks. 312 | 313 | `IndentContext` now provides a `simulatedBreak` property through which client code can query whether the context has a simulated line break. 314 | 315 | ## 0.18.2 (2021-06-01) 316 | 317 | ### Bug fixes 318 | 319 | Fix an issue where asynchronous re-parsing (with dynamically loaded languages) sometimes failed to fully happen. 320 | 321 | ## 0.18.1 (2021-03-31) 322 | 323 | ### Breaking changes 324 | 325 | `EditorParseContext.getSkippingParser` now replaces `EditorParseContext.skippingParser` and allows you to provide a promise that'll cause parsing to start again. (The old property remains available until the next major release.) 326 | 327 | ### Bug fixes 328 | 329 | Fix an issue where nested parsers could see past the end of the nested region. 330 | 331 | ## 0.18.0 (2021-03-03) 332 | 333 | ### Breaking changes 334 | 335 | Update dependencies to 0.18. 336 | 337 | ### Breaking changes 338 | 339 | The `Language` constructor takes an additional argument that provides the top node type. 340 | 341 | ### New features 342 | 343 | `Language` instances now have a `topNode` property giving their top node type. 344 | 345 | `TreeIndentContext` now has a `continue` method that allows an indenter to defer to the indentation of the parent nodes. 346 | 347 | ## 0.17.5 (2021-02-19) 348 | 349 | ### New features 350 | 351 | This package now exports a `foldInside` helper function, a fold function that should work for most delimited node types. 352 | 353 | ## 0.17.4 (2021-01-15) 354 | 355 | ## 0.17.3 (2021-01-15) 356 | 357 | ### Bug fixes 358 | 359 | Parse scheduling has been improved to reduce the likelyhood of the user looking at unparsed code in big documents. 360 | 361 | Prevent parser from running too far past the current viewport in huge documents. 362 | 363 | ## 0.17.2 (2021-01-06) 364 | 365 | ### New features 366 | 367 | The package now also exports a CommonJS module. 368 | 369 | ## 0.17.1 (2020-12-30) 370 | 371 | ### Bug fixes 372 | 373 | Fix a bug where changing the editor configuration wouldn't update the language parser used. 374 | 375 | ## 0.17.0 (2020-12-29) 376 | 377 | ### Breaking changes 378 | 379 | First numbered release. 380 | 381 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (C) 2018-2021 by Marijn Haverbeke and others 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # @codemirror/language [![NPM version](https://img.shields.io/npm/v/@codemirror/language.svg)](https://www.npmjs.org/package/@codemirror/language) 2 | 3 | [ [**WEBSITE**](https://codemirror.net/) | [**DOCS**](https://codemirror.net/docs/ref/#language) | [**ISSUES**](https://github.com/codemirror/dev/issues) | [**FORUM**](https://discuss.codemirror.net/c/next/) | [**CHANGELOG**](https://github.com/codemirror/language/blob/main/CHANGELOG.md) ] 4 | 5 | This package implements the language support infrastructure for the 6 | [CodeMirror](https://codemirror.net/) code editor. 7 | 8 | The [project page](https://codemirror.net/) has more information, a 9 | number of [examples](https://codemirror.net/examples/) and the 10 | [documentation](https://codemirror.net/docs/). 11 | 12 | This code is released under an 13 | [MIT license](https://github.com/codemirror/language/tree/main/LICENSE). 14 | 15 | We aim to be an inclusive, welcoming community. To make that explicit, 16 | we have a [code of 17 | conduct](http://contributor-covenant.org/version/1/1/0/) that applies 18 | to communication around the project. 19 | 20 | ## Usage 21 | 22 | Setting up a language from a [Lezer](https://lezer.codemirror.net) 23 | parser looks like this: 24 | 25 | ```javascript 26 | import {parser} from "@lezer/json" 27 | import {LRLanguage, continuedIndent, indentNodeProp, 28 | foldNodeProp, foldInside} from "@codemirror/language" 29 | 30 | export const jsonLanguage = LRLanguage.define({ 31 | name: "json", 32 | parser: parser.configure({ 33 | props: [ 34 | indentNodeProp.add({ 35 | Object: continuedIndent({except: /^\s*\}/}), 36 | Array: continuedIndent({except: /^\s*\]/}) 37 | }), 38 | foldNodeProp.add({ 39 | "Object Array": foldInside 40 | }) 41 | ] 42 | }), 43 | languageData: { 44 | closeBrackets: {brackets: ["[", "{", '"']}, 45 | indentOnInput: /^\s*[\}\]]$/ 46 | } 47 | }) 48 | ``` 49 | 50 | Often, you'll also use this package just to access some specific 51 | language-related features, such as accessing the editor's syntax 52 | tree... 53 | 54 | ```javascript 55 | import {syntaxTree} from "@codemirror/language" 56 | 57 | const tree = syntaxTree(view) 58 | ``` 59 | 60 | ... or computing the appriate indentation at a given point. 61 | 62 | ```javascript 63 | import {getIndentation} from "@codemirror/language" 64 | 65 | console.log(getIndentation(view.state, view.state.selection.main.head)) 66 | ``` 67 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@codemirror/language", 3 | "version": "6.11.1", 4 | "description": "Language support infrastructure for the CodeMirror code editor", 5 | "scripts": { 6 | "test": "cm-runtests", 7 | "prepare": "cm-buildhelper src/index.ts" 8 | }, 9 | "keywords": [ 10 | "editor", 11 | "code" 12 | ], 13 | "author": { 14 | "name": "Marijn Haverbeke", 15 | "email": "marijn@haverbeke.berlin", 16 | "url": "http://marijnhaverbeke.nl" 17 | }, 18 | "type": "module", 19 | "main": "dist/index.cjs", 20 | "exports": { 21 | "import": "./dist/index.js", 22 | "require": "./dist/index.cjs" 23 | }, 24 | "types": "dist/index.d.ts", 25 | "module": "dist/index.js", 26 | "sideEffects": false, 27 | "license": "MIT", 28 | "dependencies": { 29 | "@codemirror/state": "^6.0.0", 30 | "@codemirror/view": "^6.23.0", 31 | "@lezer/common": "^1.1.0", 32 | "@lezer/highlight": "^1.0.0", 33 | "@lezer/lr": "^1.0.0", 34 | "style-mod": "^4.0.0" 35 | }, 36 | "devDependencies": { 37 | "@codemirror/buildhelper": "^1.0.0", 38 | "@lezer/javascript": "^1.0.0" 39 | }, 40 | "repository": { 41 | "type": "git", 42 | "url": "https://github.com/codemirror/language.git" 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/README.md: -------------------------------------------------------------------------------- 1 | @languageDataProp 2 | 3 | @Language 4 | 5 | @defineLanguageFacet 6 | 7 | @Sublanguage 8 | 9 | @sublanguageProp 10 | 11 | @language 12 | 13 | @LRLanguage 14 | 15 | @ParseContext 16 | 17 | @syntaxTree 18 | 19 | @ensureSyntaxTree 20 | 21 | @syntaxTreeAvailable 22 | 23 | @forceParsing 24 | 25 | @syntaxParserRunning 26 | 27 | @LanguageSupport 28 | 29 | @LanguageDescription 30 | 31 | @DocInput 32 | 33 | ### Highlighting 34 | 35 | @HighlightStyle 36 | 37 | @syntaxHighlighting 38 | 39 | @TagStyle 40 | 41 | @defaultHighlightStyle 42 | 43 | @highlightingFor 44 | 45 | @bidiIsolates 46 | 47 | ### Folding 48 | 49 | These exports provide commands and other functionality related to code 50 | folding (temporarily hiding pieces of code). 51 | 52 | @foldService 53 | 54 | @foldNodeProp 55 | 56 | @foldInside 57 | 58 | @foldable 59 | 60 | @foldCode 61 | 62 | @unfoldCode 63 | 64 | @toggleFold 65 | 66 | @foldAll 67 | 68 | @unfoldAll 69 | 70 | @foldKeymap 71 | 72 | @codeFolding 73 | 74 | @foldGutter 75 | 76 | The following functions provide more direct, low-level control over 77 | the fold state. 78 | 79 | @foldedRanges 80 | 81 | @foldState 82 | 83 | @foldEffect 84 | 85 | @unfoldEffect 86 | 87 | ### Indentation 88 | 89 | @indentService 90 | 91 | @indentNodeProp 92 | 93 | @getIndentation 94 | 95 | @indentRange 96 | 97 | @indentUnit 98 | 99 | @getIndentUnit 100 | 101 | @indentString 102 | 103 | @IndentContext 104 | 105 | @TreeIndentContext 106 | 107 | @delimitedIndent 108 | 109 | @continuedIndent 110 | 111 | @flatIndent 112 | 113 | @indentOnInput 114 | 115 | ### Bracket Matching 116 | 117 | @bracketMatching 118 | 119 | @Config 120 | 121 | @matchBrackets 122 | 123 | @MatchResult 124 | 125 | @bracketMatchingHandle 126 | 127 | ### Stream Parser 128 | 129 | Stream parsers provide a way to adapt language modes written in the 130 | CodeMirror 5 style (see 131 | [@codemirror/legacy-modes](https://github.com/codemirror/legacy-modes)) 132 | to the `Language` interface. 133 | 134 | @StreamLanguage 135 | 136 | @StreamParser 137 | 138 | @StringStream 139 | -------------------------------------------------------------------------------- /src/fold.ts: -------------------------------------------------------------------------------- 1 | import {NodeProp, SyntaxNode, NodeIterator} from "@lezer/common" 2 | import {combineConfig, EditorState, StateEffect, ChangeDesc, Facet, StateField, Extension, 3 | RangeSet, RangeSetBuilder} from "@codemirror/state" 4 | import {EditorView, BlockInfo, Command, Decoration, DecorationSet, WidgetType, 5 | KeyBinding, ViewPlugin, ViewUpdate, gutter, GutterMarker} from "@codemirror/view" 6 | import {language, syntaxTree} from "./language" 7 | 8 | /// A facet that registers a code folding service. When called with 9 | /// the extent of a line, such a function should return a foldable 10 | /// range that starts on that line (but continues beyond it), if one 11 | /// can be found. 12 | export const foldService = Facet.define< 13 | (state: EditorState, lineStart: number, lineEnd: number) => ({from: number, to: number} | null) 14 | >() 15 | 16 | /// This node prop is used to associate folding information with 17 | /// syntax node types. Given a syntax node, it should check whether 18 | /// that tree is foldable and return the range that can be collapsed 19 | /// when it is. 20 | export const foldNodeProp = new NodeProp<(node: SyntaxNode, state: EditorState) => ({from: number, to: number} | null)>() 21 | 22 | /// [Fold](#language.foldNodeProp) function that folds everything but 23 | /// the first and the last child of a syntax node. Useful for nodes 24 | /// that start and end with delimiters. 25 | export function foldInside(node: SyntaxNode): {from: number, to: number} | null { 26 | let first = node.firstChild, last = node.lastChild 27 | return first && first.to < last!.from ? {from: first.to, to: last!.type.isError ? node.to : last!.from} : null 28 | } 29 | 30 | function syntaxFolding(state: EditorState, start: number, end: number) { 31 | let tree = syntaxTree(state) 32 | if (tree.length < end) return null 33 | let stack = tree.resolveStack(end, 1) 34 | let found: null | {from: number, to: number} = null 35 | for (let iter: NodeIterator | null = stack; iter; iter = iter.next) { 36 | let cur = iter.node 37 | if (cur.to <= end || cur.from > end) continue 38 | if (found && cur.from < start) break 39 | let prop = cur.type.prop(foldNodeProp) 40 | if (prop && (cur.to < tree.length - 50 || tree.length == state.doc.length || !isUnfinished(cur))) { 41 | let value = prop(cur, state) 42 | if (value && value.from <= end && value.from >= start && value.to > end) found = value 43 | } 44 | } 45 | return found 46 | } 47 | 48 | function isUnfinished(node: SyntaxNode) { 49 | let ch = node.lastChild 50 | return ch && ch.to == node.to && ch.type.isError 51 | } 52 | 53 | /// Check whether the given line is foldable. First asks any fold 54 | /// services registered through 55 | /// [`foldService`](#language.foldService), and if none of them return 56 | /// a result, tries to query the [fold node 57 | /// prop](#language.foldNodeProp) of syntax nodes that cover the end 58 | /// of the line. 59 | export function foldable(state: EditorState, lineStart: number, lineEnd: number) { 60 | for (let service of state.facet(foldService)) { 61 | let result = service(state, lineStart, lineEnd) 62 | if (result) return result 63 | } 64 | return syntaxFolding(state, lineStart, lineEnd) 65 | } 66 | 67 | type DocRange = {from: number, to: number} 68 | 69 | function mapRange(range: DocRange, mapping: ChangeDesc) { 70 | let from = mapping.mapPos(range.from, 1), to = mapping.mapPos(range.to, -1) 71 | return from >= to ? undefined : {from, to} 72 | } 73 | 74 | /// State effect that can be attached to a transaction to fold the 75 | /// given range. (You probably only need this in exceptional 76 | /// circumstances—usually you'll just want to let 77 | /// [`foldCode`](#language.foldCode) and the [fold 78 | /// gutter](#language.foldGutter) create the transactions.) 79 | export const foldEffect = StateEffect.define({map: mapRange}) 80 | 81 | /// State effect that unfolds the given range (if it was folded). 82 | export const unfoldEffect = StateEffect.define({map: mapRange}) 83 | 84 | function selectedLines(view: EditorView) { 85 | let lines: BlockInfo[] = [] 86 | for (let {head} of view.state.selection.ranges) { 87 | if (lines.some(l => l.from <= head && l.to >= head)) continue 88 | lines.push(view.lineBlockAt(head)) 89 | } 90 | return lines 91 | } 92 | 93 | /// The state field that stores the folded ranges (as a [decoration 94 | /// set](#view.DecorationSet)). Can be passed to 95 | /// [`EditorState.toJSON`](#state.EditorState.toJSON) and 96 | /// [`fromJSON`](#state.EditorState^fromJSON) to serialize the fold 97 | /// state. 98 | export const foldState = StateField.define({ 99 | create() { 100 | return Decoration.none 101 | }, 102 | update(folded, tr) { 103 | folded = folded.map(tr.changes) 104 | for (let e of tr.effects) { 105 | if (e.is(foldEffect) && !foldExists(folded, e.value.from, e.value.to)) { 106 | let {preparePlaceholder} = tr.state.facet(foldConfig) 107 | let widget = !preparePlaceholder ? foldWidget : 108 | Decoration.replace({widget: new PreparedFoldWidget(preparePlaceholder(tr.state, e.value))}) 109 | folded = folded.update({add: [widget.range(e.value.from, e.value.to)]}) 110 | } else if (e.is(unfoldEffect)) { 111 | folded = folded.update({filter: (from, to) => e.value.from != from || e.value.to != to, 112 | filterFrom: e.value.from, filterTo: e.value.to}) 113 | } 114 | } 115 | // Clear folded ranges that cover the selection head 116 | if (tr.selection) { 117 | let onSelection = false, {head} = tr.selection.main 118 | folded.between(head, head, (a, b) => { if (a < head && b > head) onSelection = true }) 119 | if (onSelection) folded = folded.update({ 120 | filterFrom: head, 121 | filterTo: head, 122 | filter: (a, b) => b <= head || a >= head 123 | }) 124 | } 125 | return folded 126 | }, 127 | provide: f => EditorView.decorations.from(f), 128 | toJSON(folded, state) { 129 | let ranges: number[] = [] 130 | folded.between(0, state.doc.length, (from, to) => {ranges.push(from, to)}) 131 | return ranges 132 | }, 133 | fromJSON(value) { 134 | if (!Array.isArray(value) || value.length % 2) throw new RangeError("Invalid JSON for fold state") 135 | let ranges = [] 136 | for (let i = 0; i < value.length;) { 137 | let from = value[i++], to = value[i++] 138 | if (typeof from != "number" || typeof to != "number") throw new RangeError("Invalid JSON for fold state") 139 | ranges.push(foldWidget.range(from, to)) 140 | } 141 | return Decoration.set(ranges, true) 142 | } 143 | }) 144 | 145 | /// Get a [range set](#state.RangeSet) containing the folded ranges 146 | /// in the given state. 147 | export function foldedRanges(state: EditorState): DecorationSet { 148 | return state.field(foldState, false) || RangeSet.empty 149 | } 150 | 151 | function findFold(state: EditorState, from: number, to: number) { 152 | let found: {from: number, to: number} | null = null 153 | state.field(foldState, false)?.between(from, to, (from, to) => { 154 | if (!found || found.from > from) found = {from, to} 155 | }) 156 | return found 157 | } 158 | 159 | function foldExists(folded: DecorationSet, from: number, to: number) { 160 | let found = false 161 | folded.between(from, from, (a, b) => { if (a == from && b == to) found = true }) 162 | return found 163 | } 164 | 165 | function maybeEnable(state: EditorState, other: readonly StateEffect[]) { 166 | return state.field(foldState, false) ? other : other.concat(StateEffect.appendConfig.of(codeFolding())) 167 | } 168 | 169 | /// Fold the lines that are selected, if possible. 170 | export const foldCode: Command = view => { 171 | for (let line of selectedLines(view)) { 172 | let range = foldable(view.state, line.from, line.to) 173 | if (range) { 174 | view.dispatch({effects: maybeEnable(view.state, [foldEffect.of(range), announceFold(view, range)])}) 175 | return true 176 | } 177 | } 178 | return false 179 | } 180 | 181 | /// Unfold folded ranges on selected lines. 182 | export const unfoldCode: Command = view => { 183 | if (!view.state.field(foldState, false)) return false 184 | let effects = [] 185 | for (let line of selectedLines(view)) { 186 | let folded = findFold(view.state, line.from, line.to) 187 | if (folded) effects.push(unfoldEffect.of(folded), announceFold(view, folded, false)) 188 | } 189 | if (effects.length) view.dispatch({effects}) 190 | return effects.length > 0 191 | } 192 | 193 | function announceFold(view: EditorView, range: {from: number, to: number}, fold = true) { 194 | let lineFrom = view.state.doc.lineAt(range.from).number, lineTo = view.state.doc.lineAt(range.to).number 195 | return EditorView.announce.of(`${view.state.phrase(fold ? "Folded lines" : "Unfolded lines")} ${lineFrom} ${ 196 | view.state.phrase("to")} ${lineTo}.`) 197 | } 198 | 199 | /// Fold all top-level foldable ranges. Note that, in most cases, 200 | /// folding information will depend on the [syntax 201 | /// tree](#language.syntaxTree), and folding everything may not work 202 | /// reliably when the document hasn't been fully parsed (either 203 | /// because the editor state was only just initialized, or because the 204 | /// document is so big that the parser decided not to parse it 205 | /// entirely). 206 | export const foldAll: Command = view => { 207 | let {state} = view, effects = [] 208 | for (let pos = 0; pos < state.doc.length;) { 209 | let line = view.lineBlockAt(pos), range = foldable(state, line.from, line.to) 210 | if (range) effects.push(foldEffect.of(range)) 211 | pos = (range ? view.lineBlockAt(range.to) : line).to + 1 212 | } 213 | if (effects.length) view.dispatch({effects: maybeEnable(view.state, effects)}) 214 | return !!effects.length 215 | } 216 | 217 | /// Unfold all folded code. 218 | export const unfoldAll: Command = view => { 219 | let field = view.state.field(foldState, false) 220 | if (!field || !field.size) return false 221 | let effects: StateEffect[] = [] 222 | field.between(0, view.state.doc.length, (from, to) => { effects.push(unfoldEffect.of({from, to})) }) 223 | view.dispatch({effects}) 224 | return true 225 | } 226 | 227 | // Find the foldable region containing the given line, if one exists 228 | function foldableContainer(view: EditorView, lineBlock: BlockInfo) { 229 | // Look backwards through line blocks until we find a foldable region that 230 | // intersects with the line 231 | for (let line = lineBlock;;) { 232 | let foldableRegion = foldable(view.state, line.from, line.to) 233 | if (foldableRegion && foldableRegion.to > lineBlock.from) return foldableRegion 234 | if (!line.from) return null 235 | line = view.lineBlockAt(line.from - 1) 236 | } 237 | } 238 | 239 | /// Toggle folding at cursors. Unfolds if there is an existing fold 240 | /// starting in that line, tries to find a foldable range around it 241 | /// otherwise. 242 | export const toggleFold: Command = (view) => { 243 | let effects: StateEffect[] = [] 244 | for (let line of selectedLines(view)) { 245 | let folded = findFold(view.state, line.from, line.to) 246 | if (folded) { 247 | effects.push(unfoldEffect.of(folded), announceFold(view, folded, false)) 248 | } else { 249 | let foldRange = foldableContainer(view, line) 250 | if (foldRange) effects.push(foldEffect.of(foldRange), announceFold(view, foldRange)) 251 | } 252 | } 253 | if (effects.length > 0) view.dispatch({effects: maybeEnable(view.state, effects)}) 254 | return !!effects.length 255 | } 256 | 257 | /// Default fold-related key bindings. 258 | /// 259 | /// - Ctrl-Shift-[ (Cmd-Alt-[ on macOS): [`foldCode`](#language.foldCode). 260 | /// - Ctrl-Shift-] (Cmd-Alt-] on macOS): [`unfoldCode`](#language.unfoldCode). 261 | /// - Ctrl-Alt-[: [`foldAll`](#language.foldAll). 262 | /// - Ctrl-Alt-]: [`unfoldAll`](#language.unfoldAll). 263 | export const foldKeymap: readonly KeyBinding[] = [ 264 | {key: "Ctrl-Shift-[", mac: "Cmd-Alt-[", run: foldCode}, 265 | {key: "Ctrl-Shift-]", mac: "Cmd-Alt-]", run: unfoldCode}, 266 | {key: "Ctrl-Alt-[", run: foldAll}, 267 | {key: "Ctrl-Alt-]", run: unfoldAll} 268 | ] 269 | 270 | interface FoldConfig { 271 | /// A function that creates the DOM element used to indicate the 272 | /// position of folded code. The `onclick` argument is the default 273 | /// click event handler, which toggles folding on the line that 274 | /// holds the element, and should probably be added as an event 275 | /// handler to the returned element. If 276 | /// [`preparePlaceholder`](#language.FoldConfig.preparePlaceholder) 277 | /// is given, its result will be passed as 3rd argument. Otherwise, 278 | /// this will be null. 279 | /// 280 | /// When this option isn't given, the `placeholderText` option will 281 | /// be used to create the placeholder element. 282 | placeholderDOM?: ((view: EditorView, onclick: (event: Event) => void, prepared: any) => HTMLElement) | null, 283 | /// Text to use as placeholder for folded text. Defaults to `"…"`. 284 | /// Will be styled with the `"cm-foldPlaceholder"` class. 285 | placeholderText?: string 286 | /// Given a range that is being folded, create a value that 287 | /// describes it, to be used by `placeholderDOM` to render a custom 288 | /// widget that, for example, indicates something about the folded 289 | /// range's size or type. 290 | preparePlaceholder?: (state: EditorState, range: {from: number, to: number}) => any 291 | } 292 | 293 | const defaultConfig: Required = { 294 | placeholderDOM: null, 295 | preparePlaceholder: null as any, 296 | placeholderText: "…" 297 | } 298 | 299 | const foldConfig = Facet.define>({ 300 | combine(values) { return combineConfig(values, defaultConfig) } 301 | }) 302 | 303 | /// Create an extension that configures code folding. 304 | export function codeFolding(config?: FoldConfig): Extension { 305 | let result = [foldState, baseTheme] 306 | if (config) result.push(foldConfig.of(config)) 307 | return result 308 | } 309 | 310 | function widgetToDOM(view: EditorView, prepared: any) { 311 | let {state} = view, conf = state.facet(foldConfig) 312 | let onclick = (event: Event) => { 313 | let line = view.lineBlockAt(view.posAtDOM(event.target as HTMLElement)) 314 | let folded = findFold(view.state, line.from, line.to) 315 | if (folded) view.dispatch({effects: unfoldEffect.of(folded)}) 316 | event.preventDefault() 317 | } 318 | if (conf.placeholderDOM) return conf.placeholderDOM(view, onclick, prepared) 319 | let element = document.createElement("span") 320 | element.textContent = conf.placeholderText 321 | element.setAttribute("aria-label", state.phrase("folded code")) 322 | element.title = state.phrase("unfold") 323 | element.className = "cm-foldPlaceholder" 324 | element.onclick = onclick 325 | return element 326 | } 327 | 328 | const foldWidget = Decoration.replace({widget: new class extends WidgetType { 329 | toDOM(view: EditorView) { return widgetToDOM(view, null) } 330 | }}) 331 | 332 | class PreparedFoldWidget extends WidgetType { 333 | constructor(readonly value: any) { super() } 334 | eq(other: PreparedFoldWidget) { return this.value == other.value } 335 | toDOM(view: EditorView) { return widgetToDOM(view, this.value) } 336 | } 337 | 338 | type Handlers = {[event: string]: (view: EditorView, line: BlockInfo, event: Event) => boolean} 339 | 340 | interface FoldGutterConfig { 341 | /// A function that creates the DOM element used to indicate a 342 | /// given line is folded or can be folded. 343 | /// When not given, the `openText`/`closeText` option will be used instead. 344 | markerDOM?: ((open: boolean) => HTMLElement) | null 345 | /// Text used to indicate that a given line can be folded. 346 | /// Defaults to `"⌄"`. 347 | openText?: string 348 | /// Text used to indicate that a given line is folded. 349 | /// Defaults to `"›"`. 350 | closedText?: string 351 | /// Supply event handlers for DOM events on this gutter. 352 | domEventHandlers?: Handlers 353 | /// When given, if this returns true for a given view update, 354 | /// recompute the fold markers. 355 | foldingChanged?: (update: ViewUpdate) => boolean 356 | } 357 | 358 | const foldGutterDefaults: Required = { 359 | openText: "⌄", 360 | closedText: "›", 361 | markerDOM: null, 362 | domEventHandlers: {}, 363 | foldingChanged: () => false 364 | } 365 | 366 | class FoldMarker extends GutterMarker { 367 | constructor(readonly config: Required, 368 | readonly open: boolean) { super() } 369 | 370 | eq(other: FoldMarker) { return this.config == other.config && this.open == other.open } 371 | 372 | toDOM(view: EditorView) { 373 | if (this.config.markerDOM) return this.config.markerDOM(this.open) 374 | 375 | let span = document.createElement("span") 376 | span.textContent = this.open ? this.config.openText : this.config.closedText 377 | span.title = view.state.phrase(this.open ? "Fold line" : "Unfold line") 378 | return span 379 | } 380 | } 381 | 382 | /// Create an extension that registers a fold gutter, which shows a 383 | /// fold status indicator before foldable lines (which can be clicked 384 | /// to fold or unfold the line). 385 | export function foldGutter(config: FoldGutterConfig = {}): Extension { 386 | let fullConfig = {...foldGutterDefaults, ...config} 387 | let canFold = new FoldMarker(fullConfig, true), canUnfold = new FoldMarker(fullConfig, false) 388 | 389 | let markers = ViewPlugin.fromClass(class { 390 | markers: RangeSet 391 | from: number 392 | 393 | constructor(view: EditorView) { 394 | this.from = view.viewport.from 395 | this.markers = this.buildMarkers(view) 396 | } 397 | 398 | update(update: ViewUpdate) { 399 | if (update.docChanged || update.viewportChanged || 400 | update.startState.facet(language) != update.state.facet(language) || 401 | update.startState.field(foldState, false) != update.state.field(foldState, false) || 402 | syntaxTree(update.startState) != syntaxTree(update.state) || 403 | fullConfig.foldingChanged(update)) 404 | this.markers = this.buildMarkers(update.view) 405 | } 406 | 407 | buildMarkers(view: EditorView) { 408 | let builder = new RangeSetBuilder() 409 | for (let line of view.viewportLineBlocks) { 410 | let mark = findFold(view.state, line.from, line.to) ? canUnfold 411 | : foldable(view.state, line.from, line.to) ? canFold : null 412 | if (mark) builder.add(line.from, line.from, mark) 413 | } 414 | return builder.finish() 415 | } 416 | }) 417 | 418 | let { domEventHandlers } = fullConfig; 419 | 420 | return [ 421 | markers, 422 | gutter({ 423 | class: "cm-foldGutter", 424 | markers(view) { return view.plugin(markers)?.markers || RangeSet.empty }, 425 | initialSpacer() { 426 | return new FoldMarker(fullConfig, false) 427 | }, 428 | domEventHandlers: { 429 | ...domEventHandlers, 430 | click: (view, line, event) => { 431 | if (domEventHandlers.click && domEventHandlers.click(view, line, event)) return true 432 | 433 | let folded = findFold(view.state, line.from, line.to) 434 | if (folded) { 435 | view.dispatch({effects: unfoldEffect.of(folded)}) 436 | return true 437 | } 438 | let range = foldable(view.state, line.from, line.to) 439 | if (range) { 440 | view.dispatch({effects: foldEffect.of(range)}) 441 | return true 442 | } 443 | return false 444 | } 445 | } 446 | }), 447 | codeFolding() 448 | ] 449 | } 450 | 451 | const baseTheme = EditorView.baseTheme({ 452 | ".cm-foldPlaceholder": { 453 | backgroundColor: "#eee", 454 | border: "1px solid #ddd", 455 | color: "#888", 456 | borderRadius: ".2em", 457 | margin: "0 1px", 458 | padding: "0 1px", 459 | cursor: "pointer" 460 | }, 461 | 462 | ".cm-foldGutter span": { 463 | padding: "0 1px", 464 | cursor: "pointer" 465 | } 466 | }) 467 | -------------------------------------------------------------------------------- /src/highlight.ts: -------------------------------------------------------------------------------- 1 | import {Tree, NodeType} from "@lezer/common" 2 | import {Tag, tags, tagHighlighter, Highlighter, highlightTree} from "@lezer/highlight" 3 | import {StyleSpec, StyleModule} from "style-mod" 4 | import {EditorView, ViewPlugin, ViewUpdate, Decoration, DecorationSet} from "@codemirror/view" 5 | import {EditorState, Prec, Facet, Extension, RangeSetBuilder} from "@codemirror/state" 6 | import {syntaxTree, Language, languageDataProp} from "./language" 7 | 8 | /// A highlight style associates CSS styles with higlighting 9 | /// [tags](https://lezer.codemirror.net/docs/ref#highlight.Tag). 10 | export class HighlightStyle implements Highlighter { 11 | /// A style module holding the CSS rules for this highlight style. 12 | /// When using 13 | /// [`highlightTree`](https://lezer.codemirror.net/docs/ref#highlight.highlightTree) 14 | /// outside of the editor, you may want to manually mount this 15 | /// module to show the highlighting. 16 | readonly module: StyleModule | null 17 | 18 | /// @internal 19 | readonly themeType: "dark" | "light" | undefined 20 | 21 | readonly style: (tags: readonly Tag[]) => string | null 22 | readonly scope?: (type: NodeType) => boolean 23 | 24 | private constructor( 25 | /// The tag styles used to create this highlight style. 26 | readonly specs: readonly TagStyle[], 27 | options: {scope?: NodeType | Language, all?: string | StyleSpec, themeType?: "dark" | "light"} 28 | ) { 29 | let modSpec: {[name: string]: StyleSpec} | undefined 30 | function def(spec: StyleSpec) { 31 | let cls = StyleModule.newName() 32 | ;(modSpec || (modSpec = Object.create(null)))["." + cls] = spec 33 | return cls 34 | } 35 | 36 | const all = typeof options.all == "string" ? options.all : options.all ? def(options.all) : undefined 37 | 38 | const scopeOpt = options.scope 39 | this.scope = scopeOpt instanceof Language ? (type: NodeType) => type.prop(languageDataProp) == scopeOpt.data 40 | : scopeOpt ? (type: NodeType) => type == scopeOpt : undefined 41 | 42 | this.style = tagHighlighter(specs.map(style => ({ 43 | tag: style.tag, 44 | class: style.class as string || def(Object.assign({}, style, {tag: null})) 45 | })), { 46 | all, 47 | }).style 48 | 49 | this.module = modSpec ? new StyleModule(modSpec) : null 50 | this.themeType = options.themeType 51 | } 52 | 53 | /// Create a highlighter style that associates the given styles to 54 | /// the given tags. The specs must be objects that hold a style tag 55 | /// or array of tags in their `tag` property, and either a single 56 | /// `class` property providing a static CSS class (for highlighter 57 | /// that rely on external styling), or a 58 | /// [`style-mod`](https://github.com/marijnh/style-mod#documentation)-style 59 | /// set of CSS properties (which define the styling for those tags). 60 | /// 61 | /// The CSS rules created for a highlighter will be emitted in the 62 | /// order of the spec's properties. That means that for elements that 63 | /// have multiple tags associated with them, styles defined further 64 | /// down in the list will have a higher CSS precedence than styles 65 | /// defined earlier. 66 | static define(specs: readonly TagStyle[], options?: { 67 | /// By default, highlighters apply to the entire document. You can 68 | /// scope them to a single language by providing the language 69 | /// object or a language's top node type here. 70 | scope?: Language | NodeType, 71 | /// Add a style to _all_ content. Probably only useful in 72 | /// combination with `scope`. 73 | all?: string | StyleSpec, 74 | /// Specify that this highlight style should only be active then 75 | /// the theme is dark or light. By default, it is active 76 | /// regardless of theme. 77 | themeType?: "dark" | "light" 78 | }) { 79 | return new HighlightStyle(specs, options || {}) 80 | } 81 | } 82 | 83 | const highlighterFacet = Facet.define() 84 | 85 | const fallbackHighlighter = Facet.define({ 86 | combine(values) { return values.length ? [values[0]] : null } 87 | }) 88 | 89 | function getHighlighters(state: EditorState): readonly Highlighter[] | null { 90 | let main = state.facet(highlighterFacet) 91 | return main.length ? main : state.facet(fallbackHighlighter) 92 | } 93 | 94 | /// Wrap a highlighter in an editor extension that uses it to apply 95 | /// syntax highlighting to the editor content. 96 | /// 97 | /// When multiple (non-fallback) styles are provided, the styling 98 | /// applied is the union of the classes they emit. 99 | export function syntaxHighlighting(highlighter: Highlighter, options?: { 100 | /// When enabled, this marks the highlighter as a fallback, which 101 | /// only takes effect if no other highlighters are registered. 102 | fallback: boolean 103 | }): Extension { 104 | let ext: Extension[] = [treeHighlighter], themeType: string | undefined 105 | if (highlighter instanceof HighlightStyle) { 106 | if (highlighter.module) ext.push(EditorView.styleModule.of(highlighter.module)) 107 | themeType = highlighter.themeType 108 | } 109 | if (options?.fallback) 110 | ext.push(fallbackHighlighter.of(highlighter)) 111 | else if (themeType) 112 | ext.push(highlighterFacet.computeN([EditorView.darkTheme], state => { 113 | return state.facet(EditorView.darkTheme) == (themeType == "dark") ? [highlighter] : [] 114 | })) 115 | else 116 | ext.push(highlighterFacet.of(highlighter)) 117 | return ext 118 | } 119 | 120 | /// Returns the CSS classes (if any) that the highlighters active in 121 | /// the state would assign to the given style 122 | /// [tags](https://lezer.codemirror.net/docs/ref#highlight.Tag) and 123 | /// (optional) language 124 | /// [scope](#language.HighlightStyle^define^options.scope). 125 | export function highlightingFor(state: EditorState, tags: readonly Tag[], scope?: NodeType): string | null { 126 | let highlighters = getHighlighters(state) 127 | let result = null 128 | if (highlighters) for (let highlighter of highlighters) { 129 | if (!highlighter.scope || scope && highlighter.scope(scope)) { 130 | let cls = highlighter.style(tags) 131 | if (cls) result = result ? result + " " + cls : cls 132 | } 133 | } 134 | return result 135 | } 136 | 137 | /// The type of object used in 138 | /// [`HighlightStyle.define`](#language.HighlightStyle^define). 139 | /// Assigns a style to one or more highlighting 140 | /// [tags](https://lezer.codemirror.net/docs/ref#highlight.Tag), which can either be a fixed class name 141 | /// (which must be defined elsewhere), or a set of CSS properties, for 142 | /// which the library will define an anonymous class. 143 | export interface TagStyle { 144 | /// The tag or tags to target. 145 | tag: Tag | readonly Tag[], 146 | /// If given, this maps the tags to a fixed class name. 147 | class?: string, 148 | /// Any further properties (if `class` isn't given) will be 149 | /// interpreted as in style objects given to 150 | /// [style-mod](https://github.com/marijnh/style-mod#documentation). 151 | /// (The type here is `any` because of TypeScript limitations.) 152 | [styleProperty: string]: any 153 | } 154 | 155 | class TreeHighlighter { 156 | decorations: DecorationSet 157 | decoratedTo: number 158 | tree: Tree 159 | markCache: {[cls: string]: Decoration} = Object.create(null) 160 | 161 | constructor(view: EditorView) { 162 | this.tree = syntaxTree(view.state) 163 | this.decorations = this.buildDeco(view, getHighlighters(view.state)) 164 | this.decoratedTo = view.viewport.to 165 | } 166 | 167 | update(update: ViewUpdate) { 168 | let tree = syntaxTree(update.state), highlighters = getHighlighters(update.state) 169 | let styleChange = highlighters != getHighlighters(update.startState) 170 | let {viewport} = update.view, decoratedToMapped = update.changes.mapPos(this.decoratedTo, 1) 171 | if (tree.length < viewport.to && !styleChange && tree.type == this.tree.type && decoratedToMapped >= viewport.to) { 172 | this.decorations = this.decorations.map(update.changes) 173 | this.decoratedTo = decoratedToMapped 174 | } else if (tree != this.tree || update.viewportChanged || styleChange) { 175 | this.tree = tree 176 | this.decorations = this.buildDeco(update.view, highlighters) 177 | this.decoratedTo = viewport.to 178 | } 179 | } 180 | 181 | buildDeco(view: EditorView, highlighters: readonly Highlighter[] | null) { 182 | if (!highlighters || !this.tree.length) return Decoration.none 183 | 184 | let builder = new RangeSetBuilder() 185 | for (let {from, to} of view.visibleRanges) { 186 | highlightTree(this.tree, highlighters, (from, to, style) => { 187 | builder.add(from, to, this.markCache[style] || (this.markCache[style] = Decoration.mark({class: style}))) 188 | }, from, to) 189 | } 190 | return builder.finish() 191 | } 192 | } 193 | 194 | const treeHighlighter = Prec.high(ViewPlugin.fromClass(TreeHighlighter, { 195 | decorations: v => v.decorations 196 | })) 197 | 198 | /// A default highlight style (works well with light themes). 199 | export const defaultHighlightStyle = HighlightStyle.define([ 200 | {tag: tags.meta, 201 | color: "#404740"}, 202 | {tag: tags.link, 203 | textDecoration: "underline"}, 204 | {tag: tags.heading, 205 | textDecoration: "underline", 206 | fontWeight: "bold"}, 207 | {tag: tags.emphasis, 208 | fontStyle: "italic"}, 209 | {tag: tags.strong, 210 | fontWeight: "bold"}, 211 | {tag: tags.strikethrough, 212 | textDecoration: "line-through"}, 213 | {tag: tags.keyword, 214 | color: "#708"}, 215 | {tag: [tags.atom, tags.bool, tags.url, tags.contentSeparator, tags.labelName], 216 | color: "#219"}, 217 | {tag: [tags.literal, tags.inserted], 218 | color: "#164"}, 219 | {tag: [tags.string, tags.deleted], 220 | color: "#a11"}, 221 | {tag: [tags.regexp, tags.escape, tags.special(tags.string)], 222 | color: "#e40"}, 223 | {tag: tags.definition(tags.variableName), 224 | color: "#00f"}, 225 | {tag: tags.local(tags.variableName), 226 | color: "#30a"}, 227 | {tag: [tags.typeName, tags.namespace], 228 | color: "#085"}, 229 | {tag: tags.className, 230 | color: "#167"}, 231 | {tag: [tags.special(tags.variableName), tags.macroName], 232 | color: "#256"}, 233 | {tag: tags.definition(tags.propertyName), 234 | color: "#00c"}, 235 | {tag: tags.comment, 236 | color: "#940"}, 237 | {tag: tags.invalid, 238 | color: "#f00"} 239 | ]) 240 | -------------------------------------------------------------------------------- /src/indent.ts: -------------------------------------------------------------------------------- 1 | import {NodeProp, SyntaxNode, NodeIterator, Tree} from "@lezer/common" 2 | import {EditorState, Extension, Facet, countColumn, ChangeSpec} from "@codemirror/state" 3 | import {syntaxTree} from "./language" 4 | 5 | /// Facet that defines a way to provide a function that computes the 6 | /// appropriate indentation depth, as a column number (see 7 | /// [`indentString`](#language.indentString)), at the start of a given 8 | /// line. A return value of `null` indicates no indentation can be 9 | /// determined, and the line should inherit the indentation of the one 10 | /// above it. A return value of `undefined` defers to the next indent 11 | /// service. 12 | export const indentService = Facet.define<(context: IndentContext, pos: number) => number | null | undefined>() 13 | 14 | /// Facet for overriding the unit by which indentation happens. Should 15 | /// be a string consisting entirely of the same whitespace character. 16 | /// When not set, this defaults to 2 spaces. 17 | export const indentUnit = Facet.define({ 18 | combine: values => { 19 | if (!values.length) return " " 20 | let unit = values[0] 21 | if (!unit || /\S/.test(unit) || Array.from(unit).some(e => e != unit[0])) 22 | throw new Error("Invalid indent unit: " + JSON.stringify(values[0])) 23 | return unit 24 | } 25 | }) 26 | 27 | /// Return the _column width_ of an indent unit in the state. 28 | /// Determined by the [`indentUnit`](#language.indentUnit) 29 | /// facet, and [`tabSize`](#state.EditorState^tabSize) when that 30 | /// contains tabs. 31 | export function getIndentUnit(state: EditorState) { 32 | let unit = state.facet(indentUnit) 33 | return unit.charCodeAt(0) == 9 ? state.tabSize * unit.length : unit.length 34 | } 35 | 36 | /// Create an indentation string that covers columns 0 to `cols`. 37 | /// Will use tabs for as much of the columns as possible when the 38 | /// [`indentUnit`](#language.indentUnit) facet contains 39 | /// tabs. 40 | export function indentString(state: EditorState, cols: number) { 41 | let result = "", ts = state.tabSize, ch = state.facet(indentUnit)[0] 42 | if (ch == "\t") { 43 | while (cols >= ts) { 44 | result += "\t" 45 | cols -= ts 46 | } 47 | ch = " " 48 | } 49 | for (let i = 0; i < cols; i++) result += ch 50 | return result 51 | } 52 | 53 | /// Get the indentation, as a column number, at the given position. 54 | /// Will first consult any [indent services](#language.indentService) 55 | /// that are registered, and if none of those return an indentation, 56 | /// this will check the syntax tree for the [indent node 57 | /// prop](#language.indentNodeProp) and use that if found. Returns a 58 | /// number when an indentation could be determined, and null 59 | /// otherwise. 60 | export function getIndentation(context: IndentContext | EditorState, pos: number): number | null { 61 | if (context instanceof EditorState) context = new IndentContext(context) 62 | for (let service of context.state.facet(indentService)) { 63 | let result = service(context, pos) 64 | if (result !== undefined) return result 65 | } 66 | let tree = syntaxTree(context.state) 67 | return tree.length >= pos ? syntaxIndentation(context, tree, pos) : null 68 | } 69 | 70 | /// Create a change set that auto-indents all lines touched by the 71 | /// given document range. 72 | export function indentRange(state: EditorState, from: number, to: number) { 73 | let updated: {[lineStart: number]: number} = Object.create(null) 74 | let context = new IndentContext(state, {overrideIndentation: start => updated[start] ?? -1}) 75 | let changes: ChangeSpec[] = [] 76 | for (let pos = from; pos <= to;) { 77 | let line = state.doc.lineAt(pos) 78 | pos = line.to + 1 79 | let indent = getIndentation(context, line.from) 80 | if (indent == null) continue 81 | if (!/\S/.test(line.text)) indent = 0 82 | let cur = /^\s*/.exec(line.text)![0] 83 | let norm = indentString(state, indent) 84 | if (cur != norm) { 85 | updated[line.from] = indent 86 | changes.push({from: line.from, to: line.from + cur.length, insert: norm}) 87 | } 88 | } 89 | return state.changes(changes) 90 | } 91 | 92 | /// Indentation contexts are used when calling [indentation 93 | /// services](#language.indentService). They provide helper utilities 94 | /// useful in indentation logic, and can selectively override the 95 | /// indentation reported for some lines. 96 | export class IndentContext { 97 | /// The indent unit (number of columns per indentation level). 98 | unit: number 99 | 100 | /// Create an indent context. 101 | constructor( 102 | /// The editor state. 103 | readonly state: EditorState, 104 | /// @internal 105 | readonly options: { 106 | /// Override line indentations provided to the indentation 107 | /// helper function, which is useful when implementing region 108 | /// indentation, where indentation for later lines needs to refer 109 | /// to previous lines, which may have been reindented compared to 110 | /// the original start state. If given, this function should 111 | /// return -1 for lines (given by start position) that didn't 112 | /// change, and an updated indentation otherwise. 113 | overrideIndentation?: (pos: number) => number, 114 | /// Make it look, to the indent logic, like a line break was 115 | /// added at the given position (which is mostly just useful for 116 | /// implementing something like 117 | /// [`insertNewlineAndIndent`](#commands.insertNewlineAndIndent)). 118 | simulateBreak?: number, 119 | /// When `simulateBreak` is given, this can be used to make the 120 | /// simulated break behave like a double line break. 121 | simulateDoubleBreak?: boolean 122 | } = {} 123 | ) { 124 | this.unit = getIndentUnit(state) 125 | } 126 | 127 | /// Get a description of the line at the given position, taking 128 | /// [simulated line 129 | /// breaks](#language.IndentContext.constructor^options.simulateBreak) 130 | /// into account. If there is such a break at `pos`, the `bias` 131 | /// argument determines whether the part of the line line before or 132 | /// after the break is used. 133 | lineAt(pos: number, bias: -1 | 1 = 1): {text: string, from: number} { 134 | let line = this.state.doc.lineAt(pos) 135 | let {simulateBreak, simulateDoubleBreak} = this.options 136 | if (simulateBreak != null && simulateBreak >= line.from && simulateBreak <= line.to) { 137 | if (simulateDoubleBreak && simulateBreak == pos) 138 | return {text: "", from: pos} 139 | else if (bias < 0 ? simulateBreak < pos : simulateBreak <= pos) 140 | return {text: line.text.slice(simulateBreak - line.from), from: simulateBreak} 141 | else 142 | return {text: line.text.slice(0, simulateBreak - line.from), from: line.from} 143 | } 144 | return line 145 | } 146 | 147 | /// Get the text directly after `pos`, either the entire line 148 | /// or the next 100 characters, whichever is shorter. 149 | textAfterPos(pos: number, bias: -1 | 1 = 1) { 150 | if (this.options.simulateDoubleBreak && pos == this.options.simulateBreak) return "" 151 | let {text, from} = this.lineAt(pos, bias) 152 | return text.slice(pos - from, Math.min(text.length, pos + 100 - from)) 153 | } 154 | 155 | /// Find the column for the given position. 156 | column(pos: number, bias: -1 | 1 = 1) { 157 | let {text, from} = this.lineAt(pos, bias) 158 | let result = this.countColumn(text, pos - from) 159 | let override = this.options.overrideIndentation ? this.options.overrideIndentation(from) : -1 160 | if (override > -1) result += override - this.countColumn(text, text.search(/\S|$/)) 161 | return result 162 | } 163 | 164 | /// Find the column position (taking tabs into account) of the given 165 | /// position in the given string. 166 | countColumn(line: string, pos: number = line.length) { 167 | return countColumn(line, this.state.tabSize, pos) 168 | } 169 | 170 | /// Find the indentation column of the line at the given point. 171 | lineIndent(pos: number, bias: -1 | 1 = 1) { 172 | let {text, from} = this.lineAt(pos, bias) 173 | let override = this.options.overrideIndentation 174 | if (override) { 175 | let overriden = override(from) 176 | if (overriden > -1) return overriden 177 | } 178 | return this.countColumn(text, text.search(/\S|$/)) 179 | } 180 | 181 | /// Returns the [simulated line 182 | /// break](#language.IndentContext.constructor^options.simulateBreak) 183 | /// for this context, if any. 184 | get simulatedBreak(): number | null { 185 | return this.options.simulateBreak || null 186 | } 187 | } 188 | 189 | /// A syntax tree node prop used to associate indentation strategies 190 | /// with node types. Such a strategy is a function from an indentation 191 | /// context to a column number (see also 192 | /// [`indentString`](#language.indentString)) or null, where null 193 | /// indicates that no definitive indentation can be determined. 194 | export const indentNodeProp = new NodeProp<(context: TreeIndentContext) => number | null>() 195 | 196 | // Compute the indentation for a given position from the syntax tree. 197 | function syntaxIndentation(cx: IndentContext, ast: Tree, pos: number) { 198 | let stack = ast.resolveStack(pos) 199 | let inner = ast.resolveInner(pos, -1).resolve(pos, 0).enterUnfinishedNodesBefore(pos) 200 | if (inner != stack.node) { 201 | let add = [] 202 | for (let cur = inner; cur && !( 203 | cur.from < stack.node.from || cur.to > stack.node.to || 204 | cur.from == stack.node.from && cur.type == stack.node.type 205 | ); cur = cur.parent!) add.push(cur) 206 | for (let i = add.length - 1; i >= 0; i--) stack = {node: add[i], next: stack} 207 | } 208 | return indentFor(stack, cx, pos) 209 | } 210 | 211 | function indentFor(stack: NodeIterator | null, cx: IndentContext, pos: number): number | null { 212 | for (let cur: NodeIterator | null = stack; cur; cur = cur.next) { 213 | let strategy = indentStrategy(cur.node) 214 | if (strategy) return strategy(TreeIndentContext.create(cx, pos, cur)) 215 | } 216 | return 0 217 | } 218 | 219 | function ignoreClosed(cx: TreeIndentContext) { 220 | return cx.pos == cx.options.simulateBreak && cx.options.simulateDoubleBreak 221 | } 222 | 223 | function indentStrategy(tree: SyntaxNode): ((context: TreeIndentContext) => number | null) | null { 224 | let strategy = tree.type.prop(indentNodeProp) 225 | if (strategy) return strategy 226 | let first = tree.firstChild, close: readonly string[] | undefined 227 | if (first && (close = first.type.prop(NodeProp.closedBy))) { 228 | let last = tree.lastChild, closed = last && close.indexOf(last.name) > -1 229 | return cx => delimitedStrategy(cx, true, 1, undefined, closed && !ignoreClosed(cx) ? last!.from : undefined) 230 | } 231 | return tree.parent == null ? topIndent : null 232 | } 233 | 234 | function topIndent() { return 0 } 235 | 236 | /// Objects of this type provide context information and helper 237 | /// methods to indentation functions registered on syntax nodes. 238 | export class TreeIndentContext extends IndentContext { 239 | private constructor( 240 | private base: IndentContext, 241 | /// The position at which indentation is being computed. 242 | readonly pos: number, 243 | /// @internal 244 | readonly context: NodeIterator 245 | ) { 246 | super(base.state, base.options) 247 | } 248 | 249 | /// The syntax tree node to which the indentation strategy 250 | /// applies. 251 | get node(): SyntaxNode { return this.context.node } 252 | 253 | 254 | /// @internal 255 | static create(base: IndentContext, pos: number, context: NodeIterator) { 256 | return new TreeIndentContext(base, pos, context) 257 | } 258 | 259 | /// Get the text directly after `this.pos`, either the entire line 260 | /// or the next 100 characters, whichever is shorter. 261 | get textAfter() { 262 | return this.textAfterPos(this.pos) 263 | } 264 | 265 | /// Get the indentation at the reference line for `this.node`, which 266 | /// is the line on which it starts, unless there is a node that is 267 | /// _not_ a parent of this node covering the start of that line. If 268 | /// so, the line at the start of that node is tried, again skipping 269 | /// on if it is covered by another such node. 270 | get baseIndent() { 271 | return this.baseIndentFor(this.node) 272 | } 273 | 274 | /// Get the indentation for the reference line of the given node 275 | /// (see [`baseIndent`](#language.TreeIndentContext.baseIndent)). 276 | baseIndentFor(node: SyntaxNode) { 277 | let line = this.state.doc.lineAt(node.from) 278 | // Skip line starts that are covered by a sibling (or cousin, etc) 279 | for (;;) { 280 | let atBreak = node.resolve(line.from) 281 | while (atBreak.parent && atBreak.parent.from == atBreak.from) atBreak = atBreak.parent 282 | if (isParent(atBreak, node)) break 283 | line = this.state.doc.lineAt(atBreak.from) 284 | } 285 | return this.lineIndent(line.from) 286 | } 287 | 288 | /// Continue looking for indentations in the node's parent nodes, 289 | /// and return the result of that. 290 | continue() { 291 | return indentFor(this.context.next, this.base, this.pos) 292 | } 293 | } 294 | 295 | function isParent(parent: SyntaxNode, of: SyntaxNode) { 296 | for (let cur: SyntaxNode | null = of; cur; cur = cur.parent) if (parent == cur) return true 297 | return false 298 | } 299 | 300 | // Check whether a delimited node is aligned (meaning there are 301 | // non-skipped nodes on the same line as the opening delimiter). And 302 | // if so, return the opening token. 303 | function bracketedAligned(context: TreeIndentContext) { 304 | let tree = context.node 305 | let openToken = tree.childAfter(tree.from), last = tree.lastChild 306 | if (!openToken) return null 307 | let sim = context.options.simulateBreak 308 | let openLine = context.state.doc.lineAt(openToken.from) 309 | let lineEnd = sim == null || sim <= openLine.from ? openLine.to : Math.min(openLine.to, sim) 310 | for (let pos = openToken.to;;) { 311 | let next = tree.childAfter(pos) 312 | if (!next || next == last) return null 313 | if (!next.type.isSkipped) { 314 | if (next.from >= lineEnd) return null 315 | let space = /^ */.exec(openLine.text.slice(openToken.to - openLine.from))![0].length 316 | return {from: openToken.from, to: openToken.to + space} 317 | } 318 | pos = next.to 319 | } 320 | } 321 | 322 | /// An indentation strategy for delimited (usually bracketed) nodes. 323 | /// Will, by default, indent one unit more than the parent's base 324 | /// indent unless the line starts with a closing token. When `align` 325 | /// is true and there are non-skipped nodes on the node's opening 326 | /// line, the content of the node will be aligned with the end of the 327 | /// opening node, like this: 328 | /// 329 | /// foo(bar, 330 | /// baz) 331 | export function delimitedIndent({closing, align = true, units = 1}: {closing: string, align?: boolean, units?: number}) { 332 | return (context: TreeIndentContext) => delimitedStrategy(context, align, units, closing) 333 | } 334 | 335 | function delimitedStrategy(context: TreeIndentContext, align: boolean, units: number, closing?: string, closedAt?: number) { 336 | let after = context.textAfter, space = after.match(/^\s*/)![0].length 337 | let closed = closing && after.slice(space, space + closing.length) == closing || closedAt == context.pos + space 338 | let aligned = align ? bracketedAligned(context) : null 339 | if (aligned) return closed ? context.column(aligned.from) : context.column(aligned.to) 340 | return context.baseIndent + (closed ? 0 : context.unit * units) 341 | } 342 | 343 | /// An indentation strategy that aligns a node's content to its base 344 | /// indentation. 345 | export const flatIndent = (context: TreeIndentContext) => context.baseIndent 346 | 347 | /// Creates an indentation strategy that, by default, indents 348 | /// continued lines one unit more than the node's base indentation. 349 | /// You can provide `except` to prevent indentation of lines that 350 | /// match a pattern (for example `/^else\b/` in `if`/`else` 351 | /// constructs), and you can change the amount of units used with the 352 | /// `units` option. 353 | export function continuedIndent({except, units = 1}: {except?: RegExp, units?: number} = {}) { 354 | return (context: TreeIndentContext) => { 355 | let matchExcept = except && except.test(context.textAfter) 356 | return context.baseIndent + (matchExcept ? 0 : units * context.unit) 357 | } 358 | } 359 | 360 | const DontIndentBeyond = 200 361 | 362 | /// Enables reindentation on input. When a language defines an 363 | /// `indentOnInput` field in its [language 364 | /// data](#state.EditorState.languageDataAt), which must hold a regular 365 | /// expression, the line at the cursor will be reindented whenever new 366 | /// text is typed and the input from the start of the line up to the 367 | /// cursor matches that regexp. 368 | /// 369 | /// To avoid unneccesary reindents, it is recommended to start the 370 | /// regexp with `^` (usually followed by `\s*`), and end it with `$`. 371 | /// For example, `/^\s*\}$/` will reindent when a closing brace is 372 | /// added at the start of a line. 373 | export function indentOnInput(): Extension { 374 | return EditorState.transactionFilter.of(tr => { 375 | if (!tr.docChanged || !tr.isUserEvent("input.type") && !tr.isUserEvent("input.complete")) return tr 376 | let rules = tr.startState.languageDataAt("indentOnInput", tr.startState.selection.main.head) 377 | if (!rules.length) return tr 378 | let doc = tr.newDoc, {head} = tr.newSelection.main, line = doc.lineAt(head) 379 | if (head > line.from + DontIndentBeyond) return tr 380 | let lineStart = doc.sliceString(line.from, head) 381 | if (!rules.some(r => r.test(lineStart))) return tr 382 | let {state} = tr, last = -1, changes = [] 383 | for (let {head} of state.selection.ranges) { 384 | let line = state.doc.lineAt(head) 385 | if (line.from == last) continue 386 | last = line.from 387 | let indent = getIndentation(state, line.from) 388 | if (indent == null) continue 389 | let cur = /^\s*/.exec(line.text)![0] 390 | let norm = indentString(state, indent) 391 | if (cur != norm) 392 | changes.push({from: line.from, to: line.from + cur.length, insert: norm}) 393 | } 394 | return changes.length ? [tr, {changes, sequential: true}] : tr 395 | }) 396 | } 397 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | export {language, Language, LRLanguage, Sublanguage, sublanguageProp, defineLanguageFacet, 2 | syntaxTree, ensureSyntaxTree, languageDataProp, 3 | ParseContext, LanguageSupport, LanguageDescription, 4 | syntaxTreeAvailable, syntaxParserRunning, forceParsing, DocInput} from "./language" 5 | 6 | export {IndentContext, getIndentUnit, indentString, indentOnInput, indentService, getIndentation, indentRange, indentUnit, 7 | TreeIndentContext, indentNodeProp, delimitedIndent, continuedIndent, flatIndent} from "./indent" 8 | 9 | export {foldService, foldNodeProp, foldInside, foldable, foldCode, unfoldCode, toggleFold, foldAll, unfoldAll, 10 | foldKeymap, codeFolding, foldGutter, foldedRanges, foldEffect, unfoldEffect, foldState} from "./fold" 11 | 12 | export {HighlightStyle, syntaxHighlighting, highlightingFor, TagStyle, defaultHighlightStyle} from "./highlight" 13 | 14 | export {bracketMatching, Config, matchBrackets, MatchResult, bracketMatchingHandle} from "./matchbrackets" 15 | 16 | export {StreamLanguage, StreamParser} from "./stream-parser" 17 | 18 | export {StringStream} from "./stringstream" 19 | 20 | export {bidiIsolates} from "./isolate" 21 | -------------------------------------------------------------------------------- /src/isolate.ts: -------------------------------------------------------------------------------- 1 | import {EditorView, ViewUpdate, ViewPlugin, DecorationSet, Decoration, Direction} from "@codemirror/view" 2 | import {syntaxTree} from "./language" 3 | import {NodeProp, Tree} from "@lezer/common" 4 | import {RangeSetBuilder, Prec, Text, Extension, ChangeSet, Facet} from "@codemirror/state" 5 | 6 | function buildForLine(line: string) { 7 | return line.length <= 4096 && /[\u0590-\u05f4\u0600-\u06ff\u0700-\u08ac\ufb50-\ufdff]/.test(line) 8 | } 9 | 10 | function textHasRTL(text: Text) { 11 | for (let i = text.iter(); !i.next().done;) 12 | if (buildForLine(i.value)) return true 13 | return false 14 | } 15 | 16 | function changeAddsRTL(change: ChangeSet) { 17 | let added = false 18 | change.iterChanges((fA, tA, fB, tB, ins) => { 19 | if (!added && textHasRTL(ins)) added = true 20 | }) 21 | return added 22 | } 23 | 24 | const alwaysIsolate = Facet.define({combine: values => values.some(x => x)}) 25 | 26 | /// Make sure nodes 27 | /// [marked](https://lezer.codemirror.net/docs/ref/#common.NodeProp^isolate) 28 | /// as isolating for bidirectional text are rendered in a way that 29 | /// isolates them from the surrounding text. 30 | export function bidiIsolates(options: { 31 | /// By default, isolating elements are only added when the editor 32 | /// direction isn't uniformly left-to-right, or if it is, on lines 33 | /// that contain right-to-left character. When true, disable this 34 | /// optimization and add them everywhere. 35 | alwaysIsolate?: boolean 36 | } = {}): Extension { 37 | let extensions: Extension[] = [isolateMarks] 38 | if (options.alwaysIsolate) extensions.push(alwaysIsolate.of(true)) 39 | return extensions 40 | } 41 | 42 | const isolateMarks = ViewPlugin.fromClass(class { 43 | decorations: DecorationSet 44 | tree: Tree 45 | hasRTL: boolean 46 | always: boolean 47 | 48 | constructor(view: EditorView) { 49 | this.always = view.state.facet(alwaysIsolate) || 50 | view.textDirection != Direction.LTR || 51 | view.state.facet(EditorView.perLineTextDirection) 52 | this.hasRTL = !this.always && textHasRTL(view.state.doc) 53 | this.tree = syntaxTree(view.state) 54 | this.decorations = this.always || this.hasRTL ? buildDeco(view, this.tree, this.always) : Decoration.none 55 | } 56 | 57 | update(update: ViewUpdate) { 58 | let always = update.state.facet(alwaysIsolate) || 59 | update.view.textDirection != Direction.LTR || 60 | update.state.facet(EditorView.perLineTextDirection) 61 | if (!always && !this.hasRTL && changeAddsRTL(update.changes)) 62 | this.hasRTL = true 63 | 64 | if (!always && !this.hasRTL) return 65 | 66 | let tree = syntaxTree(update.state) 67 | if (always != this.always || tree != this.tree || update.docChanged || update.viewportChanged) { 68 | this.tree = tree 69 | this.always = always 70 | this.decorations = buildDeco(update.view, tree, always) 71 | } 72 | } 73 | }, { 74 | provide: plugin => { 75 | function access(view: EditorView) { 76 | return view.plugin(plugin)?.decorations ?? Decoration.none 77 | } 78 | return [EditorView.outerDecorations.of(access), 79 | Prec.lowest(EditorView.bidiIsolatedRanges.of(access))] 80 | } 81 | }) 82 | 83 | function buildDeco(view: EditorView, tree: Tree, always: boolean) { 84 | let deco = new RangeSetBuilder() 85 | let ranges = view.visibleRanges 86 | if (!always) ranges = clipRTLLines(ranges, view.state.doc) 87 | for (let {from, to} of ranges) { 88 | tree.iterate({ 89 | enter: node => { 90 | let iso = node.type.prop(NodeProp.isolate) 91 | if (iso) deco.add(node.from, node.to, marks[iso]) 92 | }, 93 | from, to 94 | }) 95 | } 96 | return deco.finish() 97 | } 98 | 99 | function clipRTLLines(ranges: readonly {from: number, to: number}[], doc: Text) { 100 | let cur = doc.iter(), pos = 0, result: {from: number, to: number}[] = [], last = null 101 | for (let {from, to} of ranges) { 102 | if (last && last.to > from) { 103 | from = last.to 104 | if (from >= to) continue 105 | } 106 | if (pos + cur.value.length < from) { 107 | cur.next(from - (pos + cur.value.length)) 108 | pos = from 109 | } 110 | for (;;) { 111 | let start = pos, end = pos + cur.value.length 112 | if (!cur.lineBreak && buildForLine(cur.value)) { 113 | if (last && last.to > start - 10) last.to = Math.min(to, end) 114 | else result.push(last = {from: start, to: Math.min(to, end)}) 115 | } 116 | if (end >= to) break 117 | pos = end 118 | cur.next() 119 | } 120 | } 121 | return result 122 | } 123 | 124 | const marks = { 125 | rtl: Decoration.mark({class: "cm-iso", inclusive: true, attributes: {dir: "rtl"}, bidiIsolate: Direction.RTL}), 126 | ltr: Decoration.mark({class: "cm-iso", inclusive: true, attributes: {dir: "ltr"}, bidiIsolate: Direction.LTR}), 127 | auto: Decoration.mark({class: "cm-iso", inclusive: true, attributes: {dir: "auto"}, bidiIsolate: null}) 128 | } 129 | -------------------------------------------------------------------------------- /src/language.ts: -------------------------------------------------------------------------------- 1 | import {Tree, SyntaxNode, ChangedRange, TreeFragment, NodeProp, NodeType, Input, 2 | PartialParse, Parser, IterMode} from "@lezer/common" 3 | import type {LRParser, ParserConfig} from "@lezer/lr" 4 | import {EditorState, StateField, Transaction, Extension, StateEffect, Facet, 5 | ChangeDesc, Text, TextIterator} from "@codemirror/state" 6 | import {ViewPlugin, ViewUpdate, EditorView, logException} from "@codemirror/view" 7 | 8 | /// Node prop stored in a parser's top syntax node to provide the 9 | /// facet that stores language-specific data for that language. 10 | export const languageDataProp = new NodeProp>() 11 | 12 | /// Helper function to define a facet (to be added to the top syntax 13 | /// node(s) for a language via 14 | /// [`languageDataProp`](#language.languageDataProp)), that will be 15 | /// used to associate language data with the language. You 16 | /// probably only need this when subclassing 17 | /// [`Language`](#language.Language). 18 | export function defineLanguageFacet(baseData?: {[name: string]: any}) { 19 | return Facet.define<{[name: string]: any}>({ 20 | combine: baseData ? values => values.concat(baseData!) : undefined 21 | }) 22 | } 23 | 24 | /// Some languages need to return different [language 25 | /// data](#state.EditorState.languageDataAt) for some parts of their 26 | /// tree. Sublanguages, registered by adding a [node 27 | /// prop](#language.sublanguageProp) to the language's top syntax 28 | /// node, provide a mechanism to do this. 29 | /// 30 | /// (Note that when using nested parsing, where nested syntax is 31 | /// parsed by a different parser and has its own top node type, you 32 | /// don't need a sublanguage.) 33 | export interface Sublanguage { 34 | /// Determines whether the data provided by this sublanguage should 35 | /// completely replace the regular data or be added to it (with 36 | /// higher-precedence). The default is `"extend"`. 37 | type?: "replace" | "extend", 38 | /// A predicate that returns whether the node at the queried 39 | /// position is part of the sublanguage. 40 | test: (node: SyntaxNode, state: EditorState) => boolean, 41 | /// The language data facet that holds the sublanguage's data. 42 | /// You'll want to use 43 | /// [`defineLanguageFacet`](#language.defineLanguageFacet) to create 44 | /// this. 45 | facet: Facet<{[name: string]: any}> 46 | } 47 | 48 | /// Syntax node prop used to register sublanguages. Should be added to 49 | /// the top level node type for the language. 50 | export const sublanguageProp = new NodeProp() 51 | 52 | /// A language object manages parsing and per-language 53 | /// [metadata](#state.EditorState.languageDataAt). Parse data is 54 | /// managed as a [Lezer](https://lezer.codemirror.net) tree. The class 55 | /// can be used directly, via the [`LRLanguage`](#language.LRLanguage) 56 | /// subclass for [Lezer](https://lezer.codemirror.net/) LR parsers, or 57 | /// via the [`StreamLanguage`](#language.StreamLanguage) subclass 58 | /// for stream parsers. 59 | export class Language { 60 | /// The extension value to install this as the document language. 61 | readonly extension: Extension 62 | 63 | /// The parser object. Can be useful when using this as a [nested 64 | /// parser](https://lezer.codemirror.net/docs/ref#common.Parser). 65 | parser: Parser 66 | 67 | /// Construct a language object. If you need to invoke this 68 | /// directly, first define a data facet with 69 | /// [`defineLanguageFacet`](#language.defineLanguageFacet), and then 70 | /// configure your parser to [attach](#language.languageDataProp) it 71 | /// to the language's outer syntax node. 72 | constructor( 73 | /// The [language data](#state.EditorState.languageDataAt) facet 74 | /// used for this language. 75 | readonly data: Facet<{[name: string]: any}>, 76 | parser: Parser, 77 | extraExtensions: Extension[] = [], 78 | /// A language name. 79 | readonly name: string = "" 80 | ) { 81 | // Kludge to define EditorState.tree as a debugging helper, 82 | // without the EditorState package actually knowing about 83 | // languages and lezer trees. 84 | if (!EditorState.prototype.hasOwnProperty("tree")) 85 | Object.defineProperty(EditorState.prototype, "tree", {get() { return syntaxTree(this) }}) 86 | 87 | this.parser = parser 88 | this.extension = [ 89 | language.of(this), 90 | EditorState.languageData.of((state, pos, side) => { 91 | let top = topNodeAt(state, pos, side), data = top.type.prop(languageDataProp) 92 | if (!data) return [] 93 | let base = state.facet(data), sub = top.type.prop(sublanguageProp) 94 | if (sub) { 95 | let innerNode = top.resolve(pos - top.from, side) 96 | for (let sublang of sub) if (sublang.test(innerNode, state)) { 97 | let data = state.facet(sublang.facet) 98 | return sublang.type == "replace" ? data : data.concat(base) 99 | } 100 | } 101 | return base 102 | }) 103 | ].concat(extraExtensions) 104 | } 105 | 106 | /// Query whether this language is active at the given position. 107 | isActiveAt(state: EditorState, pos: number, side: -1 | 0 | 1 = -1) { 108 | return topNodeAt(state, pos, side).type.prop(languageDataProp) == this.data 109 | } 110 | 111 | /// Find the document regions that were parsed using this language. 112 | /// The returned regions will _include_ any nested languages rooted 113 | /// in this language, when those exist. 114 | findRegions(state: EditorState) { 115 | let lang = state.facet(language) 116 | if (lang?.data == this.data) return [{from: 0, to: state.doc.length}] 117 | if (!lang || !lang.allowsNesting) return [] 118 | let result: {from: number, to: number}[] = [] 119 | let explore = (tree: Tree, from: number) => { 120 | if (tree.prop(languageDataProp) == this.data) { 121 | result.push({from, to: from + tree.length}) 122 | return 123 | } 124 | let mount = tree.prop(NodeProp.mounted) 125 | if (mount) { 126 | if (mount.tree.prop(languageDataProp) == this.data) { 127 | if (mount.overlay) for (let r of mount.overlay) result.push({from: r.from + from, to: r.to + from}) 128 | else result.push({from: from, to: from + tree.length}) 129 | return 130 | } else if (mount.overlay) { 131 | let size = result.length 132 | explore(mount.tree, mount.overlay[0].from + from) 133 | if (result.length > size) return 134 | } 135 | } 136 | for (let i = 0; i < tree.children.length; i++) { 137 | let ch = tree.children[i] 138 | if (ch instanceof Tree) explore(ch, tree.positions[i] + from) 139 | } 140 | } 141 | explore(syntaxTree(state), 0) 142 | return result 143 | } 144 | 145 | /// Indicates whether this language allows nested languages. The 146 | /// default implementation returns true. 147 | get allowsNesting() { return true } 148 | 149 | /// @internal 150 | static state: StateField 151 | 152 | /// @internal 153 | static setState = StateEffect.define() 154 | } 155 | 156 | function topNodeAt(state: EditorState, pos: number, side: -1 | 0 | 1) { 157 | let topLang = state.facet(language), tree = syntaxTree(state).topNode 158 | if (!topLang || topLang.allowsNesting) { 159 | for (let node: SyntaxNode | null = tree; node; node = node.enter(pos, side, IterMode.ExcludeBuffers)) 160 | if (node.type.isTop) tree = node 161 | } 162 | return tree 163 | } 164 | 165 | /// A subclass of [`Language`](#language.Language) for use with Lezer 166 | /// [LR parsers](https://lezer.codemirror.net/docs/ref#lr.LRParser) 167 | /// parsers. 168 | export class LRLanguage extends Language { 169 | private constructor(data: Facet<{[name: string]: any}>, readonly parser: LRParser, name?: string) { 170 | super(data, parser, [], name) 171 | } 172 | 173 | /// Define a language from a parser. 174 | static define(spec: { 175 | /// The [name](#Language.name) of the language. 176 | name?: string, 177 | /// The parser to use. Should already have added editor-relevant 178 | /// node props (and optionally things like dialect and top rule) 179 | /// configured. 180 | parser: LRParser, 181 | /// [Language data](#state.EditorState.languageDataAt) 182 | /// to register for this language. 183 | languageData?: {[name: string]: any} 184 | }) { 185 | let data = defineLanguageFacet(spec.languageData) 186 | return new LRLanguage(data, spec.parser.configure({ 187 | props: [languageDataProp.add(type => type.isTop ? data : undefined)] 188 | }), spec.name) 189 | } 190 | 191 | /// Create a new instance of this language with a reconfigured 192 | /// version of its parser and optionally a new name. 193 | configure(options: ParserConfig, name?: string): LRLanguage { 194 | return new LRLanguage(this.data, this.parser.configure(options), name || this.name) 195 | } 196 | 197 | get allowsNesting() { return this.parser.hasWrappers() } 198 | } 199 | 200 | /// Get the syntax tree for a state, which is the current (possibly 201 | /// incomplete) parse tree of the active 202 | /// [language](#language.Language), or the empty tree if there is no 203 | /// language available. 204 | export function syntaxTree(state: EditorState): Tree { 205 | let field = state.field(Language.state, false) 206 | return field ? field.tree : Tree.empty 207 | } 208 | 209 | /// Try to get a parse tree that spans at least up to `upto`. The 210 | /// method will do at most `timeout` milliseconds of work to parse 211 | /// up to that point if the tree isn't already available. 212 | export function ensureSyntaxTree(state: EditorState, upto: number, timeout = 50): Tree | null { 213 | let parse = state.field(Language.state, false)?.context 214 | if (!parse) return null 215 | let oldVieport = parse.viewport 216 | parse.updateViewport({from: 0, to: upto}) 217 | let result = parse.isDone(upto) || parse.work(timeout, upto) ? parse.tree : null 218 | parse.updateViewport(oldVieport) 219 | return result 220 | } 221 | 222 | /// Queries whether there is a full syntax tree available up to the 223 | /// given document position. If there isn't, the background parse 224 | /// process _might_ still be working and update the tree further, but 225 | /// there is no guarantee of that—the parser will [stop 226 | /// working](#language.syntaxParserRunning) when it has spent a 227 | /// certain amount of time or has moved beyond the visible viewport. 228 | /// Always returns false if no language has been enabled. 229 | export function syntaxTreeAvailable(state: EditorState, upto = state.doc.length) { 230 | return state.field(Language.state, false)?.context.isDone(upto) || false 231 | } 232 | 233 | /// Move parsing forward, and update the editor state afterwards to 234 | /// reflect the new tree. Will work for at most `timeout` 235 | /// milliseconds. Returns true if the parser managed get to the given 236 | /// position in that time. 237 | export function forceParsing(view: EditorView, upto = view.viewport.to, timeout = 100): boolean { 238 | let success = ensureSyntaxTree(view.state, upto, timeout) 239 | if (success != syntaxTree(view.state)) view.dispatch({}) 240 | return !!success 241 | } 242 | 243 | /// Tells you whether the language parser is planning to do more 244 | /// parsing work (in a `requestIdleCallback` pseudo-thread) or has 245 | /// stopped running, either because it parsed the entire document, 246 | /// because it spent too much time and was cut off, or because there 247 | /// is no language parser enabled. 248 | export function syntaxParserRunning(view: EditorView) { 249 | return view.plugin(parseWorker)?.isWorking() || false 250 | } 251 | 252 | /// Lezer-style 253 | /// [`Input`](https://lezer.codemirror.net/docs/ref#common.Input) 254 | /// object for a [`Text`](#state.Text) object. 255 | export class DocInput implements Input { 256 | private cursor: TextIterator 257 | private cursorPos = 0 258 | private string = "" 259 | 260 | /// Create an input object for the given document. 261 | constructor(readonly doc: Text) { 262 | this.cursor = doc.iter() 263 | } 264 | 265 | get length() { return this.doc.length } 266 | 267 | private syncTo(pos: number) { 268 | this.string = this.cursor.next(pos - this.cursorPos).value 269 | this.cursorPos = pos + this.string.length 270 | return this.cursorPos - this.string.length 271 | } 272 | 273 | chunk(pos: number) { 274 | this.syncTo(pos) 275 | return this.string 276 | } 277 | 278 | get lineChunks() { return true } 279 | 280 | read(from: number, to: number) { 281 | let stringStart = this.cursorPos - this.string.length 282 | if (from < stringStart || to >= this.cursorPos) 283 | return this.doc.sliceString(from, to) 284 | else 285 | return this.string.slice(from - stringStart, to - stringStart) 286 | } 287 | } 288 | 289 | const enum Work { 290 | // Milliseconds of work time to perform immediately for a state doc change 291 | Apply = 20, 292 | // Minimum amount of work time to perform in an idle callback 293 | MinSlice = 25, 294 | // Amount of work time to perform in pseudo-thread when idle callbacks aren't supported 295 | Slice = 100, 296 | // Minimum pause between pseudo-thread slices 297 | MinPause = 100, 298 | // Maximum pause (timeout) for the pseudo-thread 299 | MaxPause = 500, 300 | // Parse time budgets are assigned per chunk—the parser can run for 301 | // ChunkBudget milliseconds at most during ChunkTime milliseconds. 302 | // After that, no further background parsing is scheduled until the 303 | // next chunk in which the editor is active. 304 | ChunkBudget = 3000, 305 | ChunkTime = 30000, 306 | // For every change the editor receives while focused, it gets a 307 | // small bonus to its parsing budget (as a way to allow active 308 | // editors to continue doing work). 309 | ChangeBonus = 50, 310 | // Don't eagerly parse this far beyond the end of the viewport 311 | MaxParseAhead = 1e5, 312 | // When initializing the state field (before viewport info is 313 | // available), pretend the viewport goes from 0 to here. 314 | InitViewport = 3000, 315 | } 316 | 317 | let currentContext: ParseContext | null = null 318 | 319 | /// A parse context provided to parsers working on the editor content. 320 | export class ParseContext { 321 | private parse: PartialParse | null = null 322 | /// @internal 323 | tempSkipped: {from: number, to: number}[] = [] 324 | 325 | private constructor( 326 | private parser: Parser, 327 | /// The current editor state. 328 | readonly state: EditorState, 329 | /// Tree fragments that can be reused by incremental re-parses. 330 | public fragments: readonly TreeFragment[] = [], 331 | /// @internal 332 | public tree: Tree, 333 | /// @internal 334 | public treeLen: number, 335 | /// The current editor viewport (or some overapproximation 336 | /// thereof). Intended to be used for opportunistically avoiding 337 | /// work (in which case 338 | /// [`skipUntilInView`](#language.ParseContext.skipUntilInView) 339 | /// should be called to make sure the parser is restarted when the 340 | /// skipped region becomes visible). 341 | public viewport: {from: number, to: number}, 342 | /// @internal 343 | public skipped: {from: number, to: number}[], 344 | /// This is where skipping parsers can register a promise that, 345 | /// when resolved, will schedule a new parse. It is cleared when 346 | /// the parse worker picks up the promise. @internal 347 | public scheduleOn: Promise | null 348 | ) {} 349 | 350 | /// @internal 351 | static create(parser: Parser, state: EditorState, viewport: {from: number, to: number}) { 352 | return new ParseContext(parser, state, [], Tree.empty, 0, viewport, [], null) 353 | } 354 | 355 | private startParse() { 356 | return this.parser.startParse(new DocInput(this.state.doc), this.fragments) 357 | } 358 | 359 | /// @internal 360 | work(until: number | (() => boolean), upto?: number) { 361 | if (upto != null && upto >= this.state.doc.length) upto = undefined 362 | if (this.tree != Tree.empty && this.isDone(upto ?? this.state.doc.length)) { 363 | this.takeTree() 364 | return true 365 | } 366 | return this.withContext(() => { 367 | if (typeof until == "number") { 368 | let endTime = Date.now() + until 369 | until = () => Date.now() > endTime 370 | } 371 | if (!this.parse) this.parse = this.startParse() 372 | if (upto != null && (this.parse.stoppedAt == null || this.parse.stoppedAt > upto) && 373 | upto < this.state.doc.length) this.parse.stopAt(upto) 374 | for (;;) { 375 | let done = this.parse.advance() 376 | if (done) { 377 | this.fragments = this.withoutTempSkipped(TreeFragment.addTree(done, this.fragments, this.parse.stoppedAt != null)) 378 | this.treeLen = this.parse.stoppedAt ?? this.state.doc.length 379 | this.tree = done 380 | this.parse = null 381 | if (this.treeLen < (upto ?? this.state.doc.length)) 382 | this.parse = this.startParse() 383 | else 384 | return true 385 | } 386 | if (until()) return false 387 | } 388 | }) 389 | } 390 | 391 | /// @internal 392 | takeTree() { 393 | let pos, tree: Tree | undefined | null 394 | if (this.parse && (pos = this.parse.parsedPos) >= this.treeLen) { 395 | if (this.parse.stoppedAt == null || this.parse.stoppedAt > pos) this.parse.stopAt(pos) 396 | this.withContext(() => { while (!(tree = this.parse!.advance())) {} }) 397 | this.treeLen = pos 398 | this.tree = tree! 399 | this.fragments = this.withoutTempSkipped(TreeFragment.addTree(this.tree, this.fragments, true)) 400 | this.parse = null 401 | } 402 | } 403 | 404 | private withContext(f: () => T): T { 405 | let prev = currentContext 406 | currentContext = this 407 | try { return f() } 408 | finally { currentContext = prev } 409 | } 410 | 411 | private withoutTempSkipped(fragments: readonly TreeFragment[]) { 412 | for (let r; r = this.tempSkipped.pop();) 413 | fragments = cutFragments(fragments, r.from, r.to) 414 | return fragments 415 | } 416 | 417 | /// @internal 418 | changes(changes: ChangeDesc, newState: EditorState) { 419 | let {fragments, tree, treeLen, viewport, skipped} = this 420 | this.takeTree() 421 | if (!changes.empty) { 422 | let ranges: ChangedRange[] = [] 423 | changes.iterChangedRanges((fromA, toA, fromB, toB) => ranges.push({fromA, toA, fromB, toB})) 424 | fragments = TreeFragment.applyChanges(fragments, ranges) 425 | tree = Tree.empty 426 | treeLen = 0 427 | viewport = {from: changes.mapPos(viewport.from, -1), to: changes.mapPos(viewport.to, 1)} 428 | if (this.skipped.length) { 429 | skipped = [] 430 | for (let r of this.skipped) { 431 | let from = changes.mapPos(r.from, 1), to = changes.mapPos(r.to, -1) 432 | if (from < to) skipped.push({from, to}) 433 | } 434 | } 435 | } 436 | return new ParseContext(this.parser, newState, fragments, tree, treeLen, viewport, skipped, this.scheduleOn) 437 | } 438 | 439 | /// @internal 440 | updateViewport(viewport: {from: number, to: number}) { 441 | if (this.viewport.from == viewport.from && this.viewport.to == viewport.to) return false 442 | this.viewport = viewport 443 | let startLen = this.skipped.length 444 | for (let i = 0; i < this.skipped.length; i++) { 445 | let {from, to} = this.skipped[i] 446 | if (from < viewport.to && to > viewport.from) { 447 | this.fragments = cutFragments(this.fragments, from, to) 448 | this.skipped.splice(i--, 1) 449 | } 450 | } 451 | if (this.skipped.length >= startLen) return false 452 | this.reset() 453 | return true 454 | } 455 | 456 | /// @internal 457 | reset() { 458 | if (this.parse) { 459 | this.takeTree() 460 | this.parse = null 461 | } 462 | } 463 | 464 | /// Notify the parse scheduler that the given region was skipped 465 | /// because it wasn't in view, and the parse should be restarted 466 | /// when it comes into view. 467 | skipUntilInView(from: number, to: number) { 468 | this.skipped.push({from, to}) 469 | } 470 | 471 | /// Returns a parser intended to be used as placeholder when 472 | /// asynchronously loading a nested parser. It'll skip its input and 473 | /// mark it as not-really-parsed, so that the next update will parse 474 | /// it again. 475 | /// 476 | /// When `until` is given, a reparse will be scheduled when that 477 | /// promise resolves. 478 | static getSkippingParser(until?: Promise): Parser { 479 | return new class extends Parser { 480 | createParse( 481 | input: Input, 482 | fragments: readonly TreeFragment[], 483 | ranges: readonly {from: number, to: number}[] 484 | ): PartialParse { 485 | let from = ranges[0].from, to = ranges[ranges.length - 1].to 486 | let parser = { 487 | parsedPos: from, 488 | advance() { 489 | let cx = currentContext 490 | if (cx) { 491 | for (let r of ranges) cx.tempSkipped.push(r) 492 | if (until) cx.scheduleOn = cx.scheduleOn ? Promise.all([cx.scheduleOn, until]) : until 493 | } 494 | this.parsedPos = to 495 | return new Tree(NodeType.none, [], [], to - from) 496 | }, 497 | stoppedAt: null, 498 | stopAt() {} 499 | } 500 | return parser 501 | } 502 | } 503 | } 504 | 505 | /// @internal 506 | isDone(upto: number) { 507 | upto = Math.min(upto, this.state.doc.length) 508 | let frags = this.fragments 509 | return this.treeLen >= upto && frags.length && frags[0].from == 0 && frags[0].to >= upto 510 | } 511 | 512 | /// Get the context for the current parse, or `null` if no editor 513 | /// parse is in progress. 514 | static get() { return currentContext } 515 | } 516 | 517 | function cutFragments(fragments: readonly TreeFragment[], from: number, to: number) { 518 | return TreeFragment.applyChanges(fragments, [{fromA: from, toA: to, fromB: from, toB: to}]) 519 | } 520 | 521 | class LanguageState { 522 | // The current tree. Immutable, because directly accessible from 523 | // the editor state. 524 | readonly tree: Tree 525 | 526 | constructor( 527 | // A mutable parse state that is used to preserve work done during 528 | // the lifetime of a state when moving to the next state. 529 | readonly context: ParseContext 530 | ) { 531 | this.tree = context.tree 532 | } 533 | 534 | apply(tr: Transaction) { 535 | if (!tr.docChanged && this.tree == this.context.tree) return this 536 | let newCx = this.context.changes(tr.changes, tr.state) 537 | // If the previous parse wasn't done, go forward only up to its 538 | // end position or the end of the viewport, to avoid slowing down 539 | // state updates with parse work beyond the viewport. 540 | let upto = this.context.treeLen == tr.startState.doc.length ? undefined 541 | : Math.max(tr.changes.mapPos(this.context.treeLen), newCx.viewport.to) 542 | if (!newCx.work(Work.Apply, upto)) newCx.takeTree() 543 | return new LanguageState(newCx) 544 | } 545 | 546 | static init(state: EditorState) { 547 | let vpTo = Math.min(Work.InitViewport, state.doc.length) 548 | let parseState = ParseContext.create(state.facet(language)!.parser, state, {from: 0, to: vpTo}) 549 | if (!parseState.work(Work.Apply, vpTo)) parseState.takeTree() 550 | return new LanguageState(parseState) 551 | } 552 | } 553 | 554 | Language.state = StateField.define({ 555 | create: LanguageState.init, 556 | update(value, tr) { 557 | for (let e of tr.effects) if (e.is(Language.setState)) return e.value 558 | if (tr.startState.facet(language) != tr.state.facet(language)) return LanguageState.init(tr.state) 559 | return value.apply(tr) 560 | } 561 | }) 562 | 563 | let requestIdle = (callback: (deadline?: IdleDeadline) => void) => { 564 | let timeout = setTimeout(() => callback(), Work.MaxPause) 565 | return () => clearTimeout(timeout) 566 | } 567 | 568 | if (typeof requestIdleCallback != "undefined") requestIdle = (callback: (deadline?: IdleDeadline) => void) => { 569 | let idle = -1, timeout = setTimeout(() => { 570 | idle = requestIdleCallback(callback, {timeout: Work.MaxPause - Work.MinPause}) 571 | }, Work.MinPause) 572 | return () => idle < 0 ? clearTimeout(timeout) : cancelIdleCallback(idle) 573 | } 574 | 575 | const isInputPending = typeof navigator != "undefined" && (navigator as any).scheduling?.isInputPending 576 | ? () => (navigator as any).scheduling.isInputPending() : null 577 | 578 | const parseWorker = ViewPlugin.fromClass(class ParseWorker { 579 | working: (() => void) | null = null 580 | workScheduled = 0 581 | // End of the current time chunk 582 | chunkEnd = -1 583 | // Milliseconds of budget left for this chunk 584 | chunkBudget = -1 585 | 586 | constructor(readonly view: EditorView) { 587 | this.work = this.work.bind(this) 588 | this.scheduleWork() 589 | } 590 | 591 | update(update: ViewUpdate) { 592 | let cx = this.view.state.field(Language.state).context 593 | if (cx.updateViewport(update.view.viewport) || this.view.viewport.to > cx.treeLen) 594 | this.scheduleWork() 595 | if (update.docChanged || update.selectionSet) { 596 | if (this.view.hasFocus) this.chunkBudget += Work.ChangeBonus 597 | this.scheduleWork() 598 | } 599 | this.checkAsyncSchedule(cx) 600 | } 601 | 602 | scheduleWork() { 603 | if (this.working) return 604 | let {state} = this.view, field = state.field(Language.state) 605 | if (field.tree != field.context.tree || !field.context.isDone(state.doc.length)) 606 | this.working = requestIdle(this.work) 607 | } 608 | 609 | work(deadline?: IdleDeadline) { 610 | this.working = null 611 | 612 | let now = Date.now() 613 | if (this.chunkEnd < now && (this.chunkEnd < 0 || this.view.hasFocus)) { // Start a new chunk 614 | this.chunkEnd = now + Work.ChunkTime 615 | this.chunkBudget = Work.ChunkBudget 616 | } 617 | if (this.chunkBudget <= 0) return // No more budget 618 | 619 | let {state, viewport: {to: vpTo}} = this.view, field = state.field(Language.state) 620 | if (field.tree == field.context.tree && field.context.isDone(vpTo + Work.MaxParseAhead)) return 621 | let endTime = Date.now() + Math.min( 622 | this.chunkBudget, Work.Slice, deadline && !isInputPending ? Math.max(Work.MinSlice, deadline.timeRemaining() - 5) : 1e9) 623 | let viewportFirst = field.context.treeLen < vpTo && state.doc.length > vpTo + 1000 624 | let done = field.context.work(() => { 625 | return isInputPending && isInputPending() || Date.now() > endTime 626 | } , vpTo + (viewportFirst ? 0 : Work.MaxParseAhead)) 627 | this.chunkBudget -= Date.now() - now 628 | if (done || this.chunkBudget <= 0) { 629 | field.context.takeTree() 630 | this.view.dispatch({effects: Language.setState.of(new LanguageState(field.context))}) 631 | } 632 | if (this.chunkBudget > 0 && !(done && !viewportFirst)) this.scheduleWork() 633 | this.checkAsyncSchedule(field.context) 634 | } 635 | 636 | checkAsyncSchedule(cx: ParseContext) { 637 | if (cx.scheduleOn) { 638 | this.workScheduled++ 639 | cx.scheduleOn 640 | .then(() => this.scheduleWork()) 641 | .catch(err => logException(this.view.state, err)) 642 | .then(() => this.workScheduled--) 643 | cx.scheduleOn = null 644 | } 645 | } 646 | 647 | destroy() { 648 | if (this.working) this.working() 649 | } 650 | 651 | isWorking() { 652 | return !!(this.working || this.workScheduled > 0) 653 | } 654 | }, { 655 | eventHandlers: {focus() { this.scheduleWork() }} 656 | }) 657 | 658 | /// The facet used to associate a language with an editor state. Used 659 | /// by `Language` object's `extension` property (so you don't need to 660 | /// manually wrap your languages in this). Can be used to access the 661 | /// current language on a state. 662 | export const language = Facet.define({ 663 | combine(languages) { return languages.length ? languages[0] : null }, 664 | enables: language => [ 665 | Language.state, 666 | parseWorker, 667 | EditorView.contentAttributes.compute([language], state => { 668 | let lang = state.facet(language) 669 | return lang && lang.name ? {"data-language": lang.name} : {} as {} 670 | }) 671 | ] 672 | }) 673 | 674 | /// This class bundles a [language](#language.Language) with an 675 | /// optional set of supporting extensions. Language packages are 676 | /// encouraged to export a function that optionally takes a 677 | /// configuration object and returns a `LanguageSupport` instance, as 678 | /// the main way for client code to use the package. 679 | export class LanguageSupport { 680 | /// An extension including both the language and its support 681 | /// extensions. (Allowing the object to be used as an extension 682 | /// value itself.) 683 | extension: Extension 684 | 685 | /// Create a language support object. 686 | constructor( 687 | /// The language object. 688 | readonly language: Language, 689 | /// An optional set of supporting extensions. When nesting a 690 | /// language in another language, the outer language is encouraged 691 | /// to include the supporting extensions for its inner languages 692 | /// in its own set of support extensions. 693 | readonly support: Extension = [] 694 | ) { 695 | this.extension = [language, support] 696 | } 697 | } 698 | 699 | /// Language descriptions are used to store metadata about languages 700 | /// and to dynamically load them. Their main role is finding the 701 | /// appropriate language for a filename or dynamically loading nested 702 | /// parsers. 703 | export class LanguageDescription { 704 | private loading: Promise | null = null 705 | 706 | private constructor( 707 | /// The name of this language. 708 | readonly name: string, 709 | /// Alternative names for the mode (lowercased, includes `this.name`). 710 | readonly alias: readonly string[], 711 | /// File extensions associated with this language. 712 | readonly extensions: readonly string[], 713 | /// Optional filename pattern that should be associated with this 714 | /// language. 715 | readonly filename: RegExp | undefined, 716 | private loadFunc: () => Promise, 717 | /// If the language has been loaded, this will hold its value. 718 | public support: LanguageSupport | undefined = undefined 719 | ) {} 720 | 721 | /// Start loading the the language. Will return a promise that 722 | /// resolves to a [`LanguageSupport`](#language.LanguageSupport) 723 | /// object when the language successfully loads. 724 | load(): Promise { 725 | return this.loading || (this.loading = this.loadFunc().then( 726 | support => this.support = support, 727 | err => { this.loading = null; throw err } 728 | )) 729 | } 730 | 731 | /// Create a language description. 732 | static of(spec: { 733 | /// The language's name. 734 | name: string, 735 | /// An optional array of alternative names. 736 | alias?: readonly string[], 737 | /// An optional array of filename extensions associated with this 738 | /// language. 739 | extensions?: readonly string[], 740 | /// An optional filename pattern associated with this language. 741 | filename?: RegExp, 742 | /// A function that will asynchronously load the language. 743 | load?: () => Promise, 744 | /// Alternatively to `load`, you can provide an already loaded 745 | /// support object. Either this or `load` should be provided. 746 | support?: LanguageSupport 747 | }) { 748 | let {load, support} = spec 749 | if (!load) { 750 | if (!support) throw new RangeError("Must pass either 'load' or 'support' to LanguageDescription.of") 751 | load = () => Promise.resolve(support!) 752 | } 753 | return new LanguageDescription(spec.name, (spec.alias || []).concat(spec.name).map(s => s.toLowerCase()), 754 | spec.extensions || [], spec.filename, load, support) 755 | } 756 | 757 | /// Look for a language in the given array of descriptions that 758 | /// matches the filename. Will first match 759 | /// [`filename`](#language.LanguageDescription.filename) patterns, 760 | /// and then [extensions](#language.LanguageDescription.extensions), 761 | /// and return the first language that matches. 762 | static matchFilename(descs: readonly LanguageDescription[], filename: string) { 763 | for (let d of descs) if (d.filename && d.filename.test(filename)) return d 764 | let ext = /\.([^.]+)$/.exec(filename) 765 | if (ext) for (let d of descs) if (d.extensions.indexOf(ext[1]) > -1) return d 766 | return null 767 | } 768 | 769 | /// Look for a language whose name or alias matches the the given 770 | /// name (case-insensitively). If `fuzzy` is true, and no direct 771 | /// matchs is found, this'll also search for a language whose name 772 | /// or alias occurs in the string (for names shorter than three 773 | /// characters, only when surrounded by non-word characters). 774 | static matchLanguageName(descs: readonly LanguageDescription[], name: string, fuzzy = true) { 775 | name = name.toLowerCase() 776 | for (let d of descs) if (d.alias.some(a => a == name)) return d 777 | if (fuzzy) for (let d of descs) for (let a of d.alias) { 778 | let found = name.indexOf(a) 779 | if (found > -1 && (a.length > 2 || !/\w/.test(name[found - 1]) && !/\w/.test(name[found + a.length]))) 780 | return d 781 | } 782 | return null 783 | } 784 | } 785 | -------------------------------------------------------------------------------- /src/matchbrackets.ts: -------------------------------------------------------------------------------- 1 | import {combineConfig, EditorState, Facet, StateField, Extension, Range} from "@codemirror/state" 2 | import {syntaxTree} from "./language" 3 | import {EditorView, Decoration, DecorationSet} from "@codemirror/view" 4 | import {Tree, SyntaxNode, SyntaxNodeRef, NodeType, NodeProp} from "@lezer/common" 5 | 6 | export interface Config { 7 | /// Whether the bracket matching should look at the character after 8 | /// the cursor when matching (if the one before isn't a bracket). 9 | /// Defaults to true. 10 | afterCursor?: boolean 11 | /// The bracket characters to match, as a string of pairs. Defaults 12 | /// to `"()[]{}"`. Note that these are only used as fallback when 13 | /// there is no [matching 14 | /// information](https://lezer.codemirror.net/docs/ref/#common.NodeProp^closedBy) 15 | /// in the syntax tree. 16 | brackets?: string 17 | /// The maximum distance to scan for matching brackets. This is only 18 | /// relevant for brackets not encoded in the syntax tree. Defaults 19 | /// to 10 000. 20 | maxScanDistance?: number 21 | /// Can be used to configure the way in which brackets are 22 | /// decorated. The default behavior is to add the 23 | /// `cm-matchingBracket` class for matching pairs, and 24 | /// `cm-nonmatchingBracket` for mismatched pairs or single brackets. 25 | renderMatch?: (match: MatchResult, state: EditorState) => readonly Range[] 26 | } 27 | 28 | const baseTheme = EditorView.baseTheme({ 29 | "&.cm-focused .cm-matchingBracket": {backgroundColor: "#328c8252"}, 30 | "&.cm-focused .cm-nonmatchingBracket": {backgroundColor: "#bb555544"} 31 | }) 32 | 33 | const DefaultScanDist = 10000, DefaultBrackets = "()[]{}" 34 | 35 | const bracketMatchingConfig = Facet.define>({ 36 | combine(configs) { 37 | return combineConfig(configs, { 38 | afterCursor: true, 39 | brackets: DefaultBrackets, 40 | maxScanDistance: DefaultScanDist, 41 | renderMatch: defaultRenderMatch 42 | }) 43 | } 44 | }) 45 | 46 | const matchingMark = Decoration.mark({class: "cm-matchingBracket"}), 47 | nonmatchingMark = Decoration.mark({class: "cm-nonmatchingBracket"}) 48 | 49 | function defaultRenderMatch(match: MatchResult) { 50 | let decorations = [] 51 | let mark = match.matched ? matchingMark : nonmatchingMark 52 | decorations.push(mark.range(match.start.from, match.start.to)) 53 | if (match.end) decorations.push(mark.range(match.end.from, match.end.to)) 54 | return decorations 55 | } 56 | 57 | const bracketMatchingState = StateField.define({ 58 | create() { return Decoration.none }, 59 | update(deco, tr) { 60 | if (!tr.docChanged && !tr.selection) return deco 61 | let decorations: Range[] = [] 62 | let config = tr.state.facet(bracketMatchingConfig) 63 | for (let range of tr.state.selection.ranges) { 64 | if (!range.empty) continue 65 | let match = matchBrackets(tr.state, range.head, -1, config) 66 | || (range.head > 0 && matchBrackets(tr.state, range.head - 1, 1, config)) 67 | || (config.afterCursor && 68 | (matchBrackets(tr.state, range.head, 1, config) || 69 | (range.head < tr.state.doc.length && matchBrackets(tr.state, range.head + 1, -1, config)))) 70 | if (match) 71 | decorations = decorations.concat(config.renderMatch(match, tr.state)) 72 | } 73 | return Decoration.set(decorations, true) 74 | }, 75 | provide: f => EditorView.decorations.from(f) 76 | }) 77 | 78 | const bracketMatchingUnique = [ 79 | bracketMatchingState, 80 | baseTheme 81 | ] 82 | 83 | /// Create an extension that enables bracket matching. Whenever the 84 | /// cursor is next to a bracket, that bracket and the one it matches 85 | /// are highlighted. Or, when no matching bracket is found, another 86 | /// highlighting style is used to indicate this. 87 | export function bracketMatching(config: Config = {}): Extension { 88 | return [bracketMatchingConfig.of(config), bracketMatchingUnique] 89 | } 90 | 91 | /// When larger syntax nodes, such as HTML tags, are marked as 92 | /// opening/closing, it can be a bit messy to treat the whole node as 93 | /// a matchable bracket. This node prop allows you to define, for such 94 | /// a node, a ‘handle’—the part of the node that is highlighted, and 95 | /// that the cursor must be on to activate highlighting in the first 96 | /// place. 97 | export const bracketMatchingHandle = new NodeProp<(node: SyntaxNode) => SyntaxNode | null>() 98 | 99 | function matchingNodes(node: NodeType, dir: -1 | 1, brackets: string): null | readonly string[] { 100 | let byProp = node.prop(dir < 0 ? NodeProp.openedBy : NodeProp.closedBy) 101 | if (byProp) return byProp 102 | if (node.name.length == 1) { 103 | let index = brackets.indexOf(node.name) 104 | if (index > -1 && index % 2 == (dir < 0 ? 1 : 0)) 105 | return [brackets[index + dir]] 106 | } 107 | return null 108 | } 109 | 110 | 111 | /// The result returned from `matchBrackets`. 112 | export interface MatchResult { 113 | /// The extent of the bracket token found. 114 | start: {from: number, to: number}, 115 | /// The extent of the matched token, if any was found. 116 | end?: {from: number, to: number}, 117 | /// Whether the tokens match. This can be false even when `end` has 118 | /// a value, if that token doesn't match the opening token. 119 | matched: boolean 120 | } 121 | 122 | function findHandle(node: SyntaxNodeRef) { 123 | let hasHandle = node.type.prop(bracketMatchingHandle) 124 | return hasHandle ? hasHandle(node.node) : node 125 | } 126 | 127 | /// Find the matching bracket for the token at `pos`, scanning 128 | /// direction `dir`. Only the `brackets` and `maxScanDistance` 129 | /// properties are used from `config`, if given. Returns null if no 130 | /// bracket was found at `pos`, or a match result otherwise. 131 | export function matchBrackets(state: EditorState, pos: number, dir: -1 | 1, config: Config = {}): MatchResult | null { 132 | let maxScanDistance = config.maxScanDistance || DefaultScanDist, brackets = config.brackets || DefaultBrackets 133 | let tree = syntaxTree(state), node = tree.resolveInner(pos, dir) 134 | for (let cur: SyntaxNode | null = node; cur; cur = cur.parent) { 135 | let matches = matchingNodes(cur.type, dir, brackets) 136 | if (matches && cur.from < cur.to) { 137 | let handle = findHandle(cur) 138 | if (handle && (dir > 0 ? pos >= handle.from && pos < handle.to : pos > handle.from && pos <= handle.to)) 139 | return matchMarkedBrackets(state, pos, dir, cur, handle, matches, brackets) 140 | } 141 | } 142 | return matchPlainBrackets(state, pos, dir, tree, node.type, maxScanDistance, brackets) 143 | } 144 | 145 | function matchMarkedBrackets(_state: EditorState, _pos: number, dir: -1 | 1, token: SyntaxNode, 146 | handle: SyntaxNodeRef, matching: readonly string[], brackets: string) { 147 | let parent = token.parent, firstToken = {from: handle.from, to: handle.to} 148 | let depth = 0, cursor = parent?.cursor() 149 | if (cursor && (dir < 0 ? cursor.childBefore(token.from) : cursor.childAfter(token.to))) do { 150 | if (dir < 0 ? cursor.to <= token.from : cursor.from >= token.to) { 151 | if (depth == 0 && matching.indexOf(cursor.type.name) > -1 && cursor.from < cursor.to) { 152 | let endHandle = findHandle(cursor) 153 | return {start: firstToken, end: endHandle ? {from: endHandle.from, to: endHandle.to} : undefined, matched: true} 154 | } else if (matchingNodes(cursor.type, dir, brackets)) { 155 | depth++ 156 | } else if (matchingNodes(cursor.type, -dir as -1 | 1, brackets)) { 157 | if (depth == 0) { 158 | let endHandle = findHandle(cursor) 159 | return { 160 | start: firstToken, 161 | end: endHandle && endHandle.from < endHandle.to ? {from: endHandle.from, to: endHandle.to} : undefined, 162 | matched: false 163 | } 164 | } 165 | depth-- 166 | } 167 | } 168 | } while (dir < 0 ? cursor.prevSibling() : cursor.nextSibling()) 169 | return {start: firstToken, matched: false} 170 | } 171 | 172 | function matchPlainBrackets(state: EditorState, pos: number, dir: number, tree: Tree, 173 | tokenType: NodeType, maxScanDistance: number, brackets: string) { 174 | let startCh = dir < 0 ? state.sliceDoc(pos - 1, pos) : state.sliceDoc(pos, pos + 1) 175 | let bracket = brackets.indexOf(startCh) 176 | if (bracket < 0 || (bracket % 2 == 0) != (dir > 0)) return null 177 | 178 | let startToken = {from: dir < 0 ? pos - 1 : pos, to: dir > 0 ? pos + 1 : pos} 179 | let iter = state.doc.iterRange(pos, dir > 0 ? state.doc.length : 0), depth = 0 180 | for (let distance = 0; !(iter.next()).done && distance <= maxScanDistance;) { 181 | let text = iter.value 182 | if (dir < 0) distance += text.length 183 | let basePos = pos + distance * dir 184 | for (let pos = dir > 0 ? 0 : text.length - 1, end = dir > 0 ? text.length : -1; pos != end; pos += dir) { 185 | let found = brackets.indexOf(text[pos]) 186 | if (found < 0 || tree.resolveInner(basePos + pos, 1).type != tokenType) continue 187 | if ((found % 2 == 0) == (dir > 0)) { 188 | depth++ 189 | } else if (depth == 1) { // Closing 190 | return {start: startToken, end: {from: basePos + pos, to: basePos + pos + 1}, matched: (found >> 1) == (bracket >> 1)} 191 | } else { 192 | depth-- 193 | } 194 | } 195 | if (dir > 0) distance += text.length 196 | } 197 | return iter.done ? {start: startToken, matched: false} : null 198 | } 199 | -------------------------------------------------------------------------------- /src/stream-parser.ts: -------------------------------------------------------------------------------- 1 | import {Tree, Input, TreeFragment, NodeType, NodeSet, PartialParse, Parser, NodeProp} from "@lezer/common" 2 | import {Tag, tags as highlightTags, styleTags} from "@lezer/highlight" 3 | import {EditorState, Facet} from "@codemirror/state" 4 | import {Language, defineLanguageFacet, languageDataProp, ParseContext} from "./language" 5 | import {TreeIndentContext, IndentContext, indentNodeProp, getIndentUnit} from "./indent" 6 | import {StringStream} from "./stringstream" 7 | 8 | export {StringStream} 9 | 10 | /// A stream parser parses or tokenizes content from start to end, 11 | /// emitting tokens as it goes over it. It keeps a mutable (but 12 | /// copyable) object with state, in which it can store information 13 | /// about the current context. 14 | export interface StreamParser { 15 | /// A name for this language. 16 | name?: string 17 | /// Produce a start state for the parser. 18 | startState?(indentUnit: number): State 19 | /// Read one token, advancing the stream past it, and returning a 20 | /// string indicating the token's style tag—either the name of one 21 | /// of the tags in 22 | /// [`tags`](https://lezer.codemirror.net/docs/ref#highlight.tags) 23 | /// or [`tokenTable`](#language.StreamParser.tokenTable), or such a 24 | /// name suffixed by one or more tag 25 | /// [modifier](https://lezer.codemirror.net/docs/ref#highlight.Tag^defineModifier) 26 | /// names, separated by periods. For example `"keyword"` or 27 | /// "`variableName.constant"`, or a space-separated set of such 28 | /// token types. 29 | /// 30 | /// It is okay to return a zero-length token, but only if that 31 | /// updates the state so that the next call will return a non-empty 32 | /// token again. 33 | token(stream: StringStream, state: State): string | null 34 | /// This notifies the parser of a blank line in the input. It can 35 | /// update its state here if it needs to. 36 | blankLine?(state: State, indentUnit: number): void 37 | /// Copy a given state. By default, a shallow object copy is done 38 | /// which also copies arrays held at the top level of the object. 39 | copyState?(state: State): State 40 | /// Compute automatic indentation for the line that starts with the 41 | /// given state and text. 42 | indent?(state: State, textAfter: string, context: IndentContext): number | null 43 | /// Default [language data](#state.EditorState.languageDataAt) to 44 | /// attach to this language. 45 | languageData?: {[name: string]: any} 46 | /// Extra tokens to use in this parser. When the tokenizer returns a 47 | /// token name that exists as a property in this object, the 48 | /// corresponding tags will be assigned to the token. 49 | tokenTable?: {[name: string]: Tag | readonly Tag[]} 50 | /// By default, adjacent tokens of the same type are merged in the 51 | /// output tree. Set this to false to disable that. 52 | mergeTokens?: boolean 53 | } 54 | 55 | function fullParser(spec: StreamParser): Required> { 56 | return { 57 | name: spec.name || "", 58 | token: spec.token, 59 | blankLine: spec.blankLine || (() => {}), 60 | startState: spec.startState || (() => (true as any)), 61 | copyState: spec.copyState || defaultCopyState, 62 | indent: spec.indent || (() => null), 63 | languageData: spec.languageData || {}, 64 | tokenTable: spec.tokenTable || noTokens, 65 | mergeTokens: spec.mergeTokens !== false 66 | } 67 | } 68 | 69 | function defaultCopyState(state: State) { 70 | if (typeof state != "object") return state 71 | let newState = {} as State 72 | for (let prop in state) { 73 | let val = state[prop] 74 | newState[prop] = (val instanceof Array ? val.slice() : val) as any 75 | } 76 | return newState 77 | } 78 | 79 | const IndentedFrom = new WeakMap() 80 | 81 | /// A [language](#language.Language) class based on a CodeMirror 82 | /// 5-style [streaming parser](#language.StreamParser). 83 | export class StreamLanguage extends Language { 84 | /// @internal 85 | streamParser: Required> 86 | /// @internal 87 | stateAfter: NodeProp 88 | /// @internal 89 | tokenTable: TokenTable 90 | /// @internal 91 | topNode: NodeType 92 | 93 | private constructor(parser: StreamParser) { 94 | let data = defineLanguageFacet(parser.languageData) 95 | let p = fullParser(parser), self: StreamLanguage 96 | let impl = new class extends Parser { 97 | createParse(input: Input, fragments: readonly TreeFragment[], ranges: readonly {from: number, to: number}[]) { 98 | return new Parse(self, input, fragments, ranges) 99 | } 100 | } 101 | super(data, impl, [], parser.name) 102 | this.topNode = docID(data, this) 103 | self = this 104 | this.streamParser = p 105 | this.stateAfter = new NodeProp({perNode: true}) 106 | this.tokenTable = parser.tokenTable ? new TokenTable(p.tokenTable) : defaultTokenTable 107 | } 108 | 109 | /// Define a stream language. 110 | static define(spec: StreamParser) { return new StreamLanguage(spec) } 111 | 112 | /// @internal 113 | getIndent(cx: TreeIndentContext) { 114 | let from = undefined 115 | let {overrideIndentation} = cx.options 116 | if (overrideIndentation) { 117 | from = IndentedFrom.get(cx.state) 118 | if (from != null && from < cx.pos - 1e4) from = undefined 119 | } 120 | let start = findState(this, cx.node.tree!, cx.node.from, cx.node.from, from ?? cx.pos), statePos, state 121 | if (start) { state = start.state; statePos = start.pos + 1 } 122 | else { state = this.streamParser.startState(cx.unit) ; statePos = cx.node.from } 123 | if (cx.pos - statePos > C.MaxIndentScanDist) return null 124 | while (statePos < cx.pos) { 125 | let line = cx.state.doc.lineAt(statePos), end = Math.min(cx.pos, line.to) 126 | if (line.length) { 127 | let indentation = overrideIndentation ? overrideIndentation(line.from) : -1 128 | let stream = new StringStream(line.text, cx.state.tabSize, cx.unit, indentation < 0 ? undefined : indentation) 129 | while (stream.pos < end - line.from) 130 | readToken(this.streamParser.token, stream, state) 131 | } else { 132 | this.streamParser.blankLine(state, cx.unit) 133 | } 134 | if (end == cx.pos) break 135 | statePos = line.to + 1 136 | } 137 | let line = cx.lineAt(cx.pos) 138 | if (overrideIndentation && from == null) IndentedFrom.set(cx.state, line.from) 139 | return this.streamParser.indent(state, /^\s*(.*)/.exec(line.text)![1], cx) 140 | } 141 | 142 | get allowsNesting() { return false } 143 | } 144 | 145 | function findState( 146 | lang: StreamLanguage, tree: Tree, off: number, startPos: number, before: number 147 | ): {state: State, pos: number} | null { 148 | let state = off >= startPos && off + tree.length <= before && tree.prop(lang.stateAfter) 149 | if (state) return {state: lang.streamParser.copyState(state), pos: off + tree.length} 150 | for (let i = tree.children.length - 1; i >= 0; i--) { 151 | let child = tree.children[i], pos = off + tree.positions[i] 152 | let found = child instanceof Tree && pos < before && findState(lang, child, pos, startPos, before) 153 | if (found) return found 154 | } 155 | return null 156 | } 157 | 158 | function cutTree(lang: StreamLanguage, tree: Tree, from: number, to: number, inside: boolean): Tree | null { 159 | if (inside && from <= 0 && to >= tree.length) return tree 160 | if (!inside && from == 0 && tree.type == lang.topNode) inside = true 161 | for (let i = tree.children.length - 1; i >= 0; i--) { 162 | let pos = tree.positions[i], child = tree.children[i], inner 163 | if (pos < to && child instanceof Tree) { 164 | if (!(inner = cutTree(lang, child, from - pos, to - pos, inside))) break 165 | return !inside ? inner 166 | : new Tree(tree.type, tree.children.slice(0, i).concat(inner), tree.positions.slice(0, i + 1), pos + inner.length) 167 | } 168 | } 169 | return null 170 | } 171 | 172 | function findStartInFragments(lang: StreamLanguage, fragments: readonly TreeFragment[], 173 | startPos: number, endPos: number, editorState?: EditorState) { 174 | for (let f of fragments) { 175 | let from = f.from + (f.openStart ? 25 : 0), to = f.to - (f.openEnd ? 25 : 0) 176 | let found = from <= startPos && to > startPos && findState(lang, f.tree, 0 - f.offset, startPos, to), tree 177 | if (found && found.pos <= endPos && (tree = cutTree(lang, f.tree, startPos + f.offset, found.pos + f.offset, false))) 178 | return {state: found.state, tree} 179 | } 180 | return {state: lang.streamParser.startState(editorState ? getIndentUnit(editorState) : 4), tree: Tree.empty} 181 | } 182 | 183 | const enum C { 184 | ChunkSize = 2048, 185 | MaxDistanceBeforeViewport = 1e5, 186 | MaxIndentScanDist = 1e4, 187 | MaxLineLength = 1e4 188 | } 189 | 190 | class Parse implements PartialParse { 191 | state: State 192 | parsedPos: number 193 | stoppedAt: number | null = null 194 | chunks: Tree[] = [] 195 | chunkPos: number[] = [] 196 | chunkStart: number 197 | chunk: number[] = [] 198 | chunkReused: undefined | Tree[] = undefined 199 | rangeIndex = 0 200 | to: number 201 | 202 | constructor(readonly lang: StreamLanguage, 203 | readonly input: Input, 204 | readonly fragments: readonly TreeFragment[], 205 | readonly ranges: readonly {from: number, to: number}[]) { 206 | this.to = ranges[ranges.length - 1].to 207 | let context = ParseContext.get(), from = ranges[0].from 208 | let {state, tree} = findStartInFragments(lang, fragments, from, this.to, context?.state) 209 | this.state = state 210 | this.parsedPos = this.chunkStart = from + tree.length 211 | for (let i = 0; i < tree.children.length; i++) { 212 | this.chunks.push(tree.children[i] as Tree) 213 | this.chunkPos.push(tree.positions[i]) 214 | } 215 | if (context && this.parsedPos < context.viewport.from - C.MaxDistanceBeforeViewport && 216 | ranges.some(r => r.from <= context!.viewport.from && r.to >= context!.viewport.from)) { 217 | this.state = this.lang.streamParser.startState(getIndentUnit(context.state)) 218 | context.skipUntilInView(this.parsedPos, context.viewport.from) 219 | this.parsedPos = context.viewport.from 220 | } 221 | this.moveRangeIndex() 222 | } 223 | 224 | advance() { 225 | let context = ParseContext.get() 226 | let parseEnd = this.stoppedAt == null ? this.to : Math.min(this.to, this.stoppedAt) 227 | let end = Math.min(parseEnd, this.chunkStart + C.ChunkSize) 228 | if (context) end = Math.min(end, context.viewport.to) 229 | while (this.parsedPos < end) this.parseLine(context) 230 | if (this.chunkStart < this.parsedPos) this.finishChunk() 231 | if (this.parsedPos >= parseEnd) return this.finish() 232 | if (context && this.parsedPos >= context.viewport.to) { 233 | context.skipUntilInView(this.parsedPos, parseEnd) 234 | return this.finish() 235 | } 236 | return null 237 | } 238 | 239 | stopAt(pos: number) { 240 | this.stoppedAt = pos 241 | } 242 | 243 | lineAfter(pos: number) { 244 | let chunk = this.input.chunk(pos) 245 | if (!this.input.lineChunks) { 246 | let eol = chunk.indexOf("\n") 247 | if (eol > -1) chunk = chunk.slice(0, eol) 248 | } else if (chunk == "\n") { 249 | chunk = "" 250 | } 251 | return pos + chunk.length <= this.to ? chunk : chunk.slice(0, this.to - pos) 252 | } 253 | 254 | nextLine() { 255 | let from = this.parsedPos, line = this.lineAfter(from), end = from + line.length 256 | for (let index = this.rangeIndex;;) { 257 | let rangeEnd = this.ranges[index].to 258 | if (rangeEnd >= end) break 259 | line = line.slice(0, rangeEnd - (end - line.length)) 260 | index++ 261 | if (index == this.ranges.length) break 262 | let rangeStart = this.ranges[index].from 263 | let after = this.lineAfter(rangeStart) 264 | line += after 265 | end = rangeStart + after.length 266 | } 267 | return {line, end} 268 | } 269 | 270 | skipGapsTo(pos: number, offset: number, side: -1 | 1) { 271 | for (;;) { 272 | let end = this.ranges[this.rangeIndex].to, offPos = pos + offset 273 | if (side > 0 ? end > offPos : end >= offPos) break 274 | let start = this.ranges[++this.rangeIndex].from 275 | offset += start - end 276 | } 277 | return offset 278 | } 279 | 280 | moveRangeIndex() { 281 | while (this.ranges[this.rangeIndex].to < this.parsedPos) this.rangeIndex++ 282 | } 283 | 284 | emitToken(id: number, from: number, to: number, offset: number) { 285 | let size = 4 286 | if (this.ranges.length > 1) { 287 | offset = this.skipGapsTo(from, offset, 1) 288 | from += offset 289 | let len0 = this.chunk.length 290 | offset = this.skipGapsTo(to, offset, -1) 291 | to += offset 292 | size += this.chunk.length - len0 293 | } 294 | let last = this.chunk.length - 4 295 | if (this.lang.streamParser.mergeTokens && size == 4 && last >= 0 && 296 | this.chunk[last] == id && this.chunk[last + 2] == from) 297 | this.chunk[last + 2] = to 298 | else 299 | this.chunk.push(id, from, to, size) 300 | return offset 301 | } 302 | 303 | parseLine(context: ParseContext | null) { 304 | let {line, end} = this.nextLine(), offset = 0, {streamParser} = this.lang 305 | let stream = new StringStream(line, context ? context.state.tabSize : 4, context ? getIndentUnit(context.state) : 2) 306 | if (stream.eol()) { 307 | streamParser.blankLine(this.state, stream.indentUnit) 308 | } else { 309 | while (!stream.eol()) { 310 | let token = readToken(streamParser.token, stream, this.state) 311 | if (token) 312 | offset = this.emitToken(this.lang.tokenTable.resolve(token), this.parsedPos + stream.start, 313 | this.parsedPos + stream.pos, offset) 314 | if (stream.start > C.MaxLineLength) break 315 | } 316 | } 317 | this.parsedPos = end 318 | this.moveRangeIndex() 319 | if (this.parsedPos < this.to) this.parsedPos++ 320 | } 321 | 322 | finishChunk() { 323 | let tree = Tree.build({ 324 | buffer: this.chunk, 325 | start: this.chunkStart, 326 | length: this.parsedPos - this.chunkStart, 327 | nodeSet, 328 | topID: 0, 329 | maxBufferLength: C.ChunkSize, 330 | reused: this.chunkReused 331 | }) 332 | tree = new Tree(tree.type, tree.children, tree.positions, tree.length, 333 | [[this.lang.stateAfter, this.lang.streamParser.copyState(this.state)]]) 334 | this.chunks.push(tree) 335 | this.chunkPos.push(this.chunkStart - this.ranges[0].from) 336 | this.chunk = [] 337 | this.chunkReused = undefined 338 | this.chunkStart = this.parsedPos 339 | } 340 | 341 | finish() { 342 | return new Tree(this.lang.topNode, this.chunks, this.chunkPos, this.parsedPos - this.ranges[0].from).balance() 343 | } 344 | } 345 | 346 | function readToken(token: (stream: StringStream, state: State) => string | null, 347 | stream: StringStream, state: State) { 348 | stream.start = stream.pos 349 | for (let i = 0; i < 10; i++) { 350 | let result = token(stream, state) 351 | if (stream.pos > stream.start) return result 352 | } 353 | throw new Error("Stream parser failed to advance stream.") 354 | } 355 | 356 | const noTokens: {[name: string]: Tag} = Object.create(null) 357 | 358 | const typeArray: NodeType[] = [NodeType.none] 359 | const nodeSet = new NodeSet(typeArray) 360 | const warned: string[] = [] 361 | 362 | // Cache of node types by name and tags 363 | const byTag: {[key: string]: NodeType} = Object.create(null) 364 | 365 | const defaultTable: {[name: string]: number} = Object.create(null) 366 | for (let [legacyName, name] of [ 367 | ["variable", "variableName"], 368 | ["variable-2", "variableName.special"], 369 | ["string-2", "string.special"], 370 | ["def", "variableName.definition"], 371 | ["tag", "tagName"], 372 | ["attribute", "attributeName"], 373 | ["type", "typeName"], 374 | ["builtin", "variableName.standard"], 375 | ["qualifier", "modifier"], 376 | ["error", "invalid"], 377 | ["header", "heading"], 378 | ["property", "propertyName"] 379 | ]) defaultTable[legacyName] = createTokenType(noTokens, name) 380 | 381 | class TokenTable { 382 | table: {[name: string]: number} = Object.assign(Object.create(null), defaultTable) 383 | 384 | constructor(readonly extra: {[name: string]: Tag | readonly Tag[]}) {} 385 | 386 | resolve(tag: string) { 387 | return !tag ? 0 : this.table[tag] || (this.table[tag] = createTokenType(this.extra, tag)) 388 | } 389 | } 390 | 391 | const defaultTokenTable = new TokenTable(noTokens) 392 | 393 | function warnForPart(part: string, msg: string) { 394 | if (warned.indexOf(part) > -1) return 395 | warned.push(part) 396 | console.warn(msg) 397 | } 398 | 399 | function createTokenType(extra: {[name: string]: Tag | readonly Tag[]}, tagStr: string) { 400 | let tags = [] 401 | for (let name of tagStr.split(" ")) { 402 | let found: readonly Tag[] = [] 403 | for (let part of name.split(".")) { 404 | let value = (extra[part] || (highlightTags as any)[part]) as Tag | readonly Tag[] | ((t: Tag) => Tag) | undefined 405 | if (!value) { 406 | warnForPart(part, `Unknown highlighting tag ${part}`) 407 | } else if (typeof value == "function") { 408 | if (!found.length) warnForPart(part, `Modifier ${part} used at start of tag`) 409 | else found = found.map(value) as Tag[] 410 | } else { 411 | if (found.length) warnForPart(part, `Tag ${part} used as modifier`) 412 | else found = Array.isArray(value) ? value : [value] 413 | } 414 | } 415 | for (let tag of found) tags.push(tag) 416 | } 417 | if (!tags.length) return 0 418 | 419 | let name = tagStr.replace(/ /g, "_"), key = name + " " + tags.map(t => (t as any).id) 420 | let known = byTag[key] 421 | if (known) return known.id 422 | let type = byTag[key] = NodeType.define({ 423 | id: typeArray.length, 424 | name, 425 | props: [styleTags({[name]: tags})] 426 | }) 427 | typeArray.push(type) 428 | return type.id 429 | } 430 | 431 | function docID(data: Facet<{[name: string]: any}>, lang: StreamLanguage) { 432 | let type = NodeType.define({id: typeArray.length, name: "Document", props: [ 433 | languageDataProp.add(() => data), 434 | indentNodeProp.add(() => cx => lang.getIndent(cx)) 435 | ], top: true}) 436 | typeArray.push(type) 437 | return type 438 | } 439 | -------------------------------------------------------------------------------- /src/stringstream.ts: -------------------------------------------------------------------------------- 1 | // Counts the column offset in a string, taking tabs into account. 2 | // Used mostly to find indentation. 3 | function countCol(string: string, end: number | null, tabSize: number, startIndex = 0, startValue = 0): number { 4 | if (end == null) { 5 | end = string.search(/[^\s\u00a0]/) 6 | if (end == -1) end = string.length 7 | } 8 | let n = startValue 9 | for (let i = startIndex; i < end; i++) { 10 | if (string.charCodeAt(i) == 9) n += tabSize - (n % tabSize) 11 | else n++ 12 | } 13 | return n 14 | } 15 | 16 | /// Encapsulates a single line of input. Given to stream syntax code, 17 | /// which uses it to tokenize the content. 18 | export class StringStream { 19 | /// The current position on the line. 20 | pos: number = 0 21 | /// The start position of the current token. 22 | start: number = 0 23 | private lastColumnPos: number = 0 24 | private lastColumnValue: number = 0 25 | 26 | /// Create a stream. 27 | constructor( 28 | /// The line. 29 | public string: string, 30 | private tabSize: number, 31 | /// The current indent unit size. 32 | public indentUnit: number, 33 | private overrideIndent?: number 34 | ) {} 35 | 36 | /// True if we are at the end of the line. 37 | eol(): boolean {return this.pos >= this.string.length} 38 | 39 | /// True if we are at the start of the line. 40 | sol(): boolean {return this.pos == 0} 41 | 42 | /// Get the next code unit after the current position, or undefined 43 | /// if we're at the end of the line. 44 | peek() {return this.string.charAt(this.pos) || undefined} 45 | 46 | /// Read the next code unit and advance `this.pos`. 47 | next(): string | void { 48 | if (this.pos < this.string.length) 49 | return this.string.charAt(this.pos++) 50 | } 51 | 52 | /// Match the next character against the given string, regular 53 | /// expression, or predicate. Consume and return it if it matches. 54 | eat(match: string | RegExp | ((ch: string) => boolean)): string | void { 55 | let ch = this.string.charAt(this.pos) 56 | let ok 57 | if (typeof match == "string") ok = ch == match 58 | else ok = ch && (match instanceof RegExp ? match.test(ch) : match(ch)) 59 | if (ok) {++this.pos; return ch} 60 | } 61 | 62 | /// Continue matching characters that match the given string, 63 | /// regular expression, or predicate function. Return true if any 64 | /// characters were consumed. 65 | eatWhile(match: string | RegExp | ((ch: string) => boolean)): boolean { 66 | let start = this.pos 67 | while (this.eat(match)){} 68 | return this.pos > start 69 | } 70 | 71 | /// Consume whitespace ahead of `this.pos`. Return true if any was 72 | /// found. 73 | eatSpace() { 74 | let start = this.pos 75 | while (/[\s\u00a0]/.test(this.string.charAt(this.pos))) ++this.pos 76 | return this.pos > start 77 | } 78 | 79 | /// Move to the end of the line. 80 | skipToEnd() {this.pos = this.string.length} 81 | 82 | /// Move to directly before the given character, if found on the 83 | /// current line. 84 | skipTo(ch: string): boolean | void { 85 | let found = this.string.indexOf(ch, this.pos) 86 | if (found > -1) {this.pos = found; return true} 87 | } 88 | 89 | /// Move back `n` characters. 90 | backUp(n: number) {this.pos -= n} 91 | 92 | /// Get the column position at `this.pos`. 93 | column() { 94 | if (this.lastColumnPos < this.start) { 95 | this.lastColumnValue = countCol(this.string, this.start, this.tabSize, this.lastColumnPos, this.lastColumnValue) 96 | this.lastColumnPos = this.start 97 | } 98 | return this.lastColumnValue 99 | } 100 | 101 | /// Get the indentation column of the current line. 102 | indentation() { 103 | return this.overrideIndent ?? countCol(this.string, null, this.tabSize) 104 | } 105 | 106 | /// Match the input against the given string or regular expression 107 | /// (which should start with a `^`). Return true or the regexp match 108 | /// if it matches. 109 | /// 110 | /// Unless `consume` is set to `false`, this will move `this.pos` 111 | /// past the matched text. 112 | /// 113 | /// When matching a string `caseInsensitive` can be set to true to 114 | /// make the match case-insensitive. 115 | match(pattern: string | RegExp, consume?: boolean, caseInsensitive?: boolean): boolean | RegExpMatchArray | null { 116 | if (typeof pattern == "string") { 117 | let cased = (str: string) => caseInsensitive ? str.toLowerCase() : str 118 | let substr = this.string.substr(this.pos, pattern.length) 119 | if (cased(substr) == cased(pattern)) { 120 | if (consume !== false) this.pos += pattern.length 121 | return true 122 | } else return null 123 | } else { 124 | let match = this.string.slice(this.pos).match(pattern) 125 | if (match && match.index! > 0) return null 126 | if (match && consume !== false) this.pos += match[0].length 127 | return match 128 | } 129 | } 130 | 131 | /// Get the current token. 132 | current(){return this.string.slice(this.start, this.pos)} 133 | } 134 | -------------------------------------------------------------------------------- /test/test-fold.ts: -------------------------------------------------------------------------------- 1 | import ist from "ist" 2 | import {foldEffect, unfoldEffect, foldState} from "@codemirror/language" 3 | import {EditorState} from "@codemirror/state" 4 | import {DecorationSet} from "@codemirror/view" 5 | 6 | let doc = "1\n2\n3\n4\n5\n6\n7\n8\n" 7 | 8 | function ranges(set: DecorationSet) { 9 | let result: string[] = [] 10 | set.between(0, 1e8, (f, t) => {result.push(`${f}-${t}`)}) 11 | return result.join(" ") 12 | } 13 | 14 | describe("Folding", () => { 15 | it("stores fold state", () => { 16 | let state = EditorState.create({doc, extensions: foldState}).update({ 17 | effects: [foldEffect.of({from: 0, to: 3}), foldEffect.of({from: 4, to: 7})] 18 | }).state 19 | ist(ranges(state.field(foldState)), "0-3 4-7") 20 | state = state.update({ 21 | effects: unfoldEffect.of({from: 4, to: 7}) 22 | }).state 23 | ist(ranges(state.field(foldState)), "0-3") 24 | }) 25 | 26 | it("can store fold state as JSON", () => { 27 | let state = EditorState.create({doc, extensions: foldState}).update({ 28 | effects: [foldEffect.of({from: 4, to: 7}), foldEffect.of({from: 8, to: 11})] 29 | }).state 30 | let fields = {fold: foldState} 31 | state = EditorState.fromJSON(state.toJSON(fields), {}, fields) 32 | ist(ranges(state.field(foldState)), "4-7 8-11") 33 | }) 34 | }) 35 | -------------------------------------------------------------------------------- /test/test-stream-parser.ts: -------------------------------------------------------------------------------- 1 | import ist from "ist" 2 | import {EditorState} from "@codemirror/state" 3 | import {Tag} from "@lezer/highlight" 4 | import {StreamLanguage, syntaxTree, getIndentation, Language} from "@codemirror/language" 5 | import {SyntaxNode} from "@lezer/common" 6 | 7 | let startStates = 0, keywords = ["if", "else", "return"] 8 | 9 | const language = StreamLanguage.define<{count: number}>({ 10 | startState() { 11 | startStates++ 12 | return {count: 0} 13 | }, 14 | 15 | token(stream, state) { 16 | if (stream.eatSpace()) return null 17 | state.count++ 18 | if (stream.match(/^\/\/.*/)) return "lineComment" 19 | if (stream.match(/^"[^"]*"/)) return "string" 20 | if (stream.match(/^\d+/)) return "number" 21 | if (stream.match(/^\w+/)) return keywords.indexOf(stream.current()) >= 0 ? "keyword" : "variableName" 22 | if (stream.match(/^[();{}]/)) return "punctuation" 23 | stream.next() 24 | return "invalid" 25 | }, 26 | 27 | indent(state) { 28 | return state.count 29 | } 30 | }) 31 | 32 | describe("StreamLanguage", () => { 33 | it("can parse content", () => { 34 | ist(language.parser.parse("if (x) return 500").toString(), 35 | "Document(keyword,punctuation,variableName,punctuation,keyword,number)") 36 | }) 37 | 38 | it("can reuse state on updates", () => { 39 | let state = EditorState.create({ 40 | doc: "// filler content\nif (a) foo()\nelse if (b) bar()\nelse quux()\n\n".repeat(100), 41 | extensions: language 42 | }) 43 | 44 | startStates = 0 45 | state = state.update({changes: {from: 5000, to: 5001}}).state 46 | ist(startStates, 0) 47 | }) 48 | 49 | it("can find the correct parse state for indentation", () => { 50 | let state = EditorState.create({ 51 | doc: '"abcdefg"\n'.repeat(200), 52 | extensions: language 53 | }) 54 | ist(getIndentation(state, 0), 0) 55 | ist(getIndentation(state, 10), 1) 56 | ist(getIndentation(state, 100), 10) 57 | ist(getIndentation(state, 1000), 100) 58 | }) 59 | 60 | // Fragile kludge to set the parser context viewport without 61 | // actually having access to the relevant field 62 | function setViewport(state: EditorState, from: number, to: number) { 63 | let field = (Language as any).state 64 | ;(state.field(field) as any).context.updateViewport({from, to}) 65 | } 66 | 67 | it("will make up a state when the viewport is far away from the frontier", () => { 68 | let line = "1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0\n" 69 | let state = EditorState.create({doc: line.repeat(100), extensions: language}) 70 | setViewport(state, 4000, 8000) 71 | state = state.update({changes: {from: 3000, insert: line.repeat(10000)}}).state 72 | let tree = syntaxTree(state) 73 | // No nodes in the skipped range 74 | ist(tree.resolve(10000, 1).name, "Document") 75 | // But the viewport is populated 76 | ist(tree.resolve(805000, 1).name, "number") 77 | let treeSize = 0 78 | tree.iterate({enter() { treeSize++ }}) 79 | ist(treeSize, 2000, ">") 80 | ist(treeSize, 4000, "<") 81 | setViewport(state, 4000, 8000) 82 | state = state.update({changes: {from: 100000, insert: "?"}}).state 83 | tree = syntaxTree(state) 84 | ist(tree.resolve(5000, 1).name, "number") 85 | ist(tree.resolve(50000, 1).name, "Document") 86 | }) 87 | 88 | it("doesn't parse beyond the viewport", () => { 89 | let line = "1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0\n" 90 | let state = EditorState.create({doc: line.repeat(100), extensions: language}) 91 | setViewport(state, 0, 4000) 92 | state = state.update({changes: {from: 5000, insert: line.repeat(100)}}).state 93 | ist(syntaxTree(state).resolve(2000, 1).name, "number") 94 | ist(syntaxTree(state).resolve(6000, 1).name, "Document") 95 | }) 96 | 97 | function isNode(node: SyntaxNode | null, name: string, from: number, to: number) { 98 | ist(node) 99 | ist(node!.type.name, name) 100 | ist(node!.from, from) 101 | ist(node!.to, to) 102 | } 103 | 104 | it("supports gaps", () => { 105 | let text = "1 50 xxx\nxxx\nxxx 60\n70 xxx80xxx 9xxx0" 106 | let ranges = [{from: 0, to: 5}, {from: 16, to: 23}, {from: 26, to: 28}, {from: 31, to: 33}, {from: 36, to: 37}] 107 | let tree = language.parser.parse(text, [], ranges) 108 | ist(tree.toString(), "Document(number,number,number,number,number,number)") 109 | isNode(tree.resolve(17, 1), "number", 17, 19) 110 | isNode(tree.resolve(20, 1), "number", 20, 22) 111 | isNode(tree.resolve(26, 1), "number", 26, 28) 112 | isNode(tree.resolve(32, 1), "number", 32, 37) 113 | }) 114 | 115 | it("accepts custom token types", () => { 116 | let tag = Tag.define() 117 | let lang = StreamLanguage.define({ 118 | token(stream) { 119 | if (stream.match(/^\w+/)) return "foo" 120 | stream.next() 121 | return null 122 | }, 123 | tokenTable: {foo: tag} 124 | }) 125 | ist(lang.parser.parse("hello").toString(), "Document(foo)") 126 | }) 127 | }) 128 | -------------------------------------------------------------------------------- /test/test-syntax.ts: -------------------------------------------------------------------------------- 1 | import ist from "ist" 2 | import {getIndentUnit, indentString, indentUnit, ParseContext} from "@codemirror/language" 3 | import {EditorState, ChangeSet, Text} from "@codemirror/state" 4 | import {parser} from "@lezer/javascript" 5 | 6 | let lines = `const {readFile} = require("fs"); 7 | readFile("package.json", "utf8", (err, data) => { 8 | console.log(data); 9 | }); 10 | `.split("\n") 11 | for (let l0 = lines.length, i = l0; i < 5000; i++) lines[i] = lines[i % l0] 12 | let doc = Text.of(lines) 13 | 14 | function pContext(doc: Text) { 15 | return ParseContext.create(parser, EditorState.create({doc}), {from: 0, to: doc.length}) 16 | } 17 | 18 | describe("ParseContext", () => { 19 | it("can parse a document", () => { 20 | let cx = pContext(Text.of(["let x = 10"])) 21 | cx.work(1e8) 22 | ist(cx.tree.toString(), "Script(VariableDeclaration(let,VariableDefinition,Equals,Number))") 23 | }) 24 | 25 | it("can parse incrementally", () => { 26 | let cx = pContext(doc), t0 = Date.now() 27 | if (cx.work(10)) { 28 | console.warn("Machine too fast for the incremental parsing test, skipping") 29 | return 30 | } 31 | ist(Date.now() - t0, 25, "<") 32 | ist(cx.work(1e8)) 33 | ist(cx.tree.length, doc.length) 34 | let change = ChangeSet.of({from: 0, to: 5, insert: "let"}, doc.length) 35 | let newDoc = change.apply(doc) 36 | cx = cx.changes(change, EditorState.create({doc: newDoc})) 37 | ist(cx.work(50)) 38 | ist(cx.tree.length, newDoc.length) 39 | ist(cx.tree.toString().slice(0, 31), "Script(VariableDeclaration(let,") 40 | }) 41 | }) 42 | 43 | describe("Indentation", () => { 44 | it("tracks indent units", () => { 45 | let s0 = EditorState.create({}) 46 | ist(getIndentUnit(s0), 2) 47 | ist(indentString(s0, 4), " ") 48 | let s1 = EditorState.create({extensions: indentUnit.of(" ")}) 49 | ist(getIndentUnit(s1), 3) 50 | ist(indentString(s1, 4), " ") 51 | let s2 = EditorState.create({extensions: [indentUnit.of("\t"), EditorState.tabSize.of(8)]}) 52 | ist(getIndentUnit(s2), 8) 53 | ist(indentString(s2, 16), "\t\t") 54 | let s3 = EditorState.create({extensions: indentUnit.of("　")}) 55 | ist(getIndentUnit(s3), 1) 56 | ist(indentString(s3, 2), "　　") 57 | }) 58 | 59 | it("errors for bad indent units", () => { 60 | ist.throws(() => EditorState.create({extensions: indentUnit.of("")}), /Invalid indent unit/) 61 | ist.throws(() => EditorState.create({extensions: indentUnit.of("\t ")}), /Invalid indent unit/) 62 | ist.throws(() => EditorState.create({extensions: indentUnit.of("hello")}), /Invalid indent unit/) 63 | }) 64 | }) 65 | --------------------------------------------------------------------------------