├── .gitignore ├── .npmignore ├── .vscode ├── launch.json ├── settings.json └── tasks.json ├── LICENSE.md ├── README.md ├── ThirdPartyNotices.txt ├── package.json ├── src ├── debug.ts ├── grammar.ts ├── grammarReader.ts ├── json.ts ├── main.ts ├── matcher.ts ├── registry.ts ├── rule.ts ├── theme.ts ├── types.ts └── utils.ts ├── tsconfig.json └── tslint.json /.gitignore: -------------------------------------------------------------------------------- 1 | /node_modules/ 2 | /dist/ 3 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | /.vscode/ 2 | /.npmignore 3 | /.travis.yml 4 | /gulpfile.js 5 | /npm-debug.log 6 | /ThirdPartyNotices.txt 7 | /tsconfig.json 8 | /tslint.json -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.1.0", 3 | "configurations": [ 4 | { 5 | "name": "Launch tests", 6 | "type": "node", 7 | "request": "launch", 8 | "program": "${workspaceRoot}/node_modules/mocha/bin/_mocha", 9 | "stopOnEntry": false, 10 | "args": [ "out/tests/tests", "-g" , "Issue #46"], 11 | "cwd": "${workspaceRoot}", 12 | "runtimeExecutable": null, 13 | "runtimeArgs": ["--nolazy"], 14 | "env": { 15 | "NODE_ENV": "development" 16 | }, 17 | "console": "internalConsole", 18 | "sourceMaps": true, 19 | "outFiles": [ "out/**" ], 20 | "preLaunchTask": "compile" 21 | } 22 | ] 23 | } 24 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | // Place your settings in this file to overwrite default and user settings. 2 | { 3 | "files.trimTrailingWhitespace": true, 4 | "editor.insertSpaces": false, 5 | "editor.tabSize": 4, 6 | "search.exclude": { 7 | "**/node_modules": true, 8 | "**/bower_components": true, 9 | "out/**": true, 10 | "release/**": true, 11 | "coverage/**": true 12 | }, 13 | "lcov.sourceMaps": true, 14 | "typescript.tsdk": "./node_modules/typescript/lib", 15 | "tslint.enable": true 16 | } -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | // See https://go.microsoft.com/fwlink/?LinkId=733558 3 | // for the documentation about the tasks.json format 4 | "version": "0.1.0", 5 | "command": "npm", 6 | "isShellCommand": true, 7 | "showOutput": "always", 8 | "suppressTaskName": true, 9 | "tasks": [ 10 | { 11 | "taskName": "test", 12 | "args": ["run", "test"] 13 | }, 14 | { 15 | "taskName": "compile", 16 | "args": ["run", "compile"] 17 | } 18 | ] 19 | } -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) Microsoft Corporation 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | > This repository is a heavily modified version of original `vscode-textmate` package. It's been adjusted here and there to run inside web browsers. All of the file system calls have been removed, most of the API now uses `Promises` and grammars can no longer be loaded synchronously. 2 | 3 | ⚠ I'd prefer to see this repository where it really belongs. I request anyone from Microsoft to adopt this package as soon as possible. 4 | 5 | # Monaco TextMate 6 | 7 | An interpreter for grammar files as defined by TextMate that runs on the web. Supports loading grammar files from JSON or PLIST format. Cross - grammar injections are currently not supported. 8 | 9 | ## Installing 10 | 11 | ```sh 12 | npm install monaco-textmate 13 | ``` 14 | 15 | ## Using 16 | 17 | `monaco-textmate` relies on `onigasm` package to provide `oniguruma` regex engine in browsers. `onigasm` itself relies on `WebAssembly`. Therefore to 18 | get `monaco-textmate` working in your browser, it must have `WebAssembly` support and `onigasm` loaded and ready-to-go. 19 | 20 | Make sure the example code below runs *after* `onigasm` bootstraping sequence described [here](https://www.npmjs.com/package/onigasm#light-it-up) has finished. 21 | 22 | > Example below is just a demostration of available API. To wire it up with `monaco-editor` use [`monaco-editor-textmate`](https://github.com/NeekSandhu/monaco-editor-textmate). 23 | 24 | ```javascript 25 | import { Registry } from 'monaco-textmate' 26 | 27 | (async function test() { 28 | const registry = new Registry({ 29 | // Since we're in browser, `getFilePath` has been removed, therefore you must provide `getGrammarDefinition` hook for things to work 30 | getGrammarDefinition: async (scopeName) => { 31 | // Whenever `Registry.loadGrammar` is called first time per scope name, this function will be called asking you to provide 32 | // raw grammar definition. Both JSON and plist formats are accepted. 33 | if (scopeName === 'source.css') { 34 | return { 35 | format: 'json', // can also be `plist` 36 | content: await (await fetch(`static/grammars/css.tmGrammar.json`)).text() // when format is 'json', parsed JSON also works 37 | } 38 | } 39 | } 40 | }) 41 | 42 | const grammar = await registry.loadGrammar('source.css') 43 | 44 | console.log(grammar.tokenizeLine('html, body { height: 100%; margin: 0 }')) 45 | // > {tokens: Array(19), ruleStack: StackElement} 46 | })() 47 | 48 | ``` 49 | 50 | > `onigasm` is peer dependency that you must install yourself 51 | 52 | ## Tokenizing multiple lines 53 | 54 | To tokenize multiple lines, you must pass in the previous returned `ruleStack`. 55 | 56 | ```javascript 57 | var ruleStack = null; 58 | for (var i = 0; i < lines.length; i++) { 59 | var r = grammar.tokenizeLine(lines[i], ruleStack); 60 | console.log('Line: #' + i + ', tokens: ' + r.tokens); 61 | ruleStack = r.ruleStack; 62 | } 63 | ``` 64 | 65 | ## API doc 66 | 67 | See [the main.ts file](./src/main.ts) 68 | 69 | ## Developing 70 | 71 | * Clone the repository 72 | * Run `npm install` 73 | * Compile in the background with `npm run watch` 74 | 75 | ## Credits 76 | 99% of the code in this repository is extracted straight from [`vscode-textmate`](https://github.com/Microsoft/vscode-textmate), which is MIT licensed. 77 | Other external licenses used can be found in [`ThirdPartyNotices.txt`](https://github.com/NeekSandhu/monaco-textmate/blob/master/ThirdPartyNotices.txt) 78 | 79 | ## License 80 | [MIT](https://github.com/Microsoft/vscode-textmate/blob/master/LICENSE.md) 81 | -------------------------------------------------------------------------------- /ThirdPartyNotices.txt: -------------------------------------------------------------------------------- 1 | THIRD-PARTY SOFTWARE NOTICES AND INFORMATION 2 | Do Not Translate or Localize 3 | 4 | This project incorporates material from the project(s) listed below (collectively, “Third Party Code”). 5 | Microsoft is not the original author of the Third Party Code. The original copyright notice and license 6 | under which Microsoft received such Third Party Code are set out below. This Third Party Code is licensed 7 | to you under their original license terms set forth below. Microsoft reserves all other rights not 8 | expressly granted, whether by implication, estoppel or otherwise. 9 | 10 | The following files/folders contain third party software used for development-time testing: 11 | 12 | ========================================================================================================= 13 | benchmark/bootstrap.css, benchmark/bootstrap.min.css 14 | --------------------------------------------------------------------------------------------------------- 15 | The MIT License (MIT) 16 | 17 | Copyright (c) 2011-2015 Twitter, Inc 18 | 19 | Permission is hereby granted, free of charge, to any person obtaining a copy 20 | of this software and associated documentation files (the "Software"), to deal 21 | in the Software without restriction, including without limitation the rights 22 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 23 | copies of the Software, and to permit persons to whom the Software is 24 | furnished to do so, subject to the following conditions: 25 | 26 | The above copyright notice and this permission notice shall be included in 27 | all copies or substantial portions of the Software. 28 | 29 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 30 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 31 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 32 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 33 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 34 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 35 | THE SOFTWARE. 36 | ========================================================================================================= 37 | 38 | 39 | 40 | ========================================================================================================= 41 | benchmark/large.js, benchmark/large.min.js 42 | --------------------------------------------------------------------------------------------------------- 43 | The MIT License (MIT) 44 | 45 | Copyright 2005, 2013 jQuery Foundation, Inc. and other contributors 46 | 47 | Permission is hereby granted, free of charge, to any person obtaining a copy 48 | of this software and associated documentation files (the "Software"), to deal 49 | in the Software without restriction, including without limitation the rights 50 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 51 | copies of the Software, and to permit persons to whom the Software is 52 | furnished to do so, subject to the following conditions: 53 | 54 | The above copyright notice and this permission notice shall be included in 55 | all copies or substantial portions of the Software. 56 | 57 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 58 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 59 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 60 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 61 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 62 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 63 | THE SOFTWARE. 64 | ========================================================================================================= 65 | 66 | 67 | 68 | ========================================================================================================= 69 | test-cases/first-mate/** 70 | --------------------------------------------------------------------------------------------------------- 71 | The MIT License (MIT) 72 | 73 | Copyright (c) 2013 GitHub Inc. 74 | 75 | Permission is hereby granted, free of charge, to any person obtaining a copy 76 | of this software and associated documentation files (the "Software"), to deal 77 | in the Software without restriction, including without limitation the rights 78 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 79 | copies of the Software, and to permit persons to whom the Software is 80 | furnished to do so, subject to the following conditions: 81 | 82 | The above copyright notice and this permission notice shall be included in 83 | all copies or substantial portions of the Software. 84 | 85 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 86 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 87 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 88 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 89 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 90 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 91 | THE SOFTWARE. 92 | ========================================================================================================= 93 | 94 | 95 | 96 | ========================================================================================================= 97 | test-cases/suite1/** 98 | --------------------------------------------------------------------------------------------------------- 99 | Copyright (c) TextMate project authors 100 | 101 | If not otherwise specified (see below), files in this repository fall under the following license: 102 | 103 | Permission to copy, use, modify, sell and distribute this 104 | software is granted. This software is provided "as is" without 105 | express or implied warranty, and with no claim as to its 106 | suitability for any purpose. 107 | 108 | An exception is made for files in readable text which contain their own license information, 109 | or files where an accompanying file exists (in the same directory) with a "-license" suffix added 110 | to the base-name name of the original file, and an extension of txt, html, or similar. For example 111 | "tidy" is accompanied by "tidy-license.txt". 112 | ========================================================================================================= -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "monaco-textmate", 3 | "version": "3.0.1", 4 | "description": "Monaco TextMate grammar helpers", 5 | "main": "./dist/main.js", 6 | "typings": "./dist/typings/main.d.ts", 7 | "repository": { 8 | "type": "git", 9 | "url": "https://github.com/NeekSandhu/monaco-textmate" 10 | }, 11 | "license": "MIT", 12 | "bugs": { 13 | "url": "https://github.com/NeekSandhu/monaco-textmate/issues" 14 | }, 15 | "scripts": { 16 | "watch": "tsc -watch", 17 | "compile": "tsc", 18 | "prepublish": "npm run compile" 19 | }, 20 | "dependencies": { 21 | "fast-plist": "^0.1.2" 22 | }, 23 | "peerDependencies": { 24 | "onigasm": "^2.0.0" 25 | }, 26 | "devDependencies": { 27 | "@types/node": "^10.0.4", 28 | "tslint": "^3.15.1", 29 | "typescript": "^2.0.8" 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/debug.ts: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------- 2 | * Copyright (C) Microsoft Corporation. All rights reserved. 3 | *--------------------------------------------------------*/ 4 | 5 | export const CAPTURE_METADATA = typeof process === 'undefined' ? false : !!process.env['VSCODE_TEXTMATE_DEBUG']; 6 | export const IN_DEBUG_MODE = typeof process === 'undefined' ? false : !!process.env['VSCODE_TEXTMATE_DEBUG']; 7 | -------------------------------------------------------------------------------- /src/grammar.ts: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------- 2 | * Copyright (C) Microsoft Corporation. All rights reserved. 3 | *--------------------------------------------------------*/ 4 | 5 | import { clone } from './utils'; 6 | import { IRawGrammar, IRawRepository, IRawRule } from './types'; 7 | import { IRuleRegistry, IRuleFactoryHelper, RuleFactory, Rule, CaptureRule, BeginEndRule, BeginWhileRule, MatchRule, ICompiledRule, createOnigString, getString } from './rule'; 8 | import { IOnigCaptureIndex, OnigString } from 'onigasm'; 9 | import { createMatchers, Matcher } from './matcher'; 10 | import { MetadataConsts, IGrammar, ITokenizeLineResult, ITokenizeLineResult2, IToken, IEmbeddedLanguagesMap, StandardTokenType, StackElement as StackElementDef, ITokenTypeMap } from './main'; 11 | import { IN_DEBUG_MODE } from './debug'; 12 | import { FontStyle, ThemeTrieElementRule } from './theme'; 13 | 14 | export const enum TemporaryStandardTokenType { 15 | Other = 0, 16 | Comment = 1, 17 | String = 2, 18 | RegEx = 4, 19 | MetaEmbedded = 8 20 | } 21 | 22 | export function createGrammar(grammar: IRawGrammar, initialLanguage: number, embeddedLanguages: IEmbeddedLanguagesMap, tokenTypes: ITokenTypeMap, grammarRepository: IGrammarRepository & IThemeProvider): Grammar { 23 | return new Grammar(grammar, initialLanguage, embeddedLanguages, tokenTypes, grammarRepository); 24 | } 25 | 26 | export interface IThemeProvider { 27 | themeMatch(scopeName: string): ThemeTrieElementRule[]; 28 | getDefaults(): ThemeTrieElementRule; 29 | } 30 | 31 | export interface IGrammarRepository { 32 | lookup(scopeName: string): IRawGrammar; 33 | injections(scopeName: string): string[]; 34 | } 35 | 36 | export interface IScopeNameSet { 37 | [scopeName: string]: boolean; 38 | } 39 | 40 | /** 41 | * Fill in `result` all external included scopes in `patterns` 42 | */ 43 | function _extractIncludedScopesInPatterns(result: IScopeNameSet, patterns: IRawRule[]): void { 44 | for (let i = 0, len = patterns.length; i < len; i++) { 45 | 46 | if (Array.isArray(patterns[i].patterns)) { 47 | _extractIncludedScopesInPatterns(result, patterns[i].patterns); 48 | } 49 | 50 | let include = patterns[i].include; 51 | 52 | if (!include) { 53 | continue; 54 | } 55 | 56 | if (include === '$base' || include === '$self') { 57 | // Special includes that can be resolved locally in this grammar 58 | continue; 59 | } 60 | 61 | if (include.charAt(0) === '#') { 62 | // Local include from this grammar 63 | continue; 64 | } 65 | 66 | let sharpIndex = include.indexOf('#'); 67 | if (sharpIndex >= 0) { 68 | result[include.substring(0, sharpIndex)] = true; 69 | } else { 70 | result[include] = true; 71 | } 72 | } 73 | } 74 | 75 | /** 76 | * Fill in `result` all external included scopes in `repository` 77 | */ 78 | function _extractIncludedScopesInRepository(result: IScopeNameSet, repository: IRawRepository): void { 79 | for (let name in repository) { 80 | let rule = repository[name]; 81 | 82 | if (rule.patterns && Array.isArray(rule.patterns)) { 83 | _extractIncludedScopesInPatterns(result, rule.patterns); 84 | } 85 | 86 | if (rule.repository) { 87 | _extractIncludedScopesInRepository(result, rule.repository); 88 | } 89 | } 90 | } 91 | 92 | /** 93 | * Collects the list of all external included scopes in `grammar`. 94 | */ 95 | export function collectIncludedScopes(result: IScopeNameSet, grammar: IRawGrammar): void { 96 | if (grammar.patterns && Array.isArray(grammar.patterns)) { 97 | _extractIncludedScopesInPatterns(result, grammar.patterns); 98 | } 99 | 100 | if (grammar.repository) { 101 | _extractIncludedScopesInRepository(result, grammar.repository); 102 | } 103 | 104 | // remove references to own scope (avoid recursion) 105 | delete result[grammar.scopeName]; 106 | } 107 | 108 | export interface Injection { 109 | readonly matcher: Matcher; 110 | readonly priority: -1 | 0 | 1; // 0 is the default. -1 for 'L' and 1 for 'R' 111 | readonly ruleId: number; 112 | readonly grammar: IRawGrammar; 113 | } 114 | 115 | function scopesAreMatching(thisScopeName: string, scopeName: string): boolean { 116 | if (!thisScopeName) { 117 | return false; 118 | } 119 | if (thisScopeName === scopeName) { 120 | return true; 121 | } 122 | var len = scopeName.length; 123 | return thisScopeName.length > len && thisScopeName.substr(0, len) === scopeName && thisScopeName[len] === '.'; 124 | } 125 | 126 | function nameMatcher(identifers: string[], scopes: string[]) { 127 | if (scopes.length < identifers.length) { 128 | return false; 129 | } 130 | var lastIndex = 0; 131 | return identifers.every(identifier => { 132 | for (var i = lastIndex; i < scopes.length; i++) { 133 | if (scopesAreMatching(scopes[i], identifier)) { 134 | lastIndex = i + 1; 135 | return true; 136 | } 137 | } 138 | return false; 139 | }); 140 | }; 141 | 142 | function collectInjections(result: Injection[], selector: string, rule: IRawRule, ruleFactoryHelper: IRuleFactoryHelper, grammar: IRawGrammar): void { 143 | let matchers = createMatchers(selector, nameMatcher); 144 | let ruleId = RuleFactory.getCompiledRuleId(rule, ruleFactoryHelper, grammar.repository); 145 | for (let matcher of matchers) { 146 | result.push({ 147 | matcher: matcher.matcher, 148 | ruleId: ruleId, 149 | grammar: grammar, 150 | priority: matcher.priority 151 | }); 152 | } 153 | } 154 | 155 | export class ScopeMetadata { 156 | public readonly scopeName: string; 157 | public readonly languageId: number; 158 | public readonly tokenType: TemporaryStandardTokenType; 159 | public readonly themeData: ThemeTrieElementRule[]; 160 | 161 | constructor(scopeName: string, languageId: number, tokenType: TemporaryStandardTokenType, themeData: ThemeTrieElementRule[]) { 162 | this.scopeName = scopeName; 163 | this.languageId = languageId; 164 | this.tokenType = tokenType; 165 | this.themeData = themeData; 166 | } 167 | } 168 | 169 | class ScopeMetadataProvider { 170 | 171 | private readonly _initialLanguage: number; 172 | private readonly _themeProvider: IThemeProvider; 173 | private _cache: { [scopeName: string]: ScopeMetadata; }; 174 | private _defaultMetaData: ScopeMetadata; 175 | private readonly _embeddedLanguages: IEmbeddedLanguagesMap; 176 | private readonly _embeddedLanguagesRegex: RegExp; 177 | 178 | constructor(initialLanguage: number, themeProvider: IThemeProvider, embeddedLanguages: IEmbeddedLanguagesMap) { 179 | this._initialLanguage = initialLanguage; 180 | this._themeProvider = themeProvider; 181 | this.onDidChangeTheme(); 182 | 183 | // embeddedLanguages handling 184 | this._embeddedLanguages = Object.create(null); 185 | 186 | if (embeddedLanguages) { 187 | // If embeddedLanguages are configured, fill in `this._embeddedLanguages` 188 | let scopes = Object.keys(embeddedLanguages); 189 | for (let i = 0, len = scopes.length; i < len; i++) { 190 | let scope = scopes[i]; 191 | let language = embeddedLanguages[scope]; 192 | if (typeof language !== 'number' || language === 0) { 193 | console.warn('Invalid embedded language found at scope ' + scope + ': <<' + language + '>>'); 194 | // never hurts to be too careful 195 | continue; 196 | } 197 | this._embeddedLanguages[scope] = language; 198 | } 199 | } 200 | 201 | // create the regex 202 | let escapedScopes = Object.keys(this._embeddedLanguages).map((scopeName) => ScopeMetadataProvider._escapeRegExpCharacters(scopeName)); 203 | if (escapedScopes.length === 0) { 204 | // no scopes registered 205 | this._embeddedLanguagesRegex = null; 206 | } else { 207 | escapedScopes.sort(); 208 | escapedScopes.reverse(); 209 | this._embeddedLanguagesRegex = new RegExp(`^((${escapedScopes.join(')|(')}))($|\\.)`, ''); 210 | } 211 | } 212 | 213 | public onDidChangeTheme(): void { 214 | this._cache = Object.create(null); 215 | this._defaultMetaData = new ScopeMetadata( 216 | '', 217 | this._initialLanguage, 218 | TemporaryStandardTokenType.Other, 219 | [this._themeProvider.getDefaults()] 220 | ); 221 | } 222 | 223 | public getDefaultMetadata(): ScopeMetadata { 224 | return this._defaultMetaData; 225 | } 226 | 227 | /** 228 | * Escapes regular expression characters in a given string 229 | */ 230 | private static _escapeRegExpCharacters(value: string): string { 231 | return value.replace(/[\-\\\{\}\*\+\?\|\^\$\.\,\[\]\(\)\#\s]/g, '\\$&'); 232 | } 233 | 234 | private static _NULL_SCOPE_METADATA = new ScopeMetadata('', 0, 0, null); 235 | public getMetadataForScope(scopeName: string): ScopeMetadata { 236 | if (scopeName === null) { 237 | return ScopeMetadataProvider._NULL_SCOPE_METADATA; 238 | } 239 | let value = this._cache[scopeName]; 240 | if (value) { 241 | return value; 242 | } 243 | value = this._doGetMetadataForScope(scopeName); 244 | this._cache[scopeName] = value; 245 | return value; 246 | } 247 | 248 | private _doGetMetadataForScope(scopeName: string): ScopeMetadata { 249 | let languageId = this._scopeToLanguage(scopeName); 250 | let standardTokenType = this._toStandardTokenType(scopeName); 251 | let themeData = this._themeProvider.themeMatch(scopeName); 252 | 253 | return new ScopeMetadata(scopeName, languageId, standardTokenType, themeData); 254 | } 255 | 256 | /** 257 | * Given a produced TM scope, return the language that token describes or null if unknown. 258 | * e.g. source.html => html, source.css.embedded.html => css, punctuation.definition.tag.html => null 259 | */ 260 | private _scopeToLanguage(scope: string): number { 261 | if (!scope) { 262 | return 0; 263 | } 264 | if (!this._embeddedLanguagesRegex) { 265 | // no scopes registered 266 | return 0; 267 | } 268 | let m = scope.match(this._embeddedLanguagesRegex); 269 | if (!m) { 270 | // no scopes matched 271 | return 0; 272 | } 273 | 274 | let language = this._embeddedLanguages[m[1]] || 0; 275 | if (!language) { 276 | return 0; 277 | } 278 | 279 | return language; 280 | } 281 | 282 | private static STANDARD_TOKEN_TYPE_REGEXP = /\b(comment|string|regex|meta\.embedded)\b/; 283 | private _toStandardTokenType(tokenType: string): TemporaryStandardTokenType { 284 | let m = tokenType.match(ScopeMetadataProvider.STANDARD_TOKEN_TYPE_REGEXP); 285 | if (!m) { 286 | return TemporaryStandardTokenType.Other; 287 | } 288 | switch (m[1]) { 289 | case 'comment': 290 | return TemporaryStandardTokenType.Comment; 291 | case 'string': 292 | return TemporaryStandardTokenType.String; 293 | case 'regex': 294 | return TemporaryStandardTokenType.RegEx; 295 | case 'meta.embedded': 296 | return TemporaryStandardTokenType.MetaEmbedded; 297 | } 298 | throw new Error('Unexpected match for standard token type!'); 299 | } 300 | } 301 | 302 | export class Grammar implements IGrammar, IRuleFactoryHelper { 303 | 304 | private _rootId: number; 305 | private _lastRuleId: number; 306 | private readonly _ruleId2desc: Rule[]; 307 | private readonly _includedGrammars: { [scopeName: string]: IRawGrammar; }; 308 | private readonly _grammarRepository: IGrammarRepository; 309 | private readonly _grammar: IRawGrammar; 310 | private _injections: Injection[]; 311 | private readonly _scopeMetadataProvider: ScopeMetadataProvider; 312 | private readonly _tokenTypeMatchers: TokenTypeMatcher[]; 313 | 314 | constructor(grammar: IRawGrammar, initialLanguage: number, embeddedLanguages: IEmbeddedLanguagesMap, tokenTypes: ITokenTypeMap, grammarRepository: IGrammarRepository & IThemeProvider) { 315 | this._scopeMetadataProvider = new ScopeMetadataProvider(initialLanguage, grammarRepository, embeddedLanguages); 316 | 317 | this._rootId = -1; 318 | this._lastRuleId = 0; 319 | this._ruleId2desc = []; 320 | this._includedGrammars = {}; 321 | this._grammarRepository = grammarRepository; 322 | this._grammar = initGrammar(grammar, null); 323 | 324 | this._tokenTypeMatchers = []; 325 | if (tokenTypes) { 326 | for (const selector of Object.keys(tokenTypes)) { 327 | const matchers = createMatchers(selector, nameMatcher); 328 | for (const matcher of matchers) { 329 | this._tokenTypeMatchers.push({ 330 | matcher: matcher.matcher, 331 | type: tokenTypes[selector] 332 | }); 333 | } 334 | } 335 | } 336 | } 337 | 338 | public onDidChangeTheme(): void { 339 | this._scopeMetadataProvider.onDidChangeTheme(); 340 | } 341 | 342 | public getMetadataForScope(scope: string): ScopeMetadata { 343 | return this._scopeMetadataProvider.getMetadataForScope(scope); 344 | } 345 | 346 | public getInjections(): Injection[] { 347 | if (!this._injections) { 348 | this._injections = []; 349 | // add injections from the current grammar 350 | var rawInjections = this._grammar.injections; 351 | if (rawInjections) { 352 | for (var expression in rawInjections) { 353 | collectInjections(this._injections, expression, rawInjections[expression], this, this._grammar); 354 | } 355 | } 356 | 357 | // add injection grammars contributed for the current scope 358 | if (this._grammarRepository) { 359 | let injectionScopeNames = this._grammarRepository.injections(this._grammar.scopeName); 360 | if (injectionScopeNames) { 361 | injectionScopeNames.forEach(injectionScopeName => { 362 | let injectionGrammar = this.getExternalGrammar(injectionScopeName); 363 | if (injectionGrammar) { 364 | let selector = injectionGrammar.injectionSelector; 365 | if (selector) { 366 | collectInjections(this._injections, selector, injectionGrammar, this, injectionGrammar); 367 | } 368 | } 369 | }); 370 | } 371 | } 372 | this._injections.sort((i1, i2) => i1.priority - i2.priority); // sort by priority 373 | } 374 | if (this._injections.length === 0) { 375 | return this._injections; 376 | } 377 | return this._injections; 378 | } 379 | 380 | public registerRule(factory: (id: number) => T): T { 381 | let id = (++this._lastRuleId); 382 | let result = factory(id); 383 | this._ruleId2desc[id] = result; 384 | return result; 385 | } 386 | 387 | public getRule(patternId: number): Rule { 388 | return this._ruleId2desc[patternId]; 389 | } 390 | 391 | public getExternalGrammar(scopeName: string, repository?: IRawRepository): IRawGrammar { 392 | if (this._includedGrammars[scopeName]) { 393 | return this._includedGrammars[scopeName]; 394 | } else if (this._grammarRepository) { 395 | let rawIncludedGrammar = this._grammarRepository.lookup(scopeName); 396 | if (rawIncludedGrammar) { 397 | // console.log('LOADED GRAMMAR ' + pattern.include); 398 | this._includedGrammars[scopeName] = initGrammar(rawIncludedGrammar, repository && repository.$base); 399 | return this._includedGrammars[scopeName]; 400 | } 401 | } 402 | } 403 | 404 | public tokenizeLine(lineText: string, prevState: StackElement): ITokenizeLineResult { 405 | let r = this._tokenize(lineText, prevState, false); 406 | return { 407 | tokens: r.lineTokens.getResult(r.ruleStack, r.lineLength), 408 | ruleStack: r.ruleStack 409 | }; 410 | } 411 | 412 | public tokenizeLine2(lineText: string, prevState: StackElement): ITokenizeLineResult2 { 413 | let r = this._tokenize(lineText, prevState, true); 414 | return { 415 | tokens: r.lineTokens.getBinaryResult(r.ruleStack, r.lineLength), 416 | ruleStack: r.ruleStack 417 | }; 418 | } 419 | 420 | private _tokenize(lineText: string, prevState: StackElement, emitBinaryTokens: boolean): { lineLength: number; lineTokens: LineTokens; ruleStack: StackElement; } { 421 | if (this._rootId === -1) { 422 | this._rootId = RuleFactory.getCompiledRuleId(this._grammar.repository.$self, this, this._grammar.repository); 423 | } 424 | 425 | let isFirstLine: boolean; 426 | if (!prevState || prevState === StackElement.NULL) { 427 | isFirstLine = true; 428 | let rawDefaultMetadata = this._scopeMetadataProvider.getDefaultMetadata(); 429 | let defaultTheme = rawDefaultMetadata.themeData[0]; 430 | let defaultMetadata = StackElementMetadata.set(0, rawDefaultMetadata.languageId, rawDefaultMetadata.tokenType, defaultTheme.fontStyle, defaultTheme.foreground, defaultTheme.background); 431 | 432 | let rootScopeName = this.getRule(this._rootId).getName(null, null); 433 | let rawRootMetadata = this._scopeMetadataProvider.getMetadataForScope(rootScopeName); 434 | let rootMetadata = ScopeListElement.mergeMetadata(defaultMetadata, null, rawRootMetadata); 435 | 436 | let scopeList = new ScopeListElement(null, rootScopeName, rootMetadata); 437 | 438 | prevState = new StackElement(null, this._rootId, -1, null, scopeList, scopeList); 439 | } else { 440 | isFirstLine = false; 441 | prevState.reset(); 442 | } 443 | 444 | lineText = lineText + '\n'; 445 | let onigLineText = createOnigString(lineText); 446 | let lineLength = getString(onigLineText).length; 447 | let lineTokens = new LineTokens(emitBinaryTokens, lineText, this._tokenTypeMatchers); 448 | let nextState = _tokenizeString(this, onigLineText, isFirstLine, 0, prevState, lineTokens); 449 | 450 | return { 451 | lineLength: lineLength, 452 | lineTokens: lineTokens, 453 | ruleStack: nextState 454 | }; 455 | } 456 | } 457 | 458 | function initGrammar(grammar: IRawGrammar, base: IRawRule): IRawGrammar { 459 | grammar = clone(grammar); 460 | 461 | grammar.repository = grammar.repository || {}; 462 | grammar.repository.$self = { 463 | $vscodeTextmateLocation: grammar.$vscodeTextmateLocation, 464 | patterns: grammar.patterns, 465 | name: grammar.scopeName 466 | }; 467 | grammar.repository.$base = base || grammar.repository.$self; 468 | return grammar; 469 | } 470 | 471 | function handleCaptures(grammar: Grammar, lineText: OnigString, isFirstLine: boolean, stack: StackElement, lineTokens: LineTokens, captures: CaptureRule[], captureIndices: IOnigCaptureIndex[]): void { 472 | if (captures.length === 0) { 473 | return; 474 | } 475 | 476 | let len = Math.min(captures.length, captureIndices.length); 477 | let localStack: LocalStackElement[] = []; 478 | let maxEnd = captureIndices[0].end; 479 | 480 | for (let i = 0; i < len; i++) { 481 | let captureRule = captures[i]; 482 | if (captureRule === null) { 483 | // Not interested 484 | continue; 485 | } 486 | 487 | let captureIndex = captureIndices[i]; 488 | 489 | if (captureIndex.length === 0) { 490 | // Nothing really captured 491 | continue; 492 | } 493 | 494 | if (captureIndex.start > maxEnd) { 495 | // Capture going beyond consumed string 496 | break; 497 | } 498 | 499 | // pop captures while needed 500 | while (localStack.length > 0 && localStack[localStack.length - 1].endPos <= captureIndex.start) { 501 | // pop! 502 | lineTokens.produceFromScopes(localStack[localStack.length - 1].scopes, localStack[localStack.length - 1].endPos); 503 | localStack.pop(); 504 | } 505 | 506 | if (localStack.length > 0) { 507 | lineTokens.produceFromScopes(localStack[localStack.length - 1].scopes, captureIndex.start); 508 | } else { 509 | lineTokens.produce(stack, captureIndex.start); 510 | } 511 | 512 | if (captureRule.retokenizeCapturedWithRuleId) { 513 | // the capture requires additional matching 514 | let scopeName = captureRule.getName(getString(lineText), captureIndices); 515 | let nameScopesList = stack.contentNameScopesList.push(grammar, scopeName); 516 | let contentName = captureRule.getContentName(getString(lineText), captureIndices); 517 | let contentNameScopesList = nameScopesList.push(grammar, contentName); 518 | 519 | let stackClone = stack.push(captureRule.retokenizeCapturedWithRuleId, captureIndex.start, null, nameScopesList, contentNameScopesList); 520 | _tokenizeString(grammar, 521 | createOnigString( 522 | getString(lineText).substring(0, captureIndex.end) 523 | ), 524 | (isFirstLine && captureIndex.start === 0), captureIndex.start, stackClone, lineTokens 525 | ); 526 | continue; 527 | } 528 | 529 | let captureRuleScopeName = captureRule.getName(getString(lineText), captureIndices); 530 | if (captureRuleScopeName !== null) { 531 | // push 532 | let base = localStack.length > 0 ? localStack[localStack.length - 1].scopes : stack.contentNameScopesList; 533 | let captureRuleScopesList = base.push(grammar, captureRuleScopeName); 534 | localStack.push(new LocalStackElement(captureRuleScopesList, captureIndex.end)); 535 | } 536 | } 537 | 538 | while (localStack.length > 0) { 539 | // pop! 540 | lineTokens.produceFromScopes(localStack[localStack.length - 1].scopes, localStack[localStack.length - 1].endPos); 541 | localStack.pop(); 542 | } 543 | } 544 | 545 | interface IMatchInjectionsResult { 546 | readonly priorityMatch: boolean; 547 | readonly captureIndices: IOnigCaptureIndex[]; 548 | readonly matchedRuleId: number; 549 | } 550 | 551 | function debugCompiledRuleToString(ruleScanner: ICompiledRule): string { 552 | let r: string[] = []; 553 | for (let i = 0, len = ruleScanner.rules.length; i < len; i++) { 554 | r.push(' - ' + ruleScanner.rules[i] + ': ' + ruleScanner.debugRegExps[i]); 555 | } 556 | return r.join('\n'); 557 | } 558 | 559 | function matchInjections(injections: Injection[], grammar: Grammar, lineText: OnigString, isFirstLine: boolean, linePos: number, stack: StackElement, anchorPosition: number): IMatchInjectionsResult { 560 | // The lower the better 561 | let bestMatchRating = Number.MAX_VALUE; 562 | let bestMatchCaptureIndices: IOnigCaptureIndex[] = null; 563 | let bestMatchRuleId: number; 564 | let bestMatchResultPriority: number = 0; 565 | 566 | let scopes = stack.contentNameScopesList.generateScopes(); 567 | 568 | for (let i = 0, len = injections.length; i < len; i++) { 569 | let injection = injections[i]; 570 | if (!injection.matcher(scopes)) { 571 | // injection selector doesn't match stack 572 | continue; 573 | } 574 | let ruleScanner = grammar.getRule(injection.ruleId).compile(grammar, null, isFirstLine, linePos === anchorPosition); 575 | let matchResult = ruleScanner.scanner.findNextMatchSync(lineText, linePos); 576 | if (IN_DEBUG_MODE) { 577 | console.log(' scanning for injections'); 578 | console.log(debugCompiledRuleToString(ruleScanner)); 579 | } 580 | 581 | if (!matchResult) { 582 | continue; 583 | } 584 | 585 | let matchRating = matchResult.captureIndices[0].start; 586 | if (matchRating >= bestMatchRating) { 587 | // Injections are sorted by priority, so the previous injection had a better or equal priority 588 | continue; 589 | } 590 | 591 | bestMatchRating = matchRating; 592 | bestMatchCaptureIndices = matchResult.captureIndices; 593 | bestMatchRuleId = ruleScanner.rules[matchResult.index]; 594 | bestMatchResultPriority = injection.priority; 595 | 596 | if (bestMatchRating === linePos) { 597 | // No more need to look at the rest of the injections. 598 | break; 599 | } 600 | } 601 | 602 | if (bestMatchCaptureIndices) { 603 | return { 604 | priorityMatch: bestMatchResultPriority === -1, 605 | captureIndices: bestMatchCaptureIndices, 606 | matchedRuleId: bestMatchRuleId 607 | }; 608 | } 609 | 610 | return null; 611 | } 612 | 613 | interface IMatchResult { 614 | readonly captureIndices: IOnigCaptureIndex[]; 615 | readonly matchedRuleId: number; 616 | } 617 | 618 | function matchRule(grammar: Grammar, lineText: OnigString, isFirstLine: boolean, linePos: number, stack: StackElement, anchorPosition: number): IMatchResult { 619 | let rule = stack.getRule(grammar); 620 | let ruleScanner = rule.compile(grammar, stack.endRule, isFirstLine, linePos === anchorPosition); 621 | let r = ruleScanner.scanner.findNextMatchSync(lineText, linePos); 622 | if (IN_DEBUG_MODE) { 623 | console.log(' scanning for'); 624 | console.log(debugCompiledRuleToString(ruleScanner)); 625 | } 626 | 627 | if (r) { 628 | return { 629 | captureIndices: r.captureIndices, 630 | matchedRuleId: ruleScanner.rules[r.index] 631 | }; 632 | } 633 | return null; 634 | } 635 | 636 | function matchRuleOrInjections(grammar: Grammar, lineText: OnigString, isFirstLine: boolean, linePos: number, stack: StackElement, anchorPosition: number): IMatchResult { 637 | // Look for normal grammar rule 638 | let matchResult = matchRule(grammar, lineText, isFirstLine, linePos, stack, anchorPosition); 639 | 640 | // Look for injected rules 641 | let injections = grammar.getInjections(); 642 | if (injections.length === 0) { 643 | // No injections whatsoever => early return 644 | return matchResult; 645 | } 646 | 647 | let injectionResult = matchInjections(injections, grammar, lineText, isFirstLine, linePos, stack, anchorPosition); 648 | if (!injectionResult) { 649 | // No injections matched => early return 650 | return matchResult; 651 | } 652 | 653 | if (!matchResult) { 654 | // Only injections matched => early return 655 | return injectionResult; 656 | } 657 | 658 | // Decide if `matchResult` or `injectionResult` should win 659 | let matchResultScore = matchResult.captureIndices[0].start; 660 | let injectionResultScore = injectionResult.captureIndices[0].start; 661 | 662 | if (injectionResultScore < matchResultScore || (injectionResult.priorityMatch && injectionResultScore === matchResultScore)) { 663 | // injection won! 664 | return injectionResult; 665 | } 666 | return matchResult; 667 | } 668 | 669 | interface IWhileStack { 670 | readonly stack: StackElement; 671 | readonly rule: BeginWhileRule; 672 | } 673 | 674 | interface IWhileCheckResult { 675 | readonly stack: StackElement; 676 | readonly linePos: number; 677 | readonly anchorPosition: number; 678 | readonly isFirstLine: boolean; 679 | } 680 | 681 | /** 682 | * Walk the stack from bottom to top, and check each while condition in this order. 683 | * If any fails, cut off the entire stack above the failed while condition. While conditions 684 | * may also advance the linePosition. 685 | */ 686 | function _checkWhileConditions(grammar: Grammar, lineText: OnigString, isFirstLine: boolean, linePos: number, stack: StackElement, lineTokens: LineTokens): IWhileCheckResult { 687 | let anchorPosition = -1; 688 | let whileRules: IWhileStack[] = []; 689 | for (let node = stack; node; node = node.pop()) { 690 | let nodeRule = node.getRule(grammar); 691 | if (nodeRule instanceof BeginWhileRule) { 692 | whileRules.push({ 693 | rule: nodeRule, 694 | stack: node 695 | }); 696 | } 697 | } 698 | 699 | for (let whileRule = whileRules.pop(); whileRule; whileRule = whileRules.pop()) { 700 | let ruleScanner = whileRule.rule.compileWhile(grammar, whileRule.stack.endRule, isFirstLine, anchorPosition === linePos); 701 | let r = ruleScanner.scanner.findNextMatchSync(lineText, linePos); 702 | if (IN_DEBUG_MODE) { 703 | console.log(' scanning for while rule'); 704 | console.log(debugCompiledRuleToString(ruleScanner)); 705 | } 706 | 707 | if (r) { 708 | let matchedRuleId = ruleScanner.rules[r.index]; 709 | if (matchedRuleId !== -2) { 710 | // we shouldn't end up here 711 | stack = whileRule.stack.pop(); 712 | break; 713 | } 714 | if (r.captureIndices && r.captureIndices.length) { 715 | lineTokens.produce(whileRule.stack, r.captureIndices[0].start); 716 | handleCaptures(grammar, lineText, isFirstLine, whileRule.stack, lineTokens, whileRule.rule.whileCaptures, r.captureIndices); 717 | lineTokens.produce(whileRule.stack, r.captureIndices[0].end); 718 | anchorPosition = r.captureIndices[0].end; 719 | if (r.captureIndices[0].end > linePos) { 720 | linePos = r.captureIndices[0].end; 721 | isFirstLine = false; 722 | } 723 | } 724 | } else { 725 | stack = whileRule.stack.pop(); 726 | break; 727 | } 728 | } 729 | 730 | return { stack: stack, linePos: linePos, anchorPosition: anchorPosition, isFirstLine: isFirstLine }; 731 | } 732 | 733 | function _tokenizeString(grammar: Grammar, lineText: OnigString, isFirstLine: boolean, linePos: number, stack: StackElement, lineTokens: LineTokens): StackElement { 734 | const lineLength = getString(lineText).length; 735 | 736 | let STOP = false; 737 | 738 | let whileCheckResult = _checkWhileConditions(grammar, lineText, isFirstLine, linePos, stack, lineTokens); 739 | stack = whileCheckResult.stack; 740 | linePos = whileCheckResult.linePos; 741 | isFirstLine = whileCheckResult.isFirstLine; 742 | let anchorPosition = whileCheckResult.anchorPosition; 743 | 744 | while (!STOP) { 745 | scanNext(); // potentially modifies linePos && anchorPosition 746 | } 747 | 748 | function scanNext(): void { 749 | if (IN_DEBUG_MODE) { 750 | console.log(''); 751 | console.log('@@scanNext: |' + getString(lineText).replace(/\n$/, '\\n').substr(linePos) + '|'); 752 | } 753 | let r = matchRuleOrInjections(grammar, lineText, isFirstLine, linePos, stack, anchorPosition); 754 | 755 | if (!r) { 756 | if (IN_DEBUG_MODE) { 757 | console.log(' no more matches.'); 758 | } 759 | // No match 760 | lineTokens.produce(stack, lineLength); 761 | STOP = true; 762 | return; 763 | } 764 | 765 | let captureIndices: IOnigCaptureIndex[] = r.captureIndices; 766 | let matchedRuleId: number = r.matchedRuleId; 767 | 768 | let hasAdvanced = (captureIndices && captureIndices.length > 0) ? (captureIndices[0].end > linePos) : false; 769 | 770 | if (matchedRuleId === -1) { 771 | // We matched the `end` for this rule => pop it 772 | let poppedRule = stack.getRule(grammar); 773 | 774 | if (IN_DEBUG_MODE) { 775 | console.log(' popping ' + poppedRule.debugName + ' - ' + poppedRule.debugEndRegExp); 776 | } 777 | 778 | lineTokens.produce(stack, captureIndices[0].start); 779 | stack = stack.setContentNameScopesList(stack.nameScopesList); 780 | handleCaptures(grammar, lineText, isFirstLine, stack, lineTokens, poppedRule.endCaptures, captureIndices); 781 | lineTokens.produce(stack, captureIndices[0].end); 782 | 783 | // pop 784 | let popped = stack; 785 | stack = stack.pop(); 786 | 787 | if (!hasAdvanced && popped.getEnterPos() === linePos) { 788 | // Grammar pushed & popped a rule without advancing 789 | console.error('[1] - Grammar is in an endless loop - Grammar pushed & popped a rule without advancing'); 790 | 791 | // See https://github.com/Microsoft/vscode-textmate/issues/12 792 | // Let's assume this was a mistake by the grammar author and the intent was to continue in this state 793 | stack = popped; 794 | 795 | lineTokens.produce(stack, lineLength); 796 | STOP = true; 797 | return; 798 | } 799 | } else { 800 | // We matched a rule! 801 | let _rule = grammar.getRule(matchedRuleId); 802 | 803 | lineTokens.produce(stack, captureIndices[0].start); 804 | 805 | let beforePush = stack; 806 | // push it on the stack rule 807 | let scopeName = _rule.getName(getString(lineText), captureIndices); 808 | let nameScopesList = stack.contentNameScopesList.push(grammar, scopeName); 809 | stack = stack.push(matchedRuleId, linePos, null, nameScopesList, nameScopesList); 810 | 811 | if (_rule instanceof BeginEndRule) { 812 | let pushedRule = _rule; 813 | if (IN_DEBUG_MODE) { 814 | console.log(' pushing ' + pushedRule.debugName + ' - ' + pushedRule.debugBeginRegExp); 815 | } 816 | 817 | handleCaptures(grammar, lineText, isFirstLine, stack, lineTokens, pushedRule.beginCaptures, captureIndices); 818 | lineTokens.produce(stack, captureIndices[0].end); 819 | anchorPosition = captureIndices[0].end; 820 | 821 | let contentName = pushedRule.getContentName(getString(lineText), captureIndices); 822 | let contentNameScopesList = nameScopesList.push(grammar, contentName); 823 | stack = stack.setContentNameScopesList(contentNameScopesList); 824 | 825 | if (pushedRule.endHasBackReferences) { 826 | stack = stack.setEndRule(pushedRule.getEndWithResolvedBackReferences(getString(lineText), captureIndices)); 827 | } 828 | 829 | if (!hasAdvanced && beforePush.hasSameRuleAs(stack)) { 830 | // Grammar pushed the same rule without advancing 831 | console.error('[2] - Grammar is in an endless loop - Grammar pushed the same rule without advancing'); 832 | stack = stack.pop(); 833 | lineTokens.produce(stack, lineLength); 834 | STOP = true; 835 | return; 836 | } 837 | } else if (_rule instanceof BeginWhileRule) { 838 | let pushedRule = _rule; 839 | if (IN_DEBUG_MODE) { 840 | console.log(' pushing ' + pushedRule.debugName); 841 | } 842 | 843 | handleCaptures(grammar, lineText, isFirstLine, stack, lineTokens, pushedRule.beginCaptures, captureIndices); 844 | lineTokens.produce(stack, captureIndices[0].end); 845 | anchorPosition = captureIndices[0].end; 846 | let contentName = pushedRule.getContentName(getString(lineText), captureIndices); 847 | let contentNameScopesList = nameScopesList.push(grammar, contentName); 848 | stack = stack.setContentNameScopesList(contentNameScopesList); 849 | 850 | if (pushedRule.whileHasBackReferences) { 851 | stack = stack.setEndRule(pushedRule.getWhileWithResolvedBackReferences(getString(lineText), captureIndices)); 852 | } 853 | 854 | if (!hasAdvanced && beforePush.hasSameRuleAs(stack)) { 855 | // Grammar pushed the same rule without advancing 856 | console.error('[3] - Grammar is in an endless loop - Grammar pushed the same rule without advancing'); 857 | stack = stack.pop(); 858 | lineTokens.produce(stack, lineLength); 859 | STOP = true; 860 | return; 861 | } 862 | } else { 863 | let matchingRule = _rule; 864 | if (IN_DEBUG_MODE) { 865 | console.log(' matched ' + matchingRule.debugName + ' - ' + matchingRule.debugMatchRegExp); 866 | } 867 | 868 | handleCaptures(grammar, lineText, isFirstLine, stack, lineTokens, matchingRule.captures, captureIndices); 869 | lineTokens.produce(stack, captureIndices[0].end); 870 | 871 | // pop rule immediately since it is a MatchRule 872 | stack = stack.pop(); 873 | 874 | if (!hasAdvanced) { 875 | // Grammar is not advancing, nor is it pushing/popping 876 | console.error('[4] - Grammar is in an endless loop - Grammar is not advancing, nor is it pushing/popping'); 877 | stack = stack.safePop(); 878 | lineTokens.produce(stack, lineLength); 879 | STOP = true; 880 | return; 881 | } 882 | } 883 | } 884 | 885 | if (captureIndices[0].end > linePos) { 886 | // Advance stream 887 | linePos = captureIndices[0].end; 888 | isFirstLine = false; 889 | } 890 | } 891 | 892 | return stack; 893 | } 894 | 895 | 896 | export class StackElementMetadata { 897 | 898 | public static toBinaryStr(metadata: number): string { 899 | let r = metadata.toString(2); 900 | while (r.length < 32) { 901 | r = '0' + r; 902 | } 903 | return r; 904 | } 905 | 906 | public static printMetadata(metadata: number): void { 907 | let languageId = StackElementMetadata.getLanguageId(metadata); 908 | let tokenType = StackElementMetadata.getTokenType(metadata); 909 | let fontStyle = StackElementMetadata.getFontStyle(metadata); 910 | let foreground = StackElementMetadata.getForeground(metadata); 911 | let background = StackElementMetadata.getBackground(metadata); 912 | 913 | console.log({ 914 | languageId: languageId, 915 | tokenType: tokenType, 916 | fontStyle: fontStyle, 917 | foreground: foreground, 918 | background: background, 919 | }); 920 | } 921 | 922 | public static getLanguageId(metadata: number): number { 923 | return (metadata & MetadataConsts.LANGUAGEID_MASK) >>> MetadataConsts.LANGUAGEID_OFFSET; 924 | } 925 | 926 | public static getTokenType(metadata: number): number { 927 | return (metadata & MetadataConsts.TOKEN_TYPE_MASK) >>> MetadataConsts.TOKEN_TYPE_OFFSET; 928 | } 929 | 930 | public static getFontStyle(metadata: number): number { 931 | return (metadata & MetadataConsts.FONT_STYLE_MASK) >>> MetadataConsts.FONT_STYLE_OFFSET; 932 | } 933 | 934 | public static getForeground(metadata: number): number { 935 | return (metadata & MetadataConsts.FOREGROUND_MASK) >>> MetadataConsts.FOREGROUND_OFFSET; 936 | } 937 | 938 | public static getBackground(metadata: number): number { 939 | return (metadata & MetadataConsts.BACKGROUND_MASK) >>> MetadataConsts.BACKGROUND_OFFSET; 940 | } 941 | 942 | public static set(metadata: number, languageId: number, tokenType: TemporaryStandardTokenType, fontStyle: FontStyle, foreground: number, background: number): number { 943 | let _languageId = StackElementMetadata.getLanguageId(metadata); 944 | let _tokenType = StackElementMetadata.getTokenType(metadata); 945 | let _fontStyle = StackElementMetadata.getFontStyle(metadata); 946 | let _foreground = StackElementMetadata.getForeground(metadata); 947 | let _background = StackElementMetadata.getBackground(metadata); 948 | 949 | if (languageId !== 0) { 950 | _languageId = languageId; 951 | } 952 | if (tokenType !== TemporaryStandardTokenType.Other) { 953 | _tokenType = tokenType === TemporaryStandardTokenType.MetaEmbedded ? StandardTokenType.Other : tokenType; 954 | } 955 | if (fontStyle !== FontStyle.NotSet) { 956 | _fontStyle = fontStyle; 957 | } 958 | if (foreground !== 0) { 959 | _foreground = foreground; 960 | } 961 | if (background !== 0) { 962 | _background = background; 963 | } 964 | 965 | return ( 966 | (_languageId << MetadataConsts.LANGUAGEID_OFFSET) 967 | | (_tokenType << MetadataConsts.TOKEN_TYPE_OFFSET) 968 | | (_fontStyle << MetadataConsts.FONT_STYLE_OFFSET) 969 | | (_foreground << MetadataConsts.FOREGROUND_OFFSET) 970 | | (_background << MetadataConsts.BACKGROUND_OFFSET) 971 | ) >>> 0; 972 | } 973 | } 974 | 975 | export class ScopeListElement { 976 | _scopeListElementBrand: void; 977 | 978 | public readonly parent: ScopeListElement; 979 | public readonly scope: string; 980 | public readonly metadata: number; 981 | 982 | constructor(parent: ScopeListElement, scope: string, metadata: number) { 983 | this.parent = parent; 984 | this.scope = scope; 985 | this.metadata = metadata; 986 | } 987 | 988 | private static _equals(a: ScopeListElement, b: ScopeListElement): boolean { 989 | do { 990 | if (a === b) { 991 | return true; 992 | } 993 | 994 | if (a.scope !== b.scope || a.metadata !== b.metadata) { 995 | return false; 996 | } 997 | 998 | // Go to previous pair 999 | a = a.parent; 1000 | b = b.parent; 1001 | 1002 | if (!a && !b) { 1003 | // End of list reached for both 1004 | return true; 1005 | } 1006 | 1007 | if (!a || !b) { 1008 | // End of list reached only for one 1009 | return false; 1010 | } 1011 | 1012 | } while (true); 1013 | } 1014 | 1015 | public equals(other: ScopeListElement): boolean { 1016 | return ScopeListElement._equals(this, other); 1017 | } 1018 | 1019 | private static _matchesScope(scope: string, selector: string, selectorWithDot: string): boolean { 1020 | return (selector === scope || scope.substring(0, selectorWithDot.length) === selectorWithDot); 1021 | } 1022 | 1023 | private static _matches(target: ScopeListElement, parentScopes: string[]): boolean { 1024 | if (parentScopes === null) { 1025 | return true; 1026 | } 1027 | 1028 | let len = parentScopes.length; 1029 | let index = 0; 1030 | let selector = parentScopes[index]; 1031 | let selectorWithDot = selector + '.'; 1032 | 1033 | while (target) { 1034 | if (this._matchesScope(target.scope, selector, selectorWithDot)) { 1035 | index++; 1036 | if (index === len) { 1037 | return true; 1038 | } 1039 | selector = parentScopes[index]; 1040 | selectorWithDot = selector + '.'; 1041 | } 1042 | target = target.parent; 1043 | } 1044 | 1045 | return false; 1046 | } 1047 | 1048 | public static mergeMetadata(metadata: number, scopesList: ScopeListElement, source: ScopeMetadata): number { 1049 | if (source === null) { 1050 | return metadata; 1051 | } 1052 | 1053 | let fontStyle = FontStyle.NotSet; 1054 | let foreground = 0; 1055 | let background = 0; 1056 | 1057 | if (source.themeData !== null) { 1058 | // Find the first themeData that matches 1059 | for (let i = 0, len = source.themeData.length; i < len; i++) { 1060 | let themeData = source.themeData[i]; 1061 | 1062 | if (this._matches(scopesList, themeData.parentScopes)) { 1063 | fontStyle = themeData.fontStyle; 1064 | foreground = themeData.foreground; 1065 | background = themeData.background; 1066 | break; 1067 | } 1068 | } 1069 | } 1070 | 1071 | return StackElementMetadata.set(metadata, source.languageId, source.tokenType, fontStyle, foreground, background); 1072 | } 1073 | 1074 | private static _push(target: ScopeListElement, grammar: Grammar, scopes: string[]): ScopeListElement { 1075 | for (let i = 0, len = scopes.length; i < len; i++) { 1076 | let scope = scopes[i]; 1077 | let rawMetadata = grammar.getMetadataForScope(scope); 1078 | let metadata = ScopeListElement.mergeMetadata(target.metadata, target, rawMetadata); 1079 | target = new ScopeListElement(target, scope, metadata); 1080 | } 1081 | return target; 1082 | } 1083 | 1084 | public push(grammar: Grammar, scope: string): ScopeListElement { 1085 | if (scope === null) { 1086 | return this; 1087 | } 1088 | if (scope.indexOf(' ') >= 0) { 1089 | // there are multiple scopes to push 1090 | return ScopeListElement._push(this, grammar, scope.split(/ /g)); 1091 | } 1092 | // there is a single scope to push 1093 | return ScopeListElement._push(this, grammar, [scope]); 1094 | } 1095 | 1096 | private static _generateScopes(scopesList: ScopeListElement): string[] { 1097 | let result: string[] = [], resultLen = 0; 1098 | while (scopesList) { 1099 | result[resultLen++] = scopesList.scope; 1100 | scopesList = scopesList.parent; 1101 | } 1102 | result.reverse(); 1103 | return result; 1104 | } 1105 | 1106 | public generateScopes(): string[] { 1107 | return ScopeListElement._generateScopes(this); 1108 | } 1109 | } 1110 | 1111 | /** 1112 | * Represents a "pushed" state on the stack (as a linked list element). 1113 | */ 1114 | export class StackElement implements StackElementDef { 1115 | _stackElementBrand: void; 1116 | 1117 | public static NULL = new StackElement(null, 0, 0, null, null, null); 1118 | 1119 | /** 1120 | * The position on the current line where this state was pushed. 1121 | * This is relevant only while tokenizing a line, to detect endless loops. 1122 | * Its value is meaningless across lines. 1123 | */ 1124 | private _enterPos: number; 1125 | 1126 | /** 1127 | * The previous state on the stack (or null for the root state). 1128 | */ 1129 | public readonly parent: StackElement; 1130 | /** 1131 | * The depth of the stack. 1132 | */ 1133 | public readonly depth: number; 1134 | 1135 | /** 1136 | * The state (rule) that this element represents. 1137 | */ 1138 | public readonly ruleId: number; 1139 | /** 1140 | * The "pop" (end) condition for this state in case that it was dynamically generated through captured text. 1141 | */ 1142 | public readonly endRule: string; 1143 | /** 1144 | * The list of scopes containing the "name" for this state. 1145 | */ 1146 | public readonly nameScopesList: ScopeListElement; 1147 | /** 1148 | * The list of scopes containing the "contentName" (besides "name") for this state. 1149 | * This list **must** contain as an element `scopeName`. 1150 | */ 1151 | public readonly contentNameScopesList: ScopeListElement; 1152 | 1153 | constructor(parent: StackElement, ruleId: number, enterPos: number, endRule: string, nameScopesList: ScopeListElement, contentNameScopesList: ScopeListElement) { 1154 | this.parent = parent; 1155 | this.depth = (this.parent ? this.parent.depth + 1 : 1); 1156 | this.ruleId = ruleId; 1157 | this._enterPos = enterPos; 1158 | this.endRule = endRule; 1159 | this.nameScopesList = nameScopesList; 1160 | this.contentNameScopesList = contentNameScopesList; 1161 | } 1162 | 1163 | /** 1164 | * A structural equals check. Does not take into account `scopes`. 1165 | */ 1166 | private static _structuralEquals(a: StackElement, b: StackElement): boolean { 1167 | do { 1168 | if (a === b) { 1169 | return true; 1170 | } 1171 | 1172 | if (a.depth !== b.depth || a.ruleId !== b.ruleId || a.endRule !== b.endRule) { 1173 | return false; 1174 | } 1175 | 1176 | // Go to previous pair 1177 | a = a.parent; 1178 | b = b.parent; 1179 | 1180 | if (!a && !b) { 1181 | // End of list reached for both 1182 | return true; 1183 | } 1184 | 1185 | if (!a || !b) { 1186 | // End of list reached only for one 1187 | return false; 1188 | } 1189 | 1190 | } while (true); 1191 | } 1192 | 1193 | private static _equals(a: StackElement, b: StackElement): boolean { 1194 | if (a === b) { 1195 | return true; 1196 | } 1197 | if (!this._structuralEquals(a, b)) { 1198 | return false; 1199 | } 1200 | return a.contentNameScopesList.equals(b.contentNameScopesList); 1201 | } 1202 | 1203 | public clone(): StackElement { 1204 | return this; 1205 | } 1206 | 1207 | public equals(other: StackElement): boolean { 1208 | if (other === null) { 1209 | return false; 1210 | } 1211 | return StackElement._equals(this, other); 1212 | } 1213 | 1214 | private static _reset(el: StackElement): void { 1215 | while (el) { 1216 | el._enterPos = -1; 1217 | el = el.parent; 1218 | } 1219 | } 1220 | 1221 | public reset(): void { 1222 | StackElement._reset(this); 1223 | } 1224 | 1225 | public pop(): StackElement { 1226 | return this.parent; 1227 | } 1228 | 1229 | public safePop(): StackElement { 1230 | if (this.parent) { 1231 | return this.parent; 1232 | } 1233 | return this; 1234 | } 1235 | 1236 | public push(ruleId: number, enterPos: number, endRule: string, nameScopesList: ScopeListElement, contentNameScopesList: ScopeListElement): StackElement { 1237 | return new StackElement(this, ruleId, enterPos, endRule, nameScopesList, contentNameScopesList); 1238 | } 1239 | 1240 | public getEnterPos(): number { 1241 | return this._enterPos; 1242 | } 1243 | 1244 | public getRule(grammar: IRuleRegistry): Rule { 1245 | return grammar.getRule(this.ruleId); 1246 | } 1247 | 1248 | private _writeString(res: string[], outIndex: number): number { 1249 | if (this.parent) { 1250 | outIndex = this.parent._writeString(res, outIndex); 1251 | } 1252 | 1253 | res[outIndex++] = `(${this.ruleId}, TODO-${this.nameScopesList}, TODO-${this.contentNameScopesList})`; 1254 | 1255 | return outIndex; 1256 | } 1257 | 1258 | public toString(): string { 1259 | let r: string[] = []; 1260 | this._writeString(r, 0); 1261 | return '[' + r.join(',') + ']'; 1262 | } 1263 | 1264 | public setContentNameScopesList(contentNameScopesList: ScopeListElement): StackElement { 1265 | if (this.contentNameScopesList === contentNameScopesList) { 1266 | return this; 1267 | } 1268 | return this.parent.push(this.ruleId, this._enterPos, this.endRule, this.nameScopesList, contentNameScopesList); 1269 | } 1270 | 1271 | public setEndRule(endRule: string): StackElement { 1272 | if (this.endRule === endRule) { 1273 | return this; 1274 | } 1275 | return new StackElement(this.parent, this.ruleId, this._enterPos, endRule, this.nameScopesList, this.contentNameScopesList); 1276 | } 1277 | 1278 | public hasSameRuleAs(other: StackElement): boolean { 1279 | return this.ruleId === other.ruleId; 1280 | } 1281 | } 1282 | 1283 | export class LocalStackElement { 1284 | public readonly scopes: ScopeListElement; 1285 | public readonly endPos: number; 1286 | 1287 | constructor(scopes: ScopeListElement, endPos: number) { 1288 | this.scopes = scopes; 1289 | this.endPos = endPos; 1290 | } 1291 | } 1292 | 1293 | interface TokenTypeMatcher { 1294 | readonly matcher: Matcher; 1295 | readonly type: StandardTokenType; 1296 | } 1297 | 1298 | class LineTokens { 1299 | 1300 | private readonly _emitBinaryTokens: boolean; 1301 | /** 1302 | * defined only if `IN_DEBUG_MODE`. 1303 | */ 1304 | private readonly _lineText: string; 1305 | /** 1306 | * used only if `_emitBinaryTokens` is false. 1307 | */ 1308 | private readonly _tokens: IToken[]; 1309 | /** 1310 | * used only if `_emitBinaryTokens` is true. 1311 | */ 1312 | private readonly _binaryTokens: number[]; 1313 | 1314 | private _lastTokenEndIndex: number; 1315 | 1316 | private readonly _tokenTypeOverrides: TokenTypeMatcher[]; 1317 | 1318 | constructor(emitBinaryTokens: boolean, lineText: string, tokenTypeOverrides: TokenTypeMatcher[]) { 1319 | this._emitBinaryTokens = emitBinaryTokens; 1320 | this._tokenTypeOverrides = tokenTypeOverrides; 1321 | if (IN_DEBUG_MODE) { 1322 | this._lineText = lineText; 1323 | } 1324 | if (this._emitBinaryTokens) { 1325 | this._binaryTokens = []; 1326 | } else { 1327 | this._tokens = []; 1328 | } 1329 | this._lastTokenEndIndex = 0; 1330 | } 1331 | 1332 | public produce(stack: StackElement, endIndex: number): void { 1333 | this.produceFromScopes(stack.contentNameScopesList, endIndex); 1334 | } 1335 | 1336 | public produceFromScopes(scopesList: ScopeListElement, endIndex: number): void { 1337 | if (this._lastTokenEndIndex >= endIndex) { 1338 | return; 1339 | } 1340 | 1341 | if (this._emitBinaryTokens) { 1342 | let metadata = scopesList.metadata; 1343 | 1344 | for (const tokenType of this._tokenTypeOverrides) { 1345 | if (tokenType.matcher(scopesList.generateScopes())) { 1346 | metadata = StackElementMetadata.set(metadata, 0, toTemporaryType(tokenType.type), FontStyle.NotSet, 0, 0); 1347 | } 1348 | } 1349 | 1350 | if (this._binaryTokens.length > 0 && this._binaryTokens[this._binaryTokens.length - 1] === metadata) { 1351 | // no need to push a token with the same metadata 1352 | this._lastTokenEndIndex = endIndex; 1353 | return; 1354 | } 1355 | 1356 | this._binaryTokens.push(this._lastTokenEndIndex); 1357 | this._binaryTokens.push(metadata); 1358 | 1359 | this._lastTokenEndIndex = endIndex; 1360 | return; 1361 | } 1362 | 1363 | let scopes = scopesList.generateScopes(); 1364 | 1365 | if (IN_DEBUG_MODE) { 1366 | console.log(' token: |' + this._lineText.substring(this._lastTokenEndIndex, endIndex).replace(/\n$/, '\\n') + '|'); 1367 | for (var k = 0; k < scopes.length; k++) { 1368 | console.log(' * ' + scopes[k]); 1369 | } 1370 | } 1371 | 1372 | this._tokens.push({ 1373 | startIndex: this._lastTokenEndIndex, 1374 | endIndex: endIndex, 1375 | // value: lineText.substring(lastTokenEndIndex, endIndex), 1376 | scopes: scopes 1377 | }); 1378 | 1379 | this._lastTokenEndIndex = endIndex; 1380 | } 1381 | 1382 | public getResult(stack: StackElement, lineLength: number): IToken[] { 1383 | if (this._tokens.length > 0 && this._tokens[this._tokens.length - 1].startIndex === lineLength - 1) { 1384 | // pop produced token for newline 1385 | this._tokens.pop(); 1386 | } 1387 | 1388 | if (this._tokens.length === 0) { 1389 | this._lastTokenEndIndex = -1; 1390 | this.produce(stack, lineLength); 1391 | this._tokens[this._tokens.length - 1].startIndex = 0; 1392 | } 1393 | 1394 | return this._tokens; 1395 | } 1396 | 1397 | public getBinaryResult(stack: StackElement, lineLength: number): Uint32Array { 1398 | if (this._binaryTokens.length > 0 && this._binaryTokens[this._binaryTokens.length - 2] === lineLength - 1) { 1399 | // pop produced token for newline 1400 | this._binaryTokens.pop(); 1401 | this._binaryTokens.pop(); 1402 | } 1403 | 1404 | if (this._binaryTokens.length === 0) { 1405 | this._lastTokenEndIndex = -1; 1406 | this.produce(stack, lineLength); 1407 | this._binaryTokens[this._binaryTokens.length - 2] = 0; 1408 | } 1409 | 1410 | let result = new Uint32Array(this._binaryTokens.length); 1411 | for (let i = 0, len = this._binaryTokens.length; i < len; i++) { 1412 | result[i] = this._binaryTokens[i]; 1413 | } 1414 | 1415 | return result; 1416 | } 1417 | } 1418 | 1419 | function toTemporaryType(standardType: StandardTokenType): TemporaryStandardTokenType { 1420 | switch (standardType) { 1421 | case StandardTokenType.RegEx: 1422 | return TemporaryStandardTokenType.RegEx; 1423 | case StandardTokenType.String: 1424 | return TemporaryStandardTokenType.String; 1425 | case StandardTokenType.Comment: 1426 | return TemporaryStandardTokenType.Comment; 1427 | case StandardTokenType.Other: 1428 | default: 1429 | // `MetaEmbedded` is the same scope as `Other` 1430 | // but it overwrites existing token types in the stack. 1431 | return TemporaryStandardTokenType.MetaEmbedded; 1432 | } 1433 | } -------------------------------------------------------------------------------- /src/grammarReader.ts: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------- 2 | * Copyright (C) Microsoft Corporation. All rights reserved. 3 | *--------------------------------------------------------*/ 4 | 5 | import { IRawGrammar } from './types'; 6 | import * as plist from 'fast-plist'; 7 | import { CAPTURE_METADATA } from './debug'; 8 | import { parse as manualParseJSON } from './json'; 9 | 10 | export function parseJSONGrammar(contents: string, filename: string): IRawGrammar { 11 | if (CAPTURE_METADATA) { 12 | return manualParseJSON(contents, filename, true); 13 | } 14 | return JSON.parse(contents); 15 | } 16 | 17 | export function parsePLISTGrammar(contents: string, filename: string): IRawGrammar { 18 | if (CAPTURE_METADATA) { 19 | return plist.parseWithLocation(contents, filename, '$vscodeTextmateLocation'); 20 | } 21 | return plist.parse(contents); 22 | } 23 | -------------------------------------------------------------------------------- /src/json.ts: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------- 2 | * Copyright (C) Microsoft Corporation. All rights reserved. 3 | *--------------------------------------------------------*/ 4 | 5 | function doFail(streamState: JSONStreamState, msg: string): void { 6 | // console.log('Near offset ' + streamState.pos + ': ' + msg + ' ~~~' + streamState.source.substr(streamState.pos, 50) + '~~~'); 7 | throw new Error('Near offset ' + streamState.pos + ': ' + msg + ' ~~~' + streamState.source.substr(streamState.pos, 50) + '~~~'); 8 | } 9 | 10 | export interface ILocation { 11 | readonly filename: string; 12 | readonly line: number; 13 | readonly char: number; 14 | } 15 | 16 | export function parse(source: string, filename: string, withMetadata: boolean): any { 17 | let streamState = new JSONStreamState(source); 18 | let token = new JSONToken(); 19 | let state = JSONState.ROOT_STATE; 20 | let cur: any = null; 21 | let stateStack: JSONState[] = []; 22 | let objStack: any[] = []; 23 | 24 | function pushState(): void { 25 | stateStack.push(state); 26 | objStack.push(cur); 27 | } 28 | 29 | function popState(): void { 30 | state = stateStack.pop(); 31 | cur = objStack.pop(); 32 | } 33 | 34 | function fail(msg: string): void { 35 | doFail(streamState, msg); 36 | } 37 | 38 | while (nextJSONToken(streamState, token)) { 39 | 40 | if (state === JSONState.ROOT_STATE) { 41 | if (cur !== null) { 42 | fail('too many constructs in root'); 43 | } 44 | 45 | if (token.type === JSONTokenType.LEFT_CURLY_BRACKET) { 46 | cur = {}; 47 | if (withMetadata) { 48 | cur.$vscodeTextmateLocation = token.toLocation(filename); 49 | } 50 | pushState(); 51 | state = JSONState.DICT_STATE; 52 | continue; 53 | } 54 | 55 | if (token.type === JSONTokenType.LEFT_SQUARE_BRACKET) { 56 | cur = []; 57 | pushState(); 58 | state = JSONState.ARR_STATE; 59 | continue; 60 | } 61 | 62 | fail('unexpected token in root'); 63 | 64 | } 65 | 66 | if (state === JSONState.DICT_STATE_COMMA) { 67 | 68 | if (token.type === JSONTokenType.RIGHT_CURLY_BRACKET) { 69 | popState(); 70 | continue; 71 | } 72 | 73 | if (token.type === JSONTokenType.COMMA) { 74 | state = JSONState.DICT_STATE_NO_CLOSE; 75 | continue; 76 | } 77 | 78 | fail('expected , or }'); 79 | 80 | } 81 | 82 | if (state === JSONState.DICT_STATE || state === JSONState.DICT_STATE_NO_CLOSE) { 83 | 84 | if (state === JSONState.DICT_STATE && token.type === JSONTokenType.RIGHT_CURLY_BRACKET) { 85 | popState(); 86 | continue; 87 | } 88 | 89 | if (token.type === JSONTokenType.STRING) { 90 | let keyValue = token.value; 91 | 92 | if (!nextJSONToken(streamState, token) || (/*TS bug*/token.type) !== JSONTokenType.COLON) { 93 | fail('expected colon'); 94 | } 95 | if (!nextJSONToken(streamState, token)) { 96 | fail('expected value'); 97 | } 98 | 99 | state = JSONState.DICT_STATE_COMMA; 100 | 101 | if (token.type === JSONTokenType.STRING) { 102 | cur[keyValue] = token.value; 103 | continue; 104 | } 105 | if (token.type === JSONTokenType.NULL) { 106 | cur[keyValue] = null; 107 | continue; 108 | } 109 | if (token.type === JSONTokenType.TRUE) { 110 | cur[keyValue] = true; 111 | continue; 112 | } 113 | if (token.type === JSONTokenType.FALSE) { 114 | cur[keyValue] = false; 115 | continue; 116 | } 117 | if (token.type === JSONTokenType.NUMBER) { 118 | cur[keyValue] = parseFloat(token.value); 119 | continue; 120 | } 121 | if (token.type === JSONTokenType.LEFT_SQUARE_BRACKET) { 122 | let newArr: any[] = []; 123 | cur[keyValue] = newArr; 124 | pushState(); 125 | state = JSONState.ARR_STATE; 126 | cur = newArr; 127 | continue; 128 | } 129 | if (token.type === JSONTokenType.LEFT_CURLY_BRACKET) { 130 | let newDict: any = {}; 131 | if (withMetadata) { 132 | newDict.$vscodeTextmateLocation = token.toLocation(filename); 133 | } 134 | cur[keyValue] = newDict; 135 | pushState(); 136 | state = JSONState.DICT_STATE; 137 | cur = newDict; 138 | continue; 139 | } 140 | } 141 | 142 | fail('unexpected token in dict'); 143 | } 144 | 145 | if (state === JSONState.ARR_STATE_COMMA) { 146 | 147 | if (token.type === JSONTokenType.RIGHT_SQUARE_BRACKET) { 148 | popState(); 149 | continue; 150 | } 151 | 152 | if (token.type === JSONTokenType.COMMA) { 153 | state = JSONState.ARR_STATE_NO_CLOSE; 154 | continue; 155 | } 156 | 157 | fail('expected , or ]'); 158 | } 159 | 160 | if (state === JSONState.ARR_STATE || state === JSONState.ARR_STATE_NO_CLOSE) { 161 | 162 | if (state === JSONState.ARR_STATE && token.type === JSONTokenType.RIGHT_SQUARE_BRACKET) { 163 | popState(); 164 | continue; 165 | } 166 | 167 | state = JSONState.ARR_STATE_COMMA; 168 | 169 | if (token.type === JSONTokenType.STRING) { 170 | cur.push(token.value); 171 | continue; 172 | } 173 | if (token.type === JSONTokenType.NULL) { 174 | cur.push(null); 175 | continue; 176 | } 177 | if (token.type === JSONTokenType.TRUE) { 178 | cur.push(true); 179 | continue; 180 | } 181 | if (token.type === JSONTokenType.FALSE) { 182 | cur.push(false); 183 | continue; 184 | } 185 | if (token.type === JSONTokenType.NUMBER) { 186 | cur.push(parseFloat(token.value)); 187 | continue; 188 | } 189 | 190 | if (token.type === JSONTokenType.LEFT_SQUARE_BRACKET) { 191 | let newArr: any[] = []; 192 | cur.push(newArr); 193 | pushState(); 194 | state = JSONState.ARR_STATE; 195 | cur = newArr; 196 | continue; 197 | } 198 | if (token.type === JSONTokenType.LEFT_CURLY_BRACKET) { 199 | let newDict: any = {}; 200 | if (withMetadata) { 201 | newDict.$vscodeTextmateLocation = token.toLocation(filename); 202 | } 203 | cur.push(newDict); 204 | pushState(); 205 | state = JSONState.DICT_STATE; 206 | cur = newDict; 207 | continue; 208 | } 209 | 210 | fail('unexpected token in array'); 211 | } 212 | 213 | fail('unknown state'); 214 | } 215 | 216 | if (objStack.length !== 0) { 217 | fail('unclosed constructs'); 218 | } 219 | 220 | return cur; 221 | } 222 | 223 | class JSONStreamState { 224 | source: string; 225 | 226 | pos: number; 227 | len: number; 228 | 229 | line: number; 230 | char: number; 231 | 232 | constructor(source: string) { 233 | this.source = source; 234 | this.pos = 0; 235 | this.len = source.length; 236 | this.line = 1; 237 | this.char = 0; 238 | } 239 | } 240 | 241 | const enum JSONTokenType { 242 | UNKNOWN = 0, 243 | STRING = 1, 244 | LEFT_SQUARE_BRACKET = 2, // [ 245 | LEFT_CURLY_BRACKET = 3, // { 246 | RIGHT_SQUARE_BRACKET = 4, // ] 247 | RIGHT_CURLY_BRACKET = 5, // } 248 | COLON = 6, // : 249 | COMMA = 7, // , 250 | NULL = 8, 251 | TRUE = 9, 252 | FALSE = 10, 253 | NUMBER = 11 254 | } 255 | 256 | const enum JSONState { 257 | ROOT_STATE = 0, 258 | DICT_STATE = 1, 259 | DICT_STATE_COMMA = 2, 260 | DICT_STATE_NO_CLOSE = 3, 261 | ARR_STATE = 4, 262 | ARR_STATE_COMMA = 5, 263 | ARR_STATE_NO_CLOSE = 6, 264 | } 265 | 266 | const enum ChCode { 267 | SPACE = 0x20, 268 | HORIZONTAL_TAB = 0x09, 269 | CARRIAGE_RETURN = 0x0D, 270 | LINE_FEED = 0x0A, 271 | QUOTATION_MARK = 0x22, 272 | BACKSLASH = 0x5C, 273 | 274 | LEFT_SQUARE_BRACKET = 0x5B, 275 | LEFT_CURLY_BRACKET = 0x7B, 276 | RIGHT_SQUARE_BRACKET = 0x5D, 277 | RIGHT_CURLY_BRACKET = 0x7D, 278 | COLON = 0x3A, 279 | COMMA = 0x2C, 280 | DOT = 0x2E, 281 | 282 | D0 = 0x30, 283 | D9 = 0x39, 284 | 285 | MINUS = 0x2D, 286 | PLUS = 0x2B, 287 | 288 | E = 0x45, 289 | 290 | a = 0x61, 291 | e = 0x65, 292 | f = 0x66, 293 | l = 0x6C, 294 | n = 0x6E, 295 | r = 0x72, 296 | s = 0x73, 297 | t = 0x74, 298 | u = 0x75, 299 | } 300 | 301 | class JSONToken { 302 | value: string; 303 | type: JSONTokenType; 304 | 305 | offset: number; 306 | len: number; 307 | 308 | line: number; /* 1 based line number */ 309 | char: number; 310 | 311 | constructor() { 312 | this.value = null; 313 | this.offset = -1; 314 | this.len = -1; 315 | this.line = -1; 316 | this.char = -1; 317 | } 318 | 319 | toLocation(filename: string): ILocation { 320 | return { 321 | filename: filename, 322 | line: this.line, 323 | char: this.char 324 | }; 325 | } 326 | } 327 | 328 | /** 329 | * precondition: the string is known to be valid JSON (https://www.ietf.org/rfc/rfc4627.txt) 330 | */ 331 | function nextJSONToken(_state: JSONStreamState, _out: JSONToken): boolean { 332 | _out.value = null; 333 | _out.type = JSONTokenType.UNKNOWN; 334 | _out.offset = -1; 335 | _out.len = -1; 336 | _out.line = -1; 337 | _out.char = -1; 338 | 339 | let source = _state.source; 340 | let pos = _state.pos; 341 | let len = _state.len; 342 | let line = _state.line; 343 | let char = _state.char; 344 | 345 | //------------------------ skip whitespace 346 | let chCode: number; 347 | do { 348 | if (pos >= len) { 349 | return false; /*EOS*/ 350 | } 351 | 352 | chCode = source.charCodeAt(pos); 353 | if (chCode === ChCode.SPACE || chCode === ChCode.HORIZONTAL_TAB || chCode === ChCode.CARRIAGE_RETURN) { 354 | // regular whitespace 355 | pos++; char++; 356 | continue; 357 | } 358 | 359 | if (chCode === ChCode.LINE_FEED) { 360 | // newline 361 | pos++; line++; char = 0; 362 | continue; 363 | } 364 | 365 | // not whitespace 366 | break; 367 | } while (true); 368 | 369 | _out.offset = pos; 370 | _out.line = line; 371 | _out.char = char; 372 | 373 | if (chCode === ChCode.QUOTATION_MARK) { 374 | //------------------------ strings 375 | _out.type = JSONTokenType.STRING; 376 | 377 | pos++; char++; 378 | 379 | do { 380 | if (pos >= len) { 381 | return false; /*EOS*/ 382 | } 383 | 384 | chCode = source.charCodeAt(pos); 385 | pos++; char++; 386 | 387 | if (chCode === ChCode.BACKSLASH) { 388 | // skip next char 389 | pos++; char++; 390 | continue; 391 | } 392 | 393 | if (chCode === ChCode.QUOTATION_MARK) { 394 | // end of the string 395 | break; 396 | } 397 | } while (true); 398 | 399 | _out.value = source.substring(_out.offset + 1, pos - 1).replace(/\\u([0-9A-Fa-f]{4})/g, (_, m0) => { 400 | return (String).fromCodePoint(parseInt(m0, 16)); 401 | }).replace(/\\(.)/g, (_, m0) => { 402 | switch (m0) { 403 | case '"': return '"'; 404 | case '\\': return '\\'; 405 | case '/': return '/'; 406 | case 'b': return '\b'; 407 | case 'f': return '\f'; 408 | case 'n': return '\n'; 409 | case 'r': return '\r'; 410 | case 't': return '\t'; 411 | default: doFail(_state, 'invalid escape sequence'); 412 | } 413 | }); 414 | 415 | } else if (chCode === ChCode.LEFT_SQUARE_BRACKET) { 416 | 417 | _out.type = JSONTokenType.LEFT_SQUARE_BRACKET; 418 | pos++; char++; 419 | 420 | } else if (chCode === ChCode.LEFT_CURLY_BRACKET) { 421 | 422 | _out.type = JSONTokenType.LEFT_CURLY_BRACKET; 423 | pos++; char++; 424 | 425 | } else if (chCode === ChCode.RIGHT_SQUARE_BRACKET) { 426 | 427 | _out.type = JSONTokenType.RIGHT_SQUARE_BRACKET; 428 | pos++; char++; 429 | 430 | } else if (chCode === ChCode.RIGHT_CURLY_BRACKET) { 431 | 432 | _out.type = JSONTokenType.RIGHT_CURLY_BRACKET; 433 | pos++; char++; 434 | 435 | } else if (chCode === ChCode.COLON) { 436 | 437 | _out.type = JSONTokenType.COLON; 438 | pos++; char++; 439 | 440 | } else if (chCode === ChCode.COMMA) { 441 | 442 | _out.type = JSONTokenType.COMMA; 443 | pos++; char++; 444 | 445 | } else if (chCode === ChCode.n) { 446 | //------------------------ null 447 | 448 | _out.type = JSONTokenType.NULL; 449 | pos++; char++; chCode = source.charCodeAt(pos); 450 | if (chCode !== ChCode.u) { return false; /* INVALID */ } 451 | pos++; char++; chCode = source.charCodeAt(pos); 452 | if (chCode !== ChCode.l) { return false; /* INVALID */ } 453 | pos++; char++; chCode = source.charCodeAt(pos); 454 | if (chCode !== ChCode.l) { return false; /* INVALID */ } 455 | pos++; char++; 456 | 457 | } else if (chCode === ChCode.t) { 458 | //------------------------ true 459 | 460 | _out.type = JSONTokenType.TRUE; 461 | pos++; char++; chCode = source.charCodeAt(pos); 462 | if (chCode !== ChCode.r) { return false; /* INVALID */ } 463 | pos++; char++; chCode = source.charCodeAt(pos); 464 | if (chCode !== ChCode.u) { return false; /* INVALID */ } 465 | pos++; char++; chCode = source.charCodeAt(pos); 466 | if (chCode !== ChCode.e) { return false; /* INVALID */ } 467 | pos++; char++; 468 | 469 | } else if (chCode === ChCode.f) { 470 | //------------------------ false 471 | 472 | _out.type = JSONTokenType.FALSE; 473 | pos++; char++; chCode = source.charCodeAt(pos); 474 | if (chCode !== ChCode.a) { return false; /* INVALID */ } 475 | pos++; char++; chCode = source.charCodeAt(pos); 476 | if (chCode !== ChCode.l) { return false; /* INVALID */ } 477 | pos++; char++; chCode = source.charCodeAt(pos); 478 | if (chCode !== ChCode.s) { return false; /* INVALID */ } 479 | pos++; char++; chCode = source.charCodeAt(pos); 480 | if (chCode !== ChCode.e) { return false; /* INVALID */ } 481 | pos++; char++; 482 | 483 | } else { 484 | //------------------------ numbers 485 | 486 | _out.type = JSONTokenType.NUMBER; 487 | do { 488 | if (pos >= len) { return false; /*EOS*/ } 489 | 490 | chCode = source.charCodeAt(pos); 491 | if ( 492 | chCode === ChCode.DOT 493 | || (chCode >= ChCode.D0 && chCode <= ChCode.D9) 494 | || (chCode === ChCode.e || chCode === ChCode.E) 495 | || (chCode === ChCode.MINUS || chCode === ChCode.PLUS) 496 | ) { 497 | // looks like a piece of a number 498 | pos++; char++; 499 | continue; 500 | } 501 | 502 | // pos--; char--; 503 | break; 504 | } while (true); 505 | } 506 | 507 | _out.len = pos - _out.offset; 508 | if (_out.value === null) { 509 | _out.value = source.substr(_out.offset, _out.len); 510 | } 511 | 512 | _state.pos = pos; 513 | _state.line = line; 514 | _state.char = char; 515 | 516 | // console.log('PRODUCING TOKEN: ', _out.value, JSONTokenType[_out.type]); 517 | 518 | return true; 519 | } 520 | -------------------------------------------------------------------------------- /src/main.ts: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------- 2 | * Copyright (C) Microsoft Corporation. All rights reserved. 3 | *--------------------------------------------------------*/ 4 | 5 | import { SyncRegistry } from './registry'; 6 | import { parseJSONGrammar, parsePLISTGrammar } from './grammarReader'; 7 | import { Theme } from './theme'; 8 | import { StackElement as StackElementImpl } from './grammar'; 9 | import { IGrammarDefinition, IRawGrammar } from './types'; 10 | 11 | export { IGrammarDefinition, IRawGrammar } 12 | 13 | let DEFAULT_OPTIONS: RegistryOptions = { 14 | getGrammarDefinition: (scopeName: string) => null, 15 | getInjections: (scopeName: string) => null 16 | }; 17 | 18 | /** 19 | * A single theme setting. 20 | */ 21 | export interface IRawThemeSetting { 22 | readonly name?: string; 23 | readonly scope?: string | string[]; 24 | readonly settings: { 25 | readonly fontStyle?: string; 26 | readonly foreground?: string; 27 | readonly background?: string; 28 | }; 29 | } 30 | 31 | /** 32 | * A TextMate theme. 33 | */ 34 | export interface IRawTheme { 35 | readonly name?: string; 36 | readonly settings: IRawThemeSetting[]; 37 | } 38 | 39 | /** 40 | * A registry helper that can locate grammar file paths given scope names. 41 | */ 42 | export interface RegistryOptions { 43 | theme?: IRawTheme; 44 | getGrammarDefinition(scopeName: string, dependentScope: string): Promise; 45 | getInjections?(scopeName: string): string[]; 46 | } 47 | 48 | /** 49 | * A map from scope name to a language id. Please do not use language id 0. 50 | */ 51 | export interface IEmbeddedLanguagesMap { 52 | [scopeName: string]: number; 53 | } 54 | 55 | /** 56 | * A map from selectors to token types. 57 | */ 58 | export interface ITokenTypeMap { 59 | [selector: string]: StandardTokenType; 60 | } 61 | 62 | export const enum StandardTokenType { 63 | Other = 0, 64 | Comment = 1, 65 | String = 2, 66 | RegEx = 4 67 | } 68 | 69 | export interface IGrammarConfiguration { 70 | embeddedLanguages?: IEmbeddedLanguagesMap; 71 | tokenTypes?: ITokenTypeMap; 72 | } 73 | 74 | /** 75 | * The registry that will hold all grammars. 76 | */ 77 | export class Registry { 78 | 79 | private readonly _locator: RegistryOptions; 80 | private readonly _syncRegistry: SyncRegistry; 81 | private readonly installationQueue: Map>; 82 | 83 | constructor(locator: RegistryOptions = DEFAULT_OPTIONS) { 84 | this._locator = locator; 85 | this._syncRegistry = new SyncRegistry(Theme.createFromRawTheme(locator.theme)); 86 | this.installationQueue = new Map(); 87 | } 88 | 89 | /** 90 | * Change the theme. Once called, no previous `ruleStack` should be used anymore. 91 | */ 92 | public setTheme(theme: IRawTheme): void { 93 | this._syncRegistry.setTheme(Theme.createFromRawTheme(theme)); 94 | } 95 | 96 | /** 97 | * Returns a lookup array for color ids. 98 | */ 99 | public getColorMap(): string[] { 100 | return this._syncRegistry.getColorMap(); 101 | } 102 | 103 | /** 104 | * Load the grammar for `scopeName` and all referenced included grammars asynchronously. 105 | * Please do not use language id 0. 106 | */ 107 | public loadGrammarWithEmbeddedLanguages(initialScopeName: string, initialLanguage: number, embeddedLanguages: IEmbeddedLanguagesMap): Promise { 108 | return this.loadGrammarWithConfiguration(initialScopeName, initialLanguage, { embeddedLanguages }); 109 | } 110 | 111 | /** 112 | * Load the grammar for `scopeName` and all referenced included grammars asynchronously. 113 | * Please do not use language id 0. 114 | */ 115 | public async loadGrammarWithConfiguration(initialScopeName: string, initialLanguage: number, configuration: IGrammarConfiguration): Promise { 116 | await this._loadGrammar(initialScopeName); 117 | return this.grammarForScopeName(initialScopeName, initialLanguage, configuration.embeddedLanguages, configuration.tokenTypes); 118 | } 119 | 120 | /** 121 | * Load the grammar for `scopeName` and all referenced included grammars asynchronously. 122 | */ 123 | public async loadGrammar(initialScopeName: string): Promise { 124 | return this._loadGrammar(initialScopeName); 125 | } 126 | 127 | private async _loadGrammar(initialScopeName: string, dependentScope: string = null): Promise { 128 | 129 | // already installed 130 | if (this._syncRegistry.lookup(initialScopeName)) { 131 | return this.grammarForScopeName(initialScopeName); 132 | } 133 | // installation in progress 134 | if (this.installationQueue.has(initialScopeName)) { 135 | return this.installationQueue.get(initialScopeName); 136 | } 137 | // start installation process 138 | const prom = new Promise(async (resolve, reject) => { 139 | let grammarDefinition = await this._locator.getGrammarDefinition(initialScopeName, dependentScope); 140 | if (!grammarDefinition) { 141 | throw new Error(`A tmGrammar load was requested but registry host failed to provide grammar definition`); 142 | } 143 | if ((grammarDefinition.format !== 'json' && grammarDefinition.format !== 'plist') || 144 | (grammarDefinition.format === 'json' && typeof grammarDefinition.content !== 'object' && typeof grammarDefinition.content !== 'string') || 145 | (grammarDefinition.format === 'plist' && typeof grammarDefinition.content !== 'string') 146 | ) { 147 | throw new TypeError('Grammar definition must be an object, either `{ content: string | object, format: "json" }` OR `{ content: string, format: "plist" }`)'); 148 | } 149 | const rawGrammar: IRawGrammar = grammarDefinition.format === 'json' 150 | ? typeof grammarDefinition.content === 'string' 151 | ? parseJSONGrammar(grammarDefinition.content, 'c://fakepath/grammar.json') 152 | : grammarDefinition.content as IRawGrammar 153 | : parsePLISTGrammar(grammarDefinition.content as string, 'c://fakepath/grammar.plist'); 154 | let injections = (typeof this._locator.getInjections === 'function') && this._locator.getInjections(initialScopeName); 155 | 156 | (rawGrammar as any).scopeName = initialScopeName; 157 | let deps = this._syncRegistry.addGrammar(rawGrammar, injections); 158 | await Promise.all(deps.map(async (scopeNameD) => { 159 | try { 160 | return this._loadGrammar(scopeNameD, initialScopeName); 161 | } catch (error) { 162 | throw new Error(`While trying to load tmGrammar with scopeId: '${initialScopeName}', it's dependency (scopeId: ${scopeNameD}) loading errored: ${error.message}`); 163 | } 164 | })); 165 | resolve(this.grammarForScopeName(initialScopeName)); 166 | }); 167 | this.installationQueue.set(initialScopeName, prom); 168 | await prom; 169 | this.installationQueue.delete(initialScopeName); 170 | return prom; 171 | } 172 | 173 | /** 174 | * Get the grammar for `scopeName`. The grammar must first be created via `loadGrammar` or `loadGrammarFromPathSync`. 175 | */ 176 | public grammarForScopeName(scopeName: string, initialLanguage: number = 0, embeddedLanguages: IEmbeddedLanguagesMap = null, tokenTypes: ITokenTypeMap = null): IGrammar { 177 | return this._syncRegistry.grammarForScopeName(scopeName, initialLanguage, embeddedLanguages, tokenTypes); 178 | } 179 | } 180 | 181 | /** 182 | * A grammar 183 | */ 184 | export interface IGrammar { 185 | /** 186 | * Tokenize `lineText` using previous line state `prevState`. 187 | */ 188 | tokenizeLine(lineText: string, prevState: StackElement): ITokenizeLineResult; 189 | 190 | /** 191 | * Tokenize `lineText` using previous line state `prevState`. 192 | * The result contains the tokens in binary format, resolved with the following information: 193 | * - language 194 | * - token type (regex, string, comment, other) 195 | * - font style 196 | * - foreground color 197 | * - background color 198 | * e.g. for getting the languageId: `(metadata & MetadataConsts.LANGUAGEID_MASK) >>> MetadataConsts.LANGUAGEID_OFFSET` 199 | */ 200 | tokenizeLine2(lineText: string, prevState: StackElement): ITokenizeLineResult2; 201 | } 202 | 203 | export interface ITokenizeLineResult { 204 | readonly tokens: IToken[]; 205 | /** 206 | * The `prevState` to be passed on to the next line tokenization. 207 | */ 208 | readonly ruleStack: StackElement; 209 | } 210 | 211 | /** 212 | * Helpers to manage the "collapsed" metadata of an entire StackElement stack. 213 | * The following assumptions have been made: 214 | * - languageId < 256 => needs 8 bits 215 | * - unique color count < 512 => needs 9 bits 216 | * 217 | * The binary format is: 218 | * - ------------------------------------------- 219 | * 3322 2222 2222 1111 1111 1100 0000 0000 220 | * 1098 7654 3210 9876 5432 1098 7654 3210 221 | * - ------------------------------------------- 222 | * xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx 223 | * bbbb bbbb bfff ffff ffFF FTTT LLLL LLLL 224 | * - ------------------------------------------- 225 | * - L = LanguageId (8 bits) 226 | * - T = StandardTokenType (3 bits) 227 | * - F = FontStyle (3 bits) 228 | * - f = foreground color (9 bits) 229 | * - b = background color (9 bits) 230 | */ 231 | export const enum MetadataConsts { 232 | LANGUAGEID_MASK = 0b00000000000000000000000011111111, 233 | TOKEN_TYPE_MASK = 0b00000000000000000000011100000000, 234 | FONT_STYLE_MASK = 0b00000000000000000011100000000000, 235 | FOREGROUND_MASK = 0b00000000011111111100000000000000, 236 | BACKGROUND_MASK = 0b11111111100000000000000000000000, 237 | 238 | LANGUAGEID_OFFSET = 0, 239 | TOKEN_TYPE_OFFSET = 8, 240 | FONT_STYLE_OFFSET = 11, 241 | FOREGROUND_OFFSET = 14, 242 | BACKGROUND_OFFSET = 23 243 | } 244 | 245 | export interface ITokenizeLineResult2 { 246 | /** 247 | * The tokens in binary format. Each token occupies two array indices. For token i: 248 | * - at offset 2*i => startIndex 249 | * - at offset 2*i + 1 => metadata 250 | * 251 | */ 252 | readonly tokens: Uint32Array; 253 | /** 254 | * The `prevState` to be passed on to the next line tokenization. 255 | */ 256 | readonly ruleStack: StackElement; 257 | } 258 | 259 | export interface IToken { 260 | startIndex: number; 261 | readonly endIndex: number; 262 | readonly scopes: string[]; 263 | } 264 | 265 | /** 266 | * **IMPORTANT** - Immutable! 267 | */ 268 | export interface StackElement { 269 | _stackElementBrand: void; 270 | readonly depth: number; 271 | 272 | clone(): StackElement; 273 | equals(other: StackElement): boolean; 274 | } 275 | 276 | export const INITIAL: StackElement = StackElementImpl.NULL; 277 | -------------------------------------------------------------------------------- /src/matcher.ts: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------- 2 | * Copyright (C) Microsoft Corporation. All rights reserved. 3 | *--------------------------------------------------------*/ 4 | 5 | export interface MatcherWithPriority { 6 | matcher: Matcher; 7 | priority: -1 | 0 | 1; 8 | } 9 | 10 | export interface Matcher { 11 | (matcherInput: T): boolean; 12 | } 13 | 14 | export function createMatchers(selector: string, matchesName: (names: string[], matcherInput: T) => boolean): MatcherWithPriority[] { 15 | var results = []> []; 16 | var tokenizer = newTokenizer(selector); 17 | var token = tokenizer.next(); 18 | while (token !== null) { 19 | let priority : -1 | 0 | 1 = 0; 20 | if (token.length === 2 && token.charAt(1) === ':') { 21 | switch (token.charAt(0)) { 22 | case 'R': priority = 1; break; 23 | case 'L': priority = -1; break; 24 | default: 25 | console.log(`Unknown priority ${token} in scope selector`); 26 | } 27 | token = tokenizer.next(); 28 | } 29 | let matcher = parseConjunction(); 30 | if (matcher) { 31 | results.push({ matcher, priority }); 32 | } 33 | if (token !== ',') { 34 | break; 35 | } 36 | token = tokenizer.next(); 37 | } 38 | return results; 39 | 40 | function parseOperand(): Matcher { 41 | if (token === '-') { 42 | token = tokenizer.next(); 43 | var expressionToNegate = parseOperand(); 44 | return matcherInput => expressionToNegate && !expressionToNegate(matcherInput); 45 | } 46 | if (token === '(') { 47 | token = tokenizer.next(); 48 | var expressionInParents = parseInnerExpression(); 49 | if (token === ')') { 50 | token = tokenizer.next(); 51 | } 52 | return expressionInParents; 53 | } 54 | if (isIdentifier(token)) { 55 | var identifiers: string[] = []; 56 | do { 57 | identifiers.push(token); 58 | token = tokenizer.next(); 59 | } while (isIdentifier(token)); 60 | return matcherInput => matchesName(identifiers, matcherInput); 61 | } 62 | return null; 63 | } 64 | function parseConjunction(): Matcher { 65 | var matchers: Matcher[] = []; 66 | var matcher = parseOperand(); 67 | while (matcher) { 68 | matchers.push(matcher); 69 | matcher = parseOperand(); 70 | } 71 | return matcherInput => matchers.every(matcher => matcher(matcherInput)); // and 72 | } 73 | function parseInnerExpression(): Matcher { 74 | var matchers: Matcher[] = []; 75 | var matcher = parseConjunction(); 76 | while (matcher) { 77 | matchers.push(matcher); 78 | if (token === '|' || token === ',') { 79 | do { 80 | token = tokenizer.next(); 81 | } while (token === '|' || token === ','); // ignore subsequent commas 82 | } else { 83 | break; 84 | } 85 | matcher = parseConjunction(); 86 | } 87 | return matcherInput => matchers.some(matcher => matcher(matcherInput)); // or 88 | } 89 | } 90 | 91 | function isIdentifier(token: string) { 92 | return token && token.match(/[\w\.:]+/); 93 | } 94 | 95 | function newTokenizer(input: string): { next: () => string } { 96 | let regex = /([LR]:|[\w\.:][\w\.:\-]*|[\,\|\-\(\)])/g; 97 | var match = regex.exec(input); 98 | return { 99 | next: () => { 100 | if (!match) { 101 | return null; 102 | } 103 | var res = match[0]; 104 | match = regex.exec(input); 105 | return res; 106 | } 107 | }; 108 | } 109 | -------------------------------------------------------------------------------- /src/registry.ts: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------- 2 | * Copyright (C) Microsoft Corporation. All rights reserved. 3 | *--------------------------------------------------------*/ 4 | 5 | import { createGrammar, Grammar, collectIncludedScopes, IGrammarRepository, IScopeNameSet } from './grammar'; 6 | import { IRawGrammar } from './types'; 7 | import { IGrammar, IEmbeddedLanguagesMap, ITokenTypeMap } from './main'; 8 | import { Theme, ThemeTrieElementRule } from './theme'; 9 | 10 | export class SyncRegistry implements IGrammarRepository { 11 | 12 | private readonly _grammars: { [scopeName: string]: Grammar; }; 13 | private readonly _rawGrammars: { [scopeName: string]: IRawGrammar; }; 14 | private readonly _injectionGrammars: { [scopeName: string]: string[]; }; 15 | private _theme: Theme; 16 | 17 | constructor(theme: Theme) { 18 | this._theme = theme; 19 | this._grammars = {}; 20 | this._rawGrammars = {}; 21 | this._injectionGrammars = {}; 22 | } 23 | 24 | public setTheme(theme: Theme): void { 25 | this._theme = theme; 26 | Object.keys(this._grammars).forEach((scopeName) => { 27 | let grammar = this._grammars[scopeName]; 28 | grammar.onDidChangeTheme(); 29 | }); 30 | } 31 | 32 | public getColorMap(): string[] { 33 | return this._theme.getColorMap(); 34 | } 35 | 36 | /** 37 | * Add `grammar` to registry and return a list of referenced scope names 38 | */ 39 | public addGrammar(grammar: IRawGrammar, injectionScopeNames?: string[]): string[] { 40 | this._rawGrammars[grammar.scopeName] = grammar; 41 | 42 | let includedScopes: IScopeNameSet = {}; 43 | collectIncludedScopes(includedScopes, grammar); 44 | 45 | if (injectionScopeNames) { 46 | this._injectionGrammars[grammar.scopeName] = injectionScopeNames; 47 | injectionScopeNames.forEach(scopeName => { 48 | includedScopes[scopeName] = true; 49 | }); 50 | } 51 | return Object.keys(includedScopes); 52 | } 53 | 54 | /** 55 | * Lookup a raw grammar. 56 | */ 57 | public lookup(scopeName: string): IRawGrammar { 58 | return this._rawGrammars[scopeName]; 59 | } 60 | 61 | /** 62 | * Returns the injections for the given grammar 63 | */ 64 | public injections(targetScope: string): string[] { 65 | return this._injectionGrammars[targetScope]; 66 | } 67 | 68 | /** 69 | * Get the default theme settings 70 | */ 71 | public getDefaults(): ThemeTrieElementRule { 72 | return this._theme.getDefaults(); 73 | } 74 | 75 | /** 76 | * Match a scope in the theme. 77 | */ 78 | public themeMatch(scopeName: string): ThemeTrieElementRule[] { 79 | return this._theme.match(scopeName); 80 | } 81 | 82 | 83 | /** 84 | * Lookup a grammar. 85 | */ 86 | public grammarForScopeName(scopeName: string, initialLanguage: number, embeddedLanguages: IEmbeddedLanguagesMap, tokenTypes: ITokenTypeMap): IGrammar { 87 | if (!this._grammars[scopeName]) { 88 | let rawGrammar = this._rawGrammars[scopeName]; 89 | if (!rawGrammar) { 90 | return null; 91 | } 92 | 93 | this._grammars[scopeName] = createGrammar(rawGrammar, initialLanguage, embeddedLanguages, tokenTypes, this); 94 | } 95 | return this._grammars[scopeName]; 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /src/rule.ts: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------- 2 | * Copyright (C) Microsoft Corporation. All rights reserved. 3 | *--------------------------------------------------------*/ 4 | 5 | import * as path from 'path'; 6 | import { RegexSource, mergeObjects } from './utils'; 7 | import { ILocation, IRawGrammar, IRawRepository, IRawRule, IRawCaptures } from './types'; 8 | import { OnigString, OnigScanner, IOnigCaptureIndex } from 'onigasm'; 9 | 10 | const HAS_BACK_REFERENCES = /\\(\d+)/; 11 | const BACK_REFERENCING_END = /\\(\d+)/g; 12 | 13 | export interface IRuleRegistry { 14 | getRule(patternId: number): Rule; 15 | registerRule(factory: (id: number) => T): T; 16 | } 17 | 18 | export interface IGrammarRegistry { 19 | getExternalGrammar(scopeName: string, repository: IRawRepository): IRawGrammar; 20 | } 21 | 22 | export interface IRuleFactoryHelper extends IRuleRegistry, IGrammarRegistry { 23 | } 24 | 25 | export interface ICompiledRule { 26 | readonly scanner: OnigScanner; 27 | readonly rules: number[]; 28 | readonly debugRegExps: string[]; 29 | } 30 | 31 | export abstract class Rule { 32 | 33 | public readonly $location: ILocation; 34 | public readonly id: number; 35 | 36 | private readonly _nameIsCapturing: boolean; 37 | private readonly _name: string; 38 | 39 | private readonly _contentNameIsCapturing: boolean; 40 | private readonly _contentName: string; 41 | 42 | constructor($location: ILocation, id: number, name: string, contentName: string) { 43 | this.$location = $location; 44 | this.id = id; 45 | this._name = name || null; 46 | this._nameIsCapturing = RegexSource.hasCaptures(this._name); 47 | this._contentName = contentName || null; 48 | this._contentNameIsCapturing = RegexSource.hasCaptures(this._contentName); 49 | } 50 | 51 | public get debugName(): string { 52 | return `${(this.constructor).name}#${this.id} @ ${path.basename(this.$location.filename)}:${this.$location.line}`; 53 | } 54 | 55 | public getName(lineText: string, captureIndices: IOnigCaptureIndex[]): string { 56 | if (!this._nameIsCapturing) { 57 | return this._name; 58 | } 59 | return RegexSource.replaceCaptures(this._name, lineText, captureIndices); 60 | } 61 | 62 | public getContentName(lineText: string, captureIndices: IOnigCaptureIndex[]): string { 63 | if (!this._contentNameIsCapturing) { 64 | return this._contentName; 65 | } 66 | return RegexSource.replaceCaptures(this._contentName, lineText, captureIndices); 67 | } 68 | 69 | public collectPatternsRecursive(grammar: IRuleRegistry, out: RegExpSourceList, isFirst: boolean) { 70 | throw new Error('Implement me!'); 71 | } 72 | 73 | public compile(grammar: IRuleRegistry, endRegexSource: string, allowA: boolean, allowG: boolean): ICompiledRule { 74 | throw new Error('Implement me!'); 75 | } 76 | } 77 | 78 | export interface ICompilePatternsResult { 79 | readonly patterns: number[]; 80 | readonly hasMissingPatterns: boolean; 81 | } 82 | 83 | export class CaptureRule extends Rule { 84 | 85 | public readonly retokenizeCapturedWithRuleId: number; 86 | 87 | constructor($location: ILocation, id: number, name: string, contentName: string, retokenizeCapturedWithRuleId: number) { 88 | super($location, id, name, contentName); 89 | this.retokenizeCapturedWithRuleId = retokenizeCapturedWithRuleId; 90 | } 91 | } 92 | 93 | interface IRegExpSourceAnchorCache { 94 | readonly A0_G0: string; 95 | readonly A0_G1: string; 96 | readonly A1_G0: string; 97 | readonly A1_G1: string; 98 | } 99 | 100 | export class RegExpSource { 101 | 102 | public source: string; 103 | public readonly ruleId: number; 104 | public hasAnchor: boolean; 105 | public readonly hasBackReferences: boolean; 106 | private _anchorCache: IRegExpSourceAnchorCache; 107 | 108 | constructor(regExpSource: string, ruleId: number, handleAnchors: boolean = true) { 109 | if (handleAnchors) { 110 | this._handleAnchors(regExpSource); 111 | } else { 112 | this.source = regExpSource; 113 | this.hasAnchor = false; 114 | } 115 | 116 | if (this.hasAnchor) { 117 | this._anchorCache = this._buildAnchorCache(); 118 | } 119 | 120 | this.ruleId = ruleId; 121 | this.hasBackReferences = HAS_BACK_REFERENCES.test(this.source); 122 | 123 | // console.log('input: ' + regExpSource + ' => ' + this.source + ', ' + this.hasAnchor); 124 | } 125 | 126 | public clone(): RegExpSource { 127 | return new RegExpSource(this.source, this.ruleId, true); 128 | } 129 | 130 | public setSource(newSource: string): void { 131 | if (this.source === newSource) { 132 | return; 133 | } 134 | this.source = newSource; 135 | 136 | if (this.hasAnchor) { 137 | this._anchorCache = this._buildAnchorCache(); 138 | } 139 | } 140 | 141 | private _handleAnchors(regExpSource: string): void { 142 | if (regExpSource) { 143 | let pos: number, 144 | len: number, 145 | ch: string, 146 | nextCh: string, 147 | lastPushedPos = 0, 148 | output: string[] = []; 149 | 150 | let hasAnchor = false; 151 | for (pos = 0, len = regExpSource.length; pos < len; pos++) { 152 | ch = regExpSource.charAt(pos); 153 | 154 | if (ch === '\\') { 155 | if (pos + 1 < len) { 156 | nextCh = regExpSource.charAt(pos + 1); 157 | if (nextCh === 'z') { 158 | output.push(regExpSource.substring(lastPushedPos, pos)); 159 | output.push('$(?!\\n)(? { 185 | return lineText.substring(capture.start, capture.end); 186 | }); 187 | BACK_REFERENCING_END.lastIndex = 0; 188 | return this.source.replace(BACK_REFERENCING_END, (match, g1) => { 189 | return escapeRegExpCharacters(capturedValues[parseInt(g1, 10)] || ''); 190 | }); 191 | } 192 | 193 | private _buildAnchorCache(): IRegExpSourceAnchorCache { 194 | let A0_G0_result: string[] = []; 195 | let A0_G1_result: string[] = []; 196 | let A1_G0_result: string[] = []; 197 | let A1_G1_result: string[] = []; 198 | 199 | let pos: number, 200 | len: number, 201 | ch: string, 202 | nextCh: string; 203 | 204 | for (pos = 0, len = this.source.length; pos < len; pos++) { 205 | ch = this.source.charAt(pos); 206 | A0_G0_result[pos] = ch; 207 | A0_G1_result[pos] = ch; 208 | A1_G0_result[pos] = ch; 209 | A1_G1_result[pos] = ch; 210 | 211 | if (ch === '\\') { 212 | if (pos + 1 < len) { 213 | nextCh = this.source.charAt(pos + 1); 214 | if (nextCh === 'A') { 215 | A0_G0_result[pos + 1] = '\uFFFF'; 216 | A0_G1_result[pos + 1] = '\uFFFF'; 217 | A1_G0_result[pos + 1] = 'A'; 218 | A1_G1_result[pos + 1] = 'A'; 219 | } else if (nextCh === 'G') { 220 | A0_G0_result[pos + 1] = '\uFFFF'; 221 | A0_G1_result[pos + 1] = 'G'; 222 | A1_G0_result[pos + 1] = '\uFFFF'; 223 | A1_G1_result[pos + 1] = 'G'; 224 | } else { 225 | A0_G0_result[pos + 1] = nextCh; 226 | A0_G1_result[pos + 1] = nextCh; 227 | A1_G0_result[pos + 1] = nextCh; 228 | A1_G1_result[pos + 1] = nextCh; 229 | } 230 | pos++; 231 | } 232 | } 233 | } 234 | 235 | return { 236 | A0_G0: A0_G0_result.join(''), 237 | A0_G1: A0_G1_result.join(''), 238 | A1_G0: A1_G0_result.join(''), 239 | A1_G1: A1_G1_result.join('') 240 | }; 241 | } 242 | 243 | public resolveAnchors(allowA: boolean, allowG: boolean): string { 244 | if (!this.hasAnchor) { 245 | return this.source; 246 | } 247 | 248 | if (allowA) { 249 | if (allowG) { 250 | return this._anchorCache.A1_G1; 251 | } else { 252 | return this._anchorCache.A1_G0; 253 | } 254 | } else { 255 | if (allowG) { 256 | return this._anchorCache.A0_G1; 257 | } else { 258 | return this._anchorCache.A0_G0; 259 | } 260 | } 261 | } 262 | } 263 | 264 | interface IRegExpSourceListAnchorCache { 265 | A0_G0: ICompiledRule; 266 | A0_G1: ICompiledRule; 267 | A1_G0: ICompiledRule; 268 | A1_G1: ICompiledRule; 269 | } 270 | 271 | function createOnigScanner(sources: string[]): OnigScanner { 272 | return new OnigScanner(sources); 273 | } 274 | 275 | export function createOnigString(sources: string): OnigString { 276 | var r = new OnigString(sources); 277 | (r).$str = sources; 278 | return r; 279 | } 280 | 281 | export function getString(str: OnigString): string { 282 | return (str).$str; 283 | } 284 | 285 | export class RegExpSourceList { 286 | 287 | private readonly _items: RegExpSource[]; 288 | private _hasAnchors: boolean; 289 | private _cached: ICompiledRule; 290 | private _anchorCache: IRegExpSourceListAnchorCache; 291 | private readonly _cachedSources: string[]; 292 | 293 | constructor() { 294 | this._items = []; 295 | this._hasAnchors = false; 296 | this._cached = null; 297 | this._cachedSources = null; 298 | this._anchorCache = { 299 | A0_G0: null, 300 | A0_G1: null, 301 | A1_G0: null, 302 | A1_G1: null 303 | }; 304 | } 305 | 306 | public push(item: RegExpSource): void { 307 | this._items.push(item); 308 | this._hasAnchors = this._hasAnchors || item.hasAnchor; 309 | } 310 | 311 | public unshift(item: RegExpSource): void { 312 | this._items.unshift(item); 313 | this._hasAnchors = this._hasAnchors || item.hasAnchor; 314 | } 315 | 316 | public length(): number { 317 | return this._items.length; 318 | } 319 | 320 | public setSource(index: number, newSource: string): void { 321 | if (this._items[index].source !== newSource) { 322 | // bust the cache 323 | this._cached = null; 324 | this._anchorCache.A0_G0 = null; 325 | this._anchorCache.A0_G1 = null; 326 | this._anchorCache.A1_G0 = null; 327 | this._anchorCache.A1_G1 = null; 328 | this._items[index].setSource(newSource); 329 | } 330 | } 331 | 332 | public compile(grammar: IRuleRegistry, allowA: boolean, allowG: boolean): ICompiledRule { 333 | if (!this._hasAnchors) { 334 | if (!this._cached) { 335 | let regExps = this._items.map(e => e.source); 336 | this._cached = { 337 | scanner: createOnigScanner(regExps), 338 | rules: this._items.map(e => e.ruleId), 339 | debugRegExps: regExps 340 | }; 341 | } 342 | return this._cached; 343 | } else { 344 | this._anchorCache = { 345 | A0_G0: this._anchorCache.A0_G0 || (allowA === false && allowG === false ? this._resolveAnchors(allowA, allowG) : null), 346 | A0_G1: this._anchorCache.A0_G1 || (allowA === false && allowG === true ? this._resolveAnchors(allowA, allowG) : null), 347 | A1_G0: this._anchorCache.A1_G0 || (allowA === true && allowG === false ? this._resolveAnchors(allowA, allowG) : null), 348 | A1_G1: this._anchorCache.A1_G1 || (allowA === true && allowG === true ? this._resolveAnchors(allowA, allowG) : null), 349 | }; 350 | if (allowA) { 351 | if (allowG) { 352 | return this._anchorCache.A1_G1; 353 | } else { 354 | return this._anchorCache.A1_G0; 355 | } 356 | } else { 357 | if (allowG) { 358 | return this._anchorCache.A0_G1; 359 | } else { 360 | return this._anchorCache.A0_G0; 361 | } 362 | } 363 | } 364 | 365 | } 366 | 367 | private _resolveAnchors(allowA: boolean, allowG: boolean): ICompiledRule { 368 | let regExps = this._items.map(e => e.resolveAnchors(allowA, allowG)); 369 | return { 370 | scanner: createOnigScanner(regExps), 371 | rules: this._items.map(e => e.ruleId), 372 | debugRegExps: regExps 373 | }; 374 | } 375 | } 376 | 377 | export class MatchRule extends Rule { 378 | private readonly _match: RegExpSource; 379 | public readonly captures: CaptureRule[]; 380 | private _cachedCompiledPatterns: RegExpSourceList; 381 | 382 | constructor($location: ILocation, id: number, name: string, match: string, captures: CaptureRule[]) { 383 | super($location, id, name, null); 384 | this._match = new RegExpSource(match, this.id); 385 | this.captures = captures; 386 | this._cachedCompiledPatterns = null; 387 | } 388 | 389 | public get debugMatchRegExp(): string { 390 | return `${this._match.source}`; 391 | } 392 | 393 | public collectPatternsRecursive(grammar: IRuleRegistry, out: RegExpSourceList, isFirst: boolean) { 394 | out.push(this._match); 395 | } 396 | 397 | public compile(grammar: IRuleRegistry, endRegexSource: string, allowA: boolean, allowG: boolean): ICompiledRule { 398 | if (!this._cachedCompiledPatterns) { 399 | this._cachedCompiledPatterns = new RegExpSourceList(); 400 | this.collectPatternsRecursive(grammar, this._cachedCompiledPatterns, true); 401 | } 402 | return this._cachedCompiledPatterns.compile(grammar, allowA, allowG); 403 | } 404 | } 405 | 406 | export class IncludeOnlyRule extends Rule { 407 | public readonly hasMissingPatterns: boolean; 408 | public readonly patterns: number[]; 409 | private _cachedCompiledPatterns: RegExpSourceList; 410 | 411 | constructor($location: ILocation, id: number, name: string, contentName: string, patterns: ICompilePatternsResult) { 412 | super($location, id, name, contentName); 413 | this.patterns = patterns.patterns; 414 | this.hasMissingPatterns = patterns.hasMissingPatterns; 415 | this._cachedCompiledPatterns = null; 416 | } 417 | 418 | public collectPatternsRecursive(grammar: IRuleRegistry, out: RegExpSourceList, isFirst: boolean) { 419 | let i: number, 420 | len: number, 421 | rule: Rule; 422 | 423 | for (i = 0, len = this.patterns.length; i < len; i++) { 424 | rule = grammar.getRule(this.patterns[i]); 425 | rule.collectPatternsRecursive(grammar, out, false); 426 | } 427 | } 428 | 429 | public compile(grammar: IRuleRegistry, endRegexSource: string, allowA: boolean, allowG: boolean): ICompiledRule { 430 | if (!this._cachedCompiledPatterns) { 431 | this._cachedCompiledPatterns = new RegExpSourceList(); 432 | this.collectPatternsRecursive(grammar, this._cachedCompiledPatterns, true); 433 | } 434 | return this._cachedCompiledPatterns.compile(grammar, allowA, allowG); 435 | } 436 | } 437 | 438 | function escapeRegExpCharacters(value: string): string { 439 | return value.replace(/[\-\\\{\}\*\+\?\|\^\$\.\,\[\]\(\)\#\s]/g, '\\$&'); 440 | } 441 | 442 | export class BeginEndRule extends Rule { 443 | private readonly _begin: RegExpSource; 444 | public readonly beginCaptures: CaptureRule[]; 445 | private readonly _end: RegExpSource; 446 | public readonly endHasBackReferences: boolean; 447 | public readonly endCaptures: CaptureRule[]; 448 | public readonly applyEndPatternLast: boolean; 449 | public readonly hasMissingPatterns: boolean; 450 | public readonly patterns: number[]; 451 | private _cachedCompiledPatterns: RegExpSourceList; 452 | 453 | constructor($location: ILocation, id: number, name: string, contentName: string, begin: string, beginCaptures: CaptureRule[], end: string, endCaptures: CaptureRule[], applyEndPatternLast: boolean, patterns: ICompilePatternsResult) { 454 | super($location, id, name, contentName); 455 | this._begin = new RegExpSource(begin, this.id); 456 | this.beginCaptures = beginCaptures; 457 | this._end = new RegExpSource(end, -1); 458 | this.endHasBackReferences = this._end.hasBackReferences; 459 | this.endCaptures = endCaptures; 460 | this.applyEndPatternLast = applyEndPatternLast || false; 461 | this.patterns = patterns.patterns; 462 | this.hasMissingPatterns = patterns.hasMissingPatterns; 463 | this._cachedCompiledPatterns = null; 464 | } 465 | 466 | public get debugBeginRegExp(): string { 467 | return `${this._begin.source}`; 468 | } 469 | 470 | public get debugEndRegExp(): string { 471 | return `${this._end.source}`; 472 | } 473 | 474 | public getEndWithResolvedBackReferences(lineText: string, captureIndices: IOnigCaptureIndex[]): string { 475 | return this._end.resolveBackReferences(lineText, captureIndices); 476 | } 477 | 478 | public collectPatternsRecursive(grammar: IRuleRegistry, out: RegExpSourceList, isFirst: boolean) { 479 | if (isFirst) { 480 | let i: number, 481 | len: number, 482 | rule: Rule; 483 | 484 | for (i = 0, len = this.patterns.length; i < len; i++) { 485 | rule = grammar.getRule(this.patterns[i]); 486 | rule.collectPatternsRecursive(grammar, out, false); 487 | } 488 | } else { 489 | out.push(this._begin); 490 | } 491 | } 492 | 493 | public compile(grammar: IRuleRegistry, endRegexSource: string, allowA: boolean, allowG: boolean): ICompiledRule { 494 | let precompiled = this._precompile(grammar); 495 | 496 | if (this._end.hasBackReferences) { 497 | if (this.applyEndPatternLast) { 498 | precompiled.setSource(precompiled.length() - 1, endRegexSource); 499 | } else { 500 | precompiled.setSource(0, endRegexSource); 501 | } 502 | } 503 | return this._cachedCompiledPatterns.compile(grammar, allowA, allowG); 504 | } 505 | 506 | private _precompile(grammar: IRuleRegistry): RegExpSourceList { 507 | if (!this._cachedCompiledPatterns) { 508 | this._cachedCompiledPatterns = new RegExpSourceList(); 509 | 510 | this.collectPatternsRecursive(grammar, this._cachedCompiledPatterns, true); 511 | 512 | if (this.applyEndPatternLast) { 513 | this._cachedCompiledPatterns.push(this._end.hasBackReferences ? this._end.clone() : this._end); 514 | } else { 515 | this._cachedCompiledPatterns.unshift(this._end.hasBackReferences ? this._end.clone() : this._end); 516 | } 517 | } 518 | return this._cachedCompiledPatterns; 519 | } 520 | } 521 | 522 | export class BeginWhileRule extends Rule { 523 | private readonly _begin: RegExpSource; 524 | public readonly beginCaptures: CaptureRule[]; 525 | public readonly whileCaptures: CaptureRule[]; 526 | private readonly _while: RegExpSource; 527 | public readonly whileHasBackReferences: boolean; 528 | public readonly hasMissingPatterns: boolean; 529 | public readonly patterns: number[]; 530 | private _cachedCompiledPatterns: RegExpSourceList; 531 | private _cachedCompiledWhilePatterns: RegExpSourceList; 532 | 533 | constructor($location: ILocation, id: number, name: string, contentName: string, begin: string, beginCaptures: CaptureRule[], _while: string, whileCaptures: CaptureRule[], patterns: ICompilePatternsResult) { 534 | super($location, id, name, contentName); 535 | this._begin = new RegExpSource(begin, this.id); 536 | this.beginCaptures = beginCaptures; 537 | this.whileCaptures = whileCaptures; 538 | this._while = new RegExpSource(_while, -2); 539 | this.whileHasBackReferences = this._while.hasBackReferences; 540 | this.patterns = patterns.patterns; 541 | this.hasMissingPatterns = patterns.hasMissingPatterns; 542 | this._cachedCompiledPatterns = null; 543 | this._cachedCompiledWhilePatterns = null; 544 | } 545 | 546 | public getWhileWithResolvedBackReferences(lineText: string, captureIndices: IOnigCaptureIndex[]): string { 547 | return this._while.resolveBackReferences(lineText, captureIndices); 548 | } 549 | 550 | public collectPatternsRecursive(grammar: IRuleRegistry, out: RegExpSourceList, isFirst: boolean) { 551 | if (isFirst) { 552 | let i: number, 553 | len: number, 554 | rule: Rule; 555 | 556 | for (i = 0, len = this.patterns.length; i < len; i++) { 557 | rule = grammar.getRule(this.patterns[i]); 558 | rule.collectPatternsRecursive(grammar, out, false); 559 | } 560 | } else { 561 | out.push(this._begin); 562 | } 563 | } 564 | 565 | public compile(grammar: IRuleRegistry, endRegexSource: string, allowA: boolean, allowG: boolean): ICompiledRule { 566 | this._precompile(grammar); 567 | return this._cachedCompiledPatterns.compile(grammar, allowA, allowG); 568 | } 569 | 570 | private _precompile(grammar: IRuleRegistry): void { 571 | if (!this._cachedCompiledPatterns) { 572 | this._cachedCompiledPatterns = new RegExpSourceList(); 573 | this.collectPatternsRecursive(grammar, this._cachedCompiledPatterns, true); 574 | } 575 | } 576 | 577 | 578 | public compileWhile(grammar: IRuleRegistry, endRegexSource: string, allowA: boolean, allowG: boolean): ICompiledRule { 579 | this._precompileWhile(grammar); 580 | if (this._while.hasBackReferences) { 581 | this._cachedCompiledWhilePatterns.setSource(0, endRegexSource); 582 | } 583 | return this._cachedCompiledWhilePatterns.compile(grammar, allowA, allowG); 584 | } 585 | 586 | 587 | private _precompileWhile(grammar: IRuleRegistry): void { 588 | if (!this._cachedCompiledWhilePatterns) { 589 | this._cachedCompiledWhilePatterns = new RegExpSourceList(); 590 | this._cachedCompiledWhilePatterns.push(this._while.hasBackReferences ? this._while.clone() : this._while); 591 | } 592 | } 593 | } 594 | 595 | export class RuleFactory { 596 | 597 | public static createCaptureRule(helper: IRuleFactoryHelper, $location: ILocation, name: string, contentName: string, retokenizeCapturedWithRuleId: number): CaptureRule { 598 | return helper.registerRule((id) => { 599 | return new CaptureRule($location, id, name, contentName, retokenizeCapturedWithRuleId); 600 | }); 601 | } 602 | 603 | public static getCompiledRuleId(desc: IRawRule, helper: IRuleFactoryHelper, repository: IRawRepository): number { 604 | if (!desc.id) { 605 | helper.registerRule((id) => { 606 | desc.id = id; 607 | 608 | if (desc.match) { 609 | return new MatchRule( 610 | desc.$vscodeTextmateLocation, 611 | desc.id, 612 | desc.name, 613 | desc.match, 614 | RuleFactory._compileCaptures(desc.captures, helper, repository) 615 | ); 616 | } 617 | 618 | if (!desc.begin) { 619 | if (desc.repository) { 620 | repository = mergeObjects({}, repository, desc.repository); 621 | } 622 | return new IncludeOnlyRule( 623 | desc.$vscodeTextmateLocation, 624 | desc.id, 625 | desc.name, 626 | desc.contentName, 627 | RuleFactory._compilePatterns(desc.patterns, helper, repository) 628 | ); 629 | } 630 | 631 | if (desc.while) { 632 | return new BeginWhileRule( 633 | desc.$vscodeTextmateLocation, 634 | desc.id, 635 | desc.name, 636 | desc.contentName, 637 | desc.begin, RuleFactory._compileCaptures(desc.beginCaptures || desc.captures, helper, repository), 638 | desc.while, RuleFactory._compileCaptures(desc.whileCaptures || desc.captures, helper, repository), 639 | RuleFactory._compilePatterns(desc.patterns, helper, repository) 640 | ); 641 | } 642 | 643 | return new BeginEndRule( 644 | desc.$vscodeTextmateLocation, 645 | desc.id, 646 | desc.name, 647 | desc.contentName, 648 | desc.begin, RuleFactory._compileCaptures(desc.beginCaptures || desc.captures, helper, repository), 649 | desc.end, RuleFactory._compileCaptures(desc.endCaptures || desc.captures, helper, repository), 650 | desc.applyEndPatternLast, 651 | RuleFactory._compilePatterns(desc.patterns, helper, repository) 652 | ); 653 | }); 654 | } 655 | 656 | return desc.id; 657 | } 658 | 659 | private static _compileCaptures(captures: IRawCaptures, helper: IRuleFactoryHelper, repository: IRawRepository): CaptureRule[] { 660 | let r: CaptureRule[] = [], 661 | numericCaptureId: number, 662 | maximumCaptureId: number, 663 | i: number, 664 | captureId: string; 665 | 666 | if (captures) { 667 | // Find the maximum capture id 668 | maximumCaptureId = 0; 669 | for (captureId in captures) { 670 | if (captureId === '$vscodeTextmateLocation') { 671 | continue; 672 | } 673 | numericCaptureId = parseInt(captureId, 10); 674 | if (numericCaptureId > maximumCaptureId) { 675 | maximumCaptureId = numericCaptureId; 676 | } 677 | } 678 | 679 | // Initialize result 680 | for (i = 0; i <= maximumCaptureId; i++) { 681 | r[i] = null; 682 | } 683 | 684 | // Fill out result 685 | for (captureId in captures) { 686 | if (captureId === '$vscodeTextmateLocation') { 687 | continue; 688 | } 689 | numericCaptureId = parseInt(captureId, 10); 690 | let retokenizeCapturedWithRuleId = 0; 691 | if (captures[captureId].patterns) { 692 | retokenizeCapturedWithRuleId = RuleFactory.getCompiledRuleId(captures[captureId], helper, repository); 693 | } 694 | r[numericCaptureId] = RuleFactory.createCaptureRule(helper, captures[captureId].$vscodeTextmateLocation, captures[captureId].name, captures[captureId].contentName, retokenizeCapturedWithRuleId); 695 | } 696 | } 697 | 698 | return r; 699 | } 700 | 701 | private static _compilePatterns(patterns: IRawRule[], helper: IRuleFactoryHelper, repository: IRawRepository): ICompilePatternsResult { 702 | let r: number[] = [], 703 | pattern: IRawRule, 704 | i: number, 705 | len: number, 706 | patternId: number, 707 | externalGrammar: IRawGrammar, 708 | rule: Rule, 709 | skipRule: boolean; 710 | 711 | if (patterns) { 712 | for (i = 0, len = patterns.length; i < len; i++) { 713 | pattern = patterns[i]; 714 | patternId = -1; 715 | 716 | if (pattern.include) { 717 | if (pattern.include.charAt(0) === '#') { 718 | // Local include found in `repository` 719 | let localIncludedRule = repository[pattern.include.substr(1)]; 720 | if (localIncludedRule) { 721 | patternId = RuleFactory.getCompiledRuleId(localIncludedRule, helper, repository); 722 | } else { 723 | // console.warn('CANNOT find rule for scopeName: ' + pattern.include + ', I am: ', repository['$base'].name); 724 | } 725 | } else if (pattern.include === '$base' || pattern.include === '$self') { 726 | // Special include also found in `repository` 727 | patternId = RuleFactory.getCompiledRuleId(repository[pattern.include], helper, repository); 728 | } else { 729 | let externalGrammarName: string = null, 730 | externalGrammarInclude: string = null, 731 | sharpIndex = pattern.include.indexOf('#'); 732 | if (sharpIndex >= 0) { 733 | externalGrammarName = pattern.include.substring(0, sharpIndex); 734 | externalGrammarInclude = pattern.include.substring(sharpIndex + 1); 735 | } else { 736 | externalGrammarName = pattern.include; 737 | } 738 | // External include 739 | externalGrammar = helper.getExternalGrammar(externalGrammarName, repository); 740 | 741 | if (externalGrammar) { 742 | if (externalGrammarInclude) { 743 | let externalIncludedRule = externalGrammar.repository[externalGrammarInclude]; 744 | if (externalIncludedRule) { 745 | patternId = RuleFactory.getCompiledRuleId(externalIncludedRule, helper, externalGrammar.repository); 746 | } else { 747 | // console.warn('CANNOT find rule for scopeName: ' + pattern.include + ', I am: ', repository['$base'].name); 748 | } 749 | } else { 750 | patternId = RuleFactory.getCompiledRuleId(externalGrammar.repository.$self, helper, externalGrammar.repository); 751 | } 752 | } else { 753 | // console.warn('CANNOT find grammar for scopeName: ' + pattern.include + ', I am: ', repository['$base'].name); 754 | } 755 | 756 | } 757 | } else { 758 | patternId = RuleFactory.getCompiledRuleId(pattern, helper, repository); 759 | } 760 | 761 | if (patternId !== -1) { 762 | rule = helper.getRule(patternId); 763 | 764 | skipRule = false; 765 | 766 | if (rule instanceof IncludeOnlyRule || rule instanceof BeginEndRule || rule instanceof BeginWhileRule) { 767 | if (rule.hasMissingPatterns && rule.patterns.length === 0) { 768 | skipRule = true; 769 | } 770 | } 771 | 772 | if (skipRule) { 773 | // console.log('REMOVING RULE ENTIRELY DUE TO EMPTY PATTERNS THAT ARE MISSING'); 774 | continue; 775 | } 776 | 777 | r.push(patternId); 778 | } 779 | } 780 | } 781 | 782 | return { 783 | patterns: r, 784 | hasMissingPatterns: ((patterns ? patterns.length : 0) !== r.length) 785 | }; 786 | } 787 | } 788 | -------------------------------------------------------------------------------- /src/theme.ts: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------- 2 | * Copyright (C) Microsoft Corporation. All rights reserved. 3 | *--------------------------------------------------------*/ 4 | 5 | import { IRawTheme } from './main'; 6 | 7 | export const enum FontStyle { 8 | NotSet = -1, 9 | None = 0, 10 | Italic = 1, 11 | Bold = 2, 12 | Underline = 4 13 | } 14 | 15 | export class ParsedThemeRule { 16 | _parsedThemeRuleBrand: void; 17 | 18 | readonly scope: string; 19 | readonly parentScopes: string[]; 20 | readonly index: number; 21 | 22 | /** 23 | * -1 if not set. An or mask of `FontStyle` otherwise. 24 | */ 25 | readonly fontStyle: number; 26 | readonly foreground: string; 27 | readonly background: string; 28 | 29 | constructor( 30 | scope: string, 31 | parentScopes: string[], 32 | index: number, 33 | fontStyle: number, 34 | foreground: string, 35 | background: string, 36 | ) { 37 | this.scope = scope; 38 | this.parentScopes = parentScopes; 39 | this.index = index; 40 | this.fontStyle = fontStyle; 41 | this.foreground = foreground; 42 | this.background = background; 43 | } 44 | } 45 | 46 | function isValidHexColor(hex: string): boolean { 47 | if (/^#[0-9a-f]{6}$/i.test(hex)) { 48 | // #rrggbb 49 | return true; 50 | } 51 | 52 | if (/^#[0-9a-f]{8}$/i.test(hex)) { 53 | // #rrggbbaa 54 | return true; 55 | } 56 | 57 | if (/^#[0-9a-f]{3}$/i.test(hex)) { 58 | // #rgb 59 | return true; 60 | } 61 | 62 | if (/^#[0-9a-f]{4}$/i.test(hex)) { 63 | // #rgba 64 | return true; 65 | } 66 | 67 | return false; 68 | } 69 | 70 | /** 71 | * Parse a raw theme into rules. 72 | */ 73 | export function parseTheme(source: IRawTheme): ParsedThemeRule[] { 74 | if (!source) { 75 | return []; 76 | } 77 | if (!source.settings || !Array.isArray(source.settings)) { 78 | return []; 79 | } 80 | let settings = source.settings; 81 | let result: ParsedThemeRule[] = [], resultLen = 0; 82 | for (let i = 0, len = settings.length; i < len; i++) { 83 | let entry = settings[i]; 84 | 85 | if (!entry.settings) { 86 | continue; 87 | } 88 | 89 | let scopes: string[]; 90 | if (typeof entry.scope === 'string') { 91 | let _scope = entry.scope; 92 | 93 | // remove leading commas 94 | _scope = _scope.replace(/^[,]+/, ''); 95 | 96 | // remove trailing commans 97 | _scope = _scope.replace(/[,]+$/, ''); 98 | 99 | scopes = _scope.split(','); 100 | } else if (Array.isArray(entry.scope)) { 101 | scopes = entry.scope; 102 | } else { 103 | scopes = ['']; 104 | } 105 | 106 | let fontStyle: number = FontStyle.NotSet; 107 | if (typeof entry.settings.fontStyle === 'string') { 108 | fontStyle = FontStyle.None; 109 | 110 | let segments = entry.settings.fontStyle.split(' '); 111 | for (let j = 0, lenJ = segments.length; j < lenJ; j++) { 112 | let segment = segments[j]; 113 | switch (segment) { 114 | case 'italic': 115 | fontStyle = fontStyle | FontStyle.Italic; 116 | break; 117 | case 'bold': 118 | fontStyle = fontStyle | FontStyle.Bold; 119 | break; 120 | case 'underline': 121 | fontStyle = fontStyle | FontStyle.Underline; 122 | break; 123 | } 124 | } 125 | } 126 | 127 | let foreground: string = null; 128 | if (typeof entry.settings.foreground === 'string' && isValidHexColor(entry.settings.foreground)) { 129 | foreground = entry.settings.foreground; 130 | } 131 | 132 | let background: string = null; 133 | if (typeof entry.settings.background === 'string' && isValidHexColor(entry.settings.background)) { 134 | background = entry.settings.background; 135 | } 136 | 137 | for (let j = 0, lenJ = scopes.length; j < lenJ; j++) { 138 | let _scope = scopes[j].trim(); 139 | 140 | let segments = _scope.split(' '); 141 | 142 | let scope = segments[segments.length - 1]; 143 | let parentScopes: string[] = null; 144 | if (segments.length > 1) { 145 | parentScopes = segments.slice(0, segments.length - 1); 146 | parentScopes.reverse(); 147 | } 148 | 149 | result[resultLen++] = new ParsedThemeRule( 150 | scope, 151 | parentScopes, 152 | i, 153 | fontStyle, 154 | foreground, 155 | background 156 | ); 157 | } 158 | } 159 | 160 | return result; 161 | } 162 | 163 | /** 164 | * Resolve rules (i.e. inheritance). 165 | */ 166 | function resolveParsedThemeRules(parsedThemeRules: ParsedThemeRule[]): Theme { 167 | 168 | // Sort rules lexicographically, and then by index if necessary 169 | parsedThemeRules.sort((a, b) => { 170 | let r = strcmp(a.scope, b.scope); 171 | if (r !== 0) { 172 | return r; 173 | } 174 | r = strArrCmp(a.parentScopes, b.parentScopes); 175 | if (r !== 0) { 176 | return r; 177 | } 178 | return a.index - b.index; 179 | }); 180 | 181 | // Determine defaults 182 | let defaultFontStyle = FontStyle.None; 183 | let defaultForeground = '#000000'; 184 | let defaultBackground = '#ffffff'; 185 | while (parsedThemeRules.length >= 1 && parsedThemeRules[0].scope === '') { 186 | let incomingDefaults = parsedThemeRules.shift(); 187 | if (incomingDefaults.fontStyle !== FontStyle.NotSet) { 188 | defaultFontStyle = incomingDefaults.fontStyle; 189 | } 190 | if (incomingDefaults.foreground !== null) { 191 | defaultForeground = incomingDefaults.foreground; 192 | } 193 | if (incomingDefaults.background !== null) { 194 | defaultBackground = incomingDefaults.background; 195 | } 196 | } 197 | let colorMap = new ColorMap(); 198 | let defaults = new ThemeTrieElementRule(0, null, defaultFontStyle, colorMap.getId(defaultForeground), colorMap.getId(defaultBackground)); 199 | 200 | let root = new ThemeTrieElement(new ThemeTrieElementRule(0, null, FontStyle.NotSet, 0, 0), []); 201 | for (let i = 0, len = parsedThemeRules.length; i < len; i++) { 202 | let rule = parsedThemeRules[i]; 203 | root.insert(0, rule.scope, rule.parentScopes, rule.fontStyle, colorMap.getId(rule.foreground), colorMap.getId(rule.background)); 204 | } 205 | 206 | return new Theme(colorMap, defaults, root); 207 | } 208 | 209 | export class ColorMap { 210 | 211 | private _lastColorId: number; 212 | private _id2color: string[]; 213 | private _color2id: { [color: string]: number; }; 214 | 215 | constructor() { 216 | this._lastColorId = 0; 217 | this._id2color = []; 218 | this._color2id = Object.create(null); 219 | } 220 | 221 | public getId(color: string): number { 222 | if (color === null) { 223 | return 0; 224 | } 225 | color = color.toUpperCase(); 226 | let value = this._color2id[color]; 227 | if (value) { 228 | return value; 229 | } 230 | value = ++this._lastColorId; 231 | this._color2id[color] = value; 232 | this._id2color[value] = color; 233 | return value; 234 | } 235 | 236 | public getColorMap(): string[] { 237 | return this._id2color.slice(0); 238 | } 239 | 240 | } 241 | 242 | export class Theme { 243 | 244 | public static createFromRawTheme(source: IRawTheme): Theme { 245 | return this.createFromParsedTheme(parseTheme(source)); 246 | } 247 | 248 | public static createFromParsedTheme(source: ParsedThemeRule[]): Theme { 249 | return resolveParsedThemeRules(source); 250 | } 251 | 252 | private readonly _colorMap: ColorMap; 253 | private readonly _root: ThemeTrieElement; 254 | private readonly _defaults: ThemeTrieElementRule; 255 | private readonly _cache: { [scopeName: string]: ThemeTrieElementRule[]; }; 256 | 257 | constructor(colorMap: ColorMap, defaults: ThemeTrieElementRule, root: ThemeTrieElement) { 258 | this._colorMap = colorMap; 259 | this._root = root; 260 | this._defaults = defaults; 261 | this._cache = {}; 262 | } 263 | 264 | public getColorMap(): string[] { 265 | return this._colorMap.getColorMap(); 266 | } 267 | 268 | public getDefaults(): ThemeTrieElementRule { 269 | return this._defaults; 270 | } 271 | 272 | public match(scopeName: string): ThemeTrieElementRule[] { 273 | if (!this._cache.hasOwnProperty(scopeName)) { 274 | this._cache[scopeName] = this._root.match(scopeName); 275 | } 276 | return this._cache[scopeName]; 277 | } 278 | } 279 | 280 | export function strcmp(a: string, b: string): number { 281 | if (a < b) { 282 | return -1; 283 | } 284 | if (a > b) { 285 | return 1; 286 | } 287 | return 0; 288 | } 289 | 290 | export function strArrCmp(a: string[], b: string[]): number { 291 | if (a === null && b === null) { 292 | return 0; 293 | } 294 | if (!a) { 295 | return -1; 296 | } 297 | if (!b) { 298 | return 1; 299 | } 300 | let len1 = a.length; 301 | let len2 = b.length; 302 | if (len1 === len2) { 303 | for (let i = 0; i < len1; i++) { 304 | let res = strcmp(a[i], b[i]); 305 | if (res !== 0) { 306 | return res; 307 | } 308 | } 309 | return 0; 310 | } 311 | return len1 - len2; 312 | } 313 | 314 | export class ThemeTrieElementRule { 315 | _themeTrieElementRuleBrand: void; 316 | 317 | scopeDepth: number; 318 | parentScopes: string[]; 319 | fontStyle: number; 320 | foreground: number; 321 | background: number; 322 | 323 | constructor(scopeDepth: number, parentScopes: string[], fontStyle: number, foreground: number, background: number) { 324 | this.scopeDepth = scopeDepth; 325 | this.parentScopes = parentScopes; 326 | this.fontStyle = fontStyle; 327 | this.foreground = foreground; 328 | this.background = background; 329 | } 330 | 331 | public clone(): ThemeTrieElementRule { 332 | return new ThemeTrieElementRule(this.scopeDepth, this.parentScopes, this.fontStyle, this.foreground, this.background); 333 | } 334 | 335 | public static cloneArr(arr:ThemeTrieElementRule[]): ThemeTrieElementRule[] { 336 | let r: ThemeTrieElementRule[] = []; 337 | for (let i = 0, len = arr.length; i < len; i++) { 338 | r[i] = arr[i].clone(); 339 | } 340 | return r; 341 | } 342 | 343 | public acceptOverwrite(scopeDepth: number, fontStyle: number, foreground: number, background: number): void { 344 | if (this.scopeDepth > scopeDepth) { 345 | console.log('how did this happen?'); 346 | } else { 347 | this.scopeDepth = scopeDepth; 348 | } 349 | // console.log('TODO -> my depth: ' + this.scopeDepth + ', overwriting depth: ' + scopeDepth); 350 | if (fontStyle !== FontStyle.NotSet) { 351 | this.fontStyle = fontStyle; 352 | } 353 | if (foreground !== 0) { 354 | this.foreground = foreground; 355 | } 356 | if (background !== 0) { 357 | this.background = background; 358 | } 359 | } 360 | } 361 | 362 | export interface ITrieChildrenMap { 363 | [segment: string]: ThemeTrieElement; 364 | } 365 | 366 | export class ThemeTrieElement { 367 | _themeTrieElementBrand: void; 368 | 369 | private readonly _mainRule: ThemeTrieElementRule; 370 | private readonly _rulesWithParentScopes: ThemeTrieElementRule[]; 371 | private readonly _children: ITrieChildrenMap; 372 | 373 | constructor( 374 | mainRule: ThemeTrieElementRule, 375 | rulesWithParentScopes: ThemeTrieElementRule[] = [], 376 | children: ITrieChildrenMap = {} 377 | ) { 378 | this._mainRule = mainRule; 379 | this._rulesWithParentScopes = rulesWithParentScopes; 380 | this._children = children; 381 | } 382 | 383 | private static _sortBySpecificity(arr: ThemeTrieElementRule[]): ThemeTrieElementRule[] { 384 | if (arr.length === 1) { 385 | return arr; 386 | } 387 | 388 | arr.sort(this._cmpBySpecificity); 389 | 390 | return arr; 391 | } 392 | 393 | private static _cmpBySpecificity(a: ThemeTrieElementRule, b: ThemeTrieElementRule): number { 394 | if (a.scopeDepth === b.scopeDepth) { 395 | const aParentScopes = a.parentScopes; 396 | const bParentScopes = b.parentScopes; 397 | let aParentScopesLen = aParentScopes === null ? 0 : aParentScopes.length; 398 | let bParentScopesLen = bParentScopes === null ? 0 : bParentScopes.length; 399 | if (aParentScopesLen === bParentScopesLen) { 400 | for (let i = 0; i < aParentScopesLen; i++) { 401 | const aLen = aParentScopes[i].length; 402 | const bLen = bParentScopes[i].length; 403 | if (aLen !== bLen) { 404 | return bLen - aLen; 405 | } 406 | } 407 | } 408 | return bParentScopesLen - aParentScopesLen; 409 | } 410 | return b.scopeDepth - a.scopeDepth; 411 | } 412 | 413 | public match(scope: string): ThemeTrieElementRule[] { 414 | if (scope === '') { 415 | return ThemeTrieElement._sortBySpecificity([].concat(this._mainRule).concat(this._rulesWithParentScopes)); 416 | } 417 | 418 | let dotIndex = scope.indexOf('.'); 419 | let head: string; 420 | let tail: string; 421 | if (dotIndex === -1) { 422 | head = scope; 423 | tail = ''; 424 | } else { 425 | head = scope.substring(0, dotIndex); 426 | tail = scope.substring(dotIndex + 1); 427 | } 428 | 429 | if (this._children.hasOwnProperty(head)) { 430 | return this._children[head].match(tail); 431 | } 432 | 433 | return ThemeTrieElement._sortBySpecificity([].concat(this._mainRule).concat(this._rulesWithParentScopes)); 434 | } 435 | 436 | public insert(scopeDepth: number, scope: string, parentScopes: string[], fontStyle: number, foreground: number, background: number): void { 437 | if (scope === '') { 438 | this._doInsertHere(scopeDepth, parentScopes, fontStyle, foreground, background); 439 | return; 440 | } 441 | 442 | let dotIndex = scope.indexOf('.'); 443 | let head: string; 444 | let tail: string; 445 | if (dotIndex === -1) { 446 | head = scope; 447 | tail = ''; 448 | } else { 449 | head = scope.substring(0, dotIndex); 450 | tail = scope.substring(dotIndex + 1); 451 | } 452 | 453 | let child: ThemeTrieElement; 454 | if (this._children.hasOwnProperty(head)) { 455 | child = this._children[head]; 456 | } else { 457 | child = new ThemeTrieElement(this._mainRule.clone(), ThemeTrieElementRule.cloneArr(this._rulesWithParentScopes)); 458 | this._children[head] = child; 459 | } 460 | 461 | child.insert(scopeDepth + 1, tail, parentScopes, fontStyle, foreground, background); 462 | } 463 | 464 | private _doInsertHere(scopeDepth: number, parentScopes: string[], fontStyle: number, foreground: number, background: number): void { 465 | 466 | if (parentScopes === null) { 467 | // Merge into the main rule 468 | this._mainRule.acceptOverwrite(scopeDepth, fontStyle, foreground, background); 469 | return; 470 | } 471 | 472 | // Try to merge into existing rule 473 | for (let i = 0, len = this._rulesWithParentScopes.length; i < len; i++) { 474 | let rule = this._rulesWithParentScopes[i]; 475 | 476 | if (strArrCmp(rule.parentScopes, parentScopes) === 0) { 477 | // bingo! => we get to merge this into an existing one 478 | rule.acceptOverwrite(scopeDepth, fontStyle, foreground, background); 479 | return; 480 | } 481 | } 482 | 483 | // Must add a new rule 484 | 485 | // Inherit from main rule 486 | if (fontStyle === FontStyle.NotSet) { 487 | fontStyle = this._mainRule.fontStyle; 488 | } 489 | if (foreground === 0) { 490 | foreground = this._mainRule.foreground; 491 | } 492 | if (background === 0) { 493 | background = this._mainRule.background; 494 | } 495 | 496 | this._rulesWithParentScopes.push(new ThemeTrieElementRule(scopeDepth, parentScopes, fontStyle, foreground, background)); 497 | } 498 | } 499 | -------------------------------------------------------------------------------- /src/types.ts: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------- 2 | * Copyright (C) Microsoft Corporation. All rights reserved. 3 | *--------------------------------------------------------*/ 4 | 5 | // -- raw grammar typings 6 | 7 | export interface ILocation { 8 | readonly filename: string; 9 | readonly line: number; 10 | readonly char: number; 11 | } 12 | 13 | export interface ILocatable { 14 | readonly $vscodeTextmateLocation?: ILocation; 15 | } 16 | 17 | export interface IGrammarDefinition { 18 | format: 'json' | 'plist'; 19 | content: string | object; 20 | } 21 | 22 | export interface IRawGrammar extends ILocatable { 23 | repository: IRawRepository; 24 | readonly scopeName: string; 25 | readonly patterns: IRawRule[]; 26 | readonly injections?: { [expression: string]: IRawRule }; 27 | readonly injectionSelector?: string; 28 | 29 | readonly fileTypes?: string[]; 30 | readonly name?: string; 31 | readonly firstLineMatch?: string; 32 | } 33 | 34 | export interface IRawRepositoryMap { 35 | [name: string]: IRawRule; 36 | $self: IRawRule; 37 | $base: IRawRule; 38 | } 39 | 40 | export type IRawRepository = IRawRepositoryMap & ILocatable; 41 | 42 | export interface IRawRule extends ILocatable { 43 | id?: number; 44 | 45 | readonly include?: string; 46 | 47 | readonly name?: string; 48 | readonly contentName?: string; 49 | 50 | readonly match?: string; 51 | readonly captures?: IRawCaptures; 52 | readonly begin?: string; 53 | readonly beginCaptures?: IRawCaptures; 54 | readonly end?: string; 55 | readonly endCaptures?: IRawCaptures; 56 | readonly while?: string; 57 | readonly whileCaptures?: IRawCaptures; 58 | readonly patterns?: IRawRule[]; 59 | 60 | readonly repository?: IRawRepository; 61 | 62 | readonly applyEndPatternLast?: boolean; 63 | } 64 | 65 | export interface IRawCapturesMap { 66 | [captureId: string]: IRawRule; 67 | } 68 | 69 | export type IRawCaptures = IRawCapturesMap & ILocatable; 70 | -------------------------------------------------------------------------------- /src/utils.ts: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------- 2 | * Copyright (C) Microsoft Corporation. All rights reserved. 3 | *--------------------------------------------------------*/ 4 | 5 | import { IOnigCaptureIndex } from 'onigasm'; 6 | 7 | export function clone(something: T): T { 8 | return doClone(something); 9 | } 10 | 11 | function doClone(something: any): any { 12 | if (Array.isArray(something)) { 13 | return cloneArray(something); 14 | } 15 | if (typeof something === 'object') { 16 | return cloneObj(something); 17 | } 18 | return something; 19 | } 20 | 21 | function cloneArray(arr: any[]): any[] { 22 | let r: any[] = []; 23 | for (let i = 0, len = arr.length; i < len; i++) { 24 | r[i] = doClone(arr[i]); 25 | } 26 | return r; 27 | } 28 | 29 | function cloneObj(obj: any): any { 30 | let r: any = {}; 31 | for (let key in obj) { 32 | r[key] = doClone(obj[key]); 33 | } 34 | return r; 35 | } 36 | 37 | export function mergeObjects(target: any, ...sources: any[]): any { 38 | sources.forEach(source => { 39 | for (let key in source) { 40 | target[key] = source[key]; 41 | } 42 | }); 43 | return target; 44 | } 45 | 46 | let CAPTURING_REGEX_SOURCE = /\$(\d+)|\${(\d+):\/(downcase|upcase)}/; 47 | 48 | export class RegexSource { 49 | 50 | public static hasCaptures(regexSource: string): boolean { 51 | return CAPTURING_REGEX_SOURCE.test(regexSource); 52 | } 53 | 54 | public static replaceCaptures(regexSource: string, captureSource: string, captureIndices: IOnigCaptureIndex[]): string { 55 | return regexSource.replace(CAPTURING_REGEX_SOURCE, (match: string, index: string, commandIndex: string, command: string) => { 56 | let capture = captureIndices[parseInt(index || commandIndex, 10)]; 57 | if (capture) { 58 | let result = captureSource.substring(capture.start, capture.end); 59 | // Remove leading dots that would make the selector invalid 60 | while (result[0] === '.') { 61 | result = result.substring(1); 62 | } 63 | switch (command) { 64 | case 'downcase': 65 | return result.toLowerCase(); 66 | case 'upcase': 67 | return result.toUpperCase(); 68 | default: 69 | return result; 70 | } 71 | } else { 72 | return match; 73 | } 74 | }); 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es5", 4 | "module": "commonjs", 5 | "outDir": "dist", 6 | "declarationDir": "dist/typings", 7 | "noImplicitAny": false, 8 | "sourceMap": true, 9 | "declaration": true, 10 | "lib": ["esnext"] 11 | }, 12 | "exclude": [ 13 | "node_modules", 14 | "dist", 15 | ] 16 | } -------------------------------------------------------------------------------- /tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "rules": { 3 | "no-unused-expression": true, 4 | "no-duplicate-variable": true, 5 | "no-duplicate-key": true, 6 | "no-unused-variable": true, 7 | "curly": true, 8 | "class-name": true, 9 | "semicolon": [ 10 | "always" 11 | ], 12 | "triple-equals": true 13 | } 14 | } --------------------------------------------------------------------------------