├── .gitignore ├── package.json ├── lib ├── main.coffee ├── javascript-semantic-grammar.coffee └── acorn-modified.js ├── LICENSE.md ├── styles └── semantic-colors.atom-text-editor.less └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | npm-debug.log 3 | node_modules 4 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "language-javascript-semantic", 3 | "main": "./lib/main", 4 | "version": "0.2.1", 5 | "description": "Semantic highlighting for JavaScript", 6 | "repository": "https://github.com/p-e-w/language-javascript-semantic", 7 | "license": "MIT", 8 | "engines": { 9 | "atom": ">=0.174.0 <2.0.0" 10 | }, 11 | "dependencies": { 12 | "jquery": "^2", 13 | "first-mate": ">=1.7.0" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /lib/main.coffee: -------------------------------------------------------------------------------- 1 | # JavaScript Semantic Highlighting Package for Atom 2 | # 3 | # Copyright (c) 2014-2015 Philipp Emanuel Weidmann 4 | # 5 | # Nemo vir est qui mundum non reddat meliorem. 6 | # 7 | # Released under the terms of the MIT License (http://opensource.org/licenses/MIT) 8 | 9 | JavaScriptSemanticGrammar = require "./javascript-semantic-grammar" 10 | 11 | module.exports = 12 | activate: (state) -> 13 | atom.grammars.addGrammar(new JavaScriptSemanticGrammar(atom.grammars)) 14 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright © 2014-2015 Philipp Emanuel Weidmann () 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /styles/semantic-colors.atom-text-editor.less: -------------------------------------------------------------------------------- 1 | /* 2 | * JavaScript Semantic Highlighting Package for Atom 3 | * 4 | * Copyright (c) 2014-2015 Philipp Emanuel Weidmann 5 | * 6 | * Nemo vir est qui mundum non reddat meliorem. 7 | * 8 | * Released under the terms of the MIT License (http://opensource.org/licenses/MIT) 9 | */ 10 | 11 | @import "syntax-variables"; 12 | 13 | @bright-text-color: 14 | contrast(@syntax-background-color, 15 | lighten(@syntax-text-color, 20%), 16 | darken(@syntax-text-color, 20%)); 17 | 18 | .text-color(@contrast) { 19 | color: mix(@bright-text-color, @syntax-background-color, @contrast); 20 | } 21 | 22 | .source.js-semantic { 23 | // Reset font attributes to prevent interference from main syntax theme 24 | .text-color(80%); 25 | background-color: transparent; 26 | font-style: normal; 27 | font-weight: normal; 28 | text-decoration: none; 29 | 30 | &.identifier { 31 | .color-indices(8); 32 | 33 | .color-indices(@n, @i: 1) when (@i =< @n) { 34 | @hue: @i * (360 / @n); 35 | &.color-index-@{i} { 36 | // Choose a color of the given hue with good contrast 37 | color: contrast(@syntax-background-color, 38 | hsl(@hue, 100%, 25%), 39 | hsl(@hue, 65%, 75%)); 40 | } 41 | .color-indices(@n, @i + 1); 42 | } 43 | } 44 | 45 | &.comment { 46 | .text-color(40%); 47 | } 48 | 49 | &.keyword { 50 | font-weight: bold; 51 | } 52 | 53 | &.number { 54 | .text-color(100%); 55 | } 56 | 57 | &.string, 58 | &.regex { 59 | // TODO: Currently, these are simply the string colors from Atom's default syntax themes 60 | color: contrast(@syntax-background-color, #a8ff60, #dd1144); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ### :red_circle: This project is *on hold* pending resolution of [atom#8669](https://github.com/atom/atom/issues/8669). I'm tired of chasing an undocumented private API. 2 | 3 | --- 4 | 5 | 6 | ## See Your JavaScript Code in a New Light 7 | 8 | This [Atom](https://atom.io/) package enables [semantic highlighting](https://medium.com/programming-ideas-tutorial-and-experience/coding-in-color-3a6db2743a1e) for JavaScript code. Identifiers are highlighted in different colors (the same identifier always in the same color) while everything else (like language keywords) is displayed in various shades of gray. This approach runs contrary to classical ideas about syntax highlighting but is rapidly becoming very popular. 9 | 10 | ![With Dark Theme](https://raw.githubusercontent.com/p-e-w/language-javascript-semantic/images/screenshot-dark-theme.png) 11 | ![With Light Theme](https://raw.githubusercontent.com/p-e-w/language-javascript-semantic/images/screenshot-light-theme.png) 12 | 13 | In addition to being a useful tool for actual, productive JavaScript coding, the package also demonstrates some techniques that might serve other developers when creating similar packages for other languages: 14 | 15 | * Advanced use of Less to **dynamically create a syntax theme** that goes well with the existing one 16 | * **A language grammar that is defined programmatically** rather than through a `.cson` grammar file 17 | * Connecting an **external parser** ([Acorn](https://github.com/marijnh/acorn) in this case) 18 | 19 | Acorn is a mature, fast JavaScript parser that is available through [npm](https://www.npmjs.org/). Unfortunately, the requirements for an Atom grammar necessitated two customizations: 20 | 21 | * Support for tokenizing unterminated multi-line comments (required to allow incremental, i.e. line-by-line, tokenizing) 22 | * Reverting to regex- rather than `eval`-based keyword recognition to work around [Atom Shell](https://github.com/atom/atom-shell)'s CSP restrictions (this problem is being tracked in https://github.com/marijnh/acorn/issues/90) 23 | 24 | As a result, this package ships with a modified version of Acorn, but it would be preferable if those issues could be worked out so that Acorn can be pulled from npm in the future. 25 | 26 | ## Acknowledgments 27 | 28 | ### Prior Art 29 | 30 | * [Coding in Color](https://medium.com/programming-ideas-tutorial-and-experience/coding-in-color-3a6db2743a1e): The blog post that started the current semantic highlighting craze, which in turn acknowledges [Semantic Highlighting in KDevelop](http://zwabel.wordpress.com/2009/01/08/c-ide-evolution-from-syntax-highlighting-to-semantic-highlighting/) 31 | * [Sublime-Colorcoder](https://github.com/vprimachenko/Sublime-Colorcoder): Ingenious plugin to enable semantic highlighting through Sublime Text's highly limited plugin API by *dynamically generating a TextMate theme!* 32 | * [recognizer](https://github.com/equiet/recognizer): A very advanced semantic editing plugin for [Brackets](http://brackets.io/) 33 | * [Polychromatic](https://github.com/kolinkrewinkel/Polychromatic): Semantic highlighting plugin for Xcode 34 | 35 | ### Dependencies 36 | 37 | * [Acorn](https://github.com/marijnh/acorn): JavaScript parser 38 | 39 | ## License 40 | 41 | Copyright © 2014-2015 Philipp Emanuel Weidmann () 42 | 43 | Released under the terms of the [MIT License](http://opensource.org/licenses/MIT) 44 | -------------------------------------------------------------------------------- /lib/javascript-semantic-grammar.coffee: -------------------------------------------------------------------------------- 1 | # JavaScript Semantic Highlighting Package for Atom 2 | # 3 | # Copyright (c) 2014-2015 Philipp Emanuel Weidmann 4 | # 5 | # Nemo vir est qui mundum non reddat meliorem. 6 | # 7 | # Released under the terms of the MIT License (http://opensource.org/licenses/MIT) 8 | 9 | $ = require "jquery" 10 | {Grammar} = require "first-mate" 11 | acorn = require "./acorn-modified.js" 12 | 13 | numberOfColors = 8 14 | 15 | module.exports = 16 | class JavaScriptSemanticGrammar extends Grammar 17 | constructor: (registry) -> 18 | name = "JavaScript (Semantic Highlighting)" 19 | scopeName = "source.js-semantic" 20 | super(registry, {name, scopeName}) 21 | 22 | # Ensures that grammar takes precedence over standard JavaScript grammar 23 | getScore: -> 24 | jsGrammar = @registry.grammarForScopeName("source.js") 25 | return if jsGrammar? then (jsGrammar.getScore.apply(jsGrammar, arguments) + 1) else 0 26 | 27 | acornTokenize: (line) -> 28 | tokens = [] 29 | rules = [] 30 | 31 | onComment = (block, unterminated, text, start, end) -> 32 | # Add a faux-token for comment since Acorn doesn't tokenize comments 33 | tokens.push { start: start, end: end, type: { type: "comment", unterminated: unterminated } } 34 | if unterminated 35 | rules.push "unterminated_comment" 36 | 37 | try 38 | tokenizer = acorn.tokenize(line, { locations: true, onComment: onComment }) 39 | catch error 40 | # Error in initTokenState 41 | return { tokens: tokens, rules: rules } 42 | 43 | while true 44 | try 45 | token = tokenizer() 46 | catch error 47 | return { tokens: tokens, rules: rules } 48 | # Object is mutable, therefore it must be cloned 49 | token = $.extend(true, {}, token) 50 | if token.type.type is "eof" 51 | return { tokens: tokens, rules: rules } 52 | tokens.push token 53 | 54 | # Converted from http://stackoverflow.com/a/7616484 55 | # with the help of http://js2coffee.org/ 56 | hash: (string) -> 57 | hash = 0 58 | return hash if string.length is 0 59 | i = 0 60 | len = string.length 61 | while i < len 62 | chr = string.charCodeAt(i) 63 | hash = ((hash << 5) - hash) + chr 64 | hash |= 0 65 | i++ 66 | return hash 67 | 68 | colorIndex: (string) -> 69 | (Math.abs(@hash(string)) % numberOfColors) + 1 70 | 71 | tokenScopes: (token, text) -> 72 | if token.type.type is "name" 73 | colorIndexScope = "color-index-" + @colorIndex(text) 74 | return "identifier." + colorIndexScope 75 | else if token.type.type is "comment" 76 | return "comment" 77 | else if token.type.hasOwnProperty("keyword") 78 | return "keyword" 79 | else if token.type.type is "num" 80 | return "number" 81 | else if token.type.type is "string" 82 | return "string" 83 | else if token.type.type is "regexp" 84 | return "regex" 85 | return null 86 | 87 | tokenizeLine: (line, ruleStack, firstLine = false) -> 88 | tags = [] 89 | tokens = [] 90 | 91 | outerRegistry = @registry 92 | addToken = (text, scopes = null) -> 93 | fullScopes = "source.js-semantic" + (if scopes? then ("." + scopes) else "") 94 | tags.push outerRegistry.startIdForScope(fullScopes) 95 | tags.push text.length 96 | tags.push outerRegistry.endIdForScope(fullScopes) 97 | tokens.push { value: text, scopes: [fullScopes] } 98 | 99 | acornStartOffset = 0 100 | if ruleStack? and "unterminated_comment" in ruleStack 101 | # Help Acorn tokenize multi-line comments correctly 102 | commentEnd = line.indexOf("*/") 103 | if commentEnd is -1 104 | # Multi-line comment continues 105 | addToken line, "comment" 106 | return { line: line, tags: tags, tokens: tokens, ruleStack: ruleStack } 107 | else 108 | # Make Acorn skip over partial comment 109 | acornStartOffset = commentEnd + 2 110 | addToken line.substring(0, acornStartOffset), "comment" 111 | 112 | acornLine = line.substring(acornStartOffset) 113 | 114 | tokenizeResult = @acornTokenize(acornLine) 115 | acornTokens = tokenizeResult.tokens 116 | # Comment tokens might have been inserted in the wrong place 117 | acornTokens.sort((a, b) -> a.start - b.start) 118 | 119 | tokenPos = 0 120 | for token in acornTokens 121 | text = acornLine.substring(token.start, token.end) 122 | tokenScopes = @tokenScopes(token, text) 123 | if tokenScopes? 124 | if token.start > tokenPos 125 | addToken acornLine.substring(tokenPos, token.start) 126 | addToken text, tokenScopes 127 | tokenPos = token.end 128 | 129 | if tokenPos < acornLine.length 130 | addToken acornLine.substring(tokenPos) 131 | 132 | if tokens.length is 0 133 | addToken "" 134 | 135 | return { line: line, tags: tags, tokens: tokens, ruleStack: tokenizeResult.rules } 136 | -------------------------------------------------------------------------------- /lib/acorn-modified.js: -------------------------------------------------------------------------------- 1 | // Modified version of Acorn for Atom package "language-javascript-semantic". 2 | // Branched off from Acorn commit 7f0f07e1f9c227d61147f2962f3418149b55e66a. 3 | 4 | // Acorn is a tiny, fast JavaScript parser written in JavaScript. 5 | // 6 | // Acorn was written by Marijn Haverbeke and released under an MIT 7 | // license. The Unicode regexps (for identifiers and whitespace) were 8 | // taken from [Esprima](http://esprima.org) by Ariya Hidayat. 9 | // 10 | // Git repositories for Acorn are available at 11 | // 12 | // http://marijnhaverbeke.nl/git/acorn 13 | // https://github.com/marijnh/acorn.git 14 | // 15 | // Please use the [github bug tracker][ghbt] to report issues. 16 | // 17 | // [ghbt]: https://github.com/marijnh/acorn/issues 18 | // 19 | // This file defines the main parser interface. The library also comes 20 | // with a [error-tolerant parser][dammit] and an 21 | // [abstract syntax tree walker][walk], defined in other files. 22 | // 23 | // [dammit]: acorn_loose.js 24 | // [walk]: util/walk.js 25 | 26 | (function(root, mod) { 27 | if (typeof exports == "object" && typeof module == "object") return mod(exports); // CommonJS 28 | if (typeof define == "function" && define.amd) return define(["exports"], mod); // AMD 29 | mod(root.acorn || (root.acorn = {})); // Plain browser env 30 | })(this, function(exports) { 31 | "use strict"; 32 | 33 | exports.version = "0.5.1"; 34 | 35 | // The main exported interface (under `self.acorn` when in the 36 | // browser) is a `parse` function that takes a code string and 37 | // returns an abstract syntax tree as specified by [Mozilla parser 38 | // API][api], with the caveat that the SpiderMonkey-specific syntax 39 | // (`let`, `yield`, inline XML, etc) is not recognized. 40 | // 41 | // [api]: https://developer.mozilla.org/en-US/docs/SpiderMonkey/Parser_API 42 | 43 | var options, input, inputLen, sourceFile; 44 | 45 | exports.parse = function(inpt, opts) { 46 | input = String(inpt); inputLen = input.length; 47 | setOptions(opts); 48 | initTokenState(); 49 | return parseTopLevel(options.program); 50 | }; 51 | 52 | // A second optional argument can be given to further configure 53 | // the parser process. These options are recognized: 54 | 55 | var defaultOptions = exports.defaultOptions = { 56 | // `ecmaVersion` indicates the ECMAScript version to parse. Must 57 | // be either 3 or 5. This 58 | // influences support for strict mode, the set of reserved words, and 59 | // support for getters and setter. 60 | ecmaVersion: 5, 61 | // Turn on `strictSemicolons` to prevent the parser from doing 62 | // automatic semicolon insertion. 63 | strictSemicolons: false, 64 | // When `allowTrailingCommas` is false, the parser will not allow 65 | // trailing commas in array and object literals. 66 | allowTrailingCommas: true, 67 | // By default, reserved words are not enforced. Enable 68 | // `forbidReserved` to enforce them. When this option has the 69 | // value "everywhere", reserved words and keywords can also not be 70 | // used as property names. 71 | forbidReserved: false, 72 | // When enabled, a return at the top level is not considered an 73 | // error. 74 | allowReturnOutsideFunction: false, 75 | // When `locations` is on, `loc` properties holding objects with 76 | // `start` and `end` properties in `{line, column}` form (with 77 | // line being 1-based and column 0-based) will be attached to the 78 | // nodes. 79 | locations: false, 80 | // A function can be passed as `onComment` option, which will 81 | // cause Acorn to call that function with `(block, unterminated, text, start, 82 | // end)` parameters whenever a comment is skipped. `block` is a 83 | // boolean indicating whether this is a block (`/* */`) comment, 84 | // `text` is the content of the comment, and `start` and `end` are 85 | // character offsets that denote the start and end of the comment. 86 | // When the `locations` option is on, two more parameters are 87 | // passed, the full `{line, column}` locations of the start and 88 | // end of the comments. Note that you are not allowed to call the 89 | // parser from the callback—that will corrupt its internal state. 90 | onComment: null, 91 | // Nodes have their start and end characters offsets recorded in 92 | // `start` and `end` properties (directly on the node, rather than 93 | // the `loc` object, which holds line/column data. To also add a 94 | // [semi-standardized][range] `range` property holding a `[start, 95 | // end]` array with the same numbers, set the `ranges` option to 96 | // `true`. 97 | // 98 | // [range]: https://bugzilla.mozilla.org/show_bug.cgi?id=745678 99 | ranges: false, 100 | // It is possible to parse multiple files into a single AST by 101 | // passing the tree produced by parsing the first file as 102 | // `program` option in subsequent parses. This will add the 103 | // toplevel forms of the parsed file to the `Program` (top) node 104 | // of an existing parse tree. 105 | program: null, 106 | // When `locations` is on, you can pass this to record the source 107 | // file in every node's `loc` object. 108 | sourceFile: null, 109 | // This value, if given, is stored in every node, whether 110 | // `locations` is on or off. 111 | directSourceFile: null 112 | }; 113 | 114 | function setOptions(opts) { 115 | options = opts || {}; 116 | for (var opt in defaultOptions) if (!Object.prototype.hasOwnProperty.call(options, opt)) 117 | options[opt] = defaultOptions[opt]; 118 | sourceFile = options.sourceFile || null; 119 | } 120 | 121 | // The `getLineInfo` function is mostly useful when the 122 | // `locations` option is off (for performance reasons) and you 123 | // want to find the line/column position for a given character 124 | // offset. `input` should be the code string that the offset refers 125 | // into. 126 | 127 | var getLineInfo = exports.getLineInfo = function(input, offset) { 128 | for (var line = 1, cur = 0;;) { 129 | lineBreak.lastIndex = cur; 130 | var match = lineBreak.exec(input); 131 | if (match && match.index < offset) { 132 | ++line; 133 | cur = match.index + match[0].length; 134 | } else break; 135 | } 136 | return {line: line, column: offset - cur}; 137 | }; 138 | 139 | // Acorn is organized as a tokenizer and a recursive-descent parser. 140 | // The `tokenize` export provides an interface to the tokenizer. 141 | // Because the tokenizer is optimized for being efficiently used by 142 | // the Acorn parser itself, this interface is somewhat crude and not 143 | // very modular. Performing another parse or call to `tokenize` will 144 | // reset the internal state, and invalidate existing tokenizers. 145 | 146 | exports.tokenize = function(inpt, opts) { 147 | input = String(inpt); inputLen = input.length; 148 | setOptions(opts); 149 | initTokenState(); 150 | 151 | var t = {}; 152 | function getToken(forceRegexp) { 153 | lastEnd = tokEnd; 154 | readToken(forceRegexp); 155 | t.start = tokStart; t.end = tokEnd; 156 | t.startLoc = tokStartLoc; t.endLoc = tokEndLoc; 157 | t.type = tokType; t.value = tokVal; 158 | return t; 159 | } 160 | getToken.jumpTo = function(pos, reAllowed) { 161 | tokPos = pos; 162 | if (options.locations) { 163 | tokCurLine = 1; 164 | tokLineStart = lineBreak.lastIndex = 0; 165 | var match; 166 | while ((match = lineBreak.exec(input)) && match.index < pos) { 167 | ++tokCurLine; 168 | tokLineStart = match.index + match[0].length; 169 | } 170 | } 171 | tokRegexpAllowed = reAllowed; 172 | skipSpace(); 173 | }; 174 | return getToken; 175 | }; 176 | 177 | // State is kept in (closure-)global variables. We already saw the 178 | // `options`, `input`, and `inputLen` variables above. 179 | 180 | // The current position of the tokenizer in the input. 181 | 182 | var tokPos; 183 | 184 | // The start and end offsets of the current token. 185 | 186 | var tokStart, tokEnd; 187 | 188 | // When `options.locations` is true, these hold objects 189 | // containing the tokens start and end line/column pairs. 190 | 191 | var tokStartLoc, tokEndLoc; 192 | 193 | // The type and value of the current token. Token types are objects, 194 | // named by variables against which they can be compared, and 195 | // holding properties that describe them (indicating, for example, 196 | // the precedence of an infix operator, and the original name of a 197 | // keyword token). The kind of value that's held in `tokVal` depends 198 | // on the type of the token. For literals, it is the literal value, 199 | // for operators, the operator name, and so on. 200 | 201 | var tokType, tokVal; 202 | 203 | // Interal state for the tokenizer. To distinguish between division 204 | // operators and regular expressions, it remembers whether the last 205 | // token was one that is allowed to be followed by an expression. 206 | // (If it is, a slash is probably a regexp, if it isn't it's a 207 | // division operator. See the `parseStatement` function for a 208 | // caveat.) 209 | 210 | var tokRegexpAllowed; 211 | 212 | // When `options.locations` is true, these are used to keep 213 | // track of the current line, and know when a new line has been 214 | // entered. 215 | 216 | var tokCurLine, tokLineStart; 217 | 218 | // These store the position of the previous token, which is useful 219 | // when finishing a node and assigning its `end` position. 220 | 221 | var lastStart, lastEnd, lastEndLoc; 222 | 223 | // This is the parser's state. `inFunction` is used to reject 224 | // `return` statements outside of functions, `labels` to verify that 225 | // `break` and `continue` have somewhere to jump to, and `strict` 226 | // indicates whether strict mode is on. 227 | 228 | var inFunction, labels, strict; 229 | 230 | // This function is used to raise exceptions on parse errors. It 231 | // takes an offset integer (into the current `input`) to indicate 232 | // the location of the error, attaches the position to the end 233 | // of the error message, and then raises a `SyntaxError` with that 234 | // message. 235 | 236 | function raise(pos, message) { 237 | var loc = getLineInfo(input, pos); 238 | message += " (" + loc.line + ":" + loc.column + ")"; 239 | var err = new SyntaxError(message); 240 | err.pos = pos; err.loc = loc; err.raisedAt = tokPos; 241 | throw err; 242 | } 243 | 244 | // Reused empty array added for node fields that are always empty. 245 | 246 | var empty = []; 247 | 248 | // ## Token types 249 | 250 | // The assignment of fine-grained, information-carrying type objects 251 | // allows the tokenizer to store the information it has about a 252 | // token in a way that is very cheap for the parser to look up. 253 | 254 | // All token type variables start with an underscore, to make them 255 | // easy to recognize. 256 | 257 | // These are the general types. The `type` property is only used to 258 | // make them recognizeable when debugging. 259 | 260 | var _num = {type: "num"}, _regexp = {type: "regexp"}, _string = {type: "string"}; 261 | var _name = {type: "name"}, _eof = {type: "eof"}; 262 | 263 | // Keyword tokens. The `keyword` property (also used in keyword-like 264 | // operators) indicates that the token originated from an 265 | // identifier-like word, which is used when parsing property names. 266 | // 267 | // The `beforeExpr` property is used to disambiguate between regular 268 | // expressions and divisions. It is set on all token types that can 269 | // be followed by an expression (thus, a slash after them would be a 270 | // regular expression). 271 | // 272 | // `isLoop` marks a keyword as starting a loop, which is important 273 | // to know when parsing a label, in order to allow or disallow 274 | // continue jumps to that label. 275 | 276 | var _break = {keyword: "break"}, _case = {keyword: "case", beforeExpr: true}, _catch = {keyword: "catch"}; 277 | var _continue = {keyword: "continue"}, _debugger = {keyword: "debugger"}, _default = {keyword: "default"}; 278 | var _do = {keyword: "do", isLoop: true}, _else = {keyword: "else", beforeExpr: true}; 279 | var _finally = {keyword: "finally"}, _for = {keyword: "for", isLoop: true}, _function = {keyword: "function"}; 280 | var _if = {keyword: "if"}, _return = {keyword: "return", beforeExpr: true}, _switch = {keyword: "switch"}; 281 | var _throw = {keyword: "throw", beforeExpr: true}, _try = {keyword: "try"}, _var = {keyword: "var"}; 282 | var _while = {keyword: "while", isLoop: true}, _with = {keyword: "with"}, _new = {keyword: "new", beforeExpr: true}; 283 | var _this = {keyword: "this"}; 284 | 285 | // The keywords that denote values. 286 | 287 | var _null = {keyword: "null", atomValue: null}, _true = {keyword: "true", atomValue: true}; 288 | var _false = {keyword: "false", atomValue: false}; 289 | 290 | // Some keywords are treated as regular operators. `in` sometimes 291 | // (when parsing `for`) needs to be tested against specifically, so 292 | // we assign a variable name to it for quick comparing. 293 | 294 | var _in = {keyword: "in", binop: 7, beforeExpr: true}; 295 | 296 | // Map keyword names to token types. 297 | 298 | var keywordTypes = {"break": _break, "case": _case, "catch": _catch, 299 | "continue": _continue, "debugger": _debugger, "default": _default, 300 | "do": _do, "else": _else, "finally": _finally, "for": _for, 301 | "function": _function, "if": _if, "return": _return, "switch": _switch, 302 | "throw": _throw, "try": _try, "var": _var, "while": _while, "with": _with, 303 | "null": _null, "true": _true, "false": _false, "new": _new, "in": _in, 304 | "instanceof": {keyword: "instanceof", binop: 7, beforeExpr: true}, "this": _this, 305 | "typeof": {keyword: "typeof", prefix: true, beforeExpr: true}, 306 | "void": {keyword: "void", prefix: true, beforeExpr: true}, 307 | "delete": {keyword: "delete", prefix: true, beforeExpr: true}}; 308 | 309 | // Punctuation token types. Again, the `type` property is purely for debugging. 310 | 311 | var _bracketL = {type: "[", beforeExpr: true}, _bracketR = {type: "]"}, _braceL = {type: "{", beforeExpr: true}; 312 | var _braceR = {type: "}"}, _parenL = {type: "(", beforeExpr: true}, _parenR = {type: ")"}; 313 | var _comma = {type: ",", beforeExpr: true}, _semi = {type: ";", beforeExpr: true}; 314 | var _colon = {type: ":", beforeExpr: true}, _dot = {type: "."}, _ellipsis = {type: "..."}, _question = {type: "?", beforeExpr: true}; 315 | 316 | // Operators. These carry several kinds of properties to help the 317 | // parser use them properly (the presence of these properties is 318 | // what categorizes them as operators). 319 | // 320 | // `binop`, when present, specifies that this operator is a binary 321 | // operator, and will refer to its precedence. 322 | // 323 | // `prefix` and `postfix` mark the operator as a prefix or postfix 324 | // unary operator. `isUpdate` specifies that the node produced by 325 | // the operator should be of type UpdateExpression rather than 326 | // simply UnaryExpression (`++` and `--`). 327 | // 328 | // `isAssign` marks all of `=`, `+=`, `-=` etcetera, which act as 329 | // binary operators with a very low precedence, that should result 330 | // in AssignmentExpression nodes. 331 | 332 | var _slash = {binop: 10, beforeExpr: true}, _eq = {isAssign: true, beforeExpr: true}; 333 | var _assign = {isAssign: true, beforeExpr: true}; 334 | var _incDec = {postfix: true, prefix: true, isUpdate: true}, _prefix = {prefix: true, beforeExpr: true}; 335 | var _logicalOR = {binop: 1, beforeExpr: true}; 336 | var _logicalAND = {binop: 2, beforeExpr: true}; 337 | var _bitwiseOR = {binop: 3, beforeExpr: true}; 338 | var _bitwiseXOR = {binop: 4, beforeExpr: true}; 339 | var _bitwiseAND = {binop: 5, beforeExpr: true}; 340 | var _equality = {binop: 6, beforeExpr: true}; 341 | var _relational = {binop: 7, beforeExpr: true}; 342 | var _bitShift = {binop: 8, beforeExpr: true}; 343 | var _plusMin = {binop: 9, prefix: true, beforeExpr: true}; 344 | var _multiplyModulo = {binop: 10, beforeExpr: true}; 345 | 346 | // Provide access to the token types for external users of the 347 | // tokenizer. 348 | 349 | exports.tokTypes = {bracketL: _bracketL, bracketR: _bracketR, braceL: _braceL, braceR: _braceR, 350 | parenL: _parenL, parenR: _parenR, comma: _comma, semi: _semi, colon: _colon, 351 | dot: _dot, ellipsis: _ellipsis, question: _question, slash: _slash, eq: _eq, 352 | name: _name, eof: _eof, num: _num, regexp: _regexp, string: _string}; 353 | for (var kw in keywordTypes) exports.tokTypes["_" + kw] = keywordTypes[kw]; 354 | 355 | // This is a trick taken from Esprima. It turns out that, on 356 | // non-Chrome browsers, to check whether a string is in a set, a 357 | // predicate containing a big ugly `switch` statement is faster than 358 | // a regular expression, and on Chrome the two are about on par. 359 | // This function uses `eval` (non-lexical) to produce such a 360 | // predicate from a space-separated string of words. 361 | // 362 | // It starts by sorting the words by length. 363 | 364 | function makePredicate(words) { 365 | words = words.split(" "); 366 | var f = "", cats = []; 367 | out: for (var i = 0; i < words.length; ++i) { 368 | for (var j = 0; j < cats.length; ++j) 369 | if (cats[j][0].length == words[i].length) { 370 | cats[j].push(words[i]); 371 | continue out; 372 | } 373 | cats.push([words[i]]); 374 | } 375 | function compareTo(arr) { 376 | if (arr.length == 1) return f += "return str === " + JSON.stringify(arr[0]) + ";"; 377 | f += "switch(str){"; 378 | for (var i = 0; i < arr.length; ++i) f += "case " + JSON.stringify(arr[i]) + ":"; 379 | f += "return true}return false;"; 380 | } 381 | 382 | // When there are more than three length categories, an outer 383 | // switch first dispatches on the lengths, to save on comparisons. 384 | 385 | if (cats.length > 3) { 386 | cats.sort(function(a, b) {return b.length - a.length;}); 387 | f += "switch(str.length){"; 388 | for (var i = 0; i < cats.length; ++i) { 389 | var cat = cats[i]; 390 | f += "case " + cat[0].length + ":"; 391 | compareTo(cat); 392 | } 393 | f += "}"; 394 | 395 | // Otherwise, simply generate a flat `switch` statement. 396 | 397 | } else { 398 | compareTo(words); 399 | } 400 | return new Function("str", f); 401 | } 402 | 403 | // The ECMAScript 3 reserved word list. 404 | 405 | // Regexes from https://github.com/marijnh/acorn/blob/a1d958751911faa06a18d0f99d5ca98d053ce655/acorn.js 406 | 407 | var reservedWords3 = /^(?:abstract|boolean|byte|char|class|double|enum|export|extends|final|float|goto|implements|import|int|interface|long|native|package|private|protected|public|short|static|super|synchronized|throws|transient|volatile)$/; 408 | var isReservedWord3 = function(word) { return reservedWords3.test(word); }; 409 | 410 | // ECMAScript 5 reserved words. 411 | 412 | var reservedWords5 = /^(?:class|enum|extends|super|const|export|import)$/; 413 | var isReservedWord5 = function(word) { return reservedWords5.test(word); }; 414 | 415 | // The additional reserved words in strict mode. 416 | 417 | var strictReservedWords = /^(?:implements|interface|let|package|private|protected|public|static|yield)$/; 418 | var isStrictReservedWord = function(word) { return strictReservedWords.test(word); }; 419 | 420 | // The forbidden variable names in strict mode. 421 | 422 | var strictBadWords = /^(?:eval|arguments)$/; 423 | var isStrictBadIdWord = function(word) { return strictBadWords.test(word); }; 424 | 425 | // And the keywords. 426 | 427 | var keywords = /^(?:break|case|catch|continue|debugger|default|do|else|finally|for|function|if|return|switch|throw|try|var|while|with|null|true|false|instanceof|typeof|void|delete|new|in|this)$/; 428 | var isKeyword = function(word) { return keywords.test(word); }; 429 | 430 | // ## Character categories 431 | 432 | // Big ugly regular expressions that match characters in the 433 | // whitespace, identifier, and identifier-start categories. These 434 | // are only applied when a character is found to actually have a 435 | // code point above 128. 436 | 437 | var nonASCIIwhitespace = /[\u1680\u180e\u2000-\u200a\u202f\u205f\u3000\ufeff]/; 438 | var nonASCIIidentifierStartChars = "\xaa\xb5\xba\xc0-\xd6\xd8-\xf6\xf8-\u02c1\u02c6-\u02d1\u02e0-\u02e4\u02ec\u02ee\u0370-\u0374\u0376\u0377\u037a-\u037d\u0386\u0388-\u038a\u038c\u038e-\u03a1\u03a3-\u03f5\u03f7-\u0481\u048a-\u0527\u0531-\u0556\u0559\u0561-\u0587\u05d0-\u05ea\u05f0-\u05f2\u0620-\u064a\u066e\u066f\u0671-\u06d3\u06d5\u06e5\u06e6\u06ee\u06ef\u06fa-\u06fc\u06ff\u0710\u0712-\u072f\u074d-\u07a5\u07b1\u07ca-\u07ea\u07f4\u07f5\u07fa\u0800-\u0815\u081a\u0824\u0828\u0840-\u0858\u08a0\u08a2-\u08ac\u0904-\u0939\u093d\u0950\u0958-\u0961\u0971-\u0977\u0979-\u097f\u0985-\u098c\u098f\u0990\u0993-\u09a8\u09aa-\u09b0\u09b2\u09b6-\u09b9\u09bd\u09ce\u09dc\u09dd\u09df-\u09e1\u09f0\u09f1\u0a05-\u0a0a\u0a0f\u0a10\u0a13-\u0a28\u0a2a-\u0a30\u0a32\u0a33\u0a35\u0a36\u0a38\u0a39\u0a59-\u0a5c\u0a5e\u0a72-\u0a74\u0a85-\u0a8d\u0a8f-\u0a91\u0a93-\u0aa8\u0aaa-\u0ab0\u0ab2\u0ab3\u0ab5-\u0ab9\u0abd\u0ad0\u0ae0\u0ae1\u0b05-\u0b0c\u0b0f\u0b10\u0b13-\u0b28\u0b2a-\u0b30\u0b32\u0b33\u0b35-\u0b39\u0b3d\u0b5c\u0b5d\u0b5f-\u0b61\u0b71\u0b83\u0b85-\u0b8a\u0b8e-\u0b90\u0b92-\u0b95\u0b99\u0b9a\u0b9c\u0b9e\u0b9f\u0ba3\u0ba4\u0ba8-\u0baa\u0bae-\u0bb9\u0bd0\u0c05-\u0c0c\u0c0e-\u0c10\u0c12-\u0c28\u0c2a-\u0c33\u0c35-\u0c39\u0c3d\u0c58\u0c59\u0c60\u0c61\u0c85-\u0c8c\u0c8e-\u0c90\u0c92-\u0ca8\u0caa-\u0cb3\u0cb5-\u0cb9\u0cbd\u0cde\u0ce0\u0ce1\u0cf1\u0cf2\u0d05-\u0d0c\u0d0e-\u0d10\u0d12-\u0d3a\u0d3d\u0d4e\u0d60\u0d61\u0d7a-\u0d7f\u0d85-\u0d96\u0d9a-\u0db1\u0db3-\u0dbb\u0dbd\u0dc0-\u0dc6\u0e01-\u0e30\u0e32\u0e33\u0e40-\u0e46\u0e81\u0e82\u0e84\u0e87\u0e88\u0e8a\u0e8d\u0e94-\u0e97\u0e99-\u0e9f\u0ea1-\u0ea3\u0ea5\u0ea7\u0eaa\u0eab\u0ead-\u0eb0\u0eb2\u0eb3\u0ebd\u0ec0-\u0ec4\u0ec6\u0edc-\u0edf\u0f00\u0f40-\u0f47\u0f49-\u0f6c\u0f88-\u0f8c\u1000-\u102a\u103f\u1050-\u1055\u105a-\u105d\u1061\u1065\u1066\u106e-\u1070\u1075-\u1081\u108e\u10a0-\u10c5\u10c7\u10cd\u10d0-\u10fa\u10fc-\u1248\u124a-\u124d\u1250-\u1256\u1258\u125a-\u125d\u1260-\u1288\u128a-\u128d\u1290-\u12b0\u12b2-\u12b5\u12b8-\u12be\u12c0\u12c2-\u12c5\u12c8-\u12d6\u12d8-\u1310\u1312-\u1315\u1318-\u135a\u1380-\u138f\u13a0-\u13f4\u1401-\u166c\u166f-\u167f\u1681-\u169a\u16a0-\u16ea\u16ee-\u16f0\u1700-\u170c\u170e-\u1711\u1720-\u1731\u1740-\u1751\u1760-\u176c\u176e-\u1770\u1780-\u17b3\u17d7\u17dc\u1820-\u1877\u1880-\u18a8\u18aa\u18b0-\u18f5\u1900-\u191c\u1950-\u196d\u1970-\u1974\u1980-\u19ab\u19c1-\u19c7\u1a00-\u1a16\u1a20-\u1a54\u1aa7\u1b05-\u1b33\u1b45-\u1b4b\u1b83-\u1ba0\u1bae\u1baf\u1bba-\u1be5\u1c00-\u1c23\u1c4d-\u1c4f\u1c5a-\u1c7d\u1ce9-\u1cec\u1cee-\u1cf1\u1cf5\u1cf6\u1d00-\u1dbf\u1e00-\u1f15\u1f18-\u1f1d\u1f20-\u1f45\u1f48-\u1f4d\u1f50-\u1f57\u1f59\u1f5b\u1f5d\u1f5f-\u1f7d\u1f80-\u1fb4\u1fb6-\u1fbc\u1fbe\u1fc2-\u1fc4\u1fc6-\u1fcc\u1fd0-\u1fd3\u1fd6-\u1fdb\u1fe0-\u1fec\u1ff2-\u1ff4\u1ff6-\u1ffc\u2071\u207f\u2090-\u209c\u2102\u2107\u210a-\u2113\u2115\u2119-\u211d\u2124\u2126\u2128\u212a-\u212d\u212f-\u2139\u213c-\u213f\u2145-\u2149\u214e\u2160-\u2188\u2c00-\u2c2e\u2c30-\u2c5e\u2c60-\u2ce4\u2ceb-\u2cee\u2cf2\u2cf3\u2d00-\u2d25\u2d27\u2d2d\u2d30-\u2d67\u2d6f\u2d80-\u2d96\u2da0-\u2da6\u2da8-\u2dae\u2db0-\u2db6\u2db8-\u2dbe\u2dc0-\u2dc6\u2dc8-\u2dce\u2dd0-\u2dd6\u2dd8-\u2dde\u2e2f\u3005-\u3007\u3021-\u3029\u3031-\u3035\u3038-\u303c\u3041-\u3096\u309d-\u309f\u30a1-\u30fa\u30fc-\u30ff\u3105-\u312d\u3131-\u318e\u31a0-\u31ba\u31f0-\u31ff\u3400-\u4db5\u4e00-\u9fcc\ua000-\ua48c\ua4d0-\ua4fd\ua500-\ua60c\ua610-\ua61f\ua62a\ua62b\ua640-\ua66e\ua67f-\ua697\ua6a0-\ua6ef\ua717-\ua71f\ua722-\ua788\ua78b-\ua78e\ua790-\ua793\ua7a0-\ua7aa\ua7f8-\ua801\ua803-\ua805\ua807-\ua80a\ua80c-\ua822\ua840-\ua873\ua882-\ua8b3\ua8f2-\ua8f7\ua8fb\ua90a-\ua925\ua930-\ua946\ua960-\ua97c\ua984-\ua9b2\ua9cf\uaa00-\uaa28\uaa40-\uaa42\uaa44-\uaa4b\uaa60-\uaa76\uaa7a\uaa80-\uaaaf\uaab1\uaab5\uaab6\uaab9-\uaabd\uaac0\uaac2\uaadb-\uaadd\uaae0-\uaaea\uaaf2-\uaaf4\uab01-\uab06\uab09-\uab0e\uab11-\uab16\uab20-\uab26\uab28-\uab2e\uabc0-\uabe2\uac00-\ud7a3\ud7b0-\ud7c6\ud7cb-\ud7fb\uf900-\ufa6d\ufa70-\ufad9\ufb00-\ufb06\ufb13-\ufb17\ufb1d\ufb1f-\ufb28\ufb2a-\ufb36\ufb38-\ufb3c\ufb3e\ufb40\ufb41\ufb43\ufb44\ufb46-\ufbb1\ufbd3-\ufd3d\ufd50-\ufd8f\ufd92-\ufdc7\ufdf0-\ufdfb\ufe70-\ufe74\ufe76-\ufefc\uff21-\uff3a\uff41-\uff5a\uff66-\uffbe\uffc2-\uffc7\uffca-\uffcf\uffd2-\uffd7\uffda-\uffdc"; 439 | var nonASCIIidentifierChars = "\u0300-\u036f\u0483-\u0487\u0591-\u05bd\u05bf\u05c1\u05c2\u05c4\u05c5\u05c7\u0610-\u061a\u0620-\u0649\u0672-\u06d3\u06e7-\u06e8\u06fb-\u06fc\u0730-\u074a\u0800-\u0814\u081b-\u0823\u0825-\u0827\u0829-\u082d\u0840-\u0857\u08e4-\u08fe\u0900-\u0903\u093a-\u093c\u093e-\u094f\u0951-\u0957\u0962-\u0963\u0966-\u096f\u0981-\u0983\u09bc\u09be-\u09c4\u09c7\u09c8\u09d7\u09df-\u09e0\u0a01-\u0a03\u0a3c\u0a3e-\u0a42\u0a47\u0a48\u0a4b-\u0a4d\u0a51\u0a66-\u0a71\u0a75\u0a81-\u0a83\u0abc\u0abe-\u0ac5\u0ac7-\u0ac9\u0acb-\u0acd\u0ae2-\u0ae3\u0ae6-\u0aef\u0b01-\u0b03\u0b3c\u0b3e-\u0b44\u0b47\u0b48\u0b4b-\u0b4d\u0b56\u0b57\u0b5f-\u0b60\u0b66-\u0b6f\u0b82\u0bbe-\u0bc2\u0bc6-\u0bc8\u0bca-\u0bcd\u0bd7\u0be6-\u0bef\u0c01-\u0c03\u0c46-\u0c48\u0c4a-\u0c4d\u0c55\u0c56\u0c62-\u0c63\u0c66-\u0c6f\u0c82\u0c83\u0cbc\u0cbe-\u0cc4\u0cc6-\u0cc8\u0cca-\u0ccd\u0cd5\u0cd6\u0ce2-\u0ce3\u0ce6-\u0cef\u0d02\u0d03\u0d46-\u0d48\u0d57\u0d62-\u0d63\u0d66-\u0d6f\u0d82\u0d83\u0dca\u0dcf-\u0dd4\u0dd6\u0dd8-\u0ddf\u0df2\u0df3\u0e34-\u0e3a\u0e40-\u0e45\u0e50-\u0e59\u0eb4-\u0eb9\u0ec8-\u0ecd\u0ed0-\u0ed9\u0f18\u0f19\u0f20-\u0f29\u0f35\u0f37\u0f39\u0f41-\u0f47\u0f71-\u0f84\u0f86-\u0f87\u0f8d-\u0f97\u0f99-\u0fbc\u0fc6\u1000-\u1029\u1040-\u1049\u1067-\u106d\u1071-\u1074\u1082-\u108d\u108f-\u109d\u135d-\u135f\u170e-\u1710\u1720-\u1730\u1740-\u1750\u1772\u1773\u1780-\u17b2\u17dd\u17e0-\u17e9\u180b-\u180d\u1810-\u1819\u1920-\u192b\u1930-\u193b\u1951-\u196d\u19b0-\u19c0\u19c8-\u19c9\u19d0-\u19d9\u1a00-\u1a15\u1a20-\u1a53\u1a60-\u1a7c\u1a7f-\u1a89\u1a90-\u1a99\u1b46-\u1b4b\u1b50-\u1b59\u1b6b-\u1b73\u1bb0-\u1bb9\u1be6-\u1bf3\u1c00-\u1c22\u1c40-\u1c49\u1c5b-\u1c7d\u1cd0-\u1cd2\u1d00-\u1dbe\u1e01-\u1f15\u200c\u200d\u203f\u2040\u2054\u20d0-\u20dc\u20e1\u20e5-\u20f0\u2d81-\u2d96\u2de0-\u2dff\u3021-\u3028\u3099\u309a\ua640-\ua66d\ua674-\ua67d\ua69f\ua6f0-\ua6f1\ua7f8-\ua800\ua806\ua80b\ua823-\ua827\ua880-\ua881\ua8b4-\ua8c4\ua8d0-\ua8d9\ua8f3-\ua8f7\ua900-\ua909\ua926-\ua92d\ua930-\ua945\ua980-\ua983\ua9b3-\ua9c0\uaa00-\uaa27\uaa40-\uaa41\uaa4c-\uaa4d\uaa50-\uaa59\uaa7b\uaae0-\uaae9\uaaf2-\uaaf3\uabc0-\uabe1\uabec\uabed\uabf0-\uabf9\ufb20-\ufb28\ufe00-\ufe0f\ufe20-\ufe26\ufe33\ufe34\ufe4d-\ufe4f\uff10-\uff19\uff3f"; 440 | var nonASCIIidentifierStart = new RegExp("[" + nonASCIIidentifierStartChars + "]"); 441 | var nonASCIIidentifier = new RegExp("[" + nonASCIIidentifierStartChars + nonASCIIidentifierChars + "]"); 442 | 443 | // Whether a single character denotes a newline. 444 | 445 | var newline = /[\n\r\u2028\u2029]/; 446 | 447 | // Matches a whole line break (where CRLF is considered a single 448 | // line break). Used to count lines. 449 | 450 | var lineBreak = /\r\n|[\n\r\u2028\u2029]/g; 451 | 452 | // Test whether a given character code starts an identifier. 453 | 454 | var isIdentifierStart = exports.isIdentifierStart = function(code) { 455 | if (code < 65) return code === 36; 456 | if (code < 91) return true; 457 | if (code < 97) return code === 95; 458 | if (code < 123)return true; 459 | return code >= 0xaa && nonASCIIidentifierStart.test(String.fromCharCode(code)); 460 | }; 461 | 462 | // Test whether a given character is part of an identifier. 463 | 464 | var isIdentifierChar = exports.isIdentifierChar = function(code) { 465 | if (code < 48) return code === 36; 466 | if (code < 58) return true; 467 | if (code < 65) return false; 468 | if (code < 91) return true; 469 | if (code < 97) return code === 95; 470 | if (code < 123)return true; 471 | return code >= 0xaa && nonASCIIidentifier.test(String.fromCharCode(code)); 472 | }; 473 | 474 | // ## Tokenizer 475 | 476 | // These are used when `options.locations` is on, for the 477 | // `tokStartLoc` and `tokEndLoc` properties. 478 | 479 | function Position() { 480 | this.line = tokCurLine; 481 | this.column = tokPos - tokLineStart; 482 | } 483 | 484 | // Reset the token state. Used at the start of a parse. 485 | 486 | function initTokenState() { 487 | tokCurLine = 1; 488 | tokPos = tokLineStart = 0; 489 | tokRegexpAllowed = true; 490 | skipSpace(); 491 | } 492 | 493 | // Called at the end of every token. Sets `tokEnd`, `tokVal`, and 494 | // `tokRegexpAllowed`, and skips the space after the token, so that 495 | // the next one's `tokStart` will point at the right position. 496 | 497 | function finishToken(type, val) { 498 | tokEnd = tokPos; 499 | if (options.locations) tokEndLoc = new Position; 500 | tokType = type; 501 | skipSpace(); 502 | tokVal = val; 503 | tokRegexpAllowed = type.beforeExpr; 504 | } 505 | 506 | function skipBlockComment() { 507 | var startLoc = options.onComment && options.locations && new Position; 508 | var start = tokPos, end = input.indexOf("*/", tokPos += 2); 509 | var unterminated = (end === -1); 510 | tokPos = unterminated ? input.length : end + 2; 511 | 512 | if (options.locations) { 513 | lineBreak.lastIndex = start; 514 | var match; 515 | while ((match = lineBreak.exec(input)) && match.index < tokPos) { 516 | ++tokCurLine; 517 | tokLineStart = match.index + match[0].length; 518 | } 519 | } 520 | if (options.onComment) 521 | options.onComment(true, unterminated, input.slice(start + 2, end), start, tokPos, 522 | startLoc, options.locations && new Position); 523 | } 524 | 525 | function skipLineComment() { 526 | var start = tokPos; 527 | var startLoc = options.onComment && options.locations && new Position; 528 | var ch = input.charCodeAt(tokPos+=2); 529 | while (tokPos < inputLen && ch !== 10 && ch !== 13 && ch !== 8232 && ch !== 8233) { 530 | ++tokPos; 531 | ch = input.charCodeAt(tokPos); 532 | } 533 | if (options.onComment) 534 | options.onComment(false, false, input.slice(start + 2, tokPos), start, tokPos, 535 | startLoc, options.locations && new Position); 536 | } 537 | 538 | // Called at the start of the parse and after every token. Skips 539 | // whitespace and comments, and. 540 | 541 | function skipSpace() { 542 | while (tokPos < inputLen) { 543 | var ch = input.charCodeAt(tokPos); 544 | if (ch === 32) { // ' ' 545 | ++tokPos; 546 | } else if (ch === 13) { 547 | ++tokPos; 548 | var next = input.charCodeAt(tokPos); 549 | if (next === 10) { 550 | ++tokPos; 551 | } 552 | if (options.locations) { 553 | ++tokCurLine; 554 | tokLineStart = tokPos; 555 | } 556 | } else if (ch === 10 || ch === 8232 || ch === 8233) { 557 | ++tokPos; 558 | if (options.locations) { 559 | ++tokCurLine; 560 | tokLineStart = tokPos; 561 | } 562 | } else if (ch > 8 && ch < 14) { 563 | ++tokPos; 564 | } else if (ch === 47) { // '/' 565 | var next = input.charCodeAt(tokPos + 1); 566 | if (next === 42) { // '*' 567 | skipBlockComment(); 568 | } else if (next === 47) { // '/' 569 | skipLineComment(); 570 | } else break; 571 | } else if (ch === 160) { // '\xa0' 572 | ++tokPos; 573 | } else if (ch >= 5760 && nonASCIIwhitespace.test(String.fromCharCode(ch))) { 574 | ++tokPos; 575 | } else { 576 | break; 577 | } 578 | } 579 | } 580 | 581 | // ### Token reading 582 | 583 | // This is the function that is called to fetch the next token. It 584 | // is somewhat obscure, because it works in character codes rather 585 | // than characters, and because operator parsing has been inlined 586 | // into it. 587 | // 588 | // All in the name of speed. 589 | // 590 | // The `forceRegexp` parameter is used in the one case where the 591 | // `tokRegexpAllowed` trick does not work. See `parseStatement`. 592 | 593 | function readToken_dot() { 594 | var next = input.charCodeAt(tokPos + 1); 595 | if (next >= 48 && next <= 57) return readNumber(true); 596 | var next2 = input.charCodeAt(tokPos + 2); 597 | if (options.ecmaVersion >= 6 && next === 46 && next2 === 46) { // 46 = dot '.' 598 | tokPos += 3; 599 | return finishToken(_ellipsis); 600 | } else { 601 | ++tokPos; 602 | return finishToken(_dot); 603 | } 604 | } 605 | 606 | function readToken_slash() { // '/' 607 | var next = input.charCodeAt(tokPos + 1); 608 | if (tokRegexpAllowed) {++tokPos; return readRegexp();} 609 | if (next === 61) return finishOp(_assign, 2); 610 | return finishOp(_slash, 1); 611 | } 612 | 613 | function readToken_mult_modulo() { // '%*' 614 | var next = input.charCodeAt(tokPos + 1); 615 | if (next === 61) return finishOp(_assign, 2); 616 | return finishOp(_multiplyModulo, 1); 617 | } 618 | 619 | function readToken_pipe_amp(code) { // '|&' 620 | var next = input.charCodeAt(tokPos + 1); 621 | if (next === code) return finishOp(code === 124 ? _logicalOR : _logicalAND, 2); 622 | if (next === 61) return finishOp(_assign, 2); 623 | return finishOp(code === 124 ? _bitwiseOR : _bitwiseAND, 1); 624 | } 625 | 626 | function readToken_caret() { // '^' 627 | var next = input.charCodeAt(tokPos + 1); 628 | if (next === 61) return finishOp(_assign, 2); 629 | return finishOp(_bitwiseXOR, 1); 630 | } 631 | 632 | function readToken_plus_min(code) { // '+-' 633 | var next = input.charCodeAt(tokPos + 1); 634 | if (next === code) { 635 | if (next == 45 && input.charCodeAt(tokPos + 2) == 62 && 636 | newline.test(input.slice(lastEnd, tokPos))) { 637 | // A `-->` line comment 638 | tokPos += 3; 639 | skipLineComment(); 640 | skipSpace(); 641 | return readToken(); 642 | } 643 | return finishOp(_incDec, 2); 644 | } 645 | if (next === 61) return finishOp(_assign, 2); 646 | return finishOp(_plusMin, 1); 647 | } 648 | 649 | function readToken_lt_gt(code) { // '<>' 650 | var next = input.charCodeAt(tokPos + 1); 651 | var size = 1; 652 | if (next === code) { 653 | size = code === 62 && input.charCodeAt(tokPos + 2) === 62 ? 3 : 2; 654 | if (input.charCodeAt(tokPos + size) === 61) return finishOp(_assign, size + 1); 655 | return finishOp(_bitShift, size); 656 | } 657 | if (next == 33 && code == 60 && input.charCodeAt(tokPos + 2) == 45 && 658 | input.charCodeAt(tokPos + 3) == 45) { 659 | // `