├── .vscode └── launch.json ├── README.md ├── compiler.js ├── emitjs.js ├── parser.js ├── shuntingyard.js └── tokenizer.js /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | 8 | { 9 | "type": "node", 10 | "request": "launch", 11 | "name": "Launch Compiler", 12 | "program": "${workspaceFolder}/compiler.js" 13 | }, 14 | { 15 | "type": "node", 16 | "request": "launch", 17 | "name": "Launch Tokenizer", 18 | "program": "${workspaceFolder}/tokenizer.js" 19 | } 20 | ] 21 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Simple PL/0 Compiler 2 | 3 | This converts PL/0 code to JavaScript via an AST. 4 | 5 | The Shunting Yard Algorithm is used for mathematical expressions. 6 | 7 | Includes these bits: 8 | 9 | * tokenizer 10 | * parser 11 | * emitter (JavaScript) 12 | * shunting yard 13 | 14 | ## Usage 15 | 16 | Right now, the PL/0 source is hardcoded in `compiler.js`. (Obviously 17 | that's scheduled to change.) 18 | 19 | node compiler > foo.js 20 | node foo.js 21 | 22 | ## Caveats 23 | 24 | This isn't the most efficient code, by any means. The tokenizer and 25 | emitter constantly make new strings, for example. 26 | 27 | ## TODO 28 | 29 | * Accept input filename on command line 30 | * Finish implementing `ODD` 31 | * `?` statment 32 | * Report symbol table-related errors 33 | * Add a C emitter 34 | * Moar testing! 35 | 36 | ## References 37 | 38 | * [PL/0 at Wikipedia](https://en.wikipedia.org/wiki/PL/0) 39 | * [Recursive Descent Parsing at Wikipedia](https://en.wikipedia.org/wiki/Recursive_descent_parser) 40 | * [Shunting Yard Algorithm at Wikipedia](https://en.wikipedia.org/wiki/Shunting-yard_algorithm) 41 | * [Write your own (LISP-y) Compiler](http://blog.klipse.tech/javascript/2017/02/08/tiny-compiler-intro.html) -------------------------------------------------------------------------------- /compiler.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Main compiler executor 3 | */ 4 | 5 | const tokenizer = require('./tokenizer'); 6 | const parser = require('./parser'); 7 | const emitJS = require('./emitjs'); 8 | 9 | let input = `VAR x, sqr, temp; 10 | PROCEDURE square; 11 | BEGIN 12 | sqr:= x * x 13 | END; 14 | 15 | BEGIN 16 | x := 1; 17 | WHILE x <= 10 DO 18 | BEGIN 19 | CALL square; 20 | ! sqr; 21 | x := x + 1 22 | END 23 | END. 24 | `; 25 | 26 | let tokens = tokenizer(input); 27 | //console.log(JSON.stringify(tokens, null, 4)); 28 | 29 | let ast = parser(tokens); 30 | //console.log(JSON.stringify(ast, null, 4)); 31 | 32 | let js = emitJS(ast); 33 | console.log(js); -------------------------------------------------------------------------------- /emitjs.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Emitter (JavaScript) 3 | */ 4 | 5 | let output; 6 | 7 | /** 8 | * Write output inefficiently 9 | */ 10 | function out(s) { 11 | output += s; 12 | } 13 | 14 | /** 15 | * Procedure emitter 16 | */ 17 | function procedureEmitter(procedure) { 18 | //out('PROCEDURE'); 19 | out('function ' + procedure.name + '() {\n'); 20 | blockEmitter(procedure.block); 21 | out('}\n'); 22 | } 23 | 24 | /** 25 | * Condition emitter 26 | */ 27 | function expressionEmitter(expression) { 28 | const operMap = { 29 | UNARY_MINUS: '-', 30 | UNARY_PLUS: '+', 31 | MINUS: '-', 32 | PLUS: '+', 33 | MULT: '*', 34 | DIV: '/' 35 | }; 36 | 37 | //out('EXPR'); 38 | 39 | let root = expression.tree; 40 | 41 | function inOrder(node) { 42 | if (!node) { return; } 43 | 44 | if (node.type == 'Operator') { 45 | out('('); 46 | 47 | // Go left 48 | inOrder(node.operand[0]); 49 | 50 | let operSym = operMap[node.value]; 51 | 52 | if (!operSym) { 53 | throw 'unknown operator type: ' + node.value; 54 | } 55 | 56 | out(operSym); 57 | 58 | // Go right 59 | inOrder(node.operand[1]); 60 | 61 | out(')'); 62 | } 63 | 64 | else if (node.type == 'Operand') { 65 | out(node.value); 66 | } 67 | 68 | else { 69 | throw 'unknown expression node type: ' + node.type; 70 | } 71 | } 72 | 73 | inOrder(root); 74 | } 75 | 76 | /** 77 | * Expression emitter 78 | */ 79 | function conditionEmitter(condition) { 80 | const compMap = { 81 | EQ: '=', 82 | NE: '!=', 83 | LT: '<', 84 | GT: '>', 85 | LE: '<=', 86 | GE: '>=' 87 | }; 88 | 89 | if (condition.type == 'Odd') { 90 | out('((('); 91 | expressionEmitter(condition.expression); 92 | out(')%2) == 1)'); 93 | } 94 | 95 | else if (condition.type in compMap) { 96 | let comp = compMap[condition.type]; 97 | 98 | out('(('); 99 | expressionEmitter(condition.expression[0]); 100 | out(`) ${comp} (`); 101 | expressionEmitter(condition.expression[1]); 102 | out('))'); 103 | } 104 | 105 | else { 106 | throw 'unknown condition type: ' + condition.type; 107 | } 108 | 109 | } 110 | 111 | /** 112 | * Statement emitter 113 | */ 114 | function statementEmitter(statement) { 115 | //out('STATEMENT'); 116 | 117 | if (statement.type == 'CompoundStatement') { 118 | out('{\n'); 119 | for (let s of statement.statements) { 120 | statementEmitter(s); 121 | } 122 | out('}\n'); 123 | } 124 | 125 | else if (statement.type == 'Call') { 126 | out(statement.identifier + '();\n'); 127 | } 128 | 129 | else if (statement.type == 'Assignment') { 130 | out(statement.identifier + ' = '); 131 | expressionEmitter(statement.expression); 132 | out(';\n'); 133 | } 134 | 135 | else if (statement.type == 'Write') { 136 | out(`console.log(${statement.identifier});\n`); 137 | } 138 | 139 | else if (statement.type == 'IfThen') { 140 | out('if ('); 141 | conditionEmitter(statement.condition); 142 | out(') '); 143 | statementEmitter(statement.statement); 144 | out('\n'); 145 | } 146 | 147 | else if (statement.type == 'While') { 148 | out('while ('); 149 | conditionEmitter(statement.condition); 150 | out(') '); 151 | statementEmitter(statement.statement); 152 | out('\n'); 153 | } 154 | 155 | else { 156 | throw 'Unknown statement type: ' + statement.type; 157 | } 158 | } 159 | 160 | /** 161 | * Block emitter 162 | */ 163 | function blockEmitter(block) { 164 | //out('BLOCK\n'); 165 | //out('{\n'); 166 | 167 | for (let sym in block.symbol) { 168 | let symData = block.symbol[sym]; 169 | if (symData.const) { 170 | out(`const ${sym} = ${symData.val};\n`); 171 | } else { 172 | out(`let ${sym};\n`); 173 | } 174 | } 175 | 176 | for (let procedure of block.procedure) { 177 | procedureEmitter(procedure); 178 | } 179 | 180 | statementEmitter(block.statement); 181 | //out('}\n'); 182 | } 183 | 184 | /** 185 | * Program emitter 186 | */ 187 | function programEmitter(program) { 188 | //out('PROGRAM'); 189 | out(';(function () {\n'); 190 | 191 | blockEmitter(program.block); 192 | 193 | out('}());\n'); 194 | } 195 | 196 | /** 197 | * Convert AST into JS 198 | */ 199 | function emitJS(ast) { 200 | output = ''; 201 | 202 | if (ast.type != 'Program') { 203 | throw 'root node needs to be of type Program'; 204 | } 205 | 206 | // Start at the root node 207 | programEmitter(ast); 208 | 209 | return output; 210 | } 211 | 212 | module.exports = emitJS; -------------------------------------------------------------------------------- /parser.js: -------------------------------------------------------------------------------- 1 | /** 2 | * PL/0 recursive descent parser 3 | * 4 | * https://en.wikipedia.org/wiki/Recursive_descent_parser 5 | * 6 | * program = block "." . 7 | * 8 | * block = [ "const" ident "=" number {"," ident "=" number} ";"] 9 | * [ "var" ident {"," ident} ";"] 10 | * { "procedure" ident ";" block ";" } statement . 11 | * 12 | * statement = [ ident ":=" expression | "call" ident 13 | * | "?" ident | "!" expression 14 | * | "begin" statement {";" statement } "end" 15 | * | "if" condition "then" statement 16 | * | "while" condition "do" statement ]. 17 | * 18 | * condition = "odd" expression | 19 | * expression ("="|"#"|"<"|"<="|">"|">=") expression . 20 | * 21 | * expression = [ "+"|"-"] term { ("+"|"-") term}. 22 | * 23 | * term = factor {("*"|"/") factor}. 24 | * 25 | * factor = ident | number | "(" expression ")". 26 | * 27 | */ 28 | 29 | const ShuntingYard = require('./shuntingyard.js'); 30 | 31 | let curToken; 32 | let curIndex; 33 | let allTokens; 34 | 35 | /** 36 | * Get previous tokens without changing curToken 37 | */ 38 | function prevToken(n=0) { 39 | return allTokens[curIndex - n - 2]; 40 | } 41 | 42 | /** 43 | * Take the next token off the front of the list 44 | */ 45 | function nextToken() { 46 | curToken = allTokens[curIndex]; 47 | curIndex++; 48 | } 49 | 50 | /** 51 | * Return true if the next token matches 52 | */ 53 | function accept(t) { 54 | if (curToken.token == t) { 55 | nextToken(); 56 | return true; 57 | } 58 | 59 | return false; 60 | } 61 | 62 | /** 63 | * Return true iif next token matches, otherwise throw an exception 64 | */ 65 | function expect(t) { 66 | if (!accept(t)) { 67 | throw 'unexpected symbol: ' + curToken.strval; 68 | } 69 | 70 | return true; 71 | } 72 | 73 | /** 74 | * Parse a condition 75 | * 76 | * condition = 77 | * "odd" expression 78 | * | expression ("="|"#"|"<"|"<="|">"|">=") expression . 79 | */ 80 | function parseCondition() { 81 | let astNode; 82 | 83 | if (accept('ODD')) { 84 | astNode = { 85 | type: 'Odd', 86 | expression: parseExpression() 87 | }; 88 | } else { 89 | let expression1 = parseExpression(); 90 | 91 | if (['EQ', 'NE', 'LT', 'LE', 'GT', 'GE'].includes(curToken.token)) { 92 | nextToken(); 93 | astNode = { 94 | type: prevToken().token, // EQ, NE, etc. 95 | expression: [ 96 | expression1, 97 | parseExpression() 98 | ] 99 | }; 100 | } else { 101 | throw 'condition: invalid operator: ' + curToken.strval; 102 | } 103 | } 104 | 105 | return astNode; 106 | } 107 | 108 | /** 109 | * Parse a factor 110 | * 111 | * factor = 112 | * ident 113 | * | number 114 | * | "(" expression ")" . 115 | */ 116 | function parseFactor(sy) { 117 | if (accept('IDENTIFIER')) { 118 | sy.process(prevToken()); 119 | } else if (accept('NUMBER')) { 120 | sy.process(prevToken()); 121 | } else if (accept('LPAREN')) { 122 | sy.process(prevToken()); 123 | parseExpression(sy); 124 | expect('RPAREN'); 125 | sy.process(prevToken()); 126 | } else { 127 | throw 'syntax error: ' + curToken.strval; 128 | } 129 | } 130 | 131 | /** 132 | * Parse a product term 133 | * 134 | * term = factor {("*"|"/") factor} . 135 | */ 136 | function parseTerm(sy) { 137 | parseFactor(sy); 138 | 139 | while (['MULT', 'DIV'].includes(curToken.token)) { 140 | sy.process(curToken); 141 | nextToken(); 142 | parseFactor(sy); 143 | } 144 | } 145 | 146 | /** 147 | * Parse an expression 148 | * 149 | * expression = ["+"|"-"] term {("+"|"-") term} . 150 | */ 151 | function parseExpression(shuntingYard) { 152 | let sy; 153 | 154 | if (shuntingYard) { 155 | sy = shuntingYard; 156 | } else { 157 | sy = new ShuntingYard(); 158 | } 159 | 160 | let astNode = { 161 | type: 'Expression' 162 | }; 163 | 164 | // This is the section for negation ... I THINK 165 | if (curToken.token == 'PLUS' || curToken.token == 'MINUS') { 166 | curToken.token = 'UNARY_' + curToken.token; 167 | sy.process(curToken); 168 | nextToken(); 169 | } 170 | 171 | parseTerm(sy); 172 | 173 | while (curToken.token == 'PLUS' || curToken.token == 'MINUS') { 174 | sy.process(curToken); 175 | nextToken(); 176 | parseTerm(sy); 177 | } 178 | 179 | if (!shuntingYard) { 180 | // if shuntingYard is undefined, it means we created a new on in 181 | // this stack frame and we're responsible for finishing it up 182 | sy.complete(); 183 | astNode.rpn = sy.getRPN(); 184 | astNode.tree = sy.getAST(); 185 | } 186 | 187 | return astNode; 188 | } 189 | 190 | /** 191 | * Parse a statement 192 | * 193 | * statement = 194 | * ident ":=" expression 195 | * | "call" ident 196 | * | "begin" statement {";" statement } "end" 197 | * | "if" condition "then" statement 198 | * | "while" condition "do" statement . 199 | */ 200 | function parseStatement() { 201 | let astNode; 202 | 203 | if (accept('IDENTIFIER')) { 204 | expect('ASSIGNMENT'); 205 | astNode = { 206 | type: 'Assignment', 207 | identifier: prevToken(1).strval, 208 | expression: parseExpression() 209 | }; 210 | 211 | } else if (accept('CALL')) { 212 | expect('IDENTIFIER'); 213 | astNode = { 214 | type: 'Call', 215 | identifier: prevToken().strval 216 | }; 217 | 218 | } else if (accept('WRITE')) { 219 | parseExpression(); 220 | astNode = { 221 | type: 'Write', 222 | identifier: prevToken().strval 223 | }; 224 | 225 | } else if (accept('BEGIN')) { 226 | astNode = { 227 | type: 'CompoundStatement', 228 | statements: [] 229 | }; 230 | 231 | do { 232 | astNode.statements.push(parseStatement()); 233 | } while (accept('SEMICOLON')); 234 | expect('END'); 235 | 236 | } else if (accept('IF')) { 237 | astNode = { 238 | type: 'IfThen', 239 | }; 240 | 241 | astNode.condition = parseCondition(); 242 | 243 | expect('THEN'); 244 | 245 | astNode.statement = parseStatement(); 246 | 247 | } else if (accept('WHILE')) { 248 | astNode = { 249 | type: 'While' 250 | }; 251 | 252 | astNode.condition = parseCondition(); 253 | 254 | expect('DO'); 255 | 256 | astNode.statement = parseStatement(); 257 | 258 | } else { 259 | throw 'syntax error: ' + curToken.strval; 260 | } 261 | 262 | return astNode; 263 | } 264 | 265 | /** 266 | * Parse a block 267 | * 268 | * block = 269 | * ["const" ident "=" number {"," ident "=" number} ";"] 270 | * ["var" ident {"," ident} ";"] 271 | * {"procedure" ident ";" block ";"} statement . 272 | */ 273 | function parseBlock() { 274 | let astNode = { 275 | type: 'Block', 276 | procedure: [], 277 | statement: null, 278 | symbol: {} 279 | }; 280 | 281 | if (accept('CONST')) { 282 | do { 283 | expect('IDENTIFIER'); 284 | expect('EQ'); 285 | expect('NUMBER'); 286 | astNode.symbol[prevToken(2).strval] = { 287 | const: true, 288 | val: prevToken().intval 289 | }; 290 | } while (accept('COMMA')); 291 | 292 | expect('SEMICOLON'); 293 | } 294 | 295 | if (accept('VAR')) { 296 | do { 297 | expect('IDENTIFIER'); 298 | astNode.symbol[prevToken().strval] = { const: false }; 299 | } while (accept('COMMA')); 300 | expect('SEMICOLON'); 301 | } 302 | 303 | while (accept('PROCEDURE')) { 304 | expect('IDENTIFIER'); 305 | let id = prevToken().strval; 306 | expect('SEMICOLON'); 307 | let b = parseBlock(); 308 | expect('SEMICOLON'); 309 | 310 | astNode.procedure.push({ 311 | type: 'Procedure', 312 | name: id, 313 | block: b 314 | }); 315 | } 316 | 317 | astNode.statement = parseStatement(); 318 | 319 | return astNode; 320 | } 321 | 322 | /** 323 | * Parse the main program block 324 | * 325 | * program = block "." . 326 | */ 327 | function parseProgram() { 328 | let astNode = { 329 | type: 'Program' 330 | }; 331 | 332 | nextToken(); 333 | astNode.block = parseBlock(); 334 | expect('PERIOD'); 335 | 336 | return astNode; 337 | } 338 | 339 | /** 340 | * Main exported function 341 | */ 342 | function parse(tokens) { 343 | allTokens = tokens; 344 | curIndex = 0; 345 | 346 | return parseProgram(); 347 | } 348 | 349 | module.exports = parse; -------------------------------------------------------------------------------- /shuntingyard.js: -------------------------------------------------------------------------------- 1 | /** 2 | * The Shunting Yard algorithm 3 | * 4 | * https://en.wikipedia.org/wiki/Shunting-yard_algorithm 5 | */ 6 | 7 | const precedence = { 8 | 'UNARY_MINUS': 0, 9 | 'UNARY_PLUS': 0, 10 | 'PLUS': 10, 11 | 'MINUS': 10, 12 | 'MULT': 20, 13 | 'DIV': 20, 14 | }; 15 | 16 | /** 17 | * True if the token is an operator 18 | */ 19 | function isOperator(token) { 20 | let t = token.token; 21 | 22 | // A little hackish, but this allows you to use a token or a token 23 | // name: 24 | if (t === undefined) { 25 | t = token; 26 | } 27 | 28 | return precedence[t] !== undefined; 29 | } 30 | 31 | /** 32 | * True if the token is a unary operator 33 | */ 34 | function isUnary(token) { 35 | return ['UNARY_PLUS', 'UNARY_MINUS'].includes(token.token); 36 | } 37 | 38 | class ShuntingYard { 39 | constructor() { 40 | this.reset(); 41 | } 42 | 43 | reset() { 44 | this.queue = []; // RPN output queue, 1, 2, x, PLUS, etc. 45 | this.stack = []; // operators, MULT, PLUS, etc. 46 | } 47 | 48 | stackPeek() { 49 | return this.stack[this.stack.length - 1]; 50 | } 51 | 52 | process(token) { 53 | if (token.token == 'IDENTIFIER') { 54 | this.queue.push(token.strval); 55 | } 56 | 57 | else if (token.token == 'NUMBER') { 58 | this.queue.push(token.intval); 59 | } 60 | 61 | else if (isOperator(token.token)) { 62 | let topOp = this.stackPeek(); 63 | 64 | while (this.stack.length > 0 && precedence[topOp] > precedence[token.token]) { 65 | topOp = this.stack.pop(); 66 | this.queue.push(topOp); 67 | 68 | topOp = this.stackPeek(); 69 | } 70 | 71 | this.stack.push(token.token); 72 | } 73 | 74 | else if (token.token == 'LPAREN') { 75 | this.stack.push(token.token); 76 | } 77 | 78 | else if (token.token == 'RPAREN') { 79 | let topOp = this.stackPeek(); 80 | 81 | while (topOp != 'LPAREN') { 82 | topOp = this.stack.pop(); 83 | if (topOp === undefined) { 84 | throw 'mismatched parens'; 85 | } 86 | this.queue.push(topOp); 87 | 88 | topOp = this.stackPeek(); 89 | } 90 | 91 | // Pop and discard left paren 92 | this.stack.pop(); 93 | } 94 | } 95 | 96 | /** 97 | * Call this after sending the last token to process() 98 | */ 99 | complete() { 100 | let topOp = this.stackPeek(); 101 | 102 | while (topOp !== undefined) { 103 | topOp = this.stack.pop(); 104 | 105 | if (topOp == 'LPAREN' || topOp == 'RPAREN') { 106 | throw 'mismatched parens'; 107 | } 108 | 109 | this.queue.push(topOp); 110 | 111 | topOp = this.stackPeek(); 112 | } 113 | } 114 | 115 | /** 116 | * Return RPN of expression 117 | * 118 | * NOTE: Only valid after complete() called 119 | */ 120 | getRPN() { 121 | return this.queue; 122 | } 123 | 124 | /** 125 | * Return AST of expression 126 | * 127 | * NOTE: Only valid after complete() called 128 | */ 129 | getAST() { 130 | let convStack = []; 131 | let topOp; 132 | let queueCopy = this.queue.slice(); // Be nondestructive 133 | 134 | topOp = queueCopy.shift(); 135 | 136 | while (topOp !== undefined) { 137 | 138 | if (isOperator(topOp)) { 139 | let node = { 140 | type: 'Operator', 141 | value: topOp 142 | }; 143 | 144 | if (isUnary(topOp)) { 145 | node.operand = [ null, convStack.pop() ]; 146 | 147 | } else { 148 | // Binary 149 | let op2 = convStack.pop(); 150 | let op1 = convStack.pop(); 151 | node.operand = [ 152 | op1, 153 | op2, 154 | ]; 155 | } 156 | 157 | convStack.push(node); 158 | 159 | } else { 160 | // Not an operator 161 | convStack.push({ 162 | type: 'Operand', 163 | value: topOp 164 | }); 165 | } 166 | 167 | topOp = queueCopy.shift(); 168 | } 169 | 170 | if (convStack.length != 1) { 171 | throw 'expected convStack to be length 1'; 172 | } 173 | 174 | return convStack[0]; 175 | 176 | } 177 | } 178 | 179 | module.exports = ShuntingYard; -------------------------------------------------------------------------------- /tokenizer.js: -------------------------------------------------------------------------------- 1 | /** 2 | * PL/0 tokenizer 3 | * 4 | * https://en.wikipedia.org/wiki/PL/0 5 | */ 6 | 7 | let patterns = { 8 | VAR: /VAR/, 9 | CONST: /CONST/, 10 | PROCEDURE: /PROCEDURE/, 11 | CALL: /CALL/, 12 | WHILE: /WHILE/, 13 | DO: /DO/, 14 | BEGIN: /BEGIN/, 15 | END: /END/, 16 | IDENTIFIER: /[a-z_][a-z0-9_]*/, 17 | NUMBER: /[0-9]+/, 18 | ASSIGNMENT: /:=/, 19 | WRITE: /!/, 20 | LPAREN: /\(/, 21 | RPAREN: /\)/, 22 | MULT: /\*/, 23 | DIV: /\//, 24 | PLUS: /\+/, 25 | MINUS: /-/, 26 | EQ: /=/, 27 | NE: /#/, 28 | LE: /<=/, 29 | GE: />=/, 30 | LT: //, 32 | COMMA: /,/, 33 | PERIOD: /\./, 34 | SEMICOLON: /;/, 35 | WHITESPACE: /\s/ 36 | }; 37 | 38 | 39 | /** 40 | * Tokenize a string 41 | * 42 | * @param {string} input String to tokenize 43 | * @param {boolean} [ignoreWhitespace=true] If true, no WHITESPACE tokens will be produced 44 | */ 45 | function tokenize(input, ignoreWhitespace=true) { 46 | let tokens = []; 47 | let normPatterns = {}; 48 | 49 | while (input != '') { 50 | let tokenized = false; 51 | 52 | for (let token of Object.keys(patterns)) { 53 | // Modify user patterns to search from start of string 54 | if (!(token in normPatterns)) { 55 | normPatterns[token] = new RegExp('^' + patterns[token].source, patterns[token].flags + 'i'); 56 | } 57 | 58 | // Match a token 59 | let m = input.match(normPatterns[token]); 60 | 61 | if (m !== null && m.index === 0) { 62 | // Found a match 63 | tokenized = true; 64 | 65 | let strval = m[0]; 66 | let len = strval.length; 67 | 68 | if (token != 'WHITESPACE' || !ignoreWhitespace) { 69 | tokens.push({ 70 | token: token, 71 | length: len, 72 | strval: strval, 73 | intval: parseInt(strval), 74 | floatval: parseFloat(strval) 75 | }); 76 | } 77 | 78 | // On to the next token 79 | input = input.slice(len); 80 | break; 81 | } 82 | } 83 | 84 | if (!tokenized) { 85 | throw `Unrecognized token at: "${input}"`; 86 | } 87 | } 88 | 89 | return tokens; 90 | } 91 | 92 | module.exports = tokenize; --------------------------------------------------------------------------------