├── .gitattributes ├── .gitignore ├── .vscode ├── launch.json └── tasks.json ├── LICENSE ├── PowerQuery.YAML-tmLanguage ├── PowerQuery.tmLanguage ├── PowerQuery.tmLanguage.json ├── README.md ├── SECURITY.md ├── build ├── build.ts └── tsconfig.json ├── package-lock.json ├── package.json ├── tests ├── basic.ts ├── common.ts ├── incremental.ts ├── parser.ts └── tsconfig.json └── tsconfig.json /.gitattributes: -------------------------------------------------------------------------------- 1 | # Make sure baseline files have consistent line endings 2 | *.txt text eol=lf -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Node and JS files 2 | node_modules/ 3 | *.js 4 | *.js.map 5 | xunit.xml -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "type": "node", 6 | "request": "launch", 7 | "name": "Mocha Tests", 8 | "program": "${workspaceFolder}/node_modules/mocha/bin/_mocha", 9 | "args": [ 10 | "--debug", 11 | "--colors", 12 | "--timeout", 13 | "999999", 14 | "-r", 15 | "ts-node/register", 16 | "${workspaceFolder}/tests/**/*.ts" 17 | ], 18 | "preLaunchTask": "build", 19 | "internalConsoleOptions": "openOnSessionStart" 20 | } 21 | ], 22 | "compounds": [] 23 | } 24 | -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | // See https://go.microsoft.com/fwlink/?LinkId=733558 3 | // for the documentation about the tasks.json format 4 | "version": "2.0.0", 5 | "tasks": [ 6 | { 7 | "label": "build", 8 | "type": "typescript", 9 | "tsconfig": "tsconfig.json", 10 | "problemMatcher": ["$tsc"], 11 | "group": { 12 | "kind": "build", 13 | "isDefault": true 14 | } 15 | } 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. All rights reserved. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /PowerQuery.YAML-tmLanguage: -------------------------------------------------------------------------------- 1 | # [PackageDev] target_format: plist, ext: tmLanguage 2 | --- 3 | name: powerquery 4 | scopeName: source.powerquery 5 | fileTypes: [pq,pqm] 6 | uuid: 41968B57-12E6-4AC5-92A4-A837010E8B0A 7 | 8 | patterns: 9 | - include: '#Noise' 10 | - include: '#LiteralExpression' 11 | - include: '#Keywords' 12 | - include: '#ImplicitVariable' 13 | - include: '#IntrinsicVariable' 14 | - include: '#Operators' 15 | - include: '#DotOperators' 16 | - include: '#TypeName' 17 | - include: '#RecordExpression' 18 | - include: '#Punctuation' 19 | - include: '#QuotedIdentifier' 20 | - include: '#Identifier' 21 | 22 | repository: 23 | Keywords: 24 | match: '\b(?:(and|or|not)|(if|then|else)|(try|otherwise)|(as|each|in|is|let|meta|type|error)|(section|shared))\b' 25 | captures: 26 | '1': 27 | name: keyword.operator.word.logical.powerquery 28 | '2': 29 | name: keyword.control.conditional.powerquery 30 | '3': 31 | name: keyword.control.exception.powerquery 32 | '4': 33 | name: keyword.other.powerquery 34 | '5': 35 | name: keyword.powerquery 36 | TypeName: 37 | match: '\b(?:(optional|nullable)|(action|any|anynonnull|binary|date|datetime|datetimezone|duration|function|list|logical|none|null|number|record|table|text|type))\b' 38 | captures: 39 | '1': 40 | name: storage.modifier.powerquery 41 | '2': 42 | name: storage.type.powerquery 43 | LiteralExpression: 44 | patterns: 45 | - include: '#String' 46 | - include: '#NumericConstant' 47 | - include: '#LogicalConstant' 48 | - include: '#NullConstant' 49 | - include: '#FloatNumber' 50 | - include: '#DecimalNumber' 51 | - include: '#HexNumber' 52 | - include: '#IntNumber' 53 | Noise: 54 | patterns: 55 | - include: '#BlockComment' 56 | - include: '#LineComment' 57 | - include: '#Whitespace' 58 | Whitespace: 59 | match: \s+ 60 | BlockComment: 61 | begin: '/\*' 62 | end: '\*/' 63 | name: comment.block.powerquery 64 | LineComment: 65 | match: '//.*' 66 | name: comment.line.double-slash.powerquery 67 | String: 68 | begin: '"' 69 | beginCaptures: 70 | '0': 71 | name: punctuation.definition.string.begin.powerquery 72 | end: '"(?!")' 73 | endCaptures: 74 | '0': 75 | name: punctuation.definition.string.end.powerquery 76 | patterns: 77 | - match: '""' 78 | name: constant.character.escape.quote.powerquery 79 | - include: '#EscapeSequence' 80 | name: string.quoted.double.powerquery 81 | QuotedIdentifier: 82 | begin: '#"' 83 | beginCaptures: 84 | '0': 85 | name: punctuation.definition.quotedidentifier.begin.powerquery 86 | end: '"(?!")' 87 | endCaptures: 88 | '0': 89 | name: punctuation.definition.quotedidentifier.end.powerquery 90 | patterns: 91 | - match: '""' 92 | name: constant.character.escape.quote.powerquery 93 | - include: '#EscapeSequence' 94 | name: entity.name.powerquery 95 | EscapeSequence: 96 | begin: '#\(' 97 | beginCaptures: 98 | '0': 99 | name: punctuation.definition.escapesequence.begin.powerquery 100 | end: \) 101 | endCaptures: 102 | '0': 103 | name: punctuation.definition.escapesequence.end.powerquery 104 | patterns: 105 | - match: '(#|\h{4}|\h{8}|cr|lf|tab)(?:,(#|\h{4}|\h{8}|cr|lf|tab))*' 106 | - match: '[^\)]' 107 | name: invalid.illegal.escapesequence.powerquery 108 | name: constant.character.escapesequence.powerquery 109 | LogicalConstant: 110 | match: \b(true|false)\b 111 | name: constant.language.logical.powerquery 112 | NullConstant: 113 | match: \b(null)\b 114 | name: constant.language.null.powerquery 115 | NumericConstant: 116 | match: '(?)|(=)|(<>|<|>|<=|>=)|(&)|(\+|-|\*|\/)|(!)|(\?)' 148 | captures: 149 | '1': 150 | name: keyword.operator.function.powerquery 151 | '2': 152 | name: keyword.operator.assignment-or-comparison.powerquery 153 | '3': 154 | name: keyword.operator.comparison.powerquery 155 | '4': 156 | name: keyword.operator.combination.powerquery 157 | '5': 158 | name: keyword.operator.arithmetic.powerquery 159 | '6': 160 | name: keyword.operator.sectionaccess.powerquery 161 | '7': 162 | name: keyword.operator.optional.powerquery 163 | DotOperators: 164 | match: '(? 2 | 3 | 4 | 5 | name 6 | powerquery 7 | scopeName 8 | source.powerquery 9 | fileTypes 10 | 11 | pq 12 | pqm 13 | 14 | uuid 15 | 41968B57-12E6-4AC5-92A4-A837010E8B0A 16 | patterns 17 | 18 | 19 | include 20 | #Noise 21 | 22 | 23 | include 24 | #LiteralExpression 25 | 26 | 27 | include 28 | #Keywords 29 | 30 | 31 | include 32 | #ImplicitVariable 33 | 34 | 35 | include 36 | #IntrinsicVariable 37 | 38 | 39 | include 40 | #Operators 41 | 42 | 43 | include 44 | #DotOperators 45 | 46 | 47 | include 48 | #TypeName 49 | 50 | 51 | include 52 | #RecordExpression 53 | 54 | 55 | include 56 | #Punctuation 57 | 58 | 59 | include 60 | #QuotedIdentifier 61 | 62 | 63 | include 64 | #Identifier 65 | 66 | 67 | repository 68 | 69 | Keywords 70 | 71 | match 72 | \b(?:(and|or|not)|(if|then|else)|(try|otherwise)|(as|each|in|is|let|meta|type|error)|(section|shared))\b 73 | captures 74 | 75 | 1 76 | 77 | name 78 | keyword.operator.word.logical.powerquery 79 | 80 | 2 81 | 82 | name 83 | keyword.control.conditional.powerquery 84 | 85 | 3 86 | 87 | name 88 | keyword.control.exception.powerquery 89 | 90 | 4 91 | 92 | name 93 | keyword.other.powerquery 94 | 95 | 5 96 | 97 | name 98 | keyword.powerquery 99 | 100 | 101 | 102 | TypeName 103 | 104 | match 105 | \b(?:(optional|nullable)|(action|any|anynonnull|binary|date|datetime|datetimezone|duration|function|list|logical|none|null|number|record|table|text|type))\b 106 | captures 107 | 108 | 1 109 | 110 | name 111 | storage.modifier.powerquery 112 | 113 | 2 114 | 115 | name 116 | storage.type.powerquery 117 | 118 | 119 | 120 | LiteralExpression 121 | 122 | patterns 123 | 124 | 125 | include 126 | #String 127 | 128 | 129 | include 130 | #NumericConstant 131 | 132 | 133 | include 134 | #LogicalConstant 135 | 136 | 137 | include 138 | #NullConstant 139 | 140 | 141 | include 142 | #FloatNumber 143 | 144 | 145 | include 146 | #DecimalNumber 147 | 148 | 149 | include 150 | #HexNumber 151 | 152 | 153 | include 154 | #IntNumber 155 | 156 | 157 | 158 | Noise 159 | 160 | patterns 161 | 162 | 163 | include 164 | #BlockComment 165 | 166 | 167 | include 168 | #LineComment 169 | 170 | 171 | include 172 | #Whitespace 173 | 174 | 175 | 176 | Whitespace 177 | 178 | match 179 | \s+ 180 | 181 | BlockComment 182 | 183 | begin 184 | /\* 185 | end 186 | \*/ 187 | name 188 | comment.block.powerquery 189 | 190 | LineComment 191 | 192 | match 193 | //.* 194 | name 195 | comment.line.double-slash.powerquery 196 | 197 | String 198 | 199 | begin 200 | " 201 | beginCaptures 202 | 203 | 0 204 | 205 | name 206 | punctuation.definition.string.begin.powerquery 207 | 208 | 209 | end 210 | "(?!") 211 | endCaptures 212 | 213 | 0 214 | 215 | name 216 | punctuation.definition.string.end.powerquery 217 | 218 | 219 | patterns 220 | 221 | 222 | match 223 | "" 224 | name 225 | constant.character.escape.quote.powerquery 226 | 227 | 228 | include 229 | #EscapeSequence 230 | 231 | 232 | name 233 | string.quoted.double.powerquery 234 | 235 | QuotedIdentifier 236 | 237 | begin 238 | #" 239 | beginCaptures 240 | 241 | 0 242 | 243 | name 244 | punctuation.definition.quotedidentifier.begin.powerquery 245 | 246 | 247 | end 248 | "(?!") 249 | endCaptures 250 | 251 | 0 252 | 253 | name 254 | punctuation.definition.quotedidentifier.end.powerquery 255 | 256 | 257 | patterns 258 | 259 | 260 | match 261 | "" 262 | name 263 | constant.character.escape.quote.powerquery 264 | 265 | 266 | include 267 | #EscapeSequence 268 | 269 | 270 | name 271 | entity.name.powerquery 272 | 273 | EscapeSequence 274 | 275 | begin 276 | #\( 277 | beginCaptures 278 | 279 | 0 280 | 281 | name 282 | punctuation.definition.escapesequence.begin.powerquery 283 | 284 | 285 | end 286 | \) 287 | endCaptures 288 | 289 | 0 290 | 291 | name 292 | punctuation.definition.escapesequence.end.powerquery 293 | 294 | 295 | patterns 296 | 297 | 298 | match 299 | (#|\h{4}|\h{8}|cr|lf|tab)(?:,(#|\h{4}|\h{8}|cr|lf|tab))* 300 | 301 | 302 | match 303 | [^\)] 304 | name 305 | invalid.illegal.escapesequence.powerquery 306 | 307 | 308 | name 309 | constant.character.escapesequence.powerquery 310 | 311 | LogicalConstant 312 | 313 | match 314 | \b(true|false)\b 315 | name 316 | constant.language.logical.powerquery 317 | 318 | NullConstant 319 | 320 | match 321 | \b(null)\b 322 | name 323 | constant.language.null.powerquery 324 | 325 | NumericConstant 326 | 327 | match 328 | (?<![\d\w])(#infinity|#nan)\b 329 | captures 330 | 331 | 1 332 | 333 | name 334 | constant.language.numeric.float.powerquery 335 | 336 | 337 | 338 | HexNumber 339 | 340 | match 341 | 0(x|X)\h+ 342 | name 343 | constant.numeric.integer.hexadecimal.powerquery 344 | 345 | IntNumber 346 | 347 | match 348 | \b(\d+)\b 349 | captures 350 | 351 | 1 352 | 353 | name 354 | constant.numeric.integer.powerquery 355 | 356 | 357 | 358 | DecimalNumber 359 | 360 | match 361 | (?<![\d\w])(\d*\.\d+)\b 362 | name 363 | constant.numeric.decimal.powerquery 364 | 365 | FloatNumber 366 | 367 | match 368 | (\d*\.)?\d+(e|E)(\+|-)?\d+ 369 | name 370 | constant.numeric.float.powerquery 371 | 372 | InclusiveIdentifier 373 | 374 | match 375 | @ 376 | captures 377 | 378 | 0 379 | 380 | name 381 | inclusiveidentifier.powerquery 382 | 383 | 384 | 385 | Identifier 386 | 387 | match 388 | (?x:(?<![\._\p{Lu}\p{Ll}\p{Lt}\p{Lm}\p{Lo}\p{Nl}\p{Nd}\p{Pc}\p{Mn}\p{Mc}\p{Cf}])(@?)([_\p{Lu}\p{Ll}\p{Lt}\p{Lm}\p{Lo}\p{Nl}][_\p{Lu}\p{Ll}\p{Lt}\p{Lm}\p{Lo}\p{Nl}\p{Nd}\p{Pc}\p{Mn}\p{Mc}\p{Cf}]*(?:\.[_\p{Lu}\p{Ll}\p{Lt}\p{Lm}\p{Lo}\p{Nl}][_\p{Lu}\p{Ll}\p{Lt}\p{Lm}\p{Lo}\p{Nl}\p{Nd}\p{Pc}\p{Mn}\p{Mc}\p{Cf}])*)\b) 389 | captures 390 | 391 | 1 392 | 393 | name 394 | keyword.operator.inclusiveidentifier.powerquery 395 | 396 | 2 397 | 398 | name 399 | entity.name.powerquery 400 | 401 | 402 | 403 | Operators 404 | 405 | match 406 | (=>)|(=)|(<>|<|>|<=|>=)|(&)|(\+|-|\*|\/)|(!)|(\?) 407 | captures 408 | 409 | 1 410 | 411 | name 412 | keyword.operator.function.powerquery 413 | 414 | 2 415 | 416 | name 417 | keyword.operator.assignment-or-comparison.powerquery 418 | 419 | 3 420 | 421 | name 422 | keyword.operator.comparison.powerquery 423 | 424 | 4 425 | 426 | name 427 | keyword.operator.combination.powerquery 428 | 429 | 5 430 | 431 | name 432 | keyword.operator.arithmetic.powerquery 433 | 434 | 6 435 | 436 | name 437 | keyword.operator.sectionaccess.powerquery 438 | 439 | 7 440 | 441 | name 442 | keyword.operator.optional.powerquery 443 | 444 | 445 | 446 | DotOperators 447 | 448 | match 449 | (?<!\.)(?:(\.\.\.)|(\.\.))(?!\.) 450 | captures 451 | 452 | 1 453 | 454 | name 455 | keyword.operator.ellipsis.powerquery 456 | 457 | 2 458 | 459 | name 460 | keyword.operator.list.powerquery 461 | 462 | 463 | 464 | RecordExpression 465 | 466 | begin 467 | \[ 468 | beginCaptures 469 | 470 | 0 471 | 472 | name 473 | punctuation.section.brackets.begin.powerquery 474 | 475 | 476 | end 477 | \] 478 | endCaptures 479 | 480 | 0 481 | 482 | name 483 | punctuation.section.brackets.end.powerquery 484 | 485 | 486 | patterns 487 | 488 | 489 | include 490 | $self 491 | 492 | 493 | contentName 494 | meta.recordexpression.powerquery 495 | 496 | Punctuation 497 | 498 | match 499 | (,)|(\()|(\))|({)|(}) 500 | captures 501 | 502 | 1 503 | 504 | name 505 | punctuation.separator.powerquery 506 | 507 | 2 508 | 509 | name 510 | punctuation.section.parens.begin.powerquery 511 | 512 | 3 513 | 514 | name 515 | punctuation.section.parens.end.powerquery 516 | 517 | 4 518 | 519 | name 520 | punctuation.section.braces.begin.powerquery 521 | 522 | 5 523 | 524 | name 525 | punctuation.section.braces.end.powerquery 526 | 527 | 528 | 529 | ImplicitVariable 530 | 531 | match 532 | \b_\b 533 | name 534 | keyword.operator.implicitvariable.powerquery 535 | 536 | IntrinsicVariable 537 | 538 | match 539 | (?<![\d\w])(#sections|#shared)\b 540 | captures 541 | 542 | 1 543 | 544 | name 545 | constant.language.intrinsicvariable.powerquery 546 | 547 | 548 | 549 | 550 | 551 | -------------------------------------------------------------------------------- /PowerQuery.tmLanguage.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "powerquery", 3 | "scopeName": "source.powerquery", 4 | "fileTypes": [ 5 | "pq", 6 | "pqm" 7 | ], 8 | "uuid": "41968B57-12E6-4AC5-92A4-A837010E8B0A", 9 | "patterns": [ 10 | { 11 | "include": "#Noise" 12 | }, 13 | { 14 | "include": "#LiteralExpression" 15 | }, 16 | { 17 | "include": "#Keywords" 18 | }, 19 | { 20 | "include": "#ImplicitVariable" 21 | }, 22 | { 23 | "include": "#IntrinsicVariable" 24 | }, 25 | { 26 | "include": "#Operators" 27 | }, 28 | { 29 | "include": "#DotOperators" 30 | }, 31 | { 32 | "include": "#TypeName" 33 | }, 34 | { 35 | "include": "#RecordExpression" 36 | }, 37 | { 38 | "include": "#Punctuation" 39 | }, 40 | { 41 | "include": "#QuotedIdentifier" 42 | }, 43 | { 44 | "include": "#Identifier" 45 | } 46 | ], 47 | "repository": { 48 | "Keywords": { 49 | "match": "\\b(?:(and|or|not)|(if|then|else)|(try|otherwise)|(as|each|in|is|let|meta|type|error)|(section|shared))\\b", 50 | "captures": { 51 | "1": { 52 | "name": "keyword.operator.word.logical.powerquery" 53 | }, 54 | "2": { 55 | "name": "keyword.control.conditional.powerquery" 56 | }, 57 | "3": { 58 | "name": "keyword.control.exception.powerquery" 59 | }, 60 | "4": { 61 | "name": "keyword.other.powerquery" 62 | }, 63 | "5": { 64 | "name": "keyword.powerquery" 65 | } 66 | } 67 | }, 68 | "TypeName": { 69 | "match": "\\b(?:(optional|nullable)|(action|any|anynonnull|binary|date|datetime|datetimezone|duration|function|list|logical|none|null|number|record|table|text|type))\\b", 70 | "captures": { 71 | "1": { 72 | "name": "storage.modifier.powerquery" 73 | }, 74 | "2": { 75 | "name": "storage.type.powerquery" 76 | } 77 | } 78 | }, 79 | "LiteralExpression": { 80 | "patterns": [ 81 | { 82 | "include": "#String" 83 | }, 84 | { 85 | "include": "#NumericConstant" 86 | }, 87 | { 88 | "include": "#LogicalConstant" 89 | }, 90 | { 91 | "include": "#NullConstant" 92 | }, 93 | { 94 | "include": "#FloatNumber" 95 | }, 96 | { 97 | "include": "#DecimalNumber" 98 | }, 99 | { 100 | "include": "#HexNumber" 101 | }, 102 | { 103 | "include": "#IntNumber" 104 | } 105 | ] 106 | }, 107 | "Noise": { 108 | "patterns": [ 109 | { 110 | "include": "#BlockComment" 111 | }, 112 | { 113 | "include": "#LineComment" 114 | }, 115 | { 116 | "include": "#Whitespace" 117 | } 118 | ] 119 | }, 120 | "Whitespace": { 121 | "match": "\\s+" 122 | }, 123 | "BlockComment": { 124 | "begin": "/\\*", 125 | "end": "\\*/", 126 | "name": "comment.block.powerquery" 127 | }, 128 | "LineComment": { 129 | "match": "//.*", 130 | "name": "comment.line.double-slash.powerquery" 131 | }, 132 | "String": { 133 | "begin": "\"", 134 | "beginCaptures": { 135 | "0": { 136 | "name": "punctuation.definition.string.begin.powerquery" 137 | } 138 | }, 139 | "end": "\"(?!\")", 140 | "endCaptures": { 141 | "0": { 142 | "name": "punctuation.definition.string.end.powerquery" 143 | } 144 | }, 145 | "patterns": [ 146 | { 147 | "match": "\"\"", 148 | "name": "constant.character.escape.quote.powerquery" 149 | }, 150 | { 151 | "include": "#EscapeSequence" 152 | } 153 | ], 154 | "name": "string.quoted.double.powerquery" 155 | }, 156 | "QuotedIdentifier": { 157 | "begin": "#\"", 158 | "beginCaptures": { 159 | "0": { 160 | "name": "punctuation.definition.quotedidentifier.begin.powerquery" 161 | } 162 | }, 163 | "end": "\"(?!\")", 164 | "endCaptures": { 165 | "0": { 166 | "name": "punctuation.definition.quotedidentifier.end.powerquery" 167 | } 168 | }, 169 | "patterns": [ 170 | { 171 | "match": "\"\"", 172 | "name": "constant.character.escape.quote.powerquery" 173 | }, 174 | { 175 | "include": "#EscapeSequence" 176 | } 177 | ], 178 | "name": "entity.name.powerquery" 179 | }, 180 | "EscapeSequence": { 181 | "begin": "#\\(", 182 | "beginCaptures": { 183 | "0": { 184 | "name": "punctuation.definition.escapesequence.begin.powerquery" 185 | } 186 | }, 187 | "end": "\\)", 188 | "endCaptures": { 189 | "0": { 190 | "name": "punctuation.definition.escapesequence.end.powerquery" 191 | } 192 | }, 193 | "patterns": [ 194 | { 195 | "match": "(#|\\h{4}|\\h{8}|cr|lf|tab)(?:,(#|\\h{4}|\\h{8}|cr|lf|tab))*" 196 | }, 197 | { 198 | "match": "[^\\)]", 199 | "name": "invalid.illegal.escapesequence.powerquery" 200 | } 201 | ], 202 | "name": "constant.character.escapesequence.powerquery" 203 | }, 204 | "LogicalConstant": { 205 | "match": "\\b(true|false)\\b", 206 | "name": "constant.language.logical.powerquery" 207 | }, 208 | "NullConstant": { 209 | "match": "\\b(null)\\b", 210 | "name": "constant.language.null.powerquery" 211 | }, 212 | "NumericConstant": { 213 | "match": "(?)|(=)|(<>|<|>|<=|>=)|(&)|(\\+|-|\\*|\\/)|(!)|(\\?)", 261 | "captures": { 262 | "1": { 263 | "name": "keyword.operator.function.powerquery" 264 | }, 265 | "2": { 266 | "name": "keyword.operator.assignment-or-comparison.powerquery" 267 | }, 268 | "3": { 269 | "name": "keyword.operator.comparison.powerquery" 270 | }, 271 | "4": { 272 | "name": "keyword.operator.combination.powerquery" 273 | }, 274 | "5": { 275 | "name": "keyword.operator.arithmetic.powerquery" 276 | }, 277 | "6": { 278 | "name": "keyword.operator.sectionaccess.powerquery" 279 | }, 280 | "7": { 281 | "name": "keyword.operator.optional.powerquery" 282 | } 283 | } 284 | }, 285 | "DotOperators": { 286 | "match": "(? 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd). 40 | 41 | 42 | -------------------------------------------------------------------------------- /build/build.ts: -------------------------------------------------------------------------------- 1 | import fs = require('fs'); 2 | import path = require('path'); 3 | import yaml = require('js-yaml'); 4 | import plist = require('plist'); 5 | 6 | enum Language { 7 | PowerQuery = "PowerQuery" 8 | } 9 | 10 | enum Extension { 11 | TmLanguage = "tmLanguage", 12 | TmTheme = "tmTheme", 13 | YamlTmLangauge = "YAML-tmLanguage", 14 | YamlTmTheme = "YAML-tmTheme" 15 | } 16 | 17 | function file(language: Language, extension: Extension) { 18 | return path.join(__dirname, '..', `${language}.${extension}`); 19 | } 20 | 21 | function writePlistFile(grammar: any, fileName: string) { 22 | const text = plist.build(grammar); 23 | fs.writeFileSync(fileName, text); 24 | } 25 | 26 | function readYaml(fileName: string) { 27 | const text = fs.readFileSync(fileName, "utf8"); 28 | return yaml.safeLoad(text); 29 | } 30 | 31 | function buildGrammar() { 32 | const grammar = readYaml(file(Language.PowerQuery, Extension.YamlTmLangauge)) 33 | writePlistFile(grammar, file(Language.PowerQuery, Extension.TmLanguage)); 34 | } 35 | 36 | buildGrammar(); -------------------------------------------------------------------------------- /build/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES5", 4 | "module": "commonjs", 5 | "noImplicitAny": true, 6 | "skipLibCheck": true 7 | }, 8 | "exclude": [ 9 | "node_modules" 10 | ] 11 | } -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "powerquery-language", 3 | "version": "1.0.0", 4 | "private": true, 5 | "description": "TextMate grammar files for Power Query / M language", 6 | "main": "build/build.js", 7 | "scripts": { 8 | "build": "npm run build:grammar && npm run build:tests", 9 | "build:grammar": "tsc -b build -v && node build/build.js && .\\node_modules\\.bin\\npx js-yaml PowerQuery.YAML-tmLanguage > PowerQuery.tmLanguage.json", 10 | "build:tests": "tsc -b tests -v", 11 | "test": "mocha -r ts-node/register tests/**/*.ts" 12 | }, 13 | "devDependencies": { 14 | "@types/chai": "latest", 15 | "@types/js-yaml": "latest", 16 | "@types/mocha": "latest", 17 | "@types/plist": "latest", 18 | "chai": "latest", 19 | "js-yaml": "latest", 20 | "mocha": "latest", 21 | "mocha-multi-reporters": "latest", 22 | "npx": "^10.2.0", 23 | "plist": "latest", 24 | "powerquery-parser": "latest", 25 | "ts-node": "^8.1.0", 26 | "typescript": "latest", 27 | "vscode-textmate": "^3.3.3" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /tests/basic.ts: -------------------------------------------------------------------------------- 1 | import { expect } from "chai"; 2 | import "mocha"; 3 | import * as Shared from "./common"; 4 | 5 | // Query constants 6 | const QueryMultiline = `let 7 | a = 1, 8 | b = "text", 9 | c = Text.From(a) 10 | in 11 | c`; 12 | 13 | describe("Basic grammar tests", () => { 14 | it("Grammar loaded", () => { 15 | expect(Shared.grammar).to.exist; 16 | }); 17 | 18 | it("Tokens", () => { 19 | let state = null; 20 | let lines = QueryMultiline.split("\n"); 21 | 22 | for (let i = 0; i < lines.length; i++) { 23 | let r = Shared.grammar.tokenizeLine(lines[i], state); 24 | state = r.ruleStack; 25 | expect(r.tokens.length).to.be.greaterThan(0); 26 | } 27 | }); 28 | }); -------------------------------------------------------------------------------- /tests/common.ts: -------------------------------------------------------------------------------- 1 | import path = require('path'); 2 | import vsctm = require('vscode-textmate'); 3 | import { Lexer, Token } from "powerquery-parser"; 4 | import { expect } from "chai"; 5 | import "mocha"; 6 | 7 | function getGrammarFilePath(): string { 8 | return path.join(__dirname, '..', "PowerQuery.tmLanguage"); 9 | } 10 | 11 | export const registry = new vsctm.Registry(); 12 | export const grammar = registry.loadGrammarFromPathSync(getGrammarFilePath()); 13 | 14 | // Grammar constants 15 | export class Scopes { 16 | public static QuoteStringBegin = "punctuation.definition.string.begin.powerquery"; 17 | public static QuoteStringEnd = "punctuation.definition.string.end.powerquery"; 18 | public static String = "string.quoted.double.powerquery"; 19 | public static QuotedIdentifierBegin = "punctuation.definition.quotedidentifier.begin.powerquery"; 20 | public static QuotedIdentifierEnd = "punctuation.definition.quotedidentifier.end.powerquery"; 21 | public static Identifier = "entity.name.powerquery"; 22 | } 23 | 24 | export class TokenComparer { 25 | public readonly grammarTokens: vsctm.IToken[]; 26 | public readonly parserTokens: readonly Token[]; 27 | 28 | constructor(grammarTokens: vsctm.IToken[], parserTokens: readonly Token[]) { 29 | this.grammarTokens = grammarTokens; 30 | this.parserTokens = parserTokens; 31 | 32 | expect(this.parserTokens.length).greaterThan(0, "parserTokens should have at least one token"); 33 | expect(this.grammarTokens.length).greaterThan(0, "grammarTokens should have at least one token"); 34 | } 35 | 36 | public assertSame() { 37 | this.assertTokenCount(); 38 | this.assertTokenOffsets(); 39 | } 40 | 41 | public assertTokenCount() { 42 | expect(this.parserTokens.length).equals(this.grammarTokens.length, "token counts are not equal"); 43 | } 44 | 45 | public assertTokenOffsets() { 46 | for (let i = 0; i < this.parserTokens.length; i++) { 47 | const pt = this.parserTokens[i]; 48 | const gt = this.grammarTokens[i]; 49 | 50 | expect(pt.documentStartIndex).eq(gt.startIndex, "startIndex does not match"); 51 | expect(pt.documentEndIndex).eq(gt.endIndex, "endIndex does not match"); 52 | } 53 | } 54 | 55 | public static NormalizeGrammarTokens(tokens: vsctm.IToken[]): vsctm.IToken[] { 56 | let result: vsctm.IToken[] = []; 57 | 58 | for (let i = 0; i < tokens.length; i++) { 59 | const token = tokens[i]; 60 | 61 | // PQ doesn't return tokens for whitespace. 62 | // Remove them from the grammar results. 63 | if (token.scopes.length == 1) { 64 | continue; 65 | } 66 | 67 | // PQ returns strings as a single token. 68 | // Grammar returns separate tokens for punctuation. 69 | // TODO: multiline string tokens? 70 | if (token.scopes.includes(Scopes.QuoteStringBegin) || token.scopes.includes(Scopes.QuotedIdentifierBegin)) { 71 | var startIndex = token.startIndex; 72 | 73 | // next token should be string 74 | var middleToken = tokens[++i]; 75 | var scopes = middleToken.scopes; 76 | 77 | // final token should be end quote 78 | var endToken = tokens[++i]; 79 | var endIndex = endToken.endIndex; 80 | 81 | // sanity 82 | if (token.scopes.includes(Scopes.QuoteStringBegin)) { 83 | expect(scopes).contains(Scopes.String, "middle token should be a string"); 84 | expect(endToken.scopes).contains(Scopes.QuoteStringEnd, "third token should be end quote"); 85 | } else { 86 | expect(scopes).contains(Scopes.Identifier, "middle token should be an identifier"); 87 | expect(endToken.scopes).contains(Scopes.QuotedIdentifierEnd, "third token should be end quote"); 88 | } 89 | 90 | result.push({ 91 | startIndex: startIndex, 92 | scopes: scopes, 93 | endIndex: endIndex 94 | }); 95 | 96 | continue; 97 | } 98 | 99 | result.push(token); 100 | } 101 | 102 | return result; 103 | } 104 | } 105 | 106 | export class SingleLineTokenComparer extends TokenComparer { 107 | constructor(query: string, normalize: boolean = true) { 108 | let pqlex: Lexer.TLexer = Lexer.from(query); 109 | pqlex = Lexer.remaining(pqlex); 110 | 111 | // remove whitespace tokens from grammar result 112 | let r = grammar.tokenizeLine(query, null); 113 | let grammarTokens = normalize ? TokenComparer.NormalizeGrammarTokens(r.tokens) : r.tokens; 114 | 115 | super(grammarTokens, pqlex.tokens); 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /tests/incremental.ts: -------------------------------------------------------------------------------- 1 | import vsctm = require('vscode-textmate'); 2 | import { Lexer, Token } from "powerquery-parser"; 3 | import { expect } from "chai"; 4 | import "mocha"; 5 | import * as Shared from "./common"; 6 | 7 | const QueryMultiline = `let 8 | a = 1, 9 | b = "text", 10 | c = Text.From(a) 11 | in 12 | c`; 13 | 14 | describe("Incremental parsing", () => { 15 | it("Simple", () => { 16 | let grammarState: vsctm.StackElement = null; 17 | let pqState: Lexer.TLexer = null; 18 | let lastTokenCount: number = 0; 19 | 20 | let lines = QueryMultiline.split("\n"); 21 | 22 | for (let i = 0; i < lines.length; i++) { 23 | let r = Shared.grammar.tokenizeLine(lines[i], grammarState); 24 | grammarState = r.ruleStack; 25 | 26 | if (pqState == null) { 27 | pqState = Lexer.from(lines[i]); 28 | } else { 29 | pqState = Lexer.appendToDocument(pqState, lines[i]); 30 | } 31 | 32 | lastTokenCount = pqState.tokens.length; 33 | 34 | pqState = Lexer.remaining(pqState); 35 | 36 | let gTokens = Shared.TokenComparer.NormalizeGrammarTokens(r.tokens); 37 | let comparer = new Shared.TokenComparer(gTokens, pqState.tokens.slice(lastTokenCount)); 38 | 39 | comparer.assertTokenCount(); 40 | // TODO: PQ offsets are based on document position rather than line position 41 | // comparer.assertTokenOffsets(); 42 | } 43 | }); 44 | 45 | // TODO: add PQ equivalent 46 | xit("Multiline string literal", () => { 47 | const query = "\"multi\nline\nstring\""; 48 | 49 | let grammarState: vsctm.StackElement = null; 50 | let lines = query.split("\n"); 51 | 52 | for (let i = 0; i < lines.length; i++) { 53 | let r = Shared.grammar.tokenizeLine(lines[i], grammarState); 54 | grammarState = r.ruleStack; 55 | } 56 | }); 57 | }); -------------------------------------------------------------------------------- /tests/parser.ts: -------------------------------------------------------------------------------- 1 | import vsctm = require('vscode-textmate'); 2 | import { Lexer, Token } from "powerquery-parser"; 3 | import { expect } from "chai"; 4 | import "mocha"; 5 | import * as Shared from "./common"; 6 | 7 | describe("Compare parser tokens", () => { 8 | it("Logical", () => { 9 | const query = "if true then true else false"; 10 | const r = new Shared.SingleLineTokenComparer(query); 11 | r.assertSame(); 12 | }); 13 | it("Text", () => { 14 | const query = "\"string one\" & \"string 2\""; 15 | const r = new Shared.SingleLineTokenComparer(query); 16 | r.assertSame(); 17 | }); 18 | it("Numbers", () => { 19 | const query = "1 1.1 5e123 534.1223 2.2e555 -1.3"; 20 | const r = new Shared.SingleLineTokenComparer(query); 21 | r.assertSame(); 22 | }); 23 | it("Quoted identifier", () => { 24 | const query = "#\"identifier with spaces\""; 25 | const r = new Shared.SingleLineTokenComparer(query); 26 | r.assertSame(); 27 | }); 28 | // TODO: Grammar returns single token (good), but it ends at '.' (bad) 29 | xit("Identifier", () => { 30 | const query = "Table.FromRecords"; 31 | const r = new Shared.SingleLineTokenComparer(query); 32 | r.assertSame(); 33 | }); 34 | it("simple function", () => { 35 | const query = "x = () => 1"; 36 | const r = new Shared.SingleLineTokenComparer(query); 37 | r.assertSame(); 38 | }); 39 | // TODO: duration is flagged as type, but starting # is ignored 40 | xit("duration constructor", () => { 41 | const query = "#duration(1,1,1,1)"; 42 | const r = new Shared.SingleLineTokenComparer(query); 43 | r.assertSame(); 44 | }); 45 | // TODO: PQ parser doesn't handle comments in the same way 46 | xit("line comment", () => { 47 | const query = "1; // comment"; 48 | const r = new Shared.SingleLineTokenComparer(query); 49 | r.assertSame(); 50 | }); 51 | // TODO: PQ parser doesn't handle comments in the same way 52 | xit("block comment", () => { 53 | const query = "1 + /* just a comment */ 1"; 54 | const r = new Shared.SingleLineTokenComparer(query); 55 | r.assertSame(); 56 | }); 57 | }); -------------------------------------------------------------------------------- /tests/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es5", 4 | "module": "commonjs", 5 | "sourceMap": true, 6 | "noImplicitAny": true, 7 | "skipLibCheck": true 8 | }, 9 | "exclude": [ 10 | "node_modules" 11 | ] 12 | } -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es5", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017','ES2018' or 'ESNEXT'. */ 4 | "module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ 5 | "sourceMap": true, /* Generates corresponding '.map' file. */ 6 | "esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */ 7 | } 8 | } 9 | --------------------------------------------------------------------------------