├── .eslintignore ├── .eslintrc.js ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md └── workflows │ └── pr-gated.yml ├── .gitignore ├── .pipelines ├── 1loc.yml └── pr-pipeline.yml ├── .prettierrc ├── .vscode ├── extensions.json ├── launch.json ├── settings.json └── tasks.json ├── LICENSE ├── NOTICE.txt ├── README.md ├── SECURITY.md ├── package-lock.json ├── package.json ├── specification.md ├── src ├── example.ts ├── index.ts ├── powerquery-parser │ ├── common │ │ ├── arrayUtils.ts │ │ ├── assert.ts │ │ ├── cancellationToken │ │ │ ├── ICancellationToken.ts │ │ │ ├── cancellationToken.ts │ │ │ └── index.ts │ │ ├── commonSettings.ts │ │ ├── error.ts │ │ ├── immutableSet.ts │ │ ├── index.ts │ │ ├── mapUtils.ts │ │ ├── orderedMap.ts │ │ ├── partialResult │ │ │ ├── index.ts │ │ │ ├── partialResult.ts │ │ │ └── partialResultUtils.ts │ │ ├── patterns.ts │ │ ├── result │ │ │ ├── index.ts │ │ │ ├── result.ts │ │ │ └── resultUtils.ts │ │ ├── setUtils.ts │ │ ├── stringUtils.ts │ │ ├── trace.ts │ │ ├── traversal.ts │ │ └── typeScriptTypeUtils.ts │ ├── index.ts │ ├── language │ │ ├── ast │ │ │ ├── ast.ts │ │ │ ├── astUtils │ │ │ │ ├── assert.ts │ │ │ │ ├── astUtils.ts │ │ │ │ ├── index.ts │ │ │ │ └── typeGuards.ts │ │ │ └── index.ts │ │ ├── comment.ts │ │ ├── constant │ │ │ ├── constant.ts │ │ │ ├── constantUtils.ts │ │ │ └── index.ts │ │ ├── identifierUtils.ts │ │ ├── index.ts │ │ ├── keyword │ │ │ ├── index.ts │ │ │ ├── keyword.ts │ │ │ └── keywordUtils.ts │ │ ├── textUtils.ts │ │ ├── token.ts │ │ └── type │ │ │ ├── index.ts │ │ │ ├── type.ts │ │ │ └── typeUtils │ │ │ ├── assert.ts │ │ │ ├── categorize.ts │ │ │ ├── factories.ts │ │ │ ├── index.ts │ │ │ ├── isCompatible.ts │ │ │ ├── isEqualType.ts │ │ │ ├── isType.ts │ │ │ ├── nameOf.ts │ │ │ ├── primitive.ts │ │ │ ├── simplify.ts │ │ │ ├── typeCheck.ts │ │ │ ├── typeTraceConstant.ts │ │ │ └── typeUtils.ts │ ├── lexer │ │ ├── error.ts │ │ ├── index.ts │ │ ├── lexSettings.ts │ │ ├── lexer.ts │ │ └── lexerSnapshot.ts │ ├── localization │ │ ├── LocProject.json │ │ ├── index.ts │ │ ├── loc │ │ │ ├── bg-BG │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── ca-ES │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── cs-CZ │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── da-DK │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── de-DE │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── el-GR │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── es-ES │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── et-EE │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── eu-ES │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── fi-FI │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── fr-FR │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── gl-ES │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── hi-IN │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── hr-HR │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── hu-HU │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── id-ID │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── it-IT │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── ja-JP │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── kk-KZ │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── ko-KR │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── lt-LT │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── lv-LV │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── ms-MY │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── nb-NO │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── nl-NL │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── pl-PL │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── pt-BR │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── pt-PT │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── ro-RO │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── ru-RU │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── sk-SK │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── sl-SI │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── sr-Cyrl-RS │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── sr-Latn-RS │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── sv-SE │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── th-TH │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── tr-TR │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── uk-UA │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── vi-VN │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ ├── zh-CN │ │ │ │ └── templates │ │ │ │ │ └── template.json.lcl │ │ │ └── zh-TW │ │ │ │ └── templates │ │ │ │ └── template.json.lcl │ │ ├── locale.ts │ │ ├── localization.ts │ │ ├── localizationUtils.ts │ │ ├── templates.ts │ │ └── templates │ │ │ ├── template.bg-BG.json │ │ │ ├── template.ca-ES.json │ │ │ ├── template.cs-CZ.json │ │ │ ├── template.da-DK.json │ │ │ ├── template.de-DE.json │ │ │ ├── template.el-GR.json │ │ │ ├── template.es-ES.json │ │ │ ├── template.et-EE.json │ │ │ ├── template.eu-ES.json │ │ │ ├── template.fi-FI.json │ │ │ ├── template.fr-FR.json │ │ │ ├── template.gl-ES.json │ │ │ ├── template.hi-IN.json │ │ │ ├── template.hr-HR.json │ │ │ ├── template.hu-HU.json │ │ │ ├── template.id-ID.json │ │ │ ├── template.it-IT.json │ │ │ ├── template.ja-JP.json │ │ │ ├── template.json │ │ │ ├── template.kk-KZ.json │ │ │ ├── template.ko-KR.json │ │ │ ├── template.lt-LT.json │ │ │ ├── template.lv-LV.json │ │ │ ├── template.ms-MY.json │ │ │ ├── template.nb-NO.json │ │ │ ├── template.nl-NL.json │ │ │ ├── template.pl-PL.json │ │ │ ├── template.pt-BR.json │ │ │ ├── template.pt-PT.json │ │ │ ├── template.ro-RO.json │ │ │ ├── template.ru-RU.json │ │ │ ├── template.sk-SK.json │ │ │ ├── template.sl-SI.json │ │ │ ├── template.sr-Cyrl-RS.json │ │ │ ├── template.sr-Latn-RS.json │ │ │ ├── template.sv-SE.json │ │ │ ├── template.th-TH.json │ │ │ ├── template.tr-TR.json │ │ │ ├── template.uk-UA.json │ │ │ ├── template.vi-VN.json │ │ │ ├── template.zh-CN.json │ │ │ └── template.zh-TW.json │ ├── parser │ │ ├── context │ │ │ ├── context.ts │ │ │ ├── contextUtils.ts │ │ │ └── index.ts │ │ ├── disambiguation │ │ │ ├── disambiguation.ts │ │ │ ├── disambiguationUtils.ts │ │ │ └── index.ts │ │ ├── error.ts │ │ ├── index.ts │ │ ├── nodeIdMap │ │ │ ├── ancestryUtils.ts │ │ │ ├── index.ts │ │ │ ├── nodeIdMap.ts │ │ │ ├── nodeIdMapIterator.ts │ │ │ ├── nodeIdMapUtils │ │ │ │ ├── childSelectors.ts │ │ │ │ ├── commonSelectors.ts │ │ │ │ ├── idUtils.ts │ │ │ │ ├── index.ts │ │ │ │ ├── leafSelectors.ts │ │ │ │ ├── nodeIdMapUtils.ts │ │ │ │ ├── parentSelectors.ts │ │ │ │ └── specializedSelectors.ts │ │ │ ├── xorNode.ts │ │ │ └── xorNodeUtils.ts │ │ ├── parseSettings.ts │ │ ├── parseState │ │ │ ├── index.ts │ │ │ ├── parseState.ts │ │ │ └── parseStateUtils.ts │ │ ├── parser │ │ │ ├── index.ts │ │ │ ├── parser.ts │ │ │ └── parserUtils.ts │ │ └── parsers │ │ │ ├── combinatorialParserV2 │ │ │ ├── combinatorialParserV2.ts │ │ │ ├── combineOperatorsAndOperands.ts │ │ │ ├── commonTypes.ts │ │ │ ├── index.ts │ │ │ └── readOperatorsAndOperands.ts │ │ │ ├── index.ts │ │ │ ├── naiveParseSteps.ts │ │ │ └── recursiveDescentParser.ts │ ├── settings.ts │ └── task │ │ ├── index.ts │ │ ├── task.ts │ │ └── taskUtils.ts └── test │ ├── helperUtils.ts │ ├── index.ts │ ├── libraryTest │ ├── common │ │ ├── cancellationToken.test.ts │ │ └── stringUtils.test.ts │ ├── identifierUtils.test.ts │ ├── language │ │ ├── astUtils.test.ts │ │ └── typeUtils │ │ │ ├── isCompatible.test.ts │ │ │ ├── nameOf.test.ts │ │ │ ├── typeCheck.test.ts │ │ │ └── typeUtils.test.ts │ ├── lexer │ │ ├── lexError.test.ts │ │ ├── lexIncremental.test.ts │ │ ├── lexMultilineTokens.test.ts │ │ └── lexSimple.test.ts │ ├── parser │ │ ├── customParser.test.ts │ │ ├── error.test.ts │ │ ├── identifierContextKind.test.ts │ │ ├── parseChildren.test.ts │ │ ├── parseColumnNumber.test.ts │ │ ├── parseIdUtils.test.ts │ │ ├── parseNodeIdMapUtils.test.ts │ │ └── parseSimple.test.ts │ ├── textUtils.test.ts │ └── tokenizer │ │ ├── tokenizerIncremental.test.ts │ │ └── tokenizerSimple.test.ts │ ├── mochaConfig.json │ ├── resourceTest │ └── lexParseResource.test.ts │ ├── resources │ └── microsoft-DataConnectors │ │ ├── DataWorldSwagger │ │ ├── DataWorldSwagger.pq │ │ └── DataWorldSwagger.query.pq │ │ ├── DirectQueryForSQL │ │ ├── DirectQueryForSQL.pq │ │ └── DirectQueryForSQL.query.pq │ │ ├── Github │ │ ├── github.pq │ │ └── github.query.pq │ │ ├── HelloWorld │ │ ├── HelloWorld.pq │ │ └── HelloWorld.query.pq │ │ ├── HelloWorldWithDocs │ │ ├── HelloWorldWithDocs.pq │ │ └── HelloWorldWithDocs.query.pq │ │ ├── NativeQuery │ │ └── ODBC │ │ │ └── SQL ODBC │ │ │ ├── Finish │ │ │ ├── Diagnostics.pqm │ │ │ ├── OdbcConstants.pqm │ │ │ └── SqlODBC.pq │ │ │ └── Start │ │ │ ├── Diagnostics.pqm │ │ │ ├── OdbcConstants.pqm │ │ │ └── SqlODBC.pq │ │ ├── NavigationTable │ │ ├── NavigationTable.pq │ │ └── NavigationTable.query.pq │ │ ├── OAuthPKCE │ │ └── PKCESample.pq │ │ ├── ODBC │ │ ├── HiveSample │ │ │ ├── Diagnostics.pqm │ │ │ ├── HiveSample.pq │ │ │ ├── HiveSample.query.pq │ │ │ └── OdbcConstants.pqm │ │ ├── ImpalaODBC │ │ │ ├── ImpalaODBC.pq │ │ │ └── ImpalaODBC.query.pq │ │ ├── RedshiftODBC │ │ │ ├── Diagnostics.pqm │ │ │ ├── OdbcConstants.pqm │ │ │ ├── RedshiftODBC.pq │ │ │ └── RedshiftODBC.query.pq │ │ ├── SnowflakeODBC │ │ │ ├── Diagnostics.pqm │ │ │ ├── OdbcConstants.pqm │ │ │ ├── SnowflakeODBC.pq │ │ │ └── SnowflakeODBC.query.pq │ │ └── SqlODBC │ │ │ ├── .vscode │ │ │ └── settings.json │ │ │ ├── Diagnostics.pqm │ │ │ ├── OdbcConstants.pqm │ │ │ ├── SqlODBC.pq │ │ │ └── SqlODBC.query.pq │ │ ├── OData │ │ └── AnnotationsSample │ │ │ ├── AnnotationsSample.pq │ │ │ └── AnnotationsSample.query.pq │ │ ├── OpenApiSample │ │ ├── OpenApiSample.pq │ │ ├── OpenApiSample.query.pq │ │ └── apisGuru.json │ │ ├── TripPin │ │ ├── 1-OData │ │ │ ├── TripPin.pq │ │ │ └── TripPin.query.pq │ │ ├── 10-TableView1 │ │ │ ├── Diagnostics.pqm │ │ │ ├── Table.ChangeType.pqm │ │ │ ├── Table.GenerateByPage.pqm │ │ │ ├── Table.ToNavigationTable.pqm │ │ │ ├── TripPin.pq │ │ │ └── TripPin.query.pq │ │ ├── 2-Rest │ │ │ ├── TripPin.pq │ │ │ └── TripPin.query.pq │ │ ├── 3-NavTables │ │ │ ├── TripPin.pq │ │ │ └── TripPin.query.pq │ │ ├── 4-Paths │ │ │ ├── TripPin.pq │ │ │ └── TripPin.query.pq │ │ ├── 5-Paging │ │ │ ├── TripPin.pq │ │ │ └── TripPin.query.pq │ │ ├── 6-Schema │ │ │ ├── TripPin.pq │ │ │ └── TripPin.query.pq │ │ ├── 7-AdvancedSchema │ │ │ ├── Table.ChangeType.pqm │ │ │ ├── Table.GenerateByPage.pqm │ │ │ ├── Table.ToNavigationTable.pqm │ │ │ ├── TripPin.pq │ │ │ └── TripPin.query.pq │ │ ├── 8-Diagnostics │ │ │ ├── Diagnostics.pqm │ │ │ ├── Table.ChangeType.pqm │ │ │ ├── Table.GenerateByPage.pqm │ │ │ ├── Table.ToNavigationTable.pqm │ │ │ ├── TripPin.pq │ │ │ └── TripPin.query.pq │ │ └── 9-TestConnection │ │ │ ├── Diagnostics.pqm │ │ │ ├── Table.ChangeType.pqm │ │ │ ├── Table.GenerateByPage.pqm │ │ │ ├── Table.ToNavigationTable.pqm │ │ │ ├── TripPin.pq │ │ │ └── TripPin.query.pq │ │ └── UnitTesting │ │ ├── UnitTesting.pq │ │ └── UnitTesting.query.pq │ ├── scripts │ ├── createBenchmark.ts │ └── createNodeDump.ts │ └── testUtils │ ├── assertTestUtils.ts │ ├── fileTestUtils.ts │ ├── index.ts │ ├── lexTestUtils.ts │ ├── resourceTestUtils.ts │ ├── testConstants.ts │ └── tokenizerTestUtils.ts ├── style.md └── tsconfig.json /.eslintignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | lib 3 | **/*.js -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: "[BUG]" 5 | labels: bug 6 | assignees: JordanBoltonMN 7 | 8 | --- 9 | 10 | **Expected behavior** 11 | A clear and concise description of what you expected to happen. 12 | 13 | **Actual behavior** 14 | A clear and concise description of the description of what the bug is. 15 | 16 | **To Reproduce** 17 | Please include the following: 18 | * (Required) The Power Query script that triggers the issue. 19 | * (Required) Any non-default settings used in the API call(s) which trigger the issue. 20 | * (Ideally) A minimal reproducible example. Can you reproduce the problem by calling a function in `src/example.ts`? 21 | 22 | **Additional context** 23 | Add any other context about the problem here. 24 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: "[Enhancement]" 5 | labels: enhancement 6 | assignees: JordanBoltonMN 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/workflows/pr-gated.yml: -------------------------------------------------------------------------------- 1 | name: Gated pull request 2 | on: 3 | workflow_dispatch: 4 | pull_request: 5 | branches: [master] 6 | jobs: 7 | build-and-test: 8 | runs-on: windows-latest 9 | steps: 10 | - name: checkout 11 | uses: actions/checkout@v4.1.2 12 | - name: setup node 13 | uses: actions/setup-node@v4.0.2 14 | with: 15 | node-version: "18.17" 16 | - run: node -v 17 | - run: npm ci 18 | - run: npm audit 19 | - run: npm run build 20 | - run: npm run test 21 | -------------------------------------------------------------------------------- /.pipelines/1loc.yml: -------------------------------------------------------------------------------- 1 | # Use the below to create a PAT token for the ceapex organization. 2 | # https://ceapex.visualstudio.com/CEINTL/_wiki/wikis/CEINTL.wiki/107/Localization-with-OneLocBuild-Task 3 | # 4 | # Cron Schedules have been converted using UTC Time Zone and may need to be updated for your location. 5 | 6 | name: 1loc_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r) 7 | 8 | schedules: 9 | - cron: "0 0 * * *" 10 | displayName: Daily midnight build 11 | branches: 12 | include: 13 | - master 14 | always: true 15 | 16 | jobs: 17 | - job: Job_1 18 | displayName: Agent job 1 19 | pool: 20 | vmImage: windows-latest 21 | steps: 22 | - checkout: self 23 | - task: cesve.one-loc-build.one-loc-build.OneLocBuild@2 24 | displayName: "Localization Build: src/powerquery-parser-localization/LocProject.json" 25 | inputs: 26 | locProj: "src/powerquery-parser/localization/LocProject.json" 27 | isCreatePrSelected: true 28 | repoType: gitHub 29 | gitHubPatVariable: "$(GitHubPat)" 30 | packageSourceAuth: patAuth 31 | patVariable: "$(OneLocBuildPat)" 32 | env: 33 | SYSTEM_ACCESSTOKEN: $(System.AccessToken) 34 | - task: PublishBuildArtifacts@1 35 | displayName: "Publish Artifact: drop" 36 | -------------------------------------------------------------------------------- /.pipelines/pr-pipeline.yml: -------------------------------------------------------------------------------- 1 | # Node.js 2 | # Build a general Node.js project with npm. 3 | # Add steps that analyze code, save build artifacts, deploy, and more: 4 | # https://docs.microsoft.com/azure/devops/pipelines/languages/javascript 5 | 6 | trigger: 7 | - master 8 | 9 | pr: 10 | - master 11 | 12 | pool: 13 | vmImage: 'windows-latest' 14 | 15 | steps: 16 | - task: NodeTool@0 17 | inputs: 18 | versionSpec: '16.x' 19 | displayName: 'Install Node.js' 20 | 21 | - task: Npm@1 22 | displayName: 'npm install' 23 | inputs: 24 | command: 'install' 25 | 26 | - task: Npm@1 27 | displayName: 'build' 28 | inputs: 29 | command: 'custom' 30 | customCommand: 'run build' 31 | 32 | - task: Npm@1 33 | displayName: 'npm test' 34 | inputs: 35 | command: 'custom' 36 | customCommand: 'test' 37 | 38 | - task: Npm@1 39 | displayName: 'pack' 40 | inputs: 41 | command: 'custom' 42 | customCommand: 'pack' 43 | 44 | - task: CopyFiles@2 45 | displayName: 'copy package to out directory' 46 | inputs: 47 | SourceFolder: '$(Build.SourcesDirectory)' 48 | Contents: '*.tgz' 49 | TargetFolder: '$(Build.SourcesDirectory)/out' 50 | OverWrite: true 51 | 52 | - task: PoliCheck@1 53 | inputs: 54 | inputType: 'Basic' 55 | targetType: 'F' 56 | targetArgument: 'src' 57 | result: 'PoliCheck.xml' 58 | 59 | - task: PublishTestResults@2 60 | condition: succeededOrFailed() 61 | inputs: 62 | testRunner: JUnit 63 | testResultsFiles: '**/test-results.xml' 64 | 65 | - task: PublishBuildArtifacts@1 66 | inputs: 67 | PathtoPublish: '$(System.DefaultWorkingDirectory)/lib' 68 | ArtifactName: lib 69 | displayName: 'publish lib' 70 | 71 | - task: PublishBuildArtifacts@1 72 | inputs: 73 | PathtoPublish: '$(System.DefaultWorkingDirectory)/out' 74 | ArtifactName: out 75 | displayName: 'publish out directory' 76 | 77 | - task: PublishBuildArtifacts@1 78 | inputs: 79 | PathtoPublish: '$(System.DefaultWorkingDirectory)/../_sdt/logs/PoliCheck' 80 | ArtifactName: PoliCheck 81 | displayName: 'publish policheck results' 82 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "arrowParens": "avoid", 3 | "endOfLine": "auto", 4 | "printWidth": 120, 5 | "tabWidth": 4, 6 | "trailingComma": "all" 7 | } 8 | -------------------------------------------------------------------------------- /.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": [ 3 | "dbaeumer.vscode-eslint", 4 | "esbenp.prettier-vscode", 5 | ], 6 | "unwantedRecommendations": [] 7 | } 8 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "type": "node", 9 | "request": "launch", 10 | "name": "Debug Library Tests", 11 | "program": "${workspaceFolder}/node_modules/mocha/bin/_mocha", 12 | "args": ["--inspect", "--colors", "--timeout", "999999", "${workspaceFolder}/lib/test/libraryTest/**/*.js"], 13 | "preLaunchTask": "watch", 14 | "internalConsoleOptions": "openOnSessionStart" 15 | }, 16 | { 17 | "type": "node", 18 | "request": "launch", 19 | "name": "Debug Resource Tests", 20 | "program": "${workspaceFolder}/node_modules/mocha/bin/_mocha", 21 | "args": [ 22 | "--inspect", 23 | "--colors", 24 | "--timeout", 25 | "999999", 26 | "-r", 27 | "ts-node/register", 28 | "${workspaceFolder}/src/test/resourceTest/**/*.ts" 29 | ], 30 | "preLaunchTask": "build", 31 | "internalConsoleOptions": "openOnSessionStart" 32 | }, 33 | { 34 | "type": "node", 35 | "request": "launch", 36 | "name": "Launch Example", 37 | "program": "${workspaceFolder}\\src\\example.ts", 38 | "outFiles": ["${workspaceFolder}/**/*.js"], 39 | "preLaunchTask": "build" 40 | }, 41 | { 42 | "name": "Launch Benchmark", 43 | "type": "node", 44 | "request": "launch", 45 | "runtimeExecutable": "node", 46 | "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"], 47 | "args": ["${workspaceFolder}/src/test/scripts/createBenchmark.ts"], 48 | "cwd": "${workspaceRoot}", 49 | "internalConsoleOptions": "openOnSessionStart", 50 | "skipFiles": ["/**", "node_modules/**"] 51 | }, 52 | { 53 | "name": "Launch NodeDump", 54 | "type": "node", 55 | "request": "launch", 56 | "runtimeExecutable": "node", 57 | "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"], 58 | "args": ["${workspaceFolder}/src/test/scripts/createNodeDump.ts"], 59 | "cwd": "${workspaceRoot}", 60 | "internalConsoleOptions": "openOnSessionStart", 61 | "skipFiles": ["/**", "node_modules/**"] 62 | } 63 | ] 64 | } 65 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "typescript.tsdk": "node_modules\\typescript\\lib", 3 | "files.exclude": { 4 | "src/**/*.js": true 5 | }, 6 | "editor.formatOnSave": true, 7 | "sarif-viewer.connectToGithubCodeScanning": "off" 8 | } 9 | -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | // See https://go.microsoft.com/fwlink/?LinkId=733558 3 | // for the documentation about the tasks.json format 4 | "version": "2.0.0", 5 | "tasks": [ 6 | { 7 | "label": "build", 8 | "type": "typescript", 9 | "tsconfig": "tsconfig.json", 10 | "problemMatcher": [ 11 | "$tsc" 12 | ], 13 | "group": { 14 | "kind": "build", 15 | "isDefault": true 16 | } 17 | }, 18 | { 19 | "label": "watch", 20 | "type": "typescript", 21 | "tsconfig": "tsconfig.json", 22 | "option": "watch", 23 | "problemMatcher": "$tsc-watch", 24 | "group": "build", 25 | "isBackground": true 26 | } 27 | ] 28 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. All rights reserved. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /NOTICE.txt: -------------------------------------------------------------------------------- 1 | NOTICES AND INFORMATION 2 | Do Not Translate or Localize 3 | 4 | This software incorporates material from third parties. 5 | Microsoft makes certain open source code available at https://3rdpartysource.microsoft.com, 6 | or you may send a check or money order for US $5.00, including the product name, 7 | the open source component name, platform, and version number, to: 8 | 9 | Source Code Compliance Team 10 | Microsoft Corporation 11 | One Microsoft Way 12 | Redmond, WA 98052 13 | USA 14 | 15 | Notwithstanding any other terms, you may reverse engineer this software to the extent 16 | required to debug changes to any libraries licensed under the GNU Lesser General Public License. 17 | 18 | --------------------------------------------------------- 19 | 20 | grapheme-splitter 1.0.4 - MIT 21 | https://github.com/orling/grapheme-splitter 22 | 23 | (c) 2017 Unicode(r), Inc. 24 | Copyright (c) 2015 Orlin Georgiev 25 | 26 | The MIT License (MIT) 27 | 28 | Copyright (c) 2015 Orlin Georgiev 29 | 30 | Permission is hereby granted, free of charge, to any person obtaining a copy 31 | of this software and associated documentation files (the "Software"), to deal 32 | in the Software without restriction, including without limitation the rights 33 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 34 | copies of the Software, and to permit persons to whom the Software is 35 | furnished to do so, subject to the following conditions: 36 | 37 | The above copyright notice and this permission notice shall be included in all 38 | copies or substantial portions of the Software. 39 | 40 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 41 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 42 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 43 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 44 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 45 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 46 | SOFTWARE. 47 | 48 | 49 | 50 | --------------------------------------------------------- 51 | 52 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # powerquery-parser 2 | 3 | [![Build Status](https://dev.azure.com/ms/powerquery-parser/_apis/build/status/microsoft.powerquery-parser?branchName=master)](https://dev.azure.com/ms/powerquery-parser/_build/latest?definitionId=134&branchName=master) 4 | 5 | A parser for the [Power Query/M](https://docs.microsoft.com/en-us/power-query/) language, written in TypeScript. Designed to be consumed by other projects. 6 | 7 | ## How to use 8 | 9 | The most common way to consume the project is to interact with the helper functions found in [taskUtils.ts](src/powerquery-parser/task/taskUtils.ts). There are all-in-one functions, such as `tryLexParse`, which does a full pass on a given document. There are also incremental functions, such as `tryLex` and `tryParse`, which perform one step at a time. Minimal code samples can be found in [example.ts](src/example.ts). 10 | 11 | ## Related projects 12 | 13 | - [powerquery-formatter](https://github.com/microsoft/powerquery-formatter): A code formatter for Power Query which is bundled in the VSCode extension. 14 | - [powerquery-language-services](https://github.com/microsoft/powerquery-language-services): A high level library that wraps the parser for external projects, such as the VSCode extension. Includes features such as Intellisense. 15 | - [vscode-powerquery](https://github.com/microsoft/vscode-powerquery): The VSCode extension for Power Query language support. 16 | - [vscode-powerquery-sdk](https://github.com/microsoft/vscode-powerquery-sdk): The VSCode extension for Power Query connector SDK. 17 | 18 | ## Things to note 19 | 20 | ### Parser 21 | 22 | The parser started off as a naive recursive descent parser with limited backtracking. It mostly followed the [official specification](https://docs.microsoft.com/en-us/powerquery-m/power-query-m-language-specification) released in October 2016. Deviations from the specification should be marked down in [specification.md](specification.md). A combinatorial parser has since been added which uses the naive parser as its base. 23 | 24 | ### Style 25 | 26 | This project uses [prettier](https://github.com/prettier/prettier) as the primary source of style enforcement. Additional style requirements are located in [style.md](style.md). 27 | 28 | ## How to build 29 | 30 | - Install NodeJS 31 | - `npm install` 32 | - `npm run-script build` 33 | 34 | ## How to run tests 35 | 36 | - Install NodeJS 37 | - `npm install` 38 | - `npm test` 39 | 40 | ## Contributing 41 | 42 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 43 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 44 | the rights to use your contribution. For details, visit https://cla.microsoft.com. 45 | 46 | When you submit a pull request, a CLA-bot will automatically determine whether you need to provide 47 | a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions 48 | provided by the bot. You will only need to do this once across all repos using our CLA. 49 | 50 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 51 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 52 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 53 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd). 40 | 41 | 42 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@microsoft/powerquery-parser", 3 | "version": "0.16.1", 4 | "description": "A parser for the Power Query/M formula language.", 5 | "author": "Microsoft", 6 | "license": "MIT", 7 | "keywords": [ 8 | "power query", 9 | "power bi" 10 | ], 11 | "scripts": { 12 | "build": "node_modules\\.bin\\tsc", 13 | "test": "mocha --reporter mocha-multi-reporters --reporter-options configFile=src/test/mochaConfig.json -r ts-node/register src/test/libraryTest/**/*.ts", 14 | "test:resources": "mocha --reporter mocha-multi-reporters --reporter-options configFile=src/test/mochaConfig.json -r ts-node/register src/test/resourceTest/**/*.ts", 15 | "script:benchmark": "npx ts-node src/test/scripts/createBenchmark.ts", 16 | "script:nodeDump": "npx ts-node src/test/scripts/createNodeDump.ts", 17 | "lint": "eslint src --ext ts", 18 | "prepublishOnly": "git clean -xdf && npm install-clean && npm run lint && npm run build && npm run test && npm run test:resources" 19 | }, 20 | "homepage": "https://github.com/microsoft/powerquery-parser#readme", 21 | "repository": { 22 | "type": "git", 23 | "url": "git+https://github.com/microsoft/powerquery-parser.git" 24 | }, 25 | "bugs": { 26 | "url": "https://github.com/microsoft/powerquery-parser/issues" 27 | }, 28 | "main": "lib/powerquery-parser/index.js", 29 | "types": "lib/powerquery-parser/index.d.ts", 30 | "engines": { 31 | "node": ">=16.13.1" 32 | }, 33 | "dependencies": { 34 | "grapheme-splitter": "^1.0.4", 35 | "performance-now": "^2.1.0" 36 | }, 37 | "devDependencies": { 38 | "@types/chai": "^4.3.0", 39 | "@types/mocha": "^9.0.0", 40 | "@types/node": "^17.0.5", 41 | "@typescript-eslint/eslint-plugin": "5.8.1", 42 | "@typescript-eslint/parser": "5.8.1", 43 | "chai": "^4.3.4", 44 | "eslint": "8.5.0", 45 | "eslint-config-prettier": "8.3.0", 46 | "eslint-plugin-prettier": "4.0.0", 47 | "eslint-plugin-promise": "6.0.0", 48 | "eslint-plugin-security": "1.4.0", 49 | "mocha": "^11.1.0", 50 | "mocha-junit-reporter": "^2.0.2", 51 | "mocha-multi-reporters": "^1.5.1", 52 | "prettier": "^2.5.1", 53 | "ts-loader": "^9.2.6", 54 | "ts-node": "^10.4.0", 55 | "typescript": "^4.5.4" 56 | }, 57 | "files": [ 58 | "lib/powerquery-parser/**/*" 59 | ] 60 | } 61 | -------------------------------------------------------------------------------- /specification.md: -------------------------------------------------------------------------------- 1 | # Specification Notes 2 | 3 | There are a few differences between the [Power Query / M Language Specification](https://docs.microsoft.com/en-us/powerquery-m/power-query-m-language-specification), the Power Query implementation, and this implementation. 4 | 5 | ## Where the Power Query parser differs from the specification 6 | 7 | - An additional match of `primary-expression` is added on the following tokens: `@`, `identifier`, or `left-parenthesis`. 8 | - The `identifier` construct was changed so that after a period instead of matching `identifier-start-character` it now matches `identifier-part-character`. 9 | - The `generalized-identifier` construct was changed so that `identifier-start-character` was replaced with `identifier-part-character`. It also accepts quoted identifiers. 10 | - The try-otherwise is normally transformed internally into a try-catch. Instead we need to preserve the original structure so two additional constructs exist, one for try-catch and the other for try-otherwise. 11 | -------------------------------------------------------------------------------- /src/example.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | /* eslint-disable @typescript-eslint/ban-ts-comment */ 5 | /* eslint-disable @typescript-eslint/no-floating-promises */ 6 | /* eslint-disable @typescript-eslint/no-unused-vars */ 7 | /* eslint-disable no-unused-vars */ 8 | 9 | import { DefaultSettings, Lexer, ResultUtils, Task, TaskUtils } from "."; 10 | 11 | parseText(`let x = 1 in try x otherwise 2`); 12 | 13 | async function parseText(text: string): Promise { 14 | // Try lexing and parsing the argument which returns a Result object. 15 | // A Result is the union (Ok | Error). 16 | const task: Task.TriedLexParseTask = await TaskUtils.tryLexParse(DefaultSettings, text); 17 | 18 | // If it was a success then dump the abstract syntax tree (AST) as verbose JSON to console. 19 | if (TaskUtils.isParseStageOk(task)) { 20 | console.log(JSON.stringify(task.ast, undefined, 4)); 21 | } 22 | // Else if the error was during lexing then dump the error to console. 23 | else if (TaskUtils.isLexStageError(task)) { 24 | console.log(task.error.message); 25 | } 26 | // Else if the error was during parsing then dump the error to the console. 27 | else if (TaskUtils.isParseStageError(task)) { 28 | // If we branch on isCommonError we can know if a CommonError or ParseError was thrown. 29 | console.log( 30 | `a ${task.isCommonError ? "CommonError" : "ParseError"} was thrown during parsing: ${task.error.message}`, 31 | ); 32 | 33 | if (!task.isCommonError) { 34 | console.log(`parsed ${task.nodeIdMapCollection.leafIds.size} leaf nodes`); 35 | } 36 | } 37 | } 38 | 39 | // @ts-ignore 40 | function lexText(text: string): void { 41 | // Notice that the Lexer.State variable is declared using let instead of const. 42 | // This is because calling Lexer functions return a new state object. 43 | 44 | // An error state is returned if the argument can't be lexed. 45 | // Use either the typeguard Lexer.isErrorState to discover if it's an error state, 46 | // or Lexer.errorLineMap to quickly get all lines with errors. 47 | // Note: At this point all errors are isolated to a single line. 48 | // Checks for multiline errors, such as an unterminated string, have not been processed. 49 | let triedLex: Lexer.TriedLex = Lexer.tryLex(DefaultSettings, text); 50 | 51 | if (ResultUtils.isError(triedLex)) { 52 | console.log(`An error occured while lexing: ${triedLex.error.message}`); 53 | 54 | return; 55 | } 56 | 57 | let lexerState: Lexer.State = triedLex.value; 58 | 59 | // The lexer state might have an error. 60 | // To be sure either use the typeguard Lexer.isErrorState, 61 | // or Lexer.errorLineMap to get an option containing a map of all lines with errors. 62 | const errorLineMap: Lexer.ErrorLineMap | undefined = Lexer.errorLineMap(lexerState); 63 | 64 | if (errorLineMap !== undefined) { 65 | for (const [lineNumber, errorLine] of errorLineMap.entries()) { 66 | console.log(`lineNumber ${lineNumber} has the following error: ${errorLine.error.message}`); 67 | } 68 | 69 | return; 70 | } 71 | 72 | // Appending a line is easy. 73 | // Be aware that this is a Result due to the potential of errors such as 74 | // a cancellation request from the CancellationToken. 75 | triedLex = Lexer.tryAppendLine(lexerState, "// hello world", "\n"); 76 | ResultUtils.assertIsOk(triedLex); 77 | lexerState = triedLex.value; 78 | 79 | // Updating a line number is also easy. 80 | // Be aware that this is a Result due the potential of errors such as 81 | // invalid line numbers or a cancellation request from the CancellationToken. 82 | // For fine-grained control there is also the method Lexer.tryUpdateRange, 83 | // which is how Lexer.tryUpdateLine is implemented. 84 | const triedUpdate: Lexer.TriedLex = Lexer.tryUpdateLine( 85 | lexerState, 86 | lexerState.lines.length - 1, 87 | "// goodbye world", 88 | ); 89 | 90 | if (ResultUtils.isError(triedUpdate)) { 91 | console.log("Failed to update line"); 92 | 93 | return; 94 | } 95 | 96 | lexerState = triedUpdate.value; 97 | 98 | // Once no more changes will occur a LexerSnapshot should be created, which is an immutable copy that: 99 | // * combines multiline tokens together 100 | // (eg. TextLiteralStart + TextLiteralContent + TextLiteralEnd) 101 | // * checks for multiline errors 102 | // (eg. unterminated string error) 103 | 104 | // If you intend to only lex the document once (perform no append/updates), 105 | // then use the `jobs.tryLex` helper function to perform both actions in one go. 106 | 107 | // Creating a LexerSnapshot is a Result due to potential multiline token errors. 108 | const triedLexerSnapshot: Lexer.TriedLexerSnapshot = Lexer.trySnapshot(lexerState); 109 | 110 | if (ResultUtils.isOk(triedLexerSnapshot)) { 111 | const snapshot: Lexer.LexerSnapshot = triedLexerSnapshot.value; 112 | console.log(`numTokens: ${snapshot.tokens}`); 113 | console.log(`numComments: ${snapshot.comments}`); 114 | } 115 | // A multiline token validation error was thrown. 116 | else { 117 | const error: Lexer.LexError.LexError = triedLexerSnapshot.error; 118 | console.log(error.innerError.message); 119 | console.log(JSON.stringify(error.innerError, undefined, 4)); 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | export * from "./powerquery-parser"; 5 | -------------------------------------------------------------------------------- /src/powerquery-parser/common/arrayUtils.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import { Assert } from "."; 5 | 6 | export function all( 7 | collection: ReadonlyArray, 8 | predicate: (value: T) => boolean = (value: T): boolean => Boolean(value), 9 | ): boolean { 10 | for (const element of collection) { 11 | if (!predicate(element)) { 12 | return false; 13 | } 14 | } 15 | 16 | return true; 17 | } 18 | 19 | export function assertGet(collection: ReadonlyArray, index: number, message?: string, details?: object): T { 20 | return Assert.asDefined(collection[index], message, details); 21 | } 22 | 23 | export function assertIncludes(collection: ReadonlyArray, element: T, message?: string, details?: object): void { 24 | Assert.isTrue( 25 | collection.includes(element), 26 | message ?? "collection.includes(element) failed", 27 | details ?? { collection, element }, 28 | ); 29 | } 30 | 31 | export function assertNonZeroLength(collection: ReadonlyArray, message?: string, details?: object): void { 32 | Assert.isTrue( 33 | collection.length > 0, 34 | message ?? `collection should have at least one element in it`, 35 | details ?? { 36 | collectionLength: collection.length, 37 | }, 38 | ); 39 | } 40 | 41 | export function assertReplaceAtIndex(collection: ReadonlyArray, value: T, index: number): T[] { 42 | Assert.isFalse(index < 0 || index >= collection.length, "index < 0 || index >= collection.length", { 43 | index, 44 | collectionLength: collection.length, 45 | }); 46 | 47 | return [...collection.slice(0, index), value, ...collection.slice(index + 1)]; 48 | } 49 | 50 | export function assertRemoveFirstInstance(collection: ReadonlyArray, element: T): T[] { 51 | return assertRemoveAtIndex(collection, collection.indexOf(element)); 52 | } 53 | 54 | export function assertReplaceFirstInstance(collection: ReadonlyArray, oldValue: T, newValue: T): T[] { 55 | return assertReplaceAtIndex(collection, newValue, collection.indexOf(oldValue)); 56 | } 57 | 58 | export function assertRemoveAtIndex(collection: ReadonlyArray, index: number): T[] { 59 | Assert.isFalse(index < 0 || index >= collection.length, "index < 0 || index >= collection.length", { 60 | index, 61 | collectionLength: collection.length, 62 | }); 63 | 64 | return [...collection.slice(0, index), ...collection.slice(index + 1)]; 65 | } 66 | 67 | export function concatUnique(left: ReadonlyArray, right: ReadonlyArray): ReadonlyArray { 68 | const partial: T[] = [...left]; 69 | 70 | for (const element of right) { 71 | if (partial.indexOf(element) === -1) { 72 | partial.push(element); 73 | } 74 | } 75 | 76 | return partial; 77 | } 78 | 79 | export function enumerate(collection: ReadonlyArray): ReadonlyArray<[T, number]> { 80 | return collection.map((value: T, index: number) => [value, index]); 81 | } 82 | 83 | export function isSubset( 84 | largerCollection: ReadonlyArray, 85 | smallerCollection: ReadonlyArray, 86 | comparer: (left: T, right: T) => boolean = (left: T, right: T): boolean => left === right, 87 | ): boolean { 88 | if (smallerCollection.length > largerCollection.length) { 89 | return false; 90 | } 91 | 92 | for (const smallerCollectionValue of smallerCollection) { 93 | let foundMatch: boolean = false; 94 | 95 | for (const largerCollectionValue of largerCollection) { 96 | if (comparer(smallerCollectionValue, largerCollectionValue)) { 97 | foundMatch = true; 98 | break; 99 | } 100 | } 101 | 102 | if (!foundMatch) { 103 | return false; 104 | } 105 | } 106 | 107 | return true; 108 | } 109 | 110 | export async function mapAsync( 111 | collection: ReadonlyArray, 112 | map: (value: T) => Promise, 113 | ): Promise> { 114 | const tasks: ReadonlyArray> = collection.map(map); 115 | 116 | return await Promise.all(tasks); 117 | } 118 | 119 | export function range(size: number, startAt: number = 0): ReadonlyArray { 120 | // tslint:disable-next-line: prefer-array-literal 121 | return [...Array(size).keys()].map((index: number) => index + startAt); 122 | } 123 | -------------------------------------------------------------------------------- /src/powerquery-parser/common/assert.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import { CommonError } from "."; 5 | 6 | export function asDefined(value: T | undefined, message?: string, details?: object): NonNullable { 7 | isDefined(value, message, details); 8 | 9 | return value; 10 | } 11 | 12 | export function asInstanceofError(value: T): Error { 13 | isInstanceofError(value); 14 | 15 | return value; 16 | } 17 | 18 | export function isTrue(value: boolean, message?: string, details?: object): asserts value is true { 19 | if (value !== true) { 20 | throw new CommonError.InvariantError(message ?? `assert failed, expected value to be true`, details); 21 | } 22 | } 23 | 24 | export function isFalse(value: boolean, message?: string, details?: object): asserts value is false { 25 | if (value !== false) { 26 | throw new CommonError.InvariantError(message ?? `assert failed, expected value to be false`, details); 27 | } 28 | } 29 | 30 | export function isNever(_: never): never { 31 | throw new CommonError.InvariantError(`Should never be reached. Stack trace: ${new Error().stack}`); 32 | } 33 | 34 | export function isInstanceofError(value: T | Error): asserts value is Error { 35 | if (!(value instanceof Error)) { 36 | throw new CommonError.InvariantError(`Expected value to be instanceof Error`, { 37 | typeof: typeof value, 38 | value, 39 | }); 40 | } 41 | } 42 | 43 | export function isDefined( 44 | value: T | undefined, 45 | message?: string, 46 | details?: object, 47 | ): asserts value is NonNullable { 48 | if (value === undefined) { 49 | throw new CommonError.InvariantError(message ?? `assert failed, expected value to be defined`, details); 50 | } 51 | } 52 | 53 | export function isUndefined(value: T | undefined, message?: string, details?: object): asserts value is undefined { 54 | if (value !== undefined) { 55 | throw new CommonError.InvariantError(message ?? `assert failed, expected value to be undefined`, details); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/powerquery-parser/common/cancellationToken/ICancellationToken.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | export interface ICancellationToken { 5 | isCancelled: () => boolean; 6 | throwIfCancelled: () => void; 7 | cancel: (reason: string) => void; 8 | } 9 | -------------------------------------------------------------------------------- /src/powerquery-parser/common/cancellationToken/cancellationToken.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import { CommonError } from ".."; 5 | import { ICancellationToken } from "./ICancellationToken"; 6 | 7 | // Cancelled after X calls are made to isCancelled. 8 | // Not really useful other than as an example of how to create your own cancellation token. 9 | export class CounterCancellationToken implements ICancellationToken { 10 | private cancelReason: string | undefined; 11 | private counter: number; 12 | private wasForceCancelled: boolean; 13 | 14 | constructor(private readonly cancellationThreshold: number) { 15 | this.wasForceCancelled = false; 16 | this.counter = 0; 17 | } 18 | 19 | public isCancelled(): boolean { 20 | this.counter += 1; 21 | 22 | return this.wasForceCancelled || this.counter >= this.cancellationThreshold; 23 | } 24 | 25 | public throwIfCancelled(): void { 26 | if (this.isCancelled()) { 27 | throw new CommonError.CancellationError( 28 | this, 29 | this.cancelReason ?? `Exceeded ${this.cancellationThreshold} calls`, 30 | ); 31 | } 32 | } 33 | 34 | public cancel(reason: string): void { 35 | this.wasForceCancelled = true; 36 | this.cancelReason = reason; 37 | } 38 | } 39 | 40 | // Cancelled after X milliseconds. 41 | export class TimedCancellationToken implements ICancellationToken { 42 | private readonly threshold: number; 43 | private cancelReason: string | undefined; 44 | private wasForceCancelled: boolean; 45 | 46 | constructor(private readonly milliseconds: number) { 47 | this.threshold = Date.now() + milliseconds; 48 | this.wasForceCancelled = false; 49 | } 50 | 51 | public throwIfCancelled(): void { 52 | if (this.isCancelled()) { 53 | throw new CommonError.CancellationError(this, this.cancelReason ?? `Exceeded ${this.milliseconds}ms`); 54 | } 55 | } 56 | 57 | public isCancelled(): boolean { 58 | return this.wasForceCancelled || Date.now() >= this.threshold; 59 | } 60 | 61 | public cancel(reason: string): void { 62 | this.wasForceCancelled = true; 63 | this.cancelReason = reason; 64 | } 65 | } 66 | 67 | export const ExpiredCancellationToken: ICancellationToken = new TimedCancellationToken(0); 68 | 69 | // In case you need to provide a cancellation token but don't want to support cancellation. 70 | export const NoOpCancellationToken: ICancellationToken = { 71 | isCancelled: () => false, 72 | // eslint-disable-next-line @typescript-eslint/no-empty-function 73 | throwIfCancelled: () => {}, 74 | // eslint-disable-next-line @typescript-eslint/no-empty-function 75 | cancel: () => {}, 76 | }; 77 | -------------------------------------------------------------------------------- /src/powerquery-parser/common/cancellationToken/index.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | export * from "./cancellationToken"; 5 | export * from "./ICancellationToken"; 6 | -------------------------------------------------------------------------------- /src/powerquery-parser/common/commonSettings.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import { ICancellationToken } from "./cancellationToken"; 5 | import { TraceManager } from "./trace"; 6 | 7 | export interface CommonSettings { 8 | readonly cancellationToken: ICancellationToken | undefined; 9 | readonly initialCorrelationId: number | undefined; 10 | readonly locale: string; 11 | readonly traceManager: TraceManager; 12 | } 13 | -------------------------------------------------------------------------------- /src/powerquery-parser/common/error.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import { Localization, LocalizationUtils, Templates } from "../localization"; 5 | import { Assert } from "."; 6 | import { ICancellationToken } from "./cancellationToken/ICancellationToken"; 7 | 8 | export type TInnerCommonError = CancellationError | InvariantError | UnknownError; 9 | 10 | export class CommonError extends Error { 11 | constructor(readonly innerError: TInnerCommonError) { 12 | super(innerError.message); 13 | Object.setPrototypeOf(this, CommonError.prototype); 14 | } 15 | } 16 | 17 | export class CancellationError extends Error { 18 | constructor(readonly cancellationToken: ICancellationToken, readonly reason: string) { 19 | super(Localization.error_common_cancellationError(Templates.DefaultTemplates, reason)); 20 | Object.setPrototypeOf(this, CancellationError.prototype); 21 | } 22 | } 23 | 24 | export class InvariantError extends Error { 25 | constructor(readonly invariantBroken: string, readonly details?: object) { 26 | super(Localization.error_common_invariantError(Templates.DefaultTemplates, invariantBroken, details)); 27 | Object.setPrototypeOf(this, InvariantError.prototype); 28 | } 29 | } 30 | 31 | export class UnknownError extends Error { 32 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 33 | constructor(readonly innerError: any, locale: string) { 34 | super(Localization.error_common_unknown(LocalizationUtils.getLocalizationTemplates(locale), innerError)); 35 | Object.setPrototypeOf(this, UnknownError.prototype); 36 | } 37 | } 38 | 39 | export function assertIsCommonError(error: unknown): error is CommonError { 40 | Assert.isTrue(isCommonError(error), "isCommonError(error)"); 41 | 42 | return true; 43 | } 44 | 45 | export function isCommonError(error: unknown): error is CommonError { 46 | return error instanceof CommonError; 47 | } 48 | 49 | export function isCancellationError(error: Error): error is CancellationError { 50 | return error instanceof CancellationError; 51 | } 52 | 53 | export function isTInnerCommonError(error: unknown): error is TInnerCommonError { 54 | return error instanceof CancellationError || error instanceof InvariantError || error instanceof UnknownError; 55 | } 56 | 57 | export function throwIfCancellationError(error: Error): void { 58 | if (isCancellationError(error)) { 59 | throw error; 60 | } 61 | } 62 | 63 | export function ensureCommonError(error: Error, locale: string): CommonError { 64 | if (error instanceof CommonError) { 65 | return error; 66 | } else if (isTInnerCommonError(error)) { 67 | return new CommonError(error); 68 | } else { 69 | return new CommonError(new UnknownError(error, locale)); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/powerquery-parser/common/immutableSet.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | // TODO: this needs to be benchmarked 5 | export class ImmutableSet { 6 | public readonly size: number; 7 | private internalCollection: ReadonlyArray; 8 | 9 | public constructor( 10 | iterable: Iterable = [], 11 | private readonly comparer: (left: T, right: T) => boolean = (left: T, right: T): boolean => left === right, 12 | ) { 13 | this.internalCollection = [...iterable]; 14 | this.size = this.internalCollection.length; 15 | } 16 | 17 | public add(value: T): ImmutableSet { 18 | if (this.has(value)) { 19 | return this; 20 | } else { 21 | return new ImmutableSet(new Set([...this.values(), value]), this.comparer); 22 | } 23 | } 24 | 25 | public addMany(values: Iterable): ImmutableSet { 26 | // eslint-disable-next-line @typescript-eslint/no-this-alias 27 | let result: ImmutableSet = this; 28 | 29 | for (const value of values) { 30 | result = result.add(value); 31 | } 32 | 33 | return result; 34 | } 35 | 36 | public clear(): void { 37 | this.internalCollection = []; 38 | } 39 | 40 | public delete(value: T): ImmutableSet { 41 | const values: ReadonlyArray = [...this.internalCollection.values()].filter( 42 | (item: T) => !this.comparer(item, value), 43 | ); 44 | 45 | if (values.length === this.internalCollection.length) { 46 | return this; 47 | } else { 48 | return new ImmutableSet(new Set(values), this.comparer); 49 | } 50 | } 51 | 52 | public has(value: T): boolean { 53 | for (const element of this.internalCollection) { 54 | if (this.comparer(element, value)) { 55 | return true; 56 | } 57 | } 58 | 59 | return false; 60 | } 61 | 62 | public values(): IterableIterator { 63 | return this.internalCollection.values(); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/powerquery-parser/common/index.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | export * as ArrayUtils from "./arrayUtils"; 5 | export * as Assert from "./assert"; 6 | export * as CommonError from "./error"; 7 | export * as MapUtils from "./mapUtils"; 8 | export * as Pattern from "./patterns"; 9 | export * as SetUtils from "./setUtils"; 10 | export * as StringUtils from "./stringUtils"; 11 | export * as Trace from "./trace"; 12 | export * as Traverse from "./traversal"; 13 | export * as TypeScriptUtils from "./typeScriptTypeUtils"; 14 | 15 | export * from "./cancellationToken"; 16 | export * from "./commonSettings"; 17 | export * from "./immutableSet"; 18 | export * from "./orderedMap"; 19 | export * from "./partialResult"; 20 | export * from "./result"; 21 | -------------------------------------------------------------------------------- /src/powerquery-parser/common/mapUtils.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import { Assert } from "."; 5 | 6 | export function assertDelete(map: Map, key: K, message?: string, details?: object): void { 7 | Assert.isTrue(map.delete(key), message ?? `failed to delete, key is absent`, details ?? { key }); 8 | } 9 | 10 | export function assertGet(map: Map | ReadonlyMap, key: K, message?: string, details?: object): V { 11 | return Assert.asDefined(map.get(key), message ?? `key not found in given map`, details ?? { key }); 12 | } 13 | 14 | export function assertHas(map: Map | ReadonlyMap, key: K, message?: string): void { 15 | Assert.isTrue(map.has(key), message ?? `key is absent`, { key }); 16 | } 17 | 18 | export function assertNotIn(map: Map | ReadonlyMap, key: K, message?: string): void { 19 | Assert.isFalse(map.has(key), message ?? `key is present`, { key }); 20 | } 21 | 22 | export function filter(map: Map | ReadonlyMap, predicate: (key: K, value: V) => boolean): Map { 23 | const filtered: Map = new Map(); 24 | 25 | for (const [key, value] of map.entries()) { 26 | if (predicate(key, value)) { 27 | filtered.set(key, value); 28 | } 29 | } 30 | 31 | return filtered; 32 | } 33 | 34 | export function hasKeys(map: Map | ReadonlyMap, keys: ReadonlyArray): boolean { 35 | for (const key of keys) { 36 | if (!map.has(key)) { 37 | return false; 38 | } 39 | } 40 | 41 | return true; 42 | } 43 | 44 | export function isEqualMap( 45 | left: Map | ReadonlyMap, 46 | right: Map | ReadonlyMap, 47 | comparer: (left: V, right: V) => boolean, 48 | ): boolean { 49 | if (left.size !== right.size) { 50 | return false; 51 | } 52 | 53 | for (const [leftKey, leftValue] of left.entries()) { 54 | const value: V | undefined = right.get(leftKey); 55 | 56 | if (value === undefined || !comparer(leftValue, value)) { 57 | return false; 58 | } 59 | } 60 | 61 | return true; 62 | } 63 | 64 | export function isSubsetMap( 65 | left: Map | ReadonlyMap, 66 | right: Map | ReadonlyMap, 67 | comparer: (left: V, right: V) => boolean, 68 | ): boolean { 69 | if (left.size > right.size) { 70 | return false; 71 | } 72 | 73 | for (const [key, leftType] of left.entries()) { 74 | const rightType: V | undefined = right.get(key); 75 | 76 | if (rightType === undefined || !comparer(leftType, rightType)) { 77 | return false; 78 | } 79 | } 80 | 81 | return true; 82 | } 83 | 84 | export function pick(map: Map | ReadonlyMap, keys: ReadonlyArray): Map { 85 | const newMap: Map = new Map(); 86 | 87 | for (const key of keys) { 88 | newMap.set(key, Assert.asDefined(map.get(key), `key from keys is not found in map`, { key })); 89 | } 90 | 91 | return newMap; 92 | } 93 | -------------------------------------------------------------------------------- /src/powerquery-parser/common/orderedMap.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import { ArrayUtils, Assert } from "."; 5 | 6 | export class OrderedMap implements Map { 7 | public size: number; 8 | public [Symbol.toStringTag]: string; 9 | 10 | private readonly map: Map; 11 | private order: ReadonlyArray; 12 | 13 | constructor(entries?: readonly (readonly [K, V])[] | null | undefined | Map) { 14 | if (!entries) { 15 | this.map = new Map(); 16 | this.order = []; 17 | this.size = 0; 18 | } else { 19 | this.map = new Map(entries); 20 | 21 | if (entries instanceof Map) { 22 | this.order = [...entries.keys()]; 23 | this.size = entries.size; 24 | } else { 25 | this.order = entries.map((pair: readonly [K, V]) => pair[0]); 26 | this.size = entries.length; 27 | } 28 | } 29 | } 30 | 31 | public [Symbol.iterator](): IterableIterator<[K, V]> { 32 | return this.entries(); 33 | } 34 | 35 | public clear(): void { 36 | this.map.clear(); 37 | this.order = []; 38 | } 39 | 40 | public delete(key: K): boolean { 41 | if (this.map.delete(key)) { 42 | this.order = ArrayUtils.assertRemoveFirstInstance(this.order, key); 43 | 44 | return true; 45 | } else { 46 | return false; 47 | } 48 | } 49 | 50 | public *entries(): IterableIterator<[K, V]> { 51 | for (const key of this.order) { 52 | yield [key, Assert.asDefined(this.map.get(key))]; 53 | } 54 | } 55 | 56 | public forEach(callback: (value: V, key: K, map: Map) => void): void { 57 | for (const [key, value] of this.entries()) { 58 | callback(value, key, this.map); 59 | } 60 | } 61 | 62 | public get(key: K): V | undefined { 63 | return this.map.get(key); 64 | } 65 | 66 | public has(key: K): boolean { 67 | return this.map.has(key); 68 | } 69 | 70 | public keys(): IterableIterator { 71 | return this.map.keys(); 72 | } 73 | 74 | public set(key: K, value: V, maintainIndex?: boolean): this { 75 | if (this.has(key)) { 76 | if (!maintainIndex) { 77 | this.order = [...ArrayUtils.assertRemoveFirstInstance(this.order, key), key]; 78 | } 79 | } else { 80 | this.order = [...this.order, key]; 81 | } 82 | 83 | this.map.set(key, value); 84 | 85 | return this; 86 | } 87 | 88 | public values(): IterableIterator { 89 | return this.map.values(); 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /src/powerquery-parser/common/partialResult/index.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import * as PartialResultUtils from "./partialResultUtils"; 5 | 6 | export { PartialResultUtils }; 7 | export * from "./partialResult"; 8 | -------------------------------------------------------------------------------- /src/powerquery-parser/common/partialResult/partialResult.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | // A tri-state Result, also known as a partial. 5 | // The third state is for when a job was partially completed but an error occured partway through. 6 | // 7 | // Example usage: 8 | // type DividedNumbersPartial = { results: number[], errorIndex: number }; 9 | // type DividedNumbersPartialResult = PartialResult; 10 | // 11 | // function divideNumbers(numbers: [number, number][]): DividedNumbersPartialResult { 12 | // try { 13 | // const results: number[] = []; 14 | // for (let i = 0; i < numbers.length; i++) { 15 | // const [numerator, denominator] = numbers[i]; 16 | // 17 | // if (denominator === 0) { 18 | // return PartialResultUtils.incomplete({ results, errorIndex: i }); 19 | // } 20 | // 21 | // results.push(numerator / denominator); 22 | // } 23 | // 24 | // return PartialResultUtils.ok(results); 25 | // } 26 | // catch (error) { 27 | // return PartialResultUtils.error(error); 28 | // } 29 | // } 30 | 31 | export type PartialResult = 32 | | PartialResultOk 33 | | PartialResultIncomplete 34 | | PartialResultError; 35 | 36 | export enum PartialResultKind { 37 | Ok = "Ok", 38 | Incomplete = "Incomplete", 39 | Error = "Error", 40 | } 41 | 42 | export interface PartialResultOk { 43 | readonly kind: PartialResultKind.Ok; 44 | readonly value: Value; 45 | } 46 | 47 | export interface PartialResultIncomplete { 48 | readonly kind: PartialResultKind.Incomplete; 49 | readonly partial: Partial; 50 | } 51 | 52 | export interface PartialResultError { 53 | readonly kind: PartialResultKind.Error; 54 | readonly error: Error; 55 | } 56 | -------------------------------------------------------------------------------- /src/powerquery-parser/common/partialResult/partialResultUtils.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import { 5 | PartialResult, 6 | PartialResultError, 7 | PartialResultIncomplete, 8 | PartialResultKind, 9 | PartialResultOk, 10 | } from "./partialResult"; 11 | 12 | export function ok(value: Ok): PartialResultOk { 13 | return { 14 | kind: PartialResultKind.Ok, 15 | value, 16 | }; 17 | } 18 | 19 | export function incomplete(partial: Partial): PartialResultIncomplete { 20 | return { 21 | kind: PartialResultKind.Incomplete, 22 | partial, 23 | }; 24 | } 25 | 26 | export function error(error: Error): PartialResultError { 27 | return { 28 | kind: PartialResultKind.Error, 29 | error, 30 | }; 31 | } 32 | 33 | export function isOk(result: PartialResult): result is PartialResultOk { 34 | return result.kind === PartialResultKind.Ok; 35 | } 36 | 37 | export function isIncomplete( 38 | result: PartialResult, 39 | ): result is PartialResultIncomplete { 40 | return result.kind === PartialResultKind.Incomplete; 41 | } 42 | 43 | export function isError( 44 | result: PartialResult, 45 | ): result is PartialResultError { 46 | return result.kind === PartialResultKind.Error; 47 | } 48 | -------------------------------------------------------------------------------- /src/powerquery-parser/common/patterns.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | export const IdentifierStartCharacter: RegExp = 5 | /(?:[\p{Uppercase_Letter}|\p{Lowercase_Letter}|\p{Titlecase_Letter}|\p{Modifier_Letter}|\p{Other_Letter}|\p{Letter_Number}|\u{5F}]+)/gu; 6 | 7 | export const IdentifierPartCharacters: RegExp = 8 | /(?:[\p{Uppercase_Letter}|\p{Lowercase_Letter}|\p{Titlecase_Letter}|\p{Modifier_Letter}|\p{Other_Letter}|\p{Letter_Number}|\p{Decimal_Number}|\p{Connector_Punctuation}|\p{Spacing_Mark}|\p{Nonspacing_Mark}|\p{Format}]+)/gu; 9 | 10 | export const Whitespace: RegExp = 11 | // eslint-disable-next-line no-control-regex 12 | /(:?[\u000b-\u000c\u2000-\u200a])|(?:\u0009)|(?:\u0020)|(?:\u00a0)|(?:\u1680)|(?:\u202f)|(?:\u205f)|(?:\u3000)/g; 13 | 14 | export const Hex: RegExp = /0[xX][a-fA-F0-9]+/g; 15 | 16 | // eslint-disable-next-line security/detect-unsafe-regex 17 | export const Numeric: RegExp = /(([0-9]*\.[0-9]+)|([0-9]+))([eE][\\+\\-]?[0-9]+)?/g; 18 | -------------------------------------------------------------------------------- /src/powerquery-parser/common/result/index.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import * as ResultUtils from "./resultUtils"; 5 | 6 | export { ResultUtils }; 7 | export * from "./result"; 8 | -------------------------------------------------------------------------------- /src/powerquery-parser/common/result/result.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | export type Result = OkResult | ErrorResult; 5 | 6 | export enum ResultKind { 7 | Ok = "Ok", 8 | Error = "Error", 9 | } 10 | 11 | export interface OkResult { 12 | readonly kind: ResultKind.Ok; 13 | readonly value: T; 14 | } 15 | 16 | export interface ErrorResult { 17 | readonly kind: ResultKind.Error; 18 | readonly error: E; 19 | } 20 | -------------------------------------------------------------------------------- /src/powerquery-parser/common/result/resultUtils.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import { Assert, CommonError } from ".."; 5 | import { ErrorResult, OkResult, Result, ResultKind } from "./result"; 6 | 7 | export function assertIsOk(result: Result): asserts result is OkResult { 8 | if (!isOk(result)) { 9 | throw new CommonError.InvariantError(`assert failed, result expected to be an Ok`, { 10 | error: result.error, 11 | }); 12 | } 13 | } 14 | 15 | export function assertIsError(result: Result): asserts result is ErrorResult { 16 | if (!isError(result)) { 17 | throw new CommonError.InvariantError(`assert failed, result expected to be an Error`); 18 | } 19 | } 20 | 21 | export function assertOk(result: Result): T { 22 | assertIsOk(result); 23 | 24 | return result.value; 25 | } 26 | 27 | export function assertError(result: Result): E { 28 | assertIsError(result); 29 | 30 | return result.error; 31 | } 32 | 33 | export function ok(value: T): OkResult { 34 | return { 35 | kind: ResultKind.Ok, 36 | value, 37 | }; 38 | } 39 | 40 | export function error(error: E): ErrorResult { 41 | return { 42 | kind: ResultKind.Error, 43 | error, 44 | }; 45 | } 46 | 47 | export function ensureResult(callback: () => T, locale: string): Result { 48 | try { 49 | return ok(callback()); 50 | } catch (caught: unknown) { 51 | Assert.isInstanceofError(caught); 52 | 53 | return error(CommonError.ensureCommonError(caught, locale)); 54 | } 55 | } 56 | 57 | export async function ensureResultAsync( 58 | callback: () => Promise, 59 | locale: string, 60 | ): Promise> { 61 | try { 62 | return ok(await callback()); 63 | } catch (caught: unknown) { 64 | Assert.isInstanceofError(caught); 65 | 66 | return error(CommonError.ensureCommonError(caught, locale)); 67 | } 68 | } 69 | 70 | export function isOk(result: Result): result is OkResult { 71 | return result.kind === ResultKind.Ok; 72 | } 73 | 74 | export function isError(result: Result): result is ErrorResult { 75 | return result.kind === ResultKind.Error; 76 | } 77 | 78 | export function okOrDefault(result: Result, defaultValue: T): T { 79 | return isOk(result) ? result.value : defaultValue; 80 | } 81 | -------------------------------------------------------------------------------- /src/powerquery-parser/common/setUtils.ts: -------------------------------------------------------------------------------- 1 | import { Assert } from "."; 2 | 3 | export function assertAddUnique(collection: Set, item: T, message?: string, details?: object): void { 4 | Assert.isFalse( 5 | collection.has(item), 6 | message ?? `collection expected to not already contain the given item`, 7 | details ?? { item }, 8 | ); 9 | 10 | collection.add(item); 11 | } 12 | 13 | export function assertDelete(collection: Set, item: T, message?: string, details?: object): void { 14 | Assert.isTrue( 15 | collection.delete(item), 16 | message ?? `collection expected to contain the given item`, 17 | details ?? { item }, 18 | ); 19 | } 20 | -------------------------------------------------------------------------------- /src/powerquery-parser/common/typeScriptTypeUtils.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | // removes `readonly` from T's attributes 5 | export type StripReadonly = { -readonly [K in keyof T]: T[K] }; 6 | 7 | export function isDefined(x: T | undefined): x is T { 8 | return x !== undefined; 9 | } 10 | -------------------------------------------------------------------------------- /src/powerquery-parser/index.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | export * as Language from "./language"; 5 | export * as Lexer from "./lexer"; 6 | export * as Parser from "./parser"; 7 | export * from "./common"; 8 | export * from "./localization"; 9 | export * from "./settings"; 10 | export * from "./task"; 11 | -------------------------------------------------------------------------------- /src/powerquery-parser/language/ast/astUtils/assert.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import * as TypeGuards from "./typeGuards"; 5 | import { Ast } from ".."; 6 | import { CommonError } from "../../../common"; 7 | 8 | export function assertAsNodeKind( 9 | node: Ast.TNode, 10 | expectedNodeKinds: ReadonlyArray | T["kind"], 11 | ): T { 12 | assertIsNodeKind(node, expectedNodeKinds); 13 | 14 | return node; 15 | } 16 | 17 | export function assertAsTBinOpExpression(node: Ast.TNode): Ast.TBinOpExpression { 18 | return assertAsNodeKind(node, [ 19 | Ast.NodeKind.ArithmeticExpression, 20 | Ast.NodeKind.AsExpression, 21 | Ast.NodeKind.EqualityExpression, 22 | Ast.NodeKind.IsExpression, 23 | Ast.NodeKind.LogicalExpression, 24 | Ast.NodeKind.MetadataExpression, 25 | Ast.NodeKind.RelationalExpression, 26 | ]); 27 | } 28 | 29 | export function assertAsTKeyValuePair(node: Ast.TNode): Ast.TKeyValuePair { 30 | return assertAsNodeKind(node, [ 31 | Ast.NodeKind.GeneralizedIdentifierPairedAnyLiteral, 32 | Ast.NodeKind.GeneralizedIdentifierPairedExpression, 33 | Ast.NodeKind.IdentifierPairedExpression, 34 | ]); 35 | } 36 | 37 | export function assertAsTPairedConstant(node: Ast.TNode): Ast.TPairedConstant { 38 | return assertAsNodeKind(node, [ 39 | Ast.NodeKind.AsNullablePrimitiveType, 40 | Ast.NodeKind.AsType, 41 | Ast.NodeKind.ErrorRaisingExpression, 42 | Ast.NodeKind.IsNullablePrimitiveType, 43 | Ast.NodeKind.NullablePrimitiveType, 44 | Ast.NodeKind.NullableType, 45 | Ast.NodeKind.OtherwiseExpression, 46 | Ast.NodeKind.TypePrimaryType, 47 | ]); 48 | } 49 | 50 | export function assertIsNodeKind( 51 | node: Ast.TNode, 52 | expectedNodeKinds: ReadonlyArray | T["kind"], 53 | ): asserts node is T { 54 | if (!TypeGuards.isNodeKind(node, expectedNodeKinds)) { 55 | throw new CommonError.InvariantError(`unexpected node kind`, { 56 | nodeId: node.id, 57 | nodeKind: node.kind, 58 | expectedNodeKinds, 59 | }); 60 | } 61 | } 62 | 63 | export function assertIsTBinOpExpression(node: Ast.TNode): asserts node is Ast.TBinOpExpression { 64 | assertIsNodeKind(node, [ 65 | Ast.NodeKind.ArithmeticExpression, 66 | Ast.NodeKind.AsExpression, 67 | Ast.NodeKind.EqualityExpression, 68 | Ast.NodeKind.IsExpression, 69 | Ast.NodeKind.LogicalExpression, 70 | Ast.NodeKind.MetadataExpression, 71 | Ast.NodeKind.RelationalExpression, 72 | ]); 73 | } 74 | 75 | export function assertIsTKeyValuePair(node: Ast.TNode): asserts node is Ast.TKeyValuePair { 76 | assertIsNodeKind(node, [ 77 | Ast.NodeKind.GeneralizedIdentifierPairedAnyLiteral, 78 | Ast.NodeKind.GeneralizedIdentifierPairedExpression, 79 | Ast.NodeKind.IdentifierPairedExpression, 80 | ]); 81 | } 82 | 83 | export function assertIsTPairedConstant(node: Ast.TNode): asserts node is Ast.TPairedConstant { 84 | assertIsNodeKind(node, [ 85 | Ast.NodeKind.AsNullablePrimitiveType, 86 | Ast.NodeKind.AsType, 87 | Ast.NodeKind.ErrorRaisingExpression, 88 | Ast.NodeKind.IsNullablePrimitiveType, 89 | Ast.NodeKind.NullablePrimitiveType, 90 | Ast.NodeKind.NullableType, 91 | Ast.NodeKind.OtherwiseExpression, 92 | Ast.NodeKind.TypePrimaryType, 93 | ]); 94 | } 95 | -------------------------------------------------------------------------------- /src/powerquery-parser/language/ast/astUtils/index.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | export * from "./assert"; 5 | export * from "./astUtils"; 6 | export * from "./typeGuards"; 7 | -------------------------------------------------------------------------------- /src/powerquery-parser/language/ast/astUtils/typeGuards.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import { Ast } from ".."; 5 | 6 | export function isNodeKind( 7 | node: Ast.TNode, 8 | expectedNodeKinds: ReadonlyArray | T["kind"], 9 | ): node is T { 10 | return Array.isArray(expectedNodeKinds) ? expectedNodeKinds.includes(node.kind) : node.kind === expectedNodeKinds; 11 | } 12 | 13 | export function isTBinOpExpression(node: Ast.TNode): node is Ast.TBinOpExpression { 14 | return isNodeKind< 15 | | Ast.ArithmeticExpression 16 | | Ast.AsExpression 17 | | Ast.EqualityExpression 18 | | Ast.IsExpression 19 | | Ast.LogicalExpression 20 | | Ast.NullCoalescingExpression 21 | | Ast.MetadataExpression 22 | | Ast.RelationalExpression 23 | >(node, [ 24 | Ast.NodeKind.ArithmeticExpression, 25 | Ast.NodeKind.AsExpression, 26 | Ast.NodeKind.EqualityExpression, 27 | Ast.NodeKind.IsExpression, 28 | Ast.NodeKind.LogicalExpression, 29 | Ast.NodeKind.NullCoalescingExpression, 30 | Ast.NodeKind.MetadataExpression, 31 | Ast.NodeKind.RelationalExpression, 32 | ]); 33 | } 34 | 35 | export function isTBinOpExpressionKind(nodeKind: Ast.NodeKind): nodeKind is Ast.TBinOpExpressionNodeKind { 36 | switch (nodeKind) { 37 | case Ast.NodeKind.ArithmeticExpression: 38 | case Ast.NodeKind.AsExpression: 39 | case Ast.NodeKind.EqualityExpression: 40 | case Ast.NodeKind.IsExpression: 41 | case Ast.NodeKind.LogicalExpression: 42 | case Ast.NodeKind.NullCoalescingExpression: 43 | case Ast.NodeKind.MetadataExpression: 44 | case Ast.NodeKind.RelationalExpression: 45 | return true; 46 | 47 | default: 48 | return false; 49 | } 50 | } 51 | 52 | export function isTKeyValuePair(node: Ast.TNode): node is Ast.TKeyValuePair { 53 | switch (node.kind) { 54 | case Ast.NodeKind.GeneralizedIdentifierPairedAnyLiteral: 55 | case Ast.NodeKind.GeneralizedIdentifierPairedExpression: 56 | case Ast.NodeKind.IdentifierPairedExpression: 57 | return true; 58 | 59 | default: 60 | return false; 61 | } 62 | } 63 | 64 | export function isTPairedConstant(node: Ast.TNode): node is Ast.TPairedConstant { 65 | switch (node.kind) { 66 | case Ast.NodeKind.AsNullablePrimitiveType: 67 | case Ast.NodeKind.AsType: 68 | case Ast.NodeKind.ErrorRaisingExpression: 69 | case Ast.NodeKind.IsNullablePrimitiveType: 70 | case Ast.NodeKind.NullablePrimitiveType: 71 | case Ast.NodeKind.NullableType: 72 | case Ast.NodeKind.OtherwiseExpression: 73 | case Ast.NodeKind.TypePrimaryType: 74 | return true; 75 | 76 | default: 77 | return false; 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/powerquery-parser/language/ast/index.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import * as Ast from "./ast"; 5 | import * as AstUtils from "./astUtils"; 6 | 7 | export { Ast, AstUtils }; 8 | -------------------------------------------------------------------------------- /src/powerquery-parser/language/comment.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import { TokenPosition } from "./token"; 5 | 6 | export type TComment = LineComment | MultilineComment; 7 | 8 | export enum CommentKind { 9 | Line = "Line", 10 | Multiline = "Multiline", 11 | } 12 | 13 | export interface IComment { 14 | readonly kind: CommentKind; 15 | readonly data: string; 16 | readonly containsNewline: boolean; 17 | readonly positionStart: TokenPosition; 18 | readonly positionEnd: TokenPosition; 19 | } 20 | 21 | export interface LineComment extends IComment { 22 | readonly kind: CommentKind.Line; 23 | } 24 | 25 | export interface MultilineComment extends IComment { 26 | readonly kind: CommentKind.Multiline; 27 | } 28 | -------------------------------------------------------------------------------- /src/powerquery-parser/language/constant/index.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import * as Constant from "./constant"; 5 | import * as ConstantUtils from "./constantUtils"; 6 | export { Constant, ConstantUtils }; 7 | -------------------------------------------------------------------------------- /src/powerquery-parser/language/index.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import * as Comment from "./comment"; 5 | export * as IdentifierUtils from "./identifierUtils"; 6 | export * as TextUtils from "./textUtils"; 7 | import * as Token from "./token"; 8 | 9 | export { Comment, Token }; 10 | export * from "./ast"; 11 | export * from "./constant"; 12 | export * from "./keyword"; 13 | export * from "./type"; 14 | -------------------------------------------------------------------------------- /src/powerquery-parser/language/keyword/index.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import * as Keyword from "./keyword"; 5 | import * as KeywordUtils from "./keywordUtils"; 6 | 7 | export { Keyword, KeywordUtils }; 8 | -------------------------------------------------------------------------------- /src/powerquery-parser/language/keyword/keyword.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | export enum KeywordKind { 5 | And = "and", 6 | As = "as", 7 | Each = "each", 8 | Else = "else", 9 | Error = "error", 10 | False = "false", 11 | If = "if", 12 | In = "in", 13 | Is = "is", 14 | Let = "let", 15 | Meta = "meta", 16 | Not = "not", 17 | Or = "or", 18 | Otherwise = "otherwise", 19 | Section = "section", 20 | Shared = "shared", 21 | Then = "then", 22 | True = "true", 23 | Try = "try", 24 | Type = "type", 25 | HashBinary = "#binary", 26 | HashDate = "#date", 27 | HashDateTime = "#datetime", 28 | HashDateTimeZone = "#datetimezone", 29 | HashDuration = "#duration", 30 | HashInfinity = "#infinity", 31 | HashNan = "#nan", 32 | HashSections = "#sections", 33 | HashShared = "#shared", 34 | HashTable = "#table", 35 | HashTime = "#time", 36 | } 37 | 38 | export const KeywordKinds: ReadonlyArray = [ 39 | KeywordKind.And, 40 | KeywordKind.As, 41 | KeywordKind.Each, 42 | KeywordKind.Else, 43 | KeywordKind.Error, 44 | KeywordKind.False, 45 | KeywordKind.If, 46 | KeywordKind.In, 47 | KeywordKind.Is, 48 | KeywordKind.Let, 49 | KeywordKind.Meta, 50 | KeywordKind.Not, 51 | KeywordKind.Or, 52 | KeywordKind.Otherwise, 53 | KeywordKind.Section, 54 | KeywordKind.Shared, 55 | KeywordKind.Then, 56 | KeywordKind.True, 57 | KeywordKind.Try, 58 | KeywordKind.Type, 59 | KeywordKind.HashBinary, 60 | KeywordKind.HashDate, 61 | KeywordKind.HashDateTime, 62 | KeywordKind.HashDateTimeZone, 63 | KeywordKind.HashDuration, 64 | KeywordKind.HashInfinity, 65 | KeywordKind.HashNan, 66 | KeywordKind.HashSections, 67 | KeywordKind.HashShared, 68 | KeywordKind.HashTable, 69 | KeywordKind.HashTime, 70 | ]; 71 | 72 | export const ExpressionKeywordKinds: ReadonlyArray = [ 73 | KeywordKind.Each, 74 | KeywordKind.Error, 75 | KeywordKind.False, 76 | KeywordKind.If, 77 | KeywordKind.Let, 78 | KeywordKind.Not, 79 | KeywordKind.True, 80 | KeywordKind.Try, 81 | KeywordKind.Type, 82 | KeywordKind.HashBinary, 83 | KeywordKind.HashDate, 84 | KeywordKind.HashDateTime, 85 | KeywordKind.HashDateTimeZone, 86 | KeywordKind.HashDuration, 87 | KeywordKind.HashInfinity, 88 | KeywordKind.HashNan, 89 | KeywordKind.HashTable, 90 | KeywordKind.HashTime, 91 | ]; 92 | 93 | export const StartOfDocumentKeywords: ReadonlyArray = [...ExpressionKeywordKinds, KeywordKind.Section]; 94 | -------------------------------------------------------------------------------- /src/powerquery-parser/language/keyword/keywordUtils.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import { KeywordKind, KeywordKinds } from "./keyword"; 5 | 6 | export function isKeyword(text: string): text is KeywordKind { 7 | return KeywordKinds.includes(text as KeywordKind); 8 | } 9 | -------------------------------------------------------------------------------- /src/powerquery-parser/language/textUtils.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | export function escape(text: string): string { 5 | let result: string = text; 6 | 7 | for (const [regexp, escaped] of UnescapedWhitespaceRegexp) { 8 | result = result.replace(regexp, escaped); 9 | } 10 | 11 | return result; 12 | } 13 | 14 | export function unescape(text: string): string { 15 | let result: string = text; 16 | 17 | for (const [regexp, literal] of EscapedWhitespaceRegexp) { 18 | result = result.replace(regexp, literal); 19 | } 20 | 21 | return result; 22 | } 23 | 24 | const EscapedWhitespaceRegexp: ReadonlyArray<[RegExp, string]> = [ 25 | [/#\(cr,lf\)/gm, "\r\n"], 26 | [/#\(cr\)/gm, "\r"], 27 | [/#\(lf\)/gm, "\n"], 28 | [/#\(tab\)/gm, "\t"], 29 | [/""/gm, '"'], 30 | [/#\(#\)\(/gm, "#("], 31 | ]; 32 | 33 | const UnescapedWhitespaceRegexp: ReadonlyArray<[RegExp, string]> = [ 34 | [/#/gm, "#(#)"], 35 | [/\r\n/gm, `#(cr,lf)`], 36 | [/\r/gm, `#(cr)`], 37 | [/\n/gm, `#(lf)`], 38 | [/\t/gm, `#(tab)`], 39 | [/"/gm, `""`], 40 | ]; 41 | -------------------------------------------------------------------------------- /src/powerquery-parser/language/type/index.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | export * as Type from "./type"; 5 | export * as TypeUtils from "./typeUtils"; 6 | -------------------------------------------------------------------------------- /src/powerquery-parser/language/type/typeUtils/index.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | export * from "./assert"; 5 | export * from "./categorize"; 6 | export * from "./factories"; 7 | export * from "./isCompatible"; 8 | export * from "./isEqualType"; 9 | export * from "./isType"; 10 | export * from "./nameOf"; 11 | export * from "./primitive"; 12 | export * from "./simplify"; 13 | export * from "./typeCheck"; 14 | export * from "./typeUtils"; 15 | -------------------------------------------------------------------------------- /src/powerquery-parser/language/type/typeUtils/typeTraceConstant.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | export enum TypeUtilsTraceConstant { 5 | AnyUnion = "CreateAnyUnion", 6 | Categorize = "Categorize", 7 | IsCompatible = "IsCompatible", 8 | NameOf = "NameOf", 9 | Simplify = "Simplify", 10 | TypeCheck = "TypeCheck", 11 | TypeUtils = "TypeUtils", 12 | } 13 | -------------------------------------------------------------------------------- /src/powerquery-parser/lexer/index.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import * as LexError from "./error"; 5 | 6 | export { LexError }; 7 | export * from "./lexer"; 8 | export * from "./lexerSnapshot"; 9 | export * from "./lexSettings"; 10 | -------------------------------------------------------------------------------- /src/powerquery-parser/lexer/lexSettings.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import { CommonSettings } from "../common"; 5 | 6 | export type LexSettings = CommonSettings; 7 | -------------------------------------------------------------------------------- /src/powerquery-parser/localization/LocProject.json: -------------------------------------------------------------------------------- 1 | { 2 | "Projects": [ 3 | { 4 | "LanguageSet": "Azure_Languages", 5 | "LocItems": [ 6 | { 7 | "SourceFile": "src\\powerquery-parser\\localization\\templates\\template.json", 8 | "Languages": "bg-BG;ca-ES;cs-CZ;da-DK;de-DE;el-GR;es-ES;et-EE;eu-ES;fi-FI;fr-FR;gl-ES;hi-IN;hr-HR;hu-HU;id-ID;it-IT;ja-JP;kk-KZ;ko-KR;lt-LT;lv-LV;ms-MY;nb-NO;nl-NL;pl-PL;pt-BR;pt-PT;ro-RO;ru-RU;sk-SK;sl-SI;sr-Cyrl-RS;sr-Latn-RS;sv-SE;th-TH;tr-TR;uk-UA;vi-VN;zh-CN;zh-TW", 9 | "LclFile": "src\\powerquery-parser\\localization\\loc\\{Lang}\\templates\\template.json.lcl", 10 | "CopyOption": "LangIDOnName", 11 | "OutputPath": "src\\powerquery-parser\\localization\\templates" 12 | } 13 | ] 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /src/powerquery-parser/localization/index.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import * as LocalizationUtils from "./localizationUtils"; 5 | import * as Templates from "./templates"; 6 | 7 | export { LocalizationUtils, Templates }; 8 | export { Localization } from "./localization"; 9 | export { DefaultLocale, Locale } from "./locale"; 10 | -------------------------------------------------------------------------------- /src/powerquery-parser/localization/locale.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | export enum Locale { 5 | bg_BG = "bg-BG", 6 | ca_EZ = "ca-EZ", 7 | cs_CZ = "cs-CZ", 8 | da_DK = "da-DK", 9 | de_DE = "de-DE", 10 | el_GR = "el-GR", 11 | en_US = "en-US", 12 | es_ES = "es-ES", 13 | et_EE = "et-EE", 14 | eu_ES = "eu-ES", 15 | fi_FI = "fi-FI", 16 | fr_FR = "fr-FR", 17 | gl_ES = "gl-ES", 18 | hi_IN = "hi-IN", 19 | hr_HR = "hr-HR", 20 | hu_HU = "hu-HU", 21 | id_ID = "id-ID", 22 | it_IT = "it-IT", 23 | ja_JP = "ja-JP", 24 | kk_KZ = "kk-KZ", 25 | ko_KR = "ko-KR", 26 | lt_LT = "lt-LT", 27 | lv_LV = "lv-LV", 28 | ms_MY = "ms-MY", 29 | nb_NO = "nb-NO", 30 | nl_NL = "nl-NL", 31 | pl_PL = "pl-PL", 32 | pt_BR = "pt-BR", 33 | pt_PT = "pt-PT", 34 | ro_RO = "ro-RO", 35 | ru_RU = "ru-RU", 36 | sk_SK = "sk-SK", 37 | sl_SI = "sl-SI", 38 | sr_Cyrl_RS = "sr-Cyrl-RS", 39 | sr_Latn_RS = "sr-Latn-RS", 40 | sv_SE = "sv-SE", 41 | th_TH = "th-TH", 42 | tr_TR = "tr-TR", 43 | uk_UA = "uk-UA", 44 | vi_VN = "vi-VN", 45 | zh_CN = "zh-CN", 46 | zh_TW = "zh-TW", 47 | } 48 | 49 | export const DefaultLocale: Locale = Locale.en_US; 50 | -------------------------------------------------------------------------------- /src/powerquery-parser/localization/localizationUtils.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import { DefaultTemplates, ILocalizationTemplates, TemplatesByLocale } from "./templates"; 5 | 6 | export function getLocalizationTemplates(locale: string): ILocalizationTemplates { 7 | return TemplatesByLocale.get(locale.toLowerCase()) ?? DefaultTemplates; 8 | } 9 | -------------------------------------------------------------------------------- /src/powerquery-parser/parser/context/context.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import { Ast, Token } from "../../language"; 5 | import { NodeIdMap } from ".."; 6 | 7 | // Parsing use to be one giant evaluation, leading to an all-or-nothing outcome which was unsuitable for a 8 | // document that was being live edited. 9 | // 10 | // Take the scenario where a user is in the process of adding another element to a ListExpression. 11 | // Once a comma is typed the parser would error out as it also expects the yet untyped element. 12 | // Under the one giant evaluation model there was no way to propagate what was parsed up to that point. 13 | // 14 | // Context is used as a workbook for Ast.TNodes that have started evaluation but haven't yet finished. 15 | // It starts out empty, with no children belonging to it. 16 | // Most (all non-leaf) Ast.TNode's require several sub-Ast.TNodes to be evaluated as well. 17 | // For each sub-Ast.TNode that begins evaluation another Context is created and linked as a child of the original. 18 | // This means if a Ast.TNode has N attributes of type Ast.TNode, then the Ast.TNode is fully evaluated there should be N 19 | // child contexts created belonging under the original Context. 20 | // Once the Ast.TNode evaluation is complete the result is saved on the Context under its astNode attribute. 21 | // 22 | // Back to the scenario listed above, where the user has entered `{1,}`, you could examine the context state to find: 23 | // An incomplete ListExpression context with 3 children 24 | // With the first child being an evaluated Ast.TNode of NodeKind.Constant: `{` 25 | // With the second child being an evaluated Ast.TNode of NodeKind.Csv: `1,` 26 | // With the third child being a yet-to-be evaluated Context of NodeKind.Csv 27 | 28 | export interface State { 29 | readonly nodeIdMapCollection: NodeIdMap.Collection; 30 | root: Node | undefined; 31 | idCounter: number; 32 | } 33 | 34 | export interface Node { 35 | readonly id: number; 36 | readonly kind: T["kind"]; 37 | readonly tokenIndexStart: number; 38 | readonly tokenStart: Token.Token | undefined; 39 | // Incremented for each child context created with the Node as its parent, 40 | // and decremented for each child context deleted. 41 | attributeCounter: number; 42 | attributeIndex: number | undefined; 43 | isClosed: boolean; 44 | } 45 | 46 | export type TNode = Node; 47 | -------------------------------------------------------------------------------- /src/powerquery-parser/parser/context/index.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import * as ParseContext from "./context"; 5 | import * as ParseContextUtils from "./contextUtils"; 6 | 7 | export { ParseContext, ParseContextUtils }; 8 | -------------------------------------------------------------------------------- /src/powerquery-parser/parser/disambiguation/disambiguation.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import { Ast } from "../../language"; 5 | import { ParseError } from ".."; 6 | import { ParseState } from "../parseState"; 7 | import { Result } from "../../common"; 8 | 9 | export type TAmbiguousBracketNode = Ast.FieldProjection | Ast.FieldSelector | Ast.RecordExpression; 10 | 11 | export type TAmbiguousParenthesisNode = Ast.FunctionExpression | Ast.ParenthesizedExpression | Ast.TLogicalExpression; 12 | 13 | export enum DismabiguationBehavior { 14 | Strict = "Strict", 15 | Thorough = "Thorough", 16 | } 17 | 18 | export enum BracketDisambiguation { 19 | FieldProjection = "FieldProjection", 20 | FieldSelection = "FieldSelection", 21 | RecordExpression = "RecordExpression", 22 | } 23 | 24 | export enum ParenthesisDisambiguation { 25 | FunctionExpression = "FunctionExpression", 26 | ParenthesizedExpression = "ParenthesizedExpression", 27 | } 28 | 29 | export interface AmbiguousParse { 30 | readonly parseState: ParseState; 31 | readonly result: Result; 32 | } 33 | -------------------------------------------------------------------------------- /src/powerquery-parser/parser/disambiguation/index.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import * as Disambiguation from "./disambiguation"; 5 | import * as DisambiguationUtils from "./disambiguationUtils"; 6 | 7 | export { Disambiguation, DisambiguationUtils }; 8 | -------------------------------------------------------------------------------- /src/powerquery-parser/parser/index.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | export * as ParseError from "./error"; 5 | export * from "./context"; 6 | export * from "./disambiguation"; 7 | export * from "./nodeIdMap"; 8 | export * from "./parser"; 9 | export * from "./parsers"; 10 | export * from "./parseSettings"; 11 | export * from "./parseState"; 12 | -------------------------------------------------------------------------------- /src/powerquery-parser/parser/nodeIdMap/ancestryUtils.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import { ArrayUtils, Assert } from "../../common"; 5 | import { NodeIdMap, NodeIdMapIterator, XorNodeUtils } from "."; 6 | import { TXorNode, XorNode } from "./xorNode"; 7 | import { Ast } from "../../language"; 8 | 9 | // Builds up an TXorNode path starting from nodeId and goes up to the root of the Ast. 10 | export function assertAncestry(nodeIdMapCollection: NodeIdMap.Collection, nodeId: number): ReadonlyArray { 11 | const ancestryIds: number[] = [nodeId]; 12 | 13 | let parentId: number | undefined = nodeIdMapCollection.parentIdById.get(nodeId); 14 | 15 | while (parentId) { 16 | ancestryIds.push(parentId); 17 | parentId = nodeIdMapCollection.parentIdById.get(parentId); 18 | } 19 | 20 | return NodeIdMapIterator.assertIterXor(nodeIdMapCollection, ancestryIds); 21 | } 22 | 23 | export function assertNth(ancestry: ReadonlyArray, ancestryIndex: number): TXorNode { 24 | return Assert.asDefined(nth(ancestry, ancestryIndex), "ancestryIndex was out of bounds", { 25 | ancestryLength: ancestry.length, 26 | ancestryIndex, 27 | }); 28 | } 29 | 30 | export function assertNthChecked( 31 | ancestry: ReadonlyArray, 32 | ancestryIndex: number, 33 | expectedNodeKinds: ReadonlyArray | T["kind"], 34 | ): XorNode { 35 | const xorNode: TXorNode = assertNth(ancestry, ancestryIndex); 36 | XorNodeUtils.assertIsNodeKind(xorNode, expectedNodeKinds); 37 | 38 | return xorNode; 39 | } 40 | 41 | export function findNodeKind( 42 | ancestry: ReadonlyArray, 43 | expectedNodeKinds: ReadonlyArray | T["kind"], 44 | ): XorNode | undefined { 45 | for (const xorNode of ancestry) { 46 | if (XorNodeUtils.isNodeKind(xorNode, expectedNodeKinds)) { 47 | return xorNode; 48 | } 49 | } 50 | 51 | return undefined; 52 | } 53 | 54 | export function indexOfNodeKind( 55 | ancestry: ReadonlyArray, 56 | expectedNodeKinds: ReadonlyArray | T["kind"], 57 | ): number | undefined { 58 | for (const [xorNode, index] of ArrayUtils.enumerate(ancestry)) { 59 | if (XorNodeUtils.isNodeKind(xorNode, expectedNodeKinds)) { 60 | return index; 61 | } 62 | } 63 | 64 | return undefined; 65 | } 66 | 67 | export function nth(ancestry: ReadonlyArray, ancestryIndex: number): TXorNode | undefined { 68 | return ancestry[ancestryIndex]; 69 | } 70 | 71 | export function nthChecked( 72 | ancestry: ReadonlyArray, 73 | ancestryIndex: number, 74 | expectedNodeKinds: ReadonlyArray | T["kind"], 75 | ): XorNode | undefined { 76 | const xorNode: TXorNode | undefined = nth(ancestry, ancestryIndex); 77 | 78 | return xorNode && XorNodeUtils.isNodeKind(xorNode, expectedNodeKinds) ? xorNode : undefined; 79 | } 80 | -------------------------------------------------------------------------------- /src/powerquery-parser/parser/nodeIdMap/index.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import * as AncestryUtils from "./ancestryUtils"; 5 | import * as NodeIdMap from "./nodeIdMap"; 6 | import * as NodeIdMapIterator from "./nodeIdMapIterator"; 7 | import * as NodeIdMapUtils from "./nodeIdMapUtils"; 8 | import * as XorNodeUtils from "./xorNodeUtils"; 9 | 10 | export { AncestryUtils, NodeIdMap, NodeIdMapIterator, NodeIdMapUtils, XorNodeUtils }; 11 | export * from "./xorNode"; 12 | -------------------------------------------------------------------------------- /src/powerquery-parser/parser/nodeIdMap/nodeIdMap.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import { Ast } from "../../language"; 5 | import { ParseContext } from ".."; 6 | 7 | export type AstNodeById = Map; 8 | export type ChildIdsById = Map>; 9 | export type ContextNodeById = Map; 10 | export type IdsByNodeKind = Map>; 11 | export type ParentIdById = Map; 12 | 13 | export interface Collection { 14 | // Holds all of the Ast.TNodes which have been parsed 15 | readonly astNodeById: AstNodeById; 16 | // Holds all of the ParseContext.TNodes which are being parsed 17 | readonly contextNodeById: ContextNodeById; 18 | 19 | // Maps a childId to its parentId (if one exists) 20 | readonly parentIdById: ParentIdById; 21 | // Holds all of the childIds for a given parentId. 22 | // The childIds are in the order that they were parsed. 23 | readonly childIdsById: ChildIdsById; 24 | readonly idsByNodeKind: IdsByNodeKind; 25 | 26 | // A collection of all leaf nodes (which by definition are Ast.TNodes) 27 | readonly leafIds: Set; 28 | // The right most Ast in the parse context, which can be treated as the most recently parsed node. 29 | readonly rightMostLeaf: Ast.TNode | undefined; 30 | } 31 | 32 | export interface CollectionValidation { 33 | readonly astNodes: Map; 34 | readonly contextNodes: ReadonlyMap; 35 | readonly leafIds: ReadonlyArray; 36 | readonly nodeIdsByNodeKind: Map>; 37 | 38 | readonly badParentChildLink: ReadonlyArray<[number, number]>; 39 | readonly duplicateIds: ReadonlyArray; 40 | readonly unknownByNodeKindNodeIds: ReadonlyArray; 41 | readonly unknownByNodeKindNodeKinds: ReadonlyArray; 42 | readonly unknownChildIdsKeys: ReadonlyArray; 43 | readonly unknownChildIdsValues: ReadonlyArray; 44 | readonly unknownLeafIds: ReadonlyArray; 45 | readonly unknownParentIdKeys: ReadonlyArray; 46 | readonly unknownParentIdValues: ReadonlyArray; 47 | } 48 | 49 | export interface NodeSummary { 50 | readonly nodeKind: Ast.NodeKind; 51 | readonly childIds: ReadonlyArray | undefined; 52 | readonly parentId: number | undefined; 53 | readonly isAstNode: boolean; 54 | } 55 | -------------------------------------------------------------------------------- /src/powerquery-parser/parser/nodeIdMap/nodeIdMapUtils/commonSelectors.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import { Ast, AstUtils } from "../../../language"; 5 | import { ParseContext, ParseContextUtils } from "../../context"; 6 | import { TXorNode, XorNode } from "../xorNode"; 7 | import { Assert } from "../../../common"; 8 | import { Collection } from "../nodeIdMap"; 9 | import { XorNodeUtils } from ".."; 10 | 11 | export function assertXor(nodeIdMapCollection: Collection, nodeId: number): TXorNode { 12 | return Assert.asDefined(xor(nodeIdMapCollection, nodeId), "failed to find the expected node", { nodeId }); 13 | } 14 | 15 | export function assertXorChecked( 16 | nodeIdMapCollection: Collection, 17 | nodeId: number, 18 | expectedNodeKinds: ReadonlyArray | T["kind"], 19 | ): XorNode { 20 | return XorNodeUtils.assertAsNodeKind(assertXor(nodeIdMapCollection, nodeId), expectedNodeKinds); 21 | } 22 | 23 | export function assertAst(nodeIdMapCollection: Collection, nodeId: number): Ast.TNode { 24 | return Assert.asDefined(ast(nodeIdMapCollection, nodeId), "failed to find the expected Ast node", { 25 | nodeId, 26 | }); 27 | } 28 | 29 | export function assertAstChecked( 30 | nodeIdMapCollection: Collection, 31 | nodeId: number, 32 | expectedNodeKinds: ReadonlyArray | T["kind"], 33 | ): T { 34 | return AstUtils.assertAsNodeKind(assertAst(nodeIdMapCollection, nodeId), expectedNodeKinds); 35 | } 36 | 37 | export function assertContext(nodeIdMapCollection: Collection, nodeId: number): ParseContext.TNode { 38 | return Assert.asDefined(context(nodeIdMapCollection, nodeId), "failed to find the expected Context node", { 39 | nodeId, 40 | }); 41 | } 42 | 43 | export function assertContextChecked( 44 | nodeIdMapCollection: Collection, 45 | nodeId: number, 46 | expectedNodeKinds: ReadonlyArray | T["kind"], 47 | ): ParseContext.Node { 48 | return ParseContextUtils.assertAsNodeKind(assertContext(nodeIdMapCollection, nodeId), expectedNodeKinds); 49 | } 50 | 51 | export function xor(nodeIdMapCollection: Collection, nodeId: number): TXorNode | undefined { 52 | const astNode: Ast.TNode | undefined = nodeIdMapCollection.astNodeById.get(nodeId); 53 | 54 | if (astNode !== undefined) { 55 | return XorNodeUtils.boxAst(astNode); 56 | } 57 | 58 | const contextNode: ParseContext.TNode | undefined = nodeIdMapCollection.contextNodeById.get(nodeId); 59 | 60 | if (contextNode !== undefined) { 61 | return XorNodeUtils.boxContext(contextNode); 62 | } 63 | 64 | return undefined; 65 | } 66 | 67 | export function xorChecked( 68 | nodeIdMapCollection: Collection, 69 | nodeId: number, 70 | expectedNodeKinds: ReadonlyArray | T["kind"], 71 | ): XorNode | undefined { 72 | const xorNode: TXorNode | undefined = xor(nodeIdMapCollection, nodeId); 73 | 74 | return xorNode && XorNodeUtils.isNodeKind(xorNode, expectedNodeKinds) ? xorNode : undefined; 75 | } 76 | 77 | export function ast(nodeIdMapCollection: Collection, nodeId: number): Ast.TNode | undefined { 78 | const xorNode: TXorNode | undefined = xor(nodeIdMapCollection, nodeId); 79 | 80 | return xorNode && XorNodeUtils.isAst(xorNode) ? xorNode.node : undefined; 81 | } 82 | 83 | export function astChecked( 84 | nodeIdMapCollection: Collection, 85 | nodeId: number, 86 | expectedNodeKinds: ReadonlyArray | T["kind"], 87 | ): T | undefined { 88 | const astNode: Ast.TNode | undefined = ast(nodeIdMapCollection, nodeId); 89 | 90 | return astNode && AstUtils.isNodeKind(astNode, expectedNodeKinds) ? astNode : undefined; 91 | } 92 | 93 | export function context(nodeIdMapCollection: Collection, nodeId: number): ParseContext.TNode | undefined { 94 | const xorNode: TXorNode | undefined = xor(nodeIdMapCollection, nodeId); 95 | 96 | return xorNode && XorNodeUtils.isContext(xorNode) ? xorNode.node : undefined; 97 | } 98 | 99 | export function contextChecked( 100 | nodeIdMapCollection: Collection, 101 | nodeId: number, 102 | expectedNodeKinds: ReadonlyArray | T["kind"], 103 | ): ParseContext.Node | undefined { 104 | const contextNode: ParseContext.TNode | undefined = context(nodeIdMapCollection, nodeId); 105 | 106 | return contextNode && ParseContextUtils.isNodeKind(contextNode, expectedNodeKinds) ? contextNode : undefined; 107 | } 108 | -------------------------------------------------------------------------------- /src/powerquery-parser/parser/nodeIdMap/nodeIdMapUtils/index.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | export * from "./childSelectors"; 5 | export * from "./commonSelectors"; 6 | export * from "./idUtils"; 7 | export * from "./leafSelectors"; 8 | export * from "./nodeIdMapUtils"; 9 | export * from "./parentSelectors"; 10 | export * from "./specializedSelectors"; 11 | -------------------------------------------------------------------------------- /src/powerquery-parser/parser/nodeIdMap/nodeIdMapUtils/leafSelectors.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import { AstNodeById, Collection } from "../nodeIdMap"; 5 | import { NodeIdMap, XorNodeUtils } from ".."; 6 | import { Assert } from "../../../common"; 7 | import { Ast } from "../../../language"; 8 | import { TXorNode } from "../xorNode"; 9 | import { xor } from "./commonSelectors"; 10 | 11 | // As it's a leaf it's expected to be a Ast.TNode. 12 | export function assertLeftMostLeaf(nodeIdMapCollection: Collection, nodeId: number): Ast.TNode { 13 | return XorNodeUtils.assertAst( 14 | Assert.asDefined(leftMostXor(nodeIdMapCollection, nodeId), `nodeId does not exist in nodeIdMapCollection`, { 15 | nodeId, 16 | }), 17 | ); 18 | } 19 | 20 | export function assertLeftMostXor(nodeIdMapCollection: Collection, nodeId: number): TXorNode { 21 | return Assert.asDefined(leftMostXor(nodeIdMapCollection, nodeId), `nodeId does not exist in nodeIdMapCollection`, { 22 | nodeId, 23 | }); 24 | } 25 | 26 | // Travels down the left most node under the given nodeId by way of the children collection. 27 | export function leftMostXor(nodeIdMapCollection: Collection, nodeId: number): TXorNode | undefined { 28 | const currentNode: TXorNode | undefined = xor(nodeIdMapCollection, nodeId); 29 | 30 | if (currentNode === undefined) { 31 | return undefined; 32 | } 33 | 34 | let currentNodeId: number = currentNode.node.id; 35 | let childIds: ReadonlyArray | undefined = nodeIdMapCollection.childIdsById.get(currentNodeId); 36 | 37 | while (childIds?.length) { 38 | currentNodeId = childIds[0]; 39 | childIds = nodeIdMapCollection.childIdsById.get(currentNodeId); 40 | } 41 | 42 | return xor(nodeIdMapCollection, currentNodeId); 43 | } 44 | 45 | // Same as leftMostXor but also checks if it's an Ast node. 46 | export function leftMostLeaf(nodeIdMapCollection: NodeIdMap.Collection, nodeId: number): Ast.TNode | undefined { 47 | const xorNode: TXorNode | undefined = leftMostXor(nodeIdMapCollection, nodeId); 48 | 49 | return xorNode && XorNodeUtils.isAst(xorNode) ? xorNode.node : undefined; 50 | } 51 | 52 | // There are a few assumed invariants about children: 53 | // * Children were read left to right. 54 | // * Children were placed in childIdsById in the order they were read. 55 | // * Therefore the right-most child is the most recently read which also appears last in the document. 56 | export function rightMostLeaf( 57 | nodeIdMapCollection: Collection, 58 | nodeId: number, 59 | predicate: ((node: Ast.TNode) => boolean) | undefined = undefined, 60 | ): Promise { 61 | const astNodeById: AstNodeById = nodeIdMapCollection.astNodeById; 62 | let nodeIdsToExplore: number[] = [nodeId]; 63 | let rightMost: Ast.TNode | undefined; 64 | 65 | while (nodeIdsToExplore.length) { 66 | const nodeId: number = Assert.asDefined(nodeIdsToExplore.pop()); 67 | const astNode: Ast.TNode | undefined = astNodeById.get(nodeId); 68 | 69 | let addChildren: boolean = false; 70 | 71 | // Check if Ast.TNode or ParserContext.Node 72 | if (astNode !== undefined) { 73 | if (predicate && !predicate(astNode)) { 74 | continue; 75 | } 76 | 77 | // Is leaf, check if it's more right than the previous record. 78 | // As it's a leaf there are no children to add. 79 | if (astNode.isLeaf) { 80 | // Is the first leaf encountered. 81 | if (rightMost === undefined) { 82 | rightMost = astNode; 83 | } 84 | // Compare current leaf node to the existing record. 85 | else if (astNode.tokenRange.tokenIndexStart > rightMost.tokenRange.tokenIndexStart) { 86 | rightMost = astNode; 87 | } 88 | } 89 | // Is not a leaf, no previous record exists. 90 | // Add all children to the queue. 91 | else if (rightMost === undefined) { 92 | addChildren = true; 93 | } 94 | // Is not a leaf, previous record exists. 95 | // Check if we can cull the branch, otherwise add all children to the queue. 96 | else if (astNode.tokenRange.tokenIndexEnd > rightMost.tokenRange.tokenIndexStart) { 97 | addChildren = true; 98 | } 99 | } 100 | // Must be a ParserContext.Node. 101 | // Add all children to the queue as ParserContext.Nodes can have Ast children which are leafs. 102 | else { 103 | addChildren = true; 104 | } 105 | 106 | if (addChildren) { 107 | const childIds: ReadonlyArray | undefined = nodeIdMapCollection.childIdsById.get(nodeId); 108 | 109 | if (childIds !== undefined) { 110 | // Add the child ids in reversed order to prioritize visiting the right most nodes first. 111 | const reversedChildIds: number[] = [...childIds]; 112 | reversedChildIds.reverse(); 113 | nodeIdsToExplore = [...reversedChildIds, ...nodeIdsToExplore]; 114 | } 115 | } 116 | } 117 | 118 | return Promise.resolve(rightMost); 119 | } 120 | 121 | export function rightMostLeafWhere( 122 | nodeIdMapCollection: Collection, 123 | nodeId: number, 124 | predicateFn: ((node: Ast.TNode) => boolean) | undefined, 125 | ): Promise { 126 | return rightMostLeaf(nodeIdMapCollection, nodeId, predicateFn); 127 | } 128 | -------------------------------------------------------------------------------- /src/powerquery-parser/parser/nodeIdMap/nodeIdMapUtils/parentSelectors.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import { Ast, AstUtils } from "../../../language"; 5 | import { boxAst, boxContext } from "../xorNodeUtils"; 6 | import { ParseContext, ParseContextUtils } from "../../context"; 7 | import { TXorNode, XorNode } from "../xorNode"; 8 | import { Assert } from "../../../common"; 9 | import { Collection } from "../nodeIdMap"; 10 | import { XorNodeUtils } from ".."; 11 | 12 | export function assertParentXor(nodeIdMapCollection: Collection, nodeId: number): TXorNode { 13 | return Assert.asDefined(parentXor(nodeIdMapCollection, nodeId), `nodeId doesn't have a parent`, { 14 | nodeId, 15 | }); 16 | } 17 | 18 | export function assertParentXorChecked( 19 | nodeIdMapCollection: Collection, 20 | nodeId: number, 21 | expectedNodeKinds: ReadonlyArray | T["kind"], 22 | ): XorNode { 23 | return XorNodeUtils.assertAsNodeKind(assertParentXor(nodeIdMapCollection, nodeId), expectedNodeKinds); 24 | } 25 | 26 | export function assertParentAst(nodeIdMapCollection: Collection, nodeId: number): Ast.TNode { 27 | return Assert.asDefined( 28 | parentAst(nodeIdMapCollection, nodeId), 29 | "couldn't find the expected parent Ast for nodeId", 30 | { 31 | nodeId, 32 | }, 33 | ); 34 | } 35 | 36 | export function assertParentAstChecked( 37 | nodeIdMapCollection: Collection, 38 | nodeId: number, 39 | expectedNodeKinds: ReadonlyArray | T["kind"], 40 | ): T { 41 | return AstUtils.assertAsNodeKind(assertParentAst(nodeIdMapCollection, nodeId), expectedNodeKinds); 42 | } 43 | 44 | export function assertParentContext(nodeIdMapCollection: Collection, nodeId: number): ParseContext.TNode { 45 | return Assert.asDefined( 46 | parentContext(nodeIdMapCollection, nodeId), 47 | "couldn't find the expected parent ParseContext for nodeId", 48 | { 49 | nodeId, 50 | }, 51 | ); 52 | } 53 | 54 | export function assertParentContextChecked( 55 | nodeIdMapCollection: Collection, 56 | nodeId: number, 57 | expectedNodeKinds: ReadonlyArray | T["kind"], 58 | ): ParseContext.Node { 59 | return ParseContextUtils.assertAsNodeKind(assertParentContext(nodeIdMapCollection, nodeId), expectedNodeKinds); 60 | } 61 | 62 | export function parentXor(nodeIdMapCollection: Collection, childId: number): TXorNode | undefined { 63 | const astNode: Ast.TNode | undefined = parentAst(nodeIdMapCollection, childId); 64 | 65 | if (astNode !== undefined) { 66 | return boxAst(astNode); 67 | } 68 | 69 | const context: ParseContext.TNode | undefined = parentContext(nodeIdMapCollection, childId); 70 | 71 | if (context !== undefined) { 72 | return boxContext(context); 73 | } 74 | 75 | return undefined; 76 | } 77 | 78 | export function parentXorChecked( 79 | nodeIdMapCollection: Collection, 80 | childId: number, 81 | expectedNodeKinds: ReadonlyArray | T["kind"], 82 | ): XorNode | undefined { 83 | const xorNode: TXorNode | undefined = parentXor(nodeIdMapCollection, childId); 84 | 85 | return xorNode && XorNodeUtils.isNodeKind(xorNode, expectedNodeKinds) ? xorNode : undefined; 86 | } 87 | 88 | export function parentAst(nodeIdMapCollection: Collection, childId: number): Ast.TNode | undefined { 89 | const parentId: number | undefined = nodeIdMapCollection.parentIdById.get(childId); 90 | 91 | return parentId !== undefined ? nodeIdMapCollection.astNodeById.get(parentId) : undefined; 92 | } 93 | 94 | export function parentAstChecked( 95 | nodeIdMapCollection: Collection, 96 | childId: number, 97 | expectedNodeKinds: ReadonlyArray | T["kind"], 98 | ): T | undefined { 99 | const astNode: Ast.TNode | undefined = parentAst(nodeIdMapCollection, childId); 100 | 101 | return astNode && AstUtils.isNodeKind(astNode, expectedNodeKinds) ? astNode : undefined; 102 | } 103 | 104 | export function parentContext(nodeIdMapCollection: Collection, childId: number): ParseContext.TNode | undefined { 105 | const parentId: number | undefined = nodeIdMapCollection.parentIdById.get(childId); 106 | 107 | return parentId !== undefined ? nodeIdMapCollection.contextNodeById.get(parentId) : undefined; 108 | } 109 | 110 | export function parentContextChecked( 111 | nodeIdMapCollection: Collection, 112 | childId: number, 113 | expectedNodeKinds: ReadonlyArray | T["kind"], 114 | ): ParseContext.Node | undefined { 115 | const contextNode: ParseContext.TNode | undefined = parentContext(nodeIdMapCollection, childId); 116 | 117 | return contextNode && ParseContextUtils.isNodeKind(contextNode, expectedNodeKinds) ? contextNode : undefined; 118 | } 119 | -------------------------------------------------------------------------------- /src/powerquery-parser/parser/nodeIdMap/xorNode.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import { Ast } from "../../language"; 5 | import { ParseContext } from ".."; 6 | 7 | export enum XorNodeKind { 8 | Ast = "Ast", 9 | Context = "Context", 10 | } 11 | 12 | export type TXorNode = XorNode; 13 | 14 | export type XorNode = AstXorNode | ContextXorNode; 15 | 16 | export type TAstXorNode = AstXorNode; 17 | 18 | export type AstXorNode = IXorNode; 19 | 20 | export type TContextXorNode = ContextXorNode; 21 | 22 | export type ContextXorNode = IXorNode>; 23 | 24 | export interface IXorNode { 25 | readonly kind: Kind; 26 | readonly node: T; 27 | } 28 | 29 | export interface XorNodeTokenRange { 30 | readonly tokenIndexStart: number; 31 | readonly tokenIndexEnd: number; 32 | } 33 | -------------------------------------------------------------------------------- /src/powerquery-parser/parser/parseSettings.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import { Ast } from "../language"; 5 | import { CommonSettings } from "../common"; 6 | import { LexerSnapshot } from "../lexer"; 7 | import { Parser } from "./parser"; 8 | import { ParseState } from "./parseState"; 9 | 10 | export interface ParseSettings extends CommonSettings { 11 | readonly parser: Parser; 12 | readonly newParseState: (lexerSnapshot: LexerSnapshot, overrides: Partial | undefined) => ParseState; 13 | readonly parserEntryPoint: 14 | | ((state: ParseState, parser: Parser, correlationId: number | undefined) => Promise) 15 | | undefined; 16 | } 17 | -------------------------------------------------------------------------------- /src/powerquery-parser/parser/parseState/index.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | export * as ParseStateUtils from "./parseStateUtils"; 5 | export * from "./parseState"; 6 | -------------------------------------------------------------------------------- /src/powerquery-parser/parser/parseState/parseState.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import { Disambiguation } from "../disambiguation"; 5 | import { ICancellationToken } from "../../common"; 6 | import { LexerSnapshot } from "../../lexer"; 7 | import { ParseContext } from ".."; 8 | import { Token } from "../../language"; 9 | import { TraceManager } from "../../common/trace"; 10 | 11 | export interface ParseState { 12 | readonly cancellationToken: ICancellationToken | undefined; 13 | readonly disambiguationBehavior: Disambiguation.DismabiguationBehavior; 14 | readonly lexerSnapshot: LexerSnapshot; 15 | readonly locale: string; 16 | readonly traceManager: TraceManager; 17 | contextState: ParseContext.State; 18 | currentContextNode: ParseContext.TNode | undefined; 19 | currentToken: Token.Token | undefined; 20 | currentTokenKind: Token.TokenKind | undefined; 21 | tokenIndex: number; 22 | } 23 | -------------------------------------------------------------------------------- /src/powerquery-parser/parser/parser/index.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | export * as ParserUtils from "./parserUtils"; 5 | export * from "./parser"; 6 | -------------------------------------------------------------------------------- /src/powerquery-parser/parser/parsers/combinatorialParserV2/commonTypes.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import { Ast } from "../../../language"; 5 | 6 | export type TOperand = Ast.TBinOpExpression | Ast.TUnaryExpression | Ast.TNullablePrimitiveType; 7 | 8 | export enum CombinatorialParserV2TraceConstant { 9 | CombinatorialParseV2 = "CombinatorialParseV2", 10 | } 11 | 12 | export interface OperatorsAndOperands { 13 | readonly operatorConstants: ReadonlyArray; 14 | readonly operands: ReadonlyArray; 15 | } 16 | -------------------------------------------------------------------------------- /src/powerquery-parser/parser/parsers/combinatorialParserV2/index.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | export { CombinatorialParserV2 } from "./combinatorialParserV2"; 5 | -------------------------------------------------------------------------------- /src/powerquery-parser/parser/parsers/index.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import * as NaiveParseSteps from "./naiveParseSteps"; 5 | 6 | export * from "./combinatorialParserV2"; 7 | export { NaiveParseSteps }; 8 | export { CombinatorialParserV2 } from "./combinatorialParserV2"; 9 | export { RecursiveDescentParser } from "./recursiveDescentParser"; 10 | -------------------------------------------------------------------------------- /src/powerquery-parser/parser/parsers/recursiveDescentParser.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import { Parser, ParserUtils } from "../parser"; 5 | import { NaiveParseSteps } from "."; 6 | import { ParseStateUtils } from "../parseState"; 7 | 8 | export const RecursiveDescentParser: Parser = { 9 | applyState: ParseStateUtils.applyState, 10 | copyState: ParseStateUtils.copyState, 11 | checkpoint: ParserUtils.checkpoint, 12 | restoreCheckpoint: ParserUtils.restoreCheckpoint, 13 | 14 | readIdentifier: NaiveParseSteps.readIdentifier, 15 | readGeneralizedIdentifier: NaiveParseSteps.readGeneralizedIdentifier, 16 | readKeyword: NaiveParseSteps.readKeyword, 17 | 18 | readDocument: NaiveParseSteps.readDocument, 19 | 20 | readSectionDocument: NaiveParseSteps.readSectionDocument, 21 | readSectionMembers: NaiveParseSteps.readSectionMembers, 22 | readSectionMember: NaiveParseSteps.readSectionMember, 23 | 24 | readNullCoalescingExpression: NaiveParseSteps.readNullCoalescingExpression, 25 | readExpression: NaiveParseSteps.readExpression, 26 | 27 | readLogicalExpression: NaiveParseSteps.readLogicalExpression, 28 | 29 | readIsExpression: NaiveParseSteps.readIsExpression, 30 | readNullablePrimitiveType: NaiveParseSteps.readNullablePrimitiveType, 31 | 32 | readAsExpression: NaiveParseSteps.readAsExpression, 33 | 34 | readEqualityExpression: NaiveParseSteps.readEqualityExpression, 35 | 36 | readRelationalExpression: NaiveParseSteps.readRelationalExpression, 37 | 38 | readArithmeticExpression: NaiveParseSteps.readArithmeticExpression, 39 | 40 | readMetadataExpression: NaiveParseSteps.readMetadataExpression, 41 | 42 | readUnaryExpression: NaiveParseSteps.readUnaryExpression, 43 | 44 | readPrimaryExpression: NaiveParseSteps.readPrimaryExpression, 45 | readRecursivePrimaryExpression: NaiveParseSteps.readRecursivePrimaryExpression, 46 | 47 | readLiteralExpression: NaiveParseSteps.readLiteralExpression, 48 | 49 | readIdentifierExpression: NaiveParseSteps.readIdentifierExpression, 50 | 51 | readParenthesizedExpression: NaiveParseSteps.readParenthesizedExpression, 52 | 53 | readNotImplementedExpression: NaiveParseSteps.readNotImplementedExpression, 54 | 55 | readInvokeExpression: NaiveParseSteps.readInvokeExpression, 56 | 57 | readListExpression: NaiveParseSteps.readListExpression, 58 | readListItem: NaiveParseSteps.readListItem, 59 | 60 | readRecordExpression: NaiveParseSteps.readRecordExpression, 61 | 62 | readItemAccessExpression: NaiveParseSteps.readItemAccessExpression, 63 | 64 | readFieldSelection: NaiveParseSteps.readFieldSelection, 65 | readFieldProjection: NaiveParseSteps.readFieldProjection, 66 | readFieldSelector: NaiveParseSteps.readFieldSelector, 67 | 68 | readFunctionExpression: NaiveParseSteps.readFunctionExpression, 69 | readParameterList: NaiveParseSteps.readParameterList, 70 | readAsType: NaiveParseSteps.readAsType, 71 | 72 | readEachExpression: NaiveParseSteps.readEachExpression, 73 | 74 | readLetExpression: NaiveParseSteps.readLetExpression, 75 | 76 | readIfExpression: NaiveParseSteps.readIfExpression, 77 | 78 | readTypeExpression: NaiveParseSteps.readTypeExpression, 79 | readType: NaiveParseSteps.readType, 80 | readPrimaryType: NaiveParseSteps.readPrimaryType, 81 | readRecordType: NaiveParseSteps.readRecordType, 82 | readTableType: NaiveParseSteps.readTableType, 83 | readFieldSpecificationList: NaiveParseSteps.readFieldSpecificationList, 84 | readListType: NaiveParseSteps.readListType, 85 | readFunctionType: NaiveParseSteps.readFunctionType, 86 | readParameterSpecificationList: NaiveParseSteps.readParameterSpecificationList, 87 | readNullableType: NaiveParseSteps.readNullableType, 88 | 89 | readErrorRaisingExpression: NaiveParseSteps.readErrorRaisingExpression, 90 | 91 | readErrorHandlingExpression: NaiveParseSteps.readErrorHandlingExpression, 92 | 93 | readRecordLiteral: NaiveParseSteps.readRecordLiteral, 94 | readFieldNamePairedAnyLiterals: NaiveParseSteps.readFieldNamePairedAnyLiterals, 95 | readListLiteral: NaiveParseSteps.readListLiteral, 96 | readAnyLiteral: NaiveParseSteps.readAnyLiteral, 97 | readPrimitiveType: NaiveParseSteps.readPrimitiveType, 98 | 99 | readIdentifierPairedExpressions: NaiveParseSteps.readIdentifierPairedExpressions, 100 | readIdentifierPairedExpression: NaiveParseSteps.readIdentifierPairedExpression, 101 | readGeneralizedIdentifierPairedExpressions: NaiveParseSteps.readGeneralizedIdentifierPairedExpressions, 102 | readGeneralizedIdentifierPairedExpression: NaiveParseSteps.readGeneralizedIdentifierPairedExpression, 103 | }; 104 | -------------------------------------------------------------------------------- /src/powerquery-parser/settings.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import { CombinatorialParserV2, ParseSettings, ParseState, ParseStateUtils } from "./parser"; 5 | import { LexerSnapshot, LexSettings } from "./lexer"; 6 | import { DefaultLocale } from "./localization"; 7 | import { NoOpTraceManagerInstance } from "./common/trace"; 8 | 9 | export type Settings = LexSettings & ParseSettings; 10 | 11 | export const DefaultSettings: Settings = { 12 | newParseState: (lexerSnapshot: LexerSnapshot, overrides: Partial | undefined) => 13 | ParseStateUtils.newState(lexerSnapshot, overrides), 14 | locale: DefaultLocale, 15 | cancellationToken: undefined, 16 | initialCorrelationId: undefined, 17 | parserEntryPoint: undefined, 18 | parser: CombinatorialParserV2, 19 | traceManager: NoOpTraceManagerInstance, 20 | }; 21 | -------------------------------------------------------------------------------- /src/powerquery-parser/task/index.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | export * as Task from "./task"; 5 | export * as TaskUtils from "./taskUtils"; 6 | -------------------------------------------------------------------------------- /src/powerquery-parser/task/task.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import { CommonError, ResultKind } from "../common"; 5 | import { Lexer, Parser } from ".."; 6 | import { NodeIdMap, ParseState } from "../parser"; 7 | import { Ast } from "../language"; 8 | 9 | export type TTask = TriedLexTask | TriedParseTask; 10 | 11 | export type TriedLexTask = LexTaskOk | LexTaskError; 12 | 13 | export type TriedParseTask = ParseTaskOk | ParseTaskCommonError | ParseTaskParseError; 14 | 15 | export type TriedLexParseTask = LexTaskError | TriedParseTask; 16 | 17 | export enum TaskStage { 18 | Lex = "Lex", 19 | Parse = "Parse", 20 | } 21 | 22 | export interface ITask { 23 | readonly stage: TaskStage; 24 | readonly resultKind: ResultKind; 25 | } 26 | 27 | export interface ILexTask extends ITask { 28 | readonly stage: TaskStage.Lex; 29 | } 30 | 31 | export interface LexTaskOk extends ILexTask { 32 | readonly resultKind: ResultKind.Ok; 33 | readonly lexerSnapshot: Lexer.LexerSnapshot; 34 | } 35 | 36 | export interface LexTaskError extends ILexTask { 37 | readonly resultKind: ResultKind.Error; 38 | readonly error: Lexer.LexError.TLexError; 39 | } 40 | 41 | export interface IParseTask extends ITask { 42 | readonly stage: TaskStage.Parse; 43 | readonly lexerSnapshot: Lexer.LexerSnapshot; 44 | } 45 | 46 | export interface ParseTaskOk extends IParseTask { 47 | readonly resultKind: ResultKind.Ok; 48 | readonly ast: Ast.TNode; 49 | readonly parseState: ParseState; 50 | // Indirection to parseState.contextState.nodeIdMapCollection 51 | readonly nodeIdMapCollection: NodeIdMap.Collection; 52 | } 53 | 54 | export interface IParseTaskError extends IParseTask { 55 | readonly resultKind: ResultKind.Error; 56 | readonly error: T; 57 | readonly isCommonError: boolean; 58 | } 59 | 60 | export interface ParseTaskCommonError extends IParseTaskError { 61 | readonly resultKind: ResultKind.Error; 62 | readonly isCommonError: true; 63 | } 64 | 65 | export interface ParseTaskParseError extends IParseTaskError { 66 | readonly resultKind: ResultKind.Error; 67 | readonly isCommonError: false; 68 | readonly parseState: ParseState; 69 | // Indirection to parseState.contextState.nodeIdMapCollection 70 | readonly nodeIdMapCollection: NodeIdMap.Collection; 71 | } 72 | -------------------------------------------------------------------------------- /src/test/helperUtils.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | export function zFill(currentValue: number, upperBound: number): string { 5 | return currentValue.toString().padStart(Math.ceil(Math.log10(upperBound + 1)), "0"); 6 | } 7 | -------------------------------------------------------------------------------- /src/test/index.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | export * from "./testUtils"; 5 | -------------------------------------------------------------------------------- /src/test/libraryTest/common/cancellationToken.test.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import "mocha"; 5 | 6 | import { 7 | CommonError, 8 | DefaultSettings, 9 | Lexer, 10 | Result, 11 | ResultUtils, 12 | Settings, 13 | TimedCancellationToken, 14 | TypeScriptUtils, 15 | } from "../../.."; 16 | 17 | function assertGetCancellationError(tried: Result): CommonError.CancellationError { 18 | ResultUtils.assertIsError(tried); 19 | 20 | if (!CommonError.isCommonError(tried.error)) { 21 | throw new Error(`expected error to be a ${CommonError.CommonError.name}`); 22 | } 23 | 24 | const innerError: Lexer.LexError.TInnerLexError = tried.error.innerError; 25 | 26 | if (!(innerError instanceof CommonError.CancellationError)) { 27 | throw new Error(`expected innerError to be a ${CommonError.CancellationError.name}`); 28 | } 29 | 30 | return innerError; 31 | } 32 | 33 | function assertGetLexerStateWithCancellationToken(): Lexer.State { 34 | const triedLex: Lexer.TriedLex = Lexer.tryLex(DefaultSettings, "foo"); 35 | ResultUtils.assertIsOk(triedLex); 36 | const state: TypeScriptUtils.StripReadonly = triedLex.value; 37 | state.cancellationToken = new TimedCancellationToken(0); 38 | 39 | return state; 40 | } 41 | 42 | function defaultSettingsWithExpiredCancellationToken(): Settings { 43 | return { 44 | ...DefaultSettings, 45 | cancellationToken: new TimedCancellationToken(0), 46 | }; 47 | } 48 | 49 | describe("CancellationToken", () => { 50 | describe(`lexer`, () => { 51 | it(`Lexer.tryLex`, () => { 52 | const triedLex: Lexer.TriedLex = Lexer.tryLex(defaultSettingsWithExpiredCancellationToken(), "foo"); 53 | assertGetCancellationError(triedLex); 54 | }); 55 | 56 | it(`Lexer.tryAppendLine`, () => { 57 | const triedLex: Lexer.TriedLex = Lexer.tryAppendLine( 58 | assertGetLexerStateWithCancellationToken(), 59 | "bar", 60 | "\n", 61 | ); 62 | 63 | assertGetCancellationError(triedLex); 64 | }); 65 | 66 | it(`Lexer.tryDeleteLine`, () => { 67 | const triedLex: Lexer.TriedLex = Lexer.tryDeleteLine(assertGetLexerStateWithCancellationToken(), 0); 68 | assertGetCancellationError(triedLex); 69 | }); 70 | 71 | it(`Lexer.tryUpdateLine`, () => { 72 | const triedLex: Lexer.TriedLex = Lexer.tryUpdateLine(assertGetLexerStateWithCancellationToken(), 0, ""); 73 | assertGetCancellationError(triedLex); 74 | }); 75 | 76 | it(`Lexer.tryUpdateRange`, () => { 77 | const triedLex: Lexer.TriedLex = Lexer.tryUpdateRange( 78 | assertGetLexerStateWithCancellationToken(), 79 | { 80 | start: { 81 | lineCodeUnit: 0, 82 | lineNumber: 0, 83 | }, 84 | end: { 85 | lineNumber: 0, 86 | lineCodeUnit: 1, 87 | }, 88 | }, 89 | "", 90 | ); 91 | 92 | assertGetCancellationError(triedLex); 93 | }); 94 | }); 95 | }); 96 | -------------------------------------------------------------------------------- /src/test/libraryTest/common/stringUtils.test.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import "mocha"; 5 | import { expect } from "chai"; 6 | 7 | import { StringUtils } from "../../.."; 8 | 9 | describe("StringUtils", () => { 10 | describe(`ensureQuoted`, () => { 11 | it(``, () => expect(StringUtils.ensureQuoted(``)).to.equal(`""`)); 12 | it(`a`, () => expect(StringUtils.ensureQuoted(`a`)).to.equal(`"a"`)); 13 | it(`"`, () => expect(StringUtils.ensureQuoted(`"`)).to.equal(`""""`)); 14 | it(`""`, () => expect(StringUtils.ensureQuoted(`""`)).to.equal(`""`)); 15 | it(`"a"`, () => expect(StringUtils.ensureQuoted(`"a"`)).to.equal(`"a"`)); 16 | it(`a"b"c`, () => expect(StringUtils.ensureQuoted(`a"b"c`)).to.equal(`"a""b""c"`)); 17 | }); 18 | 19 | describe(`findQuote`, () => { 20 | it(`""`, () => { 21 | const actual: StringUtils.FoundQuotes | undefined = StringUtils.findQuotes(`""`, 0); 22 | 23 | const expected: StringUtils.FoundQuotes = { 24 | indexStart: 0, 25 | indexEnd: 2, 26 | quoteLength: 2, 27 | }; 28 | 29 | expect(actual).to.deep.equal(expected); 30 | }); 31 | 32 | it(`""""`, () => { 33 | const actual: StringUtils.FoundQuotes | undefined = StringUtils.findQuotes(`""""`, 0); 34 | 35 | const expected: StringUtils.FoundQuotes = { 36 | indexStart: 0, 37 | indexEnd: 4, 38 | quoteLength: 4, 39 | }; 40 | 41 | expect(actual).to.deep.equal(expected); 42 | }); 43 | 44 | it(`"""a"""`, () => { 45 | const actual: StringUtils.FoundQuotes | undefined = StringUtils.findQuotes(`"""a"""`, 0); 46 | 47 | const expected: StringUtils.FoundQuotes = { 48 | indexStart: 0, 49 | indexEnd: 7, 50 | quoteLength: 7, 51 | }; 52 | 53 | expect(actual).to.deep.equal(expected); 54 | }); 55 | 56 | it(`"""abc"""`, () => { 57 | const actual: StringUtils.FoundQuotes | undefined = StringUtils.findQuotes(`"""abc"""`, 0); 58 | 59 | const expected: StringUtils.FoundQuotes = { 60 | indexStart: 0, 61 | indexEnd: 9, 62 | quoteLength: 9, 63 | }; 64 | 65 | expect(actual).to.deep.equal(expected); 66 | }); 67 | 68 | it(`"`, () => expect(StringUtils.findQuotes(`"`, 0)).to.be.undefined); 69 | it(`"abc`, () => expect(StringUtils.findQuotes(`"abc`, 0)).to.be.undefined); 70 | 71 | it(`_""`, () => { 72 | const actual: StringUtils.FoundQuotes | undefined = StringUtils.findQuotes(`_""`, 1); 73 | 74 | const expected: StringUtils.FoundQuotes = { 75 | indexStart: 1, 76 | indexEnd: 3, 77 | quoteLength: 2, 78 | }; 79 | 80 | expect(actual).to.deep.equal(expected); 81 | }); 82 | 83 | it(`_"a"`, () => { 84 | const actual: StringUtils.FoundQuotes | undefined = StringUtils.findQuotes(`_"a"`, 1); 85 | 86 | const expected: StringUtils.FoundQuotes = { 87 | indexStart: 1, 88 | indexEnd: 4, 89 | quoteLength: 3, 90 | }; 91 | 92 | expect(actual).to.deep.equal(expected); 93 | }); 94 | }); 95 | 96 | describe(`normalizeNumber`, () => { 97 | // tslint:disable-next-line: chai-vague-errors 98 | it(`foo`, () => expect(StringUtils.normalizeNumber(`foo`)).to.be.undefined); 99 | it(`1`, () => expect(StringUtils.normalizeNumber(`1`)).to.equal("1")); 100 | it(`-1`, () => expect(StringUtils.normalizeNumber(`-1`)).to.equal("-1")); 101 | it(`--1`, () => expect(StringUtils.normalizeNumber(`--1`)).to.equal("1")); 102 | it(`+1`, () => expect(StringUtils.normalizeNumber(`+1`)).to.equal("1")); 103 | it(`-+1`, () => expect(StringUtils.normalizeNumber(`-+1`)).to.equal("-1")); 104 | it(`+-1`, () => expect(StringUtils.normalizeNumber(`+-1`)).to.equal("-1")); 105 | it(`--1E1`, () => expect(StringUtils.normalizeNumber(`--1E1`)).to.equal("1E1")); 106 | it(`0x1`, () => expect(StringUtils.normalizeNumber(`0x1`)).to.equal("0x1")); 107 | it(`-0x1`, () => expect(StringUtils.normalizeNumber(`-0x1`)).to.equal("-0x1")); 108 | it(`0X1`, () => expect(StringUtils.normalizeNumber(`0X1`)).to.equal("0x1")); 109 | it(`-0X1`, () => expect(StringUtils.normalizeNumber(`-0X1`)).to.equal("-0x1")); 110 | }); 111 | }); 112 | -------------------------------------------------------------------------------- /src/test/libraryTest/identifierUtils.test.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import "mocha"; 5 | import { expect } from "chai"; 6 | 7 | import { IdentifierUtils } from "../../powerquery-parser/language"; 8 | 9 | describe("IdentifierUtils", () => { 10 | describe(`isRegularIdentifier`, () => { 11 | describe(`valid`, () => { 12 | it(`foo`, () => expect(IdentifierUtils.isRegularIdentifier("foo", false), "should be true").to.be.true); 13 | it(`foo`, () => expect(IdentifierUtils.isRegularIdentifier("foo", true), "should be true").to.be.true); 14 | it(`foo.`, () => expect(IdentifierUtils.isRegularIdentifier("foo.", true), "should be true").to.be.true); 15 | it(`foo.1`, () => expect(IdentifierUtils.isRegularIdentifier("foo.1", true), "should be true").to.be.true); 16 | 17 | it(`foo.bar123`, () => 18 | expect(IdentifierUtils.isRegularIdentifier("foo.bar123", true), "should be true").to.be.true); 19 | }); 20 | 21 | describe(`invalid`, () => { 22 | it(`foo.`, () => expect(IdentifierUtils.isRegularIdentifier("foo.", false), "should be false").to.be.false); 23 | }); 24 | }); 25 | 26 | describe(`isGeneralizedIdentifier`, () => { 27 | describe(`valid`, () => { 28 | it("a", () => expect(IdentifierUtils.isGeneralizedIdentifier("a"), "should be true").to.be.true); 29 | it("a.1", () => expect(IdentifierUtils.isGeneralizedIdentifier("a.1"), "should be true").to.be.true); 30 | it("a b", () => expect(IdentifierUtils.isGeneralizedIdentifier("a b"), "should be true").to.be.true); 31 | }); 32 | 33 | describe(`invalid`, () => { 34 | it("a..1", () => expect(IdentifierUtils.isGeneralizedIdentifier("a..1"), "should be false").to.be.false); 35 | }); 36 | }); 37 | 38 | describe(`isQuotedIdentifier`, () => { 39 | describe(`valid`, () => { 40 | it(`#"foo"`, () => expect(IdentifierUtils.isQuotedIdentifier(`#"foo"`), "should be true").to.be.true); 41 | it(`#""`, () => expect(IdentifierUtils.isQuotedIdentifier(`#""`), "should be true").to.be.true); 42 | it(`#""""`, () => expect(IdentifierUtils.isQuotedIdentifier(`#""""`), "should be true").to.be.true); 43 | 44 | it(`#"a""b""c"`, () => 45 | expect(IdentifierUtils.isQuotedIdentifier(`#"a""b""c"`), "should be true").to.be.true); 46 | 47 | it(`#"""b""c"`, () => expect(IdentifierUtils.isQuotedIdentifier(`#"""b""c"`), "should be true").to.be.true); 48 | it(`#"a""b"""`, () => expect(IdentifierUtils.isQuotedIdentifier(`#"a""b"""`), "should be true").to.be.true); 49 | it(`#"bar.1"`, () => expect(IdentifierUtils.isQuotedIdentifier(`#"foo"`), "should be true").to.be.true); 50 | }); 51 | 52 | describe(`invalid`, () => { 53 | it(`#"`, () => expect(IdentifierUtils.isGeneralizedIdentifier(`#"`), "should be false").to.be.false); 54 | it(`""`, () => expect(IdentifierUtils.isGeneralizedIdentifier(`""`), "should be false").to.be.false); 55 | }); 56 | }); 57 | }); 58 | -------------------------------------------------------------------------------- /src/test/libraryTest/language/astUtils.test.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import "mocha"; 5 | import { expect } from "chai"; 6 | 7 | import { Ast, AstUtils } from "../../../powerquery-parser/language"; 8 | import { AssertTestUtils } from "../../testUtils"; 9 | import { DefaultSettings } from "../../../powerquery-parser"; 10 | import { ParseOk } from "../../../powerquery-parser/parser"; 11 | 12 | describe(`AstUtils`, () => { 13 | describe(`getIdentifierLiteral`, () => { 14 | async function runTest(params: { 15 | readonly text: string; 16 | readonly expectedIdentifierLiteral: string; 17 | readonly expectedIdentifierExpressionLiteral: string; 18 | }): Promise { 19 | const parseOk: ParseOk = await AssertTestUtils.assertGetParseOk(DefaultSettings, params.text); 20 | 21 | // First, ensure the root node is an IdentifierExpression. 22 | const rootAsIdentifierExpression: Ast.IdentifierExpression = 23 | AstUtils.assertAsNodeKind(parseOk.root, Ast.NodeKind.IdentifierExpression); 24 | 25 | // Second, grab our actual literals. 26 | const actualIdentifierExpressionLiteral: string = AstUtils.getIdentifierLiteral(rootAsIdentifierExpression); 27 | 28 | const actualIdentifierLiteral: string = AstUtils.getIdentifierLiteral( 29 | rootAsIdentifierExpression.identifier, 30 | ); 31 | 32 | expect( 33 | actualIdentifierExpressionLiteral, 34 | `expected identifier expression literal to be ${params.expectedIdentifierExpressionLiteral}`, 35 | ).to.equal(params.expectedIdentifierExpressionLiteral); 36 | 37 | // Finally, assert that our actual literals match the expected literals. 38 | expect( 39 | actualIdentifierLiteral, 40 | `expected identifier literal to be ${params.expectedIdentifierLiteral}`, 41 | ).to.equal(params.expectedIdentifierLiteral); 42 | } 43 | 44 | it(`foo`, async () => { 45 | await runTest({ 46 | text: "foo", 47 | expectedIdentifierLiteral: "foo", 48 | expectedIdentifierExpressionLiteral: "foo", 49 | }); 50 | }); 51 | 52 | it(`@foo`, async () => { 53 | await runTest({ 54 | text: "@foo", 55 | expectedIdentifierExpressionLiteral: "@foo", 56 | expectedIdentifierLiteral: "foo", 57 | }); 58 | }); 59 | }); 60 | }); 61 | -------------------------------------------------------------------------------- /src/test/libraryTest/parser/customParser.test.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import "mocha"; 5 | 6 | import { DefaultSettings, Parser, Settings, Task, TaskUtils } from "../../.."; 7 | 8 | describe(`custom Parser`, () => { 9 | it(`parserEntryPoint`, async () => { 10 | const customSettings: Settings = { 11 | ...DefaultSettings, 12 | parser: Parser.RecursiveDescentParser, 13 | parserEntryPoint: Parser.RecursiveDescentParser.readParameterSpecificationList, 14 | }; 15 | 16 | const triedLexParseTask: Task.TriedLexParseTask = await TaskUtils.tryLexParse( 17 | customSettings, 18 | "(a as number, optional b as text)", 19 | ); 20 | 21 | TaskUtils.assertIsParseStageOk(triedLexParseTask); 22 | }); 23 | }); 24 | -------------------------------------------------------------------------------- /src/test/libraryTest/parser/identifierContextKind.test.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import "mocha"; 5 | import { expect } from "chai"; 6 | 7 | import { NodeIdMap, NodeIdMapUtils, ParseOk } from "../../../powerquery-parser/parser"; 8 | import { AssertTestUtils } from "../../testUtils"; 9 | import { Ast } from "../../../powerquery-parser/language"; 10 | import { DefaultSettings } from "../../.."; 11 | 12 | function assertGetIdentifierByLiteral(parseOk: ParseOk, identifierLiteral: string): Ast.Identifier { 13 | const nodeIdMapCollection: NodeIdMap.Collection = parseOk.state.contextState.nodeIdMapCollection; 14 | const matches: Ast.Identifier[] = []; 15 | 16 | for (const identifierId of nodeIdMapCollection.idsByNodeKind.get(Ast.NodeKind.Identifier) ?? []) { 17 | const identifier: Ast.Identifier = NodeIdMapUtils.assertAstChecked( 18 | nodeIdMapCollection, 19 | identifierId, 20 | Ast.NodeKind.Identifier, 21 | ); 22 | 23 | if (identifier.literal === identifierLiteral) { 24 | matches.push(identifier); 25 | } 26 | } 27 | 28 | if (matches.length === 0) { 29 | throw new Error(`could not find the following identifier in the ast: ${identifierLiteral}`); 30 | } else if (matches.length === 1) { 31 | return matches[0]; 32 | } else { 33 | throw new Error(`found multiple instances of the following identifier: ${identifierLiteral}`); 34 | } 35 | } 36 | 37 | function assertIdentifierIsInContext( 38 | parseOk: ParseOk, 39 | identifierLiteral: string, 40 | identifierContextKind: Ast.IdentifierContextKind, 41 | ): void { 42 | const identifier: Ast.Identifier = assertGetIdentifierByLiteral(parseOk, identifierLiteral); 43 | expect(identifier.identifierContextKind).to.equal(identifierContextKind); 44 | } 45 | 46 | describe("Parser.IdentifierContextKind", () => { 47 | it("let foo = 1 in bar", async () => { 48 | const parseOk: ParseOk = await AssertTestUtils.assertGetParseOk(DefaultSettings, "let foo = 1 in bar"); 49 | 50 | assertIdentifierIsInContext(parseOk, "foo", Ast.IdentifierContextKind.Key); 51 | assertIdentifierIsInContext(parseOk, "bar", Ast.IdentifierContextKind.Value); 52 | }); 53 | 54 | it("let fn = (foo as number) => 1 in 1", async () => { 55 | const parseOk: ParseOk = await AssertTestUtils.assertGetParseOk( 56 | DefaultSettings, 57 | "let fn = (foo as number) => 1 in 1", 58 | ); 59 | 60 | assertIdentifierIsInContext(parseOk, "fn", Ast.IdentifierContextKind.Key); 61 | assertIdentifierIsInContext(parseOk, "foo", Ast.IdentifierContextKind.Parameter); 62 | }); 63 | 64 | it("let foo = #date in 1", async () => { 65 | const parseOk: ParseOk = await AssertTestUtils.assertGetParseOk(DefaultSettings, "let foo = #date in 1"); 66 | 67 | assertIdentifierIsInContext(parseOk, "foo", Ast.IdentifierContextKind.Key); 68 | assertIdentifierIsInContext(parseOk, "#date", Ast.IdentifierContextKind.Keyword); 69 | }); 70 | }); 71 | -------------------------------------------------------------------------------- /src/test/libraryTest/parser/parseChildren.test.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import "mocha"; 5 | import { expect } from "chai"; 6 | 7 | import { Assert, DefaultSettings, Language, Parser, Task } from "../../.."; 8 | import { AssertTestUtils } from "../../testUtils"; 9 | 10 | interface ChildIdsByIdEntry { 11 | readonly childNodeIds: ReadonlyArray; 12 | readonly id: number; 13 | readonly kind: Language.Ast.NodeKind; 14 | } 15 | 16 | function createActual(lexParseOk: Task.ParseTaskOk): ChildIdsByIdEntry[] { 17 | const actual: ChildIdsByIdEntry[] = []; 18 | const astNodeById: Parser.NodeIdMap.AstNodeById = lexParseOk.nodeIdMapCollection.astNodeById; 19 | 20 | for (const [key, value] of lexParseOk.nodeIdMapCollection.childIdsById.entries()) { 21 | actual.push({ 22 | childNodeIds: value, 23 | id: key, 24 | kind: Assert.asDefined(astNodeById.get(key)).kind, 25 | }); 26 | } 27 | 28 | return actual; 29 | } 30 | 31 | describe("Parser.Children", () => { 32 | it(`() as number => 1`, async () => { 33 | const text: string = `() as number => 1`; 34 | 35 | const expected: ReadonlyArray = [ 36 | { 37 | childNodeIds: [2, 6, 9, 11], 38 | id: 1, 39 | kind: Language.Ast.NodeKind.FunctionExpression, 40 | }, 41 | { 42 | childNodeIds: [3, 4, 5], 43 | id: 2, 44 | kind: Language.Ast.NodeKind.ParameterList, 45 | }, 46 | { 47 | childNodeIds: [7, 8], 48 | id: 6, 49 | kind: Language.Ast.NodeKind.AsNullablePrimitiveType, 50 | }, 51 | ]; 52 | 53 | const actual: ReadonlyArray = createActual( 54 | await AssertTestUtils.assertGetLexParseOk(DefaultSettings, text), 55 | ); 56 | 57 | expect(actual).to.deep.equal(expected); 58 | }); 59 | 60 | it(`null ?? 1 ?? 2`, async () => { 61 | const text: string = `null ?? 1 ?? 2`; 62 | 63 | const expected: ReadonlyArray = [ 64 | { 65 | childNodeIds: [6, 8, 10], 66 | id: 11, 67 | kind: Language.Ast.NodeKind.NullCoalescingExpression, 68 | }, 69 | { 70 | childNodeIds: [2, 4, 11], 71 | id: 12, 72 | kind: Language.Ast.NodeKind.NullCoalescingExpression, 73 | }, 74 | ]; 75 | 76 | const actual: ReadonlyArray = createActual( 77 | await AssertTestUtils.assertGetLexParseOk(DefaultSettings, text), 78 | ); 79 | 80 | expect(actual).to.deep.equal(expected); 81 | }); 82 | }); 83 | -------------------------------------------------------------------------------- /src/test/libraryTest/parser/parseColumnNumber.test.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import "mocha"; 5 | import { expect } from "chai"; 6 | 7 | import { Assert, DefaultSettings } from "../../.."; 8 | import { AssertTestUtils } from "../../testUtils"; 9 | import { ParseError } from "../../../powerquery-parser/parser"; 10 | 11 | async function assertGetExpectedTokenKindError(text: string): Promise { 12 | const error: ParseError.ParseError = await AssertTestUtils.assertGetParseError(DefaultSettings, text); 13 | const innerError: ParseError.TInnerParseError = error.innerError; 14 | 15 | Assert.isTrue( 16 | innerError instanceof ParseError.ExpectedTokenKindError, 17 | "innerError instanceof ParseError.ExpectedTokenKindError", 18 | ); 19 | 20 | return innerError as ParseError.ExpectedTokenKindError; 21 | } 22 | 23 | async function assertErrorAt(text: string, lineNumber: number, columnNumber: number, codeUnit: number): Promise { 24 | const error: ParseError.ExpectedTokenKindError = await assertGetExpectedTokenKindError(text); 25 | const foundToken: ParseError.TokenWithColumnNumber = Assert.asDefined(error.foundToken); 26 | 27 | expect(foundToken.token.positionStart.codeUnit).to.equal(codeUnit, "codeUnit"); 28 | expect(foundToken.token.positionStart.lineNumber).to.equal(lineNumber, "lineNumber"); 29 | expect(foundToken.columnNumber).to.equal(columnNumber, "columnNumber"); 30 | } 31 | 32 | describe(`Parser.ColumnNumber`, () => { 33 | it(`if x foo`, async () => { 34 | await assertErrorAt(`if x foo`, 0, 5, 5); 35 | }); 36 | 37 | it(`if x \\nfoo`, async () => { 38 | await assertErrorAt(`if x \nfoo`, 1, 0, 6); 39 | }); 40 | 41 | it(`if x \\n foo`, async () => { 42 | await assertErrorAt(`if x \n foo`, 1, 1, 7); 43 | }); 44 | 45 | it(`if \u006E\u0303 foo`, async () => { 46 | await assertErrorAt(`if \u006E\u0303 foo`, 0, 5, 6); 47 | }); 48 | 49 | it(`if \u006E\u0303 \\nfoo`, async () => { 50 | await assertErrorAt(`if \u006E\u0303 \nfoo`, 1, 0, 7); 51 | }); 52 | 53 | it(`if \u006E\u0303 \\n foo`, async () => { 54 | await assertErrorAt(`if \u006E\u0303 \n foo`, 1, 1, 8); 55 | }); 56 | }); 57 | -------------------------------------------------------------------------------- /src/test/libraryTest/textUtils.test.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import "mocha"; 5 | import { expect } from "chai"; 6 | 7 | import { TextUtils } from "../../powerquery-parser/language"; 8 | 9 | describe("TextUtils", () => { 10 | it(`escape`, () => { 11 | const unescaped: string = 'Encode \t\t and \r\n and "quotes" but not this #(tab)'; 12 | const escaped: string = 'Encode #(tab)#(tab) and #(cr,lf) and ""quotes"" but not this #(#)(tab)'; 13 | 14 | expect(TextUtils.escape(unescaped)).to.equal(escaped); 15 | expect(TextUtils.unescape(escaped)).to.equal(unescaped); 16 | }); 17 | }); 18 | -------------------------------------------------------------------------------- /src/test/libraryTest/tokenizer/tokenizerSimple.test.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import "mocha"; 5 | import { expect } from "chai"; 6 | 7 | import { ILineTokens, IState, IToken, Tokenizer, TokenizerState } from "../../testUtils/tokenizerTestUtils"; 8 | 9 | const tokenizer: Tokenizer = new Tokenizer(`\n`); 10 | const initialState: TokenizerState = tokenizer.getInitialState() as TokenizerState; 11 | 12 | function tokenizeLines(query: string, expectedTokenCounts: number[]): void { 13 | const lines: ReadonlyArray = query.split(`\n`); 14 | let state: TokenizerState = initialState; 15 | 16 | expect(lines.length).equals(expectedTokenCounts.length); 17 | 18 | for (let index: number = 0; index < lines.length; index += 1) { 19 | const r: ILineTokens = tokenizer.tokenize(lines[index], state); 20 | expect(!state.equals(r.endState), `state should have changed.`); 21 | expect(r.tokens.length).equals(expectedTokenCounts[index], `unexpected token count`); 22 | 23 | state = r.endState as TokenizerState; 24 | 25 | r.tokens.forEach((token: IToken) => { 26 | expect(token.startIndex).is.lessThan(lines[index].length); 27 | }); 28 | } 29 | } 30 | 31 | describe(`Tokenizer`, () => { 32 | it(`Initial state`, () => { 33 | expect(tokenizer.getInitialState(), `initial state should not be null`); 34 | expect(initialState, `initial state variable should not be null`); 35 | }); 36 | 37 | it(`Initial state equality`, () => { 38 | expect(initialState.equals(initialState)); 39 | }); 40 | 41 | it(`Cloned state equality`, () => { 42 | const r: ILineTokens = tokenizer.tokenize(`let a = 1 in a`, initialState); 43 | 44 | const clonedState: IState = r.endState.clone(); 45 | expect(r.endState.equals(clonedState), `states should be equal`); 46 | }); 47 | 48 | it(`Single line token count`, () => { 49 | tokenizeLines(`1 = 1`, [3]); 50 | }); 51 | 52 | it(`Simple multiline query`, () => { 53 | tokenizeLines(`let\na = 1\nin a`, [1, 3, 2]); 54 | }); 55 | 56 | it(`Query with comments`, () => { 57 | tokenizeLines(`/* block */ let\na = 1 // line comment\nin a`, [2, 4, 2]); 58 | }); 59 | 60 | it(`Multiline comment block`, () => { 61 | tokenizeLines(`1 + /* comment\nend*/ 2`, [3, 2]); 62 | }); 63 | 64 | it(`Multiline string`, () => { 65 | tokenizeLines(`"hello\nthere\n\n" & "append"`, [1, 1, 0, 3]); 66 | }); 67 | 68 | it(`Multiline quoted identifier`, () => { 69 | tokenizeLines(`#"hello\nthere\n\n" & #"append"`, [1, 1, 0, 3]); 70 | }); 71 | }); 72 | -------------------------------------------------------------------------------- /src/test/mochaConfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "reporterEnabled": "spec, mocha-junit-reporter", 3 | "mochaJunitReporterReporterOptions": { 4 | "mochaFile": "test-results.xml" 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /src/test/resourceTest/lexParseResource.test.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import "mocha"; 5 | import { expect } from "chai"; 6 | 7 | import { ArrayUtils, TaskUtils } from "../../powerquery-parser"; 8 | import { DefaultSettings, Parser, Settings } from "../.."; 9 | import { NodeIdMap, NodeIdMapUtils } from "../../powerquery-parser/parser"; 10 | import { ResourceTestUtils, TestConstants } from "../testUtils"; 11 | 12 | function createSettings(parser: Parser.Parser): Settings { 13 | return { 14 | ...DefaultSettings, 15 | parser, 16 | }; 17 | } 18 | 19 | for (const [parserName, parser] of TestConstants.ParserByParserName.entries()) { 20 | const settings: Settings = createSettings(parser); 21 | 22 | ResourceTestUtils.runResourceTestSuite( 23 | settings, 24 | `Attempt lex and parse resources (${parserName})`, 25 | (filePath: string) => { 26 | const filePathSlice: string = ArrayUtils.assertGet(filePath.split("microsoft-DataConnectors\\"), 1); 27 | 28 | return `${filePathSlice}`; 29 | }, 30 | (testRun: ResourceTestUtils.ResourceTestRun) => { 31 | TaskUtils.assertIsParseStageOk(testRun.triedLexParse); 32 | 33 | const validation: NodeIdMap.CollectionValidation = NodeIdMapUtils.validate( 34 | testRun.triedLexParse.nodeIdMapCollection, 35 | ); 36 | 37 | expect({ 38 | badParentChildLink: validation.badParentChildLink, 39 | duplicateIds: validation.duplicateIds, 40 | unknownByNodeKindNodeIds: validation.unknownByNodeKindNodeIds, 41 | unknownByNodeKindNodeKinds: validation.unknownByNodeKindNodeKinds, 42 | unknownChildIdsKeys: validation.unknownChildIdsKeys, 43 | unknownChildIdsValues: validation.unknownChildIdsValues, 44 | unknownLeafIds: validation.unknownLeafIds, 45 | unknownParentIdKeys: validation.unknownParentIdKeys, 46 | unknownParentIdValues: validation.unknownParentIdValues, 47 | }).to.deep.equal({ 48 | badParentChildLink: [], 49 | duplicateIds: [], 50 | unknownByNodeKindNodeIds: [], 51 | unknownByNodeKindNodeKinds: [], 52 | unknownChildIdsKeys: [], 53 | unknownChildIdsValues: [], 54 | unknownLeafIds: [], 55 | unknownParentIdKeys: [], 56 | unknownParentIdValues: [], 57 | }); 58 | }, 59 | ); 60 | } 61 | -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/DataWorldSwagger/DataWorldSwagger.pq: -------------------------------------------------------------------------------- 1 | // This file contains your Data Connector logic 2 | section DataWorldSwagger; 3 | 4 | // TODO: add your client id and secret to the embedded files 5 | client_id = Text.FromBinary(Extension.Contents("client_id")); 6 | client_secret = Text.FromBinary(Extension.Contents("client_secret")); 7 | 8 | redirect_uri = "https://oauth.powerbi.com/views/oauthredirect.html"; 9 | windowWidth = 800; 10 | windowHeight = 800; 11 | 12 | BaseUrl = "https://api.data.world/v0"; 13 | OAuthBaseUrl = "https://data.world/oauth"; 14 | 15 | [DataSource.Kind="DataWorldSwagger", Publish="DataWorldSwagger.Publish"] 16 | shared DataWorldSwagger.Contents = () => 17 | let 18 | credential = Extension.CurrentCredential(), 19 | token = if (credential[AuthenticationKind] = "Key") then credential[Key] else credential[access_token], 20 | headers = [ Authorization = "Bearer " & token ], 21 | navTable = OpenApi.Document(Web.Contents("https://api.data.world/v0/swagger.json"), [ Headers = headers, ManualCredentials = true ]) 22 | in 23 | navTable; 24 | 25 | // Data Source Kind description 26 | DataWorldSwagger = [ 27 | // enable both OAuth and Key based auth 28 | Authentication = [ 29 | OAuth = [ 30 | StartLogin = StartLogin, 31 | FinishLogin = FinishLogin, 32 | Refresh=Refresh 33 | ], 34 | Key = [ 35 | ] 36 | ], 37 | Label = Extension.LoadString("DataSourceLabel") 38 | ]; 39 | 40 | // Data Source UI publishing description 41 | DataWorldSwagger.Publish = [ 42 | Beta = true, 43 | Category = "Other", 44 | ButtonText = { Extension.LoadString("ButtonTitle"), Extension.LoadString("ButtonHelp") }, 45 | LearnMoreUrl = "https://data.world/" 46 | ]; 47 | 48 | // 49 | // OAuth2 flow definition 50 | // 51 | 52 | StartLogin = (resourceUrl, state, display) => 53 | let 54 | AuthorizeUrl = OAuthBaseUrl & "/authorize?" & Uri.BuildQueryString([ 55 | client_id = client_id, 56 | response_type = "code", 57 | state = state, 58 | redirect_uri = redirect_uri]) 59 | in 60 | [ 61 | LoginUri = AuthorizeUrl, 62 | CallbackUri = redirect_uri, 63 | WindowHeight = windowHeight, 64 | WindowWidth = windowWidth, 65 | Context = null 66 | ]; 67 | 68 | FinishLogin = (context, callbackUri, state) => 69 | let 70 | Parts = Uri.Parts(callbackUri)[Query] 71 | in 72 | TokenMethod(Parts[code], "authorization_code"); 73 | 74 | TokenMethod = (code, grant_type) => 75 | let 76 | Response = Web.Contents(OAuthBaseUrl & "/access_token", [ 77 | Content = Text.ToBinary(Uri.BuildQueryString([ 78 | client_id = client_id, 79 | client_secret = client_secret, 80 | code = code, 81 | grant_type = grant_type, 82 | redirect_uri = redirect_uri])), 83 | Headers=[#"Content-type" = "application/x-www-form-urlencoded",#"Accept" = "application/json"]]), 84 | Parts = Json.Document(Response) 85 | in 86 | Parts; 87 | 88 | Refresh = (resourceUrl, refresh_token) => TokenMethod(refresh_token, "refresh_token"); 89 | -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/DataWorldSwagger/DataWorldSwagger.query.pq: -------------------------------------------------------------------------------- 1 | // Use this file to write queries to test your data connector 2 | let 3 | result = DataWorldSwagger.Contents() 4 | in 5 | result -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/DirectQueryForSQL/DirectQueryForSQL.query.pq: -------------------------------------------------------------------------------- 1 | //Use this file to write queries. 2 | let 3 | Host = "localhost", 4 | Database = "AdventureWorksDW2012", 5 | Source = DirectSQL.Database(Host, Database), 6 | dbo_Schema = Source{[Name="dbo",Kind="Schema"]}[Data] 7 | in 8 | dbo_Schema -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/Github/github.query.pq: -------------------------------------------------------------------------------- 1 | // This is where you can write sample queries to test your extension. 2 | let 3 | source = GithubSample.Contents("https://api.github.com/") 4 | in 5 | source -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/HelloWorld/HelloWorld.pq: -------------------------------------------------------------------------------- 1 | section HelloWorld; 2 | 3 | [DataSource.Kind="HelloWorld", Publish="HelloWorld.Publish"] 4 | shared HelloWorld.Contents = (optional message as text) => 5 | let 6 | message = if (message <> null) then message else "Hello world" 7 | in 8 | message; 9 | 10 | HelloWorld = [ 11 | TestConnection = (dataSourcePath) => {"HelloWorld.Contents"}, 12 | Authentication = [ 13 | Anonymous = [] 14 | ], 15 | Label = Extension.LoadString("DataSourceLabel") 16 | ]; 17 | 18 | HelloWorld.Publish = [ 19 | Beta = true, 20 | ButtonText = { Extension.LoadString("FormulaTitle"), Extension.LoadString("FormulaHelp") }, 21 | SourceImage = HelloWorld.Icons, 22 | SourceTypeImage = HelloWorld.Icons 23 | ]; 24 | 25 | HelloWorld.Icons = [ 26 | Icon16 = { Extension.Contents("HelloWorld16.png"), Extension.Contents("HelloWorld20.png"), Extension.Contents("HelloWorld24.png"), Extension.Contents("HelloWorld32.png") }, 27 | Icon32 = { Extension.Contents("HelloWorld32.png"), Extension.Contents("HelloWorld40.png"), Extension.Contents("HelloWorld48.png"), Extension.Contents("HelloWorld64.png") } 28 | ]; 29 | -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/HelloWorld/HelloWorld.query.pq: -------------------------------------------------------------------------------- 1 | //Use this file to write queries. 2 | HelloWorld.Contents("hello this is my message") -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/HelloWorldWithDocs/HelloWorldWithDocs.pq: -------------------------------------------------------------------------------- 1 | section HelloWorldWithDocs; 2 | 3 | [DataSource.Kind="HelloWorldWithDocs", Publish="HelloWorldWithDocs.Publish"] 4 | shared HelloWorldWithDocs.Contents = Value.ReplaceType(HelloWorldImpl, HelloWorldType); 5 | 6 | HelloWorldType = type function ( 7 | message as (type text meta [ 8 | Documentation.FieldCaption = "Message", 9 | Documentation.FieldDescription = "Text to display", 10 | Documentation.SampleValues = {"Hello world", "Hola mundo"} 11 | ]), 12 | optional count as (type number meta [ 13 | Documentation.FieldCaption = "Count", 14 | Documentation.FieldDescription = "Number of times to repeat the message", 15 | Documentation.AllowedValues = { 1, 2, 3 } 16 | ])) 17 | as table meta [ 18 | Documentation.Name = "Hello - Name", 19 | Documentation.LongDescription = "Hello - Long Description", 20 | Documentation.Examples = {[ 21 | Description = "Returns a table with 'Hello world' repeated 2 times", 22 | Code = "HelloWorldWithDocs.Contents(""Hello world"", 2)", 23 | Result = "#table({""Column1""}, {{""Hello world""}, {""Hello world""}})" 24 | ],[ 25 | Description = "Another example, new message, new count!", 26 | Code = "HelloWorldWithDocs.Contents(""Goodbye"", 1)", 27 | Result = "#table({""Column1""}, {{""Goodbye""}})" 28 | ]} 29 | ]; 30 | 31 | HelloWorldImpl = (message as text, optional count as number) as table => 32 | let 33 | _count = if (count <> null) then count else 5, 34 | listOfMessages = List.Repeat({message}, _count), 35 | table = Table.FromList(listOfMessages, Splitter.SplitByNothing()) 36 | in 37 | table; 38 | 39 | // Data Source Kind description 40 | HelloWorldWithDocs = [ 41 | Authentication = [ 42 | Anonymous = [] 43 | ]//, 44 | //Label = "Hello World With Docs" 45 | ]; 46 | 47 | // Data Source UI publishing description 48 | HelloWorldWithDocs.Publish = [ 49 | Beta = true, 50 | Category = "Other", 51 | ButtonText = { "Hello World With Docs", "Provides an example of how to provide function documentation" } 52 | ]; 53 | -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/HelloWorldWithDocs/HelloWorldWithDocs.query.pq: -------------------------------------------------------------------------------- 1 | // Use this file to write queries to test your data connector 2 | let 3 | result = HelloWorldWithDocs.Contents("Hello world", 2) 4 | in 5 | result -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/NavigationTable/NavigationTable.query.pq: -------------------------------------------------------------------------------- 1 | // Use this file to write queries to test your data connector 2 | let 3 | result = NavigationTable.Icons() 4 | in 5 | result -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/OAuthPKCE/PKCESample.pq: -------------------------------------------------------------------------------- 1 | // This is not a complete connector sample, but demonstrates a PKCE based OAuth flow 2 | [Version="1.0.0"] 3 | section PKCESample; 4 | 5 | // Native client flow (PKCE) 6 | // see https://tools.ietf.org/html/rfc7636 7 | // see https://oauth.net/2/pkce/ 8 | 9 | OAuthBaseUrl = "https://your-service-url/oauth"; 10 | 11 | StartLogin = (resourceUrl, state, display) => 12 | let 13 | // We'll generate our code verifier using Guids 14 | codeVerifier = Text.NewGuid() & Text.NewGuid(), 15 | AuthorizeUrl = OAuthBaseUrl & "/authorize?" & Uri.BuildQueryString([ 16 | client_id = client_id, 17 | response_type = "code", 18 | code_challenge_method = "plain", 19 | code_challenge = codeVerifier, 20 | state = state, 21 | redirect_uri = redirect_uri]) 22 | in 23 | [ 24 | LoginUri = AuthorizeUrl, 25 | CallbackUri = redirect_uri, 26 | WindowHeight = windowHeight, 27 | WindowWidth = windowWidth, 28 | // Need to roundtrip this value to FinishLogin 29 | Context = codeVerifier 30 | ]; 31 | 32 | // The code verifier will be passed in through the context parameter. 33 | FinishLogin = (context, callbackUri, state) => 34 | let 35 | Parts = Uri.Parts(callbackUri)[Query] 36 | in 37 | TokenMethod(Parts[code], "authorization_code", context); 38 | 39 | // Verifier is optional to support both the original FinishLogin call 40 | // (which has a verifier) and the Refresh call (which does not). 41 | TokenMethod = (code, grant_type, optional verifier) => 42 | let 43 | codeVerifier = if (verifier <> null) then [code_verifier = verifier] else [], 44 | codeParameter = if (grant_type = "authorization_code") then [ code = code ] else [ refresh_token = code ], 45 | query = codeVerifier & codeParameter & [ 46 | client_id = client_id, 47 | // Native client flows should not require a client_secret when using PKCE, but some still do. 48 | // client_secret = client_secret, 49 | grant_type = grant_type, 50 | redirect_uri = redirect_uri 51 | ], 52 | 53 | // Set this if your API returns a non-2xx status for login failures 54 | // ManualHandlingStatusCodes = {400, 403} 55 | ManualHandlingStatusCodes= {}, 56 | 57 | Response = Web.Contents(OAuthBaseUrl & "/access_token", [ 58 | Content = Text.ToBinary(Uri.BuildQueryString(query)), 59 | Headers = [ 60 | #"Content-type" = "application/x-www-form-urlencoded", 61 | #"Accept" = "application/json" 62 | ], 63 | ManualStatusHandling = ManualHandlingStatusCodes 64 | ]), 65 | Parts = Json.Document(Response) 66 | in 67 | // check for error in response 68 | if (Parts[error]? <> null) then 69 | error Error.Record(Parts[error], Parts[message]?) 70 | else 71 | Parts; 72 | 73 | Refresh = (resourceUrl, refresh_token) => TokenMethod(refresh_token, "refresh_token"); 74 | -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/ODBC/HiveSample/HiveSample.query.pq: -------------------------------------------------------------------------------- 1 | // Tested with Hortonworks Sandbox 2 | let 3 | Source = HiveSample.Contents("127.0.0.1", 10500), 4 | HIVE_Database = Source{[Name="HIVE",Kind="Database"]}[Data], 5 | foodmart_Schema = HIVE_Database{[Name="foodmart",Kind="Schema"]}[Data], 6 | customer_Table = foodmart_Schema{[Name="customer",Kind="Table"]}[Data], 7 | #"Kept First Rows" = Table.FirstN(customer_Table,5) 8 | in 9 | #"Kept First Rows" -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/ODBC/ImpalaODBC/ImpalaODBC.query.pq: -------------------------------------------------------------------------------- 1 | // Use this file to write queries to test your data connector 2 | let 3 | result = ImpalaODBC.Databases("localhost") 4 | in 5 | result -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/ODBC/RedshiftODBC/RedshiftODBC.query.pq: -------------------------------------------------------------------------------- 1 | let 2 | result = RedshiftODBC.Database("server.redshift.amazonaws.com:5439", "database") 3 | in 4 | result -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/ODBC/SnowflakeODBC/SnowflakeODBC.query.pq: -------------------------------------------------------------------------------- 1 | // Use this file to write queries to test your data connector 2 | let 3 | result = SnowflakeODBC.Databases("server", "warehouse") 4 | in 5 | result -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/ODBC/SqlODBC/.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "powerquery.sdk.defaultExtension": "${workspaceFolder}\\bin\\AnyCPU\\Debug\\${workspaceFolderBasename}.mez", 3 | "powerquery.sdk.defaultQueryFile": "${workspaceFolder}\\${workspaceFolderBasename}.query.pq" 4 | } -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/ODBC/SqlODBC/SqlODBC.query.pq: -------------------------------------------------------------------------------- 1 | // Use this file to write queries to test your data connector 2 | let 3 | result = SqlODBC.Contents("localhost"), 4 | db = result{[Name="master"]}[Data], 5 | schema = db{[Name="sys"]}[Data], 6 | allViews = schema{[Name="all_views"]}[Data] 7 | in 8 | Table.FirstN(allViews, 5) -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/OData/AnnotationsSample/AnnotationsSample.pq: -------------------------------------------------------------------------------- 1 | section TripPinAnnotations; 2 | 3 | // Data Source Kind description 4 | TripPin = [ 5 | // TestConnection is required to enable the connector through the Gateway 6 | TestConnection = (dataSourcePath) => { "TripPin.Annotations" }, 7 | Authentication = [ 8 | Anonymous = [] 9 | ], 10 | Label = "AnnotationsSample" 11 | ]; 12 | 13 | // Data Source UI publishing description 14 | TripPin.Publish = [ 15 | Beta = true, 16 | Category = "Other", 17 | ButtonText = { "AnnotationsSample", "AnnotationsSample" } 18 | ]; 19 | 20 | [DataSource.Kind="TripPin", Publish="TripPin.Publish"] 21 | shared TripPin.Annotations = () => let 22 | serviceDocument = TripPin.ServiceDocument() 23 | in 24 | #table({ "Location", "Data" }, { 25 | { "Entity Container Annotations", GetEntityContainerAnnotations(serviceDocument) }, 26 | { "Resources", Table.TransformColumns(serviceDocument, { { "Data", (resource) => 27 | if resource is function then #table({ "Location", "Data" }, { 28 | { "Function Import Annotations", GetFunctionImportAnnotations(resource) }, 29 | { "Parameter Annotations", Record.ToTable(GetFunctionParameterAnnotations(Value.Type(resource))) } 30 | }) else #table({ "Location", "Data" }, { 31 | { "Entity Set or Singleton", GetEntitySetOrSingletonAnnotations(resource) }, 32 | { "Entity Type", #table({ "Location", "Data" }, { 33 | { "Entity Type Annotations", GetEntityTypeAnnotations(resource) }, 34 | { "Property Annotations", Record.ToTable(GetEntityTypePropertyAnnotations(resource)) }, 35 | { "Function Annotations", Table.AddColumn( 36 | Table.FromColumns(Table.ColumnsOfType(resource, { Function.Type }), { "Name" }), 37 | "Value", 38 | each GetEntityTypeFunctionAnnotations(resource, [Name])) } 39 | }) } 40 | }) 41 | } }) } 42 | }); 43 | 44 | GetEntityContainerAnnotations = (serviceDocument) => Value.Metadata(Value.Type(serviceDocument)); 45 | 46 | GetEntitySetOrSingletonAnnotations = (entitySetOrSingleton) => Value.Metadata(entitySetOrSingleton); 47 | 48 | GetFunctionImportAnnotations = (functionImport) => Value.Metadata(Value.Type(functionImport)); 49 | 50 | GetEntityTypeAnnotations = (entitySetOrSingleton) => let 51 | entityCollectionType = Value.Type(entitySetOrSingleton), 52 | entityType = Type.TableRow(entityCollectionType), 53 | entityTypeAnnotations = Value.Metadata(entityType)[OData.Annotations] 54 | in 55 | entityTypeAnnotations; 56 | 57 | GetEntityTypePropertyAnnotations = (entitySetOrSingleton) => let 58 | entityCollectionType = Value.Type(entitySetOrSingleton), 59 | entityType = if entitySetOrSingleton is record then entityCollectionType else Type.TableRow(entityCollectionType), 60 | fieldAnnotations = Value.Metadata(entityType)[OData.FieldAnnotations] 61 | in 62 | fieldAnnotations; 63 | 64 | GetEntityTypeFunctionAnnotations = (entitySetOrSingleton, functionName) => let 65 | entityCollectionType = Value.Type(entitySetOrSingleton), 66 | entityType = if entitySetOrSingleton is record then entityCollectionType else Type.TableRow(entityCollectionType), 67 | functionType = Type.TableColumn(entityType, functionName), 68 | functionAnnotations = Value.Metadata(functionType) 69 | in 70 | functionAnnotations; 71 | 72 | GetFunctionParameterAnnotations = (functionType) => let 73 | parameters = Type.FunctionParameters(functionType), 74 | parametersTable = Record.ToTable(parameters), 75 | parameterAnnotations = Table.TransformColumns(parametersTable, { { "Value", Value.Metadata } }) 76 | in 77 | Record.FromTable(parameterAnnotations); 78 | 79 | BaseUrl = "http://services.odata.org/v4/TripPinService/"; 80 | 81 | // Without explicitly setting IncludeAnnotations or IncludeMetadataAnnotations, 82 | // annotations will not be made available 83 | TripPin.ServiceDocument = () => OData.Feed(BaseUrl, null, [ Implementation = "2.0", IncludeAnnotations = "*" ]) as table; -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/OData/AnnotationsSample/AnnotationsSample.query.pq: -------------------------------------------------------------------------------- 1 | // Use this file to write queries to test your data connector 2 | let 3 | result = TripPin.Annotations() 4 | in 5 | result -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/OpenApiSample/OpenApiSample.query.pq: -------------------------------------------------------------------------------- 1 | // Use this file to write queries to test your data connector 2 | let 3 | result = OpenApiSample.ApisGuru() 4 | in 5 | result -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/TripPin/1-OData/TripPin.pq: -------------------------------------------------------------------------------- 1 | section TripPin; 2 | 3 | [DataSource.Kind="TripPin", Publish="TripPin.Publish"] 4 | shared TripPin.Feed = Value.ReplaceType(TripPinImpl, type function (url as Uri.Type) as any); 5 | 6 | TripPinImpl = (url as text) => 7 | let 8 | source = OData.Feed(url) 9 | in 10 | source; 11 | 12 | // Data Source Kind description 13 | TripPin = [ 14 | // Declares the supported type(s) of authentication. 15 | // In this case, Implicit = Anonymous web access 16 | Authentication = [ 17 | Anonymous = [] 18 | ], 19 | // Assigns a label to the data source credential. 20 | // This will be displayed in the "Manage Data Sources" dialog. 21 | Label = "TripPin Part 1 - OData" 22 | ]; 23 | 24 | // Data Source UI publishing description 25 | TripPin.Publish = [ 26 | Beta = true, 27 | Category = "Other", 28 | ButtonText = { "TripPin OData", "TripPin OData" } 29 | ]; 30 | 31 | -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/TripPin/1-OData/TripPin.query.pq: -------------------------------------------------------------------------------- 1 | // Use this file to write queries to test your data connector 2 | let 3 | result = TripPin.Feed("http://services.odata.org/v4/TripPinService/") 4 | in 5 | result -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/TripPin/10-TableView1/Table.GenerateByPage.pqm: -------------------------------------------------------------------------------- 1 | (getNextPage as function) as table => 2 | let 3 | listOfPages = List.Generate( 4 | () => getNextPage(null), // get the first page of data 5 | (lastPage) => lastPage <> null, // stop when the function returns null 6 | (lastPage) => getNextPage(lastPage) // pass the previous page to the next function call 7 | ), 8 | // concatenate the pages together 9 | tableOfPages = Table.FromList(listOfPages, Splitter.SplitByNothing(), {"Column1"}), 10 | firstRow = tableOfPages{0}? 11 | in 12 | // if we didn't get back any pages of data, return an empty table 13 | // otherwise set the table type based on the columns of the first page 14 | if (firstRow = null) then 15 | Table.FromRows({}) 16 | // check for empty first table 17 | else if (Table.IsEmpty(firstRow[Column1])) then 18 | firstRow[Column1] 19 | else 20 | Value.ReplaceType( 21 | Table.ExpandTableColumn(tableOfPages, "Column1", Table.ColumnNames(firstRow[Column1])), 22 | Value.Type(firstRow[Column1]) 23 | ) 24 | -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/TripPin/10-TableView1/Table.ToNavigationTable.pqm: -------------------------------------------------------------------------------- 1 | ( 2 | table as table, 3 | keyColumns as list, 4 | nameColumn as text, 5 | dataColumn as text, 6 | itemKindColumn as text, 7 | itemNameColumn as text, 8 | isLeafColumn as text 9 | ) as table => 10 | let 11 | tableType = Value.Type(table), 12 | newTableType = Type.AddTableKey(tableType, keyColumns, true) meta 13 | [ 14 | NavigationTable.NameColumn = nameColumn, 15 | NavigationTable.DataColumn = dataColumn, 16 | NavigationTable.ItemKindColumn = itemKindColumn, 17 | Preview.DelayColumn = itemNameColumn, 18 | NavigationTable.IsLeafColumn = isLeafColumn 19 | ], 20 | navigationTable = Value.ReplaceType(table, newTableType) 21 | in 22 | navigationTable -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/TripPin/2-Rest/TripPin.pq: -------------------------------------------------------------------------------- 1 | section TripPin; 2 | 3 | [DataSource.Kind="TripPin", Publish="TripPin.Publish"] 4 | shared TripPin.Feed = Value.ReplaceType(TripPinImpl, type function (url as Uri.Type) as any); 5 | 6 | DefaultRequestHeaders = [ 7 | #"Accept" = "application/json;odata.metadata=minimal", // column name and values only 8 | #"OData-MaxVersion" = "4.0" // we only support v4 9 | ]; 10 | 11 | TripPinImpl = (url as text) => 12 | let 13 | source = Web.Contents(url, [ Headers = DefaultRequestHeaders ]), 14 | json = Json.Document(source) 15 | in 16 | json; 17 | 18 | // Data Source Kind description 19 | TripPin = [ 20 | Authentication = [ 21 | Anonymous = [] 22 | ], 23 | Label = "TripPin Part 2 - REST" 24 | ]; 25 | 26 | // Data Source UI publishing description 27 | TripPin.Publish = [ 28 | Beta = true, 29 | Category = "Other", 30 | ButtonText = { "TripPin REST", "TripPin REST" } 31 | ]; 32 | 33 | -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/TripPin/2-Rest/TripPin.query.pq: -------------------------------------------------------------------------------- 1 | // Use this file to write queries to test your data connector 2 | let 3 | result = TripPin.Feed("http://services.odata.org/v4/TripPinService/Me") 4 | in 5 | result -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/TripPin/3-NavTables/TripPin.pq: -------------------------------------------------------------------------------- 1 | section TripPin; 2 | 3 | // 4 | // Definition 5 | // 6 | 7 | // Data Source Kind description 8 | TripPin = [ 9 | Authentication = [ 10 | Anonymous = [] 11 | ], 12 | Label = "TripPin Part 3 - Navigator" 13 | ]; 14 | 15 | // Data Source UI publishing description 16 | TripPin.Publish = [ 17 | Beta = true, 18 | Category = "Other", 19 | ButtonText = { "TripPin Navigator", "TripPin Navigator" } 20 | ]; 21 | 22 | // 23 | // Implementation 24 | // 25 | 26 | [DataSource.Kind="TripPin"] 27 | shared TripPin.Feed = Value.ReplaceType(TripPinImpl, type function (url as Uri.Type) as any); 28 | 29 | [DataSource.Kind="TripPin", Publish="TripPin.Publish"] 30 | shared TripPin.Contents = Value.ReplaceType(TripPinNavTable, type function (url as Uri.Type) as any); 31 | 32 | DefaultRequestHeaders = [ 33 | #"Accept" = "application/json;odata.metadata=minimal", // column name and values only 34 | #"OData-MaxVersion" = "4.0" // we only support v4 35 | ]; 36 | 37 | TripPinImpl = (url as text) => 38 | let 39 | source = Web.Contents(url, [ Headers = DefaultRequestHeaders ]), 40 | json = Json.Document(source) 41 | in 42 | json; 43 | 44 | TripPinNavTable = (url as text) as table => 45 | let 46 | source = #table({"Name", "Data", "ItemKind", "ItemName", "IsLeaf"}, { 47 | { "Airlines", GetAirlinesTable(url), "Table", "Table", true }, 48 | { "Airports", GetAirportsTable(url), "Table", "Table", true } 49 | }), 50 | navTable = Table.ToNavigationTable(source, {"Name"}, "Name", "Data", "ItemKind", "ItemName", "IsLeaf") 51 | in 52 | navTable; 53 | 54 | GetAirlinesTable = (url as text) as table => 55 | let 56 | source = TripPin.Feed(url & "Airlines"), 57 | value = source[value], 58 | toTable = Table.FromList(value, Splitter.SplitByNothing(), null, null, ExtraValues.Error), 59 | expand = Table.ExpandRecordColumn(toTable, "Column1", {"AirlineCode", "Name"}, {"AirlineCode", "Name"}) 60 | in 61 | expand; 62 | 63 | GetAirportsTable = (url as text) as table => 64 | let 65 | source = TripPin.Feed(url & "Airports"), 66 | value = source[value], 67 | #"Converted to Table" = Table.FromList(value, Splitter.SplitByNothing(), null, null, ExtraValues.Error), 68 | #"Expanded Column1" = Table.ExpandRecordColumn(#"Converted to Table", "Column1", {"Name", "IcaoCode", "IataCode", "Location"}, {"Name", "IcaoCode", "IataCode", "Location"}), 69 | #"Expanded Location" = Table.ExpandRecordColumn(#"Expanded Column1", "Location", {"Address", "Loc", "City"}, {"Address", "Loc", "City"}), 70 | #"Expanded City" = Table.ExpandRecordColumn(#"Expanded Location", "City", {"Name", "CountryRegion", "Region"}, {"Name.1", "CountryRegion", "Region"}), 71 | #"Renamed Columns" = Table.RenameColumns(#"Expanded City",{{"Name.1", "City"}}), 72 | #"Expanded Loc" = Table.ExpandRecordColumn(#"Renamed Columns", "Loc", {"coordinates"}, {"coordinates"}), 73 | #"Added Custom" = Table.AddColumn(#"Expanded Loc", "Latitude", each [coordinates]{1}), 74 | #"Added Custom1" = Table.AddColumn(#"Added Custom", "Longitude", each [coordinates]{0}), 75 | #"Removed Columns" = Table.RemoveColumns(#"Added Custom1",{"coordinates"}), 76 | #"Changed Type" = Table.TransformColumnTypes(#"Removed Columns",{{"Name", type text}, {"IcaoCode", type text}, {"IataCode", type text}, {"Address", type text}, {"City", type text}, {"CountryRegion", type text}, {"Region", type text}, {"Latitude", type number}, {"Longitude", type number}}) 77 | in 78 | #"Changed Type"; 79 | 80 | // 81 | // Common functions 82 | // 83 | Table.ToNavigationTable = ( 84 | table as table, 85 | keyColumns as list, 86 | nameColumn as text, 87 | dataColumn as text, 88 | itemKindColumn as text, 89 | itemNameColumn as text, 90 | isLeafColumn as text 91 | ) as table => 92 | let 93 | tableType = Value.Type(table), 94 | newTableType = Type.AddTableKey(tableType, keyColumns, true) meta 95 | [ 96 | NavigationTable.NameColumn = nameColumn, 97 | NavigationTable.DataColumn = dataColumn, 98 | NavigationTable.ItemKindColumn = itemKindColumn, 99 | Preview.DelayColumn = itemNameColumn, 100 | NavigationTable.IsLeafColumn = isLeafColumn 101 | ], 102 | navigationTable = Value.ReplaceType(table, newTableType) 103 | in 104 | navigationTable; 105 | -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/TripPin/3-NavTables/TripPin.query.pq: -------------------------------------------------------------------------------- 1 |  TripPin.Contents("http://services.odata.org/v4/TripPinService/") -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/TripPin/4-Paths/TripPin.pq: -------------------------------------------------------------------------------- 1 | section TripPin; 2 | 3 | // 4 | // Definition 5 | // 6 | 7 | // Data Source Kind description 8 | TripPin = [ 9 | Authentication = [ 10 | Anonymous = [] 11 | ], 12 | Label = "TripPin Part 4 - Data Source Paths" 13 | ]; 14 | 15 | // Data Source UI publishing description 16 | TripPin.Publish = [ 17 | Beta = true, 18 | Category = "Other", 19 | ButtonText = { "TripPin Data Source Paths", "TripPin Data Source Paths" } 20 | ]; 21 | 22 | // 23 | // Implementation 24 | // 25 | 26 | DefaultRequestHeaders = [ 27 | #"Accept" = "application/json;odata.metadata=minimal", // column name and values only 28 | #"OData-MaxVersion" = "4.0" // we only support v4 29 | ]; 30 | 31 | BaseUrl = "http://services.odata.org/v4/TripPinService/"; 32 | 33 | RootEntities = { 34 | "Airlines", 35 | "Airports", 36 | "People" 37 | }; 38 | 39 | [DataSource.Kind="TripPin", Publish="TripPin.Publish"] 40 | shared TripPin.Contents = () => TripPinNavTable(BaseUrl) as table; 41 | 42 | TripPinNavTable = (url as text) as table => 43 | let 44 | entitiesAsTable = Table.FromList(RootEntities, Splitter.SplitByNothing()), 45 | rename = Table.RenameColumns(entitiesAsTable, {{"Column1", "Name"}}), 46 | // Add Data as a calculated column 47 | withData = Table.AddColumn(rename, "Data", each TripPin.Feed(Uri.Combine(url, [Name])), Uri.Type), 48 | // Add ItemKind and ItemName as fixed text values 49 | withItemKind = Table.AddColumn(withData, "ItemKind", each "Table", type text), 50 | withItemName = Table.AddColumn(withItemKind, "ItemName", each "Table", type text), 51 | // Indicate that the node should not be expandable 52 | withIsLeaf = Table.AddColumn(withItemName, "IsLeaf", each true, type logical), 53 | // Generate the nav table 54 | navTable = Table.ToNavigationTable(withIsLeaf, {"Name"}, "Name", "Data", "ItemKind", "ItemName", "IsLeaf") 55 | in 56 | navTable; 57 | 58 | TripPin.Feed = (url as text) => 59 | let 60 | source = Web.Contents(url, [ Headers = DefaultRequestHeaders ]), 61 | json = Json.Document(source), 62 | // The response is a JSON record - the data we want is a list of records in the "value" field 63 | value = json[value], 64 | asTable = Table.FromList(value, Splitter.SplitByNothing()), 65 | // expand all columns from the record 66 | fields = Record.FieldNames(Table.FirstValue(asTable, [Empty = null])), 67 | expandAll = Table.ExpandRecordColumn(asTable, "Column1", fields) 68 | in 69 | expandAll; 70 | 71 | // 72 | // Common functions 73 | // 74 | Table.ToNavigationTable = ( 75 | table as table, 76 | keyColumns as list, 77 | nameColumn as text, 78 | dataColumn as text, 79 | itemKindColumn as text, 80 | itemNameColumn as text, 81 | isLeafColumn as text 82 | ) as table => 83 | let 84 | tableType = Value.Type(table), 85 | newTableType = Type.AddTableKey(tableType, keyColumns, true) meta 86 | [ 87 | NavigationTable.NameColumn = nameColumn, 88 | NavigationTable.DataColumn = dataColumn, 89 | NavigationTable.ItemKindColumn = itemKindColumn, 90 | Preview.DelayColumn = itemNameColumn, 91 | NavigationTable.IsLeafColumn = isLeafColumn 92 | ], 93 | navigationTable = Value.ReplaceType(table, newTableType) 94 | in 95 | navigationTable; 96 | 97 | 98 | 99 | -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/TripPin/4-Paths/TripPin.query.pq: -------------------------------------------------------------------------------- 1 | TripPin.Contents() 2 | -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/TripPin/5-Paging/TripPin.pq: -------------------------------------------------------------------------------- 1 | section TripPin; 2 | 3 | // 4 | // Definition 5 | // 6 | 7 | // Data Source Kind description 8 | TripPin = [ 9 | Authentication = [ 10 | Anonymous = [] 11 | ], 12 | Label = "TripPin Part 5 - Paging" 13 | ]; 14 | 15 | // Data Source UI publishing description 16 | TripPin.Publish = [ 17 | Beta = true, 18 | Category = "Other", 19 | ButtonText = { "TripPin Paging", "TripPin Paging" } 20 | ]; 21 | 22 | // 23 | // Implementation 24 | // 25 | 26 | DefaultRequestHeaders = [ 27 | #"Accept" = "application/json;odata.metadata=minimal", // column name and values only 28 | #"OData-MaxVersion" = "4.0" // we only support v4 29 | ]; 30 | 31 | BaseUrl = "http://services.odata.org/v4/TripPinService/"; 32 | 33 | RootEntities = { 34 | "Airlines", 35 | "Airports", 36 | "People" 37 | }; 38 | 39 | [DataSource.Kind="TripPin", Publish="TripPin.Publish"] 40 | shared TripPin.Contents = () => TripPinNavTable(BaseUrl) as table; 41 | 42 | TripPinNavTable = (url as text) as table => 43 | let 44 | entitiesAsTable = Table.FromList(RootEntities, Splitter.SplitByNothing()), 45 | rename = Table.RenameColumns(entitiesAsTable, {{"Column1", "Name"}}), 46 | // Add Data as a calculated column 47 | withData = Table.AddColumn(rename, "Data", each TripPin.Feed(Uri.Combine(url, [Name])), type table), 48 | // Add ItemKind and ItemName as fixed text values 49 | withItemKind = Table.AddColumn(withData, "ItemKind", each "Table", type text), 50 | withItemName = Table.AddColumn(withItemKind, "ItemName", each "Table", type text), 51 | // Indicate that the node should not be expandable 52 | withIsLeaf = Table.AddColumn(withItemName, "IsLeaf", each true, type logical), 53 | // Generate the nav table 54 | navTable = Table.ToNavigationTable(withIsLeaf, {"Name"}, "Name", "Data", "ItemKind", "ItemName", "IsLeaf") 55 | in 56 | navTable; 57 | 58 | TripPin.Feed = (url as text) as table => GetAllPagesByNextLink(url); 59 | 60 | GetPage = (url as text) as table => 61 | let 62 | response = Web.Contents(url, [ Headers = DefaultRequestHeaders ]), 63 | body = Json.Document(response), 64 | nextLink = GetNextLink(body), 65 | data = Table.FromRecords(body[value]) 66 | in 67 | data meta [NextLink = nextLink]; 68 | 69 | // Read all pages of data. 70 | // After every page, we check the "NextLink" record on the metadata of the previous request. 71 | // Table.GenerateByPage will keep asking for more pages until we return null. 72 | GetAllPagesByNextLink = (url as text) as table => 73 | Table.GenerateByPage((previous) => 74 | let 75 | // if previous is null, then this is our first page of data 76 | nextLink = if (previous = null) then url else Value.Metadata(previous)[NextLink]?, 77 | // if NextLink was set to null by the previous call, we know we have no more data 78 | page = if (nextLink <> null) then GetPage(nextLink) else null 79 | in 80 | page 81 | ); 82 | 83 | // In this implementation, 'response' will be the parsed body of the response after the call to Json.Document. 84 | // We look for the '@odata.nextLink' field and simply return null if it doesn't exist. 85 | GetNextLink = (response) as nullable text => Record.FieldOrDefault(response, "@odata.nextLink"); 86 | 87 | // 88 | // Common functions 89 | // 90 | Table.ToNavigationTable = ( 91 | table as table, 92 | keyColumns as list, 93 | nameColumn as text, 94 | dataColumn as text, 95 | itemKindColumn as text, 96 | itemNameColumn as text, 97 | isLeafColumn as text 98 | ) as table => 99 | let 100 | tableType = Value.Type(table), 101 | newTableType = Type.AddTableKey(tableType, keyColumns, true) meta 102 | [ 103 | NavigationTable.NameColumn = nameColumn, 104 | NavigationTable.DataColumn = dataColumn, 105 | NavigationTable.ItemKindColumn = itemKindColumn, 106 | Preview.DelayColumn = itemNameColumn, 107 | NavigationTable.IsLeafColumn = isLeafColumn 108 | ], 109 | navigationTable = Value.ReplaceType(table, newTableType) 110 | in 111 | navigationTable; 112 | 113 | // The getNextPage function takes a single argument and is expected to return a nullable table 114 | Table.GenerateByPage = (getNextPage as function) as table => 115 | let 116 | listOfPages = List.Generate( 117 | () => getNextPage(null), // get the first page of data 118 | (lastPage) => lastPage <> null, // stop when the function returns null 119 | (lastPage) => getNextPage(lastPage) // pass the previous page to the next function call 120 | ), 121 | // concatenate the pages together 122 | tableOfPages = Table.FromList(listOfPages, Splitter.SplitByNothing(), {"Column1"}), 123 | firstRow = tableOfPages{0}? 124 | in 125 | // if we didn't get back any pages of data, return an empty table 126 | // otherwise set the table type based on the columns of the first page 127 | if (firstRow = null) then 128 | Table.FromRows({}) 129 | else 130 | Value.ReplaceType( 131 | Table.ExpandTableColumn(tableOfPages, "Column1", Table.ColumnNames(firstRow[Column1])), 132 | Value.Type(firstRow[Column1]) 133 | ); -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/TripPin/5-Paging/TripPin.query.pq: -------------------------------------------------------------------------------- 1 | let 2 | source = TripPin.Contents(), 3 | data = source{[Name="People"]}[Data], 4 | withRowCount = Table.AddIndexColumn(data, "Index") 5 | in 6 | withRowCount 7 | -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/TripPin/6-Schema/TripPin.query.pq: -------------------------------------------------------------------------------- 1 | let 2 | source = TripPin.Contents(), 3 | data = source{[Name="People"]}[Data] 4 | in 5 | Table.Schema(data) 6 | -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/TripPin/7-AdvancedSchema/Table.GenerateByPage.pqm: -------------------------------------------------------------------------------- 1 | (getNextPage as function) as table => 2 | let 3 | listOfPages = List.Generate( 4 | () => getNextPage(null), // get the first page of data 5 | (lastPage) => lastPage <> null, // stop when the function returns null 6 | (lastPage) => getNextPage(lastPage) // pass the previous page to the next function call 7 | ), 8 | // concatenate the pages together 9 | tableOfPages = Table.FromList(listOfPages, Splitter.SplitByNothing(), {"Column1"}), 10 | firstRow = tableOfPages{0}? 11 | in 12 | // if we didn't get back any pages of data, return an empty table 13 | // otherwise set the table type based on the columns of the first page 14 | if (firstRow = null) then 15 | Table.FromRows({}) 16 | // check for empty first table 17 | else if (Table.IsEmpty(firstRow[Column1])) then 18 | firstRow[Column1] 19 | else 20 | Value.ReplaceType( 21 | Table.ExpandTableColumn(tableOfPages, "Column1", Table.ColumnNames(firstRow[Column1])), 22 | Value.Type(firstRow[Column1]) 23 | ) 24 | -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/TripPin/7-AdvancedSchema/Table.ToNavigationTable.pqm: -------------------------------------------------------------------------------- 1 | ( 2 | table as table, 3 | keyColumns as list, 4 | nameColumn as text, 5 | dataColumn as text, 6 | itemKindColumn as text, 7 | itemNameColumn as text, 8 | isLeafColumn as text 9 | ) as table => 10 | let 11 | tableType = Value.Type(table), 12 | newTableType = Type.AddTableKey(tableType, keyColumns, true) meta 13 | [ 14 | NavigationTable.NameColumn = nameColumn, 15 | NavigationTable.DataColumn = dataColumn, 16 | NavigationTable.ItemKindColumn = itemKindColumn, 17 | Preview.DelayColumn = itemNameColumn, 18 | NavigationTable.IsLeafColumn = isLeafColumn 19 | ], 20 | navigationTable = Value.ReplaceType(table, newTableType) 21 | in 22 | navigationTable -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/TripPin/8-Diagnostics/Table.GenerateByPage.pqm: -------------------------------------------------------------------------------- 1 | (getNextPage as function) as table => 2 | let 3 | listOfPages = List.Generate( 4 | () => getNextPage(null), // get the first page of data 5 | (lastPage) => lastPage <> null, // stop when the function returns null 6 | (lastPage) => getNextPage(lastPage) // pass the previous page to the next function call 7 | ), 8 | // concatenate the pages together 9 | tableOfPages = Table.FromList(listOfPages, Splitter.SplitByNothing(), {"Column1"}), 10 | firstRow = tableOfPages{0}? 11 | in 12 | // if we didn't get back any pages of data, return an empty table 13 | // otherwise set the table type based on the columns of the first page 14 | if (firstRow = null) then 15 | Table.FromRows({}) 16 | // check for empty first table 17 | else if (Table.IsEmpty(firstRow[Column1])) then 18 | firstRow[Column1] 19 | else 20 | Value.ReplaceType( 21 | Table.ExpandTableColumn(tableOfPages, "Column1", Table.ColumnNames(firstRow[Column1])), 22 | Value.Type(firstRow[Column1]) 23 | ) 24 | -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/TripPin/8-Diagnostics/Table.ToNavigationTable.pqm: -------------------------------------------------------------------------------- 1 | ( 2 | table as table, 3 | keyColumns as list, 4 | nameColumn as text, 5 | dataColumn as text, 6 | itemKindColumn as text, 7 | itemNameColumn as text, 8 | isLeafColumn as text 9 | ) as table => 10 | let 11 | tableType = Value.Type(table), 12 | newTableType = Type.AddTableKey(tableType, keyColumns, true) meta 13 | [ 14 | NavigationTable.NameColumn = nameColumn, 15 | NavigationTable.DataColumn = dataColumn, 16 | NavigationTable.ItemKindColumn = itemKindColumn, 17 | Preview.DelayColumn = itemNameColumn, 18 | NavigationTable.IsLeafColumn = isLeafColumn 19 | ], 20 | navigationTable = Value.ReplaceType(table, newTableType) 21 | in 22 | navigationTable -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/TripPin/9-TestConnection/Table.GenerateByPage.pqm: -------------------------------------------------------------------------------- 1 | (getNextPage as function) as table => 2 | let 3 | listOfPages = List.Generate( 4 | () => getNextPage(null), // get the first page of data 5 | (lastPage) => lastPage <> null, // stop when the function returns null 6 | (lastPage) => getNextPage(lastPage) // pass the previous page to the next function call 7 | ), 8 | // concatenate the pages together 9 | tableOfPages = Table.FromList(listOfPages, Splitter.SplitByNothing(), {"Column1"}), 10 | firstRow = tableOfPages{0}? 11 | in 12 | // if we didn't get back any pages of data, return an empty table 13 | // otherwise set the table type based on the columns of the first page 14 | if (firstRow = null) then 15 | Table.FromRows({}) 16 | // check for empty first table 17 | else if (Table.IsEmpty(firstRow[Column1])) then 18 | firstRow[Column1] 19 | else 20 | Value.ReplaceType( 21 | Table.ExpandTableColumn(tableOfPages, "Column1", Table.ColumnNames(firstRow[Column1])), 22 | Value.Type(firstRow[Column1]) 23 | ) 24 | -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/TripPin/9-TestConnection/Table.ToNavigationTable.pqm: -------------------------------------------------------------------------------- 1 | ( 2 | table as table, 3 | keyColumns as list, 4 | nameColumn as text, 5 | dataColumn as text, 6 | itemKindColumn as text, 7 | itemNameColumn as text, 8 | isLeafColumn as text 9 | ) as table => 10 | let 11 | tableType = Value.Type(table), 12 | newTableType = Type.AddTableKey(tableType, keyColumns, true) meta 13 | [ 14 | NavigationTable.NameColumn = nameColumn, 15 | NavigationTable.DataColumn = dataColumn, 16 | NavigationTable.ItemKindColumn = itemKindColumn, 17 | Preview.DelayColumn = itemNameColumn, 18 | NavigationTable.IsLeafColumn = isLeafColumn 19 | ], 20 | navigationTable = Value.ReplaceType(table, newTableType) 21 | in 22 | navigationTable -------------------------------------------------------------------------------- /src/test/resources/microsoft-DataConnectors/UnitTesting/UnitTesting.pq: -------------------------------------------------------------------------------- 1 | section UnitTesting; 2 | 3 | shared UnitTesting.ReturnsABC = () => "ABC"; 4 | shared UnitTesting.Returns123 = () => "123"; 5 | shared UnitTesting.ReturnsTableWithFiveRows = () => Table.Repeat(#table({"a"},{{1}}), 5); 6 | 7 | 8 | -------------------------------------------------------------------------------- /src/test/testUtils/assertTestUtils.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import "mocha"; 5 | 6 | import { Assert, Lexer, Parser, Task } from "../.."; 7 | import { ResultUtils, TaskUtils } from "../../powerquery-parser"; 8 | import { LexSettings } from "../../powerquery-parser/lexer"; 9 | import { ParseSettings } from "../../powerquery-parser/parser"; 10 | 11 | export async function assertGetLexParseOk( 12 | settings: LexSettings & ParseSettings, 13 | text: string, 14 | ): Promise { 15 | const triedLexParseTask: Task.TriedLexParseTask = await TaskUtils.tryLexParse(settings, text); 16 | TaskUtils.assertIsParseStageOk(triedLexParseTask); 17 | 18 | return triedLexParseTask; 19 | } 20 | 21 | export async function assertGetLexParseError( 22 | settings: LexSettings & ParseSettings, 23 | text: string, 24 | ): Promise { 25 | const triedLexParseTask: Task.TriedLexParseTask = await TaskUtils.tryLexParse(settings, text); 26 | TaskUtils.assertIsParseStageParseError(triedLexParseTask); 27 | 28 | return triedLexParseTask; 29 | } 30 | 31 | export async function assertGetParseError( 32 | settings: LexSettings & ParseSettings, 33 | text: string, 34 | ): Promise { 35 | const triedParse: Parser.TriedParse = await assertGetTriedParse(settings, text); 36 | ResultUtils.assertIsError(triedParse); 37 | 38 | if (!Parser.ParseError.isParseError(triedParse.error)) { 39 | throw new Error(`expected triedParse to return a ParseError.ParseError: ${triedParse.error.message}`); 40 | } 41 | 42 | return triedParse.error; 43 | } 44 | 45 | export async function assertGetParseOk(settings: LexSettings & ParseSettings, text: string): Promise { 46 | const triedParse: Parser.TriedParse = await assertGetTriedParse(settings, text); 47 | ResultUtils.assertIsOk(triedParse); 48 | 49 | return triedParse.value; 50 | } 51 | 52 | // I only care about errors coming from the parse stage. 53 | // If I use tryLexParse I might get a CommonError which could have come either from lexing or parsing. 54 | function assertGetTriedParse(settings: LexSettings & ParseSettings, text: string): Promise { 55 | const triedLex: Lexer.TriedLex = Lexer.tryLex(settings, text); 56 | ResultUtils.assertIsOk(triedLex); 57 | const lexerState: Lexer.State = triedLex.value; 58 | Assert.isUndefined(Lexer.errorLineMap(lexerState)); 59 | 60 | const triedSnapshot: Lexer.TriedLexerSnapshot = Lexer.trySnapshot(lexerState); 61 | ResultUtils.assertIsOk(triedSnapshot); 62 | const lexerSnapshot: Lexer.LexerSnapshot = triedSnapshot.value; 63 | 64 | return Parser.ParserUtils.tryParse(settings, lexerSnapshot); 65 | } 66 | -------------------------------------------------------------------------------- /src/test/testUtils/fileTestUtils.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import "mocha"; 5 | import * as fs from "fs"; 6 | import * as path from "path"; 7 | 8 | import { LexSettings } from "../../powerquery-parser/lexer"; 9 | import { ParseSettings } from "../../powerquery-parser/parser"; 10 | import { Task } from "../.."; 11 | import { TaskUtils } from "../../powerquery-parser"; 12 | 13 | const PowerQueryExtensions: ReadonlyArray = [".m", ".mout", ".pq", "pqm"]; 14 | 15 | export function getPowerQueryFilePathsRecursively(rootDirectory: string): ReadonlyArray { 16 | const dirs: ReadonlyArray = getDirectoryPaths(rootDirectory); 17 | 18 | let files: ReadonlyArray = dirs 19 | // go through each directory 20 | .map(getPowerQueryFilePathsRecursively) 21 | // map returns a 2d array (array of file arrays) so flatten 22 | .reduce((a: ReadonlyArray, b: ReadonlyArray) => a.concat(b), []); 23 | 24 | // Get files in root folder 25 | files = files.concat(getPowerQueryFilePaths(rootDirectory)); 26 | 27 | return files; 28 | } 29 | 30 | export function readContents(filePath: string): string { 31 | // tslint:disable-next-line: non-literal-fs-path 32 | const contents: string = fs.readFileSync(filePath, "utf8"); 33 | 34 | return contents.replace(/^\uFEFF/, ""); 35 | } 36 | 37 | export function writeContents(filePath: string, contents: string): void { 38 | const dirPath: string = path.dirname(filePath); 39 | 40 | // tslint:disable-next-line: non-literal-fs-path 41 | if (!fs.existsSync(dirPath)) { 42 | // tslint:disable-next-line: non-literal-fs-path 43 | fs.mkdirSync(dirPath, { recursive: true }); 44 | } 45 | 46 | // tslint:disable-next-line: non-literal-fs-path 47 | fs.writeFileSync(filePath, contents, { encoding: "utf8" }); 48 | } 49 | 50 | export function tryLexParse(settings: LexSettings & ParseSettings, filePath: string): Promise { 51 | const contents: string = readContents(filePath); 52 | 53 | return TaskUtils.tryLexParse(settings, contents); 54 | } 55 | 56 | function isDirectory(path: string): boolean { 57 | // tslint:disable-next-line: non-literal-fs-path 58 | return fs.statSync(path).isDirectory(); 59 | } 60 | 61 | function isFile(filePath: string): boolean { 62 | // tslint:disable-next-line: non-literal-fs-path 63 | return fs.statSync(filePath).isFile(); 64 | } 65 | 66 | function isPowerQueryFile(filePath: string): boolean { 67 | return isFile(filePath) && isPowerQueryExtension(path.extname(filePath)); 68 | } 69 | 70 | function isPowerQueryExtension(extension: string): boolean { 71 | return PowerQueryExtensions.indexOf(extension) !== -1; 72 | } 73 | 74 | function getDirectoryPaths(rootDirectory: string): ReadonlyArray { 75 | // tslint:disable-next-line: non-literal-fs-path 76 | return ( 77 | fs 78 | // tslint:disable-next-line: non-literal-fs-path 79 | .readdirSync(rootDirectory) 80 | .map((name: string) => path.join(rootDirectory, name)) 81 | .filter(isDirectory) 82 | ); 83 | } 84 | 85 | function getPowerQueryFilePaths(filePath: string): ReadonlyArray { 86 | // tslint:disable-next-line: non-literal-fs-path 87 | return ( 88 | fs 89 | // tslint:disable-next-line: non-literal-fs-path 90 | .readdirSync(filePath) 91 | .map((name: string) => path.join(filePath, name)) 92 | .filter(isPowerQueryFile) 93 | ); 94 | } 95 | -------------------------------------------------------------------------------- /src/test/testUtils/index.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | export * as AssertTestUtils from "./assertTestUtils"; 5 | export * as FileTestUtils from "./fileTestUtils"; 6 | export * as LexTestUtils from "./lexTestUtils"; 7 | export * as ResourceTestUtils from "./resourceTestUtils"; 8 | export * as TestConstants from "./testConstants"; 9 | export * as TokenizerTestUtils from "./tokenizerTestUtils"; 10 | -------------------------------------------------------------------------------- /src/test/testUtils/lexTestUtils.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import { expect } from "chai"; 5 | 6 | import { Assert, DefaultSettings, Language, Lexer, ResultUtils } from "../.."; 7 | 8 | export type AbridgedComments = ReadonlyArray<[Language.Comment.CommentKind, string]>; 9 | 10 | export type AbridgedTokens = ReadonlyArray<[Language.Token.TokenKind, string]>; 11 | 12 | export interface AbridgedSnapshot { 13 | readonly tokens: AbridgedTokens; 14 | readonly comments: AbridgedComments; 15 | } 16 | 17 | export type AbridgedLineTokens = ReadonlyArray<[Language.Token.LineTokenKind, string]>; 18 | 19 | export function assertGetAbridgedSnapshotMatch( 20 | text: string, 21 | expected: AbridgedSnapshot, 22 | wrapped: boolean, 23 | ): Lexer.LexerSnapshot { 24 | if (wrapped) { 25 | const wrappedText: string = `wrapperOpen\n${text}\nwrapperClose`; 26 | 27 | const wrappedExpected: AbridgedSnapshot = { 28 | tokens: [ 29 | [Language.Token.TokenKind.Identifier, "wrapperOpen"], 30 | ...expected.tokens, 31 | [Language.Token.TokenKind.Identifier, "wrapperClose"], 32 | ], 33 | comments: expected.comments, 34 | }; 35 | 36 | assertGetAbridgedSnapshotMatch(wrappedText, wrappedExpected, false); 37 | } 38 | 39 | const snapshot: Lexer.LexerSnapshot = assertGetLexerSnapshot(text); 40 | const expectedTokens: AbridgedTokens = expected.tokens; 41 | const expectedComments: AbridgedComments = expected.comments; 42 | const actualTokens: AbridgedTokens = snapshot.tokens.map((token: Language.Token.Token) => [token.kind, token.data]); 43 | 44 | const actualComments: AbridgedComments = snapshot.comments.map((comment: Language.Comment.TComment) => [ 45 | comment.kind, 46 | comment.data, 47 | ]); 48 | 49 | expect(actualTokens).deep.equal(expectedTokens); 50 | expect(actualComments).deep.equal(expectedComments); 51 | 52 | return snapshot; 53 | } 54 | 55 | export function assertGetLineTokenMatch(text: string, expected: AbridgedLineTokens, wrapped: boolean): Lexer.State { 56 | if (wrapped) { 57 | const wrappedText: string = `wrapperOpen\n${text}\nwrapperClose`; 58 | 59 | const wrappedExpected: AbridgedLineTokens = [ 60 | [Language.Token.LineTokenKind.Identifier, "wrapperOpen"], 61 | ...expected, 62 | [Language.Token.LineTokenKind.Identifier, "wrapperClose"], 63 | ]; 64 | 65 | assertGetLineTokenMatch(wrappedText, wrappedExpected, false); 66 | } 67 | 68 | const state: Lexer.State = assertGetLexOk(text); 69 | 70 | const tmp: [Language.Token.LineTokenKind, string][] = []; 71 | 72 | for (const line of state.lines) { 73 | for (const token of line.tokens) { 74 | tmp.push([token.kind, token.data]); 75 | } 76 | } 77 | 78 | const actual: AbridgedLineTokens = tmp; 79 | 80 | const tokenDetails: { 81 | actual: AbridgedLineTokens; 82 | expected: AbridgedLineTokens; 83 | } = { 84 | actual, 85 | expected, 86 | }; 87 | 88 | expect(actual).deep.equal(expected, JSON.stringify(tokenDetails)); 89 | 90 | return state; 91 | } 92 | 93 | export function assertGetSnapshotAbridgedTokens( 94 | text: string, 95 | expected: AbridgedTokens, 96 | wrapped: boolean, 97 | ): Lexer.LexerSnapshot { 98 | return assertGetAbridgedSnapshotMatch( 99 | text, 100 | { 101 | tokens: expected, 102 | comments: [], 103 | }, 104 | wrapped, 105 | ); 106 | } 107 | 108 | export function assertGetSnapshotAbridgedComments( 109 | text: string, 110 | expected: AbridgedComments, 111 | wrapped: boolean, 112 | ): Lexer.LexerSnapshot { 113 | return assertGetAbridgedSnapshotMatch( 114 | text, 115 | { 116 | tokens: [], 117 | comments: expected, 118 | }, 119 | wrapped, 120 | ); 121 | } 122 | 123 | export function assertGetLexOk(text: string): Lexer.State { 124 | const triedLex: Lexer.TriedLex = Lexer.tryLex(DefaultSettings, text); 125 | ResultUtils.assertIsOk(triedLex); 126 | const lexerState: Lexer.State = triedLex.value; 127 | 128 | if (Lexer.isErrorState(lexerState)) { 129 | const errorLineMap: Lexer.ErrorLineMap = Assert.asDefined(Lexer.errorLineMap(lexerState)); 130 | const errorLines: ReadonlyArray = [...errorLineMap.keys()]; 131 | 132 | const details: { errorLines: ReadonlyArray } = { errorLines }; 133 | throw new Error(`AssertFailed: Lexer.isErrorState(state) ${JSON.stringify(details, undefined, 4)}`); 134 | } 135 | 136 | return lexerState; 137 | } 138 | 139 | export function assertGetLexerSnapshot(text: string): Lexer.LexerSnapshot { 140 | const state: Lexer.State = assertGetLexOk(text); 141 | const triedSnapshot: Lexer.TriedLexerSnapshot = Lexer.trySnapshot(state); 142 | ResultUtils.assertIsOk(triedSnapshot); 143 | 144 | return triedSnapshot.value; 145 | } 146 | -------------------------------------------------------------------------------- /src/test/testUtils/resourceTestUtils.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | // tslint:disable-next-line: no-require-imports 5 | import performanceNow = require("performance-now"); 6 | 7 | import "mocha"; 8 | import * as path from "path"; 9 | 10 | import { ArrayUtils, Settings, Task, TaskUtils } from "../../powerquery-parser"; 11 | import { FileTestUtils } from "."; 12 | 13 | export interface ResourceTestRun { 14 | readonly fileContents: string; 15 | readonly filePath: string; 16 | readonly testDuration: number; 17 | readonly triedLexParse: Task.TriedLexParseTask; 18 | } 19 | 20 | export interface TestResource { 21 | readonly fileContents: string; 22 | readonly filePath: string; 23 | readonly resourceName: string; 24 | } 25 | 26 | export function getResourceFilePaths(): ReadonlyArray { 27 | return FileTestUtils.getPowerQueryFilePathsRecursively(ResourcesDirectory); 28 | } 29 | 30 | export function getResources(): ReadonlyArray { 31 | return getResourceFilePaths().map((filePath: string) => { 32 | const fileContents: string = FileTestUtils.readContents(filePath); 33 | 34 | const resourceName: string = ArrayUtils.assertGet( 35 | filePath.split(ResourcesDirectory), 36 | 1, 37 | `expected ${filePath} to include ResourcesDirectory: ${ResourcesDirectory}}`, 38 | ) 39 | .replace(/\\/g, "-") 40 | .replace(/^-/g, ""); 41 | 42 | return { 43 | fileContents, 44 | filePath, 45 | resourceName, 46 | }; 47 | }); 48 | } 49 | 50 | export async function visitResources(visitFn: (filePath: string) => Promise): Promise { 51 | for (const filePath of FileTestUtils.getPowerQueryFilePathsRecursively(ResourcesDirectory)) { 52 | // eslint-disable-next-line no-await-in-loop 53 | await visitFn(filePath); 54 | } 55 | } 56 | 57 | export function runResourceTestSuite( 58 | settings: Settings, 59 | suiteName: string, 60 | testNameFn: (filePath: string) => string, 61 | visitFn: (testRun: ResourceTestRun) => void, 62 | ): void { 63 | describe(suiteName, () => { 64 | for (const filePath of getResourceFilePaths()) { 65 | it(testNameFn(filePath), async () => { 66 | const fileContents: string = FileTestUtils.readContents(filePath); 67 | const testStart: number = performanceNow(); 68 | 69 | // eslint-disable-next-line no-await-in-loop 70 | const triedLexParse: Task.TriedLexParseTask = await TaskUtils.tryLexParse(settings, fileContents); 71 | 72 | visitFn({ 73 | fileContents, 74 | filePath, 75 | testDuration: Math.floor(performanceNow() - testStart), 76 | triedLexParse, 77 | }); 78 | }); 79 | } 80 | }); 81 | } 82 | 83 | const ResourcesDirectory: string = path.join(path.dirname(__dirname), "resources"); 84 | -------------------------------------------------------------------------------- /src/test/testUtils/testConstants.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import { Parser } from "../../powerquery-parser"; 5 | 6 | export const ParserByParserName: ReadonlyMap = new Map([ 7 | ["CombinatorialParserV2", Parser.CombinatorialParserV2], 8 | ["RecursiveDescentParser", Parser.RecursiveDescentParser], 9 | ]); 10 | -------------------------------------------------------------------------------- /src/test/testUtils/tokenizerTestUtils.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | import { DefaultLocale, Language, ResultUtils } from "../../powerquery-parser"; 5 | import { Lexer } from "../.."; 6 | 7 | export class Tokenizer implements TokensProvider { 8 | constructor(private readonly lineTerminator: string) {} 9 | 10 | // tslint:disable-next-line: function-name 11 | public static ITokenFrom(lineToken: Language.Token.LineToken): IToken { 12 | // UNSAFE MARKER 13 | // 14 | // Purpose of code block: 15 | // Translate LineTokenKind to LineToken. 16 | // 17 | // Why are you trying to avoid a safer approach? 18 | // A proper mapping would require a switch statement, one case per kind in LineNodeKind. 19 | // 20 | // Why is it safe? 21 | // All variants of LineNodeKind are strings. 22 | return { 23 | startIndex: lineToken.positionStart, 24 | scopes: lineToken.kind as unknown as string, 25 | }; 26 | } 27 | 28 | public getInitialState(): IState { 29 | const lexerState: Lexer.State = { 30 | lines: [], 31 | locale: DefaultLocale, 32 | cancellationToken: undefined, 33 | }; 34 | 35 | return new TokenizerState(lexerState); 36 | } 37 | 38 | public tokenize(line: string, state: IState): ILineTokens { 39 | const tokenizerState: TokenizerState = state as TokenizerState; 40 | const lexerState: Lexer.State = tokenizerState.lexerState; 41 | 42 | const triedLex: Lexer.TriedLex = Lexer.tryAppendLine(lexerState, line, this.lineTerminator); 43 | ResultUtils.assertIsOk(triedLex); 44 | const newLexerState: Lexer.State = triedLex.value; 45 | 46 | return { 47 | tokens: newLexerState.lines[newLexerState.lines.length - 1].tokens.map(Tokenizer.ITokenFrom), 48 | endState: new TokenizerState(newLexerState), 49 | }; 50 | } 51 | } 52 | 53 | export class TokenizerState implements IState { 54 | constructor(public readonly lexerState: Lexer.State) {} 55 | 56 | public clone(): IState { 57 | return new TokenizerState(this.lexerState); 58 | } 59 | 60 | // For tokenizer state comparison, all we really care about is the line mode end value. 61 | // i.e. we need to know if we're ending on an unterminated comment/string as it 62 | // would impact tokenization for the following line. 63 | public equals(other: IState): boolean { 64 | if (!other) { 65 | return false; 66 | } 67 | 68 | // Check for initial state. 69 | const rightLexerState: Lexer.State = (other as TokenizerState).lexerState; 70 | 71 | if (this.lexerState.lines.length === 0) { 72 | return rightLexerState.lines.length === 0; 73 | } 74 | 75 | // Compare last line state. 76 | const leftLastLine: Lexer.TLine = this.lexerState.lines[this.lexerState.lines.length - 1]; 77 | const rightLastLine: Lexer.TLine = rightLexerState.lines[rightLexerState.lines.length - 1]; 78 | 79 | return leftLastLine.lineModeEnd === rightLastLine.lineModeEnd; 80 | } 81 | } 82 | 83 | // Taken from https://raw.githubusercontent.com/Microsoft/monaco-editor/master/monaco.d.ts 84 | export interface IState { 85 | clone(): IState; 86 | equals(other: IState): boolean; 87 | } 88 | 89 | export interface IToken { 90 | startIndex: number; 91 | scopes: string; 92 | } 93 | 94 | export interface ILineTokens { 95 | /** 96 | * The list of tokens on the line. 97 | */ 98 | tokens: IToken[]; 99 | /** 100 | * The tokenization end state. 101 | * A pointer will be held to this and the object should not be modified by the tokenizer after the pointer is returned. 102 | */ 103 | endState: IState; 104 | } 105 | 106 | export interface TokensProvider { 107 | /** 108 | * The initial state of a language. Will be the state passed in to tokenize the first line. 109 | */ 110 | getInitialState(): IState; 111 | /** 112 | * Tokenize a line given the state at the beginning of the line. 113 | */ 114 | tokenize(line: string, state: IState): ILineTokens; 115 | } 116 | -------------------------------------------------------------------------------- /style.md: -------------------------------------------------------------------------------- 1 | # Style Notes 2 | 3 | ## Exports 4 | 5 | - naive.ts, ast.ts, and naive.ts are excluded from the following rules 6 | - Always put exports before non-exports 7 | - Always order exports in the following order: types, enums, classes, interfaces, constants, functions 8 | 9 | ## If statements 10 | 11 | - Prefer to pair if with an else, except in the case of invariant checks, eg. 12 | 13 | ```typescript 14 | if (divisor === 0) { 15 | throw new Error("divisor should never be 0"); 16 | } 17 | 18 | return x / divisor; 19 | ``` 20 | 21 | ## Parameters 22 | 23 | - If one of the following parameters exist, order it in this order: locale, cancellation token, trace manager, trace, trace correlation Id 24 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "declaration": true, 4 | "downlevelIteration": true, 5 | "incremental": true, 6 | "module": "commonjs", 7 | "noEmitOnError": true, 8 | "noFallthroughCasesInSwitch": true, 9 | "noImplicitOverride": true, 10 | "noImplicitReturns": true, 11 | "noUnusedLocals": true, 12 | "noUnusedParameters": true, 13 | "outDir": "lib", 14 | "resolveJsonModule": true, 15 | "rootDir": "src", 16 | "sourceMap": true, 17 | "strict": true, 18 | "target": "es6", 19 | "tsBuildInfoFile": "./tsconfig.tsbuildinfo" 20 | }, 21 | "exclude": ["node_modules"], 22 | "include": ["src/**/*.ts", "src/powerquery-parser/localization/templates/*.json"] 23 | } 24 | --------------------------------------------------------------------------------