├── .eslintrc.json ├── .github ├── FUNDING.yml └── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── .gitignore ├── .markdownlint.json ├── .versionrc ├── .vscode-test.mjs ├── .vscode ├── extensions.json ├── launch.json ├── settings.json └── tasks.json ├── .vscodeignore ├── CHANGELOG.md ├── CONTRIBUTING.md ├── FAQ.md ├── LICENSE ├── README.md ├── language-configuration.json ├── media ├── css │ ├── query.css │ ├── reset.css │ ├── tabulator_custom_dark.css │ ├── tabulator_custom_light.css │ └── vscode.css ├── dataform.svg ├── images │ ├── cli_scope.png │ ├── compiled_query_preview.png │ ├── dataform_tools_run_and_debug.png │ ├── dependancy_tree.png │ ├── diagnostics.png │ ├── disable_save_on_compile.png │ ├── dtools.png │ ├── formatting.gif │ ├── func_def_on_hover.png │ ├── go_to_definition.gif │ ├── preferred_autocompletion.png │ ├── preview_query_results.png │ ├── schema_code_gen.gif │ ├── sources_autocompletion.gif │ └── tag_cost_estimator.png └── js │ ├── deps │ └── highlightjs-copy │ │ ├── highlightjs-copy.min.css │ │ └── highlightjs-copy.min.js │ ├── showCompiledQuery.js │ ├── showQueryResults.js │ └── sidePanel.js ├── package-lock.json ├── package.json ├── postcss.config.js ├── snippets ├── README.md ├── bigquery.code-snippets.json └── dataform.code-snippets.json ├── sqlx.grammar.json ├── src ├── bigqueryClient.ts ├── bigqueryDryRun.ts ├── bigqueryRunQuery.ts ├── codeActionProvider.ts ├── codeLensProvider.ts ├── completions.ts ├── constants.ts ├── costEstimator.ts ├── definitionProvider.ts ├── dependancyTree.ts ├── dependancyTreeNodeMeta.ts ├── extension.ts ├── formatCurrentFile.ts ├── getLineageMetadata.ts ├── globals.d.ts ├── hoverProvider.ts ├── logger.ts ├── previewQueryResults.ts ├── renameProvider.ts ├── runFiles.ts ├── runFilesTagsWtOptions.ts ├── runTag.ts ├── setDiagnostics.ts ├── sqlxFileParser.ts ├── test │ ├── suite │ │ ├── extension.test.ts │ │ └── helper.ts │ └── test-workspace │ │ ├── .sqlfluff │ │ ├── README.md │ │ ├── definitions │ │ ├── 0100_CLUBS.sqlx │ │ ├── 0100_GAMES_META.sqlx │ │ ├── 0100_PLAYERS.sqlx │ │ ├── 010_JS_MULTIPLE.js │ │ ├── 0200_GAME_DETAILS.sqlx │ │ ├── 0200_PLAYER_TRANSFERS.sqlx │ │ ├── 0300_INCREMENTAL.sqlx │ │ ├── 0500_OPERATIONS.sqlx │ │ ├── assertions │ │ │ └── 0100_CLUBS_ASSER.sqlx │ │ ├── sources.js │ │ └── tests_for_vscode_extension │ │ │ ├── 0100_MULTIPLE_PRE_POST_OPS.sqlx │ │ │ ├── 0100_SINGLE_LINE_CONFIG.sqlx │ │ │ └── 099_MULTIPLE_ERRORS.sqlx │ │ ├── includes │ │ ├── docs.js │ │ └── params.js │ │ └── workflow_settings.yaml ├── types.ts ├── utils.ts └── views │ ├── depedancyGraphPanel.ts │ ├── register-preview-compiled-panel.ts │ ├── register-query-results-panel.ts │ └── register-sidebar-panel.ts ├── tailwind.config.js ├── tsconfig.json ├── vite.config.ts ├── website ├── .gitignore ├── app │ ├── blog │ │ ├── [slug] │ │ │ └── page.tsx │ │ ├── not-found.tsx │ │ └── page.tsx │ ├── components │ │ ├── BlogContent.tsx │ │ ├── ClientBlogContent.tsx │ │ ├── admonitions.css │ │ └── site-header.tsx │ ├── faq │ │ └── page.tsx │ ├── features │ │ └── page.tsx │ ├── globals.css │ ├── install │ │ └── page.tsx │ ├── layout.tsx │ └── page.tsx ├── components.json ├── components │ ├── code-block.tsx │ ├── feature-reels.tsx │ ├── features-table.tsx │ ├── installation-step-header.tsx │ ├── theme-image.tsx │ ├── theme-provider.tsx │ ├── theme-toggle.tsx │ └── ui │ │ ├── button.tsx │ │ ├── card.tsx │ │ ├── carousel.tsx │ │ ├── dialog.tsx │ │ ├── separator.tsx │ │ ├── sheet.tsx │ │ ├── sidebar.tsx │ │ ├── skeleton.tsx │ │ ├── use-mobile.tsx │ │ └── use-toast.ts ├── hooks │ ├── use-mobile.tsx │ └── use-toast.ts ├── lib │ └── utils.ts ├── next.config.mjs ├── package-lock.json ├── package.json ├── pnpm-lock.yaml ├── postcss.config.mjs ├── public │ ├── compiled_query_preview.png │ ├── compiled_query_preview_dark_mode.png │ ├── compiled_query_preview_light_mode.png │ ├── dependancy_tree.png │ ├── diagnostics.png │ ├── formatting.gif │ ├── func_def_on_hover.png │ ├── go_to_definition.gif │ ├── preview_query_results.png │ ├── quick_fix.png │ ├── schema_code_gen.gif │ ├── sources_autocompletion.gif │ └── tag_cost_estimator.png ├── styles │ └── globals.css ├── tailwind.config.ts └── tsconfig.json └── webviews └── dependancy_graph ├── App.tsx ├── TableNode.tsx ├── components └── StyledSelect.tsx ├── index.css ├── index.tsx ├── initialEdges.ts ├── initialNodes.ts ├── nodePositioning.ts └── vscode.ts /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "root": true, 3 | "parser": "@typescript-eslint/parser", 4 | "parserOptions": { 5 | "ecmaVersion": 6, 6 | "sourceType": "module" 7 | }, 8 | "plugins": [ 9 | "@typescript-eslint" 10 | ], 11 | "rules": { 12 | "@typescript-eslint/naming-convention": [ 13 | "warn", 14 | { 15 | "selector": "import", 16 | "format": [ "camelCase", "PascalCase" ] 17 | } 18 | ], 19 | "@typescript-eslint/semi": "warn", 20 | "curly": "warn", 21 | "eqeqeq": "warn", 22 | "no-throw-literal": "warn", 23 | "semi": "off" 24 | }, 25 | "ignorePatterns": [ 26 | "out", 27 | "dist", 28 | "**/*.d.ts" 29 | ] 30 | } -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | 2 | github: [ashish10alex] 3 | buy_me_a_coffee: ashishalexj -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: 'bug: add title of bug here' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 16 | **Expected behavior** 17 | A clear and concise description of what you expected to happen. 18 | 19 | **Screenshots** 20 | If applicable, add screenshots to help explain your problem. 21 | 22 | **Environment:** 23 | - Operating system 24 | - Dataform cli version 25 | - Dataform core version 26 | - Version of the extension 27 | - ... 28 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest feature for extension 4 | title: 'feat: add title of feature here' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Description of requested feature here** 11 | 12 | **How does this help the users of the extension** 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | out 2 | dist 3 | node_modules 4 | .vscode-test/ 5 | *.vsix 6 | .DS_Store 7 | .excalidraw 8 | user_config.json 9 | src/test/test-workspace/AGENDA.md 10 | webview/initialNodes.ts 11 | src/assets/nodeMetadata.ts -------------------------------------------------------------------------------- /.markdownlint.json: -------------------------------------------------------------------------------- 1 | 2 | { 3 | "MD033": false, // Inline HTML 4 | "MD013": false, // Line length 5 | "MD051": false, // Link fragments should be valid" 6 | "MD042": false // No empty links 7 | } 8 | -------------------------------------------------------------------------------- /.versionrc: -------------------------------------------------------------------------------- 1 | { 2 | "types": [ 3 | {"type": "feat", "section": "Features"}, 4 | {"type": "fix", "section": "Bug Fixes"}, 5 | {"type": "docs", "section": "Documentation"}, 6 | {"type": "style", "section": "Styling"}, 7 | {"type": "refactor", "section": "Refactors"}, 8 | {"type": "perf", "section": "Performance"}, 9 | {"type": "test", "section": "Tests"}, 10 | {"type": "build", "section": "Build System"}, 11 | {"type": "ci", "section": "CI"}, 12 | {"type": "revert", "section": "Reverts"} 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /.vscode-test.mjs: -------------------------------------------------------------------------------- 1 | import { defineConfig } from '@vscode/test-cli'; 2 | 3 | export default defineConfig({ 4 | files: 'out/src/test/**/*.test.js', 5 | }); 6 | 7 | -------------------------------------------------------------------------------- /.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | // See http://go.microsoft.com/fwlink/?LinkId=827846 3 | // for the documentation about the extensions.json format 4 | "recommendations": [ 5 | "dbaeumer.vscode-eslint", 6 | "ms-vscode.extension-test-runner" 7 | ] 8 | } 9 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "name": "Run Extension", 6 | "type": "extensionHost", 7 | "request": "launch", 8 | "args": [ 9 | "--extensionDevelopmentPath=${workspaceFolder}" 10 | ], 11 | "outFiles": [ 12 | "${workspaceFolder}/out/**/*.js", 13 | "${workspaceFolder}/dist/**/*.js" 14 | ], 15 | "preLaunchTask": "Build All" 16 | }, 17 | { 18 | "name": "Run Extension Tests", 19 | "type": "extensionHost", 20 | "request": "launch", 21 | "runtimeExecutable": "${execPath}", 22 | "args": [ 23 | "--disable-extensions", 24 | "--extensionDevelopmentPath=${workspaceFolder}", 25 | "--extensionTestsPath=${workspaceFolder}/out/test/suite/index" 26 | ], 27 | "outFiles": ["${workspaceFolder}/out/test/**/*.js"], 28 | "preLaunchTask": "Build All" 29 | } 30 | ] 31 | } 32 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | // Place your settings in this file to overwrite default and user settings. 2 | { 3 | "files.exclude": { 4 | "out": false // set this to true to hide the "out" folder with the compiled JS files 5 | }, 6 | "search.exclude": { 7 | "out": true // set this to false to include "out" folder in search results 8 | }, 9 | // Turn off tsc task auto detection since we have the necessary tasks as npm scripts 10 | "typescript.tsc.autoDetect": "off" 11 | } 12 | -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | // See https://go.microsoft.com/fwlink/?LinkId=733558 2 | // for the documentation about the tasks.json format 3 | { 4 | "version": "2.0.0", 5 | "tasks": [ 6 | { 7 | "type": "npm", 8 | "script": "watch", 9 | "problemMatcher": "$tsc-watch", 10 | "isBackground": true, 11 | "presentation": { 12 | "reveal": "never" 13 | }, 14 | "group": "build" 15 | }, 16 | { 17 | "type": "shell", 18 | "label": "vite-build", 19 | "command": "./node_modules/.bin/vite", 20 | "args": ["build"], 21 | "isBackground": true, 22 | "problemMatcher": [ 23 | { 24 | "pattern": [ 25 | { 26 | "regexp": ".", 27 | "file": 1, 28 | "location": 2, 29 | "message": 3 30 | } 31 | ], 32 | "background": { 33 | "activeOnStart": true, 34 | "beginsPattern": ".", 35 | "endsPattern": "." 36 | } 37 | } 38 | ], 39 | "group": "build" 40 | }, 41 | { 42 | "label": "Build All", 43 | "dependsOn": ["npm: watch", "vite-build"], 44 | "group": { 45 | "kind": "build", 46 | "isDefault": true 47 | } 48 | } 49 | ] 50 | } 51 | -------------------------------------------------------------------------------- /.vscodeignore: -------------------------------------------------------------------------------- 1 | .vscode/** 2 | .vscode-test/** 3 | src/** 4 | .gitignore 5 | .yarnrc 6 | vsc-extension-quickstart.md 7 | 8 | **/tsconfig.json 9 | **/.eslintrc.json 10 | **/*.map 11 | **/*.ts 12 | **/.vscode-test.* 13 | .versionrc 14 | .markdownlint.json 15 | 16 | FAQ.md 17 | webviews/** 18 | .github/** 19 | media/images/cli_scope.png 20 | media/images/dataform_tools_run_and_debug.png 21 | media/images/disable_save_on_compile.png 22 | media/images/preferred_autocompletion.png 23 | tailwind.config.js 24 | out/test*/** 25 | out/webviews/** 26 | 27 | 28 | node_modules/react*/** 29 | node_modules/vite*/** 30 | node_modules/mocha*/** 31 | node_modules/postcss*/** 32 | node_modules/tailwindcss*/** 33 | node_modules/autoprefixer*/** 34 | node_modules/postcss*/** 35 | node_modules/eslint*/** 36 | node_modules/.bin/** 37 | node_modules/dom-helpers*/** 38 | node_modules/zustand/** 39 | node_modules/pretest/** 40 | node_modules/@babel/** 41 | node_modules/babel-plugin-macros/** 42 | node_modules/undici-types/** 43 | node_modules/@types/** 44 | node_modules/@emotion*/** 45 | node_modules/concurrently*/** 46 | website/** 47 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | 2 | # Requirements 3 | 4 | * `node v18` or higher 5 | * `npm` (gets installed alongside node) 6 | * [Dataform cli](https://cloud.google.com/dataform/docs/use-dataform-cli) 7 | * Install and setup [gcloud cli](https://cloud.google.com/sdk/docs/install) for your operating system 8 | 9 | ## How to contribute 10 | 11 | [1. Build and make changes to the extension]() 12 | 13 | * Fork this repository, clone it to your desktop and open it in VSCode 14 | * Run `npm install` in your terminal to install all the dependencies 15 | * Click the `Run and debug` icon on the left hand pane of your editor and click on `Run Extension` button 16 | 17 | compilation 18 | 19 | * This should open a new VSCode window where you can open a Dataform project. Make sure that you folder opened in the workspace is at the root of the Dataform project. For example if your Dataform project is located at `~/repos/my_dataform_project` open the workspace at `~/repos/my_dataform_project`, **NOT** `~/repos`. Such that either `workflow_settings.yaml` or `dataform.json` depending on the Dataform version you are using is located at the root of the VSCode workspace. 20 | 21 | [2. Make your changes]() 22 | 23 | Make the desired changes to the `vscode-dataform-tools` repo and re-run/refresh the compilation to see the desired outcome in the new VSCode window 24 | 25 | [Test your changes]() 26 | 27 | * **Test your changes** on your Dataform repository. If you are running linux based operating system run `npm run test` on your terminal to verify if the exsisiting tests are pasing. There are some caveats with running tests, so do not panic if the test fail to run. The test would not be able to run if your project path is very long this is a [known issue reported here](https://github.com/microsoft/vscode-test/issues/232). Also, we are having to remove `.vscode-test/user-data` before running `vscode-test` in the `npm run test` script in `package.json`. These tests currently are only tested to be running on Mac OS. We will need to change the script for `npm run test` in `package.json` for it to work in multiple operating systems. 28 | 29 | * Run `npm install markdownlint-cli2 --global` to install markdown linter and run `markdownlint-cli2 README.md` to verify the Markdown is correctly formatted if you have made any changes there. 30 | 31 | ### Open an issue / pull request 32 | 33 | [If you'd like the feature or bug fix to be merged]() 34 | 35 | * Check the exsisting issues to make sure that if it has not been already raised 36 | * [Create an issue here](https://github.com/ashish10alex/vscode-dataform-tools/issues) 37 | * [Create a pull request here](https://github.com/ashish10alex/vscode-dataform-tools/pulls) 38 | 39 | ## Dependency graph webview 40 | 41 | We are using [React Flow](https://reactflow.dev/) to create the dependency graph. To build `dist/dependancy_graph.js` run `./node_modules/.bin/vite build` in the terminal. This will watch for changes and rebuild the `dist/dependancy_graph.js` file. 42 | -------------------------------------------------------------------------------- /FAQ.md: -------------------------------------------------------------------------------- 1 | 2 | ### Frequently asked questions 3 | 4 | 1. [Unable to execute command e.g. error]() `command vscode-dataform-tools.xxx not found` 5 | 6 | * It is likely that the vscode workspace folder is not opened at the root of your dataform project. For example, if your dataform project is located at `~/Documents/repos/my_dataform_project` ensure that workspace is opened at 7 | `~/Documents/repos/my_dataform_project` NOT `~/Documents/repos/my_dataform_project` 8 | * The above design is to facilitate the exection of `dataform compile --json` command without infering the dataform root at run time 9 | 10 | 2. [Error compiling Dataform, process existed with exit code 1]() 11 | * Check if correct dataform cli version is installed by running `dataform --version` in your terminal 12 | * Ensure that dataform cli version matches the version required by the project 13 | * Try compiling the project by running `dataform compile` on your terminal from the root of your dataform project 14 | * In case you need to install a specific dataform cli version by running `npm i -g @dataform/cli@2.9.0`. Make sure you verify the version by running the `dataform --version` 15 | * In case the error is not due to all the above reasons it is likely that you have a compilation error in your pipeline 16 | 17 | 3. [Dataform encountered an error: Missing credentials JSON file; not found at path /.df-credentials.json]() 18 | * Run `dataform init-creds` from the from the root of your dataform project in your terminal 19 | * You will be promted to pick the location and type of authentication `json/adc`. Choosing adc will be use your default gcp credentials that you had setup using `gcloud` 20 | 21 | 4. [I do not want to see compiled query each time I save it]() 22 | * Open vscode settings and search for Dataform and uncheck the following setting 23 | ![disable_save_on_compile](/media/images/disable_save_on_compile.png) 24 | 25 | 5. [I want the autocompletion to be of the format `${ref('dataset_name', 'table_name)}` instead of `${ref('table_name')}`]() 26 | * Open vscode settings and search for Dataform and select the prefered autocompletion format 27 | ![disable_save_on_compile](/media/images/preferred_autocompletion.png) 28 | 29 | 6. [I want to use local installation of dataform cli instead of global one | OR | I have to use different version of Dataform cli for a workspace]() 30 | * If you install dataform cli using `npm install -g @dataform/cli` it will install the dataform cli 31 | globally making only one version available everywhere. If you have varing Dataform version requeriement for a workspace 32 | you can installation dataform cli version specific to that workspace by running `npm install @dataform/cli` notice the absence of `-g` flag as compared to the previous command. Running `npm install @dataform/cli` will install dataform cli at `./node_modules/.bin/dataform`. To make the extension use dataform cli installated in local scope open 33 | settings and select local for `Dataform Cli Scope` as shown in the screenshot below. 34 | 35 | ![cli_scope](/media/images/cli_scope.png) 36 | 37 | 7. [I do not see go to definition option when right clicking references `${ref('table_name')}`]() 38 | * Check if you language mode when sqlx file is open is set to `sqlx`. VSCode sometimes sets it as a different flavour of sql. You can change that by opening the command pallet and searching for `change language mode` 39 | followed by `Configure language association for "sqlx"` and selecting `sqlx` from the list of available options. This should also resolve hover information not being visible as the all the language specific behaviors 40 | are tied to file being inferred as `sqlx` file. 41 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Ashish Alex 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /language-configuration.json: -------------------------------------------------------------------------------- 1 | { 2 | "comments": { 3 | "lineComment": "--", 4 | "blockComment": ["/*", "*/"] 5 | }, 6 | "brackets": [ 7 | ["{", "}"], 8 | ["[", "]"], 9 | ["(", ")"] 10 | ], 11 | "autoClosingPairs": [ 12 | { 13 | "open": "{", 14 | "close": "}" 15 | }, 16 | { 17 | "open": "[", 18 | "close": "]" 19 | }, 20 | { 21 | "open": "(", 22 | "close": ")" 23 | }, 24 | { 25 | "open": "\"", 26 | "close": "\"" 27 | }, 28 | { 29 | "open": "'", 30 | "close": "'" 31 | }, 32 | { 33 | "open": "`", 34 | "close": "`" 35 | } 36 | ], 37 | "surroundingPairs": [ 38 | { 39 | "open": "{", 40 | "close": "}" 41 | }, 42 | { 43 | "open": "[", 44 | "close": "]" 45 | }, 46 | { 47 | "open": "(", 48 | "close": ")" 49 | }, 50 | { 51 | "open": "\"", 52 | "close": "\"" 53 | }, 54 | { 55 | "open": "'", 56 | "close": "'" 57 | }, 58 | { 59 | "open": "`", 60 | "close": "`", 61 | "notIn": ["string", "comment"] 62 | } 63 | ] 64 | } 65 | -------------------------------------------------------------------------------- /media/css/reset.css: -------------------------------------------------------------------------------- 1 | html { 2 | box-sizing: border-box; 3 | font-size: 13px; 4 | } 5 | 6 | *, 7 | *:before, 8 | *:after { 9 | box-sizing: inherit; 10 | } 11 | 12 | body, 13 | h1, 14 | h2, 15 | h3, 16 | h4, 17 | h5, 18 | h6, 19 | p, 20 | ol, 21 | ul { 22 | margin: 0; 23 | padding: 0; 24 | font-weight: normal; 25 | } 26 | 27 | img { 28 | max-width: 100%; 29 | height: auto; 30 | } 31 | 32 | :root { 33 | --svg-legends-height: 550px; 34 | } 35 | 36 | /*starting position of the dependancy tree graph from the top 37 | current hardcoded to be more than the height of #svg-legends #*/ 38 | .content { 39 | margin-top: calc(var(--svg-legends-height) + 5px); 40 | } 41 | 42 | .warning { 43 | display: flex; 44 | align-items: center; 45 | color: #333; 46 | } 47 | 48 | .warning svg { 49 | margin-right: 5px; 50 | } -------------------------------------------------------------------------------- /media/css/tabulator_custom_dark.css: -------------------------------------------------------------------------------- 1 | .tabulator { 2 | background-color: #1e1e1e; /* VS Code dark theme background */ 3 | color: #d4d4d4; /* VS Code default text color */ 4 | } 5 | 6 | .tabulator .tabulator-header { 7 | background-color: #252526; 8 | color: #d4d4d4; 9 | } 10 | 11 | .tabulator .tabulator-row { 12 | background-color: #1e1e1e; 13 | } 14 | 15 | .tabulator .tabulator-row:hover { 16 | background-color: #2a2d2e; 17 | } 18 | 19 | .tabulator .tabulator-cell { 20 | border-color: #3c3c3c; 21 | color: #d4d4d4; 22 | } 23 | 24 | /* Optional: Add alternating row colors for better readability */ 25 | .tabulator .tabulator-row:nth-child(even) { 26 | background-color: #252526; /* Slightly lighter background for even rows */ 27 | } 28 | 29 | /* when cell is being edited the backgroudn color should be light gray */ 30 | .tabulator .tabulator-cell.tabulator-editing { 31 | background-color: #d4d4d4; 32 | } 33 | 34 | .error-column { 35 | color: #D32F2F; 36 | } -------------------------------------------------------------------------------- /media/css/tabulator_custom_light.css: -------------------------------------------------------------------------------- 1 | .error-column { 2 | color: #d32f2f; 3 | } 4 | -------------------------------------------------------------------------------- /media/css/vscode.css: -------------------------------------------------------------------------------- 1 | :root { 2 | --container-padding: 20px; 3 | --input-padding-vertical: 6px; 4 | --input-padding-horizontal: 4px; 5 | --input-margin-vertical: 4px; 6 | --input-margin-horizontal: 0; 7 | } 8 | 9 | body { 10 | padding: 0 var(--container-padding); 11 | color: var(--vscode-foreground); 12 | font-size: var(--vscode-font-size); 13 | font-weight: var(--vscode-font-weight); 14 | font-family: var(--vscode-font-family); 15 | background-color: var(--vscode-editor-background); 16 | } 17 | 18 | ol, 19 | ul { 20 | padding-left: var(--container-padding); 21 | } 22 | 23 | body > *, 24 | form > * { 25 | margin-block-start: var(--input-margin-vertical); 26 | margin-block-end: var(--input-margin-vertical); 27 | } 28 | 29 | *:focus { 30 | outline-color: var(--vscode-focusBorder) !important; 31 | } 32 | 33 | a { 34 | color: var(--vscode-textLink-foreground); 35 | } 36 | 37 | a:hover, 38 | a:active { 39 | color: var(--vscode-textLink-activeForeground); 40 | } 41 | 42 | code { 43 | font-size: var(--vscode-editor-font-size); 44 | font-family: var(--vscode-editor-font-family); 45 | } 46 | 47 | button { 48 | border: none; 49 | padding: var(--input-padding-vertical) var(--input-padding-horizontal); 50 | width: 100%; 51 | text-align: center; 52 | outline: 1px solid transparent; 53 | outline-offset: 2px !important; 54 | color: var(--vscode-button-foreground); 55 | background: var(--vscode-button-background); 56 | } 57 | 58 | button:hover { 59 | cursor: pointer; 60 | background: var(--vscode-button-hoverBackground); 61 | } 62 | 63 | button:focus { 64 | outline-color: var(--vscode-focusBorder); 65 | } 66 | 67 | button.secondary { 68 | color: var(--vscode-button-secondaryForeground); 69 | background: var(--vscode-button-secondaryBackground); 70 | } 71 | 72 | button.secondary:hover { 73 | background: var(--vscode-button-secondaryHoverBackground); 74 | } 75 | 76 | input:not([type='checkbox']), 77 | textarea { 78 | display: block; 79 | width: 100%; 80 | border: none; 81 | font-family: var(--vscode-font-family); 82 | padding: var(--input-padding-vertical) var(--input-padding-horizontal); 83 | color: var(--vscode-input-foreground); 84 | outline-color: var(--vscode-input-border); 85 | background-color: var(--vscode-input-background); 86 | } 87 | 88 | input::placeholder, 89 | textarea::placeholder { 90 | color: var(--vscode-input-placeholderForeground); 91 | } 92 | -------------------------------------------------------------------------------- /media/dataform.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /media/images/cli_scope.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ashish10alex/vscode-dataform-tools/43f7477f8cdc8a3c3121b43418728a62b3cb0f94/media/images/cli_scope.png -------------------------------------------------------------------------------- /media/images/compiled_query_preview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ashish10alex/vscode-dataform-tools/43f7477f8cdc8a3c3121b43418728a62b3cb0f94/media/images/compiled_query_preview.png -------------------------------------------------------------------------------- /media/images/dataform_tools_run_and_debug.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ashish10alex/vscode-dataform-tools/43f7477f8cdc8a3c3121b43418728a62b3cb0f94/media/images/dataform_tools_run_and_debug.png -------------------------------------------------------------------------------- /media/images/dependancy_tree.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ashish10alex/vscode-dataform-tools/43f7477f8cdc8a3c3121b43418728a62b3cb0f94/media/images/dependancy_tree.png -------------------------------------------------------------------------------- /media/images/diagnostics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ashish10alex/vscode-dataform-tools/43f7477f8cdc8a3c3121b43418728a62b3cb0f94/media/images/diagnostics.png -------------------------------------------------------------------------------- /media/images/disable_save_on_compile.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ashish10alex/vscode-dataform-tools/43f7477f8cdc8a3c3121b43418728a62b3cb0f94/media/images/disable_save_on_compile.png -------------------------------------------------------------------------------- /media/images/dtools.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ashish10alex/vscode-dataform-tools/43f7477f8cdc8a3c3121b43418728a62b3cb0f94/media/images/dtools.png -------------------------------------------------------------------------------- /media/images/formatting.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ashish10alex/vscode-dataform-tools/43f7477f8cdc8a3c3121b43418728a62b3cb0f94/media/images/formatting.gif -------------------------------------------------------------------------------- /media/images/func_def_on_hover.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ashish10alex/vscode-dataform-tools/43f7477f8cdc8a3c3121b43418728a62b3cb0f94/media/images/func_def_on_hover.png -------------------------------------------------------------------------------- /media/images/go_to_definition.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ashish10alex/vscode-dataform-tools/43f7477f8cdc8a3c3121b43418728a62b3cb0f94/media/images/go_to_definition.gif -------------------------------------------------------------------------------- /media/images/preferred_autocompletion.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ashish10alex/vscode-dataform-tools/43f7477f8cdc8a3c3121b43418728a62b3cb0f94/media/images/preferred_autocompletion.png -------------------------------------------------------------------------------- /media/images/preview_query_results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ashish10alex/vscode-dataform-tools/43f7477f8cdc8a3c3121b43418728a62b3cb0f94/media/images/preview_query_results.png -------------------------------------------------------------------------------- /media/images/schema_code_gen.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ashish10alex/vscode-dataform-tools/43f7477f8cdc8a3c3121b43418728a62b3cb0f94/media/images/schema_code_gen.gif -------------------------------------------------------------------------------- /media/images/sources_autocompletion.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ashish10alex/vscode-dataform-tools/43f7477f8cdc8a3c3121b43418728a62b3cb0f94/media/images/sources_autocompletion.gif -------------------------------------------------------------------------------- /media/images/tag_cost_estimator.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ashish10alex/vscode-dataform-tools/43f7477f8cdc8a3c3121b43418728a62b3cb0f94/media/images/tag_cost_estimator.png -------------------------------------------------------------------------------- /media/js/deps/highlightjs-copy/highlightjs-copy.min.css: -------------------------------------------------------------------------------- 1 | .hljs-copy-wrapper{position:relative;overflow:hidden;transform:translateZ(0)}.hljs-copy-container{--hljs-theme-padding:16px;position:absolute;top:0;right:0;transition:transform 200ms ease-out}.hljs-copy-button{position:relative;margin:calc(var(--hljs-theme-padding) / 2);width:calc(16px + var(--hljs-theme-padding));height:calc(16px + var(--hljs-theme-padding));font-size:.8125rem;text-indent:-9999px;color:var(--hljs-theme-color);border-radius:.25rem;border:1px solid;border-color:color-mix(in srgb,var(--hljs-theme-color),transparent 80%);background-color:var(--hljs-theme-background);transition:background-color 200ms ease;overflow:hidden}.hljs-copy-button:not([data-copied="true"])::before{content:"";width:1rem;height:1rem;top:50%;left:50%;transform:translate(-50%,-50%);position:absolute;background-color:currentColor;mask:url('data:image/svg+xml;utf-8,');mask-repeat:no-repeat;mask-size:contain;mask-position:center center}.hljs-copy-button:hover{background-color:color-mix(in srgb,var(--hljs-theme-color),transparent 90%)}.hljs-copy-button:active{border-color:color-mix(in srgb,var(--hljs-theme-color),transparent 60%)}.hljs-copy-button[data-copied="true"]{text-indent:0;width:auto}.hljs-copy-container[data-autohide="true"]{transform:translateX(calc(100% + 1.125em))}.hljs-copy-wrapper:focus-within .hljs-copy-container{transition:none;transform:translateX(0)}.hljs-copy-wrapper:hover .hljs-copy-container{transform:translateX(0)}@media(prefers-reduced-motion){.hljs-copy-button{transition:none}}.hljs-copy-alert{clip:rect(0 0 0 0);clip-path:inset(50%);height:1px;overflow:hidden;position:absolute;white-space:nowrap;width:1px} 2 | -------------------------------------------------------------------------------- /media/js/deps/highlightjs-copy/highlightjs-copy.min.js: -------------------------------------------------------------------------------- 1 | class CopyButtonPlugin{constructor(options={}){this.hook=options.hook;this.callback=options.callback;this.lang=options.lang||document.documentElement.lang||"en";this.autohide=typeof options.autohide!=="undefined"?options.autohide:true}"after:highlightElement"({el,text}){if(el.parentElement.querySelector(".hljs-copy-button"))return;let{hook,callback,lang,autohide}=this;let container=Object.assign(document.createElement("div"),{className:"hljs-copy-container"});container.dataset.autohide=autohide;let button=Object.assign(document.createElement("button"),{innerHTML:locales[lang]?.[0]||"Copy",className:"hljs-copy-button"});button.dataset.copied=false;el.parentElement.classList.add("hljs-copy-wrapper");el.parentElement.appendChild(container);container.appendChild(button);container.style.setProperty("--hljs-theme-background",window.getComputedStyle(el).backgroundColor);container.style.setProperty("--hljs-theme-color",window.getComputedStyle(el).color);container.style.setProperty("--hljs-theme-padding",window.getComputedStyle(el).padding);button.onclick=function(){if(!navigator.clipboard)return;let newText=text;if(hook&&typeof hook==="function"){newText=hook(text,el)||text}navigator.clipboard.writeText(newText).then(function(){button.innerHTML=locales[lang]?.[1]||"Copied!";button.dataset.copied=true;let alert=Object.assign(document.createElement("div"),{role:"status",className:"hljs-copy-alert",innerHTML:locales[lang]?.[2]||"Copied to clipboard"});el.parentElement.appendChild(alert);setTimeout(()=>{button.innerHTML=locales[lang]?.[0]||"Copy";button.dataset.copied=false;el.parentElement.removeChild(alert);alert=null},2e3)}).then(function(){if(typeof callback==="function")return callback(newText,el)})}}}if(typeof module!="undefined"){module.exports=CopyButtonPlugin}const locales={en:["Copy","Copied!","Copied to clipboard"],es:["Copiar","¡Copiado!","Copiado al portapapeles"],"pt-BR":["Copiar","Copiado!","Copiado para a área de transferência"],fr:["Copier","Copié !","Copié dans le presse-papier"],de:["Kopieren","Kopiert!","In die Zwischenablage kopiert"],ja:["コピー","コピーしました！","クリップボードにコピーしました"],ko:["복사","복사됨!","클립보드에 복사됨"],ru:["Копировать","Скопировано!","Скопировано в буфер обмена"],zh:["复制","已复制!","已复制到剪贴板"],"zh-tw":["複製","已複製!","已複製到剪貼簿"]}; -------------------------------------------------------------------------------- /media/js/sidePanel.js: -------------------------------------------------------------------------------- 1 | function removeLoadingMessage(errorMessage) { 2 | var loadingMessage = document.getElementById("loadingMessage"); 3 | if (loadingMessage && !errorMessage) { 4 | loadingMessage.parentNode.removeChild(loadingMessage); 5 | } else if (errorMessage){ 6 | loadingMessage.innerHTML = `

${errorMessage}

`; 7 | } 8 | } 9 | 10 | const getUrlToNavigateToTableInBigQuery = (gcpProjectId, datasetId, tableName) => { 11 | return `https://console.cloud.google.com/bigquery?project=${gcpProjectId}&ws=!1m5!1m4!4m3!1s${gcpProjectId}!2s${datasetId}!3s${tableName}`; 12 | }; 13 | 14 | function showCurrFileMetadataInSideBar(tables) { 15 | if (!tables) { 16 | return; 17 | } 18 | 19 | //clear the previous tags from nwDiv element 20 | if (document.getElementById('newDiv')) { 21 | document.getElementById('newDiv').remove(); 22 | } 23 | 24 | const newDiv = document.createElement('div'); 25 | newDiv.id = 'newDiv'; 26 | 27 | fullTableIds = []; 28 | 29 | const tableHeader = document.createElement('h3'); 30 | tableHeader.innerHTML = "
Table / view Name
"; 31 | 32 | const tagHeader = document.createElement('h3'); 33 | tagHeader.innerHTML = "
Tags
"; 34 | 35 | const tagsList = document.createElement('ul'); // Create an unordered list 36 | let uniqueTags = []; 37 | for (let i = 0; i < tables.length; i++) { 38 | let tags = tables[i].tags; 39 | if (tags) { 40 | for (let j = 0; j < tags.length; j++) { 41 | if (!uniqueTags.includes(tags[j])) { 42 | const li = document.createElement('li'); 43 | li.textContent = tags[j]; 44 | tagsList.appendChild(li); 45 | uniqueTags.push(tags[j]); 46 | } 47 | } 48 | } 49 | } 50 | 51 | let tableTarget = tables[0]?.target; 52 | let tablelinkWtName = document.createElement('a'); 53 | if (tableTarget){ 54 | tablelinkWtName.href = getUrlToNavigateToTableInBigQuery(tableTarget.database, tableTarget.schema, tableTarget.name); 55 | tablelinkWtName.textContent = `${tableTarget.database}.${tableTarget.schema}.${tableTarget.name}`; 56 | } 57 | 58 | 59 | const targetsHeader = document.createElement('h3'); 60 | 61 | targetsHeader.innerHTML = "
Dependencies
"; 62 | 63 | const dependencyList = document.createElement('ul'); // Create an unordered list 64 | for (let i = 0; i < tables.length; i++) { 65 | let tableTargets = tables[i]?.dependencyTargets; 66 | if (!tableTargets){ 67 | continue; 68 | } 69 | for (let j = 0; j < tableTargets.length; j++) { 70 | fullTableId = `${tableTargets[j].database}.${tableTargets[j].schema}.${tableTargets[j].name}`; 71 | fullTableIds.push(fullTableId); 72 | 73 | // Create a list item 74 | const li = document.createElement('li'); 75 | 76 | // Create an anchor element 77 | const link = document.createElement('a'); 78 | link.href = getUrlToNavigateToTableInBigQuery(tableTargets[j].database, tableTargets[j].schema, tableTargets[j].name); 79 | link.textContent = fullTableId; 80 | 81 | // Append the link to the list item 82 | li.appendChild(link); 83 | 84 | // Append the list item to the unordered list 85 | dependencyList.appendChild(li); 86 | } 87 | } 88 | 89 | 90 | 91 | newDiv.appendChild(tableHeader); 92 | if (tableTarget){ 93 | newDiv.appendChild(tablelinkWtName); 94 | } 95 | newDiv.appendChild(tagHeader); 96 | newDiv.appendChild(tagsList); 97 | newDiv.appendChild(targetsHeader); 98 | newDiv.appendChild(dependencyList); 99 | 100 | // Append newDiv to the body 101 | document.body.appendChild(newDiv); 102 | 103 | 104 | } 105 | 106 | window.addEventListener('message', event => { 107 | const message = event.data; 108 | let errorMessage = message.errorMessage; 109 | if(errorMessage){ 110 | removeLoadingMessage(errorMessage); 111 | return; 112 | } 113 | let tables = message.currFileMetadata.fileMetadata.tables; 114 | showCurrFileMetadataInSideBar(tables); 115 | }); 116 | 117 | -------------------------------------------------------------------------------- /postcss.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: { 3 | 'tailwindcss/nesting': {}, 4 | tailwindcss: {}, 5 | autoprefixer: {}, 6 | }, 7 | }; -------------------------------------------------------------------------------- /snippets/README.md: -------------------------------------------------------------------------------- 1 | # BigQuery snippets & Hover docs 2 | 3 | BigQuery snippets are borrowed from [vscode-language-sql-bigquery](https://github.com/shinichi-takii/vscode-language-sql-bigquery) VSCode extension. Full credit goes to the author @shinichi-takii. 4 | This plugin adds additional descriptions to some commonly used functions so that we can use it to show on hover docs. 5 | -------------------------------------------------------------------------------- /src/bigqueryClient.ts: -------------------------------------------------------------------------------- 1 | import * as vscode from 'vscode'; 2 | import { BigQuery } from '@google-cloud/bigquery'; 3 | 4 | let bigquery: BigQuery | undefined; 5 | let authenticationCheckInterval: NodeJS.Timeout | undefined; 6 | let lastAuthCheck: number = 0; 7 | let isAuthenticated: boolean = false; 8 | 9 | export async function createBigQueryClient() { 10 | try { 11 | const projectId = vscode.workspace.getConfiguration('vscode-dataform-tools').get('gcpProjectId'); 12 | // default state will be projectId as null and the projectId will be inferred from what the user has set using gcloud cli 13 | // @ts-ignore 14 | bigquery = new BigQuery({ projectId }); 15 | await verifyAuthentication(); 16 | vscode.window.showInformationMessage('BigQuery client created successfully.'); 17 | } catch (error) { 18 | bigquery = undefined; 19 | isAuthenticated = false; 20 | vscode.window.showErrorMessage(`Error creating BigQuery client: ${error}`); 21 | } 22 | } 23 | 24 | async function verifyAuthentication() { 25 | try { 26 | if (!bigquery) { 27 | throw new Error('BigQuery client not initialized'); 28 | } 29 | await bigquery.query('SELECT 1'); 30 | isAuthenticated = true; 31 | lastAuthCheck = Date.now(); 32 | } catch (error) { 33 | isAuthenticated = false; 34 | throw error; 35 | } 36 | } 37 | 38 | export async function checkAuthentication() { 39 | if (!bigquery || !isAuthenticated) { 40 | await createBigQueryClient(); 41 | return; 42 | } 43 | 44 | const useIntervalCheck = vscode.workspace.getConfiguration('vscode-dataform-tools').get('bigqueryAuthenticationCheck', true); 45 | 46 | if (useIntervalCheck) { 47 | const timeSinceLastCheck = Date.now() - lastAuthCheck; 48 | if (timeSinceLastCheck > 55 * 60 * 1000) { // 55 minutes in milliseconds 49 | try { 50 | await verifyAuthentication(); 51 | } catch (error) { 52 | vscode.window.showWarningMessage('BigQuery authentication expired. Recreating client...'); 53 | await createBigQueryClient(); 54 | } 55 | } 56 | } 57 | } 58 | 59 | export function getBigQueryClient(): BigQuery | undefined { 60 | return isAuthenticated ? bigquery : undefined; 61 | } 62 | 63 | export function setAuthenticationCheckInterval() { 64 | const useIntervalCheck = vscode.workspace.getConfiguration('vscode-dataform-tools').get('bigqueryAuthenticationCheck', true); 65 | 66 | clearAuthenticationCheckInterval(); 67 | 68 | if (useIntervalCheck) { 69 | authenticationCheckInterval = setInterval(async () => { 70 | await checkAuthentication(); 71 | }, 60 * 60 * 1000); // Check every hour 72 | } 73 | } 74 | 75 | export function clearAuthenticationCheckInterval() { 76 | if (authenticationCheckInterval) { 77 | clearInterval(authenticationCheckInterval); 78 | authenticationCheckInterval = undefined; 79 | } 80 | } 81 | 82 | export async function handleBigQueryError(error: any): Promise { 83 | if (error?.message?.includes('authentication')) { 84 | vscode.window.showWarningMessage('BigQuery authentication error. Recreating client...'); 85 | await createBigQueryClient(); 86 | } 87 | throw error; 88 | } -------------------------------------------------------------------------------- /src/bigqueryDryRun.ts: -------------------------------------------------------------------------------- 1 | import * as vscode from 'vscode'; 2 | import { getBigQueryClient, checkAuthentication, handleBigQueryError } from './bigqueryClient'; 3 | import { bigQueryDryRunCostOneGbByCurrency } from './constants'; 4 | import { BigQueryDryRunResponse, LastModifiedTimeMeta, SupportedCurrency, Target } from './types'; 5 | 6 | export function getLineAndColumnNumberFromErrorMessage(errorMessage: string) { 7 | //e.g. error 'Unrecognized name: SSY_LOC_ID; Did you mean ASSY_LOC_ID? at [65:7]' 8 | let lineAndColumn = errorMessage.match(/\[(\d+):(\d+)\]/); 9 | if (lineAndColumn) { 10 | return { 11 | line: parseInt(lineAndColumn[1]), 12 | column: parseInt(lineAndColumn[2]) 13 | }; 14 | } 15 | return { 16 | line: 0, 17 | column: 0 18 | }; 19 | } 20 | 21 | export async function queryDryRun(query: string): Promise { 22 | if (query === "" || !query) { 23 | return { 24 | schema: undefined, 25 | location: undefined, 26 | statistics: { totalBytesProcessed: 0 }, 27 | error: { hasError: false, message: "" } 28 | }; 29 | } 30 | 31 | await checkAuthentication(); 32 | 33 | const bigqueryClient = getBigQueryClient(); 34 | if (!bigqueryClient) { 35 | return { 36 | schema: undefined, 37 | location: undefined, 38 | statistics: { totalBytesProcessed: 0 }, 39 | error: { hasError: true, message: "BigQuery client not available." } 40 | }; 41 | } 42 | 43 | // For all options, see https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/query 44 | /* 45 | const options = { 46 | query: query, 47 | Location must match that of the dataset(s) referenced in the query. 48 | location: '', 49 | dryRun: true, 50 | }; 51 | */ 52 | 53 | let currencyFoDryRunCost: SupportedCurrency | undefined = vscode.workspace.getConfiguration('vscode-dataform-tools').get('currencyFoDryRunCost'); 54 | if (!currencyFoDryRunCost) { 55 | currencyFoDryRunCost = "USD" as SupportedCurrency; 56 | } 57 | try { 58 | const [job] = await bigqueryClient.createQueryJob({ 59 | query, 60 | dryRun: true 61 | }); 62 | 63 | const totalBytesProcessed = Number(parseFloat(job.metadata.statistics.totalBytesProcessed)); 64 | const cost = Number((totalBytesProcessed) / 10 ** 9) * bigQueryDryRunCostOneGbByCurrency[currencyFoDryRunCost]; 65 | 66 | return { 67 | schema: job.metadata.statistics.query.schema, 68 | location: job.metadata.jobReference.location, 69 | statistics: { 70 | totalBytesProcessed: totalBytesProcessed, 71 | cost: { 72 | currency: currencyFoDryRunCost, 73 | value: cost 74 | }, 75 | statementType: job.metadata.statistics.query.statementType, 76 | totalBytesProcessedAccuracy: job.metadata.statistics.query.totalBytesProcessedAccuracy 77 | }, 78 | error: { hasError: false, message: "" } 79 | }; 80 | } catch (error: any) { 81 | try { 82 | await handleBigQueryError(error); 83 | return await queryDryRun(query); 84 | } catch (finalError: any) { 85 | const errorLocation = getLineAndColumnNumberFromErrorMessage(finalError.message); 86 | return { 87 | schema: undefined, 88 | location: undefined, 89 | statistics: { 90 | totalBytesProcessed: 0, 91 | cost: { 92 | currency: currencyFoDryRunCost, 93 | value: 0 94 | } 95 | }, 96 | error: { hasError: true, message: finalError.message, location: errorLocation } 97 | }; 98 | } 99 | } 100 | } 101 | 102 | 103 | function formatTimestamp(lastModifiedTime:Date) { 104 | return lastModifiedTime.toLocaleString('en-US', { 105 | month: 'short', 106 | day: 'numeric', 107 | year: 'numeric', 108 | hour: '2-digit', 109 | minute: '2-digit', 110 | second: '2-digit', 111 | hour12: true, 112 | timeZone: 'UTC' 113 | }) + ' UTC'; 114 | } 115 | 116 | function isModelWasUpdatedToday(lastModifiedTime:Date) { 117 | const today = new Date(); 118 | return lastModifiedTime.toDateString() === today.toDateString(); 119 | } 120 | 121 | 122 | export async function getModelLastModifiedTime(targetTablesOrViews: Target[]): Promise { 123 | const bigqueryClient = getBigQueryClient(); 124 | if (!bigqueryClient) { 125 | return undefined; 126 | } 127 | let lastModifiedTimeMeta: LastModifiedTimeMeta = []; 128 | 129 | for (const targetTableOrView of targetTablesOrViews) { 130 | const projectId = targetTableOrView.database; 131 | const datasetId = targetTableOrView.schema; 132 | const tableId = targetTableOrView.name; 133 | 134 | try { 135 | const [table] = await bigqueryClient.dataset(datasetId, { projectId }).table(tableId).get(); 136 | let lastModifiedTime = table?.metadata?.lastModifiedTime; 137 | lastModifiedTime = new Date(parseInt(lastModifiedTime)); 138 | const formattedLastModifiedTime = formatTimestamp(lastModifiedTime); 139 | const modelWasUpdatedToday = isModelWasUpdatedToday(lastModifiedTime); 140 | 141 | lastModifiedTimeMeta.push({ 142 | lastModifiedTime: formattedLastModifiedTime, 143 | modelWasUpdatedToday : modelWasUpdatedToday, 144 | error: { message: undefined } 145 | }); 146 | } catch (error: any) { 147 | lastModifiedTimeMeta.push({ 148 | lastModifiedTime: undefined, 149 | modelWasUpdatedToday: undefined, 150 | error: { 151 | message: `Could not retrieve lastModifiedTime for ${projectId}.${datasetId}.${tableId}` 152 | } 153 | }); 154 | } 155 | } 156 | return lastModifiedTimeMeta; 157 | } 158 | 159 | -------------------------------------------------------------------------------- /src/codeActionProvider.ts: -------------------------------------------------------------------------------- 1 | import * as vscode from 'vscode'; 2 | 3 | /** 4 | Suggestion if provided from dry is expected to along the lines of 5 | 6 | `googleapi: Error 400: Unrecognized name: MODELID; Did you mean MODEL_ID? at [27:28], invalidQuery` 7 | 8 | From the above string the function attempts to extract the suggestion which we assumed based on observations to be separated by ";" 9 | followed by `Did you mean **fix**? at [lineNumber:columnNumber]` 10 | */ 11 | function extractFixFromDiagnosticMessage(diagnosticMessage: string) { 12 | const diagnosticSuggestion = diagnosticMessage.split(';')[1]; 13 | 14 | if (!diagnosticSuggestion) { 15 | return null; 16 | } 17 | 18 | const regex = /Did you mean (\w+)\?/; 19 | const match = diagnosticSuggestion.match(regex); 20 | const fix = match ? match[1] : null; 21 | return fix; 22 | } 23 | 24 | 25 | export let dataformCodeActionProviderDisposable = () => vscode.languages.registerCodeActionsProvider('sqlx', { 26 | provideCodeActions(document: vscode.TextDocument, _: vscode.Range, context) { 27 | const diagnostics = context.diagnostics.filter(diag => diag.severity === vscode.DiagnosticSeverity.Error); 28 | if (diagnostics.length === 0) { 29 | return; 30 | } 31 | 32 | const codeActions = diagnostics.map(diagnostic => { 33 | const fix = extractFixFromDiagnosticMessage(diagnostic.message); 34 | if (fix === null || fix === undefined){ 35 | return new vscode.CodeAction(``, vscode.CodeActionKind.QuickFix); 36 | } 37 | const fixAction = new vscode.CodeAction(`Replace with ${fix}`, vscode.CodeActionKind.QuickFix); 38 | fixAction.command = { 39 | command: 'vscode-dataform-tools.fixError', 40 | title: 'Apply dry run suggestion', 41 | tooltip: 'Apply dry run suggestion', 42 | arguments: [document, diagnostic.range, diagnostic.message] 43 | }; 44 | fixAction.diagnostics = [diagnostic]; 45 | fixAction.isPreferred = true; 46 | return fixAction; 47 | }); 48 | 49 | return codeActions; 50 | } 51 | }, { 52 | providedCodeActionKinds: [vscode.CodeActionKind.QuickFix] 53 | }); 54 | 55 | 56 | export let applyCodeActionUsingDiagnosticMessage = (range: vscode.Range, diagnosticMessage: string) => { 57 | const editor = vscode.window.activeTextEditor; 58 | if (!editor) { 59 | return; 60 | } 61 | 62 | // get the position; i.e line number and start index of the error 63 | // needs to be offset by one based on testing 64 | const position = range.start; 65 | const adjustedPosition = new vscode.Position(position.line, position.character - 1); 66 | 67 | const fix = extractFixFromDiagnosticMessage(diagnosticMessage); 68 | if (fix === null){ 69 | return; 70 | } 71 | 72 | // apply the fix by only replacing the word that is incorrect 73 | editor.edit(async editBuilder => { 74 | let wordEndPosition = editor.document.getWordRangeAtPosition(adjustedPosition)?.end; 75 | if (wordEndPosition) { 76 | let myRange = new vscode.Range(adjustedPosition, wordEndPosition); 77 | editBuilder.replace(myRange, fix); 78 | } 79 | 80 | }); 81 | }; 82 | -------------------------------------------------------------------------------- /src/codeLensProvider.ts: -------------------------------------------------------------------------------- 1 | import * as vscode from 'vscode'; 2 | 3 | export class AssertionRunnerCodeLensProvider implements vscode.CodeLensProvider { 4 | async provideCodeLenses(document: vscode.TextDocument): Promise { 5 | const codeLenses: vscode.CodeLens[] = []; 6 | const assertionConfigTypeRegexExp = /type\s*:\s*(['"])assertion\1/; 7 | 8 | for (let i = 0; i < document.lineCount; i++) { 9 | const line = document.lineAt(i); 10 | if (line.text.includes('assertions')) { 11 | const range = new vscode.Range(i, 0, i, 0); 12 | const codeLens = new vscode.CodeLens(range, { 13 | title: '▶ Run assertions', 14 | command: 'vscode-dataform-tools.runAssertions', 15 | arguments: [document.uri, i] 16 | }); 17 | codeLenses.push(codeLens); 18 | } else if (assertionConfigTypeRegexExp.exec(line.text) !== null){ 19 | const range = new vscode.Range(i, 0, i, 0); 20 | const codeLens = new vscode.CodeLens(range, { 21 | title: '▶ Run assertion', 22 | command: 'vscode-dataform-tools.runQuery', 23 | arguments: [document.uri, i] 24 | }); 25 | codeLenses.push(codeLens); 26 | } 27 | } 28 | return codeLenses; 29 | } 30 | } 31 | 32 | export class TagsRunnerCodeLensProvider implements vscode.CodeLensProvider { 33 | async provideCodeLenses(document: vscode.TextDocument): Promise { 34 | const codeLenses: vscode.CodeLens[] = []; 35 | 36 | for (let i = 0; i < document.lineCount; i++) { 37 | const line = document.lineAt(i); 38 | if (line.text.includes('tags')) { 39 | const range = new vscode.Range(i, 0, i, 0); 40 | const codeLens = new vscode.CodeLens(range, { 41 | title: '▶ Run Tag', 42 | command: 'vscode-dataform-tools.runFilesTagsWtOptions', 43 | arguments: [document.uri, i] 44 | }); 45 | codeLenses.push(codeLens); 46 | } 47 | } 48 | return codeLenses; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/completions.ts: -------------------------------------------------------------------------------- 1 | import * as vscode from 'vscode'; 2 | import { SchemaMetadata } from './types'; 3 | 4 | export const sourcesAutoCompletionDisposable = () => vscode.languages.registerCompletionItemProvider( 5 | // NOTE: Could this be made more reusable, i.e. a function that takes in the trigger and the language 6 | /* 7 | you might need to set up the file association to use the auto-completion 8 | sql should be added as a file association for sqlx 9 | this will enable both sufficient syntax highlighting and auto-completion 10 | */ 11 | { language: 'sqlx', scheme: 'file' }, 12 | { 13 | provideCompletionItems(document, position) { 14 | 15 | const linePrefix = document.lineAt(position).text.substring(0, position.character); 16 | if (!linePrefix.endsWith('$')) { 17 | return undefined; 18 | } 19 | let sourceCompletionItem = (text: any) => { 20 | let item = new vscode.CompletionItem(text, vscode.CompletionItemKind.Field); 21 | item.range = new vscode.Range(position, position); 22 | return item; 23 | }; 24 | if (declarationsAndTargets.length === 0) { 25 | return undefined; 26 | } 27 | let sourceAutoCompletionPreference = vscode.workspace.getConfiguration('vscode-dataform-tools').get('sourceAutoCompletionPreference'); 28 | 29 | let sourceCompletionItems: vscode.CompletionItem[] = []; 30 | 31 | if (sourceAutoCompletionPreference === "${ref('table_name')}"){ 32 | declarationsAndTargets.forEach((source: string) => { 33 | source = `{ref("${source}")}`; 34 | sourceCompletionItems.push(sourceCompletionItem(source)); 35 | }); 36 | } else if (sourceAutoCompletionPreference === "${ref('dataset_name', 'table_name')}") { 37 | declarationsAndTargets.forEach((source: string) => { 38 | let [database, table] = source.split('.'); 39 | source = `{ref("${database}", "${table}")}`; 40 | sourceCompletionItems.push(sourceCompletionItem(source)); 41 | }); 42 | } 43 | else { 44 | declarationsAndTargets.forEach((source: string) => { 45 | let [database, table] = source.split('.'); 46 | source = `{ref({schema: "${database}", name: "${table}"})}`; 47 | sourceCompletionItems.push(sourceCompletionItem(source)); 48 | }); 49 | } 50 | return sourceCompletionItems; 51 | } 52 | }, 53 | '$' // trigger 54 | ); 55 | 56 | export const dependenciesAutoCompletionDisposable = () => vscode.languages.registerCompletionItemProvider( 57 | // NOTE: Could this be made more reusable, i.e. a function that takes in the trigger and the language 58 | { language: 'sqlx', scheme: 'file' }, 59 | { 60 | provideCompletionItems(document, position) { 61 | 62 | const linePrefix = document.lineAt(position).text.substring(0, position.character); 63 | if (!(linePrefix.includes('dependencies') && ( linePrefix.includes('"') || linePrefix.includes("'")))){ 64 | return undefined; 65 | } 66 | let sourceCompletionItem = (text: any) => { 67 | let item = new vscode.CompletionItem(text, vscode.CompletionItemKind.Field); 68 | item.range = new vscode.Range(position, position); 69 | return item; 70 | }; 71 | if (declarationsAndTargets.length === 0) { 72 | return undefined; 73 | } 74 | let sourceCompletionItems: vscode.CompletionItem[] = []; 75 | declarationsAndTargets.forEach((source: string) => { 76 | source = `${source}`; 77 | sourceCompletionItems.push(sourceCompletionItem(source)); 78 | }); 79 | return sourceCompletionItems; 80 | }, 81 | }, 82 | ...["'", '"'], 83 | ); 84 | 85 | export const tagsAutoCompletionDisposable = () => vscode.languages.registerCompletionItemProvider( 86 | // NOTE: Could this be made more reusable, i.e. a function that takes in the trigger and the language 87 | { language: 'sqlx', scheme: 'file' }, 88 | { 89 | provideCompletionItems(document, position) { 90 | 91 | const linePrefix = document.lineAt(position).text.substring(0, position.character); 92 | if (!(linePrefix.includes('tags') && ( linePrefix.includes('"') || linePrefix.includes("'")))){ 93 | return undefined; 94 | } 95 | let sourceCompletionItem = (text: any) => { 96 | let item = new vscode.CompletionItem(text, vscode.CompletionItemKind.Field); 97 | item.range = new vscode.Range(position, position); 98 | return item; 99 | }; 100 | if (dataformTags.length === 0) { 101 | return undefined; 102 | } 103 | let sourceCompletionItems: vscode.CompletionItem[] = []; 104 | dataformTags.forEach((source: string) => { 105 | source = `${source}`; 106 | sourceCompletionItems.push(sourceCompletionItem(source)); 107 | }); 108 | return sourceCompletionItems; 109 | }, 110 | }, 111 | ...["'", '"'], 112 | ); 113 | 114 | 115 | export const schemaAutoCompletionDisposable = () => vscode.languages.registerCompletionItemProvider( 116 | "*", 117 | { 118 | async provideCompletionItems() { 119 | const completionItems = schemaAutoCompletions.map((item: SchemaMetadata) => { 120 | const completionItem = new vscode.CompletionItem(`${item.name}`); 121 | completionItem.kind = vscode.CompletionItemKind.Variable; 122 | completionItem.detail = `${item.metadata.fullTableId}`; 123 | completionItem.sortText = '0'; // put it ahead of other completion objects 124 | const markdownString = new vscode.MarkdownString(`[ ${item.metadata.type} ] \n\n ${item.metadata.description}`); 125 | markdownString.isTrusted = true; 126 | markdownString.supportHtml = true; 127 | completionItem.documentation = markdownString; 128 | return completionItem; 129 | }); 130 | return completionItems; 131 | } 132 | } 133 | ); 134 | 135 | -------------------------------------------------------------------------------- /src/constants.ts: -------------------------------------------------------------------------------- 1 | import os from 'os'; 2 | import path from 'path'; 3 | import { SupportedCurrency as SupportedCurrencies } from './types'; 4 | import { getWorkspaceFolder } from './utils'; 5 | 6 | const tempDir = os.tmpdir(); 7 | export const sqlFileToFormatPath = path.join(tempDir, "format.sql"); 8 | export const executablesToCheck = ['dataform', 'gcloud']; 9 | export const tableQueryOffset = 2; 10 | export const incrementalTableOffset = 1; 11 | export const assertionQueryOffset = 4; 12 | export const windowsDataformCliNotAvailableErrorMessage = "'dataform.cmd' is not recognized as an internal or external command"; 13 | export const linuxDataformCliNotAvailableErrorMessage = "dataform: command not found"; 14 | export const costInPoundsForOneGb = 0.005; 15 | export const bigQuerytimeoutMs = 20000; 16 | 17 | export const bigQueryDryRunCostOneGbByCurrency: Record = { 18 | "USD": 0.005, 19 | "EUR": 0.0046, 20 | "GBP": 0.0039, 21 | "JPY": 0.56, 22 | "CAD": 0.0067, 23 | "AUD": 0.0075, 24 | "INR": 0.41, 25 | }; 26 | 27 | export const currencySymbolMapping = { 28 | "USD": "$", 29 | "EUR": "€", 30 | "GBP": "£", 31 | "JPY": "¥", 32 | "CAD": "C$", 33 | "AUD": "A$", 34 | "INR": "₹", 35 | }; 36 | 37 | export async function getFileNotFoundErrorMessageForWebView(relativeFilePath:string){ 38 | 39 | if(!workspaceFolder){ 40 | workspaceFolder = await getWorkspaceFolder(); 41 | } 42 | 43 | // Create a single HTML string with the error message 44 | const errorMessage = ` 45 |

46 |

File "${relativeFilePath}" not found in Dataform compiled json with workspace folder "${workspaceFolder}"

47 |

Ignore the error if the file you are in is not expected to produce a sql output

48 |

Possible resolution/fix(s):

49 |
    50 |
  1. If you are using multi-root workspace, select the correct workspace folder for the file by clicking here
    2. 51 |
  2. Check if running "dataform compile" throws an error
    3. 52 |
  3. 53 | Check if case of the file has been changed and the case does not match what is being shown in the error message above, 54 | this is a known issue with VSCode #123660. 55 | A workaround for this is: 56 |
      57 |
    1. Change the filename to something arbitrary and save it
      2. 58 |
    2. Reload the VSCode window
      3. 59 |
    3. Change the file name to the case you want and recompile Dataform by saving the file
      4. 60 |
    61 |
    4. 62 |
63 |
64 | `; 65 | 66 | return errorMessage; 67 | } -------------------------------------------------------------------------------- /src/costEstimator.ts: -------------------------------------------------------------------------------- 1 | import { queryDryRun } from "./bigqueryDryRun"; 2 | import * as vscode from 'vscode'; 3 | import { Assertion, DataformCompiledJson, TagDryRunStats, TagDryRunStatsMeta, Operation, Table, Target, SupportedCurrency } from "./types"; 4 | 5 | const createFullTargetName = (target: Target) => { 6 | return `${target.database}.${target.schema}.${target.name}`; 7 | }; 8 | 9 | export function handleSemicolonInQuery(query: string){ 10 | query = query.trimStart(); 11 | const queryWithSemicolon = /;\s*$/.test(query); 12 | if(!queryWithSemicolon && query !== "" ){ 13 | query = query + ";"; 14 | } 15 | return query; 16 | } 17 | 18 | 19 | async function getModelDryRunStats(filteredModels: Table[] | Operation[] | Assertion[], type:string|undefined): Promise>{ 20 | const modelPromises = filteredModels.map(async (curModel) => { 21 | let fullQuery = ""; 22 | let preOpsQuery = curModel.preOps ? curModel.preOps.join("\n") : ""; 23 | preOpsQuery = handleSemicolonInQuery(preOpsQuery); 24 | 25 | let incrementalPreOpsQuery = curModel.incrementalPreOps ? curModel.incrementalPreOps.join("\n") : ""; 26 | incrementalPreOpsQuery = handleSemicolonInQuery(incrementalPreOpsQuery); 27 | 28 | let incrementalQuery = curModel.incrementalQuery || ""; 29 | incrementalQuery = handleSemicolonInQuery(incrementalQuery); 30 | 31 | if (curModel.type === "view") { 32 | fullQuery = preOpsQuery + 'CREATE OR REPLACE VIEW ' + createFullTargetName(curModel.target) + ' AS ' + curModel.query; 33 | } else if (curModel.type === "table" && (curModel?.bigquery?.partitionBy || curModel?.bigquery?.clusterBy)) { 34 | fullQuery = preOpsQuery + curModel.query; 35 | } else if (curModel.type === "table") { 36 | fullQuery = preOpsQuery + 'CREATE OR REPLACE TABLE ' + createFullTargetName(curModel.target) + ' AS ' + curModel.query; 37 | } else if (curModel.type === "incremental" && (curModel?.bigquery?.partitionBy || curModel?.bigquery?.clusterBy)) { 38 | fullQuery = incrementalPreOpsQuery + incrementalQuery; 39 | } else if (curModel.type === "incremental") { 40 | fullQuery = incrementalPreOpsQuery + 'CREATE OR REPLACE TABLE ' + createFullTargetName(curModel.target) + ' AS ' + incrementalQuery; 41 | } else if (type === "assertion") { 42 | fullQuery = curModel.query || ""; 43 | } else if (type === "operation") { 44 | // @ts-ignore -- adding this to avoid type error hassle, we can revisit this later 45 | fullQuery = curModel.queries.join("\n") + ";"; 46 | } 47 | 48 | const dryRunOutput = await queryDryRun(fullQuery); 49 | const costOfRunningModel = dryRunOutput?.statistics?.cost?.value || 0; 50 | const totalGBProcessed = ((dryRunOutput?.statistics?.totalBytesProcessed) / (10 ** 9)).toFixed(3); 51 | const statementType = dryRunOutput?.statistics?.statementType; 52 | const totalBytesProcessedAccuracy = dryRunOutput?.statistics?.totalBytesProcessedAccuracy; 53 | const error = dryRunOutput?.error; 54 | 55 | return { 56 | type: curModel.type || type || "", 57 | targetName: createFullTargetName(curModel.target), 58 | costOfRunningModel: costOfRunningModel, 59 | currency: dryRunOutput?.statistics?.cost?.currency as SupportedCurrency, 60 | totalGBProcessed: totalGBProcessed || "0.000", 61 | totalBytesProcessedAccuracy: totalBytesProcessedAccuracy, 62 | statementType: statementType, 63 | error: error.message 64 | }; 65 | }); 66 | const results = await Promise.all(modelPromises); 67 | return results; 68 | } 69 | 70 | export async function costEstimator(jsonData: DataformCompiledJson, selectedTag:string): Promise { 71 | try{ 72 | const testQueryToCheckUserAccess = "SELECT 1;"; 73 | const testDryRunOutput = await queryDryRun(testQueryToCheckUserAccess); 74 | if(testDryRunOutput.error.hasError){ 75 | return { 76 | tagDryRunStatsList: undefined, 77 | error: testDryRunOutput.error.message, 78 | }; 79 | } 80 | 81 | const filteredTables = jsonData.tables.filter(table => table?.tags?.includes(selectedTag)); 82 | const filteredOperations = jsonData.operations.filter(operation => operation?.tags?.includes(selectedTag)); 83 | const filteredAssertions = jsonData.assertions.filter(assertion => assertion?.tags?.includes(selectedTag)); 84 | 85 | let allResults = []; 86 | 87 | if(filteredTables?.length > 0){ 88 | const tableResults = await getModelDryRunStats(filteredTables, undefined); 89 | allResults.push(...tableResults); 90 | } 91 | 92 | if(filteredAssertions?.length > 0){ 93 | const assertionResults = await getModelDryRunStats(filteredAssertions, "assertion"); 94 | allResults.push(...assertionResults); 95 | } 96 | 97 | if(filteredOperations?.length > 0){ 98 | const operationResults = await getModelDryRunStats(filteredOperations, "operation"); 99 | allResults.push(...operationResults); 100 | } 101 | return { 102 | tagDryRunStatsList: allResults, 103 | error: undefined, 104 | }; 105 | }catch(error:any){ 106 | //TODO: return error and show in the web view ? 107 | vscode.window.showErrorMessage(error.message); 108 | return undefined; 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /src/dependancyTree.ts: -------------------------------------------------------------------------------- 1 | import { Assertion, Declarations, DeclarationsLegendMetadata, DependancyTreeMetadata, Operation, Table, Target } from "./types"; 2 | import { getWorkspaceFolder, runCompilation } from "./utils"; 3 | 4 | function populateDependancyTree(type: string, structs: Table[] | Operation[] | Assertion[] | Declarations[], dependancyTreeMetadata: DependancyTreeMetadata[], schemaDict: any, schemaIdx: number) { 5 | let declarationsLegendMetadata: DeclarationsLegendMetadata[] = []; 6 | let addedSchemas: string[] = []; 7 | let schemaIdxTracker = 0; 8 | 9 | //_schema_idx 0 is reserved for nodes generated by Dataform pipeline 10 | declarationsLegendMetadata.push({ 11 | "_schema": "dataform", 12 | "_schema_idx": 0 13 | }); 14 | 15 | structs.forEach((struct) => { 16 | let tableName = `${struct.target.database}.${struct.target.schema}.${struct.target.name}`; 17 | let tags = struct.tags; 18 | let fileName = struct.fileName; 19 | let schema = `${struct.target.schema}`; 20 | 21 | // NOTE: Only adding colors in web panel for tables declared in declarations 22 | if (type === "declarations") { 23 | if (schemaDict.hasOwnProperty(schema)) { 24 | schemaIdx = schemaDict[schema]; 25 | } else { 26 | schemaDict[schema] = schemaIdxTracker + 1; 27 | schemaIdxTracker += 1; 28 | schemaIdx = schemaIdxTracker; 29 | } 30 | } 31 | 32 | let dependancyTargets = struct?.dependencyTargets; 33 | 34 | let depedancyList: string[] = []; 35 | if (dependancyTargets) { 36 | dependancyTargets.forEach((dep: Target) => { 37 | let dependancyTableName = `${dep.database}.${dep.schema}.${dep.name}`; 38 | depedancyList.push(dependancyTableName); 39 | }); 40 | } 41 | 42 | if (depedancyList.length === 0) { 43 | dependancyTreeMetadata.push( 44 | { 45 | "_name": tableName, 46 | "_fileName": fileName, 47 | "_schema": schema, 48 | "_tags": tags, 49 | "_schema_idx": (struct.hasOwnProperty("type")) ? 0 : schemaIdx 50 | } 51 | ); 52 | } else { 53 | dependancyTreeMetadata.push( 54 | { 55 | "_name": tableName, 56 | "_fileName": fileName, 57 | "_schema": schema, 58 | "_deps": depedancyList, 59 | "_tags": tags, 60 | "_schema_idx": (struct.hasOwnProperty("type")) ? 0 : schemaIdx 61 | } 62 | ); 63 | } 64 | 65 | if (type === "declarations") { 66 | if (!addedSchemas.includes(schema)) { 67 | declarationsLegendMetadata.push({ 68 | "_schema": schema, 69 | "_schema_idx": schemaIdx 70 | }); 71 | addedSchemas.push(schema); 72 | } 73 | } 74 | }); 75 | return { "dependancyTreeMetadata": dependancyTreeMetadata, "schemaIdx": schemaIdx, "declarationsLegendMetadata": declarationsLegendMetadata }; 76 | } 77 | 78 | export async function generateDependancyTreeMetadata(): Promise<{ dependancyTreeMetadata: DependancyTreeMetadata[], declarationsLegendMetadata: DeclarationsLegendMetadata[] } | undefined> { 79 | let dependancyTreeMetadata: DependancyTreeMetadata[] = []; 80 | let schemaDict = {}; // used to keep track of unique schema names ( gcp dataset name ) already seen in the compiled json declarations 81 | let schemaIdx = 0; // used to assign a unique index to each unique schema name for color coding dataset in the web panel 82 | 83 | if (!CACHED_COMPILED_DATAFORM_JSON) { 84 | 85 | let workspaceFolder = await getWorkspaceFolder(); 86 | if (!workspaceFolder) { 87 | return; 88 | } 89 | 90 | let {dataformCompiledJson} = await runCompilation(workspaceFolder); // Takes ~1100ms 91 | if (dataformCompiledJson) { 92 | CACHED_COMPILED_DATAFORM_JSON = dataformCompiledJson; 93 | } 94 | } 95 | 96 | let output; 97 | if (!CACHED_COMPILED_DATAFORM_JSON) { 98 | return { "dependancyTreeMetadata": output ? output["dependancyTreeMetadata"] : dependancyTreeMetadata, "declarationsLegendMetadata": output ? output["declarationsLegendMetadata"] : [] }; 99 | } 100 | let tables = CACHED_COMPILED_DATAFORM_JSON.tables; 101 | let operations = CACHED_COMPILED_DATAFORM_JSON.operations; 102 | let assertions = CACHED_COMPILED_DATAFORM_JSON.assertions; 103 | let declarations = CACHED_COMPILED_DATAFORM_JSON.declarations; 104 | 105 | if (tables) { 106 | output = populateDependancyTree("tables", tables, dependancyTreeMetadata, schemaDict, schemaIdx); 107 | } 108 | if (operations) { 109 | output = populateDependancyTree("operations", operations, output ? output["dependancyTreeMetadata"] : dependancyTreeMetadata, schemaDict, output ? output["schemaIdx"] : schemaIdx); 110 | } 111 | if (assertions) { 112 | output = populateDependancyTree("assertions", assertions, output ? output["dependancyTreeMetadata"] : dependancyTreeMetadata, schemaDict, output ? output["schemaIdx"] : schemaIdx); 113 | } 114 | if (declarations) { 115 | output = populateDependancyTree("declarations", declarations, output ? output["dependancyTreeMetadata"] : dependancyTreeMetadata, schemaDict, output ? output["schemaIdx"] : schemaIdx); 116 | } 117 | return { "dependancyTreeMetadata": output ? output["dependancyTreeMetadata"] : dependancyTreeMetadata, "declarationsLegendMetadata": output ? output["declarationsLegendMetadata"] : [] }; 118 | } 119 | -------------------------------------------------------------------------------- /src/getLineageMetadata.ts: -------------------------------------------------------------------------------- 1 | import { Target } from "./types"; 2 | const {LineageClient} = require('@google-cloud/lineage').v1; 3 | 4 | export async function getLiniageMetadata(targetToSearch: Target, location:string) { 5 | const projectId = targetToSearch.database; 6 | const datasetId = targetToSearch.schema; 7 | const tableId = targetToSearch.name; 8 | 9 | const client = new LineageClient(); // TODO: This gets created everytime this func is called. Can we use same client for longer ? 10 | 11 | const request = { 12 | parent: `projects/${projectId}/locations/${location}`, 13 | source: { 14 | fullyQualifiedName: `bigquery:${projectId}.${datasetId}.${tableId}` 15 | }, 16 | // NOTE: target seems to get upstream. We might or might not need this 17 | //target: { 18 | // fullyQualifiedName: `bigquery:${projectId}.${datasetId}.${tableId}` 19 | //}, 20 | }; 21 | 22 | try { 23 | const [response] = await client.searchLinks(request); 24 | const prefix = "bigquery:"; 25 | const dependencies = response.map((link:any) => link.target.fullyQualifiedName.slice(prefix.length)); 26 | return { 27 | dependencies: dependencies, 28 | error: undefined, 29 | }; 30 | 31 | } catch (error:any) { 32 | return { 33 | dependencies: undefined, 34 | error: error.details, 35 | }; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/globals.d.ts: -------------------------------------------------------------------------------- 1 | import { Job } from "@google-cloud/bigquery"; 2 | import { CompiledQuerySchema, DataformCompiledJson, Metadata} from "./types"; 3 | import * as vscode from 'vscode'; 4 | 5 | declare global { 6 | var CACHED_COMPILED_DATAFORM_JSON: DataformCompiledJson | undefined; 7 | } 8 | 9 | declare global { 10 | var cdnLinks : { 11 | highlightJsCssUri: string; 12 | highlightJsUri: string; 13 | highlightJsLineNoExtUri: string; 14 | tabulatorUri: string; 15 | tabulatorDarkCssUri: string; 16 | tabulatorLightCssUri: string; 17 | highlightJsOneDarkThemeUri: string; 18 | highlightJsOneLightThemeUri: string; 19 | } 20 | } 21 | 22 | declare global { 23 | var declarationsAndTargets: string[] 24 | } 25 | 26 | declare global { 27 | var dataformTags: string[] 28 | } 29 | 30 | declare global { 31 | var isRunningOnWindows: boolean 32 | } 33 | 34 | declare global { 35 | var bigQueryJob: Job | undefined 36 | } 37 | 38 | declare global { 39 | var cancelBigQueryJobSignal: boolean 40 | } 41 | 42 | declare global { 43 | var queryLimit: number 44 | } 45 | 46 | declare global { 47 | var diagnosticCollection: vscode.DiagnosticCollection | undefined; 48 | } 49 | 50 | 51 | declare global { 52 | var compiledQuerySchema: CompiledQuerySchema | undefined; 53 | } 54 | 55 | declare global { 56 | var incrementalCheckBox: boolean 57 | } 58 | 59 | declare global { 60 | var schemaAutoCompletions: {name: string, metadata: Metadata }[]; 61 | } 62 | 63 | declare global { 64 | var activeEditorFileName: string | undefined; 65 | } 66 | 67 | declare global { 68 | var activeDocumentObj: any; 69 | } 70 | 71 | declare global { 72 | var workspaceFolder: string | undefined; 73 | } 74 | 75 | declare global { 76 | var bigQuerySnippetMetadata: { 77 | [key: string]: { 78 | prefix: string; 79 | body: string; 80 | description: string[]; 81 | }; 82 | }; 83 | } 84 | 85 | export {}; 86 | -------------------------------------------------------------------------------- /src/logger.ts: -------------------------------------------------------------------------------- 1 | import * as vscode from 'vscode'; 2 | 3 | class Logger { 4 | private outputChannel: vscode.OutputChannel; 5 | private enabled: boolean; 6 | 7 | constructor() { 8 | this.outputChannel = vscode.window.createOutputChannel('Dataform Tools'); 9 | this.enabled = false; 10 | } 11 | 12 | public initialize() { 13 | this.enabled = vscode.workspace.getConfiguration('vscode-dataform-tools').get('enableLogging') || false; 14 | if (this.enabled) { 15 | this.info('Logging initialized'); 16 | } 17 | } 18 | 19 | public info(message: string) { 20 | if (this.enabled) { 21 | const timeStamp = new Date().toISOString(); 22 | this.outputChannel.appendLine(`[${timeStamp}] INFO: ${message}`); 23 | } 24 | } 25 | 26 | public error(message: string, error?: any) { 27 | if (this.enabled) { 28 | const timeStamp = new Date().toISOString(); 29 | this.outputChannel.appendLine(`[${timeStamp}] ERROR: ${message}`); 30 | if (error) { 31 | this.outputChannel.appendLine(JSON.stringify(error, null, 2)); 32 | } 33 | } 34 | } 35 | 36 | public debug(message: string) { 37 | if (this.enabled) { 38 | const timeStamp = new Date().toISOString(); 39 | this.outputChannel.appendLine(`[${timeStamp}] DEBUG: ${message}`); 40 | } 41 | } 42 | 43 | public show() { 44 | this.outputChannel.show(); 45 | } 46 | 47 | public dispose() { 48 | this.outputChannel.dispose(); 49 | } 50 | } 51 | 52 | export const logger = new Logger(); -------------------------------------------------------------------------------- /src/previewQueryResults.ts: -------------------------------------------------------------------------------- 1 | import * as vscode from 'vscode'; 2 | import { getCurrentFileMetadata, handleSemicolonPrePostOps } from "./utils"; 3 | import { CustomViewProvider } from './views/register-query-results-panel'; 4 | import { QueryWtType } from './types'; 5 | 6 | export async function runQueryInPanel(queryWtType: QueryWtType, queryResultsViewProvider: CustomViewProvider) { 7 | if (!queryResultsViewProvider._view) { 8 | queryResultsViewProvider.focusWebview(queryWtType); 9 | } else { 10 | queryResultsViewProvider.updateContent(queryWtType); 11 | } 12 | } 13 | 14 | export async function previewQueryResults(queryResultsViewProvider: CustomViewProvider) { 15 | let curFileMeta = await getCurrentFileMetadata(false); 16 | if (!curFileMeta?.fileMetadata) { 17 | return; 18 | } 19 | 20 | let fileMetadata = handleSemicolonPrePostOps(curFileMeta.fileMetadata); 21 | 22 | let query = ""; 23 | if (fileMetadata.queryMeta.type === "assertion") { 24 | query = fileMetadata.queryMeta.assertionQuery; 25 | } else if (fileMetadata.queryMeta.type === "table" || fileMetadata.queryMeta.type === "view") { 26 | query = fileMetadata.queryMeta.preOpsQuery + fileMetadata.queryMeta.tableOrViewQuery; 27 | } else if (fileMetadata.queryMeta.type === "operations") { 28 | query = fileMetadata.queryMeta.preOpsQuery + fileMetadata.queryMeta.operationsQuery; 29 | } else if (fileMetadata.queryMeta.type === "incremental") { 30 | if (incrementalCheckBox === true){ 31 | query = fileMetadata.queryMeta.incrementalPreOpsQuery + fileMetadata.queryMeta.incrementalQuery; 32 | } else { 33 | query = fileMetadata.queryMeta.preOpsQuery + fileMetadata.queryMeta.nonIncrementalQuery; 34 | } 35 | } 36 | if (query === "") { 37 | vscode.window.showWarningMessage("No query to run"); 38 | return; 39 | } 40 | runQueryInPanel({query: query, type: fileMetadata.queryMeta.type}, queryResultsViewProvider); 41 | } 42 | -------------------------------------------------------------------------------- /src/renameProvider.ts: -------------------------------------------------------------------------------- 1 | import * as vscode from 'vscode'; 2 | 3 | export const renameProvider = vscode.languages.registerRenameProvider('sqlx', { 4 | provideRenameEdits( 5 | document: vscode.TextDocument, 6 | position: vscode.Position, 7 | newName: string, 8 | _: vscode.CancellationToken 9 | ) { 10 | const wordRange = document.getWordRangeAtPosition(position); 11 | if (!wordRange) { 12 | return null; 13 | } 14 | const oldName = document.getText(wordRange); 15 | const edit = new vscode.WorkspaceEdit(); 16 | 17 | // Find all occurrences in the document and replace 18 | const wordRegex = new RegExp(`\\b${oldName}\\b`, 'g'); 19 | for (let line = 0; line < document.lineCount; line++) { 20 | const lineText = document.lineAt(line).text; 21 | let match; 22 | while ((match = wordRegex.exec(lineText)) !== null) { 23 | const range = new vscode.Range( 24 | new vscode.Position(line, match.index), 25 | new vscode.Position(line, match.index + oldName.length) 26 | ); 27 | edit.replace(document.uri, range, newName); 28 | } 29 | } 30 | return edit; 31 | }, 32 | prepareRename( 33 | document: vscode.TextDocument, 34 | position: vscode.Position, 35 | _: vscode.CancellationToken 36 | ) { 37 | const wordRange = document.getWordRangeAtPosition(position); 38 | if (!wordRange) { 39 | throw new Error('No symbol at cursor to rename'); 40 | } 41 | return wordRange; 42 | } 43 | } 44 | ); 45 | -------------------------------------------------------------------------------- /src/runFiles.ts: -------------------------------------------------------------------------------- 1 | import * as vscode from 'vscode'; 2 | import { getDataformActionCmdFromActionList, getDataformCompilationTimeoutFromConfig, getFileNameFromDocument, getQueryMetaForCurrentFile, getVSCodeDocument, getWorkspaceFolder, runCommandInTerminal, runCompilation } from "./utils"; 3 | 4 | export async function runCurrentFile(includDependencies: boolean, includeDownstreamDependents: boolean, fullRefresh: boolean) { 5 | 6 | let document = getVSCodeDocument() || activeDocumentObj; 7 | if (!document) { 8 | return; 9 | } 10 | 11 | var result = getFileNameFromDocument(document, false); 12 | if (result.success === false) { 13 | return; 14 | //{ return {errors: {errorGettingFileNameFromDocument: result.error}}; } 15 | //TODO: should we return an error here ? 16 | } 17 | //@ts-ignore 18 | const [filename, relativeFilePath, extension] = result.value; 19 | let workspaceFolder = await getWorkspaceFolder(); 20 | if (!workspaceFolder) { 21 | return; 22 | } 23 | 24 | let dataformCompilationTimeoutVal = getDataformCompilationTimeoutFromConfig(); 25 | 26 | let currFileMetadata; 27 | let {dataformCompiledJson, errors} = await runCompilation(workspaceFolder); // Takes ~1100ms 28 | 29 | if(errors && errors.length > 0){ 30 | vscode.window.showErrorMessage("Error compiling Dataform. Run `dataform compile` to see more details"); 31 | return; 32 | } 33 | 34 | if (dataformCompiledJson) { 35 | currFileMetadata = await getQueryMetaForCurrentFile(relativeFilePath, dataformCompiledJson); 36 | } 37 | 38 | if (currFileMetadata) { 39 | let actionsList: string[] = currFileMetadata.tables.map(table => `${table.target.database}.${table.target.schema}.${table.target.name}`); 40 | 41 | let dataformActionCmd = ""; 42 | 43 | // create the dataform run command for the list of actions from actionsList 44 | dataformActionCmd = getDataformActionCmdFromActionList(actionsList, workspaceFolder, dataformCompilationTimeoutVal, includDependencies, includeDownstreamDependents, fullRefresh); 45 | runCommandInTerminal(dataformActionCmd); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/runFilesTagsWtOptions.ts: -------------------------------------------------------------------------------- 1 | import * as vscode from 'vscode'; 2 | import { getDataformCompilationTimeoutFromConfig, getMultipleFileSelection, getWorkspaceFolder, runCommandInTerminal, runMultipleFilesFromSelection } from './utils'; 3 | import { getMultipleTagsSelection, getRunTagsWtOptsCommand, runMultipleTagsFromSelection } from './runTag'; 4 | import { runCurrentFile } from './runFiles'; 5 | 6 | export async function runFilesTagsWtOptions() { 7 | const firstStageOptions = ["run current file", "run a tag", "run multiple files", "run multiple tags"]; 8 | const firstStageSelection = await vscode.window.showQuickPick(firstStageOptions, { 9 | placeHolder: 'Select an option' 10 | }); 11 | 12 | if (!firstStageSelection) { 13 | return; 14 | } 15 | 16 | let tagSelection: string | undefined; 17 | if (firstStageSelection === "run a tag") { 18 | const tagOptions = dataformTags; 19 | tagSelection = await vscode.window.showQuickPick(tagOptions, { 20 | placeHolder: 'Select a tag' 21 | }); 22 | } 23 | 24 | let multipleFileSelection: string | undefined; 25 | let workspaceFolder = await getWorkspaceFolder(); 26 | if (!workspaceFolder){ return; } 27 | if (firstStageSelection === "run multiple files"){ 28 | multipleFileSelection = await getMultipleFileSelection(workspaceFolder); 29 | } 30 | 31 | let multipleTagsSelection: string | undefined; 32 | if (firstStageSelection === "run multiple tags"){ 33 | multipleTagsSelection = await getMultipleTagsSelection(); 34 | } 35 | 36 | let secondStageOptions: string[]; 37 | if (firstStageSelection || tagSelection) { 38 | secondStageOptions = ["default", "include dependents", "include dependencies"]; 39 | } else { 40 | return; 41 | } 42 | 43 | const secondStageSelection = await vscode.window.showQuickPick(secondStageOptions, { 44 | placeHolder: `select run type` 45 | }); 46 | 47 | if (!secondStageSelection) { 48 | return; 49 | } 50 | 51 | let thirdStageOptions: string[]; 52 | if (firstStageSelection) { 53 | thirdStageOptions = ["no", "yes"]; 54 | } else { 55 | return; 56 | } 57 | 58 | const thirdStageSelection = await vscode.window.showQuickPick(thirdStageOptions, { 59 | placeHolder: `full refresh` 60 | }); 61 | 62 | if (!thirdStageSelection) { 63 | return; 64 | } 65 | 66 | let includeDependents = false; 67 | let includeDependencies = false; 68 | let fullRefresh = false; 69 | if (secondStageSelection === "include dependents") { 70 | includeDependents = true; 71 | } 72 | if (secondStageSelection === "include dependencies") { 73 | includeDependencies = true; 74 | } 75 | if (thirdStageSelection === "yes") { 76 | fullRefresh = true; 77 | } 78 | 79 | if (firstStageSelection === "run current file") { 80 | runCurrentFile(includeDependencies, includeDependents, fullRefresh); 81 | } else if (firstStageSelection === "run a tag") { 82 | if(!tagSelection){return;}; 83 | let defaultDataformCompileTime = getDataformCompilationTimeoutFromConfig(); 84 | let runTagsWtDepsCommand = getRunTagsWtOptsCommand(workspaceFolder, tagSelection, defaultDataformCompileTime, includeDependencies, includeDependents, fullRefresh); 85 | runCommandInTerminal(runTagsWtDepsCommand); 86 | } else if (firstStageSelection === "run multiple files"){ 87 | if(!multipleFileSelection){return;}; 88 | runMultipleFilesFromSelection(workspaceFolder, multipleFileSelection, includeDependencies, includeDependents, fullRefresh); 89 | } else if (firstStageSelection === "run multiple tags"){ 90 | if(!multipleTagsSelection){return;}; 91 | runMultipleTagsFromSelection(workspaceFolder, multipleTagsSelection, includeDependencies, includeDependents, fullRefresh); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/runTag.ts: -------------------------------------------------------------------------------- 1 | import { getDataformCliCmdBasedOnScope, getDataformCompilationTimeoutFromConfig, getDataformCompilerOptions, getWorkspaceFolder, runCommandInTerminal } from "./utils"; 2 | import * as vscode from 'vscode'; 3 | 4 | export async function runMultipleTagsFromSelection(workspaceFolder: string, selectedTags: string, includDependencies: boolean, includeDownstreamDependents: boolean, fullRefresh: boolean) { 5 | let defaultDataformCompileTime = getDataformCompilationTimeoutFromConfig(); 6 | let runmultitagscommand = getRunTagsWtOptsCommand(workspaceFolder, selectedTags, defaultDataformCompileTime, includDependencies, includeDownstreamDependents, fullRefresh); 7 | runCommandInTerminal(runmultitagscommand); 8 | } 9 | 10 | 11 | export async function getMultipleTagsSelection() { 12 | let options = { 13 | canPickMany: true, 14 | ignoreFocusOut: true, 15 | }; 16 | let selectedTags = await vscode.window.showQuickPick(dataformTags, options); 17 | return selectedTags; 18 | } 19 | 20 | 21 | 22 | export function getRunTagsWtOptsCommand(workspaceFolder: string, tags: string | object[], dataformCompilationTimeoutVal: string, includDependencies: boolean, includeDownstreamDependents: boolean, fullRefresh: boolean): string { 23 | let dataformCompilerOptions = getDataformCompilerOptions(); 24 | const customDataformCliPath = getDataformCliCmdBasedOnScope(workspaceFolder); 25 | let cmd = `${customDataformCliPath} run "${workspaceFolder}" ${dataformCompilerOptions} --timeout=${dataformCompilationTimeoutVal}`; 26 | if (typeof tags === "object") { 27 | for (let tag of tags) { 28 | cmd += ` --tags=${tag}`; 29 | } 30 | } else { 31 | cmd += ` --tags=${tags}`; 32 | } 33 | 34 | if (includDependencies) { 35 | cmd += ` --include-deps`; 36 | } 37 | if (includeDownstreamDependents) { 38 | cmd += ` --include-dependents`; 39 | } 40 | if (fullRefresh) { 41 | cmd += ` --full-refresh`; 42 | } 43 | return cmd; 44 | } 45 | 46 | export async function runTag(includeDependencies: boolean, includeDependents: boolean) { 47 | if (dataformTags.length === 0) { 48 | vscode.window.showInformationMessage('No tags found in project'); 49 | return; 50 | } 51 | vscode.window.showQuickPick(dataformTags, { 52 | onDidSelectItem: (_) => { 53 | // This is triggered as soon as a item is hovered over 54 | } 55 | }).then(async(selection) => { 56 | if (!selection) { 57 | return; 58 | } 59 | 60 | let workspaceFolder = await getWorkspaceFolder(); 61 | if (!workspaceFolder) { return; } 62 | 63 | let defaultDataformCompileTime = getDataformCompilationTimeoutFromConfig(); 64 | let cmd = ""; 65 | if (includeDependencies) { 66 | cmd = getRunTagsWtOptsCommand(workspaceFolder, selection, defaultDataformCompileTime, true, false, false); 67 | } else if (includeDependents) { 68 | cmd = getRunTagsWtOptsCommand(workspaceFolder, selection, defaultDataformCompileTime, false, true, false); 69 | } else { 70 | cmd = getRunTagsWtOptsCommand(workspaceFolder, selection, defaultDataformCompileTime, false, false, false); 71 | } 72 | if (cmd !== "") { 73 | runCommandInTerminal(cmd); 74 | } 75 | }); 76 | } 77 | 78 | -------------------------------------------------------------------------------- /src/setDiagnostics.ts: -------------------------------------------------------------------------------- 1 | 2 | import * as vscode from 'vscode'; 3 | import {ErrorMeta, SqlxBlockMetadata} from './types'; 4 | 5 | export function setDiagnostics(document: vscode.TextDocument, errorMeta: ErrorMeta, diagnosticCollection: vscode.DiagnosticCollection, sqlxBlockMetadata: SqlxBlockMetadata, offSet:number){ 6 | 7 | const diagnostics: vscode.Diagnostic[] = []; 8 | const severity = vscode.DiagnosticSeverity.Error; 9 | 10 | let errLineNumber; 11 | let errColumnNumber = 0; 12 | 13 | if (errorMeta.mainQueryError.hasError){ 14 | let errLineNumber = errorMeta.mainQueryError.location?.line; 15 | let errColumnNumber = errorMeta.mainQueryError.location?.column; 16 | if (errLineNumber === undefined || errColumnNumber === undefined) { 17 | vscode.window.showErrorMessage(`Error in setting diagnostics. Error location is undefined.`); 18 | return; 19 | } 20 | let sqlQueryStartLineNumber = sqlxBlockMetadata.sqlBlock.startLine; 21 | //TODO: This will not work if pre_operation block is placed after main sql query. unlikely that is coding pattern used ? 22 | let preOpsOffset = 0; 23 | if (sqlxBlockMetadata.preOpsBlock.preOpsList.length > 0){ 24 | preOpsOffset = (sqlxBlockMetadata.preOpsBlock.preOpsList[0].endLine - sqlxBlockMetadata.preOpsBlock.preOpsList[0].startLine) + 1; 25 | } 26 | errLineNumber = (sqlQueryStartLineNumber + (errLineNumber - offSet)) - preOpsOffset; 27 | 28 | const range = new vscode.Range(new vscode.Position(errLineNumber, errColumnNumber), new vscode.Position(errLineNumber, errColumnNumber + 5)); 29 | const regularBlockDiagnostic = new vscode.Diagnostic(range, `(Main Query): ${errorMeta.mainQueryError.message}`, severity); 30 | diagnostics.push(regularBlockDiagnostic); 31 | } 32 | 33 | if(errorMeta?.preOpsError?.hasError){ 34 | errLineNumber = sqlxBlockMetadata.preOpsBlock.preOpsList[0].startLine - 1; 35 | const range = new vscode.Range(new vscode.Position(errLineNumber, errColumnNumber), new vscode.Position(errLineNumber, errColumnNumber + 5)); 36 | const preOpsDiagnostic = new vscode.Diagnostic(range, `(Pre-Ops): ${errorMeta.preOpsError.message}`, severity); 37 | diagnostics.push(preOpsDiagnostic); 38 | } 39 | if(errorMeta?.postOpsError?.hasError){ 40 | errLineNumber = sqlxBlockMetadata.postOpsBlock.postOpsList[0].startLine - 1; 41 | const range = new vscode.Range(new vscode.Position(errLineNumber, errColumnNumber), new vscode.Position(errLineNumber, errColumnNumber + 5)); 42 | const postOpsDiagnostic = new vscode.Diagnostic(range, `(Post-Ops): ${errorMeta.postOpsError.message}`, severity); 43 | diagnostics.push(postOpsDiagnostic); 44 | } 45 | 46 | if (errorMeta?.assertionError?.hasError){ 47 | const range = new vscode.Range(new vscode.Position(0, 0), new vscode.Position(0, 0)); 48 | const assertionDiagnostic = new vscode.Diagnostic(range, `(Assertion): ${errorMeta.assertionError.message}`, severity); 49 | diagnostics.push(assertionDiagnostic); 50 | } 51 | 52 | if (document !== undefined) { 53 | diagnosticCollection.set(document.uri, diagnostics); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/test/suite/helper.ts: -------------------------------------------------------------------------------- 1 | import path from "path"; 2 | import fs from "fs"; 3 | 4 | // Helper function to find project root (where package.json is located) 5 | export function findProjectRoot(startDir: string): string { 6 | let currentDir = startDir; 7 | 8 | // Maximum number of directories to traverse up to avoid infinite loops 9 | const maxDepth = 10; 10 | let depth = 0; 11 | 12 | while (depth < maxDepth) { 13 | if (fs.existsSync(path.join(currentDir, 'package.json'))) { 14 | return currentDir; 15 | } 16 | 17 | const parentDir = path.dirname(currentDir); 18 | if (parentDir === currentDir) { 19 | // Reached the root of the filesystem 20 | break; 21 | } 22 | 23 | currentDir = parentDir; 24 | depth++; 25 | } 26 | 27 | throw new Error('Could not find project root (package.json)'); 28 | } -------------------------------------------------------------------------------- /src/test/test-workspace/.sqlfluff: -------------------------------------------------------------------------------- 1 | [sqlfluff] 2 | templater = placeholder 3 | dialect = bigquery 4 | output_line_length = 120 5 | 6 | # EXCLUDED RULES 7 | # ============== 8 | # AL07 - Avoid table aliases in from clauses and join conditions. 9 | # ST06 - Select wildcards then simple targets before calculations and aggregates. 10 | # ST07 - Prefer specifying join keys instead of using USING. 11 | # AM03 - Ambiguous ordering directions for columns in order by clause. 12 | # ST04 - Dont mess up nested CASE statement 13 | # LT05 - Line is too long 14 | # AL05 - Tables should not be aliased if that alias is not used. 15 | exclude_rules = AL07, ST06, ST07, AM03, ST04, LT05, AL05 16 | 17 | [sqlfluff:rules] 18 | allow_scalar = False 19 | 20 | [sqlfluff:indentation] 21 | # See https://docs.sqlfluff.com/en/stable/layout.html#configuring-indent-locations 22 | indent_unit = space 23 | tab_space_size = 2 24 | indented_joins = False 25 | indented_ctes = False 26 | indented_using_on = True 27 | indented_on_contents = True 28 | indented_then = True 29 | indented_then_contents = True 30 | allow_implicit_indents = False 31 | template_blocks_indent = True 32 | # This is a comma separated list of elements to skip 33 | # indentation edits to. 34 | skip_indentation_in = script_content 35 | # If comments are found at the end of long lines, we default to moving 36 | # them to the line _before_ their current location as the convention is 37 | # that a comment precedes the line it describes. However if you prefer 38 | # comments moved _after_, this configuration setting can be set to "after". 39 | trailing_comments = before 40 | # To exclude comment lines from indentation entirely set this to "True". 41 | ignore_comment_lines = False 42 | 43 | [sqlfluff:layout:type:where_clause] 44 | line_position = alone:strict 45 | 46 | [sqlfluff:layout:type:binary_operator] 47 | line_position = leading 48 | 49 | [sqlfluff:layout:type:comparison_operator] 50 | line_position = trailing 51 | 52 | [sqlfluff:layout:type:alias_expression] 53 | # We want non-default spacing _before_ the alias expressions. 54 | spacing_before = align 55 | align_within = select_clause 56 | align_scope = bracketed 57 | 58 | [sqlfluff:rules:capitalisation.keywords] 59 | # Keywords must be capitalised 60 | capitalisation_policy = upper 61 | 62 | [sqlfluff:rules:capitalisation.literals] 63 | # Null & Boolean Literals eg: NULL, TRUE, FALSE 64 | capitalisation_policy = upper 65 | 66 | [sqlfluff:rules:capitalisation.types] 67 | # Data Types eg: INT, STR 68 | extended_capitalisation_policy = upper 69 | 70 | [sqlfluff:rules:capitalisation.identifiers] 71 | # Unquoted identifiers 72 | extended_capitalisation_policy = upper 73 | unquoted_identifiers_policy=all 74 | 75 | [sqlfluff:rules:capitalisation.functions] 76 | # Function names 77 | capitalisation_policy = upper 78 | extended_capitalisation_policy = upper 79 | 80 | [sqlfluff:rules:ambiguous.join] 81 | # Fully qualify JOIN clause 82 | fully_qualify_join_types = inner 83 | 84 | [sqlfluff:rules:aliasing.length] 85 | # Minimum string length when creating an alias 86 | min_alias_length = 3 87 | 88 | [sqlfluff:rules:aliasing.table] 89 | # Aliasing preference for tables, ie needs an AS 90 | aliasing = explicit 91 | 92 | [sqlfluff:rules:aliasing.column] 93 | # Aliasing preference for columns, ie needs an AS 94 | aliasing = explicit 95 | 96 | [sqlfluff:rules:layout.commas] 97 | # Leading or trailing commas 98 | line_position = leading 99 | 100 | [sqlfluff:layout:type:comma] 101 | line_position = leading 102 | 103 | 104 | [sqlfluff:rules:convention.select_trailing_comma] 105 | # No trailing comma at end of SELECT, ie before FROM (after last column name) 106 | select_clause_trailing_comma = forbid 107 | 108 | [sqlfluff:rules:ambiguous.column_references] 109 | # GROUP BY/ORDER BY column references (i.e. implicit by position or explicit by name) 110 | group_by_and_order_by_style = explicit 111 | 112 | [sqlfluff:rules:references.special_chars] 113 | # Special characters in identifiers 114 | unquoted_identifiers_policy = all 115 | quoted_identifiers_policy = all 116 | allow_space_in_identifier = False 117 | additional_allowed_characters = ["", $] 118 | 119 | [sqlfluff:rules:references.keywords] 120 | # Keywords should not be used as identifiers. 121 | unquoted_identifiers_policy = all 122 | quoted_identifiers_policy = none 123 | 124 | [sqlfluff:templater:placeholder] 125 | param_regex = (?s)\${\s*(?P[\w_]+)(?P(?:[^{}]+|{(?&rec)})*+)}|(?Pconfig|pre_operations|post_operations)\s*{(?&rec)} 126 | ref = ref_table_placeholder 127 | self = self_table_placeholder 128 | when = 129 | config = 130 | pre_operations = 131 | -------------------------------------------------------------------------------- /src/test/test-workspace/README.md: -------------------------------------------------------------------------------- 1 | 2 | ### How to use 3 | 4 | Option 1: 5 | 6 | Open the repository in github code spaces by clicking on the "Code" button on the github repository page and selecting the "Codespaces" tab. It uses the `devcontainer.json` file in this repo to build a 7 | container with the [Dataform tools](https://marketplace.visualstudio.com/items?itemName=ashishalex.dataform-lsp-vscode) extension and dependencies such as gcloud and dataform cli preconfigured. **Note** that it takes approximately 5 mins for the container to build, so grab a cup of coffe while its getting spun up ! 8 | 9 | Option 2: 10 | 11 | Open the repository in a [VSCode Dev Container](https://code.visualstudio.com/docs/devcontainers/containers) and run the following 12 | 13 | ```bash 14 | gcloud init 15 | gloud auth application-default login 16 | gcloud config set project drawingfire-b72a8 # replace with your gcp project id 17 | ``` 18 | 19 | #### TODOs 20 | 21 | - [ ] Add example of using a javascript function 22 | - [ ] Create another dataset in BigQuery and connect to it in the pipeline 23 | 24 | #### Personal notes 25 | 26 | Create a new Dataform project 27 | 28 | ```bash 29 | dataform init --default-database drawingfire-b72a8 --default-location europe-west2 30 | ``` 31 | -------------------------------------------------------------------------------- /src/test/test-workspace/definitions/0100_CLUBS.sqlx: -------------------------------------------------------------------------------- 1 | config { 2 | type: 'view', 3 | schema: 'dataform', 4 | description: 'data for club', 5 | tags: ["FOOTY"] 6 | } 7 | 8 | select * from 9 | ${ref("CLUBS")} WHERE 1=1 10 | -- AND LOWER(name) LIKE "%manchester%" -------------------------------------------------------------------------------- /src/test/test-workspace/definitions/0100_GAMES_META.sqlx: -------------------------------------------------------------------------------- 1 | config { 2 | type: "table", 3 | description: "games played by Manchester United in 2024", 4 | schema: "dataform", 5 | assertions: { 6 | uniqueKey: ["game_id"] 7 | }, 8 | tags: ["FOOTY"] 9 | } 10 | 11 | 12 | 13 | WITH GAMES AS ( 14 | SELECT * FROM 15 | ${ref("GAMES")} 16 | ) 17 | 18 | , GAME_EVENTS AS ( 19 | SELECT * FROM 20 | ${ref("GAME_EVENTS")} 21 | ) 22 | 23 | SELECT *, 24 | ROW_NUMBER() OVER ( 25 | PARTITION BY HOME_CLUB_ID 26 | ORDER BY ATTENDANCE DESC 27 | ) AS RN 28 | FROM GAMES 29 | WHERE 1 = 1 30 | AND COMPETITION_ID = ${params.PREMIER_LEAGUE_COMP_ID} 31 | AND HOME_CLUB_ID = ${params.MANCHESTER_UNITED_CLUB_ID} 32 | AND SEASON = 2024 33 | ORDER BY DATE desc 34 | 35 | -- SELECT * FROM GAMES AS G 36 | -- LEFT JOIN GAME_EVENTS AS E 37 | -- USING (GAME_ID) 38 | 39 | 40 | -- SELECT DISTINCT competition_id, competition_type FROM GAMES 41 | -------------------------------------------------------------------------------- /src/test/test-workspace/definitions/0100_PLAYERS.sqlx: -------------------------------------------------------------------------------- 1 | config { 2 | type: "table", 3 | columns: docs.columnDescriptions, 4 | } 5 | 6 | WITH PLAYERS AS ( 7 | select * 8 | FROM 9 | ${ref("PLAYERS")} 10 | ) 11 | 12 | 13 | , PLAYER_VALUATIONS AS ( 14 | select * 15 | FROM 16 | ${ref("PLAYER_VALUATIONS")} 17 | ) 18 | 19 | 20 | , FINAL_TABLE AS ( 21 | SELECT 22 | P.* 23 | FROM PLAYERS P LEFT JOIN 24 | PLAYER_VALUATIONS V 25 | USING(PLAYER_ID) 26 | ) 27 | 28 | SELECT * FROM FINAL_TABLE WHERE 1=1 -------------------------------------------------------------------------------- /src/test/test-workspace/definitions/010_JS_MULTIPLE.js: -------------------------------------------------------------------------------- 1 | 2 | publish("test_js_table_1").query(ctx => "select 1 as a"); 3 | 4 | publish( 5 | "test_js_table_2", 6 | ` 7 | select 12 as a 8 | ` 9 | ); 10 | 11 | 12 | operate( 13 | "test_js_ops", 14 | ` 15 | create or replace table 16 | drawingfire-b72a8.dataform.test_js_ops 17 | as 18 | select 14 as b 19 | ` 20 | ); 21 | 22 | assert( 23 | "test_js_assert", 24 | ` 25 | select 14 as b 26 | ` 27 | ); -------------------------------------------------------------------------------- /src/test/test-workspace/definitions/0200_GAME_DETAILS.sqlx: -------------------------------------------------------------------------------- 1 | config { 2 | type: 'table', 3 | schema: 'dataform', 4 | description: 'Shows who scored goals for a specific Chelsea vs Arsenal game', 5 | tags: ["FOOTY"] 6 | } 7 | 8 | 9 | js { 10 | const type = "'Goals'" 11 | } 12 | 13 | 14 | WITH CHELSEA_GAMES_META AS ( 15 | SELECT * 16 | FROM ${ref("0100_GAMES_META")} 17 | WHERE 18 | 1 = 1 19 | AND GAME_ID = ${params.CHELSEA_VS_ARSENAL_GAME_ID} 20 | ) 21 | 22 | , PLAYERS AS ( 23 | SELECT * FROM ${ref("PLAYERS")} 24 | ) 25 | 26 | , GOAL_EVENTS AS ( 27 | SELECT * FROM ${ref("GAME_EVENTS")} 28 | WHERE TYPE = ${type} 29 | ) 30 | 31 | , GAME_EVENTS AS ( 32 | SELECT E.* 33 | FROM GOAL_EVENTS AS E 34 | RIGHT JOIN CHELSEA_GAMES_META AS C USING (GAME_ID) 35 | ) 36 | 37 | 38 | SELECT 39 | P.NAME AS PLAYER_NAME 40 | , E.* 41 | FROM GAME_EVENTS AS E 42 | LEFT JOIN 43 | PLAYERS AS P 44 | USING (PLAYER_ID) 45 | -------------------------------------------------------------------------------- /src/test/test-workspace/definitions/0200_PLAYER_TRANSFERS.sqlx: -------------------------------------------------------------------------------- 1 | config { 2 | type: 'table', 3 | schema: 'dataform', 4 | description: 'Shows mox expensive manchester united player in each season', 5 | columns: {...{transfer_date: "date on which transfere from one club to another happened"}, ...docs.columnDescriptions} 6 | } 7 | 8 | pre_operations { 9 | SET @@query_label = "key:value"; 10 | } 11 | 12 | WITH TRANSFERS AS ( 13 | SELECT * FROM ${ref("TRANSFERS")} 14 | ) 15 | 16 | , TEST AS ( 17 | SELECT * 18 | FROM TRANSFERS 19 | WHERE 20 | 1 = 1 21 | AND TO_CLUB_ID = ${params.MANCHESTER_UNITED_CLUB_ID} 22 | ORDER BY TRANSFER_DATE DESC 23 | ) 24 | 25 | SELECT * 26 | FROM TEST 27 | QUALIFY 28 | ROW_NUMBER() OVER ( 29 | PARTITION BY TRANSFER_SEASON 30 | ORDER BY TRANSFER_FEE DESC 31 | ) = 1 32 | ORDER BY TRANSFER_DATE DESC 33 | -------------------------------------------------------------------------------- /src/test/test-workspace/definitions/0300_INCREMENTAL.sqlx: -------------------------------------------------------------------------------- 1 | config { 2 | type: "incremental", 3 | } 4 | 5 | pre_operations { 6 | DECLARE date_checkpoint DEFAULT ( 7 | ${when(incremental(), 8 | `SELECT MAX(date) FROM ${self()}`, 9 | `SELECT DATE("2012-01-01")`)} 10 | ); 11 | } 12 | 13 | 14 | SELECT 15 | * 16 | FROM 17 | ${ref("0100_GAMES_META")} WHERE 1=1 18 | AND date >= date_checkpoint 19 | -------------------------------------------------------------------------------- /src/test/test-workspace/definitions/0500_OPERATIONS.sqlx: -------------------------------------------------------------------------------- 1 | config { 2 | type: 'operations', 3 | hasOutput: true 4 | } 5 | 6 | DROP TABLE IF EXISTS 7 | ${self()}; 8 | 9 | CREATE OR REPLACE TABLE ${self()} AS 10 | SELECT * FROM ${ref("0100_GAMES_META")} -------------------------------------------------------------------------------- /src/test/test-workspace/definitions/assertions/0100_CLUBS_ASSER.sqlx: -------------------------------------------------------------------------------- 1 | config { 2 | type: 'assertion', 3 | } 4 | 5 | WITH TEST AS ( 6 | 7 | SELECT 8 | SUBSTRING(URL, 0,5) as initial 9 | FROM ${ref("0100_CLUBS")} 10 | ) 11 | SELECT * FROM TEST WHERE initial <> "https" -------------------------------------------------------------------------------- /src/test/test-workspace/definitions/sources.js: -------------------------------------------------------------------------------- 1 | 2 | 3 | declare({ 4 | database: 'drawingfire-b72a8', 5 | schema: 'football_data', 6 | name: 'PLAYERS', 7 | }); 8 | 9 | declare({ 10 | database: 'drawingfire-b72a8', 11 | schema: 'football_data', 12 | name: 'CLUBS', 13 | }); 14 | 15 | 16 | declare({ 17 | database: 'drawingfire-b72a8', 18 | schema: 'football_data', 19 | name: 'GAMES', 20 | }); 21 | 22 | declare({ 23 | database: 'drawingfire-b72a8', 24 | schema: 'football_data', 25 | name: 'GAME_EVENTS', 26 | }); 27 | 28 | declare({ 29 | database: 'drawingfire-b72a8', 30 | schema: 'football_data', 31 | name: 'PLAYER_VALUATIONS', 32 | }); 33 | 34 | declare({ 35 | database: 'drawingfire-b72a8', 36 | schema: 'football_data', 37 | name: 'TRANSFERS', 38 | }); -------------------------------------------------------------------------------- /src/test/test-workspace/definitions/tests_for_vscode_extension/0100_MULTIPLE_PRE_POST_OPS.sqlx: -------------------------------------------------------------------------------- 1 | config { 2 | type: 'table', 3 | } 4 | 5 | 6 | pre_operations { 7 | select 22 as someNumber 8 | } 9 | 10 | pre_operations { 11 | select 44 as someNumber 12 | } 13 | 14 | 15 | post_operations { 16 | select 55 as someNumber 17 | } 18 | 19 | post_operations { 20 | select 66 as someNumber 21 | } 22 | 23 | 24 | SELECT CURRENT_TIMESTAMP() AS LAST_UPDATED_TIMESTAMP 25 | -------------------------------------------------------------------------------- /src/test/test-workspace/definitions/tests_for_vscode_extension/0100_SINGLE_LINE_CONFIG.sqlx: -------------------------------------------------------------------------------- 1 | config { type: 'table'} 2 | 3 | pre_operations { 4 | SELECT 1 AS SOME_NUMBER 5 | } 6 | 7 | post_operations { 8 | SELECT 2 AS SOME_NUMBER 9 | } 10 | 11 | SELECT 12 | CURRENT_DATE() AS TODAYS_DATE -------------------------------------------------------------------------------- /src/test/test-workspace/definitions/tests_for_vscode_extension/099_MULTIPLE_ERRORS.sqlx: -------------------------------------------------------------------------------- 1 | config { 2 | type: 'table', 3 | assertions: { 4 | uniqueKey: ["uniqueId"] 5 | } 6 | } 7 | 8 | pre_operations { 9 | SET @@query_label = "key:value"; 10 | declare SOME_DATE DATE DEFAULT CURRENT_DATE; 11 | } 12 | 13 | post_operations { 14 | SELECT 15 | URRENT_TIMESTAMP() AS SOME_TIMESTAMP 16 | } 17 | 18 | 19 | SELECT 20 | 1 AS UNIQUEID 21 | , "myData" AS SOMECOLUMN 22 | , URRENT_DATE() AS TODAYS_DATE 23 | -------------------------------------------------------------------------------- /src/test/test-workspace/includes/docs.js: -------------------------------------------------------------------------------- 1 | const columnDescriptions = { 2 | player_id: "unique identifier for a player" 3 | } 4 | 5 | module.exports = { 6 | columnDescriptions, 7 | } -------------------------------------------------------------------------------- /src/test/test-workspace/includes/params.js: -------------------------------------------------------------------------------- 1 | 2 | const CHELSEA_CLUB_ID = 631 3 | const PREMIER_LEAGUE_COMP_ID = "'GB1'" 4 | const CHELSEA_VS_ARSENAL_GAME_ID = 2258986 5 | const MANCHESTER_UNITED_CLUB_ID = 985 6 | 7 | module.exports = { 8 | CHELSEA_CLUB_ID, 9 | PREMIER_LEAGUE_COMP_ID, 10 | CHELSEA_VS_ARSENAL_GAME_ID, 11 | MANCHESTER_UNITED_CLUB_ID, 12 | } -------------------------------------------------------------------------------- /src/test/test-workspace/workflow_settings.yaml: -------------------------------------------------------------------------------- 1 | dataformCoreVersion: 3.0.8 2 | defaultProject: drawingfire-b72a8 3 | defaultLocation: europe-west2 4 | defaultDataset: dataform 5 | defaultAssertionDataset: dataform_assertions 6 | -------------------------------------------------------------------------------- /src/views/depedancyGraphPanel.ts: -------------------------------------------------------------------------------- 1 | import * as vscode from 'vscode'; 2 | import { logger } from '../logger'; 3 | import { getNonce, getPostionOfSourceDeclaration, getWorkspaceFolder } from '../utils'; 4 | import { generateDependancyTreeMetadata } from '../dependancyTreeNodeMeta'; 5 | import path from 'path'; 6 | 7 | export function getWebViewHtmlContent(context: vscode.ExtensionContext, webview: vscode.Webview) { 8 | const scriptUri = webview.asWebviewUri(vscode.Uri.joinPath(context.extensionUri, 'dist', 'dependancy_graph.js')); 9 | const nonce = getNonce(); 10 | 11 | return ` 12 | 13 | 14 | 15 | 16 | 17 | 18 | My Extension 19 | 20 | 21 |
22 | 23 | 24 | 25 | `; 26 | } 27 | 28 | 29 | export async function createDependencyGraphPanel(context: vscode.ExtensionContext, viewColumn: vscode.ViewColumn = vscode.ViewColumn.Beside) { 30 | logger.info('Creating dependency graph panel'); 31 | const output = await generateDependancyTreeMetadata(); 32 | logger.info(`output.currentActiveEditorIdx: ${output?.currentActiveEditorIdx}`); 33 | if(!output){ 34 | logger.error('No dependency graph data found'); 35 | //TODO: show error message maybe ? 36 | return; 37 | } 38 | const panel = vscode.window.createWebviewPanel( 39 | "Dependency Graph", 40 | "Dependency Graph", 41 | viewColumn, 42 | { 43 | enableFindWidget: true, 44 | enableScripts: true, 45 | retainContextWhenHidden: true, 46 | localResourceRoots: [ 47 | vscode.Uri.joinPath(context.extensionUri, "dist") 48 | ], 49 | }, 50 | ); 51 | 52 | panel.webview.html = getWebViewHtmlContent(context, panel.webview); 53 | 54 | panel.webview.onDidReceiveMessage( 55 | async (message) => { 56 | switch (message.type) { 57 | case 'webviewReady': 58 | // Send data only after the webview signals it's ready 59 | panel.webview.postMessage({ 60 | type: 'nodeMetadata', 61 | value: { 62 | initialNodesStatic: output.dependancyTreeMetadata, 63 | initialEdgesStatic: output.initialEdgesStatic, 64 | datasetColorMap: Object.fromEntries(output.datasetColorMap), 65 | currentActiveEditorIdx: output.currentActiveEditorIdx 66 | } 67 | }); 68 | break; 69 | case 'nodeFileName': 70 | const filePath = message.value.filePath; 71 | const type = message.value.type; 72 | if (filePath) { 73 | const workspaceFolder = await getWorkspaceFolder(); 74 | if (workspaceFolder) { 75 | const fullFilePath = path.join(workspaceFolder, filePath); 76 | const filePathUri = vscode.Uri.file(fullFilePath); 77 | const document = await vscode.workspace.openTextDocument(filePathUri); 78 | if (type === 'declarations') { 79 | const position = await getPostionOfSourceDeclaration(filePathUri, message.value.modelName); 80 | if (position) { 81 | vscode.window.showTextDocument(document, vscode.ViewColumn.One, false).then(editor => { 82 | const range = new vscode.Range(position.line, 0, position.line, 0); 83 | editor.revealRange(range, vscode.TextEditorRevealType.InCenter); 84 | editor.selection = new vscode.Selection(position.line, 0, position.line, 0); 85 | }); 86 | } 87 | } else { 88 | await vscode.window.showTextDocument(document, vscode.ViewColumn.One); 89 | } 90 | } else { 91 | vscode.window.showErrorMessage('Workspace folder not found'); 92 | } 93 | } 94 | return; 95 | } 96 | }, 97 | undefined, 98 | context.subscriptions 99 | ); 100 | 101 | return panel; 102 | } 103 | -------------------------------------------------------------------------------- /src/views/register-sidebar-panel.ts: -------------------------------------------------------------------------------- 1 | import { commands, ExtensionContext, Uri, Webview, WebviewView, WebviewViewProvider, window } from "vscode"; 2 | import * as vscode from 'vscode'; 3 | import {getNonce, getCurrentFileMetadata } from '../utils'; 4 | 5 | export async function registerWebViewProvider(context: ExtensionContext) { 6 | const provider = new SidebarWebViewProvider(context.extensionUri, context); 7 | context.subscriptions.push(window.registerWebviewViewProvider('dataform-sidebar', provider)); 8 | 9 | context.subscriptions.push(commands.registerCommand('vscode-dataform-tools.getCurrFileMetadataForSidePanel', async () => { 10 | let currFileMetadata = await getCurrentFileMetadata(false); 11 | if (currFileMetadata) { 12 | provider.view?.webview.postMessage({ "currFileMetadata": currFileMetadata }); 13 | } else { 14 | provider.view?.webview.postMessage({ 15 | "errorMessage": `File type not supported. Supported file types are sqlx, js` 16 | }); 17 | } 18 | })); 19 | 20 | context.subscriptions.push(commands.registerCommand('vscode-dataform-tools.showFileMetadataForSidePanel', async () => { 21 | vscode.commands.executeCommand('dataform-sidebar.focus'); 22 | })); 23 | 24 | vscode.window.onDidChangeActiveTextEditor((editor) => { 25 | if (editor) { 26 | vscode.commands.executeCommand('vscode-dataform-tools.getCurrFileMetadataForSidePanel'); 27 | } 28 | }, null, context.subscriptions); 29 | } 30 | 31 | export class SidebarWebViewProvider implements WebviewViewProvider { 32 | constructor(private readonly _extensionUri: Uri, public extensionContext: ExtensionContext) { } 33 | view?: WebviewView; 34 | 35 | resolveWebviewView(webviewView: WebviewView) { 36 | this.view = webviewView; 37 | 38 | webviewView.onDidChangeVisibility(() => { 39 | if (webviewView.visible) { 40 | vscode.commands.executeCommand('vscode-dataform-tools.getCurrFileMetadataForSidePanel'); 41 | } 42 | }); 43 | 44 | webviewView.webview.options = { 45 | enableScripts: true, 46 | localResourceRoots: [this._extensionUri], 47 | }; 48 | 49 | webviewView.webview.html = this._getHtmlForWebview(webviewView.webview); 50 | 51 | let isFileOpen = vscode.window.activeTextEditor?.document.uri.fsPath; 52 | if (isFileOpen) { 53 | vscode.commands.executeCommand('vscode-dataform-tools.getCurrFileMetadataForSidePanel'); 54 | } 55 | } 56 | 57 | private _getHtmlForWebview(webview: Webview) { 58 | const styleResetUri = webview.asWebviewUri(Uri.joinPath(this._extensionUri, "media", "css", "reset.css")); 59 | const styleVSCodeUri = webview.asWebviewUri(Uri.joinPath(this._extensionUri, "media", "css", "vscode.css")); 60 | 61 | const scriptPanel = webview.asWebviewUri(Uri.joinPath(this._extensionUri, "media", "js", "panel.js")); 62 | const sidePanelScriptUri = webview.asWebviewUri(Uri.joinPath(this._extensionUri, "media", "js", "sidePanel.js")); 63 | 64 | const nonce = getNonce(); 65 | 66 | return /*html*/ ` 67 | 68 | 69 | 70 | 71 | 75 | 78 | 79 | 80 | 81 | 82 | 83 |

Dataform

84 |
85 |

86 |
87 | 88 | 89 | 90 | `; 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /tailwind.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('tailwindcss').Config} */ 2 | module.exports = { 3 | content: ['./webviews/dependancy_graph/**/*.{js,jsx,ts,tsx}'], 4 | theme: { 5 | extend: {}, 6 | }, 7 | plugins: [], 8 | }; -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "Node16", 4 | "target": "ES2022", 5 | "outDir": "out", 6 | "lib": [ 7 | "ES2022", 8 | "DOM" 9 | ], 10 | "jsx": "react-jsx", 11 | "sourceMap": true, 12 | "rootDir": ".", 13 | "strict": true, 14 | "esModuleInterop": true, 15 | "skipLibCheck": true, 16 | "forceConsistentCasingInFileNames": true, 17 | "resolveJsonModule": true, 18 | "isolatedModules": true, 19 | "types": ["react", "node"], 20 | /* Additional Checks */ 21 | "noImplicitReturns": true, 22 | "noFallthroughCasesInSwitch": true, 23 | "noUnusedLocals": true, 24 | "noUnusedParameters": true 25 | }, 26 | "include": [ 27 | "src", 28 | "webviews/dependancy_graph/**/*", 29 | "src/globals.d.ts" 30 | ], 31 | "exclude": [ 32 | "node_modules", 33 | ".vscode-test", 34 | "dist" 35 | ] 36 | } 37 | -------------------------------------------------------------------------------- /vite.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'vite'; 2 | import react from '@vitejs/plugin-react'; 3 | import path from 'path'; 4 | 5 | export default defineConfig({ 6 | plugins: [react()], 7 | build: { 8 | outDir: 'dist', 9 | rollupOptions: { 10 | input: './webviews/dependancy_graph/index.tsx', 11 | output: { 12 | entryFileNames: 'dependancy_graph.js', 13 | format: 'iife', 14 | }, 15 | }, 16 | sourcemap: true, 17 | watch: {} 18 | }, 19 | css: { 20 | postcss: './postcss.config.js' 21 | }, 22 | resolve: { 23 | alias: { 24 | '@': path.resolve(__dirname, './webviews/dependancy_graph') 25 | } 26 | } 27 | }); -------------------------------------------------------------------------------- /website/.gitignore: -------------------------------------------------------------------------------- 1 | # dependencies 2 | /node_modules 3 | /.pnp 4 | .pnp.js 5 | 6 | # testing 7 | /coverage 8 | 9 | # next.js 10 | /.next/ 11 | /out/ 12 | 13 | # production 14 | /build 15 | 16 | # misc 17 | .DS_Store 18 | *.pem 19 | 20 | # debug 21 | npm-debug.log* 22 | yarn-debug.log* 23 | yarn-error.log* 24 | .pnpm-debug.log* 25 | 26 | # local env files 27 | .env*.local 28 | .env 29 | 30 | # vercel 31 | .vercel 32 | 33 | # typescript 34 | *.tsbuildinfo 35 | next-env.d.ts 36 | 37 | # IDE 38 | .idea/ 39 | .vscode/ 40 | *.swp 41 | *.swo 42 | 43 | # OS generated files 44 | .DS_Store 45 | .DS_Store? 46 | ._* 47 | .Spotlight-V100 48 | .Trashes 49 | ehthumbs.db 50 | Thumbs.db -------------------------------------------------------------------------------- /website/app/blog/[slug]/page.tsx: -------------------------------------------------------------------------------- 1 | import { Metadata } from "next"; 2 | import Link from "next/link"; 3 | import { notFound } from "next/navigation"; 4 | import { ClientBlogContent } from "@/app/components/ClientBlogContent"; 5 | 6 | const getBlogPost = async (slug: string) => { 7 | const posts = { 8 | "compiler-options": { 9 | title: "Using Compiler Options in Dataform", 10 | date: "April 16, 2025", 11 | content: ` 12 | Compiler options can be used to set things like table prefix, schema prefix and adding variables the the query compilation or execution. 13 | 14 | For example, to add the table prefix to \`AA\` in all tables, you can run: 15 | 16 | \`\`\`bash 17 | dataform compile --table-prefix=AA 18 | \`\`\` 19 | 20 | \`your-project.your-dataset.your-table\` would become \`your-project.your-dataset.AA_your-table\` 21 | 22 | 23 | Compiler options can also be used with \`dataform run\` to add the table prefix to a specific execution as follows: 24 | 25 | \`\`\`bash 26 | dataform run --actions "your-project.your-dataset.your-table" --table-prefix=AA 27 | \`\`\` 28 | 29 | 30 | You can use table prefixes to isolate your development work from production tables. For example, use \`dev_\` for development and \`prod_\` for production tables. 31 | 32 | > [!TIP] 33 | > In Dataform Tools VSCode extension, you can set the table prefix directly in the compiled query webview by adding \`--table-prefix=AA\` to the text box. 34 | 35 | Available compiler options can be seen by running \`dataform compile --help\` in your terminal: 36 | 37 | \`\`\`bash 38 | 39 | Positionals: 40 | project-dir The Dataform project directory. [default: "."] 41 | 42 | Options: 43 | --help Show help [boolean] 44 | --version Show version number [boolean] 45 | --watch Whether to watch the changes in the project directory. [boolean] [default: false] 46 | --json Outputs a JSON representation of the compiled project. [boolean] [default: false] 47 | --timeout Duration to allow project compilation to complete. Examples: '1s', '10m', etc. [string] [default: null] 48 | --default-database The default database to use, equivalent to Google Cloud Project ID. If unset, the value from workflow_settings.yaml is used. [string] 49 | --default-schema Override for the default schema name. If unset, the value from workflow_settings.yaml is used. 50 | --default-location The default location to use. See https://cloud.google.com/bigquery/docs/locations for supported values. If unset, the value from workflow_settings.yaml is used. 51 | --assertion-schema Default assertion schema. If unset, the value from workflow_settings.yaml is used. 52 | --vars Override for variables to inject via '--vars=someKey=someValue,a=b', referenced by \`dataform.projectConfig.vars.someValue\`. If unset, the value from workflow_settings.yaml is used. [string] [default: null] 53 | --database-suffix Default assertion schema. If unset, the value from workflow_settings.yaml is used. 54 | --schema-suffix A suffix to be appended to output schema names. If unset, the value from workflow_settings.yaml is used. 55 | --table-prefix Adds a prefix for all table names. If unset, the value from workflow_settings.yaml is used. 56 | \`\`\` 57 | ` 58 | }, 59 | }; 60 | 61 | return posts[slug as keyof typeof posts] || null; 62 | }; 63 | 64 | export const generateMetadata = async ({ params }: { params: { slug: string } }): Promise => { 65 | const resolvedParams = await params; 66 | const post = await getBlogPost(resolvedParams.slug); 67 | 68 | if (!post) { 69 | return { 70 | title: "Blog Post Not Found", 71 | description: "The requested blog post could not be found." 72 | }; 73 | } 74 | 75 | return { 76 | title: post.title, 77 | description: `${post.title} - Published on ${post.date}` 78 | }; 79 | }; 80 | 81 | export default async function BlogPostPage({ params }: { params: { slug: string } }) { 82 | const resolvedParams = await params; 83 | const post = await getBlogPost(resolvedParams.slug); 84 | 85 | if (!post) { 86 | notFound(); 87 | } 88 | 89 | return ( 90 |
91 | 95 | ← Back to Blog 96 | 97 | 98 |
99 |

{post.title}

100 |
101 | Published on {post.date} 102 |
103 | 104 | 105 |
106 |
107 | ); 108 | } 109 | -------------------------------------------------------------------------------- /website/app/blog/not-found.tsx: -------------------------------------------------------------------------------- 1 | import Link from "next/link"; 2 | 3 | export default function BlogNotFound() { 4 | return ( 5 |
6 |

Blog Post Not Found

7 |

8 | Sorry, the blog post you're looking for doesn't exist or has been moved. 9 |

10 | 14 | Back to Blog 15 | 16 |
17 | ); 18 | } -------------------------------------------------------------------------------- /website/app/blog/page.tsx: -------------------------------------------------------------------------------- 1 | import { Metadata } from "next"; 2 | 3 | export const metadata: Metadata = { 4 | title: "Blog", 5 | }; 6 | 7 | export default function BlogPage() { 8 | return ( 9 |
10 |
11 |

Blogs

12 |
13 | 14 |
15 | 21 |
22 |
23 | ); 24 | } 25 | 26 | function BlogPostCard({ title, description, date, slug }: { title: string, description: string, date: string, slug: string }) { 27 | return ( 28 |
29 |
30 |
{date}
31 |

32 | {title} 33 |

34 |

{description}

35 | 39 | Read more → 40 | 41 |
42 |
43 | ); 44 | } 45 | -------------------------------------------------------------------------------- /website/app/components/BlogContent.tsx: -------------------------------------------------------------------------------- 1 | 'use client'; 2 | 3 | import React, { useState, useEffect } from 'react'; 4 | import ReactMarkdown from 'react-markdown'; 5 | import rehypeHighlight from 'rehype-highlight'; 6 | import rehypeRaw from 'rehype-raw'; 7 | import bash from 'highlight.js/lib/languages/bash'; 8 | import './admonitions.css'; 9 | 10 | interface BlogContentProps { 11 | content: string; 12 | } 13 | 14 | // Custom function to handle GitHub-style admonitions 15 | const transformGithubAdmonitions = (content: string): string => { 16 | // Define the admonition types we want to handle 17 | const admonitionTypes = ['NOTE', 'TIP', 'WARNING', 'IMPORTANT']; 18 | 19 | // Regular expression to match GitHub-style admonitions 20 | // Example: > [!TIP] 21 | const admonitionRegex = new RegExp( 22 | `> \\[!(${admonitionTypes.join('|')})\\]\\n((?:> .*\\n)+)`, 23 | 'g' 24 | ); 25 | 26 | // Transform GitHub-style admonitions to HTML 27 | return content.replace(admonitionRegex, (match, type, content) => { 28 | // Clean up the content lines (remove the leading '> ') 29 | const cleanContent = content 30 | .split('\n') 31 | .map(line => line.startsWith('> ') ? line.substring(2) : line) 32 | .join('\n') 33 | .trim(); 34 | 35 | // Generate HTML for the admonition 36 | return `
37 |
38 | ${getAdmonitionIcon(type)} 39 | ${type} 40 |
41 |
42 |

${cleanContent}

43 |
44 |
`; 45 | }); 46 | }; 47 | 48 | // Helper function to get the icon for a given admonition type 49 | const getAdmonitionIcon = (type: string): string => { 50 | switch (type.toUpperCase()) { 51 | case 'TIP': 52 | return '💡'; 53 | case 'NOTE': 54 | return 'ℹ️'; 55 | case 'WARNING': 56 | return '⚠️'; 57 | case 'IMPORTANT': 58 | return '❗'; 59 | default: 60 | return ''; 61 | } 62 | }; 63 | 64 | export function BlogContent({ content }: BlogContentProps) { 65 | const [processedContent, setProcessedContent] = useState(content); 66 | 67 | useEffect(() => { 68 | // Process the content on the client side 69 | setProcessedContent(transformGithubAdmonitions(content)); 70 | }, [content]); 71 | 72 | return ( 73 |
74 | rehypeHighlight({ languages: { bash } })]} 76 | components={{ 77 | pre: ({ node, ...props }) => ( 78 | 
79 |           ),
80 |           code: ({ node, ...props }) => (
81 |             
82 |           )
83 |         }}
84 |       >
85 |         {processedContent}
86 |       
87 |
88 | ); 89 | } -------------------------------------------------------------------------------- /website/app/components/ClientBlogContent.tsx: -------------------------------------------------------------------------------- 1 | 'use client'; 2 | 3 | import React from 'react'; 4 | import { BlogContent } from './BlogContent'; 5 | 6 | interface ClientBlogContentProps { 7 | content: string; 8 | } 9 | 10 | export function ClientBlogContent({ content }: ClientBlogContentProps) { 11 | return ; 12 | } -------------------------------------------------------------------------------- /website/app/components/admonitions.css: -------------------------------------------------------------------------------- 1 | .admonition { 2 | margin: 1.5rem 0; 3 | padding: 1rem; 4 | border-radius: 0.5rem; 5 | border-left: 0.25rem solid; 6 | background-color: rgba(235, 235, 235, 0.1); 7 | } 8 | 9 | .admonition-heading { 10 | font-weight: 700; 11 | text-transform: uppercase; 12 | letter-spacing: 0.05em; 13 | font-size: 0.875rem; 14 | margin-bottom: 0.5rem; 15 | display: flex; 16 | align-items: center; 17 | } 18 | 19 | .admonition-icon { 20 | margin-right: 0.5rem; 21 | } 22 | 23 | .admonition-content > p:last-child { 24 | margin-bottom: 0; 25 | } 26 | 27 | /* TIP */ 28 | .admonition-tip { 29 | border-color: #00C853; 30 | } 31 | 32 | .admonition-tip .admonition-heading { 33 | color: #00C853; 34 | } 35 | 36 | /* NOTE */ 37 | .admonition-note { 38 | border-color: #2196F3; 39 | } 40 | 41 | .admonition-note .admonition-heading { 42 | color: #2196F3; 43 | } 44 | 45 | /* WARNING */ 46 | .admonition-warning { 47 | border-color: #FF9800; 48 | } 49 | 50 | .admonition-warning .admonition-heading { 51 | color: #FF9800; 52 | } 53 | 54 | /* IMPORTANT */ 55 | .admonition-important { 56 | border-color: #F44336; 57 | } 58 | 59 | .admonition-important .admonition-heading { 60 | color: #F44336; 61 | } 62 | 63 | /* Dark mode adjustments */ 64 | :root.dark .admonition { 65 | background-color: rgba(50, 50, 50, 0.2); 66 | } -------------------------------------------------------------------------------- /website/app/components/site-header.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import { Github, Menu } from "lucide-react"; 4 | import Link from "next/link"; 5 | import { usePathname } from "next/navigation"; 6 | import { Button } from "@/components/ui/button"; 7 | import { ThemeToggle } from "@/components/theme-toggle"; 8 | import { cn } from "@/lib/utils"; 9 | import { Sheet, SheetContent, SheetTrigger, SheetTitle } from "@/components/ui/sheet"; 10 | 11 | export function SiteHeader() { 12 | const pathname = usePathname(); 13 | 14 | const navLinks = [ 15 | { href: "/", label: "Home" }, 16 | { href: "/features", label: "Features" }, 17 | { href: "/install", label: "Install" }, 18 | { href: "/blog", label: "Blog" }, 19 | { href: "/faq", label: "FAQ" }, 20 | ]; 21 | 22 | return ( 23 |
24 |
25 |
26 | {/* Mobile Menu */} 27 | 28 | 29 | 33 | 34 | 35 | Navigation Menu 36 | 50 | 51 | 52 | 53 | {/* Desktop Navigation */} 54 | 68 |
69 |
70 | 71 |
72 |
73 |
74 | ); 75 | } -------------------------------------------------------------------------------- /website/app/faq/page.tsx: -------------------------------------------------------------------------------- 1 | export default function FAQPage() { 2 | const faqs = [ 3 | { 4 | question: "What operating systems does Dataform Tools support?", 5 | answer: 6 | "Dataform Tools supports Windows, MacOS, and Linux.", 7 | }, 8 | { 9 | question: "What is the minimum version of VS Code required to use Dataform Tools?", 10 | answer: 11 | "Dataform Tools requires VS Code version 1.89.0 or higher.", 12 | }, 13 | { 14 | question: "I'm seeing 'Dataform encountered an error: Missing credentials JSON file; not found at path /.df-credentials.json'", 15 | answer: 16 | "Run `dataform init-creds` from the root of your dataform project in your terminal. You will be prompted to pick the location and type of authentication (json/adc). Choosing adc will use your default GCP credentials that you had setup using gcloud.", 17 | }, 18 | { 19 | question: "I'm getting an error: 'command vscode-dataform-tools.xxx not found'. What should I do?", 20 | answer: 21 | "It is likely that the VS Code workspace folder is not opened at the root of your dataform project. For example, if your dataform project is located at ~/Documents/repos/my_dataform_project ensure that workspace is opened at ~/Documents/repos/my_dataform_project NOT ~/Documents/repos. This design facilitates the execution of dataform compile --json command without inferring the dataform root at runtime.", 22 | }, 23 | { 24 | question: "I'm getting 'Error compiling Dataform, process exited with exit code 1'. How do I fix this?", 25 | answer: 26 | "Check if the correct dataform CLI version is installed by running dataform --version in your terminal. Ensure that dataform CLI version matches the version required by the project. Try compiling the project by running dataform compile on your terminal from the root of your dataform project. To install a specific dataform CLI version, run npm i -g @dataform/cli@x.x.x (replace with the required version). If the error persists, you likely have a compilation error in your pipeline.", 27 | }, 28 | { 29 | question: "How can I stop seeing compiled queries each time I save?", 30 | answer: 31 | "Open VS Code settings and search for Dataform. Then uncheck the 'Show compiled query on save' setting.", 32 | }, 33 | { 34 | question: "How can I change the autocompletion format for references? (default: ${ref('table_name')})", 35 | answer: 36 | "Open VS Code settings, search for Dataform, and select your preferred autocompletion format from the dropdown options. You can choose between `${ref('table_name')}`, `${ref('dataset_name', 'table_name')}`, or `${ref({schema:'dataset_name', name:'table_name'})}`.", 37 | }, 38 | { 39 | question: "How can I use a local installation of dataform CLI instead of a global one?", 40 | answer: 41 | "If you need different Dataform CLI versions for different workspaces, you can install dataform CLI locally by running `npm install @dataform/cli` (without the `-g` flag) in your project directory. This will install dataform CLI at `./node_modules/.bin/dataform`. To make the extension use the locally installed CLI, open settings and select `local` for the `Dataform CLI Scope` option.", 42 | }, 43 | { 44 | question: "I do not see go to definition option when right clicking references `${ref('table_name')}`", 45 | answer: 46 | "Check if you language mode when sqlx file is open is set to `sqlx`. VSCode sometimes sets it as a different flavour of sql. You can change that by opening the command pallet and searching for `change language mode` followed by `Configure language association for sqlx` and selecting `sqlx` from the list of available options. This should also resolve hover information not being visible as the all the language specific behaviors are tied to file being inferred as sqlx file.", 47 | } 48 | 49 | ]; 50 | 51 | return ( 52 |
53 |
54 |

55 | Frequently Asked Questions 56 |

57 |
58 | {faqs.map((faq, index) => ( 59 |
60 |

61 | {index + 1}. 62 | {faq.question} 63 |

64 |

{faq.answer}

65 |
66 | ))} 67 |
68 |
69 |
70 | ); 71 | } -------------------------------------------------------------------------------- /website/app/features/page.tsx: -------------------------------------------------------------------------------- 1 | import FeatureReels from "@/components/feature-reels"; 2 | import FeaturesTable from "@/components/features-table"; 3 | 4 | export default function FeaturesPage() { 5 | return ( 6 | 7 | ); 8 | } -------------------------------------------------------------------------------- /website/app/globals.css: -------------------------------------------------------------------------------- 1 | /* Add highlight.js styles */ 2 | @import url('https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/github-dark.min.css'); 3 | 4 | @tailwind base; 5 | @tailwind components; 6 | @tailwind utilities; 7 | 8 | body { 9 | font-family: Arial, Helvetica, sans-serif; 10 | } 11 | 12 | @layer utilities { 13 | .text-balance { 14 | text-wrap: balance; 15 | } 16 | } 17 | 18 | /* Custom styling for code blocks */ 19 | pre { 20 | margin: 1.5rem 0; 21 | padding: 1rem; 22 | border-radius: 0.5rem; 23 | overflow-x: auto; 24 | } 25 | 26 | code { 27 | font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace; 28 | font-size: 0.9rem; 29 | } 30 | 31 | @layer base { 32 | :root { 33 | --background: 0 0% 100%; 34 | --foreground: 0 0% 3.9%; 35 | --card: 0 0% 100%; 36 | --card-foreground: 0 0% 3.9%; 37 | --popover: 0 0% 100%; 38 | --popover-foreground: 0 0% 3.9%; 39 | --primary: 0 0% 9%; 40 | --primary-foreground: 0 0% 98%; 41 | --secondary: 0 0% 96.1%; 42 | --secondary-foreground: 0 0% 9%; 43 | --muted: 0 0% 96.1%; 44 | --muted-foreground: 0 0% 45.1%; 45 | --accent: 0 0% 96.1%; 46 | --accent-foreground: 0 0% 9%; 47 | --destructive: 0 84.2% 60.2%; 48 | --destructive-foreground: 0 0% 98%; 49 | --border: 0 0% 89.8%; 50 | --input: 0 0% 89.8%; 51 | --ring: 0 0% 3.9%; 52 | --chart-1: 12 76% 61%; 53 | --chart-2: 173 58% 39%; 54 | --chart-3: 197 37% 24%; 55 | --chart-4: 43 74% 66%; 56 | --chart-5: 27 87% 67%; 57 | --radius: 0.5rem; 58 | --sidebar-background: 0 0% 98%; 59 | --sidebar-foreground: 240 5.3% 26.1%; 60 | --sidebar-primary: 240 5.9% 10%; 61 | --sidebar-primary-foreground: 0 0% 98%; 62 | --sidebar-accent: 240 4.8% 95.9%; 63 | --sidebar-accent-foreground: 240 5.9% 10%; 64 | --sidebar-border: 220 13% 91%; 65 | --sidebar-ring: 217.2 91.2% 59.8%; 66 | } 67 | .dark { 68 | --background: 0 0% 3.9%; 69 | --foreground: 0 0% 98%; 70 | --card: 0 0% 3.9%; 71 | --card-foreground: 0 0% 98%; 72 | --popover: 0 0% 3.9%; 73 | --popover-foreground: 0 0% 98%; 74 | --primary: 0 0% 98%; 75 | --primary-foreground: 0 0% 9%; 76 | --secondary: 0 0% 14.9%; 77 | --secondary-foreground: 0 0% 98%; 78 | --muted: 0 0% 14.9%; 79 | --muted-foreground: 0 0% 63.9%; 80 | --accent: 0 0% 14.9%; 81 | --accent-foreground: 0 0% 98%; 82 | --destructive: 0 62.8% 30.6%; 83 | --destructive-foreground: 0 0% 98%; 84 | --border: 0 0% 14.9%; 85 | --input: 0 0% 14.9%; 86 | --ring: 0 0% 83.1%; 87 | --chart-1: 220 70% 50%; 88 | --chart-2: 160 60% 45%; 89 | --chart-3: 30 80% 55%; 90 | --chart-4: 280 65% 60%; 91 | --chart-5: 340 75% 55%; 92 | --sidebar-background: 240 5.9% 10%; 93 | --sidebar-foreground: 240 4.8% 95.9%; 94 | --sidebar-primary: 224.3 76.3% 48%; 95 | --sidebar-primary-foreground: 0 0% 100%; 96 | --sidebar-accent: 240 3.7% 15.9%; 97 | --sidebar-accent-foreground: 240 4.8% 95.9%; 98 | --sidebar-border: 240 3.7% 15.9%; 99 | --sidebar-ring: 217.2 91.2% 59.8%; 100 | } 101 | } 102 | 103 | @layer base { 104 | * { 105 | @apply border-border; 106 | } 107 | body { 108 | @apply bg-background text-foreground; 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /website/app/install/page.tsx: -------------------------------------------------------------------------------- 1 | import FeatureReels from "@/components/feature-reels"; 2 | import { CodeBlock } from "@/components/code-block"; 3 | import { InstallationStepHeader } from "@/components/installation-step-header"; 4 | 5 | export default function InstallationGuidePage() { 6 | return ( 7 |
8 |
9 |
10 | {/* Installation Section - Left Side */} 11 |
12 |
13 |

14 | Installation steps for Dataform tools VS Code extension. 15 | Once you have installed the extension on VS Code, follow the steps below. 16 | Alternatively, you can watch one of the setup videos shown on the right. 17 |

18 | 19 |
20 | {/* Dataform CLI */} 21 |
22 | 27 | 32 |

33 | Run dataform compile from the root of your Dataform project to ensure that you are able to use the CLI. 34 |

35 |
36 | 37 | {/* Install gcloud CLI */} 38 |
39 | 44 | 49 | 50 | 56 | 57 | 63 |
64 | 65 | {/* SQLFluff */} 66 |
67 | 72 | 77 |
78 | 79 | {/* Error Lens */} 80 |
81 | 87 |

88 | Install the Error Lens VS Code extension for prettier diagnostic messages. 89 |

90 |
91 | 92 | {/* Note */} 93 |
94 |
95 | 96 | 97 | 98 | 99 | 100 |

Note

101 |
102 |
103 |

104 | Trouble installing or looking for a specific customization? Please see FAQ section, if you are still stuck, please raise an issue here. 105 |

106 |
107 |
108 |
109 |
110 |
111 | 112 | {/* Feature Reels - Right Side */} 113 |
114 |
115 | 116 |
117 |
118 |
119 |
120 |
121 | ); 122 | } -------------------------------------------------------------------------------- /website/app/layout.tsx: -------------------------------------------------------------------------------- 1 | import type React from "react"; 2 | import "@/app/globals.css"; 3 | import { Inter } from "next/font/google"; 4 | import { ThemeProvider } from "@/components/theme-provider"; 5 | import { SiteHeader } from "@/app/components/site-header"; 6 | 7 | const inter = Inter({ subsets: ["latin"] }); 8 | 9 | export const metadata = { 10 | title: "Dataform Tools - VS Code Extension", 11 | description: 12 | "Enhance your Dataform development experience with powerful tools for SQL compilation, schema exploration, and more.", 13 | }; 14 | 15 | export default function RootLayout({ 16 | children, 17 | }: Readonly<{ 18 | children: React.ReactNode 19 | }>) { 20 | return ( 21 | 22 | 23 | 24 |
25 | 26 | {children} 27 |
28 | 29 | 30 | 31 | ); 32 | } 33 | -------------------------------------------------------------------------------- /website/app/page.tsx: -------------------------------------------------------------------------------- 1 | import { Github } from "lucide-react"; 2 | import Link from "next/link"; 3 | 4 | import { Button } from "@/components/ui/button"; 5 | import { ThemeImage } from "@/components/theme-image"; 6 | 7 | export default function Home() { 8 | return ( 9 |
10 |
11 |
12 |

13 | Dataform Tools VS Code Extension 14 |

15 |

16 | A powerful VS Code extension for Dataform (v2.x & v3.x) with features like compiled query previews, dependency graphs, 17 | inline diagnostics, schema generation, cost estimation, and more. 18 |

19 |
20 | 29 | 39 |
40 |
41 |
42 | 43 |
44 |
45 |
46 | 54 |
55 |
56 | 65 |
66 |
67 |
68 | 69 |
70 | ); 71 | } 72 | -------------------------------------------------------------------------------- /website/components.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://ui.shadcn.com/schema.json", 3 | "style": "default", 4 | "rsc": true, 5 | "tsx": true, 6 | "tailwind": { 7 | "config": "tailwind.config.ts", 8 | "css": "app/globals.css", 9 | "baseColor": "neutral", 10 | "cssVariables": true, 11 | "prefix": "" 12 | }, 13 | "aliases": { 14 | "components": "@/components", 15 | "utils": "@/lib/utils", 16 | "ui": "@/components/ui", 17 | "lib": "@/lib", 18 | "hooks": "@/hooks" 19 | }, 20 | "iconLibrary": "lucide" 21 | } -------------------------------------------------------------------------------- /website/components/code-block.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | import React, { useState } from "react"; 3 | import { Check, Copy } from "lucide-react"; 4 | import { cn } from "@/lib/utils"; 5 | import dynamic from 'next/dynamic'; 6 | import { useTheme } from "next-themes"; 7 | 8 | // Dynamically import SyntaxHighlighter with no SSR to prevent hydration mismatch 9 | const SyntaxHighlighter = dynamic( 10 | () => import('react-syntax-highlighter').then(mod => mod.Prism), 11 | { ssr: false } 12 | ); 13 | 14 | // Dynamically import styles with no SSR 15 | const getStyles = async () => { 16 | const styles = await import('react-syntax-highlighter/dist/esm/styles/prism'); 17 | return { vscDarkPlus: styles.vscDarkPlus, vs: styles.vs }; 18 | }; 19 | 20 | interface CodeBlockProps { 21 | code: string 22 | language?: string 23 | title?: string 24 | className?: string 25 | } 26 | 27 | export function CodeBlock({ code, language = "bash", title, className }: CodeBlockProps) { 28 | const [copied, setCopied] = useState(false); 29 | const [styles, setStyles] = useState(null); 30 | const { resolvedTheme } = useTheme(); 31 | const isDark = resolvedTheme === "dark"; 32 | 33 | // Load styles on the client side 34 | React.useEffect(() => { 35 | getStyles().then(loadedStyles => { 36 | setStyles(loadedStyles); 37 | }); 38 | }, []); 39 | 40 | const copyToClipboard = async () => { 41 | await navigator.clipboard.writeText(code); 42 | setCopied(true); 43 | setTimeout(() => setCopied(false), 2000); 44 | }; 45 | 46 | return ( 47 |
48 | {title && ( 49 |
50 |

{title}

51 |
52 | )} 53 |
54 | {styles ? ( 55 | 68 | {code} 69 | 70 | ) : ( 71 |
72 | 
73 |               {code}
74 |
75 |
76 | )} 77 | 88 |
89 |
90 | ); 91 | } -------------------------------------------------------------------------------- /website/components/feature-reels.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import { useState } from "react"; 4 | import { Play } from "lucide-react"; 5 | 6 | import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; 7 | 8 | interface Feature { 9 | id: string 10 | title: string 11 | description: string 12 | videoUrl: string 13 | videoId: string 14 | thumbnail: string 15 | } 16 | 17 | export default function FeatureReels() { 18 | const [playing, setPlaying] = useState>({}); 19 | 20 | const features: Feature[] = [ 21 | { 22 | id: "compile-sql", 23 | title: "", 24 | description: "Installation and features", 25 | videoUrl: "https://youtu.be/nb_OFh6YgOc?si=OO0Lsa7IpAUvlvJn", 26 | videoId: "nb_OFh6YgOc", 27 | thumbnail: "https://img.youtube.com/vi/nb_OFh6YgOc/maxresdefault.jpg", 28 | }, 29 | { 30 | id: "schema-explorer", 31 | title: "", 32 | description: "Windows installation", 33 | videoUrl: "https://youtu.be/8AsSwzmzhV4?si=QOPmqpvwGmQEIy96", 34 | videoId: "8AsSwzmzhV4", 35 | thumbnail: "https://img.youtube.com/vi/8AsSwzmzhV4/maxresdefault.jpg", 36 | }, 37 | { 38 | id: "dev-container", 39 | title: "", 40 | description: "Using in Dev container", 41 | videoUrl: "https://youtu.be/nb_OFh6YgOc?si=ilyrHrLDylKdRXna", 42 | videoId: "nb_OFh6YgOc", 43 | thumbnail: "https://img.youtube.com/vi/nb_OFh6YgOc/maxresdefault.jpg", 44 | }, 45 | ]; 46 | 47 | const togglePlay = (id: string) => { 48 | setPlaying((prev) => ({ 49 | ...prev, 50 | [id]: !prev[id], 51 | })); 52 | }; 53 | 54 | return ( 55 |
56 |
57 | {features.map((feature) => ( 58 | 59 |
60 | {playing[feature.id] ? ( 61 |