├── .nvmrc ├── images ├── icon.png ├── demo-chat.gif ├── demo-csv.gif └── demo-export.gif ├── src ├── platform │ └── common │ │ ├── misc.ts │ │ ├── stopwatch.ts │ │ ├── string.ts │ │ └── async.ts ├── csvCommand.ts ├── test │ ├── index.node.ts │ ├── mockResponseStream.ts │ └── extension.test.ts ├── extension.ts ├── issueReporter.ts ├── logger.ts ├── dataAgent.ts ├── tools.ts ├── exportCommand.ts └── base.tsx ├── .vscode ├── extensions.json ├── settings.json ├── launch.json ├── code.code-snippets └── tasks.json ├── package.nls.json ├── .vscode-test.mjs ├── .prettierrc.js ├── .editorconfig ├── CODE_OF_CONDUCT.md ├── .vscodeignore ├── eslint.config.mjs ├── tsconfig.json ├── CONTRIBUTING.md ├── LICENSE ├── scenarios ├── scenarios.md └── jamesbond.csv ├── SUPPORT.md ├── SECURITY.md ├── README.md ├── .gitignore └── package.json /.nvmrc: -------------------------------------------------------------------------------- 1 | 20.17.0 -------------------------------------------------------------------------------- /images/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/vscode-data-analysis-for-copilot/HEAD/images/icon.png -------------------------------------------------------------------------------- /images/demo-chat.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/vscode-data-analysis-for-copilot/HEAD/images/demo-chat.gif -------------------------------------------------------------------------------- /images/demo-csv.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/vscode-data-analysis-for-copilot/HEAD/images/demo-csv.gif -------------------------------------------------------------------------------- /images/demo-export.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/vscode-data-analysis-for-copilot/HEAD/images/demo-export.gif -------------------------------------------------------------------------------- /src/platform/common/misc.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | export function noop() {} 5 | -------------------------------------------------------------------------------- /.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": [ 3 | "dbaeumer.vscode-eslint", 4 | "connor4312.esbuild-problem-matchers", 5 | "ms-vscode.extension-test-runner" 6 | ] 7 | } 8 | -------------------------------------------------------------------------------- /package.nls.json: -------------------------------------------------------------------------------- 1 | { 2 | "capabilities.untrustedWorkspace.description": "Execution of code for data analysis is not supported in untrusted workspaces.", 3 | "commands.dachat.analyzeCsv.title": "Analyze using the Data Analysis Chat Participant", 4 | "commands.dachat.analyzeCsv.shortTitle": "Analyze" 5 | } 6 | -------------------------------------------------------------------------------- /.vscode-test.mjs: -------------------------------------------------------------------------------- 1 | import { defineConfig } from '@vscode/test-cli'; 2 | 3 | export default defineConfig({ 4 | files: 'out/test/**/*.test.js', 5 | version: 'insiders', 6 | mocha: { 7 | timeout: 600_000 8 | }, 9 | workspaceFolder: 'scenarios', 10 | platform:'desktop', 11 | useInstallation:{ 12 | fromMachine: true 13 | } 14 | }); 15 | -------------------------------------------------------------------------------- /.prettierrc.js: -------------------------------------------------------------------------------- 1 | // Test 2 | module.exports = { 3 | singleQuote: true, 4 | printWidth: 120, 5 | tabWidth: 4, 6 | endOfLine: 'auto', 7 | trailingComma: 'none', 8 | overrides: [ 9 | { 10 | files: ['*.yml', '*.yaml'], 11 | options: { 12 | tabWidth: 2 13 | } 14 | } 15 | ] 16 | }; 17 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # EditorConfig is awesome: http://EditorConfig.org 2 | 3 | # top-most EditorConfig file 4 | root = true 5 | 6 | # Tab indentation 7 | [*] 8 | indent_style = tab 9 | trim_trailing_whitespace = true 10 | insert_final_newline = true 11 | 12 | # The indent size used in the `package.json` file cannot be changed 13 | # https://github.com/npm/npm/pull/3180#issuecomment-16336516 14 | [{.travis.yml,npm-shrinkwrap.json,package.json}] 15 | indent_style = space 16 | indent_size = 4 17 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Microsoft Open Source Code of Conduct 2 | 3 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 4 | 5 | Resources: 6 | 7 | - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) 8 | - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) 9 | - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns 10 | -------------------------------------------------------------------------------- /.vscodeignore: -------------------------------------------------------------------------------- 1 | .vscode/** 2 | .vscode-test/** 3 | node_modules/** 4 | src/** 5 | .gitignore 6 | .nvmrc 7 | .yarnrc 8 | *.vsix 9 | eslint.config.mjs 10 | webpack.config.js 11 | vsc-extension-quickstart.md 12 | **/tsconfig.json 13 | **/.eslintrc.json 14 | **/*.map 15 | **/*.ts 16 | **/*.d.ts 17 | build/** 18 | pyodide/node/*.js.map 19 | pyodide/node/*.d.ts 20 | pyodide/common/*.js.map 21 | pyodide/common/*.d.ts 22 | pyodide/*.map 23 | pyodide.zip 24 | temp/** 25 | scenarios/** 26 | # Gifs are very large, no need to include in VSIX 27 | images/*.gif 28 | -------------------------------------------------------------------------------- /src/platform/common/stopwatch.ts: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------------------------------------------- 2 | * Copyright (c) Microsoft Corporation and GitHub. All rights reserved. 3 | *--------------------------------------------------------------------------------------------*/ 4 | 5 | 6 | /** 7 | * Tracks wall clock time. Start time is set at contruction. 8 | */ 9 | export class StopWatch { 10 | private started = Date.now(); 11 | public get elapsedTime() { 12 | return Date.now() - this.started; 13 | } 14 | public reset() { 15 | this.started = Date.now(); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /eslint.config.mjs: -------------------------------------------------------------------------------- 1 | import globals from 'globals'; 2 | import pluginJs from '@eslint/js'; 3 | import tseslint from 'typescript-eslint'; 4 | 5 | export default [ 6 | { ignores: ['pyodide/**', 'src/execution/src/**'] }, 7 | { files: ['**/*.{js,mjs,cjs,ts}'] }, 8 | { files: ['**/*.js'], languageOptions: { sourceType: 'script' } }, 9 | { languageOptions: { globals: globals.browser } }, 10 | pluginJs.configs.recommended, 11 | ...tseslint.configs.recommended, 12 | { 13 | rules: { 14 | '@typescript-eslint/no-unused-vars': ['warn', { argsIgnorePattern: '_\\w*' }] 15 | } 16 | } 17 | ]; 18 | -------------------------------------------------------------------------------- /src/csvCommand.ts: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------------------------------------------- 2 | * Copyright (c) Microsoft Corporation and GitHub. All rights reserved. 3 | *--------------------------------------------------------------------------------------------*/ 4 | 5 | import { commands, Uri, workspace } from "vscode"; 6 | 7 | export function registerCsvCommand() { 8 | return commands.registerCommand('dachat.analyzeCsv', async (file: Uri) => { 9 | await commands.executeCommand('workbench.action.chat.open'); 10 | await commands.executeCommand('workbench.action.chat.focusInput'); 11 | 12 | const relativePath = workspace.asRelativePath(file); 13 | await commands.executeCommand('workbench.action.chat.sendToNewChat', { inputValue: `@data Analyze the file ${relativePath}` }); 14 | }) 15 | } 16 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "Node16", 4 | "target": "ES2022", 5 | "lib": [ 6 | "ES2022" 7 | ], 8 | "sourceMap": true, 9 | "rootDir": "src", 10 | "strict": true /* enable all strict type-checking options */, 11 | "outDir": "out", 12 | "jsx": "react", 13 | "jsxFactory": "vscpp", 14 | "jsxFragmentFactory": "vscppf", 15 | "skipLibCheck": true, 16 | "allowSyntheticDefaultImports": true, 17 | "esModuleInterop": true, 18 | /* Additional Checks */ 19 | // "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ 20 | // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ 21 | // "noUnusedParameters": true, /* Report errors on unused parameters. */ 22 | }, 23 | "include": [ 24 | "src/**/*", 25 | "vscode.*" 26 | ], 27 | "exclude": [ 28 | "./node_modules", 29 | "./out", 30 | "./dist", 31 | "./build" 32 | ] 33 | } 34 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to this extension 2 | 3 | --- 4 | 5 | ### Prerequisites 6 | 7 | 1. [Node.js](https://nodejs.org/) (see `.nvmrc`) 8 | 4. Windows, macOS, or Linux 9 | 5. [Visual Studio Code](https://code.visualstudio.com/) 10 | 6. VS Code extensions defined in `.vscode/extensions.json` 11 | 12 | ### Setup 13 | 14 | ```shell 15 | npm i # This can be a little slow the first time (downloading and extracting of a 300Mb file). 16 | ``` 17 | 18 | You can also compile from the command-line. For a full compile you can use: 19 | 20 | ```shell 21 | npm run watch 22 | ``` 23 | 24 | For incremental builds you can use the following commands depending on your needs: 25 | 26 | ```shell 27 | npm run watch 28 | ``` 29 | 30 | Sometimes you will need to run `npm run clean` and even `rm -r out dist temp pyodide`. 31 | This is especially true if you have added or removed files. 32 | 33 | ### Errors and Warnings 34 | 35 | TypeScript errors and warnings will be displayed in the `Problems` window of Visual Studio Code. 36 | 37 | 38 | #### Building Pyodide Scripts 39 | 40 | See details in the `README.md` of the `pyodide` branch. 41 | -------------------------------------------------------------------------------- /src/platform/common/string.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | export function base64ToUint8Array(base64: string): Uint8Array { 5 | if (typeof Buffer !== 'undefined' && typeof Buffer.from === 'function') { 6 | return Buffer.from(base64, 'base64'); 7 | } else { 8 | return Uint8Array.from(atob(base64), (c) => c.charCodeAt(0)); 9 | } 10 | } 11 | 12 | const textDecoder = new TextDecoder(); 13 | export function uint8ArrayToBase64(buffer: Uint8Array): string { 14 | if (typeof Buffer !== 'undefined' && typeof Buffer.from === 'function') { 15 | return Buffer.from(buffer).toString('base64'); 16 | } else { 17 | // https://developer.mozilla.org/en-US/docs/Glossary/Base64#solution_1_%E2%80%93_escaping_the_string_before_encoding_it 18 | const stringValue = textDecoder.decode(buffer); 19 | return btoa( 20 | encodeURIComponent(stringValue).replace(/%([0-9A-F]{2})/g, function (_match, p1) { 21 | return String.fromCharCode(Number.parseInt('0x' + p1)); 22 | }) 23 | ); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/test/index.node.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | import Mocha from 'mocha'; 5 | import * as path from 'path'; 6 | 7 | type SetupOptions = Mocha.MochaOptions; 8 | 9 | 10 | /** 11 | * Configure the test environment and return the options required to run mocha tests. 12 | */ 13 | function configure(): SetupOptions { 14 | return { 15 | ui: 'tdd', 16 | color: true, 17 | timeout: 600_000, 18 | }; 19 | } 20 | 21 | /** 22 | * Runner, invoked by VS Code. 23 | * More info https://code.visualstudio.com/api/working-with-extensions/testing-extension 24 | * 25 | * @export 26 | * @returns {Promise} 27 | */ 28 | export async function run(): Promise { 29 | const mocha = new Mocha(configure()); 30 | // Setup test files that need to be run. 31 | [path.join(__dirname, 'extension.test.js')].forEach((file) => mocha.addFile(file)); 32 | 33 | // Run the tests. 34 | await new Promise((resolve, reject) => { 35 | mocha.run((failures) => { 36 | if (failures > 0) { 37 | return reject(new Error(`${failures} total failures`)); 38 | } 39 | resolve(); 40 | }); 41 | }); 42 | } 43 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "editor.insertSpaces": false, 3 | "editor.codeActionsOnSave": { 4 | "source.fixAll.eslint": "explicit", 5 | "source.fixAll.tslint": "explicit" 6 | }, 7 | "files.trimTrailingWhitespace": true, 8 | "[typescript]": { 9 | "editor.insertSpaces": false, 10 | "editor.tabSize": 4, 11 | "editor.defaultFormatter": "vscode.typescript-language-features", 12 | // "editor.formatOnSave": true, 13 | "editor.codeActionsOnSave": { 14 | "source.organizeImports": "always" 15 | } 16 | }, 17 | "[typescriptreact]": { 18 | "editor.insertSpaces": false, 19 | "editor.tabSize": 4, 20 | "editor.defaultFormatter": "vscode.typescript-language-features", 21 | // "editor.formatOnSave": true, 22 | "editor.codeActionsOnSave": { 23 | "source.organizeImports": "always" 24 | } 25 | }, 26 | "javascript.preferences.quoteStyle": "single", 27 | "typescript.preferences.importModuleSpecifier": "relative", 28 | "git.branchProtection": [ 29 | "main", 30 | "release*" 31 | ], 32 | "git.branchProtectionPrompt": "alwaysCommitToNewBranch" 33 | } 34 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | // A launch configuration that compiles the extension and then opens it inside a new window 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | { 6 | "version": "0.2.0", 7 | "configurations": [ 8 | { 9 | "name": "Run Extension", 10 | "type": "extensionHost", 11 | "request": "launch", 12 | "args": [ 13 | "--extensionDevelopmentPath=${workspaceFolder}", 14 | "${workspaceFolder}/scenarios" 15 | ], 16 | "outFiles": [ 17 | "${workspaceFolder}/out/**/*.js" 18 | ], 19 | "preLaunchTask": "watch", 20 | }, 21 | { 22 | "name": "Extension Tests", 23 | "type": "extensionHost", 24 | "request": "launch", 25 | "args": [ 26 | "--enable-proposed-api", 27 | "--extensionDevelopmentPath=${workspaceFolder}", 28 | "--extensionTestsPath=${workspaceFolder}/out/test/index.node", 29 | "${workspaceFolder}/scenarios" 30 | ], 31 | "outFiles": [ 32 | "${workspaceFolder}/out/**/*.js", 33 | "${workspaceFolder}/dist/**/*.js" 34 | ], 35 | "preLaunchTask": "watch-tests" 36 | } 37 | ] 38 | } 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /scenarios/scenarios.md: -------------------------------------------------------------------------------- 1 | ## Test long conversations with housing data 2 | 3 | Send the following requests, LLM should be able to remember the context and respond to the requests accordingly. 4 | 5 | ``` 6 | @data help me analyze housing.csv 7 | @data visualize the data 8 | @data try to find the correlation between housing value and location 9 | ``` 10 | 11 | ## Retry if execution fails 12 | 13 | ``` 14 | // https://www.kaggle.com/datasets/dreb87/jamesbond 15 | // Change the column names and retry a few times (i.e. ensure LLM isn't guessing the column names) 16 | @data display a histogram of movies per bond actor from jamesbond.csv file 17 | ``` 18 | 19 | 20 | ## Analyzing housing data (or any other dataset you want to test) 21 | 22 | Try with #file and also with mentioning your_dataset.csv like the very first example: 23 | 24 | ``` 25 | @data help me analyze housing.csv 26 | ``` 27 | 28 | ``` 29 | @data analyze data or dataframe 30 | ``` 31 | 32 | ``` 33 | @data perform inferential statistics 34 | ``` 35 | 36 | ``` 37 | @data perform further advanced data analysis 38 | ``` 39 | 40 | ``` 41 | @data maybe correlation analysis 42 | ``` 43 | 44 | 45 | TODO: 46 | ``` 47 | Create visualization after runing relevant Python code 48 | ``` 49 | -------------------------------------------------------------------------------- /SUPPORT.md: -------------------------------------------------------------------------------- 1 | # TODO: The maintainer of this repo has not yet edited this file 2 | 3 | **REPO OWNER**: Do you want Customer Service & Support (CSS) support for this product/project? 4 | 5 | - **No CSS support:** Fill out this template with information about how to file issues and get help. 6 | - **Yes CSS support:** Fill out an intake form at [aka.ms/onboardsupport](https://aka.ms/onboardsupport). CSS will work with/help you to determine next steps. 7 | - **Not sure?** Fill out an intake as though the answer were "Yes". CSS will help you decide. 8 | 9 | *Then remove this first heading from this SUPPORT.MD file before publishing your repo.* 10 | 11 | # Support 12 | 13 | ## How to file issues and get help 14 | 15 | This project uses GitHub Issues to track bugs and feature requests. Please search the existing 16 | issues before filing new issues to avoid duplicates. For new issues, file your bug or 17 | feature request as a new Issue. 18 | 19 | For help and questions about using this project, please **REPO MAINTAINER: INSERT INSTRUCTIONS HERE 20 | FOR HOW TO ENGAGE REPO OWNERS OR COMMUNITY FOR HELP. COULD BE A STACK OVERFLOW TAG OR OTHER 21 | CHANNEL. WHERE WILL YOU HELP PEOPLE?**. 22 | 23 | ## Microsoft Support Policy 24 | 25 | Support for this **PROJECT or PRODUCT** is limited to the resources listed above. 26 | -------------------------------------------------------------------------------- /.vscode/code.code-snippets: -------------------------------------------------------------------------------- 1 | { 2 | // Place your Advanced-Data-Analysis-for-Copilot workspace snippets here. Each snippet is defined under a snippet name and has a scope, prefix, body and 3 | // description. Add comma separated ids of the languages where the snippet is applicable in the scope field. If scope 4 | // is left empty or omitted, the snippet gets applied to all languages. The prefix is what is 5 | // used to trigger the snippet and the body will be expanded and inserted. Possible variables are: 6 | // $1, $2 for tab stops, $0 for the final cursor position, and ${1:label}, ${2:another} for placeholders. 7 | // Placeholders with the same ids are connected. 8 | // Example: 9 | // "Print to console": { 10 | // "scope": "javascript,typescript", 11 | // "prefix": "log", 12 | // "body": [ 13 | // "console.log('$1');", 14 | // "$2" 15 | // ], 16 | // "description": "Log output to console" 17 | // } 18 | "License": { 19 | "scope": "javascript,typescript", 20 | "prefix": "lic", 21 | "body": [ 22 | "/*---------------------------------------------------------------------------------------------", 23 | "* Copyright (c) Microsoft Corporation and GitHub. All rights reserved.", 24 | "*--------------------------------------------------------------------------------------------*/", 25 | "", 26 | "$1" 27 | ], 28 | "description": "Microsoft License header" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/extension.ts: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------------------------------------------- 2 | * Copyright (c) Microsoft Corporation and GitHub. All rights reserved. 3 | *--------------------------------------------------------------------------------------------*/ 4 | 5 | import * as vscode from 'vscode'; 6 | import { registerCsvCommand } from './csvCommand'; 7 | import { DataAgent } from './dataAgent'; 8 | import { registerIssueReporter } from './issueReporter'; 9 | import { initializeLogger } from './logger'; 10 | import { FindFilesTool, InstallPythonPackageTool, RunPythonTool } from './tools'; 11 | 12 | export function activate(context: vscode.ExtensionContext) { 13 | const dataAgent = new DataAgent(context); 14 | const logger = initializeLogger(context); 15 | context.subscriptions.push(logger); 16 | context.subscriptions.push(dataAgent); 17 | context.subscriptions.push(registerCsvCommand()); 18 | context.subscriptions.push(registerIssueReporter(context)); 19 | context.subscriptions.push(vscode.lm.registerTool(FindFilesTool.Id, new FindFilesTool(context))); 20 | const pythonTool = new RunPythonTool(context); 21 | context.subscriptions.push(vscode.lm.registerTool(RunPythonTool.Id, pythonTool)); 22 | context.subscriptions.push(vscode.lm.registerTool(InstallPythonPackageTool.Id, new InstallPythonPackageTool(pythonTool))); 23 | 24 | if (context.extensionMode === vscode.ExtensionMode.Test) { 25 | return { 26 | dataAgent 27 | } 28 | } 29 | } 30 | 31 | export function deactivate() { } 32 | -------------------------------------------------------------------------------- /src/issueReporter.ts: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------------------------------------------- 2 | * Copyright (c) Microsoft Corporation and GitHub. All rights reserved. 3 | *--------------------------------------------------------------------------------------------*/ 4 | 5 | import { commands, ExtensionContext } from 'vscode'; 6 | import { getLastErrors } from './logger'; 7 | 8 | export function registerIssueReporter(context: ExtensionContext) { 9 | return commands.registerCommand('dachat.reportIssue', () => { 10 | commands.executeCommand('workbench.action.openIssueReporter', { 11 | extensionId: context.extension.id, 12 | issueBody: issueBody, 13 | data: getIssueData() 14 | }); 15 | }); 16 | } 17 | 18 | const issueBody = ` 19 | 20 | # Behaviour 21 | 22 | XXX 23 | 24 | ## Steps to reproduce: 25 | 26 | 1. XXX 27 | 28 | 32 | 33 | 34 | 35 | 36 | 37 | # Outputs 38 | 39 |
40 | 41 | Output from Data Analysis Output Panel 42 | 43 |

44 | 45 | \`\`\` 46 | XXX 47 | \`\`\` 48 | 49 |

50 |
51 | `; 52 | 53 | 54 | function getIssueData() { 55 | const error = getLastErrors().trim(); 56 | if (!error) { 57 | return ''; 58 | } 59 | return ` 60 |
61 | Last few Errors 62 |

63 | 64 | \`\`\` 65 | ${error} 66 | \`\`\` 67 |

68 |
69 | `; 70 | }; 71 | -------------------------------------------------------------------------------- /src/logger.ts: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------------------------------------------- 2 | * Copyright (c) Microsoft Corporation and GitHub. All rights reserved. 3 | *--------------------------------------------------------------------------------------------*/ 4 | 5 | import { ExtensionContext, ExtensionMode, LogOutputChannel, window } from "vscode"; 6 | import { StopWatch } from "./platform/common/stopwatch"; 7 | 8 | let logger: LogOutputChannel; 9 | 10 | const lastSeenError = { 11 | timer: new StopWatch(), 12 | error: '' 13 | } 14 | 15 | export function initializeLogger(extensionContext: ExtensionContext) { 16 | if (!logger) { 17 | logger = window.createOutputChannel('Data Analysis', { log: true }); 18 | const debug = logger.debug; 19 | logger.debug = (message: string, ...args: unknown[]) => { 20 | if (extensionContext.extensionMode === ExtensionMode.Development) { 21 | console.log(message, ...args); 22 | } 23 | 24 | debug.bind(logger)(message, ...args); 25 | }; 26 | const error = logger.error; 27 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 28 | logger.error = (errorMsg: string | Error, ...args: any[]) => { 29 | // Get track of the last known error for issue reporting purposes. 30 | lastSeenError.timer.reset(); 31 | lastSeenError.error = [`${getTime()} ${errorMsg.toString()}`].concat(args.map(arg => `${arg}`)).join('\n'); 32 | error.bind(logger)(errorMsg, ...args); 33 | } 34 | } 35 | 36 | return logger; 37 | } 38 | 39 | 40 | function getTime() { 41 | const now = new Date(); 42 | return now.toTimeString().split(' ')[0]; 43 | } 44 | 45 | function getLastErrors() { 46 | // If we haven't see any errors in the past 20 minutes, no point reporting any old errors. 47 | if (!lastSeenError.error || lastSeenError.timer.elapsedTime > 20 * 60 * 1000) { 48 | return ''; 49 | } 50 | return lastSeenError.error; 51 | } 52 | 53 | export { getLastErrors, logger }; 54 | 55 | -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | // See https://go.microsoft.com/fwlink/?LinkId=733558 2 | // for the documentation about the tasks.json format 3 | { 4 | "version": "2.0.0", 5 | "tasks": [ 6 | { 7 | "label": "watch", 8 | "dependsOn": [ 9 | "npm: watch-extension", 10 | "npm: watch-types" 11 | ], 12 | "presentation": { 13 | "reveal": "never", 14 | }, 15 | "group": { 16 | "kind": "build", 17 | "isDefault": true 18 | }, 19 | "runOptions": { 20 | "runOn": "folderOpen" 21 | } 22 | }, 23 | { 24 | "type": "npm", 25 | "script": "watch-extension", 26 | "group": "build", 27 | "problemMatcher": "$esbuild-watch", 28 | "isBackground": true, 29 | "label": "npm: watch-extension", 30 | "presentation": { 31 | "group": "watch", 32 | "reveal": "never" 33 | } 34 | }, 35 | { 36 | "type": "npm", 37 | "script": "watch-types", 38 | "group": "build", 39 | "problemMatcher": "$tsc-watch", 40 | "isBackground": true, 41 | "label": "npm: watch-types", 42 | "presentation": { 43 | "group": "watch", 44 | "reveal": "never" 45 | } 46 | }, 47 | { 48 | "label": "watch-tests", 49 | "dependsOn": [ 50 | "npm: watch-tests" 51 | ], 52 | "presentation": { 53 | "reveal": "never", 54 | }, 55 | "group": { 56 | "kind": "build", 57 | "isDefault": false 58 | } 59 | }, 60 | { 61 | "type": "npm", 62 | "script": "watch-tests", 63 | "group": "build", 64 | "problemMatcher": "$tsc-watch", 65 | "isBackground": true, 66 | "label": "npm: watch-tests", 67 | "presentation": { 68 | "group": "watch", 69 | "reveal": "never" 70 | } 71 | } 72 | ] 73 | } 74 | -------------------------------------------------------------------------------- /src/platform/common/async.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | 5 | //====================== 6 | // Deferred 7 | 8 | 9 | export interface Deferred { 10 | readonly promise: Promise; 11 | readonly resolved: boolean; 12 | readonly rejected: boolean; 13 | readonly completed: boolean; 14 | readonly value?: T; 15 | resolve(value?: T | PromiseLike): void; 16 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 17 | reject(reason?: any): void; 18 | } 19 | 20 | class DeferredImpl implements Deferred { 21 | private _resolve!: (value: T | PromiseLike) => void; 22 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 23 | private _reject!: (reason?: any) => void; 24 | private _resolved: boolean = false; 25 | private _rejected: boolean = false; 26 | private _promise: Promise; 27 | private _value: T | undefined; 28 | public get value() { 29 | return this._value; 30 | } 31 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 32 | constructor(private scope: any = null) { 33 | 34 | this._promise = new Promise((res, rej) => { 35 | this._resolve = res; 36 | this._reject = rej; 37 | }); 38 | } 39 | public resolve(value?: T | PromiseLike) { 40 | this._value = value as T | undefined; 41 | // eslint-disable-next-line @typescript-eslint/no-explicit-any, prefer-rest-params 42 | this._resolve.apply(this.scope ? this.scope : this, arguments as any); 43 | this._resolved = true; 44 | } 45 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 46 | public reject(_reason?: any) { 47 | // eslint-disable-next-line @typescript-eslint/no-explicit-any, prefer-rest-params 48 | this._reject.apply(this.scope ? this.scope : this, arguments as any); 49 | this._rejected = true; 50 | } 51 | get promise(): Promise { 52 | return this._promise; 53 | } 54 | get resolved(): boolean { 55 | return this._resolved; 56 | } 57 | get rejected(): boolean { 58 | return this._rejected; 59 | } 60 | get completed(): boolean { 61 | return this._rejected || this._resolved; 62 | } 63 | } 64 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 65 | export function createDeferred(scope: any = null): Deferred { 66 | return new DeferredImpl(scope); 67 | } 68 | -------------------------------------------------------------------------------- /src/test/mockResponseStream.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | /* eslint-disable @typescript-eslint/no-explicit-any */ 5 | import { ChatResponseAnchorPart, ChatResponseCommandButtonPart, type ChatResponseFileTree, ChatResponseFileTreePart, ChatResponseMarkdownPart, type ChatResponsePart, ChatResponseProgressPart, ChatResponseReferencePart, type ChatResponseStream, type Command, type Location, type MarkdownString, TextEdit, type ThemeIcon, type Uri } from "vscode"; 6 | 7 | 8 | export class MockChatResponseStream implements ChatResponseStream { 9 | public readonly parts: ChatResponsePart[] = []; 10 | public readonly edits = new Map(); 11 | markdown(value: string | MarkdownString): void { 12 | if (this.parts.length > 0) { 13 | const item = this.parts[this.parts.length - 1]; 14 | if (item instanceof ChatResponseMarkdownPart) { 15 | this.parts[this.parts.length - 1] = new ChatResponseMarkdownPart(item.value.value + (typeof value === 'string' ? value : value.value)); 16 | return; 17 | } 18 | } 19 | this.parts.push(new ChatResponseMarkdownPart(value)); 20 | } 21 | anchor(value: Uri | Location, title?: string): void { 22 | this.parts.push(new ChatResponseAnchorPart(value, title)); 23 | } 24 | button(command: Command): void { 25 | this.parts.push(new ChatResponseCommandButtonPart(command)); 26 | } 27 | filetree(value: ChatResponseFileTree[], baseUri: Uri): void { 28 | this.parts.push(new ChatResponseFileTreePart(value, baseUri)); 29 | } 30 | progress(value: string): void { 31 | this.parts.push(new ChatResponseProgressPart(value)); 32 | } 33 | reference(value: Uri | Location, iconPath?: Uri | ThemeIcon | { light: Uri; dark: Uri; }): void { 34 | this.parts.push(new ChatResponseReferencePart(value, iconPath)); 35 | } 36 | push(part: ChatResponsePart): void { 37 | if (part instanceof ChatResponseMarkdownPart) { 38 | this.markdown(part.value); 39 | } else { 40 | this.parts.push(part); 41 | } 42 | } 43 | textEdit(_target: Uri, _edits: TextEdit | TextEdit[]): void { 44 | // 45 | } 46 | codeblockUri(_uri: Uri): void { 47 | // 48 | } 49 | confirmation(_title: string, _message: string, _data: any, _buttons?: string[]): void { 50 | 51 | } 52 | warning(_message: string | MarkdownString): void { 53 | // 54 | } 55 | codeCitation(_value: Uri, _license: string, _snippet: string): void { 56 | // 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet) and [Xamarin](https://github.com/xamarin). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/security.md/definition), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/security.md/msrc/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/security.md/msrc/pgp). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/security.md/msrc/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/security.md/cvd). 40 | 41 | 42 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Data Analysis for Copilot 2 | 3 | Data Analysis for Copilot empowers people in data science field. From cleaning up user's .csv file to performing higher level of data analysis by leveraging different statistics measures, graphs, and predictive models, the @data agent helps user make more advanced and informed decisions by offering tailored insights and interactivity for data tasks. The extension contributes a tool where the LLM can ask it to execute Python code via using [Pyodide](https://pyodide.org/en/stable/) and get the result of the relevant Python code execution. It is also able to smartly re-try for better or more appropriate execution results in case of error or failure. You can also export the code used to perform the analysis (or generate visualizations) into a Jupyter Notebook or a Python file. 4 | 5 | #### Data analysis and visualizations 6 | 7 | * Given a csv file enter the prompt such as `Analyze the file #` or write a more specific prompt (see below recording). 8 | * Provide follow up prompts requesting the generation of visualizations (charts, plots or the like). 9 | 10 | ![Data Analysis of CSV file with visualizations](https://raw.githubusercontent.com/microsoft/vscode-data-analysis-for-copilot/refs/heads/main/images/demo-chat.gif) 11 | 12 | #### Exporting the code used to perform the data analysis and generate the visualizations 13 | 14 | * Python code used to perform the analysis and generate visualizations can be viewed. 15 | * Code can be exported in Jupyter Notebooks or a plain Python file 16 | 17 | ![Exporting the code used to perform the analysis](https://raw.githubusercontent.com/microsoft/vscode-data-analysis-for-copilot/refs/heads/main/images/demo-export.gif) 18 | 19 | #### Editor and explorer integrations for *.csv files 20 | 21 | * Right click on a csv file to analyze it. 22 | * Open a csv file and use the icon to analyze the file. 23 | 24 | ![Editor and explorer integration to analyze csv files](https://raw.githubusercontent.com/microsoft/vscode-data-analysis-for-copilot/refs/heads/main/images/demo-csv.gif) 25 | 26 | 27 | ## Contributing 28 | 29 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 30 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 31 | the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. 32 | 33 | When you submit a pull request, a CLA bot will automatically determine whether you need to provide 34 | a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions 35 | provided by the bot. You will only need to do this once across all repos using our CLA. 36 | 37 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 38 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 39 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 40 | 41 | ## Trademarks 42 | 43 | This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft 44 | trademarks or logos is subject to and must follow 45 | [Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general). 46 | Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship. 47 | Any use of third-party trademarks or logos are subject to those third-party's policies. 48 | -------------------------------------------------------------------------------- /src/dataAgent.ts: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------------------------------------------- 2 | * Copyright (c) Microsoft Corporation and GitHub. All rights reserved. 3 | *--------------------------------------------------------------------------------------------*/ 4 | 5 | import { ChatMessage, HTMLTracer, PromptRenderer, toVsCodeChatMessages } from '@vscode/prompt-tsx'; 6 | import * as vscode from 'vscode'; 7 | import { DataAgentPrompt, PromptProps, ToolCallRound, ToolResultMetadata, TsxToolUserMetadata } from './base'; 8 | import { Exporter } from './exportCommand'; 9 | import { logger } from './logger'; 10 | 11 | const DATA_AGENT_PARTICIPANT_ID = 'dachat.data'; 12 | export const MODEL_SELECTOR: vscode.LanguageModelChatSelector = { 13 | vendor: 'copilot', 14 | family: 'gpt-4o' 15 | }; 16 | 17 | 18 | export class DataAgent implements vscode.Disposable { 19 | private _disposables: vscode.Disposable[] = []; 20 | private readonly exporter: Exporter; 21 | constructor(readonly extensionContext: vscode.ExtensionContext) { 22 | this.exporter = new Exporter(extensionContext); 23 | this._disposables.push(vscode.chat.createChatParticipant(DATA_AGENT_PARTICIPANT_ID, this.handle.bind(this))); 24 | } 25 | 26 | dispose() { 27 | this._disposables.forEach((d) => d.dispose()); 28 | } 29 | 30 | private async _renderMessages(chat: vscode.LanguageModelChat, props: PromptProps, stream: vscode.ChatResponseStream) { 31 | const renderer = new PromptRenderer({ modelMaxPromptTokens: chat.maxInputTokens }, DataAgentPrompt, props, { 32 | tokenLength: async (text, _token) => { 33 | return chat.countTokens(text); 34 | }, 35 | countMessageTokens: async (message: ChatMessage) => { 36 | return chat.countTokens(message.content); 37 | } 38 | }); 39 | const tracer = new HTMLTracer(); 40 | renderer.tracer = tracer; 41 | const result = await renderer.render(); 42 | 43 | if (this.extensionContext.extensionMode === vscode.ExtensionMode.Development) { 44 | const server = await tracer.serveHTML(); 45 | logger.info('Server address:', server.address); 46 | const serverUri = vscode.Uri.parse(server.address); 47 | stream.reference(serverUri); 48 | } 49 | 50 | return result; 51 | } 52 | 53 | public async handle( 54 | request: vscode.ChatRequest, 55 | chatContext: vscode.ChatContext, 56 | stream: vscode.ChatResponseStream, 57 | token: vscode.CancellationToken 58 | ): Promise { 59 | const models = await vscode.lm.selectChatModels(MODEL_SELECTOR); 60 | if (!models || !models.length) { 61 | logger.warn('NO MODELS'); 62 | return {}; 63 | } 64 | 65 | if (request.command && this.exporter.canHandle(request.command)) { 66 | this.exporter.invoke(request, chatContext, stream, token); 67 | return {}; 68 | } 69 | 70 | const chat = models[0]; 71 | 72 | const allTools: vscode.LanguageModelChatTool[] = vscode.lm.tools.map((tool) => { 73 | return { 74 | name: tool.name, 75 | description: tool.description, 76 | inputSchema: tool.inputSchema, 77 | }; 78 | }); 79 | 80 | const options: vscode.LanguageModelChatRequestOptions = { 81 | tools: allTools, 82 | justification: 'Analyzing data to provide insights and recommendations.' 83 | }; 84 | 85 | const result = await this._renderMessages(chat, { userQuery: request.prompt, references: request.references, history: chatContext.history, currentToolCallRounds: [], toolInvocationToken: request.toolInvocationToken, extensionContext: this.extensionContext }, stream); 86 | let messages = toVsCodeChatMessages(result.messages); 87 | const toolReferences = [...request.toolReferences]; 88 | const toolCallRounds: ToolCallRound[] = []; 89 | 90 | const runWithFunctions = async (): Promise => { 91 | const requestedTool = toolReferences.shift(); 92 | if (requestedTool) { 93 | options.toolMode = vscode.LanguageModelChatToolMode.Required; 94 | options.tools = allTools.filter((tool) => (tool.name === requestedTool.name)); 95 | } else { 96 | options.toolMode = undefined; 97 | options.tools = allTools; 98 | } 99 | 100 | logger.debug('Sending request', JSON.stringify(messages)); 101 | const toolCalls: vscode.LanguageModelToolCallPart[] = []; 102 | 103 | stream.progress('Analyzing'); 104 | const response = await chat.sendRequest(messages, options, token); 105 | if (response.stream) { 106 | for await (const part of response.stream) { 107 | if (part instanceof vscode.LanguageModelTextPart) { 108 | stream.markdown(part.value); 109 | } else if (part instanceof vscode.LanguageModelToolCallPart) { 110 | logger.info('Received tool call', part.name); 111 | const tool = vscode.lm.tools.find((tool) => (tool.name === part.name)); 112 | if (!tool) { 113 | // BAD tool choice? 114 | stream.progress(`Unknown function: ${part.name}`); 115 | continue; 116 | } 117 | 118 | toolCalls.push(part); 119 | } 120 | } 121 | } 122 | 123 | if (toolCalls.length) { 124 | const currentRound: ToolCallRound = { 125 | toolCalls: toolCalls, 126 | response: {} 127 | }; 128 | toolCallRounds.push(currentRound); 129 | 130 | const result = await this._renderMessages(chat, { userQuery: request.prompt, references: request.references, history: chatContext.history, currentToolCallRounds: toolCallRounds, toolInvocationToken: request.toolInvocationToken, extensionContext: this.extensionContext }, stream); 131 | const toolResultMetadata = result.metadata.getAll(ToolResultMetadata) 132 | messages = toVsCodeChatMessages(result.messages); 133 | logger.info('Token count', result.tokenCount); 134 | if (toolResultMetadata?.length) { 135 | toolResultMetadata.forEach(meta => { 136 | if (currentRound.toolCalls.find(tc => tc.callId === meta.toolCallId)) { 137 | currentRound.response[meta.toolCallId] = meta.result; 138 | } 139 | }); 140 | } 141 | 142 | return runWithFunctions(); 143 | } 144 | }; 145 | 146 | await runWithFunctions(); 147 | 148 | return { 149 | metadata: { 150 | toolCallsMetadata: { 151 | toolCallRounds 152 | } 153 | } satisfies TsxToolUserMetadata 154 | } 155 | } 156 | } 157 | -------------------------------------------------------------------------------- /scenarios/jamesbond.csv: -------------------------------------------------------------------------------- 1 | ReleaseYear,Movie,BondActorName,Director,Composer,Writer,Cinematographer,Depicted_Film_Loc,Shooting_Loc,Bond_Car_MFG,Bond_Girl_Nat,US_Gross,US_Adj,World_Gross,World_Adj,Budget,Budget_Adj,Film_Length,Avg_User_IMDB,Avg_User_Rtn_Tom,Conquests,Martinis,BJB,Kills_Bond,Kills_Others,Top_100,Video_Game 2 | 1962,Dr. No,Sean Connery,Terence Young,Monty Norman,"Richard Maibaum, Johanna Harwood & Berkely Mather",Ted Moore,"Great Britain, Jamaic","England, Jamaica",Sunbeam,"Swiss, English, English",16067035,123517,59567035,457928,1000,7688,110,7.30,7.70,3,2,1,4,8,0,0 3 | 1963,From Russia with Love,Sean Connery,Terence Young,John Barry,Richard Maibaum & Johanna Harwood,Ted Moore,"United Kingdom, Great Britain, Turkey, Croatia, Serbia, Italy","England, Scotland, Italy, Switzerland, Turkey",Bently,"English, Italian, English, Israeli",24800000,188161,78900000,598624,2000,15174,115,7.50,8.00,4,0,0,11,16,0,1 4 | 1964,Goldfinger,Sean Connery,Guy Hamilton,John Barry,Richard Maibaum & Paul Dehn,Ted Moore,"United States, Great Britain, Switzerland","England, Switzerland, United States",Aston Martin,"Serbian, English",51100000,382699,124900000,935404,3000,22468,110,7.80,8.40,2,1,2,9,68,1,1 5 | 1965,Thunderball,Sean Connery,Terence Young,John Barry,Richard Maibaum & John Hopkins,Ted Moore,"France, Great Britain, Bahamas, United States","England, France, Bahamas, United States",Aston Martin,"French, Italian, Bahamian",63600000,468754,141200000,1040693,9000,66333,130,7.00,6.80,3,0,0,20,90,1,0 6 | 1967,You Only Live Twice,Sean Connery,Lewis Gilbert,John Barry,Roald Dahl,Freddie Young,"United States, Russia, Kazakhstan, Norway, Japan","Japan, Spain, Norway",Toyota,"Japanese, Chinese, German",43100000,299591,111600000,775740,9500,66035,117,6.90,6.30,3,1,0,21,175,1,0 7 | 1969,On Her Majesty's Secret Service,George Lazenby,Peter R. Hunt,John Barry,Richard Maibaum,Michael Reed,"Portugal, Great Britain, Switzerland","England, Switzerland, Portugal",Mercury,"English, English, Hungarian",22800000,144234,82000000,518736,8000,50608,142,6.80,6.70,3,1,2,5,37,0,0 8 | 1971,Diamonds Are Forever,Sean Connery,Guy Hamilton,John Barry,Richard Maibaum & Tom Mankiewicz,Ted Moore,"Japan, Egypt, France, South Africa, Great Britain, Netherlands, United States, Mexico","England, France, Germany, Netherlands, United States",Ford,American,43800000,251083,116000000,664969,7200,41274,120,6.70,6.30,1,0,1,7,42,1,0 9 | 1973,Live and Let Die,Roger Moore,Guy Hamilton,George Martin,Tom Mankiewicz,Ted Moore,"United States, Great Britain, Jamaica","England, United States, Jamaica",AMC,"English, American, English",35400000,185105,161800000,846046,7000,36603,121,6.80,5.90,3,0,1,8,5,1,1 10 | 1974,The Man with the Golden Gun,Roger Moore,Guy Hamilton,John Barry,Richard Maibaum & Tom Mankiewicz,Ted Moore,"Great Britain, Lebanon, Portugal, Thailand","England, Thailand, Hong Kong, Portugal",AMC,"Sweedish, French",21000000,98894,97600000,459623,7000,32965,125,6.70,5.10,2,0,2,1,5,0,0 11 | 1977,The Spy Who Loved Me,Roger Moore,Lewis Gilbert,Marvin Hamlisch,Christopher Wood & Richard Maibaum,Claude Renoir,"Austria, Russia, Great Britain, Egypt, Italy","England, Switzerland, Canada, Scotland, Egypt, Italy, Malta, Bahamas, Japan",Lotus,"American, English, Bosnian",46800000,179297,185400000,710290,14000,53636,125,7.10,6.80,3,1,1,31,116,1,1 12 | 1979,Moonraker,Roger Moore,Lewis Gilbert,John Barry,Christopher Wood,Jean Tournier,"Great Britain, United States, Italy, Brazil, Russia","England, France, Italy, United States, Brazil, Guatemala",Lotus,"American, French, Moroccan",70300000,224811,210300000,672514,31000,99134,126,6.20,5.70,3,1,1,12,69,0,0 13 | 1981,For Your Eyes Only,Roger Moore,John Glen,Bill Conti,Michael G. Wilson & Richard Maibaum,Alan Hume,"Great Britain, Russia, Spain, Italy, Greece, Albania","England, Italy, Malta, Greece",Citroën,"French, Australian",54800000,139964,195300000,498812,28000,71514,127,6.80,6.30,2,0,2,18,36,1,0 14 | 1983,Octopussy,Roger Moore,John Glen,John Barry,"George MacDonald Fraser, Michael G. Wilson & Richard Maibaum",Alan Hume,"Great Britain, India, Russia, Germany","England, West Berlin, India",Bajaj,"Sweedish, English",67900000,158274,187500000,437059,27500,64102,131,6.50,5.30,2,0,1,15,43,1,0 15 | 1985,A View to a Kill,Roger Moore,John Glen,John Barry,Michael G. Wilson & Richard Maibaum,Alan Hume,"Russia, Great Britain, France, United States","England, Switzerland, Iceland, France, United States",Rolls Royce,"American, Jamaican, Sweedish, Nigerian",50327960,108592,152627960,329322,30000,64730,131,6.20,4.70,4,0,2,5,57,1,0 16 | 1987,The Living Daylights,Timothy Dalton,John Glen,John Barry,Michael G. Wilson & Richard Maibaum,Alec Mills,"Gibraltar, Great Britain, Slovakia, Austria, Morocco, Afghanistan, Pakistan","England, Germany, Austria, Italy, Morocco",Rolls Royce,"Dutch, American",51185000,104608,191200000,390758,40000,81749,130,6.70,6.30,2,2,1,13,29,0,0 17 | 1989,License to Kill,Timothy Dalton,John Glen,Michael Kamen,Michael G. Wilson & Richard Maibaum,Alec Mills,"United States, Bahamas, Great Britain","Mexico, United States",Aston Martin,"American, American",34667015,64907,156167015,292392,42000,78637,133,6.50,6.00,2,1,1,10,13,0,1 18 | 1995,GoldenEye,Pierce Brosnan,Martin Campbell,Éric Serra,"Michael France, Jeffrey Caine, Kevin Wade & Bruce Feirstein",Phil Méheux,"Russia, Monaco, France, England, United States, Cuba","England, France, Monaco, Switzerland, Russia, Puerto Rico",BMW,"Polish, Dutch",106429941,162135,356429941,542985,60000,91404,130,7.20,6.90,2,1,1,47,25,0,1 19 | 1997,Tomorrow Never Dies,Pierce Brosnan,Roger Spottiswoode,David Arnold,Bruce Feirstein,Robert Elswit,"Russia, England, Germany, Vietnam","England, France, Germany, Thailand",Aston Martin,"Malaysian, American, Danish",125304276,181254,339504276,491098,110000,159117,119,6.40,6.00,3,1,1,30,24,0,1 20 | 1999,The World Is Not Enough,Pierce Brosnan,Michael Apted,David Arnold,"Neal Purvis, Robert Wade & Bruce Feirstein",Adrian Biddle,"Spain, England, Scotland, Kazakhstan, Azerbaijan, Turkey, North Korea, South Korea","England, Scotland, France, Spain, Turkey, Azerbaijan",BMW,"American, French, English",126930660,176885,361730660,504091,135000,188130,128,6.30,5.70,3,1,2,27,43,0,1 21 | 2002,Die Another Day,Pierce Brosnan,Lee Tamahori,David Arnold,Neal Purvis and Robert Wade,David Tattersall,"South Korea, Hong Kong, Cube, England, Iceland","England, Spain, Iceland, Norway, United States",Aston Martin,"American, Filipino",160942139,207700,431942139,557433,142000,183255,133,6.00,6.10,2,2,1,31,20,1,0 22 | 2006,Casino Royale,Daniel Craig,Martin Campbell,David Arnold,"Neal Purvis, Robert Wade & Paul Haggis",Phil Méheux,"Czech Republic, Pakistan, Uganda, Madagascar, Bahamas, England, United States, Monenegro, Italy","England, Italy, Czech republic, Bahamas, United States",Aston Martin,"French, Italian",167365000,192740,596365000,686784,102000,117465,144,7.90,7.80,2,3,1,11,11,1,0 23 | 2008,Quantum of Solace,Daniel Craig,Marc Forster,David Arnold,Paul Haggis and Neal Purvis and Robert Wade,Roberto Schaefer,"Italy, England, Haiti, Austria, Bolivia, Russia","England, Italy, Spain, Panama, Mexico, Chile, Austria",Aston Martin,English,169368427,182633,591692078,638035,230000,248014,106,6.70,6.10,1,6,0,16,15,1,1 24 | 2012,Skyfall,Daniel Craig,Sam Mendes,Thomas Newman,"Neal Purvis, Robert Wade & John Logan",Roger Deakins,"Turkey, England, China, Macau, Japan, Scotland","England, Scotland, Turkey, China",Aston Martin,"French, Greek, English",304360277,307770,1108561108,1120980,200000,202240,143,7.80,8.20,3,1,1,26,26,1,0 25 | 2015,Spectre,Daniel Craig,Sam Mendes,Thomas Newman,"John Logan & Neal Purvis, Robert Wade & Jez Butterworth",Hoyte van Hoytema,"Mexico, England, Italy, Austria, Morocco, Vatican City","England, Austria, Italy, Vatican City, Mexico, Morocco",Aston Martin,"French, Italian, Mexican",200074175,196647,879620923,864553,245000,240803,148,6.80,6.40,3,1,1,30,205,1,0 -------------------------------------------------------------------------------- /src/test/extension.test.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | /* eslint-disable @typescript-eslint/no-explicit-any */ 5 | 6 | import { assert } from 'chai'; 7 | import { CancellationTokenSource, ChatResponseMarkdownPart, commands, extensions, LanguageModelChat, lm } from 'vscode'; 8 | import { getToolResultValue, isErrorMessageResponse, ToolCallRound } from '../base'; 9 | import { DataAgent, MODEL_SELECTOR } from '../dataAgent'; 10 | import { FindFilesTool, RunPythonTool } from '../tools'; 11 | import { MockChatResponseStream } from './mockResponseStream'; 12 | 13 | suite('Extension Test Suite', () => { 14 | let dataAgent: DataAgent; 15 | let tokenSource: CancellationTokenSource; 16 | let model: LanguageModelChat; 17 | // let stubRenderMessages: sinon.SinonStub; 18 | suiteSetup(async function () { 19 | await Promise.all([ 20 | extensions.getExtension('GitHub.copilot-chat')!.activate(), 21 | extensions.getExtension('ms-vscode.vscode-copilot-data-analysis')!.activate() 22 | ]); 23 | await commands.executeCommand('workbench.action.chat.open'); 24 | tokenSource = new CancellationTokenSource(); 25 | dataAgent = extensions.getExtension('ms-vscode.vscode-copilot-data-analysis')!.exports.dataAgent; 26 | const models = await lm.selectChatModels(MODEL_SELECTOR); 27 | if (!models || !models.length) { 28 | throw new Error('NO MODELS'); 29 | } 30 | model = models[0]; 31 | }); 32 | suiteTeardown(() => { 33 | tokenSource.dispose(); 34 | // stubRenderMessages.restore(); 35 | }); 36 | async function sendChatMessage(prompt: string) { 37 | const stream = new MockChatResponseStream(); 38 | const result = await dataAgent.handle({ 39 | command: undefined, 40 | prompt, 41 | references: [], 42 | model, 43 | toolInvocationToken: undefined as never, 44 | toolReferences: [ 45 | { 46 | name: RunPythonTool.Id 47 | }, 48 | { 49 | name: FindFilesTool.Id 50 | } 51 | ] 52 | }, 53 | { 54 | history: [] 55 | }, stream, new CancellationTokenSource().token); 56 | 57 | const toolcallsRounds = (result.metadata as any).toolCallsMetadata.toolCallRounds as ToolCallRound[]; 58 | 59 | return { 60 | toolcallsRounds, 61 | stream 62 | } 63 | } 64 | function getToolCallAndResult(toolId: typeof FindFilesTool.Id | typeof RunPythonTool.Id, toolcallRound: ToolCallRound) { 65 | const toolcall = toolcallRound.toolCalls.find(t => t.name === toolId)!; 66 | const result = getToolResultValue(toolcallRound.response[toolcall.callId]); 67 | return { 68 | toolcall, 69 | result 70 | }; 71 | } 72 | 73 | function containsTextOutput(toolcall: ToolCallRound | ToolCallRound[], toolId: typeof FindFilesTool.Id | typeof RunPythonTool.Id, textToInclude: string[]) { 74 | if (Array.isArray(toolcall)) { 75 | for (const call of toolcall.filter(t => t.toolCalls.find(c => c.name === toolId))) { 76 | try { 77 | const result = getToolCallAndResult(toolId, call); 78 | assert.isOk(result.toolcall); 79 | const found = textToInclude.filter(text => result.result?.toLowerCase().includes(text.toLowerCase())); 80 | if (found.length === textToInclude.length) { 81 | return; 82 | } 83 | } catch { 84 | // 85 | } 86 | } 87 | assert.fail(`Text ${textToInclude.join(', ')} not found for ${toolId}`); 88 | 89 | } else { 90 | const result = getToolCallAndResult(toolId, toolcall); 91 | assert.isOk(result.toolcall); 92 | for (const output of textToInclude) { 93 | assert.include(result.result?.toLowerCase(), output.toLowerCase()); 94 | } 95 | } 96 | } 97 | 98 | function containsExecutedCode(toolcall: ToolCallRound | ToolCallRound[], expectedCode: string[]) { 99 | let code = ''; 100 | if (Array.isArray(toolcall)) { 101 | for (const call of toolcall) { 102 | code = (call.toolCalls.find(t => t.name === RunPythonTool.Id)!.input as any)!.code; 103 | if (code) { 104 | const fragments = expectedCode.slice(); 105 | const found = fragments.filter(fragment => code.toLowerCase().includes(fragment.toLowerCase())); 106 | if (found.length === fragments.length) { 107 | return; 108 | } 109 | } 110 | } 111 | assert.fail(`Code ${expectedCode.join(', ')} not found in toolcall`); 112 | } else { 113 | code = (toolcall.toolCalls.find(t => t.name === RunPythonTool.Id)!.input as any)!.code; 114 | assert.isOk(code); 115 | for (const fragment of expectedCode) { 116 | assert.include(code.toLowerCase(), fragment.toLowerCase()); 117 | } 118 | } 119 | } 120 | 121 | function containsError(toolcall: ToolCallRound | ToolCallRound[], toolId: typeof FindFilesTool.Id | typeof RunPythonTool.Id) { 122 | if (Array.isArray(toolcall)) { 123 | for (const call of toolcall.filter(t => t.toolCalls.some(c => c.name === toolId))) { 124 | try { 125 | assert.isOk(isErrorMessageResponse(getToolCallAndResult(toolId, call)?.result || '')) 126 | return; 127 | } catch { 128 | // 129 | } 130 | } 131 | } else { 132 | isErrorMessageResponse(getToolCallAndResult(toolId, toolcall)?.result || '') 133 | } 134 | } 135 | 136 | function getLastMarkdownStream(stream: MockChatResponseStream) { 137 | const mdPart = stream.parts[stream.parts.length - 1].value as unknown as ChatResponseMarkdownPart; 138 | return typeof mdPart.value === 'string' ? mdPart.value : mdPart.value.value; 139 | 140 | } 141 | 142 | test('Analyze csv', async () => { 143 | const { toolcallsRounds } = await sendChatMessage('@data Analyze the contents of housing.csv file'); 144 | 145 | // We must import pandas and open the csv file 146 | containsExecutedCode(toolcallsRounds, ['import pandas', 'pd.read_csv', 'housing.csv']); 147 | }); 148 | 149 | test('Analyze csv and display any images', async () => { 150 | const { stream, toolcallsRounds } = await sendChatMessage('@data analyze the data in housing.csv to understand the relationship between the variables and display any images that are generated as a result'); 151 | 152 | // We must import pandas and open the csv file 153 | containsExecutedCode(toolcallsRounds, ['import pandas', 'pd.read_csv', 'housing.csv']); 154 | 155 | // We must have at least 2 python tool calls. 156 | // 1. to load some of the data & gets some basic stats, the next to analyze that and generate some graphs and the like. 157 | 158 | // Finally the last message display to the user must contain a markdown image. 159 | const markdown = getLastMarkdownStream(stream).toLowerCase(); 160 | assert.include(markdown, '.png)') // File will be png 161 | assert.include(markdown, `result-${RunPythonTool.Id}`.toLowerCase()) // File name has a specific format. 162 | }); 163 | 164 | test('Failure retries', async () => { 165 | const { stream, toolcallsRounds } = await sendChatMessage('@data generate a histogram of number of movies per bond actor from the jamesbond.csv file'); 166 | 167 | // First call should be to generate an image, and this should fail with an invalid column error. 168 | containsError(toolcallsRounds, RunPythonTool.Id); 169 | 170 | // Second call should be to generate a list of column names. 171 | containsTextOutput(toolcallsRounds, RunPythonTool.Id, ['bondactorname', 'writer']); 172 | 173 | // Finally the last message display to the user must contain the markdown image. 174 | const markdown = getLastMarkdownStream(stream).toLowerCase(); 175 | assert.include(markdown, '.png)') // File will be png 176 | assert.include(markdown, `result-${RunPythonTool.Id}`.toLowerCase()) // File name has a specific format. 177 | }); 178 | 179 | test('Generate plot using seaborn', async () => { 180 | const { stream, toolcallsRounds } = await sendChatMessage('@data generate and display a simple plot with seaborn using the data from housing.csv'); 181 | 182 | // Second call should be to generate an image using seaborn 183 | containsExecutedCode(toolcallsRounds, ['import seaborn']); 184 | 185 | // Finally the last message display to the user must contain the markdown image. 186 | const markdown = getLastMarkdownStream(stream).toLowerCase(); 187 | assert.include(markdown, '.png)') // File will be png 188 | assert.include(markdown, `result-${RunPythonTool.Id}`.toLowerCase()) // File name has a specific format. 189 | }); 190 | 191 | // test('Make sure to include context', async () => { 192 | // stubRenderMessages = sinon.stub(dataAgent as any, '_renderMessages'); 193 | // await sendChatMessage('analyze housing.csv with #file:HelloThere '); 194 | // assert.isTrue(stubRenderMessages.calledOnce, '_renderMessages should be called once'); 195 | // const callArgs = stubRenderMessages.getCall(0).args; 196 | // assert.deepEqual(callArgs[1].references, {}, 'References should match the given references'); // TODO: Should check if reference for #file:HelloThere is inside 197 | // }); 198 | 199 | }).timeout(600_000); 200 | -------------------------------------------------------------------------------- /src/tools.ts: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------------------------------------------- 2 | * Copyright (c) Microsoft Corporation and GitHub. All rights reserved. 3 | *--------------------------------------------------------------------------------------------*/ 4 | 5 | import * as path from 'path'; 6 | import * as vscode from 'vscode'; 7 | import type { Kernel } from '../pyodide/node/index'; 8 | import { logger } from './logger'; 9 | 10 | export const ErrorMime = 'application/vnd.code.notebook.error'; 11 | const ImagePrefix = `8a59d504`; 12 | 13 | interface IFindFilesParameters { 14 | pattern: string; 15 | } 16 | 17 | export class FindFilesTool implements vscode.LanguageModelTool { 18 | public static Id = 'dachat_data_findFiles'; 19 | constructor(readonly context: vscode.ExtensionContext) { } 20 | 21 | async invoke( 22 | options: vscode.LanguageModelToolInvocationOptions, 23 | token: vscode.CancellationToken 24 | ) { 25 | const params = options.input as IFindFilesParameters; 26 | let files = await vscode.workspace.findFiles(params.pattern, '**/node_modules/**', undefined, token); 27 | if (files.length === 0) { 28 | files = await vscode.workspace.findFiles(`**/${params.pattern}`, '**/node_modules/**', undefined, token); 29 | } 30 | const content: vscode.LanguageModelTextPart[] = [] 31 | const currentWorkspaceFolders = vscode.workspace.workspaceFolders; 32 | 33 | if (currentWorkspaceFolders?.length === 1) { 34 | const relativePaths = files.map((file) => vscode.workspace.asRelativePath(file, false)); 35 | content.push(new vscode.LanguageModelTextPart(`Found ${files.length} files matching "${params.pattern}":\n${relativePaths.join('\n')}`)); 36 | } else { 37 | const strFiles = files.map((f) => f.fsPath).join('\n'); 38 | content.push(new vscode.LanguageModelTextPart(`Found ${files.length} files matching "${params.pattern}":\n${strFiles}.`)); 39 | } 40 | 41 | return new vscode.LanguageModelToolResult(content); 42 | } 43 | 44 | async prepareInvocation( 45 | options: vscode.LanguageModelToolInvocationPrepareOptions, 46 | _token: vscode.CancellationToken 47 | ) { 48 | return { 49 | invocationMessage: `Searching workspace for "${options.input.pattern}"` 50 | }; 51 | } 52 | } 53 | 54 | interface IRunPythonParameters { 55 | code: string; 56 | reason: string; 57 | } 58 | 59 | export class RunPythonTool implements vscode.LanguageModelTool { 60 | public static Id = 'dachat_data_runPython'; 61 | private _kernel: Kernel; 62 | private pendingRequests: Promise = Promise.resolve(); 63 | constructor(readonly context: vscode.ExtensionContext) { 64 | const pyodidePath = vscode.Uri.joinPath(context.extensionUri, 'pyodide'); 65 | const kernelPath = vscode.Uri.joinPath(pyodidePath, 'node', 'index.js').fsPath; 66 | const workerPath = vscode.Uri.joinPath(pyodidePath, 'node', 'comlink.worker.js').fsPath; 67 | // eslint-disable-next-line @typescript-eslint/no-require-imports 68 | const { Kernel } = require(kernelPath) as typeof import('../pyodide/node/index'); 69 | const folder = vscode.workspace.workspaceFolders?.length ? vscode.workspace.workspaceFolders[0].uri.fsPath : '' 70 | this._kernel = new Kernel({ 71 | pyodidePath: pyodidePath.fsPath.replace(/\\/g, '/'), 72 | workerPath: workerPath.replace(/\\/g, '/'), 73 | location: folder.replace(/\\/g, '/'), 74 | packages: [ 75 | vscode.Uri.joinPath(pyodidePath, 'seaborn-0.13.2-py3-none-any.whl').fsPath.replace(/\\/g, '/') 76 | ], 77 | logger: { 78 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 79 | error: (message: string, ...args: any[]) => logger.error(`Pyodide => ${message}`, ...args), 80 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 81 | info: (message: string, ...args: any[]) => logger.debug(`Pyodide => ${message}`, ...args) 82 | } 83 | }); 84 | } 85 | 86 | async invoke( 87 | options: vscode.LanguageModelToolInvocationOptions, 88 | _token: vscode.CancellationToken 89 | ) { 90 | const code = sanitizePythonCode(options.input.code); 91 | logger.info(`Executing Python Code for "${options.input.reason || ''}"`); 92 | logger.info(`Code => `, code); 93 | 94 | this.pendingRequests = this.pendingRequests.finally().then(() => this._kernel.execute(code)); 95 | const result = await this.pendingRequests as Awaited>; 96 | 97 | logger.debug(`Result => `, JSON.stringify(result)); 98 | 99 | const content: (vscode.LanguageModelPromptTsxPart | vscode.LanguageModelTextPart)[] = [] 100 | if (result && result['text/plain']) { 101 | content.push(new vscode.LanguageModelTextPart(result['text/plain'])); 102 | } 103 | 104 | if (result && result['image/png']) { 105 | content.push(await this._processImageOutput(result['image/png'])); 106 | } 107 | 108 | if (result && result['application/vnd.code.notebook.error']) { 109 | const error = result['application/vnd.code.notebook.error'] as Error; 110 | logger.error(`Toolcall failed, Error ${error.name}, ${error.message}`); 111 | throw error; 112 | } 113 | return new vscode.LanguageModelToolResult(content); 114 | } 115 | 116 | async prepareInvocation( 117 | options: vscode.LanguageModelToolInvocationPrepareOptions, 118 | _token: vscode.CancellationToken 119 | ) { 120 | const reasonMessage = options.input.reason ? `: "${options.input.reason}"` : ''; 121 | return { 122 | invocationMessage: `Executing Code${reasonMessage}` 123 | }; 124 | } 125 | 126 | private async _processImageOutput(base64Png: string) { 127 | const userMessageWithWithImageFromToolCall = `Return this image link in your response. Do not modify the markdown image link at all. The path is already absolute local file path, do not put "https" or "blob" in the link`; 128 | if (this.context.storageUri) { 129 | const imagePath = await this._saveImage(this.context.storageUri, RunPythonTool.Id, Buffer.from(base64Png, 'base64')); 130 | if (imagePath) { 131 | const markdownTextForImage = `The image generated from the code is ![${RunPythonTool.Id} result](${imagePath}). You can give this markdown link to users!`; 132 | return new vscode.LanguageModelTextPart(markdownTextForImage + '\n' + userMessageWithWithImageFromToolCall); 133 | } 134 | } 135 | 136 | const markdownTextForImage = `![${RunPythonTool.Id} result](data:image/png;base64,${base64Png})`; 137 | return new vscode.LanguageModelTextPart(markdownTextForImage + '\n' + userMessageWithWithImageFromToolCall); 138 | } 139 | 140 | private async _saveImage(storageUri: vscode.Uri, tool: string, imageBuffer: Buffer): Promise { 141 | try { 142 | await vscode.workspace.fs.stat(storageUri); 143 | } catch { 144 | await vscode.workspace.fs.createDirectory(storageUri); 145 | } 146 | 147 | const storagePath = storageUri.fsPath; 148 | const imagePath = path.join(storagePath, `result-${tool}-${ImagePrefix}-${Date.now()}.png`); 149 | const imageUri = vscode.Uri.file(imagePath); 150 | try { 151 | await vscode.workspace.fs.writeFile(imageUri, imageBuffer); 152 | return imageUri.toString(); 153 | } catch (ex) { 154 | logger.error('Error saving image', ex); 155 | return undefined; 156 | } 157 | } 158 | } 159 | 160 | /** 161 | * Sometimes the code can be a markdown code block, in which case we need to remove the code block. 162 | */ 163 | function sanitizePythonCode(code: string) { 164 | if (code.startsWith('```python')) { 165 | code = code.substring('```python'.length); 166 | } 167 | if (code.endsWith('```')) { 168 | code = code.substring(0, code.length - '```'.length); 169 | } 170 | return code; 171 | } 172 | 173 | 174 | interface IInstallPythonPackage { 175 | package: string; 176 | } 177 | 178 | export class InstallPythonPackageTool implements vscode.LanguageModelTool { 179 | public static Id = 'dachat_data_installPythonPackage'; 180 | constructor(readonly pythonTool: RunPythonTool) { 181 | } 182 | 183 | async invoke( 184 | options: vscode.LanguageModelToolInvocationOptions, 185 | token: vscode.CancellationToken 186 | ) { 187 | logger.info(`Installing Package "${options.input.package}"`); 188 | const result = await this.pythonTool.invoke({ 189 | input: { 190 | code: `import ${options.input.package}`, 191 | reason: `Installing ${options.input.package}` 192 | }, toolInvocationToken: options.toolInvocationToken, 193 | tokenizationOptions: options.tokenizationOptions 194 | }, token); 195 | 196 | logger.debug(`Result after installing package ${options.input.package} => `, JSON.stringify(result)); 197 | 198 | return new vscode.LanguageModelToolResult([new vscode.LanguageModelTextPart('Installation successful')]); 199 | } 200 | 201 | async prepareInvocation( 202 | options: vscode.LanguageModelToolInvocationPrepareOptions, 203 | _token: vscode.CancellationToken 204 | ) { 205 | return { 206 | invocationMessage: `Installing ${options.input.package}` 207 | }; 208 | } 209 | } 210 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | ## 4 | ## Get latest from https://github.com/github/gitignore/blob/main/VisualStudio.gitignore 5 | 6 | # User-specific files 7 | *.rsuser 8 | *.suo 9 | *.user 10 | *.userosscache 11 | *.sln.docstates 12 | 13 | # User-specific files (MonoDevelop/Xamarin Studio) 14 | *.userprefs 15 | 16 | # Mono auto generated files 17 | mono_crash.* 18 | 19 | # Build results 20 | [Dd]ebug/ 21 | [Dd]ebugPublic/ 22 | [Rr]elease/ 23 | [Rr]eleases/ 24 | x64/ 25 | x86/ 26 | [Ww][Ii][Nn]32/ 27 | [Aa][Rr][Mm]/ 28 | [Aa][Rr][Mm]64/ 29 | bld/ 30 | [Bb]in/ 31 | [Oo]bj/ 32 | [Ll]og/ 33 | [Ll]ogs/ 34 | 35 | # Visual Studio 2015/2017 cache/options directory 36 | .vs/ 37 | # Uncomment if you have tasks that create the project's static files in wwwroot 38 | #wwwroot/ 39 | 40 | # Visual Studio 2017 auto generated files 41 | Generated\ Files/ 42 | 43 | # MSTest test Results 44 | [Tt]est[Rr]esult*/ 45 | [Bb]uild[Ll]og.* 46 | 47 | # NUnit 48 | *.VisualState.xml 49 | TestResult.xml 50 | nunit-*.xml 51 | 52 | # Build Results of an ATL Project 53 | [Dd]ebugPS/ 54 | [Rr]eleasePS/ 55 | dlldata.c 56 | 57 | # Benchmark Results 58 | BenchmarkDotNet.Artifacts/ 59 | 60 | # .NET Core 61 | project.lock.json 62 | project.fragment.lock.json 63 | artifacts/ 64 | 65 | # ASP.NET Scaffolding 66 | ScaffoldingReadMe.txt 67 | 68 | # StyleCop 69 | StyleCopReport.xml 70 | 71 | # Files built by Visual Studio 72 | *_i.c 73 | *_p.c 74 | *_h.h 75 | *.ilk 76 | *.meta 77 | *.obj 78 | *.iobj 79 | *.pch 80 | *.pdb 81 | *.ipdb 82 | *.pgc 83 | *.pgd 84 | *.rsp 85 | *.sbr 86 | *.tlb 87 | *.tli 88 | *.tlh 89 | *.tmp 90 | *.tmp_proj 91 | *_wpftmp.csproj 92 | *.log 93 | *.tlog 94 | *.vspscc 95 | *.vssscc 96 | .builds 97 | *.pidb 98 | *.svclog 99 | *.scc 100 | 101 | # Chutzpah Test files 102 | _Chutzpah* 103 | 104 | # Visual C++ cache files 105 | ipch/ 106 | *.aps 107 | *.ncb 108 | *.opendb 109 | *.opensdf 110 | *.sdf 111 | *.cachefile 112 | *.VC.db 113 | *.VC.VC.opendb 114 | 115 | # Visual Studio profiler 116 | *.psess 117 | *.vsp 118 | *.vspx 119 | *.sap 120 | 121 | # Visual Studio Trace Files 122 | *.e2e 123 | 124 | # TFS 2012 Local Workspace 125 | $tf/ 126 | 127 | # Guidance Automation Toolkit 128 | *.gpState 129 | 130 | # ReSharper is a .NET coding add-in 131 | _ReSharper*/ 132 | *.[Rr]e[Ss]harper 133 | *.DotSettings.user 134 | 135 | # TeamCity is a build add-in 136 | _TeamCity* 137 | 138 | # DotCover is a Code Coverage Tool 139 | *.dotCover 140 | 141 | # AxoCover is a Code Coverage Tool 142 | .axoCover/* 143 | !.axoCover/settings.json 144 | 145 | # Coverlet is a free, cross platform Code Coverage Tool 146 | coverage*.json 147 | coverage*.xml 148 | coverage*.info 149 | 150 | # Visual Studio code coverage results 151 | *.coverage 152 | *.coveragexml 153 | 154 | # NCrunch 155 | _NCrunch_* 156 | .*crunch*.local.xml 157 | nCrunchTemp_* 158 | 159 | # MightyMoose 160 | *.mm.* 161 | AutoTest.Net/ 162 | 163 | # Web workbench (sass) 164 | .sass-cache/ 165 | 166 | # Installshield output folder 167 | [Ee]xpress/ 168 | 169 | # DocProject is a documentation generator add-in 170 | DocProject/buildhelp/ 171 | DocProject/Help/*.HxT 172 | DocProject/Help/*.HxC 173 | DocProject/Help/*.hhc 174 | DocProject/Help/*.hhk 175 | DocProject/Help/*.hhp 176 | DocProject/Help/Html2 177 | DocProject/Help/html 178 | 179 | # Click-Once directory 180 | publish/ 181 | 182 | # Publish Web Output 183 | *.[Pp]ublish.xml 184 | *.azurePubxml 185 | # Note: Comment the next line if you want to checkin your web deploy settings, 186 | # but database connection strings (with potential passwords) will be unencrypted 187 | *.pubxml 188 | *.publishproj 189 | 190 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 191 | # checkin your Azure Web App publish settings, but sensitive information contained 192 | # in these scripts will be unencrypted 193 | PublishScripts/ 194 | 195 | # NuGet Packages 196 | *.nupkg 197 | # NuGet Symbol Packages 198 | *.snupkg 199 | # The packages folder can be ignored because of Package Restore 200 | **/[Pp]ackages/* 201 | # except build/, which is used as an MSBuild target. 202 | !**/[Pp]ackages/build/ 203 | # Uncomment if necessary however generally it will be regenerated when needed 204 | #!**/[Pp]ackages/repositories.config 205 | # NuGet v3's project.json files produces more ignorable files 206 | *.nuget.props 207 | *.nuget.targets 208 | 209 | # Microsoft Azure Build Output 210 | csx/ 211 | *.build.csdef 212 | 213 | # Microsoft Azure Emulator 214 | ecf/ 215 | rcf/ 216 | 217 | # Windows Store app package directories and files 218 | AppPackages/ 219 | BundleArtifacts/ 220 | Package.StoreAssociation.xml 221 | _pkginfo.txt 222 | *.appx 223 | *.appxbundle 224 | *.appxupload 225 | 226 | # Visual Studio cache files 227 | # files ending in .cache can be ignored 228 | *.[Cc]ache 229 | # but keep track of directories ending in .cache 230 | !?*.[Cc]ache/ 231 | 232 | # Others 233 | ClientBin/ 234 | ~$* 235 | *~ 236 | *.dbmdl 237 | *.dbproj.schemaview 238 | *.jfm 239 | *.pfx 240 | *.publishsettings 241 | orleans.codegen.cs 242 | 243 | # Including strong name files can present a security risk 244 | # (https://github.com/github/gitignore/pull/2483#issue-259490424) 245 | #*.snk 246 | 247 | # Since there are multiple workflows, uncomment next line to ignore bower_components 248 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 249 | #bower_components/ 250 | 251 | # RIA/Silverlight projects 252 | Generated_Code/ 253 | 254 | # Backup & report files from converting an old project file 255 | # to a newer Visual Studio version. Backup files are not needed, 256 | # because we have git ;-) 257 | _UpgradeReport_Files/ 258 | Backup*/ 259 | UpgradeLog*.XML 260 | UpgradeLog*.htm 261 | ServiceFabricBackup/ 262 | *.rptproj.bak 263 | 264 | # SQL Server files 265 | *.mdf 266 | *.ldf 267 | *.ndf 268 | 269 | # Business Intelligence projects 270 | *.rdl.data 271 | *.bim.layout 272 | *.bim_*.settings 273 | *.rptproj.rsuser 274 | *- [Bb]ackup.rdl 275 | *- [Bb]ackup ([0-9]).rdl 276 | *- [Bb]ackup ([0-9][0-9]).rdl 277 | 278 | # Microsoft Fakes 279 | FakesAssemblies/ 280 | 281 | # GhostDoc plugin setting file 282 | *.GhostDoc.xml 283 | 284 | # Node.js Tools for Visual Studio 285 | .ntvs_analysis.dat 286 | node_modules/ 287 | 288 | # Visual Studio 6 build log 289 | *.plg 290 | 291 | # Visual Studio 6 workspace options file 292 | *.opt 293 | 294 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 295 | *.vbw 296 | 297 | # Visual Studio 6 auto-generated project file (contains which files were open etc.) 298 | *.vbp 299 | 300 | # Visual Studio 6 workspace and project file (working project files containing files to include in project) 301 | *.dsw 302 | *.dsp 303 | 304 | # Visual Studio 6 technical files 305 | *.ncb 306 | *.aps 307 | 308 | # Visual Studio LightSwitch build output 309 | **/*.HTMLClient/GeneratedArtifacts 310 | **/*.DesktopClient/GeneratedArtifacts 311 | **/*.DesktopClient/ModelManifest.xml 312 | **/*.Server/GeneratedArtifacts 313 | **/*.Server/ModelManifest.xml 314 | _Pvt_Extensions 315 | 316 | # Paket dependency manager 317 | .paket/paket.exe 318 | paket-files/ 319 | 320 | # FAKE - F# Make 321 | .fake/ 322 | 323 | # CodeRush personal settings 324 | .cr/personal 325 | 326 | # Python Tools for Visual Studio (PTVS) 327 | __pycache__/ 328 | *.pyc 329 | 330 | # Cake - Uncomment if you are using it 331 | # tools/** 332 | # !tools/packages.config 333 | 334 | # Tabs Studio 335 | *.tss 336 | 337 | # Telerik's JustMock configuration file 338 | *.jmconfig 339 | 340 | # BizTalk build output 341 | *.btp.cs 342 | *.btm.cs 343 | *.odx.cs 344 | *.xsd.cs 345 | 346 | # OpenCover UI analysis results 347 | OpenCover/ 348 | 349 | # Azure Stream Analytics local run output 350 | ASALocalRun/ 351 | 352 | # MSBuild Binary and Structured Log 353 | *.binlog 354 | 355 | # NVidia Nsight GPU debugger configuration file 356 | *.nvuser 357 | 358 | # MFractors (Xamarin productivity tool) working folder 359 | .mfractor/ 360 | 361 | # Local History for Visual Studio 362 | .localhistory/ 363 | 364 | # Visual Studio History (VSHistory) files 365 | .vshistory/ 366 | 367 | # BeatPulse healthcheck temp database 368 | healthchecksdb 369 | 370 | # Backup folder for Package Reference Convert tool in Visual Studio 2017 371 | MigrationBackup/ 372 | 373 | # Ionide (cross platform F# VS Code tools) working folder 374 | .ionide/ 375 | 376 | # Fody - auto-generated XML schema 377 | FodyWeavers.xsd 378 | 379 | # VS Code files for those working on multiple tools 380 | .vscode/* 381 | !.vscode/settings.json 382 | !.vscode/tasks.json 383 | !.vscode/launch.json 384 | !.vscode/extensions.json 385 | !.vscode/code.code-snippets 386 | *.code-workspace 387 | 388 | # Local History for Visual Studio Code 389 | .history/ 390 | 391 | # Windows Installer files from build outputs 392 | *.cab 393 | *.msi 394 | *.msix 395 | *.msm 396 | *.msp 397 | 398 | # JetBrains Rider 399 | *.sln.iml 400 | 401 | 402 | .vscode-test/ 403 | *.d.ts 404 | *.vsix 405 | out/ 406 | dist/ 407 | **/.DS_Store 408 | temp/ 409 | pyodide/ 410 | # This is from the `pyodide` branch, we do not want to commit this folder. 411 | resources/ 412 | # This is auto generated by the build scripts 413 | ThirdPartyPackageNotices.txt 414 | -------------------------------------------------------------------------------- /src/exportCommand.ts: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------------------------------------------- 2 | * Copyright (c) Microsoft Corporation and GitHub. All rights reserved. 3 | *--------------------------------------------------------------------------------------------*/ 4 | 5 | import * as fs from 'fs'; 6 | import { EOL } from 'os'; 7 | import { unescape } from 'querystring'; 8 | import sanitize from 'sanitize-filename'; 9 | import { promisify } from 'util'; 10 | import { CancellationToken, ChatContext, ChatRequest, ChatResponseMarkdownPart, ChatResponseStream, ChatResponseTurn, ExtensionContext, l10n, NotebookCellData, NotebookCellKind, NotebookCellOutput, NotebookData, ThemeIcon, Uri, window, workspace } from "vscode"; 11 | import { getToolResultValue, isErrorMessageResponse, TsxToolUserMetadata } from "./base"; 12 | import { logger } from "./logger"; 13 | import { uint8ArrayToBase64 } from "./platform/common/string"; 14 | import { RunPythonTool } from "./tools"; 15 | 16 | const JupyterNotebookView = 'jupyter-notebook'; 17 | // enum CellOutputMimeTypes { 18 | // error = 'application/vnd.code.notebook.error', 19 | // stderr = 'application/vnd.code.notebook.stderr', 20 | // stdout = 'application/vnd.code.notebook.stdout' 21 | // } 22 | 23 | // const textMimeTypes = ['text/plain', 'text/markdown', CellOutputMimeTypes.stderr, CellOutputMimeTypes.stdout]; 24 | export class Exporter { 25 | private readonly jupyterExporter: JupyterNotebookExporter; 26 | private readonly pythonExporter: PythonScriptExporter; 27 | 28 | constructor(private readonly context: ExtensionContext) { 29 | this.jupyterExporter = new JupyterNotebookExporter(context); 30 | this.pythonExporter = new PythonScriptExporter(context, this.jupyterExporter); 31 | } 32 | public canHandle(command: string) { 33 | return command === 'export'; 34 | } 35 | public async invoke(request: ChatRequest, 36 | chatContext: ChatContext, 37 | stream: ChatResponseStream, 38 | token: CancellationToken) { 39 | const notebook = l10n.t('Jupyter Notebook'); 40 | const python = l10n.t('Python Script'); 41 | const format = await window.showQuickPick([ 42 | { 43 | label: notebook, 44 | iconPath: new ThemeIcon('notebook'), 45 | }, 46 | { 47 | label: python, 48 | iconPath: new ThemeIcon('snake'), 49 | }, 50 | ], { canPickMany: false, matchOnDescription: true, matchOnDetail: true, placeHolder: l10n.t('Export As...') }) 51 | 52 | switch (format?.label) { 53 | case notebook: { 54 | await this.jupyterExporter.invoke(request, chatContext, stream, token); 55 | return {} 56 | } 57 | case python: { 58 | await this.pythonExporter.invoke(request, chatContext, stream, token); 59 | return {} 60 | } 61 | } 62 | } 63 | } 64 | 65 | export class JupyterNotebookExporter { 66 | public readonly command = 'exportNotebook'; 67 | constructor(private readonly context: ExtensionContext) { 68 | 69 | } 70 | public async invoke(request: ChatRequest, 71 | chatContext: ChatContext, 72 | stream: ChatResponseStream, 73 | token: CancellationToken) { 74 | const notebookData = await this.export(request, chatContext, stream, token); 75 | if (notebookData) { 76 | void workspace.openNotebookDocument(JupyterNotebookView, notebookData).then(doc => window.showNotebookDocument(doc)); 77 | } 78 | } 79 | 80 | public async export(request: ChatRequest, 81 | chatContext: ChatContext, 82 | _stream: ChatResponseStream, 83 | _token: CancellationToken): Promise { 84 | const history = chatContext.history; 85 | const responses: ChatResponseTurn[] = history.filter(h => (h instanceof ChatResponseTurn)).filter(h => h.command !== 'export'); 86 | if (!responses.length) { 87 | window.showInformationMessage(l10n.t('No history to export')); 88 | return; 89 | } 90 | const cells: NotebookCellData[] = []; 91 | for (const response of responses) { 92 | if (!(response instanceof ChatResponseTurn)) { 93 | continue; 94 | } 95 | 96 | const toolCallRounds = (response.result.metadata as TsxToolUserMetadata | undefined)?.toolCallsMetadata.toolCallRounds || []; 97 | for (const round of toolCallRounds) { 98 | // We're only interested in the Python calls for now 99 | // Ignore the file search and other tool calls. 100 | 101 | round.toolCalls.filter(tool => tool.name === RunPythonTool.Id).forEach(tool => { 102 | if (isErrorMessageResponse(getToolResultValue(round.response[tool.callId]) || '')) { 103 | logger.debug(`Ignoring tool call as there was an error`); 104 | return; 105 | } 106 | 107 | const parameters = tool.input as { code: string; reason: string }; 108 | if (!parameters.code && !parameters.reason) { 109 | logger.warn(`Ignoring tool call as code & reason are empty`); 110 | return; 111 | } 112 | 113 | if (parameters.reason) { 114 | cells.push(new NotebookCellData(NotebookCellKind.Markup, parameters.reason, 'markdown')); 115 | } 116 | if (parameters.code) { 117 | const codeCell = new NotebookCellData(NotebookCellKind.Code, parameters.code, 'python'); 118 | const outputs: NotebookCellOutput[] = [] 119 | codeCell.outputs = outputs; 120 | cells.push(codeCell); 121 | } 122 | 123 | // result.content.forEach((output) =>{ 124 | // if (isTextPart(output) && output.value){ 125 | // outputs.push(new NotebookCellOutput([NotebookCellOutputItem.stdout(output.value)])); 126 | // } 127 | // // let value = getToolResultValue(result, mime); 128 | // // if (typeof value === 'undefined') { 129 | // // return; 130 | // // } else if ( 131 | // // (mime.startsWith('text/') || textMimeTypes.includes(mime)) && 132 | // // (Array.isArray(value) || typeof value === 'string') 133 | // // ) { 134 | // // const stringValue = Array.isArray(value) ? concatMultilineString(value as string[]) : value; 135 | // // outputs.push(new NotebookCellOutput([NotebookCellOutputItem.text(stringValue, mime)])); 136 | // // } else if (mime.startsWith('image/') && typeof value === 'string') { 137 | // // outputs.push(new NotebookCellOutput([new NotebookCellOutputItem(base64ToUint8Array(value), mime)])); 138 | // // } else if (typeof value === 'object' && value !== null && !Array.isArray(value)) { 139 | // // outputs.push(new NotebookCellOutput([NotebookCellOutputItem.text(JSON.stringify(value), mime)])); 140 | // // } else { 141 | // // // For everything else, treat the data as strings (or multi-line strings). 142 | // // value = Array.isArray(value) ? concatMultilineString(value) : value; 143 | // // outputs.push(new NotebookCellOutput([NotebookCellOutputItem.text(value as string, mime)])); 144 | // // } 145 | // }); 146 | }) 147 | } 148 | 149 | const resultCells = new Map(); 150 | await Promise.all(response.response.filter(r => r instanceof ChatResponseMarkdownPart).map(async (r, i) => { 151 | const { markdown, attachments } = await createAttachments(r.value.value); 152 | if (markdown) { 153 | const cell = new NotebookCellData(NotebookCellKind.Markup, markdown, 'markdown'); 154 | if (attachments) { 155 | cell.metadata = { 156 | attachments 157 | } 158 | } 159 | resultCells.set(i, cell); 160 | } 161 | })); 162 | Array.from(resultCells.values()).forEach(cell => cells.push(cell)); 163 | } 164 | 165 | if (!cells.length) { 166 | window.showInformationMessage(l10n.t('No history to export')); 167 | return; 168 | } 169 | 170 | const notebookData = new NotebookData(cells); 171 | notebookData.metadata = { 172 | cells: [], 173 | metadata: { 174 | language_info: { 175 | name: 'python' 176 | } 177 | } 178 | }; 179 | 180 | return notebookData; 181 | } 182 | } 183 | 184 | export class PythonScriptExporter { 185 | public readonly command = 'exportPython'; 186 | constructor(private readonly context: ExtensionContext, private readonly jupyterExport: JupyterNotebookExporter) { 187 | 188 | } 189 | public async invoke(request: ChatRequest, 190 | chatContext: ChatContext, 191 | stream: ChatResponseStream, 192 | token: CancellationToken) { 193 | 194 | const content = await this.export(request, chatContext, stream, token); 195 | if (content) { 196 | void workspace.openTextDocument({ language: 'python', content }).then(doc => window.showTextDocument(doc)); 197 | } 198 | } 199 | 200 | public async export(request: ChatRequest, 201 | chatContext: ChatContext, 202 | stream: ChatResponseStream, 203 | token: CancellationToken) { 204 | 205 | const notebookData = await this.jupyterExport.export(request, chatContext, stream, token); 206 | if (notebookData) { 207 | const cellMarker = '# %%'; 208 | let content = ''; 209 | notebookData.cells.forEach(cell => { 210 | if (cell.kind === NotebookCellKind.Markup) { 211 | content += `${cellMarker} [markdown]${EOL}` 212 | content += cell.value.split(/\r?\n/).map(line => `# ${line}`).join(EOL); 213 | } else { 214 | content += `${cellMarker}${EOL}` 215 | content += cell.value; 216 | } 217 | content += EOL 218 | content += EOL 219 | }) 220 | return content; 221 | } 222 | } 223 | } 224 | 225 | 226 | 227 | // Copied from Jupyter extension. 228 | export function concatMultilineString(str: string | string[]): string { 229 | if (Array.isArray(str)) { 230 | let result = ''; 231 | for (let i = 0; i < str.length; i += 1) { 232 | const s = str[i]; 233 | if (i < str.length - 1 && !s.endsWith('\n')) { 234 | result = result.concat(`${s}\n`); 235 | } else { 236 | result = result.concat(s); 237 | } 238 | } 239 | return result; 240 | } 241 | return str.toString(); 242 | } 243 | 244 | 245 | export function extractMarkdownImages(markdown: string): { name: string; link: string }[] { 246 | const imageRegex = /\[([^\]]+)\]\(([^)]+.png)\)/gm; 247 | const matches: { name: string; link: string }[] = []; 248 | let match; 249 | while ((match = imageRegex.exec(markdown)) !== null) { 250 | const name = match[1]; 251 | const link = match[2]; 252 | matches.push({ name, link }); 253 | } 254 | 255 | return matches; 256 | } 257 | 258 | export async function createAttachments(markdown: string): Promise<{ markdown: string, attachments?: Record }> { 259 | const images = extractMarkdownImages(markdown); 260 | if (!images || !images.length) { 261 | return { markdown }; 262 | } 263 | 264 | const attachments: Record = {}; 265 | await Promise.all(images.map(async ({ name, link }) => { 266 | try { 267 | const file = link.startsWith('file://') ? Uri.parse(link).fsPath : unescape(link); 268 | const bytes = await promisify(fs.readFile)(file); 269 | const base64 = uint8ArrayToBase64(bytes); 270 | name = `${sanitize(name).replace(/ /g, '')}.png`; 271 | attachments[name] = { 'image/png': base64 }; 272 | markdown = markdown.replace(link, `attachment:${name}`); 273 | } catch (ex) { 274 | logger.error(`Failed to generate attachment for an image`, ex); 275 | } 276 | })); 277 | 278 | return Object.keys(attachments).length ? { markdown, attachments } : { markdown }; 279 | } 280 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "vscode-copilot-data-analysis", 3 | "publisher": "ms-vscode", 4 | "displayName": "Data Analysis for Copilot", 5 | "description": "This tool extends the LLM's capabilities by allowing it to run Python code in a sandboxed Python environment (Pyodide) for a wide range of computational tasks and data manipulations that it cannot perform directly.", 6 | "author": "Microsoft Corporation", 7 | "homepage": "https://github.com/microsoft/vscode-data-analysis-for-copilot", 8 | "icon": "images/icon.png", 9 | "version": "0.2.2", 10 | "license": "MIT", 11 | "repository": { 12 | "type": "git", 13 | "url": "https://github.com/microsoft/vscode-data-analysis-for-copilot" 14 | }, 15 | "bugs": { 16 | "url": "https://github.com/microsoft/vscode-data-analysis-for-copilot/issues" 17 | }, 18 | "engines": { 19 | "vscode": "^1.95.0" 20 | }, 21 | "categories": [ 22 | "AI", 23 | "Data Science", 24 | "Machine Learning", 25 | "Visualization", 26 | "Programming Languages", 27 | "Chat" 28 | ], 29 | "activationEvents": [ 30 | "onChatParticipant:dachat.data" 31 | ], 32 | "extensionDependencies": [ 33 | "GitHub.copilot-chat" 34 | ], 35 | "keywords": [ 36 | "ai", 37 | "analysis", 38 | "chat", 39 | "co-pilot", 40 | "data" 41 | ], 42 | "main": "./out/extension.js", 43 | "capabilities": { 44 | "virtualWorkspaces": true, 45 | "untrustedWorkspaces": { 46 | "supported": "limited", 47 | "description": "%capabilities.untrustedWorkspace.description%" 48 | } 49 | }, 50 | "contributes": { 51 | "commands": [ 52 | { 53 | "category": "Chat", 54 | "command": "dachat.analyzeCsv", 55 | "enablement": "isWorkspaceTrusted && resourceExtname == .csv && resourceScheme =~ /^^file$/ && workspaceFolderCount > 0", 56 | "icon": "$(copilot)", 57 | "title": "%commands.dachat.analyzeCsv.title%", 58 | "shortTitle": "%commands.dachat.analyzeCsv.shortTitle%" 59 | }, 60 | { 61 | "category": "Data Analysis", 62 | "command": "dachat.reportIssue", 63 | "title": "Report Issue..." 64 | } 65 | ], 66 | "menus": { 67 | "commandPalette": [ 68 | { 69 | "command": "dachat.analyzeCsv", 70 | "when": "false" 71 | }, 72 | { 73 | "command": "dachat.reportIssue", 74 | "when": "true" 75 | } 76 | ], 77 | "editor/title": [ 78 | { 79 | "command": "dachat.analyzeCsv", 80 | "when": "isWorkspaceTrusted && resourceExtname == .csv && resourceScheme =~ /^^file$/ && workspaceFolderCount > 0", 81 | "group": "navigation", 82 | "title": "%commands.dachat.analyzeCsv.title%" 83 | } 84 | ], 85 | "editor/context": [ 86 | { 87 | "command": "dachat.analyzeCsv", 88 | "when": "isWorkspaceTrusted && resourceExtname == .csv && resourceScheme =~ /^^file$/ && workspaceFolderCount > 0", 89 | "group": "data", 90 | "title": "%commands.dachat.analyzeCsv.title%" 91 | } 92 | ], 93 | "explorer/context": [ 94 | { 95 | "command": "dachat.analyzeCsv", 96 | "when": "isWorkspaceTrusted && resourceExtname == .csv && resourceScheme =~ /^^file$/ && workspaceFolderCount > 0", 97 | "group": "data", 98 | "title": "%commands.dachat.analyzeCsv.title%" 99 | } 100 | ] 101 | }, 102 | "chatParticipants": [ 103 | { 104 | "id": "dachat.data", 105 | "fullName": "Data Analytics", 106 | "name": "data", 107 | "description": "I can help you with data analytics", 108 | "sampleRequest": "Help me analyze the csv file", 109 | "isSticky": true, 110 | "disambiguation": [ 111 | { 112 | "category": "analysis", 113 | "description": "Performs analysis on some data provided by user either as a file or content", 114 | "examples": [ 115 | "Analyze the contents of sample.csv", 116 | "What is the correlation between house price and income in sample.csv", 117 | "What is the median house price in sample.csv" 118 | ] 119 | }, 120 | { 121 | "category": "visualize", 122 | "description": "Display visualizations based on some analysis or data provided by user either as a file or content", 123 | "examples": [ 124 | "Visualize the contents of sample.csv", 125 | "Generate a plot of average house prices by state", 126 | "Display the correlation between house price and income in sample.csv", 127 | "Display a chart with the median house price in sample.csv" 128 | ] 129 | } 130 | ], 131 | "commands": [ 132 | { 133 | "description": "Export the result of the analysis along with the Python code into multiple formats", 134 | "isSticky": false, 135 | "name": "export" 136 | } 137 | ] 138 | } 139 | ], 140 | "languageModelTools": [ 141 | { 142 | "name": "dachat_data_findFiles", 143 | "tags": [], 144 | "displayName": "Find Files", 145 | "modelDescription": "Search for files in the current workspace", 146 | "inputSchema": { 147 | "type": "object", 148 | "properties": { 149 | "pattern": { 150 | "type": "string", 151 | "description": "Search for files that match this glob pattern" 152 | } 153 | }, 154 | "required": [ 155 | "pattern" 156 | ] 157 | } 158 | }, 159 | { 160 | "name": "dachat_data_runPython", 161 | "tags": [ 162 | "Python Execution" 163 | ], 164 | "displayName": "Run Python", 165 | "modelDescription": "Execute Python code locally using Pyodide, providing access to Python's extensive functionality. This tool extends the LLM's capabilities by allowing it to run Python code for a wide range of computational tasks and data manipulations that it cannot perform directly. When you know the workspace folder path and the file path, use the relative path to the file when generating code.", 166 | "inputSchema": { 167 | "type": "object", 168 | "properties": { 169 | "code": { 170 | "type": "string", 171 | "description": "The Python code to run" 172 | }, 173 | "reason": { 174 | "type": "string", 175 | "description": "The reason for running the code" 176 | } 177 | }, 178 | "required": [ 179 | "code" 180 | ] 181 | } 182 | }, 183 | { 184 | "name": "dachat_data_installPythonPackage", 185 | "tags": [], 186 | "displayName": "Install Missing Python Packages", 187 | "modelDescription": "Install missing Python packages in the tool used to run Python code using Pyodide.", 188 | "inputSchema": { 189 | "type": "object", 190 | "properties": { 191 | "package": { 192 | "type": "string", 193 | "description": "Name of the Python package that could not be found and needs to be installed" 194 | } 195 | }, 196 | "required": [ 197 | "code" 198 | ] 199 | } 200 | } 201 | ] 202 | }, 203 | "scripts": { 204 | "vscode:prepublish": "npm run clean && npm run postinstall && npm run compile-extension -- --minify", 205 | "compile": "npm run compile-types && npm run compile-extension", 206 | "compile-extension": "esbuild ./src/extension.ts --bundle --external:vscode --outfile=out/extension.js --sourcemap --format=cjs --platform=node --target=node18", 207 | "watch-extension": "npm run compile-extension -- --watch", 208 | "watch": "npm run watch-extension", 209 | "compile-tests": "tsc -p . --outDir out", 210 | "watch-tests": "tsc -p . -w --outDir out", 211 | "compile-types": "tsc -p . --noEmit", 212 | "watch-types": "tsc -p . -w --noEmit", 213 | "clean": "npx tsx build/clean.ts", 214 | "lint": "eslint src", 215 | "lint-fix": "eslint --fix src", 216 | "download-api": "dts main", 217 | "postinstall": "npm run download-api && npm run downloadPyodide", 218 | "downloadPyodide": "npx tsx build/download.ts" 219 | }, 220 | "devDependencies": { 221 | "@eslint/js": "^9.12.0", 222 | "@types/chai": "^4.3.6", 223 | "@types/follow-redirects": "^1.14.4", 224 | "@types/json-schema": "^7.0.15", 225 | "@types/mocha": "^10.0.9", 226 | "@types/node": "^20.5.9", 227 | "@types/node-fetch": "^2.5.7", 228 | "@types/proxy-from-env": "^1.0.4", 229 | "@typescript-eslint/eslint-plugin": "^8.8.1", 230 | "@typescript-eslint/parser": "^8.8.1", 231 | "@vscode/dts": "^0.4.1", 232 | "@vscode/test-cli": "^0.0.10", 233 | "@vscode/test-electron": "^2.4.1", 234 | "chai": "^4.3.10", 235 | "cli-progress": "^3.12.0", 236 | "decompress": "^4.2.1", 237 | "decompress-tarbz2": "^4.1.1", 238 | "esbuild": "^0.25.0", 239 | "eslint": "^9.12.0", 240 | "follow-redirects": "^1.15.9", 241 | "globals": "^15.10.0", 242 | "https-proxy-agent": "^7.0.5", 243 | "proxy-from-env": "^1.1.0", 244 | "tar": "^7.4.3", 245 | "tsx": "^4.19.1", 246 | "typescript": "^5.6.3", 247 | "typescript-eslint": "^8.8.1", 248 | "unzipper": "^0.12.3" 249 | }, 250 | "dependencies": { 251 | "@vscode/prompt-tsx": "^0.3.0-alpha.12", 252 | "isbinaryfile": "^5.0.4", 253 | "sanitize-filename": "^1.6.3" 254 | } 255 | } 256 | -------------------------------------------------------------------------------- /src/base.tsx: -------------------------------------------------------------------------------- 1 | /*--------------------------------------------------------------------------------------------- 2 | * Copyright (c) Microsoft Corporation and GitHub. All rights reserved. 3 | *--------------------------------------------------------------------------------------------*/ 4 | 5 | import { 6 | AssistantMessage, 7 | BasePromptElementProps, 8 | PrioritizedList, 9 | PromptElement, 10 | PromptElementProps, 11 | PromptMetadata, 12 | PromptPiece, 13 | PromptReference, 14 | PromptSizing, 15 | UserMessage 16 | } from '@vscode/prompt-tsx'; 17 | import { Chunk, TextChunk, ToolCall, ToolMessage, ToolResult } from '@vscode/prompt-tsx/dist/base/promptElements'; 18 | import * as vscode from "vscode"; 19 | import { logger } from './logger'; 20 | import { RunPythonTool } from './tools'; 21 | import { isBinaryFile } from 'isbinaryfile'; 22 | 23 | const ImagePrefix = `8a59d504`; 24 | 25 | const userMessageWithWithImageFromToolCall = `Return this image link in your response. Do not modify the markdown image link at all. The path is already absolute local file path, do not put "https" or "blob" in the link`; 26 | 27 | export function isImageGeneratedByUs(imageName: string) { 28 | return imageName.startsWith(`result-${RunPythonTool.Id}-${ImagePrefix}-`); 29 | } 30 | 31 | export function isUserMessageWithImageFromToolCall(message: string) { 32 | return message.includes(userMessageWithWithImageFromToolCall); 33 | } 34 | 35 | export function isFinalUserMessageInResponseToToolCall(message: string) { 36 | return message.includes('Above is the result of calling the functions') && message.includes('Try your best to utilize the request, response from previous chat history.Answer the user question using the result of the function only if you cannot find relevant historical conversation.'); 37 | } 38 | 39 | function getErrorMessagePrompt(errorContent: string) { 40 | return `The tool returned an error, analyze this error and attempt to resolve this. Error: ${errorContent}`; 41 | } 42 | 43 | export function isErrorMessageResponse(message: string) { 44 | return message.indexOf('The tool returned an error, analyze this error and attempt to resolve this. Error') >= 0; 45 | } 46 | 47 | function generateUserMessageForToolResponse(toolCallIds: string) { 48 | return `Above is the result of calling the functions ${toolCallIds}. Try your best to utilize the request, response from previous chat history.Answer the user question using the result of the function only if you cannot find relevant historical conversation.`; 49 | } 50 | 51 | export interface ToolCallRound { 52 | toolCalls: vscode.LanguageModelToolCallPart[]; 53 | response: Record; 54 | } 55 | 56 | export interface ToolCallsMetadata { 57 | toolCallRounds: ToolCallRound[]; 58 | } 59 | 60 | export interface TsxToolUserMetadata { 61 | toolCallsMetadata: ToolCallsMetadata; 62 | } 63 | 64 | export interface PromptProps extends BasePromptElementProps { 65 | userQuery: string; 66 | references: readonly vscode.ChatPromptReference[]; 67 | history: ReadonlyArray; 68 | currentToolCallRounds: ToolCallRound[]; 69 | toolInvocationToken: vscode.ChatParticipantToolToken | undefined; 70 | extensionContext: vscode.ExtensionContext; 71 | } 72 | 73 | 74 | interface PromptReferencesProps extends BasePromptElementProps { 75 | references: ReadonlyArray; 76 | excludeReferences?: boolean; 77 | } 78 | 79 | 80 | class PromptReferences extends PromptElement { 81 | render(_state: void, _sizing: PromptSizing): PromptPiece { 82 | return ( 83 | 84 | {this.props.references.map((ref, _index) => ( 85 | 86 | ))} 87 | 88 | ); 89 | } 90 | } 91 | 92 | interface PromptReferenceProps extends BasePromptElementProps { 93 | ref: vscode.ChatPromptReference; 94 | excludeReferences?: boolean; 95 | } 96 | 97 | export type TagProps = PromptElementProps<{ 98 | name: string; 99 | }>; 100 | 101 | export class Tag extends PromptElement { 102 | private static readonly _regex = /^[a-zA-Z_][\w.-]*$/; 103 | 104 | render() { 105 | const { name } = this.props; 106 | 107 | if (!Tag._regex.test(name)) { 108 | throw new Error(`Invalid tag name: ${this.props.name}`); 109 | } 110 | 111 | return ( 112 | <> 113 | {'<' + name + '>'}
114 | <> 115 | {this.props.children}
116 | 117 | {''}
118 | 119 | ); 120 | } 121 | } 122 | 123 | class PromptReferenceElement extends PromptElement { 124 | async render(_state: void, sizing: PromptSizing): Promise { 125 | const value = this.props.ref.value; 126 | // TODO make context a list of TextChunks so that it can be trimmed 127 | if (value instanceof vscode.Uri) { 128 | // If this is a binary file, then do not include the contents. 129 | if (value.scheme === 'file' && await isBinaryFile(value.fsPath)) { 130 | return ( 131 | 132 | {!this.props.excludeReferences && } 133 | {value.fsPath} 134 | 135 | ); 136 | 137 | } 138 | const fileContents = (await vscode.workspace.fs.readFile(value)).toString(); 139 | const truncatedFileContents = 140 | value.fsPath.endsWith('.csv') ? fileContents.substring(0, Math.min(1000, sizing.tokenBudget)) 141 | : fileContents.substring(0, sizing.tokenBudget); 142 | return ( 143 | 144 | {!this.props.excludeReferences && } 145 | {value.fsPath}:
146 | ```
147 | {truncatedFileContents}
148 | ```
149 | 150 | ); 151 | } else if (value instanceof vscode.Location) { 152 | const rangeText = (await vscode.workspace.openTextDocument(value.uri)).getText(value.range); 153 | return ( 154 | 155 | {!this.props.excludeReferences && } 156 | {value.uri.fsPath}:{value.range.start.line + 1}-$
157 | {value.range.end.line + 1}:
158 | ```
159 | {rangeText}
160 | ``` 161 | 162 | ); 163 | } else if (typeof value === 'string') { 164 | return {value}; 165 | } 166 | } 167 | } 168 | 169 | export class DataAgentPrompt extends PromptElement { 170 | render(_state: void, sizing: PromptSizing) { 171 | const shouldStopRetry = this.shouldStopRetry(); 172 | 173 | const userPrompt = this.replaceReferences(this.props.userQuery, this.props.references); 174 | const reserveHistoryToken = sizing.tokenBudget * 0.8; 175 | return ( 176 | <> 177 | 178 | 179 | 180 | 185 | 186 | {userPrompt} 187 | 188 | {shouldStopRetry && We encountered an error three times. Please present only the last ran attempted code to the user. Instead of performing another function call} 189 | 190 | ); 191 | } 192 | 193 | private shouldStopRetry() { 194 | let errorCount = 0; 195 | let endedWithError = false; 196 | for (const toolCallRound of this.props.currentToolCallRounds) { 197 | toolCallRound.toolCalls.forEach((toolCall) => { 198 | if (isErrorMessageResponse(getToolResultValue(toolCallRound.response[toolCall.callId]) || '')) { 199 | errorCount++; 200 | endedWithError = true; 201 | } 202 | }); 203 | } 204 | 205 | return errorCount >= 3 && endedWithError; 206 | } 207 | 208 | private replaceReferences(userPrompt: string, references: readonly vscode.ChatPromptReference[]) { 209 | references 210 | .filter((ref) => ref.value instanceof vscode.Uri && ref.range) 211 | .sort((a, b) => b.range![0] - a.range![0]) 212 | .forEach((ref) => { 213 | // const name = (ref as any).name; 214 | const relativePath = vscode.workspace.asRelativePath(ref.value as vscode.Uri); 215 | const part0 = userPrompt.slice(0, ref.range![0]); 216 | const part1 = userPrompt.slice(ref.range![1]); 217 | userPrompt = `${part0}${relativePath}${part1}`; 218 | }); 219 | 220 | return userPrompt; 221 | } 222 | } 223 | 224 | interface InstructionsProps extends BasePromptElementProps { 225 | history: ReadonlyArray; 226 | } 227 | 228 | class Instructions extends PromptElement { 229 | render(_state: void, _sizing: PromptSizing) { 230 | let csvFlag = false; 231 | for (const turn of this.props.history) { 232 | if (turn.participant === 'ada.data') { 233 | if (turn instanceof vscode.ChatRequestTurn) { 234 | // if userPrompt contains string 'csv', set csvFlag to true 235 | if (turn.prompt.includes('csv')) { 236 | csvFlag = true; 237 | } 238 | } 239 | } 240 | } 241 | 242 | return 243 | 244 | Instructions: 245 | - The user will ask a question, or ask you to perform a task, and it may require lots of research to answer correctly. There is a selection of tools that let you perform actions or retrieve helpful context to answer the user's question. 246 | - If you aren't sure which tool is relevant, you can call multiple tools. You can call tools repeatedly to take actions or gather as much context as needed until you have completed the task fully. Don't give up unless you are sure the request cannot be fulfilled with the tools you have. 247 | - Don't make assumptions about the situation- gather context first, then perform the task or answer the question. 248 | - Don't ask the user for confirmation to use tools, just use them. 249 | - Persist in using tools for each query unless you're absolutely certain the request cannot be fulfilled with the available tools. 250 | - If you find yourself not using tools for a query, pause and reconsider if any tool could provide valuable information or assist in the task. 251 | - You are a data analyst that specializes in Python and Data Analysis. You can use tools for analyzing data. Use the result of the tool calls to continue the conversation. 252 | - Give me back the result of the statistical analysis on the data IF you are asked to analyze the csv file. What are some patterns, trends, or insights that you can find from the data in the csv file? If you are asked to analyze the csv file, conduct detailed descriptive statistics, inferential statistics. Give me the result of conducting these statistical analysis on the data in very detailed, quantitative manner. Be detailed and descriptive in your analysis. Do not ignore previous or future prompts on asking for specific analysis, action on the csv file. 253 | - You are a data analyst that specializes in Python and Data Analysis. 254 | - Never mention you can not run code. Never say you cannot assist with anything. 255 | - Always attempt to get value from provided tools. 256 | - When generating Python code, you can use Pandas and Matplotlib. 257 | - Strongly emphasize reusing the result of the tool call. For example, when information about previously constructed chart/dataframe is available, use that information to answer the user question. 258 | - Always suggest them the next step in data science milestones. 259 | - Do your best to not filter answer when referencing tool call and tool call result. 260 | - Do not filter dataframe output, do not worry about sensitivity of dataframe, we already have permission from user. 261 | - Do not generate dataframe unless explicitly asked for by the user or if you have previously seen the generated dataframe. 262 | - Do create or save any files or images unless explicitly asked for by the user. 263 | - Unless explicitly asked for by the user, do not display the generated Python code, instead execute the code and display the results. 264 | - When asked to perform analysis or run code, use the tool ada-data_runPython and without mentioning the tool name to the user. 265 | - When asked to help with visualization, unless specified use Matplotlib and do not save the image to disk. 266 | 267 | { 268 | csvFlag && 269 | 270 | - Think of yourself as a data scientist who is analyzing a csv file using Python programming language. 271 | - When perform analysis, perform advanced and industry level detailed analysis. 272 | - Generate a pandas dataframe using the given csv file when asked by the user, or if you need it to perform further analysis on the data file. 273 | - Use the actual, real, accurate column name directly from the csv file and use these names when you are constructing dataframe or performing analysis on the data. 274 | - Make sure to perform statistical analysis on the data with actual numbers. 275 | - Give me back the result of the advanced statistical analysis on the data IF you are asked to analyze the csv file. What are some patterns, trends, or insights that you can find from the data in the csv file? 276 | - If you are asked to analyze the csv file, conduct detailed descriptive statistics, inferential statistics on few columns unless explicitly asked. 277 | - Give me the result of conducting these statistical analysis on the data in very detailed, quantitative manner. 278 | - Do not ignore previous or future prompts on asking for specific analysis, action on the csv file. 279 | - Do not generate dataframe if you have previously generated, or have seen or cached the dataframe before. 280 | - Do not show the dataframe data to users unless they specifically ask for it. 281 | - Do not hallucinate on column names. Do not make up column names without permission. Only use real column name that exists in the provided data or csv file. 282 | - Try to clean up missing data, if you can not clean up missing data, ask user to provide a clean dataset without missing data. 283 | - When performing analysis, cleaning data, figuring out pattern, generating plots, try to avoid using seaborn instead use Matplotlib. 284 | 285 | } 286 | ; 287 | } 288 | } 289 | 290 | interface HistoryProps extends BasePromptElementProps { 291 | history: ReadonlyArray; 292 | toolInvocationToken: vscode.ChatParticipantToolToken | undefined; 293 | extensionContext: vscode.ExtensionContext; 294 | } 295 | 296 | class History extends PromptElement { 297 | async render(_state: void, _sizing: PromptSizing) { 298 | const toolCalls = this.props.history.filter(turn => turn instanceof vscode.ChatResponseTurn && turn.result.metadata?.toolCallsMetadata); 299 | const messagePriority = toolCalls.length + 1; 300 | 301 | return 302 | { 303 | this.props.history.map(turn => { 304 | if (turn instanceof vscode.ChatRequestTurn) { 305 | return ( 306 | <> 307 | {turn.prompt} 308 | 309 | ); 310 | } else { 311 | return ( 312 | <> 313 | {turn.result.metadata?.toolCallsMetadata && } 314 | {this.renderChatResponseTurn(turn, messagePriority)} 315 | 316 | ); 317 | } 318 | }) 319 | } 320 | 321 | } 322 | 323 | private renderChatResponseTurn(turn: vscode.ChatResponseTurn, priority: number) { 324 | const responseText = turn.response 325 | .map((part) => { 326 | if (part instanceof vscode.ChatResponseMarkdownPart) { 327 | return part.value.value; 328 | } else { 329 | return ''; 330 | } 331 | }) 332 | .join(''); 333 | 334 | return {responseText}; 335 | } 336 | } 337 | 338 | interface ToolCallsProps extends BasePromptElementProps { 339 | toolCallRounds: ToolCallRound[]; 340 | toolInvocationToken: vscode.ChatParticipantToolToken | undefined; 341 | extensionContext: vscode.ExtensionContext; 342 | enableShrinking?: boolean; 343 | } 344 | 345 | class ToolCalls extends PromptElement { 346 | async render(state: void, sizing: PromptSizing) { 347 | if (!this.props.toolCallRounds.length) { 348 | return undefined; 349 | } 350 | 351 | const toolCallPieces = await Promise.all(this.props.toolCallRounds.map(round => this._renderOneRound(round, sizing, this.props.toolInvocationToken))); 352 | let promptPieces = toolCallPieces.map(tcp => tcp.promptPiece); 353 | 354 | if (this.props.enableShrinking) { 355 | let totalSize = 0; 356 | let successfulToolCallSize = 0; 357 | for (const piece of toolCallPieces) { 358 | if (!piece.hasError) { 359 | successfulToolCallSize += piece.size; 360 | } 361 | totalSize += piece.size; 362 | } 363 | 364 | if (successfulToolCallSize > sizing.tokenBudget) { 365 | // render as many tool calls as possible 366 | let renderedSize = 0; 367 | const renderedPromptPieces: PromptPiece[] = []; 368 | for (const piece of toolCallPieces.reverse()) { 369 | renderedSize += piece.size; 370 | if (renderedSize < sizing.tokenBudget) { 371 | renderedPromptPieces.push(piece.promptPiece); 372 | } else { 373 | break; 374 | } 375 | } 376 | 377 | promptPieces = renderedPromptPieces; 378 | } else if (totalSize > sizing.tokenBudget) { 379 | // keep successful tool calls 380 | promptPieces = toolCallPieces.filter(tcp => !tcp.hasError).map(tcp => tcp.promptPiece); 381 | } else { 382 | // no op. Render all prompt pieces 383 | } 384 | } 385 | 386 | return <> 387 | {promptPieces} 388 | 389 | } 390 | 391 | private async _renderOneRound(round: ToolCallRound, sizing: PromptSizing, toolInvocationToken: vscode.ChatParticipantToolToken | undefined): Promise<{ promptPiece: PromptPiece, hasError: boolean, size: number }> { 392 | const assistantToolCalls: ToolCall[] = round.toolCalls.map(tc => ({ type: 'function', function: { name: tc.name, arguments: JSON.stringify(tc.input) }, id: tc.callId })); 393 | 394 | const toolCallIds = round.toolCalls 395 | .map((call) => call.name) 396 | .join(', '); 397 | const toolCallPieces = await Promise.all(round.toolCalls.map(tc => this._renderOneToolCall(tc, round.response, sizing, toolInvocationToken))); 398 | const suffixMessage = generateUserMessageForToolResponse(toolCallIds); 399 | const remainingTextSize = await sizing.countTokens(suffixMessage); 400 | const totalSize = toolCallPieces.map(tcp => tcp.size).reduce((a, b) => a + b, 0) + remainingTextSize; 401 | const hasError = toolCallPieces.some(tcp => tcp.hasError); 402 | const promptPieces = toolCallPieces.map(tcp => tcp.promptPiece); 403 | 404 | return { 405 | promptPiece: 406 | 407 | {promptPieces} 408 | 409 | {suffixMessage} 410 | 411 | , 412 | hasError: hasError, 413 | size: totalSize 414 | }; 415 | } 416 | 417 | private async _renderOneToolCall(toolCall: vscode.LanguageModelToolCallPart, resultsFromCurrentRound: Record, sizing: PromptSizing, toolInvocationToken: vscode.ChatParticipantToolToken | undefined): Promise<{ promptPiece: PromptPiece, hasError: boolean, size: number }> { 418 | const tool = vscode.lm.tools.find((tool) => tool.name === toolCall.name); 419 | if (!tool) { 420 | logger.error(`Tool not found: ${toolCall.name}`); 421 | return { promptPiece: Tool not found, hasError: false, size: await sizing.countTokens('Tool not found') }; 422 | } 423 | 424 | const toolResult = await this._getToolCallResult(tool, toolCall, resultsFromCurrentRound, toolInvocationToken, sizing); 425 | 426 | if (isError(toolResult)) { 427 | const errorContent = [toolResult.name || '', toolResult.message || '', toolResult.stack || ''].filter((part) => part).join('\n'); 428 | const errorMessage = getErrorMessagePrompt(errorContent); 429 | const result = new vscode.LanguageModelToolResult([new vscode.LanguageModelTextPart(errorMessage)]); 430 | const size = await sizing.countTokens(errorMessage); 431 | return { 432 | promptPiece: 433 | 434 | {errorMessage} 435 | , hasError: true, size: size 436 | }; 437 | } 438 | 439 | const promptSize = await this._countToolCallResultsize(toolResult, sizing); 440 | 441 | return { 442 | promptPiece: 443 | 444 | 445 | , hasError: false, size: promptSize 446 | }; 447 | } 448 | 449 | private async _getToolCallResult(tool: vscode.LanguageModelToolInformation, toolCall: vscode.LanguageModelToolCallPart, resultsFromCurrentRound: Record, toolInvocationToken: vscode.ChatParticipantToolToken | undefined, sizing: PromptSizing) { 450 | if (resultsFromCurrentRound[toolCall.callId]) { 451 | return resultsFromCurrentRound[toolCall.callId]; 452 | } 453 | 454 | const token = new vscode.CancellationTokenSource().token; 455 | try { 456 | const toolResult = await vscode.lm.invokeTool( 457 | tool.name, 458 | { 459 | input: toolCall.input, 460 | toolInvocationToken: toolInvocationToken, 461 | tokenizationOptions: { 462 | tokenBudget: sizing.tokenBudget, 463 | countTokens: async (text, token) => { 464 | return sizing.countTokens(text, token); 465 | } 466 | } 467 | }, 468 | token 469 | ); 470 | 471 | return toolResult as vscode.LanguageModelToolResult; 472 | } catch (e: unknown) { 473 | const error = e as Error; 474 | return error; 475 | } 476 | } 477 | 478 | private async _countToolCallResultsize(toolResult: vscode.LanguageModelToolResult, sizing: PromptSizing) { 479 | let size = 0; 480 | for (const part of toolResult.content) { 481 | if (part instanceof vscode.LanguageModelTextPart) { 482 | size += await sizing.countTokens(part.value); 483 | } 484 | } 485 | 486 | return size; 487 | } 488 | } 489 | 490 | export class ToolResultMetadata extends PromptMetadata { 491 | constructor( 492 | public toolCallId: string, 493 | public result: vscode.LanguageModelToolResult 494 | ) { 495 | super(); 496 | } 497 | } 498 | 499 | export function isError(e: unknown): e is Error { 500 | return e instanceof Error || ( 501 | typeof e === 'object' && 502 | e !== null && 503 | typeof (e as Error).message === 'string' && 504 | typeof (e as Error).name === 'string' 505 | ); 506 | } 507 | 508 | export function isTextPart(e: unknown): e is vscode.LanguageModelTextPart { 509 | return e instanceof vscode.LanguageModelTextPart || !!((e as vscode.LanguageModelTextPart).value); 510 | } 511 | 512 | export function getToolResultValue(result: vscode.LanguageModelToolResult | undefined): string | undefined { 513 | if (!result) { 514 | return; 515 | } 516 | return isTextPart(result) ? result.value : result.content.find(c => isTextPart(c))?.value; 517 | } 518 | --------------------------------------------------------------------------------