├── .eslintignore ├── .eslintrc.js ├── .github ├── actions │ └── check-release │ │ └── action.yml └── workflows │ ├── build.yml │ ├── check-release.yml │ ├── enforce-label.yml │ ├── prep-release.yml │ ├── publish-release.yml │ └── update-integration-tests.yml ├── .gitignore ├── .prettierignore ├── .prettierrc ├── .stylelintrc ├── CHANGELOG.md ├── LICENSE ├── README.md ├── RELEASE.md ├── babel.config.js ├── install.json ├── jest.config.js ├── jupyter_voicepilot └── __init__.py ├── package-lock.json ├── package.json ├── pyproject.toml ├── schema └── plugin.json ├── setup.py ├── src ├── __tests__ │ └── jupyter_voicepilot.spec.ts ├── custom-typings.d.ts ├── extension.ts ├── index.ts ├── notebook │ ├── cmd-handler.ts │ ├── index.ts │ └── utils.ts ├── openai │ ├── actions │ │ ├── base.ts │ │ ├── chat.ts │ │ ├── code.ts │ │ ├── index.ts │ │ └── transcript.ts │ └── index.ts ├── recorder.ts ├── svg.d.ts └── voice-processor.ts ├── style ├── base.css ├── icons │ └── record-vinyl-solid.svg ├── index.css └── index.js ├── tsconfig.json ├── ui-tests ├── README.md ├── jupyter_server_test_config.py ├── package.json ├── playwright.config.js └── tests │ └── jupyter_voicepilot.spec.ts └── yarn.lock /.eslintignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | coverage 4 | **/*.d.ts 5 | tests 6 | 7 | **/__tests__ 8 | ui-tests 9 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | extends: [ 3 | 'eslint:recommended', 4 | 'plugin:@typescript-eslint/eslint-recommended', 5 | 'plugin:@typescript-eslint/recommended', 6 | 'plugin:prettier/recommended' 7 | ], 8 | parser: '@typescript-eslint/parser', 9 | parserOptions: { 10 | project: 'tsconfig.json', 11 | sourceType: 'module' 12 | }, 13 | plugins: ['@typescript-eslint'], 14 | rules: { 15 | '@typescript-eslint/naming-convention': [ 16 | 'error', 17 | { 18 | selector: 'interface', 19 | format: ['PascalCase'], 20 | custom: { 21 | regex: '^I[A-Z]', 22 | match: true 23 | } 24 | } 25 | ], 26 | '@typescript-eslint/no-unused-vars': ['warn', { args: 'none' }], 27 | '@typescript-eslint/no-explicit-any': 'off', 28 | '@typescript-eslint/no-namespace': 'off', 29 | '@typescript-eslint/no-use-before-define': 'off', 30 | '@typescript-eslint/quotes': [ 31 | 'error', 32 | 'single', 33 | { avoidEscape: true, allowTemplateLiterals: false } 34 | ], 35 | curly: ['error', 'all'], 36 | eqeqeq: 'error', 37 | 'prefer-arrow-callback': 'error' 38 | } 39 | }; 40 | -------------------------------------------------------------------------------- /.github/actions/check-release/action.yml: -------------------------------------------------------------------------------- 1 | name: Check Release 2 | uses: jupyter-server/jupyter_releaser/.github/actions/check-release@v2 3 | with: 4 | token: ${{ secrets.GITHUB_TOKEN }} 5 | 6 | - name: Upload Distributions 7 | uses: actions/upload-artifact@v2 8 | with: 9 | name: dist-${{ github.run_number }} 10 | path: .jupyter_releaser_checkout/dist -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: 4 | push: 5 | branches: master 6 | pull_request: 7 | branches: '*' 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - name: Checkout 15 | uses: actions/checkout@v3 16 | 17 | - name: Base Setup 18 | uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 19 | 20 | - name: Install dependencies 21 | run: python -m pip install -U jupyterlab~=3.1 22 | 23 | - name: Lint the extension 24 | run: | 25 | set -eux 26 | jlpm 27 | jlpm run lint:check 28 | 29 | - name: Test the extension 30 | run: | 31 | set -eux 32 | jlpm run test 33 | 34 | - name: Build the extension 35 | run: | 36 | set -eux 37 | python -m pip install .[test] 38 | 39 | jupyter labextension list 40 | jupyter labextension list 2>&1 | grep -ie "voicepilot.*OK" 41 | python -m jupyterlab.browser_check 42 | 43 | - name: Package the extension 44 | run: | 45 | set -eux 46 | 47 | pip install build 48 | python -m build 49 | pip uninstall -y "jupyter_voicepilot" jupyterlab 50 | 51 | - name: Upload extension packages 52 | uses: actions/upload-artifact@v3 53 | with: 54 | name: extension-artifacts 55 | path: dist/jupyter_voicepilot* 56 | if-no-files-found: error 57 | 58 | test_isolated: 59 | needs: build 60 | runs-on: ubuntu-latest 61 | 62 | steps: 63 | - name: Checkout 64 | uses: actions/checkout@v3 65 | - name: Install Python 66 | uses: actions/setup-python@v4 67 | with: 68 | python-version: '3.9' 69 | architecture: 'x64' 70 | - uses: actions/download-artifact@v3 71 | with: 72 | name: extension-artifacts 73 | - name: Install and Test 74 | run: | 75 | set -eux 76 | # Remove NodeJS, twice to take care of system and locally installed node versions. 77 | sudo rm -rf $(which node) 78 | sudo rm -rf $(which node) 79 | 80 | pip install "jupyterlab~=3.1" jupyter_voicepilot*.whl 81 | 82 | 83 | jupyter labextension list 84 | jupyter labextension list 2>&1 | grep -ie "voicepilot.*OK" 85 | python -m jupyterlab.browser_check --no-chrome-test 86 | 87 | integration-tests: 88 | name: Integration tests 89 | needs: build 90 | runs-on: ubuntu-latest 91 | 92 | env: 93 | PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/pw-browsers 94 | 95 | steps: 96 | - name: Checkout 97 | uses: actions/checkout@v3 98 | 99 | - name: Base Setup 100 | uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 101 | 102 | - name: Download extension package 103 | uses: actions/download-artifact@v3 104 | with: 105 | name: extension-artifacts 106 | 107 | - name: Install the extension 108 | run: | 109 | set -eux 110 | python -m pip install "jupyterlab~=3.1" jupyter_voicepilot*.whl 111 | 112 | - name: Install dependencies 113 | working-directory: ui-tests 114 | env: 115 | PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 116 | run: jlpm install 117 | 118 | - name: Set up browser cache 119 | uses: actions/cache@v3 120 | with: 121 | path: | 122 | ${{ github.workspace }}/pw-browsers 123 | key: ${{ runner.os }}-${{ hashFiles('ui-tests/yarn.lock') }} 124 | 125 | - name: Install browser 126 | run: jlpm playwright install chromium 127 | working-directory: ui-tests 128 | 129 | - name: Execute integration tests 130 | working-directory: ui-tests 131 | run: | 132 | jlpm playwright test 133 | 134 | - name: Upload Playwright Test report 135 | if: always() 136 | uses: actions/upload-artifact@v3 137 | with: 138 | name: jupyter_voicepilot-playwright-tests 139 | path: | 140 | ui-tests/test-results 141 | ui-tests/playwright-report 142 | 143 | check_links: 144 | name: Check Links 145 | runs-on: ubuntu-latest 146 | timeout-minutes: 15 147 | steps: 148 | - uses: actions/checkout@v3 149 | - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 150 | - uses: jupyterlab/maintainer-tools/.github/actions/check-links@v1 151 | with: 152 | ignore_links: "https://github.com/.*/(workflows|compare|graphs)/.*" 153 | -------------------------------------------------------------------------------- /.github/workflows/check-release.yml: -------------------------------------------------------------------------------- 1 | name: Check Release 2 | on: 3 | push: 4 | branches: ["master"] 5 | pull_request: 6 | branches: ["*"] 7 | 8 | jobs: 9 | check_release: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout 13 | uses: actions/checkout@v3 14 | - name: Base Setup 15 | uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 16 | - name: Install Dependencies 17 | run: | 18 | pip install -e . 19 | - name: Check Release 20 | uses: jupyter-server/jupyter_releaser/.github/actions/check-release@v2 21 | with: 22 | 23 | token: ${{ secrets.GITHUB_TOKEN }} 24 | 25 | - name: Upload Distributions 26 | uses: actions/upload-artifact@v3 27 | with: 28 | name: jupyter_voicepilot-releaser-dist-${{ github.run_number }} 29 | path: .jupyter_releaser_checkout/dist 30 | -------------------------------------------------------------------------------- /.github/workflows/enforce-label.yml: -------------------------------------------------------------------------------- 1 | name: Enforce PR label 2 | 3 | on: 4 | pull_request: 5 | types: [labeled, unlabeled, opened, edited, synchronize] 6 | jobs: 7 | enforce-label: 8 | runs-on: ubuntu-latest 9 | permissions: 10 | id-token: write 11 | contents: read 12 | issues: write 13 | pull-requests: write 14 | steps: 15 | - name: enforce-triage-label 16 | uses: jupyterlab/maintainer-tools/.github/actions/enforce-label@v1 17 | -------------------------------------------------------------------------------- /.github/workflows/prep-release.yml: -------------------------------------------------------------------------------- 1 | name: "Step 1: Prep Release" 2 | on: 3 | workflow_dispatch: 4 | inputs: 5 | version_spec: 6 | description: "New Version Specifier" 7 | default: "next" 8 | required: false 9 | branch: 10 | description: "The branch to target" 11 | required: false 12 | post_version_spec: 13 | description: "Post Version Specifier" 14 | required: false 15 | since: 16 | description: "Use PRs with activity since this date or git reference" 17 | required: false 18 | since_last_stable: 19 | description: "Use PRs with activity since the last stable git tag" 20 | required: false 21 | type: boolean 22 | jobs: 23 | prep_release: 24 | runs-on: ubuntu-latest 25 | steps: 26 | - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 27 | 28 | - name: Prep Release 29 | id: prep-release 30 | uses: jupyter-server/jupyter_releaser/.github/actions/prep-release@v2 31 | with: 32 | token: ${{ secrets.ADMIN_GITHUB_TOKEN }} 33 | version_spec: ${{ github.event.inputs.version_spec }} 34 | post_version_spec: ${{ github.event.inputs.post_version_spec }} 35 | target: ${{ github.event.inputs.target }} 36 | branch: ${{ github.event.inputs.branch }} 37 | since: ${{ github.event.inputs.since }} 38 | since_last_stable: ${{ github.event.inputs.since_last_stable }} 39 | 40 | - name: "** Next Step **" 41 | run: | 42 | echo "Optional): Review Draft Release: ${{ steps.prep-release.outputs.release_url }}" -------------------------------------------------------------------------------- /.github/workflows/publish-release.yml: -------------------------------------------------------------------------------- 1 | name: "Step 2: Publish Release" 2 | on: 3 | workflow_dispatch: 4 | inputs: 5 | branch: 6 | description: "The target branch" 7 | required: false 8 | release_url: 9 | description: "The URL of the draft GitHub release" 10 | required: false 11 | steps_to_skip: 12 | description: "Comma separated list of steps to skip" 13 | required: false 14 | 15 | jobs: 16 | publish_release: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 20 | 21 | - name: Populate Release 22 | id: populate-release 23 | uses: jupyter-server/jupyter_releaser/.github/actions/populate-release@v2 24 | with: 25 | token: ${{ secrets.ADMIN_GITHUB_TOKEN }} 26 | target: ${{ github.event.inputs.target }} 27 | branch: ${{ github.event.inputs.branch }} 28 | release_url: ${{ github.event.inputs.release_url }} 29 | steps_to_skip: ${{ github.event.inputs.steps_to_skip }} 30 | 31 | - name: Finalize Release 32 | id: finalize-release 33 | env: 34 | PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} 35 | PYPI_TOKEN_MAP: ${{ secrets.PYPI_TOKEN_MAP }} 36 | TWINE_USERNAME: __token__ 37 | NPM_TOKEN: ${{ secrets.NPM_TOKEN }} 38 | uses: jupyter-server/jupyter-releaser/.github/actions/finalize-release@v2 39 | with: 40 | token: ${{ secrets.ADMIN_GITHUB_TOKEN }} 41 | target: ${{ github.event.inputs.target }} 42 | release_url: ${{ steps.populate-release.outputs.release_url }} 43 | 44 | - name: "** Next Step **" 45 | if: ${{ success() }} 46 | run: | 47 | echo "Verify the final release" 48 | echo ${{ steps.finalize-release.outputs.release_url }} 49 | - name: "** Failure Message **" 50 | if: ${{ failure() }} 51 | run: | 52 | echo "Failed to Publish the Draft Release Url:" 53 | echo ${{ steps.populate-release.outputs.release_url }} -------------------------------------------------------------------------------- /.github/workflows/update-integration-tests.yml: -------------------------------------------------------------------------------- 1 | name: Update Playwright Snapshots 2 | 3 | on: 4 | issue_comment: 5 | types: [created, edited] 6 | 7 | permissions: 8 | contents: write 9 | pull-requests: write 10 | 11 | jobs: 12 | 13 | 14 | update-snapshots: 15 | if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, 'please update playwright snapshots') }} 16 | runs-on: ubuntu-latest 17 | 18 | steps: 19 | - name: Checkout 20 | uses: actions/checkout@v3 21 | with: 22 | token: ${{ secrets.GITHUB_TOKEN }} 23 | 24 | - name: Configure git to use https 25 | run: git config --global hub.protocol https 26 | 27 | - name: Checkout the branch from the PR that triggered the job 28 | run: hub pr checkout ${{ github.event.issue.number }} 29 | env: 30 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 31 | 32 | - name: Base Setup 33 | uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 34 | 35 | - name: Install dependencies 36 | run: python -m pip install -U jupyterlab~=3.1 37 | 38 | - name: Install extension 39 | run: | 40 | set -eux 41 | jlpm 42 | python -m pip install . 43 | 44 | - uses: jupyterlab/maintainer-tools/.github/actions/update-snapshots@v1 45 | with: 46 | github_token: ${{ secrets.GITHUB_TOKEN }} 47 | # Playwright knows how to start JupyterLab server 48 | start_server_script: 'null' 49 | test_folder: ui-tests 50 | 51 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.bundle.* 2 | lib/ 3 | node_modules/ 4 | *.log 5 | .eslintcache 6 | .stylelintcache 7 | *.egg-info/ 8 | .ipynb_checkpoints 9 | *.tsbuildinfo 10 | jupyter_voicepilot/labextension 11 | # Version file is handled by hatchling 12 | jupyter_voicepilot/_version.py 13 | 14 | # Integration tests 15 | ui-tests/test-results/ 16 | ui-tests/playwright-report/ 17 | 18 | # Created by https://www.gitignore.io/api/python 19 | # Edit at https://www.gitignore.io/?templates=python 20 | 21 | ### Python ### 22 | # Byte-compiled / optimized / DLL files 23 | __pycache__/ 24 | *.py[cod] 25 | *$py.class 26 | 27 | # C extensions 28 | *.so 29 | 30 | # Distribution / packaging 31 | .Python 32 | build/ 33 | develop-eggs/ 34 | dist/ 35 | downloads/ 36 | eggs/ 37 | .eggs/ 38 | lib/ 39 | lib64/ 40 | parts/ 41 | sdist/ 42 | var/ 43 | wheels/ 44 | pip-wheel-metadata/ 45 | share/python-wheels/ 46 | .installed.cfg 47 | *.egg 48 | MANIFEST 49 | 50 | # PyInstaller 51 | # Usually these files are written by a python script from a template 52 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 53 | *.manifest 54 | *.spec 55 | 56 | # Installer logs 57 | pip-log.txt 58 | pip-delete-this-directory.txt 59 | 60 | # Unit test / coverage reports 61 | htmlcov/ 62 | .tox/ 63 | .nox/ 64 | .coverage 65 | .coverage.* 66 | .cache 67 | nosetests.xml 68 | coverage/ 69 | coverage.xml 70 | *.cover 71 | .hypothesis/ 72 | .pytest_cache/ 73 | 74 | # Translations 75 | *.mo 76 | *.pot 77 | 78 | # Scrapy stuff: 79 | .scrapy 80 | 81 | # Sphinx documentation 82 | docs/_build/ 83 | 84 | # PyBuilder 85 | target/ 86 | 87 | # pyenv 88 | .python-version 89 | 90 | # celery beat schedule file 91 | celerybeat-schedule 92 | 93 | # SageMath parsed files 94 | *.sage.py 95 | 96 | # Spyder project settings 97 | .spyderproject 98 | .spyproject 99 | 100 | # Rope project settings 101 | .ropeproject 102 | 103 | # Mr Developer 104 | .mr.developer.cfg 105 | .project 106 | .pydevproject 107 | 108 | # mkdocs documentation 109 | /site 110 | 111 | # mypy 112 | .mypy_cache/ 113 | .dmypy.json 114 | dmypy.json 115 | 116 | # Pyre type checker 117 | .pyre/ 118 | 119 | # End of https://www.gitignore.io/api/python 120 | 121 | # OSX files 122 | .DS_Store 123 | 124 | # Direnv 125 | .envrc -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | **/node_modules 3 | **/lib 4 | **/package.json 5 | !/package.json 6 | jupyter_voicepilot 7 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "singleQuote": true, 3 | "trailingComma": "none", 4 | "arrowParens": "avoid", 5 | "endOfLine": "auto" 6 | } 7 | -------------------------------------------------------------------------------- /.stylelintrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "stylelint-config-recommended", 4 | "stylelint-config-standard", 5 | "stylelint-prettier/recommended" 6 | ], 7 | "rules": { 8 | "property-no-vendor-prefix": null, 9 | "selector-no-vendor-prefix": null, 10 | "value-no-vendor-prefix": null 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | 4 | 5 | ## 0.1.2 6 | 7 | ([Full Changelog](https://github.com/JovanVeljanoski/jupyter-voicepilot/compare/d4cc7e8a9963de81351d86838166022c1600bceb...5939c008c2234d40cc785f9daad8bb56e4d8e834)) 8 | 9 | ### New features added 10 | 11 | - Connect to ChatGPT [#11](https://github.com/JovanVeljanoski/jupyter-voicepilot/pull/11) ([@JovanVeljanoski](https://github.com/JovanVeljanoski)) 12 | - feat: adding notebook navigation and variety of voice commands. [#7](https://github.com/JovanVeljanoski/jupyter-voicepilot/pull/7) ([@JovanVeljanoski](https://github.com/JovanVeljanoski)) 13 | 14 | ### Enhancements made 15 | 16 | - Toggle Voice Pilot keyboard shortcut [#12](https://github.com/JovanVeljanoski/jupyter-voicepilot/pull/12) ([@itallix](https://github.com/itallix)) 17 | - feat: adding release workflows [#10](https://github.com/JovanVeljanoski/jupyter-voicepilot/pull/10) ([@JovanVeljanoski](https://github.com/JovanVeljanoski)) 18 | - feat: additional notebook nav commands [#8](https://github.com/JovanVeljanoski/jupyter-voicepilot/pull/8) ([@JovanVeljanoski](https://github.com/JovanVeljanoski)) 19 | - Animate button when recording in progress [#6](https://github.com/JovanVeljanoski/jupyter-voicepilot/pull/6) ([@itallix](https://github.com/itallix)) 20 | - Make apiKey configurable [#5](https://github.com/JovanVeljanoski/jupyter-voicepilot/pull/5) ([@itallix](https://github.com/itallix)) 21 | 22 | ### Documentation improvements 23 | 24 | - docs: add demo to readme [#13](https://github.com/JovanVeljanoski/jupyter-voicepilot/pull/13) ([@JovanVeljanoski](https://github.com/JovanVeljanoski)) 25 | - update the readme [#9](https://github.com/JovanVeljanoski/jupyter-voicepilot/pull/9) ([@JovanVeljanoski](https://github.com/JovanVeljanoski)) 26 | 27 | ### Contributors to this release 28 | 29 | ([GitHub contributors page for this release](https://github.com/JovanVeljanoski/jupyter-voicepilot/graphs/contributors?from=2023-03-16&to=2023-03-24&type=c)) 30 | 31 | [@itallix](https://github.com/search?q=repo%3AJovanVeljanoski%2Fjupyter-voicepilot+involves%3Aitallix+updated%3A2023-03-16..2023-03-24&type=Issues) | [@JovanVeljanoski](https://github.com/search?q=repo%3AJovanVeljanoski%2Fjupyter-voicepilot+involves%3AJovanVeljanoski+updated%3A2023-03-16..2023-03-24&type=Issues) 32 | 33 | 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Jovan Veljanoski 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # jupyter_voicepilot 2 | 3 | [![Github Actions Status](https://github.com/JovanVeljanoski/jupyter-voicepilot/workflows/Build/badge.svg)](https://github.com/JovanVeljanoski/jupyter-voicepilot/actions/workflows/build.yml) 4 | [![Downloads](https://static.pepy.tech/badge/jupyter-voicepilot/month)](https://pepy.tech/project/jupyter-voicepilot) 5 | 6 | A JupyterLab extension for generating code and interacting with JupyterLab via voice commands. This extension can also be used for some basic nagivation around in JupyterLab Notebook. It is built around OpenAI's `Whisper-1` and `GPT-3` APIs. You will need to have an OpenAI API key to use this extension. 7 | 8 | Click on the image below to see a demo of the extension: 9 | [![IMAGE ALT TEXT HERE](https://img.youtube.com/vi/JlHnDm4oUgA/maxresdefault.jpg)](https://www.youtube.com/watch?v=JlHnDm4oUgA) 10 | 11 | ## Requirements 12 | 13 | - JupyterLab >= 3.0 14 | 15 | ## Install 16 | 17 | To install the extension, execute: 18 | 19 | ```bash 20 | pip install jupyter_voicepilot 21 | ``` 22 | 23 | ## Uninstall 24 | 25 | To remove the extension, execute: 26 | 27 | ```bash 28 | pip uninstall jupyter_voicepilot 29 | ``` 30 | 31 | ## Usage 32 | 33 | Click the `Voice Pilot` button in the JupyterLab to start recording your instruction. When done, click the button again to stop the recording. The extension will then process your instruction and execute the appropriate action. 34 | 35 | ### Generating code 36 | 37 | If a cell is of type `code`, the extension will generate code based on the input you provided. 38 | The generated code will be inserted in the cell. An exception to this is when you provide a set phrase which is mapped to a specific Notebook navigation action. See below for more details. 39 | 40 | ### Dictation 41 | 42 | If a cell is of type `markdown`, the extension will insert the text you provided in the cell. 43 | An exception to this is when you provide a set phrase which is mapped to a specific Notebook navigation action. See below for more details. 44 | 45 | ### ChatGPT interaction 46 | 47 | If you start your voice message with "hey", the message will be sent to the `ChatGPT` model. 48 | If the current cell is empty, it will be converted to a markdown cell and the response will be added there. If the current cell is not empty, a new `markdown` cell will be inserted below the current cell, containing the response from ChatGPT. 49 | 50 | ### Notebook navigation 51 | 52 | The extension can also be used for some basic nagivation around in JupyterLab Notebook. The following table shows the Notebook actions that a supported, and the corresponding phrases that you can use to trigger them. 53 | 54 | | Action | Phrase | 55 | | -------------------------- | -------------------------------------------------------------------------------------- | 56 | | `run` | "run", "run cell", "run the cell", "execute" | 57 | | `runAll` | "run all", "run all cells", "execute all", "execute all cells" | 58 | | `runAndAdvance` | "run and advance", "run cell and advance", "execute and advance" | 59 | | `runAndInsert` | "run and insert", "run cell and insert", "execute and insert" | 60 | | `runAllAbove` | "run all above", "run all cells above", "execute all above", "execute all cells above" | 61 | | `runAllBelow` | "run all below", "run all cells below", "execute all below", "execute all cells below" | 62 | | `deleteCells` | "delete", "delete cell", "delete cells", "delete the cell", "delete the cells" | 63 | | `clearAllOutputs` | "clear all outputs", "clear all the outputs", "clear outputs" | 64 | | `selectLastRunCell` | "select last run cell", "select the last run cell" | 65 | | `undo` | "undo" | 66 | | `redo` | "redo" | 67 | | `cut` | "cut" | 68 | | `copy` | "copy" | 69 | | `paste` | "paste" | 70 | | `changeCellTypeToMarkdown` | "markdown", "to markdown", "markdown cell", "convert to markdown", "cast to markdown" | 71 | | `changeCellTypeToCode` | "code", "to code", "code cell", "convert to code", "cast to code" | 72 | | `insertMarkdownCellBelow` | "insert markdown cell below", "add markdown below" | 73 | | `insertMarkdownCellAbove` | "insert markdown cell above", "add markdown above" | 74 | | `insertCodeCellBelow` | "insert code cell below", "add code below" | 75 | | `insertCodeCellAbove` | "insert code cell above", "add code above" | 76 | 77 | ## Configuration 78 | 79 | In the advanced settings editor, you can set the following configuration options: 80 | 81 | - Open API Key (_required_): Your OpenAI API key. You can get one [here](https://platform.openai.com/overview). 82 | 83 | ## Contributing 84 | 85 | ### Development install 86 | 87 | Note: You will need NodeJS to build the extension package. 88 | 89 | The `jlpm` command is JupyterLab's pinned version of 90 | [yarn](https://yarnpkg.com/) that is installed with JupyterLab. You may use 91 | `yarn` or `npm` in lieu of `jlpm` below. 92 | 93 | ```bash 94 | # Clone the repo to your local environment 95 | # Change directory to the jupyter_voicepilot directory 96 | # Install package in development mode 97 | pip install -e "." 98 | # Link your development version of the extension with JupyterLab 99 | jupyter labextension develop . --overwrite 100 | # Rebuild extension Typescript source after making changes 101 | jlpm build 102 | ``` 103 | 104 | You can watch the source directory and run JupyterLab at the same time in different terminals to watch for changes in the extension's source and automatically rebuild the extension. 105 | 106 | ```bash 107 | # Watch the source directory in one terminal, automatically rebuilding when needed 108 | jlpm watch 109 | # Run JupyterLab in another terminal 110 | jupyter lab 111 | ``` 112 | 113 | With the watch command running, every saved change will immediately be built locally and available in your running JupyterLab. Refresh JupyterLab to load the change in your browser (you may need to wait several seconds for the extension to be rebuilt). 114 | 115 | By default, the `jlpm build` command generates the source maps for this extension to make it easier to debug using the browser dev tools. To also generate source maps for the JupyterLab core extensions, you can run the following command: 116 | 117 | ```bash 118 | jupyter lab build --minimize=False 119 | ``` 120 | 121 | ### Development uninstall 122 | 123 | ```bash 124 | pip uninstall jupyter_voicepilot 125 | ``` 126 | 127 | In development mode, you will also need to remove the symlink created by `jupyter labextension develop` 128 | command. To find its location, you can run `jupyter labextension list` to figure out where the `labextensions` 129 | folder is located. Then you can remove the symlink named `voicepilot` within that folder. 130 | 131 | 154 | -------------------------------------------------------------------------------- /RELEASE.md: -------------------------------------------------------------------------------- 1 | # Making a new release of jupyter_voicepilot 2 | 3 | The extension can be published to `PyPI` and `npm` manually or using the [Jupyter Releaser](https://github.com/jupyter-server/jupyter_releaser). 4 | 5 | ## Manual release 6 | 7 | ### Python package 8 | 9 | This extension can be distributed as Python 10 | packages. All of the Python 11 | packaging instructions in the `pyproject.toml` file to wrap your extension in a 12 | Python package. Before generating a package, we first need to install `build`. 13 | 14 | ```bash 15 | pip install build twine hatch 16 | ``` 17 | 18 | Bump the version using `hatch`. By default this will create a tag. 19 | See the docs on [hatch-nodejs-version](https://github.com/agoose77/hatch-nodejs-version#semver) for details. 20 | 21 | ```bash 22 | hatch version 23 | ``` 24 | 25 | To create a Python source package (`.tar.gz`) and the binary package (`.whl`) in the `dist/` directory, do: 26 | 27 | ```bash 28 | python -m build 29 | ``` 30 | 31 | > `python setup.py sdist bdist_wheel` is deprecated and will not work for this package. 32 | 33 | Then to upload the package to PyPI, do: 34 | 35 | ```bash 36 | twine upload dist/* 37 | ``` 38 | 39 | ### NPM package 40 | 41 | To publish the frontend part of the extension as a NPM package, do: 42 | 43 | ```bash 44 | npm login 45 | npm publish --access public 46 | ``` 47 | 48 | ## Automated releases with the Jupyter Releaser 49 | 50 | The extension repository should already be compatible with the Jupyter Releaser. 51 | 52 | Check out the [workflow documentation](https://github.com/jupyter-server/jupyter_releaser#typical-workflow) for more information. 53 | 54 | Here is a summary of the steps to cut a new release: 55 | 56 | - Fork the [`jupyter-releaser` repo](https://github.com/jupyter-server/jupyter_releaser) 57 | - Add `ADMIN_GITHUB_TOKEN`, `PYPI_TOKEN` and `NPM_TOKEN` to the Github Secrets in the fork 58 | - Go to the Actions panel 59 | - Run the "Draft Changelog" workflow 60 | - Merge the Changelog PR 61 | - Run the "Draft Release" workflow 62 | - Run the "Publish Release" workflow 63 | 64 | ## Publishing to `conda-forge` 65 | 66 | If the package is not on conda forge yet, check the documentation to learn how to add it: https://conda-forge.org/docs/maintainer/adding_pkgs.html 67 | 68 | Otherwise a bot should pick up the new version publish to PyPI, and open a new PR on the feedstock repository automatically. 69 | -------------------------------------------------------------------------------- /babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = require('@jupyterlab/testutils/lib/babel.config'); 2 | -------------------------------------------------------------------------------- /install.json: -------------------------------------------------------------------------------- 1 | { 2 | "packageManager": "python", 3 | "packageName": "jupyter_voicepilot", 4 | "uninstallInstructions": "Use your Python package manager (pip, conda, etc.) to uninstall the package jupyter_voicepilot" 5 | } 6 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | const jestJupyterLab = require('@jupyterlab/testutils/lib/jest-config'); 2 | 3 | const esModules = [ 4 | '@jupyterlab/', 5 | 'lib0', 6 | 'y\\-protocols', 7 | 'y\\-websocket', 8 | 'yjs' 9 | ].join('|'); 10 | 11 | const jlabConfig = jestJupyterLab(__dirname); 12 | 13 | const { 14 | moduleFileExtensions, 15 | moduleNameMapper, 16 | preset, 17 | setupFilesAfterEnv, 18 | setupFiles, 19 | testPathIgnorePatterns, 20 | transform 21 | } = jlabConfig; 22 | 23 | module.exports = { 24 | moduleFileExtensions, 25 | moduleNameMapper, 26 | preset, 27 | setupFilesAfterEnv, 28 | setupFiles, 29 | testPathIgnorePatterns, 30 | transform, 31 | automock: false, 32 | collectCoverageFrom: [ 33 | 'src/**/*.{ts,tsx}', 34 | '!src/**/*.d.ts', 35 | '!src/**/.ipynb_checkpoints/*' 36 | ], 37 | coverageDirectory: 'coverage', 38 | coverageReporters: ['lcov', 'text'], 39 | globals: { 40 | 'ts-jest': { 41 | tsconfig: 'tsconfig.json' 42 | } 43 | }, 44 | testRegex: 'src/.*/.*.spec.ts[x]?$', 45 | transformIgnorePatterns: [`/node_modules/(?!${esModules}).+`] 46 | }; 47 | -------------------------------------------------------------------------------- /jupyter_voicepilot/__init__.py: -------------------------------------------------------------------------------- 1 | from ._version import __version__ 2 | 3 | 4 | def _jupyter_labextension_paths(): 5 | return [{ 6 | "src": "labextension", 7 | "dest": "voicepilot" 8 | }] 9 | 10 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "voicepilot", 3 | "version": "0.1.2", 4 | "description": "A JupyterLab extension for generating code and interacting with JupyterLab via voice commands.", 5 | "keywords": [ 6 | "jupyter", 7 | "jupyterlab", 8 | "jupyterlab-extension" 9 | ], 10 | "homepage": "https://github.com/JovanVeljanoski/jupyter-voicepilot", 11 | "bugs": { 12 | "url": "https://github.com/JovanVeljanoski/jupyter-voicepilot/issues" 13 | }, 14 | "license": "BSD-3-Clause", 15 | "author": { 16 | "name": "Jovan Veljanoski", 17 | "email": "jovan.veljanoski@gmail.com" 18 | }, 19 | "files": [ 20 | "lib/**/*.{d.ts,eot,gif,html,jpg,js,js.map,json,png,svg,woff2,ttf}", 21 | "style/**/*.{css,js,eot,gif,html,jpg,json,png,svg,woff2,ttf}", 22 | "schema/*.json" 23 | ], 24 | "main": "lib/index.js", 25 | "types": "lib/index.d.ts", 26 | "style": "style/index.css", 27 | "repository": { 28 | "type": "git", 29 | "url": "https://github.com/JovanVeljanoski/jupyter-voicepilot.git" 30 | }, 31 | "scripts": { 32 | "build": "jlpm build:lib && jlpm build:labextension:dev", 33 | "build:prod": "jlpm clean && jlpm build:lib:prod && jlpm build:labextension", 34 | "build:labextension": "jupyter labextension build .", 35 | "build:labextension:dev": "jupyter labextension build --development True .", 36 | "build:lib": "tsc --sourceMap", 37 | "build:lib:prod": "tsc", 38 | "clean": "jlpm clean:lib", 39 | "clean:lib": "rimraf lib tsconfig.tsbuildinfo", 40 | "clean:lintcache": "rimraf .eslintcache .stylelintcache", 41 | "clean:labextension": "rimraf jupyter_voicepilot/labextension jupyter_voicepilot/_version.py", 42 | "clean:all": "jlpm clean:lib && jlpm clean:labextension && jlpm clean:lintcache", 43 | "eslint": "jlpm eslint:check --fix", 44 | "eslint:check": "eslint . --cache --ext .ts,.tsx", 45 | "install:extension": "jlpm build", 46 | "lint": "jlpm stylelint && jlpm prettier && jlpm eslint", 47 | "lint:check": "jlpm stylelint:check && jlpm prettier:check && jlpm eslint:check", 48 | "prettier": "jlpm prettier:base --write --list-different", 49 | "prettier:base": "prettier \"**/*{.ts,.tsx,.js,.jsx,.css,.json,.md}\"", 50 | "prettier:check": "jlpm prettier:base --check", 51 | "stylelint": "jlpm stylelint:check --fix", 52 | "stylelint:check": "stylelint --cache \"style/**/*.css\"", 53 | "test": "jest --coverage", 54 | "watch": "run-p watch:src watch:labextension", 55 | "watch:src": "tsc -w", 56 | "watch:labextension": "jupyter labextension watch ." 57 | }, 58 | "dependencies": { 59 | "@jupyterlab/application": "^3.1.0", 60 | "@jupyterlab/settingregistry": "^3.1.0", 61 | "axios": "^1.3.4", 62 | "openai": "^3.2.1" 63 | }, 64 | "devDependencies": { 65 | "@babel/core": "^7.0.0", 66 | "@babel/preset-env": "^7.0.0", 67 | "@jupyterlab/builder": "^3.1.0", 68 | "@jupyterlab/testutils": "^3.0.0", 69 | "@types/dom-mediacapture-record": "^1.0.15", 70 | "@types/jest": "^26.0.0", 71 | "@typescript-eslint/eslint-plugin": "^4.8.1", 72 | "@typescript-eslint/parser": "^4.8.1", 73 | "eslint": "^7.14.0", 74 | "eslint-config-prettier": "^6.15.0", 75 | "eslint-plugin-prettier": "^3.1.4", 76 | "jest": "^26.0.0", 77 | "npm-run-all": "^4.1.5", 78 | "prettier": "^2.1.1", 79 | "rimraf": "^3.0.2", 80 | "stylelint": "^14.3.0", 81 | "stylelint-config-prettier": "^9.0.4", 82 | "stylelint-config-recommended": "^6.0.0", 83 | "stylelint-config-standard": "~24.0.0", 84 | "stylelint-prettier": "^2.0.0", 85 | "ts-jest": "^26.0.0", 86 | "typescript": "~4.1.3" 87 | }, 88 | "sideEffects": [ 89 | "style/*.css", 90 | "style/index.js" 91 | ], 92 | "styleModule": "style/index.js", 93 | "publishConfig": { 94 | "access": "public" 95 | }, 96 | "jupyterlab": { 97 | "extension": true, 98 | "outputDir": "jupyter_voicepilot/labextension", 99 | "schemaDir": "schema" 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["hatchling>=1.4.0", "jupyterlab>=3.4.7,<4.0.0", "hatch-nodejs-version"] 3 | build-backend = "hatchling.build" 4 | 5 | [project] 6 | name = "jupyter_voicepilot" 7 | readme = "README.md" 8 | license = { file = "LICENSE" } 9 | requires-python = ">=3.7" 10 | classifiers = [ 11 | "Framework :: Jupyter", 12 | "Framework :: Jupyter :: JupyterLab", 13 | "Framework :: Jupyter :: JupyterLab :: 3", 14 | "Framework :: Jupyter :: JupyterLab :: Extensions", 15 | "Framework :: Jupyter :: JupyterLab :: Extensions :: Prebuilt", 16 | "License :: OSI Approved :: BSD License", 17 | "Programming Language :: Python", 18 | "Programming Language :: Python :: 3", 19 | "Programming Language :: Python :: 3.7", 20 | "Programming Language :: Python :: 3.8", 21 | "Programming Language :: Python :: 3.9", 22 | "Programming Language :: Python :: 3.10", 23 | "Programming Language :: Python :: 3.11", 24 | ] 25 | dependencies = [ 26 | ] 27 | dynamic = ["version", "description", "authors", "urls", "keywords"] 28 | 29 | [tool.hatch.version] 30 | source = "nodejs" 31 | 32 | [tool.hatch.metadata.hooks.nodejs] 33 | fields = ["description", "authors", "urls"] 34 | 35 | [tool.hatch.build.targets.sdist] 36 | artifacts = ["jupyter_voicepilot/labextension"] 37 | exclude = [".github", "binder"] 38 | 39 | [tool.hatch.build.targets.wheel.shared-data] 40 | "jupyter_voicepilot/labextension" = "share/jupyter/labextensions/voicepilot" 41 | "install.json" = "share/jupyter/labextensions/voicepilot/install.json" 42 | 43 | [tool.hatch.build.hooks.version] 44 | path = "jupyter_voicepilot/_version.py" 45 | 46 | [tool.hatch.build.hooks.jupyter-builder] 47 | dependencies = ["hatch-jupyter-builder>=0.5"] 48 | build-function = "hatch_jupyter_builder.npm_builder" 49 | ensured-targets = [ 50 | "jupyter_voicepilot/labextension/static/style.js", 51 | "jupyter_voicepilot/labextension/package.json", 52 | ] 53 | skip-if-exists = ["jupyter_voicepilot/labextension/static/style.js"] 54 | 55 | [tool.hatch.build.hooks.jupyter-builder.build-kwargs] 56 | build_cmd = "build:prod" 57 | npm = ["jlpm"] 58 | 59 | [tool.hatch.build.hooks.jupyter-builder.editable-build-kwargs] 60 | build_cmd = "install:extension" 61 | npm = ["jlpm"] 62 | source_dir = "src" 63 | build_dir = "jupyter_voicepilot/labextension" 64 | 65 | [tool.jupyter-releaser.options] 66 | version_cmd = "hatch version" 67 | 68 | [tool.jupyter-releaser.hooks] 69 | before-build-npm = ["python -m pip install jupyterlab~=3.1", "jlpm", "jlpm build:prod"] 70 | before-build-python = ["jlpm clean:all"] 71 | 72 | [tool.check-wheel-contents] 73 | ignore = ["W002"] 74 | -------------------------------------------------------------------------------- /schema/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "jupyter.lab.shortcuts": [ 3 | { 4 | "command": "voicepilot:toggle-button", 5 | "keys": ["Ctrl Shift V"], 6 | "selector": ".jp-Notebook:focus" 7 | } 8 | ], 9 | "title": "Voice Pilot", 10 | "description": "Voice Pilot settings.", 11 | "type": "object", 12 | "properties": { 13 | "open_api_key": { 14 | "type": "string", 15 | "title": "Open API Key", 16 | "description": "Your Open API Key", 17 | "default": "" 18 | }, 19 | "max_tokens": { 20 | "type": "integer", 21 | "title": "Max Tokens", 22 | "description": "The maximum number of tokens to generate", 23 | "default": 256, 24 | "minimum": 1 25 | }, 26 | "chat_history_length": { 27 | "type": "integer", 28 | "title": "Chat History Length", 29 | "description": "The number of messages to remember (1 - 20)", 30 | "default": 10, 31 | "minimum": 1, 32 | "maximum": 20 33 | } 34 | }, 35 | "additionalProperties": false 36 | } 37 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | __import__('setuptools').setup() 2 | -------------------------------------------------------------------------------- /src/__tests__/jupyter_voicepilot.spec.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Example of [Jest](https://jestjs.io/docs/getting-started) unit tests 3 | */ 4 | 5 | describe('voicepilot', () => { 6 | it('should be tested', () => { 7 | expect(1 + 1).toEqual(2); 8 | }); 9 | }); 10 | -------------------------------------------------------------------------------- /src/custom-typings.d.ts: -------------------------------------------------------------------------------- 1 | // Type definitions for non-npm package w3c MediaStream Recording 1.0 2 | // Project: https://w3c.github.io/mediacapture-record 3 | // Definitions by: Elias Meire 4 | // AppLover69 5 | // Definitions: https://github.com/DefinitelyTyped/DefinitelyTyped 6 | 7 | declare enum BitrateMode { 8 | VBR = 0, 9 | CBR = 1 10 | } 11 | 12 | interface MediaRecorderErrorEventInit extends EventInit { 13 | error: DOMException; 14 | } 15 | 16 | interface MediaRecorderErrorEvent extends Event { 17 | readonly error: DOMException; 18 | } 19 | 20 | declare var MediaRecorderErrorEvent: { 21 | prototype: MediaRecorderErrorEvent; 22 | new ( 23 | type: string, 24 | eventInitDict: MediaRecorderErrorEventInit 25 | ): MediaRecorderErrorEvent; 26 | }; 27 | 28 | interface BlobEventInit extends EventInit { 29 | data: Blob; 30 | timecode?: number | undefined; 31 | } 32 | 33 | interface BlobEvent extends Event { 34 | readonly data: Blob; 35 | readonly timecode: DOMHighResTimeStamp; 36 | } 37 | 38 | declare var BlobEvent: { 39 | prototype: BlobEvent; 40 | new (type: string, eventInitDict: BlobEventInit): BlobEvent; 41 | }; 42 | 43 | // type BitrateMode = 'variable' | 'constant'; 44 | 45 | interface MediaRecorderOptions { 46 | mimeType?: string | undefined; 47 | audioBitsPerSecond?: number | undefined; 48 | videoBitsPerSecond?: number | undefined; 49 | bitsPerSecond?: number | undefined; 50 | audioBitrateMode?: BitrateMode | undefined; 51 | } 52 | 53 | interface MediaRecorderEventMap { 54 | dataavailable: BlobEvent; 55 | error: Event; 56 | pause: Event; 57 | resume: Event; 58 | start: Event; 59 | stop: Event; 60 | } 61 | 62 | interface MediaRecorder extends EventTarget { 63 | readonly stream: MediaStream; 64 | readonly mimeType: string; 65 | readonly state: 'inactive' | 'recording' | 'paused'; 66 | readonly videoBitsPerSecond: number; 67 | readonly audioBitsPerSecond: number; 68 | readonly audioBitrateMode: BitrateMode; 69 | 70 | ondataavailable: ((this: MediaRecorder, event: BlobEvent) => any) | null; 71 | onerror: ((this: MediaRecorder, event: Event) => any) | null; 72 | onpause: ((this: MediaRecorder, event: Event) => any) | null; 73 | onresume: ((this: MediaRecorder, event: Event) => any) | null; 74 | onstart: ((this: MediaRecorder, event: Event) => any) | null; 75 | onstop: ((this: MediaRecorder, event: Event) => any) | null; 76 | 77 | addEventListener( 78 | type: K, 79 | listener: (this: MediaRecorder, ev: MediaRecorderEventMap[K]) => any, 80 | options?: boolean | AddEventListenerOptions 81 | ): void; 82 | addEventListener( 83 | type: string, 84 | listener: EventListenerOrEventListenerObject, 85 | options?: boolean | AddEventListenerOptions 86 | ): void; 87 | removeEventListener( 88 | type: K, 89 | listener: (this: MediaRecorder, ev: MediaRecorderEventMap[K]) => any, 90 | options?: boolean | EventListenerOptions 91 | ): void; 92 | removeEventListener( 93 | type: string, 94 | listener: EventListenerOrEventListenerObject, 95 | options?: boolean | EventListenerOptions 96 | ): void; 97 | 98 | start(timeslice?: number): void; 99 | stop(): void; 100 | resume(): void; 101 | pause(): void; 102 | requestData(): void; 103 | } 104 | 105 | declare var MediaRecorder: { 106 | prototype: MediaRecorder; 107 | new (stream: MediaStream, options?: MediaRecorderOptions): MediaRecorder; 108 | isTypeSupported(type: string): boolean; 109 | }; 110 | 111 | interface Window { 112 | MediaRecorder: typeof MediaRecorder; 113 | BlobEvent: typeof BlobEvent; 114 | MediaRecorderErrorEvent: typeof MediaRecorderErrorEvent; 115 | } 116 | -------------------------------------------------------------------------------- /src/extension.ts: -------------------------------------------------------------------------------- 1 | import { IDisposable, DisposableDelegate } from '@lumino/disposable'; 2 | import { DocumentRegistry } from '@jupyterlab/docregistry'; 3 | import { NotebookPanel, INotebookModel } from '@jupyterlab/notebook'; 4 | import { ToolbarButton } from '@jupyterlab/apputils'; 5 | import { LabIcon } from '@jupyterlab/ui-components'; 6 | import recordVinylStr from '../style/icons/record-vinyl-solid.svg'; 7 | import { VoiceProcessor } from './voice-processor'; 8 | 9 | const vynilIcon = new LabIcon({ 10 | name: 'jupyterlab:record-vinyl', 11 | svgstr: recordVinylStr 12 | }); 13 | 14 | export class ButtonExtension 15 | implements DocumentRegistry.IWidgetExtension 16 | { 17 | private button: ToolbarButton | null = null; 18 | private voiceProcessor: VoiceProcessor = new VoiceProcessor(); 19 | 20 | set apiKey(apiKey: string) { 21 | this.voiceProcessor.apiKey = apiKey; 22 | } 23 | 24 | set maxTokens(maxTokens: number) { 25 | this.voiceProcessor.maxTokens = maxTokens; 26 | } 27 | 28 | set chatHistoryMaxLength(chatHistoryMaxLength: number) { 29 | this.voiceProcessor.chatHistoryMaxLength = chatHistoryMaxLength; 30 | } 31 | 32 | toggleRecording() { 33 | this.button?.onClick(); 34 | } 35 | 36 | /** 37 | * Create a new extension for the notebook panel widget. 38 | * 39 | * @param panel Notebook panel 40 | * @param context Notebook context 41 | * @returns Disposable on the added button 42 | */ 43 | createNew( 44 | panel: NotebookPanel, 45 | context: DocumentRegistry.IContext 46 | ): IDisposable { 47 | const onClick = async () => { 48 | if (!this.voiceProcessor.isConfigured()) { 49 | return; 50 | } 51 | const is_recording = this.button!.hasClass('vp-recording'); 52 | this.button!.toggleClass('vp-recording'); 53 | if (is_recording) { 54 | this.voiceProcessor.stop(panel); 55 | } else { 56 | this.voiceProcessor.start(); 57 | } 58 | }; 59 | this.button = new ToolbarButton({ 60 | className: 'vp-button', 61 | icon: vynilIcon, 62 | pressedTooltip: 'Stop recording...', 63 | label: 'Voice Pilot', 64 | onClick: onClick, 65 | tooltip: 'Start recording...' 66 | }); 67 | 68 | panel.toolbar.insertItem(10, 'vp-button', this.button); 69 | return new DisposableDelegate(() => { 70 | this.button?.dispose(); 71 | }); 72 | } 73 | } 74 | 75 | export default ButtonExtension; 76 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import { 2 | JupyterFrontEnd, 3 | JupyterFrontEndPlugin 4 | } from '@jupyterlab/application'; 5 | 6 | import { ISettingRegistry } from '@jupyterlab/settingregistry'; 7 | import { ICommandPalette, showDialog, Dialog } from '@jupyterlab/apputils'; 8 | 9 | import { ButtonExtension } from './extension'; 10 | 11 | /** 12 | * Initialization data for the voicepilot extension. 13 | */ 14 | const PLUGIN_ID = 'voicepilot:plugin'; 15 | const plugin: JupyterFrontEndPlugin = { 16 | id: PLUGIN_ID, 17 | autoStart: true, 18 | requires: [ISettingRegistry, ICommandPalette], 19 | activate: ( 20 | app: JupyterFrontEnd, 21 | settings: ISettingRegistry, 22 | palette: ICommandPalette 23 | ) => { 24 | console.log('JupyterLab extension voicepilot is activated!'); 25 | const { commands, docRegistry } = app; 26 | const buttonExt = new ButtonExtension(); 27 | docRegistry.addWidgetExtension('Notebook', buttonExt); 28 | 29 | palette.addItem({ 30 | command: 'voicepilot:show-api-key', 31 | category: 'VoicePilot' 32 | }); 33 | palette.addItem({ 34 | command: 'voicepilot:toggle-button', 35 | category: 'VoicePilot' 36 | }); 37 | 38 | commands.addCommand('voicepilot:toggle-button', { 39 | label: 'Toggle VoicePilot', 40 | execute: () => { 41 | buttonExt.toggleRecording(); 42 | } 43 | }); 44 | 45 | /** 46 | * Load the settings for this extension 47 | * 48 | * @param setting Extension settings 49 | */ 50 | function updateExtension(settings: ISettingRegistry.ISettings): void { 51 | const apiKey = settings.get('open_api_key').composite as string; 52 | const maxTokens = settings.get('max_tokens').composite as number; 53 | const chatHistoryMaxLength = settings.get('chat_history_length') 54 | .composite as number; 55 | buttonExt.apiKey = apiKey; 56 | buttonExt.maxTokens = maxTokens; 57 | buttonExt.chatHistoryMaxLength = chatHistoryMaxLength; 58 | } 59 | 60 | // Wait for the application to be restored and 61 | // for the settings for this plugin to be loaded 62 | Promise.all([app.restored, settings?.load(PLUGIN_ID)]) 63 | .then(([, settings]) => { 64 | // Read the settings 65 | updateExtension(settings); 66 | 67 | commands.addCommand('voicepilot:show-api-key', { 68 | label: 'Show API Key', 69 | execute: () => { 70 | const apiKey = settings?.get('open_api_key').composite as string; 71 | buttonExt.apiKey = apiKey; 72 | return showDialog({ 73 | title: 'VoicePilot API Key', 74 | body: apiKey, 75 | buttons: [Dialog.okButton()] 76 | }); 77 | } 78 | }); 79 | 80 | // Listen for your plugin setting changes using Signal 81 | settings?.changed.connect(updateExtension); 82 | }) 83 | .catch(reason => { 84 | console.error( 85 | `Something went wrong when reading the settings.\n${reason}` 86 | ); 87 | }); 88 | } 89 | }; 90 | 91 | export default plugin; 92 | -------------------------------------------------------------------------------- /src/notebook/cmd-handler.ts: -------------------------------------------------------------------------------- 1 | import { NotebookActions, NotebookPanel } from '@jupyterlab/notebook'; 2 | import { Notebook } from '@jupyterlab/notebook'; 3 | 4 | type NotebookAction = CallableFunction; 5 | type CmdRegistry = Record>; 6 | 7 | function changeCellTypeToMarkdown(widget: Notebook): void { 8 | NotebookActions.changeCellType(widget, 'markdown'); 9 | } 10 | 11 | function changeCellTypeToCode(widget: Notebook): void { 12 | NotebookActions.changeCellType(widget, 'code'); 13 | } 14 | 15 | export class NotebookCmdHandler { 16 | private registry: CmdRegistry = {}; 17 | 18 | constructor() { 19 | for (const cmd of ['run cell', 'run the cell', 'run', 'execute']) { 20 | this.registry[cmd] = [NotebookActions.run]; 21 | } 22 | for (const cmd of [ 23 | 'run all', 24 | 'run all cells', 25 | 'execute all', 26 | 'execute all cells' 27 | ]) { 28 | this.registry[cmd] = [NotebookActions.runAll]; 29 | } 30 | for (const cmd of [ 31 | 'run and advance', 32 | 'run cell and advance', 33 | 'execute and advance' 34 | ]) { 35 | this.registry[cmd] = [NotebookActions.runAndAdvance]; 36 | } 37 | for (const cmd of [ 38 | 'run all above', 39 | 'run all cells above', 40 | 'execute all above', 41 | 'execute all cells above' 42 | ]) { 43 | this.registry[cmd] = [NotebookActions.runAllAbove]; 44 | } 45 | for (const cmd of [ 46 | 'run all below', 47 | 'run all cells below', 48 | 'execute all below', 49 | 'execute all cells below' 50 | ]) { 51 | this.registry[cmd] = [NotebookActions.runAllBelow]; 52 | } 53 | for (const cmd of [ 54 | 'run and insert', 55 | 'run cell and insert', 56 | 'execute and insert' 57 | ]) { 58 | this.registry[cmd] = [NotebookActions.runAndInsert]; 59 | } 60 | for (const cmd of [ 61 | 'delete', 62 | 'delete cell', 63 | 'delete the cell', 64 | 'delete cells', 65 | 'delete the cells' 66 | ]) { 67 | this.registry[cmd] = [NotebookActions.deleteCells]; 68 | } 69 | for (const cmd of [ 70 | 'clear all outputs', 71 | 'clear all the outputs', 72 | 'clear outputs' 73 | ]) { 74 | this.registry[cmd] = [NotebookActions.clearAllOutputs]; 75 | } 76 | for (const cmd of ['select last run cell', 'select the last run cell']) { 77 | this.registry[cmd] = [NotebookActions.selectLastRunCell]; 78 | } 79 | this.registry['undo'] = [NotebookActions.undo]; 80 | this.registry['redo'] = [NotebookActions.redo]; 81 | for (const cmd of ['copy', 'copy cell', 'copy cells']) { 82 | this.registry[cmd] = [NotebookActions.copy]; 83 | } 84 | for (const cmd of ['cut', 'cut cell', 'cut cells']) { 85 | this.registry[cmd] = [NotebookActions.cut]; 86 | } 87 | for (const cmd of ['paste', 'paste cell', 'paste cells']) { 88 | this.registry[cmd] = [NotebookActions.paste]; 89 | } 90 | for (const cmd of [ 91 | 'to markdown', 92 | 'convert to markdown', 93 | 'markdown', 94 | 'markdown cell', 95 | 'cast to markdown' 96 | ]) { 97 | this.registry[cmd] = [changeCellTypeToMarkdown]; 98 | } 99 | for (const cmd of [ 100 | 'to code', 101 | 'convert to code', 102 | 'code', 103 | 'code cell', 104 | 'cast to code' 105 | ]) { 106 | this.registry[cmd] = [changeCellTypeToCode]; 107 | } 108 | for (const cmd of [ 109 | 'insert code cell below', 110 | 'add code cell below', 111 | 'insert cell below', 112 | 'add cell below', 113 | 'insert cell below' 114 | ]) { 115 | this.registry[cmd] = [NotebookActions.insertBelow]; 116 | } 117 | for (const cmd of [ 118 | 'insert code cell above', 119 | 'add code cell above', 120 | 'insert cell above', 121 | 'add cell above', 122 | 'insert cell above' 123 | ]) { 124 | this.registry[cmd] = [NotebookActions.insertAbove]; 125 | } 126 | for (const cmd of [ 127 | 'insert markdown cell below', 128 | 'add markdown cell below' 129 | ]) { 130 | this.registry[cmd] = [ 131 | NotebookActions.insertBelow, 132 | changeCellTypeToMarkdown 133 | ]; 134 | } 135 | for (const cmd of [ 136 | 'insert markdown cell above', 137 | 'add markdown cell above' 138 | ]) { 139 | this.registry[cmd] = [ 140 | NotebookActions.insertAbove, 141 | changeCellTypeToMarkdown 142 | ]; 143 | } 144 | console.log(this.registry); 145 | } 146 | 147 | private preprocess_cmd(cmd: string): string { 148 | return cmd.replace(/[^\w\s]/gi, '').toLowerCase(); 149 | } 150 | 151 | execute(panel: NotebookPanel, cmd: string): boolean { 152 | cmd = this.preprocess_cmd(cmd); 153 | if (cmd in this.registry) { 154 | const actions = this.registry[cmd]; 155 | for (const action of actions) { 156 | action(panel.content, panel.sessionContext); 157 | } 158 | return true; 159 | } 160 | return false; 161 | } 162 | } 163 | -------------------------------------------------------------------------------- /src/notebook/index.ts: -------------------------------------------------------------------------------- 1 | export * from './cmd-handler'; 2 | export * as notebookUtils from './utils'; 3 | -------------------------------------------------------------------------------- /src/notebook/utils.ts: -------------------------------------------------------------------------------- 1 | import { NotebookActions, NotebookPanel } from '@jupyterlab/notebook'; 2 | 3 | function insert_code_in_cell(panel: NotebookPanel, code: string) { 4 | const cell = panel.content.activeCell; 5 | if (cell) { 6 | cell.model.value.text = code; 7 | } else { 8 | console.error('Could not insert cell because active cell is null'); 9 | } 10 | } 11 | 12 | function insert_code_cell_below(panel: NotebookPanel, code: string) { 13 | NotebookActions.insertBelow(panel.content); 14 | const cell = panel.content.activeCell; 15 | if (cell) { 16 | cell.model.value.text = code; 17 | } else { 18 | console.error('Could not insert cell because active cell is null'); 19 | } 20 | } 21 | 22 | function insert_code_cell_above(panel: NotebookPanel, code: string) { 23 | NotebookActions.insertAbove(panel.content); 24 | const cell = panel.content.activeCell; 25 | if (cell) { 26 | cell.model.value.text = code; 27 | } else { 28 | console.error('Could not insert cell because active cell is null'); 29 | } 30 | } 31 | 32 | function insert_chat_answer_in_cell(panel: NotebookPanel, answer: string) { 33 | NotebookActions.changeCellType(panel.content, 'markdown'); 34 | const cell = panel.content.activeCell; 35 | if (cell) { 36 | if (cell.model && cell.model.value && cell.model.value.text.length === 0) { 37 | if (cell) { 38 | cell.model.value.text = answer; 39 | } else { 40 | console.error('Could not insert cell because active cell is null'); 41 | } 42 | } else { 43 | NotebookActions.insertBelow(panel.content); 44 | NotebookActions.changeCellType(panel.content, 'markdown'); 45 | const cell = panel.content.activeCell; 46 | if (cell) { 47 | cell.model.value.text = answer; 48 | } else { 49 | console.error('Could not insert cell because active cell is null'); 50 | } 51 | } 52 | } 53 | } 54 | 55 | function insert_chat_answer_cell_below(panel: NotebookPanel, answer: string) { 56 | NotebookActions.insertBelow(panel.content); 57 | NotebookActions.changeCellType(panel.content, 'markdown'); 58 | const cell = panel.content.activeCell; 59 | if (cell) { 60 | cell.model.value.text = answer; 61 | } else { 62 | console.error('Could not insert cell because active cell is null'); 63 | } 64 | } 65 | 66 | export { 67 | insert_code_in_cell, 68 | insert_code_cell_below, 69 | insert_code_cell_above, 70 | insert_chat_answer_in_cell, 71 | insert_chat_answer_cell_below 72 | }; 73 | -------------------------------------------------------------------------------- /src/openai/actions/base.ts: -------------------------------------------------------------------------------- 1 | import { OpenAIApi } from 'openai'; 2 | import { showErrorMessage } from '@jupyterlab/apputils'; 3 | 4 | export interface IOpenAIAction { 5 | MODEL_ID: string; 6 | 7 | run(api: OpenAIApi | undefined, input: any): Promise; 8 | } 9 | 10 | export function showError(err: any) { 11 | const msg = err.response.data.error.message; 12 | showErrorMessage('OpenAI Error', msg); 13 | } 14 | -------------------------------------------------------------------------------- /src/openai/actions/chat.ts: -------------------------------------------------------------------------------- 1 | import { OpenAIApi } from 'openai'; 2 | import { IOpenAIAction, showError } from './base'; 3 | 4 | export class ChatAction implements IOpenAIAction { 5 | MODEL_ID = 'gpt-3.5-turbo'; 6 | 7 | async run( 8 | api: OpenAIApi | undefined, 9 | messages: any 10 | ): Promise { 11 | const answer = await api 12 | ?.createChatCompletion({ 13 | model: this.MODEL_ID, 14 | messages: messages 15 | }) 16 | .catch(err => { 17 | showError(err); 18 | return null; 19 | }); 20 | return answer?.data.choices[0].message?.content; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/openai/actions/code.ts: -------------------------------------------------------------------------------- 1 | import { OpenAIApi } from 'openai'; 2 | import { IOpenAIAction, showError } from './base'; 3 | 4 | export class CodeAction implements IOpenAIAction { 5 | MODEL_ID = 'text-davinci-003'; 6 | private maxTokens: number; 7 | 8 | constructor(maxTokens: number) { 9 | this.maxTokens = maxTokens; 10 | } 11 | 12 | private createPrompt(text: string) { 13 | const prefix = 14 | 'Context: Python, format the code output with 4 spaces \n Rules: Return the code only.'; 15 | return `${prefix}${text}`; 16 | } 17 | 18 | async run( 19 | api: OpenAIApi | undefined, 20 | input: any 21 | ): Promise { 22 | const completion = await api 23 | ?.createCompletion({ 24 | model: this.MODEL_ID, 25 | prompt: this.createPrompt(input as string), 26 | max_tokens: this.maxTokens 27 | }) 28 | .catch(err => { 29 | showError(err); 30 | return null; 31 | }); 32 | return completion?.data.choices[0].text; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/openai/actions/index.ts: -------------------------------------------------------------------------------- 1 | export * from './base'; 2 | export * from './chat'; 3 | export * from './code'; 4 | export * from './transcript'; 5 | -------------------------------------------------------------------------------- /src/openai/actions/transcript.ts: -------------------------------------------------------------------------------- 1 | import { OpenAIApi } from 'openai'; 2 | import { IOpenAIAction, showError } from './base'; 3 | 4 | export class TranscriptAction implements IOpenAIAction { 5 | MODEL_ID = 'whisper-1'; 6 | 7 | async run( 8 | api: OpenAIApi | undefined, 9 | input: any 10 | ): Promise { 11 | const audio = new File([input as Blob], 'input.webm', { 12 | type: 'audio/webm' 13 | }); 14 | const transcript = await api 15 | ?.createTranscription(audio, this.MODEL_ID) 16 | .catch(err => { 17 | showError(err); 18 | return null; 19 | }); 20 | return transcript?.data.text; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/openai/index.ts: -------------------------------------------------------------------------------- 1 | import { CodeAction, ChatAction, TranscriptAction } from './actions'; 2 | import { Configuration, OpenAIApi } from 'openai'; 3 | 4 | class CustomFormData extends FormData { 5 | getHeaders() { 6 | return {}; 7 | } 8 | } 9 | 10 | type ChatHistory = Array<{ role: string; content: string }>; 11 | 12 | export default class OpenAIClient { 13 | private api: OpenAIApi | undefined; 14 | private _apiKey = ''; 15 | private _maxTokens = 256; 16 | private _chatHistory: ChatHistory = [ 17 | { role: 'system', content: 'You are a helpful assistant.' } 18 | ]; 19 | private _chatHistoryMaxLength = 10; 20 | 21 | set apiKey(apiKey: string) { 22 | this._apiKey = apiKey; 23 | const configuration = new Configuration({ 24 | apiKey: apiKey, 25 | formDataCtor: CustomFormData 26 | }); 27 | this.api = new OpenAIApi(configuration); 28 | } 29 | 30 | get apiKey() { 31 | return this._apiKey; 32 | } 33 | 34 | set maxTokens(maxTokens: number) { 35 | this._maxTokens = maxTokens; 36 | } 37 | 38 | set chatHistoryMaxLength(chatHistoryMaxLength: number) { 39 | this._chatHistoryMaxLength = chatHistoryMaxLength; 40 | } 41 | 42 | public appendChatMessage(role: string, content: string): void { 43 | this._chatHistory.push({ role: role as string, content: content }); 44 | if (this._chatHistory.length > this._chatHistoryMaxLength) { 45 | this._chatHistory.shift(); 46 | } 47 | } 48 | 49 | async getCode(input: string) { 50 | return new CodeAction(this._maxTokens).run(this.api, input); 51 | } 52 | 53 | async getChat(input: string) { 54 | this.appendChatMessage('user', input); 55 | const answer = await new ChatAction().run(this.api, this._chatHistory); 56 | if (answer) { 57 | this.appendChatMessage('system', answer); 58 | } 59 | return answer; 60 | } 61 | 62 | async getTranscript(input: Blob) { 63 | return new TranscriptAction().run(this.api, input); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/recorder.ts: -------------------------------------------------------------------------------- 1 | interface IDataAvailableEvent extends Event { 2 | data: Blob; 3 | } 4 | 5 | class Recorder { 6 | mediaRecorder: MediaRecorder | null = null; 7 | chunks: Blob[] = []; 8 | 9 | startRecording() { 10 | const constraints = { audio: true }; 11 | navigator.mediaDevices 12 | .getUserMedia(constraints) 13 | .then(stream => { 14 | const config = { mimeType: 'audio/webm' }; 15 | this.mediaRecorder = new MediaRecorder(stream, config); 16 | this.mediaRecorder.addEventListener( 17 | 'dataavailable', 18 | (event: IDataAvailableEvent) => { 19 | this.chunks.push(event.data); 20 | } 21 | ); 22 | this.mediaRecorder.start(); 23 | }) 24 | .catch(error => console.error(error)); 25 | } 26 | 27 | async stopRecording(): Promise { 28 | return new Promise(resolve => { 29 | this.mediaRecorder?.addEventListener('stop', () => { 30 | const blob = new Blob(this.chunks, { type: 'audio/webm' }); 31 | this.chunks = []; 32 | resolve(blob); 33 | }); 34 | this.mediaRecorder?.stop(); 35 | }); 36 | } 37 | } 38 | 39 | export { Recorder }; 40 | -------------------------------------------------------------------------------- /src/svg.d.ts: -------------------------------------------------------------------------------- 1 | declare module '*.svg' { 2 | const value: string; 3 | export default value; 4 | } 5 | -------------------------------------------------------------------------------- /src/voice-processor.ts: -------------------------------------------------------------------------------- 1 | import { showErrorMessage } from '@jupyterlab/apputils'; 2 | import { Recorder } from './recorder'; 3 | import OpenAIClient from './openai'; 4 | import { NotebookCmdHandler, notebookUtils } from './notebook'; 5 | import { NotebookPanel } from '@jupyterlab/notebook'; 6 | 7 | export class VoiceProcessor { 8 | private CHAT_TRIGGER = 'hey'; 9 | private recorder: Recorder = new Recorder(); 10 | private aiClient: OpenAIClient = new OpenAIClient(); 11 | private cmdHandler: NotebookCmdHandler = new NotebookCmdHandler(); 12 | 13 | set apiKey(apiKey: string) { 14 | console.log('Setting API key'); 15 | this.aiClient.apiKey = apiKey; 16 | } 17 | 18 | set maxTokens(maxTokens: number) { 19 | console.log('Setting max tokens'); 20 | this.aiClient.maxTokens = maxTokens; 21 | } 22 | 23 | set chatHistoryMaxLength(chatHistoryMaxLength: number) { 24 | console.log('Setting chat history length'); 25 | this.aiClient.chatHistoryMaxLength = chatHistoryMaxLength; 26 | } 27 | 28 | isConfigured(): boolean { 29 | if (!this.aiClient.apiKey) { 30 | showErrorMessage( 31 | 'Voice Pilot Error', 32 | 'Please set your OpenAI API key in the settings' 33 | ); 34 | return false; 35 | } 36 | return true; 37 | } 38 | 39 | private isChatCmd(transcript: string): boolean { 40 | return transcript! 41 | .toLowerCase() 42 | .replace(/[^\w\s]/gi, '') 43 | .startsWith(this.CHAT_TRIGGER); 44 | } 45 | 46 | start() { 47 | this.recorder.startRecording(); 48 | console.log('Recording started'); 49 | } 50 | 51 | async stop(panel: NotebookPanel) { 52 | const blob = await this.recorder.stopRecording(); 53 | console.log(blob); 54 | const transcript = await this.aiClient.getTranscript(blob); 55 | console.log(transcript); 56 | const executed = this.cmdHandler.execute(panel, transcript!); 57 | if (!executed) { 58 | if (this.isChatCmd(transcript!)) { 59 | console.log('Calling ChatGPT'); 60 | const answer = await this.aiClient.getChat(transcript!); 61 | notebookUtils.insert_chat_answer_in_cell(panel, answer!); 62 | } else { 63 | if (panel.content.activeCell?.model.type === 'code') { 64 | const code = await this.aiClient.getCode(transcript!); 65 | notebookUtils.insert_code_in_cell(panel, code!); 66 | } else { 67 | notebookUtils.insert_code_in_cell(panel, transcript!); 68 | } 69 | } 70 | } else { 71 | console.log('Notebook action has been executed.'); 72 | } 73 | console.log('Recording stopped'); 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /style/base.css: -------------------------------------------------------------------------------- 1 | /* 2 | See the JupyterLab Developer Guide for useful CSS Patterns: 3 | 4 | https://jupyterlab.readthedocs.io/en/stable/developer/css.html 5 | */ 6 | 7 | @keyframes fade { 8 | 0% { 9 | opacity: 1; 10 | } 11 | 12 | 50% { 13 | opacity: 0.5; 14 | } 15 | 16 | 100% { 17 | opacity: 0; 18 | } 19 | } 20 | 21 | .vp-recording path { 22 | fill: #c11010; /* Red */ 23 | animation: fade 1s ease-in-out infinite; 24 | } 25 | 26 | .vp-button svg { 27 | margin-right: 2px; 28 | margin-bottom: 1px; 29 | } 30 | -------------------------------------------------------------------------------- /style/icons/record-vinyl-solid.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /style/index.css: -------------------------------------------------------------------------------- 1 | @import url('base.css'); 2 | -------------------------------------------------------------------------------- /style/index.js: -------------------------------------------------------------------------------- 1 | import './base.css'; 2 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "allowSyntheticDefaultImports": true, 4 | "composite": true, 5 | "declaration": true, 6 | "esModuleInterop": true, 7 | "incremental": true, 8 | "jsx": "react", 9 | "module": "esnext", 10 | "moduleResolution": "node", 11 | "noEmitOnError": true, 12 | "noImplicitAny": true, 13 | "noUnusedLocals": true, 14 | "preserveWatchOutput": true, 15 | "resolveJsonModule": true, 16 | "outDir": "lib", 17 | "rootDir": "src", 18 | "strict": true, 19 | "strictNullChecks": true, 20 | "target": "ES2018", 21 | "types": ["jest"] 22 | }, 23 | "include": ["src"] 24 | } 25 | -------------------------------------------------------------------------------- /ui-tests/README.md: -------------------------------------------------------------------------------- 1 | # Integration Testing 2 | 3 | This folder contains the integration tests of the extension. 4 | 5 | They are defined using [Playwright](https://playwright.dev/docs/intro) test runner 6 | and [Galata](https://github.com/jupyterlab/jupyterlab/tree/master/galata) helper. 7 | 8 | The Playwright configuration is defined in [playwright.config.js](./playwright.config.js). 9 | 10 | The JupyterLab server configuration to use for the integration test is defined 11 | in [jupyter_server_test_config.py](./jupyter_server_test_config.py). 12 | 13 | The default configuration will produce video for failing tests and an HTML report. 14 | 15 | ## Run the tests 16 | 17 | > All commands are assumed to be executed from the root directory 18 | 19 | To run the tests, you need to: 20 | 21 | 1. Compile the extension: 22 | 23 | ```sh 24 | jlpm install 25 | jlpm build:prod 26 | ``` 27 | 28 | > Check the extension is installed in JupyterLab. 29 | 30 | 2. Install test dependencies (needed only once): 31 | 32 | ```sh 33 | cd ./ui-tests 34 | jlpm install 35 | jlpm playwright install 36 | cd .. 37 | ``` 38 | 39 | 3. Execute the [Playwright](https://playwright.dev/docs/intro) tests: 40 | 41 | ```sh 42 | cd ./ui-tests 43 | jlpm playwright test 44 | ``` 45 | 46 | Test results will be shown in the terminal. In case of any test failures, the test report 47 | will be opened in your browser at the end of the tests execution; see 48 | [Playwright documentation](https://playwright.dev/docs/test-reporters#html-reporter) 49 | for configuring that behavior. 50 | 51 | ## Update the tests snapshots 52 | 53 | > All commands are assumed to be executed from the root directory 54 | 55 | If you are comparing snapshots to validate your tests, you may need to update 56 | the reference snapshots stored in the repository. To do that, you need to: 57 | 58 | 1. Compile the extension: 59 | 60 | ```sh 61 | jlpm install 62 | jlpm build:prod 63 | ``` 64 | 65 | > Check the extension is installed in JupyterLab. 66 | 67 | 2. Install test dependencies (needed only once): 68 | 69 | ```sh 70 | cd ./ui-tests 71 | jlpm install 72 | jlpm playwright install 73 | cd .. 74 | ``` 75 | 76 | 3. Execute the [Playwright](https://playwright.dev/docs/intro) command: 77 | 78 | ```sh 79 | cd ./ui-tests 80 | jlpm playwright test -u 81 | ``` 82 | 83 | > Some discrepancy may occurs between the snapshots generated on your computer and 84 | > the one generated on the CI. To ease updating the snapshots on a PR, you can 85 | > type `please update playwright snapshots` to trigger the update by a bot on the CI. 86 | > Once the bot has computed new snapshots, it will commit them to the PR branch. 87 | 88 | ## Create tests 89 | 90 | > All commands are assumed to be executed from the root directory 91 | 92 | To create tests, the easiest way is to use the code generator tool of playwright: 93 | 94 | 1. Compile the extension: 95 | 96 | ```sh 97 | jlpm install 98 | jlpm build:prod 99 | ``` 100 | 101 | > Check the extension is installed in JupyterLab. 102 | 103 | 2. Install test dependencies (needed only once): 104 | 105 | ```sh 106 | cd ./ui-tests 107 | jlpm install 108 | jlpm playwright install 109 | cd .. 110 | ``` 111 | 112 | 3. Execute the [Playwright code generator](https://playwright.dev/docs/codegen): 113 | 114 | ```sh 115 | cd ./ui-tests 116 | jlpm playwright codegen localhost:8888 117 | ``` 118 | 119 | ## Debug tests 120 | 121 | > All commands are assumed to be executed from the root directory 122 | 123 | To debug tests, a good way is to use the inspector tool of playwright: 124 | 125 | 1. Compile the extension: 126 | 127 | ```sh 128 | jlpm install 129 | jlpm build:prod 130 | ``` 131 | 132 | > Check the extension is installed in JupyterLab. 133 | 134 | 2. Install test dependencies (needed only once): 135 | 136 | ```sh 137 | cd ./ui-tests 138 | jlpm install 139 | jlpm playwright install 140 | cd .. 141 | ``` 142 | 143 | 3. Execute the Playwright tests in [debug mode](https://playwright.dev/docs/debug): 144 | 145 | ```sh 146 | cd ./ui-tests 147 | PWDEBUG=1 jlpm playwright test 148 | ``` 149 | -------------------------------------------------------------------------------- /ui-tests/jupyter_server_test_config.py: -------------------------------------------------------------------------------- 1 | """Server configuration for integration tests. 2 | 3 | !! Never use this configuration in production because it 4 | opens the server to the world and provide access to JupyterLab 5 | JavaScript objects through the global window variable. 6 | """ 7 | from tempfile import mkdtemp 8 | 9 | c.ServerApp.port = 8888 10 | c.ServerApp.port_retries = 0 11 | c.ServerApp.open_browser = False 12 | 13 | c.ServerApp.root_dir = mkdtemp(prefix='galata-test-') 14 | c.ServerApp.token = "" 15 | c.ServerApp.password = "" 16 | c.ServerApp.disable_check_xsrf = True 17 | c.LabApp.expose_app_in_browser = True 18 | 19 | # Uncomment to set server log level to debug level 20 | # c.ServerApp.log_level = "DEBUG" 21 | -------------------------------------------------------------------------------- /ui-tests/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "voicepilot-ui-tests", 3 | "version": "1.0.0", 4 | "description": "JupyterLab voicepilot Integration Tests", 5 | "private": true, 6 | "scripts": { 7 | "start": "jupyter lab --config jupyter_server_test_config.py", 8 | "test": "jlpm playwright test", 9 | "test:update": "jlpm playwright test --update-snapshots" 10 | }, 11 | "devDependencies": { 12 | "@jupyterlab/galata": "^4.3.0" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /ui-tests/playwright.config.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Configuration for Playwright using default from @jupyterlab/galata 3 | */ 4 | const baseConfig = require('@jupyterlab/galata/lib/playwright-config'); 5 | 6 | module.exports = { 7 | ...baseConfig, 8 | webServer: { 9 | command: 'jlpm start', 10 | url: 'http://localhost:8888/lab', 11 | timeout: 120 * 1000, 12 | reuseExistingServer: !process.env.CI 13 | } 14 | }; 15 | -------------------------------------------------------------------------------- /ui-tests/tests/jupyter_voicepilot.spec.ts: -------------------------------------------------------------------------------- 1 | import { expect, test } from '@jupyterlab/galata'; 2 | 3 | /** 4 | * Don't load JupyterLab webpage before running the tests. 5 | * This is required to ensure we capture all log messages. 6 | */ 7 | test.use({ autoGoto: false }); 8 | 9 | test('should emit an activation console message', async ({ page }) => { 10 | const logs: string[] = []; 11 | 12 | page.on('console', message => { 13 | logs.push(message.text()); 14 | }); 15 | 16 | await page.goto(); 17 | 18 | expect( 19 | logs.filter(s => s === 'JupyterLab extension voicepilot is activated!') 20 | ).toHaveLength(1); 21 | }); 22 | --------------------------------------------------------------------------------