├── .copier-answers.yml ├── .github └── workflows │ ├── build.yml │ ├── check-release.yml │ ├── enforce-label.yml │ ├── prep-release.yml │ ├── publish-release.yml │ └── update-integration-tests.yml ├── .gitignore ├── .prettierignore ├── .readthedocs.yaml ├── .yarnrc.yml ├── CHANGELOG.md ├── LICENSE ├── README.md ├── RELEASE.md ├── docs ├── _static │ └── css │ │ └── custom.css ├── advanced.md ├── build-environment.yml ├── changelog.md ├── conf.py ├── deploy.md ├── environment-cpp.yml ├── environment-other.yml ├── environment-python.yml ├── environment-r.yml ├── environment.md ├── files.md ├── index.md ├── jupyter_lite_config.json ├── jupyterlite.svg └── xeus.svg ├── environment-dev.yaml ├── example_envs ├── env_with_extension.yaml ├── env_with_many_kernels.yaml └── env_with_pip.yaml ├── install.json ├── jupyterlite_xeus ├── __init__.py ├── _pip.py ├── add_on.py ├── constants.py └── create_conda_env.py ├── lerna.json ├── package.json ├── packages ├── xeus-core │ ├── package.json │ ├── src │ │ ├── index.ts │ │ ├── interfaces.ts │ │ ├── web.worker.kernel.base.ts │ │ └── worker.base.ts │ └── tsconfig.json ├── xeus-extension │ ├── lab.webpack.config.js │ ├── package.json │ ├── schema │ │ └── xeus-kernel-status.json │ ├── src │ │ ├── index.ts │ │ └── tokens.ts │ ├── style │ │ ├── index.css │ │ └── index.js │ └── tsconfig.json └── xeus │ ├── package.json │ ├── src │ ├── coincident.worker.ts │ ├── comlink.worker.ts │ ├── index.ts │ ├── interfaces.ts │ ├── web.worker.kernel.ts │ └── worker.ts │ ├── tsconfig.json │ └── worker.webpack.config.js ├── pyproject.toml ├── scripts └── bump_version.py ├── setup.py ├── tests ├── environment-1.yml ├── environment-2.yml ├── environment-3.yml ├── test_package │ ├── environment-3.yml │ ├── pyproject.toml │ └── test_package │ │ ├── __init__.py │ │ └── hey.py └── test_xeus.py ├── tsconfig.json ├── ui-tests ├── README.md ├── build.py ├── env1.yml ├── env2.yml ├── jupyter-lite.json ├── jupyter_lite_config.json ├── package.json ├── playwright.config.js ├── tests │ ├── jupyterlite_xeus.spec.ts │ └── jupyterlite_xeus.spec.ts-snapshots │ │ ├── jupyter-xeus-execute-crossoriginisolated-linux.png │ │ ├── jupyter-xeus-execute-default-linux.png │ │ ├── jupyter-xeus-execute-env2-crossoriginisolated-linux.png │ │ ├── jupyter-xeus-execute-env2-default-linux.png │ │ ├── jupyter-xeus-launcher-crossoriginisolated-linux.png │ │ └── jupyter-xeus-launcher-default-linux.png └── yarn.lock └── yarn.lock /.copier-answers.yml: -------------------------------------------------------------------------------- 1 | # Changes here will be overwritten by Copier; NEVER EDIT MANUALLY 2 | _commit: v4.2.4 3 | _src_path: https://github.com/jupyterlab/extension-template 4 | author_email: '' 5 | author_name: JupyterLite Contributors 6 | data_format: string 7 | file_extension: '' 8 | has_binder: false 9 | has_settings: false 10 | kind: frontend 11 | labextension_name: '@jupyterlite/xeus' 12 | mimetype: '' 13 | mimetype_name: '' 14 | project_short_description: JupyterLite loader for Xeus kernels 15 | python_name: jupyterlite_xeus 16 | repository: https://github.com/jupyterlite/xeus 17 | test: true 18 | viewer_name: '' 19 | 20 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: 4 | push: 5 | branches: main 6 | pull_request: 7 | branches: '*' 8 | 9 | defaults: 10 | run: 11 | shell: bash -l {0} 12 | 13 | jobs: 14 | build: 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - name: Checkout 19 | uses: actions/checkout@v4 20 | 21 | - name: Base Setup 22 | uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 23 | 24 | - name: Install Conda environment with Micromamba 25 | uses: mamba-org/setup-micromamba@v2 26 | with: 27 | environment-name: xeus-lite-dev 28 | environment-file: environment-dev.yaml 29 | 30 | - name: Lint JS 31 | run: | 32 | set -eux 33 | jlpm 34 | jlpm run lint:check 35 | 36 | - name: Lint Python 37 | run: ruff check --output-format=github jupyterlite_xeus 38 | 39 | - name: Build the extension 40 | run: | 41 | set -eux 42 | python -m pip install .[test] 43 | 44 | jupyter labextension list 45 | jupyter labextension list 2>&1 | grep -ie "@jupyterlite/xeus.*OK" 46 | # TODO: re-enable? 47 | # python -m jupyterlab.browser_check 48 | 49 | - name: Package the extension 50 | run: | 51 | set -eux 52 | 53 | pip install build 54 | python -m build 55 | pip uninstall -y "jupyterlite_xeus" jupyterlab 56 | 57 | - name: Upload extension packages 58 | uses: actions/upload-artifact@v4 59 | with: 60 | name: extension-artifacts 61 | path: dist/jupyterlite_xeus* 62 | if-no-files-found: error 63 | 64 | test_isolated: 65 | needs: build 66 | runs-on: ubuntu-latest 67 | 68 | steps: 69 | - name: Install Python 70 | uses: actions/setup-python@v4 71 | with: 72 | python-version: '3.13' 73 | architecture: 'x64' 74 | - uses: actions/download-artifact@v4 75 | with: 76 | name: extension-artifacts 77 | - name: Install and Test 78 | run: | 79 | set -eux 80 | # Remove NodeJS, twice to take care of system and locally installed node versions. 81 | sudo rm -rf $(which node) 82 | sudo rm -rf $(which node) 83 | 84 | pip install "jupyterlab>=4.4.0.b0,<5" jupyterlite_xeus*.whl 85 | 86 | jupyter labextension list 87 | jupyter labextension list 2>&1 | grep -ie "@jupyterlite/xeus.*OK" 88 | python -m jupyterlab.browser_check --no-browser-test 89 | 90 | python-tests: 91 | needs: build 92 | runs-on: ubuntu-latest 93 | 94 | steps: 95 | - name: Checkout 96 | uses: actions/checkout@v4 97 | 98 | - uses: actions/download-artifact@v4 99 | with: 100 | name: extension-artifacts 101 | 102 | - name: Install mamba 103 | uses: mamba-org/setup-micromamba@v2 104 | with: 105 | environment-file: environment-dev.yaml 106 | environment-name: xeus-lite-dev 107 | 108 | - name: Install 109 | run: pip install jupyterlite_xeus*.whl 110 | 111 | - name: Run tests 112 | run: pytest -rP test_xeus.py 113 | working-directory: tests 114 | 115 | integration-tests: 116 | name: Integration tests 117 | needs: build 118 | runs-on: ubuntu-latest 119 | strategy: 120 | fail-fast: false 121 | matrix: 122 | # try the latest stable and potential pre-releases 123 | jupyterlite_version: ["jupyterlite-core -U", "jupyterlite-core -U --pre"] 124 | project: ["default", "crossoriginisolated"] 125 | # the latest stable release is not compatible for now 126 | exclude: 127 | - jupyterlite_version: "jupyterlite-core -U" 128 | project: "crossoriginisolated" 129 | 130 | steps: 131 | - name: Checkout 132 | uses: actions/checkout@v4 133 | 134 | - name: Base Setup 135 | uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 136 | 137 | - name: Download extension package 138 | uses: actions/download-artifact@v4 139 | with: 140 | name: extension-artifacts 141 | 142 | - name: Install Conda environment with Micromamba 143 | uses: mamba-org/setup-micromamba@v2 144 | with: 145 | environment-name: test-env 146 | create-args: >- 147 | pip 148 | 149 | - name: Install the extension 150 | run: | 151 | set -eux 152 | python -m pip install "jupyterlab>=4.4.0.b0,<5" jupyterlite_xeus*.whl 153 | python -m pip install ${{ matrix.jupyterlite_version }} 154 | 155 | - name: Install dependencies 156 | working-directory: ui-tests 157 | env: 158 | YARN_ENABLE_IMMUTABLE_INSTALLS: 0 159 | PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 160 | run: | 161 | jlpm install 162 | jlpm run build 163 | 164 | - name: Install browser 165 | run: jlpm playwright install chromium 166 | working-directory: ui-tests 167 | 168 | - name: Execute integration tests 169 | working-directory: ui-tests 170 | run: | 171 | jlpm playwright test --project ${{ matrix.project }} 172 | 173 | - name: Upload Playwright Test report 174 | if: always() 175 | uses: actions/upload-artifact@v4 176 | with: 177 | name: jupyterlite-xeus-playwright-tests (${{ matrix.jupyterlite_version }}, ${{ matrix.project }}) 178 | path: | 179 | ui-tests/test-results 180 | ui-tests/playwright-report 181 | 182 | check_links: 183 | name: Check Links 184 | runs-on: ubuntu-latest 185 | timeout-minutes: 15 186 | steps: 187 | - uses: actions/checkout@v4 188 | - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 189 | - uses: jupyterlab/maintainer-tools/.github/actions/check-links@v1 190 | -------------------------------------------------------------------------------- /.github/workflows/check-release.yml: -------------------------------------------------------------------------------- 1 | name: Check Release 2 | on: 3 | push: 4 | branches: ["main"] 5 | pull_request: 6 | branches: ["*"] 7 | 8 | jobs: 9 | check_release: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout 13 | uses: actions/checkout@v4 14 | - name: Base Setup 15 | uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 16 | - name: Check Release 17 | uses: jupyter-server/jupyter_releaser/.github/actions/check-release@v2 18 | with: 19 | 20 | token: ${{ secrets.GITHUB_TOKEN }} 21 | 22 | - name: Upload Distributions 23 | uses: actions/upload-artifact@v4 24 | with: 25 | name: jupyterlite_xeus-releaser-dist-${{ github.run_number }} 26 | path: .jupyter_releaser_checkout/dist 27 | -------------------------------------------------------------------------------- /.github/workflows/enforce-label.yml: -------------------------------------------------------------------------------- 1 | name: Enforce PR label 2 | 3 | on: 4 | pull_request: 5 | types: [labeled, unlabeled, opened, edited, synchronize] 6 | jobs: 7 | enforce-label: 8 | runs-on: ubuntu-latest 9 | permissions: 10 | pull-requests: write 11 | steps: 12 | - name: enforce-triage-label 13 | uses: jupyterlab/maintainer-tools/.github/actions/enforce-label@v1 14 | -------------------------------------------------------------------------------- /.github/workflows/prep-release.yml: -------------------------------------------------------------------------------- 1 | name: "Step 1: Prep Release" 2 | on: 3 | workflow_dispatch: 4 | inputs: 5 | version_spec: 6 | description: "New Version Specifier" 7 | default: "next" 8 | required: false 9 | branch: 10 | description: "The branch to target" 11 | required: false 12 | post_version_spec: 13 | description: "Post Version Specifier" 14 | required: false 15 | # silent: 16 | # description: "Set a placeholder in the changelog and don't publish the release." 17 | # required: false 18 | # type: boolean 19 | since: 20 | description: "Use PRs with activity since this date or git reference" 21 | required: false 22 | since_last_stable: 23 | description: "Use PRs with activity since the last stable git tag" 24 | required: false 25 | type: boolean 26 | jobs: 27 | prep_release: 28 | runs-on: ubuntu-latest 29 | permissions: 30 | contents: write 31 | steps: 32 | - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 33 | 34 | - name: Prep Release 35 | id: prep-release 36 | uses: jupyter-server/jupyter_releaser/.github/actions/prep-release@v2 37 | with: 38 | token: ${{ secrets.GITHUB_TOKEN }} 39 | version_spec: ${{ github.event.inputs.version_spec }} 40 | # silent: ${{ github.event.inputs.silent }} 41 | post_version_spec: ${{ github.event.inputs.post_version_spec }} 42 | branch: ${{ github.event.inputs.branch }} 43 | since: ${{ github.event.inputs.since }} 44 | since_last_stable: ${{ github.event.inputs.since_last_stable }} 45 | 46 | - name: "** Next Step **" 47 | run: | 48 | echo "Optional): Review Draft Release: ${{ steps.prep-release.outputs.release_url }}" 49 | -------------------------------------------------------------------------------- /.github/workflows/publish-release.yml: -------------------------------------------------------------------------------- 1 | name: "Step 2: Publish Release" 2 | on: 3 | workflow_dispatch: 4 | inputs: 5 | branch: 6 | description: "The target branch" 7 | required: false 8 | release_url: 9 | description: "The URL of the draft GitHub release" 10 | required: false 11 | steps_to_skip: 12 | description: "Comma separated list of steps to skip" 13 | required: false 14 | 15 | jobs: 16 | publish_release: 17 | runs-on: ubuntu-latest 18 | environment: release 19 | permissions: 20 | id-token: write 21 | steps: 22 | - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 23 | 24 | - uses: actions/create-github-app-token@v1 25 | id: app-token 26 | with: 27 | app-id: ${{ vars.APP_ID }} 28 | private-key: ${{ secrets.APP_PRIVATE_KEY }} 29 | 30 | - name: Populate Release 31 | id: populate-release 32 | uses: jupyter-server/jupyter_releaser/.github/actions/populate-release@v2 33 | with: 34 | token: ${{ steps.app-token.outputs.token }} 35 | branch: ${{ github.event.inputs.branch }} 36 | release_url: ${{ github.event.inputs.release_url }} 37 | steps_to_skip: ${{ github.event.inputs.steps_to_skip }} 38 | 39 | - name: Finalize Release 40 | id: finalize-release 41 | env: 42 | NPM_TOKEN: ${{ secrets.NPM_TOKEN }} 43 | uses: jupyter-server/jupyter_releaser/.github/actions/finalize-release@v2 44 | with: 45 | token: ${{ steps.app-token.outputs.token }} 46 | release_url: ${{ steps.populate-release.outputs.release_url }} 47 | 48 | - name: "** Next Step **" 49 | if: ${{ success() }} 50 | run: | 51 | echo "Verify the final release" 52 | echo ${{ steps.finalize-release.outputs.release_url }} 53 | 54 | - name: "** Failure Message **" 55 | if: ${{ failure() }} 56 | run: | 57 | echo "Failed to Publish the Draft Release Url:" 58 | echo ${{ steps.populate-release.outputs.release_url }} 59 | -------------------------------------------------------------------------------- /.github/workflows/update-integration-tests.yml: -------------------------------------------------------------------------------- 1 | name: Update Playwright Snapshots 2 | 3 | on: 4 | issue_comment: 5 | types: [created, edited] 6 | 7 | permissions: 8 | contents: write 9 | pull-requests: write 10 | 11 | jobs: 12 | update-snapshots: 13 | if: > 14 | ( 15 | github.event.comment.author_association == 'OWNER' || 16 | github.event.comment.author_association == 'COLLABORATOR' || 17 | github.event.comment.author_association == 'MEMBER' 18 | ) && github.event.issue.pull_request && contains(github.event.comment.body, 'please update snapshots') 19 | runs-on: ubuntu-latest 20 | 21 | steps: 22 | - name: React to the triggering comment 23 | run: | 24 | gh api repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }}/reactions --raw-field 'content=+1' 25 | env: 26 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 27 | 28 | - name: Checkout 29 | uses: actions/checkout@v4 30 | with: 31 | token: ${{ secrets.GITHUB_TOKEN }} 32 | 33 | - name: Checkout the branch from the PR that triggered the job 34 | run: gh pr checkout ${{ github.event.issue.number }} 35 | env: 36 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 37 | 38 | - name: Base Setup 39 | uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 40 | 41 | - name: Install micromamba 42 | uses: mamba-org/setup-micromamba@v2 43 | with: 44 | environment-name: xeus-lite-dev 45 | micromamba-version: '2.0.5-0' 46 | 47 | - name: Install dependencies 48 | run: python -m pip install -U "jupyterlab>=4.4,<5" jupyterlite-core 49 | 50 | - name: Install extension 51 | run: | 52 | set -eux 53 | jlpm 54 | python -m pip install . 55 | 56 | - name: Install ui-tests 57 | run: | 58 | jlpm 59 | jlpm run build 60 | working-directory: ui-tests 61 | 62 | - uses: jupyterlab/maintainer-tools/.github/actions/update-snapshots@v1 63 | with: 64 | github_token: ${{ secrets.GITHUB_TOKEN }} 65 | # Playwright knows how to start JupyterLab server 66 | start_server_script: 'null' 67 | test_folder: ui-tests 68 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.bundle.* 2 | lib/ 3 | node_modules/ 4 | *.log 5 | .eslintcache 6 | *.egg-info/ 7 | .ipynb_checkpoints 8 | *.tsbuildinfo 9 | jupyterlite_xeus/labextension 10 | # Version file is handled by hatchling 11 | jupyterlite_xeus/_version.py 12 | 13 | # Integration tests 14 | ui-tests/test-results/ 15 | ui-tests/playwright-report/ 16 | 17 | # Created by https://www.gitignore.io/api/python 18 | # Edit at https://www.gitignore.io/?templates=python 19 | 20 | ### Python ### 21 | # Byte-compiled / optimized / DLL files 22 | __pycache__/ 23 | *.py[cod] 24 | *$py.class 25 | 26 | # C extensions 27 | *.so 28 | 29 | # Distribution / packaging 30 | .Python 31 | build/ 32 | develop-eggs/ 33 | dist/ 34 | downloads/ 35 | eggs/ 36 | .eggs/ 37 | lib/ 38 | lib64/ 39 | parts/ 40 | sdist/ 41 | var/ 42 | wheels/ 43 | pip-wheel-metadata/ 44 | share/python-wheels/ 45 | .installed.cfg 46 | *.egg 47 | MANIFEST 48 | 49 | # PyInstaller 50 | # Usually these files are written by a python script from a template 51 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 52 | *.manifest 53 | *.spec 54 | 55 | # Installer logs 56 | pip-log.txt 57 | pip-delete-this-directory.txt 58 | 59 | # Unit test / coverage reports 60 | htmlcov/ 61 | .tox/ 62 | .nox/ 63 | .coverage 64 | .coverage.* 65 | .cache 66 | nosetests.xml 67 | coverage/ 68 | coverage.xml 69 | *.cover 70 | .hypothesis/ 71 | .pytest_cache/ 72 | 73 | # Translations 74 | *.mo 75 | *.pot 76 | 77 | # Scrapy stuff: 78 | .scrapy 79 | 80 | # Sphinx documentation 81 | docs/_build/ 82 | 83 | # PyBuilder 84 | target/ 85 | 86 | # pyenv 87 | .python-version 88 | 89 | # celery beat schedule file 90 | celerybeat-schedule 91 | 92 | # SageMath parsed files 93 | *.sage.py 94 | 95 | # Spyder project settings 96 | .spyderproject 97 | .spyproject 98 | 99 | # Rope project settings 100 | .ropeproject 101 | 102 | # Mr Developer 103 | .mr.developer.cfg 104 | .project 105 | .pydevproject 106 | 107 | # mkdocs documentation 108 | /site 109 | 110 | # mypy 111 | .mypy_cache/ 112 | .dmypy.json 113 | dmypy.json 114 | 115 | # Pyre type checker 116 | .pyre/ 117 | 118 | # End of https://www.gitignore.io/api/python 119 | 120 | # OSX files 121 | .DS_Store 122 | 123 | # Yarn cache 124 | .yarn/ 125 | 126 | # experiments 127 | experiment.sh 128 | env.yml 129 | 130 | # JupyterLite 131 | .jupyterlite.doit.db 132 | _output 133 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | **/node_modules 3 | **/lib 4 | **/package.json 5 | !/package.json 6 | jupyterlite_xeus 7 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | build: 4 | os: "ubuntu-20.04" 5 | tools: 6 | python: "mambaforge-4.10" 7 | 8 | conda: 9 | environment: docs/build-environment.yml 10 | 11 | sphinx: 12 | configuration: docs/conf.py 13 | -------------------------------------------------------------------------------- /.yarnrc.yml: -------------------------------------------------------------------------------- 1 | nodeLinker: node-modules 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2023, JupyterLite Contributors 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | 3. Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # JupyterLite Xeus 2 | 3 | [![Github Actions Status](https://github.com/jupyterlite/xeus/workflows/Build/badge.svg)](https://github.com/jupyterlite/xeus/actions/workflows/build.yml) 4 | 5 | `jupyterlite-xeus` is an extension for JupyterLite that enables fully client-side Jupyter environments powered by xeus kernels compiled to WebAssembly (Wasm). It allows users to create statically-served Jupyter deployments with custom pre-built environments — no server required. 6 | 7 | The core feature of `jupyterlite-xeus` is its integration with [emscripten-forge](https://github.com/emscripten-forge), a conda package distribution tailored for WebAssembly. This makes it possible to bundle your favorite scientific or data analysis packages directly into the browser-based environment, delivering a reproducible computing experience with zero backend dependencies. 8 | 9 | Ideal for demos, educational resources, and offline computing. Use it in combination with [Voici](https://github.com/voila-dashboards/voici)! 10 | 11 | Currently supported kernels are: 12 | 13 | - [xeus-python](https://github.com/jupyter-xeus/xeus-python) 14 | - [xeus-lua](https://github.com/jupyter-xeus/xeus-lua) 15 | - [xeus-r](https://github.com/jupyter-xeus/xeus-r) 16 | - [xeus-cpp](https://github.com/compiler-research/xeus-cpp) 17 | - [xeus-nelson](https://github.com/jupyter-xeus/xeus-nelson) 18 | - [xeus-javascript](https://github.com/jupyter-xeus/xeus-javascript) 19 | 20 | ## Requirements 21 | 22 | - JupyterLab >= 4.0.0 23 | 24 | ## Installation 25 | 26 | You can install `jupyterlite-xeus` with conda/mamba 27 | 28 | ``` 29 | mamba install -c conda-forge jupyterlite-xeus 30 | ``` 31 | 32 | Or with `pip` (you must install micromamba 2.0.5): 33 | 34 | ``` 35 | pip install jupyterlite-xeus 36 | ``` 37 | 38 | ## Documentation 39 | 40 | Learn more about `jupyterlite-xeus` and test our live demo on https://jupyterlite-xeus.readthedocs.io 41 | 42 | ## Contributing 43 | 44 | ### Development install from a conda / mamba environment 45 | 46 | Create the conda environment with `conda`/`mamba`/`micromamba` (replace `micromamba` with `conda` or `mamba` according to your preference): 47 | 48 | ```bash 49 | micromamba create -f environment-dev.yml -n xeus-lite-dev 50 | ``` 51 | 52 | Activate the environment: 53 | 54 | ```bash 55 | micromamba activate xeus-lite-dev 56 | ``` 57 | 58 | ```bash 59 | python -m pip install -e . -v --no-build-isolation 60 | ``` 61 | 62 | ### Packaging the extension 63 | 64 | See [RELEASE](RELEASE.md). 65 | -------------------------------------------------------------------------------- /RELEASE.md: -------------------------------------------------------------------------------- 1 | # Making a new release of jupyterlite_xeus 2 | 3 | The extension can be published to `PyPI` and `npm` manually or using the [Jupyter Releaser](https://github.com/jupyter-server/jupyter_releaser). 4 | 5 | ## Manual release 6 | 7 | ### Python package 8 | 9 | This extension can be distributed as Python packages. All of the Python 10 | packaging instructions are in the `pyproject.toml` file to wrap your extension in a 11 | Python package. Before generating a package, you first need to install some tools: 12 | 13 | ```bash 14 | pip install build twine hatch 15 | ``` 16 | 17 | Bump the version using `hatch`. By default this will create a tag. 18 | See the docs on [hatch-nodejs-version](https://github.com/agoose77/hatch-nodejs-version#semver) for details. 19 | 20 | ```bash 21 | hatch version 22 | ``` 23 | 24 | Make sure to clean up all the development files before building the package: 25 | 26 | ```bash 27 | jlpm clean:all 28 | ``` 29 | 30 | You could also clean up the local git repository: 31 | 32 | ```bash 33 | git clean -dfX 34 | ``` 35 | 36 | To create a Python source package (`.tar.gz`) and the binary package (`.whl`) in the `dist/` directory, do: 37 | 38 | ```bash 39 | python -m build 40 | ``` 41 | 42 | > `python setup.py sdist bdist_wheel` is deprecated and will not work for this package. 43 | 44 | Then to upload the package to PyPI, do: 45 | 46 | ```bash 47 | twine upload dist/* 48 | ``` 49 | 50 | ### NPM package 51 | 52 | To publish the frontend part of the extension as a NPM package, do: 53 | 54 | ```bash 55 | npm login 56 | npm publish --access public 57 | ``` 58 | 59 | ## Automated releases with the Jupyter Releaser 60 | 61 | The extension repository should already be compatible with the Jupyter Releaser. 62 | 63 | Check out the [workflow documentation](https://jupyter-releaser.readthedocs.io/en/latest/get_started/making_release_from_repo.html) for more information. 64 | 65 | Here is a summary of the steps to cut a new release: 66 | 67 | - Add tokens to the [Github Secrets](https://docs.github.com/en/actions/security-guides/encrypted-secrets) in the repository: 68 | - `ADMIN_GITHUB_TOKEN` (with "public_repo" and "repo:status" permissions); see the [documentation](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) 69 | - `NPM_TOKEN` (with "automation" permission); see the [documentation](https://docs.npmjs.com/creating-and-viewing-access-tokens) 70 | - Set up PyPI 71 | 72 |
Using PyPI trusted publisher (modern way) 73 | 74 | - Set up your PyPI project by [adding a trusted publisher](https://docs.pypi.org/trusted-publishers/adding-a-publisher/) 75 | - The _workflow name_ is `publish-release.yml` and the _environment_ should be left blank. 76 | - Ensure the publish release job as `permissions`: `id-token : write` (see the [documentation](https://docs.pypi.org/trusted-publishers/using-a-publisher/)) 77 | 78 |
79 | 80 |
Using PyPI token (legacy way) 81 | 82 | - If the repo generates PyPI release(s), create a scoped PyPI [token](https://packaging.python.org/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/#saving-credentials-on-github). We recommend using a scoped token for security reasons. 83 | 84 | - You can store the token as `PYPI_TOKEN` in your fork's `Secrets`. 85 | 86 | - Advanced usage: if you are releasing multiple repos, you can create a secret named `PYPI_TOKEN_MAP` instead of `PYPI_TOKEN` that is formatted as follows: 87 | 88 | ```text 89 | owner1/repo1,token1 90 | owner2/repo2,token2 91 | ``` 92 | 93 | If you have multiple Python packages in the same repository, you can point to them as follows: 94 | 95 | ```text 96 | owner1/repo1/path/to/package1,token1 97 | owner1/repo1/path/to/package2,token2 98 | ``` 99 | 100 |
101 | 102 | - Go to the Actions panel 103 | - Run the "Step 1: Prep Release" workflow 104 | - Check the draft changelog 105 | - Run the "Step 2: Publish Release" workflow 106 | 107 | ## Publishing to `conda-forge` 108 | 109 | If the package is not on conda forge yet, check the documentation to learn how to add it: https://conda-forge.org/docs/maintainer/adding_pkgs.html 110 | 111 | Otherwise a bot should pick up the new version publish to PyPI, and open a new PR on the feedstock repository automatically. 112 | -------------------------------------------------------------------------------- /docs/_static/css/custom.css: -------------------------------------------------------------------------------- 1 | @media (min-width: 960px) .bd-page-width { 2 | max-width: 92rem; 3 | } 4 | -------------------------------------------------------------------------------- /docs/advanced.md: -------------------------------------------------------------------------------- 1 | (advanced)= 2 | 3 | ## Advanced Configuration 4 | 5 | ```{warning} 6 | This section is mostly for reference and should not be needed for regular use of the `jupyterlite-xeus` kernel. 7 | ``` 8 | 9 | ### Provide a custom `empack_config.yaml` 10 | 11 | Packages sometimes ship more data than needed for the package to work (tests, documentation, data files etc). This is fine on a regular installation of the package, but in the emscripten case when running in the browser this means that starting the kernel would download more files. 12 | For this reason, `empack` allows filtering files that is are not required for the Python code to run. It does it by following a set of filtering rules available in this file: https://github.com/emscripten-forge/empack/blob/main/config/empack_config.yaml. 13 | 14 | The xeus-python kernel supports passing a custom `empack_config.yaml`. This file can be used to override the default filter rules set by the underlying `empack` tool used for packing the environment. 15 | 16 | If you would like to provide additional rules for excluding files in the packed environment, create a `empack_config.yaml` with the following content as an example: 17 | 18 | ```yaml 19 | packages: 20 | xarray: 21 | exclude_patterns: 22 | - pattern: '**/static/css/*.css' 23 | - pattern: '**/static/html/*.html' 24 | ``` 25 | 26 | This example defines a set of custom rules for the `xarray` package to make sure it excludes some static files that are not required for the code to run. 27 | 28 | You can use this file when building JupyterLite: 29 | 30 | ```shell 31 | jupyter lite build --XeusAddon.empack_config=empack_config.yaml 32 | ``` 33 | 34 | ```{note} 35 | Filtering files helps reduce the size of the assets to download and as a consequence reduce network traffic. 36 | ``` 37 | 38 | ### Build your xeus-kernel locally 39 | 40 | #### Create a local environment / prefix 41 | 42 | This workflow usually starts with creating a local conda environment / prefix for the `emscripten-wasm32` platform with all the dependencies required to build your kernel (here we install dependencies for `xeus-python`). 43 | 44 | ```bash 45 | micromamba create -n xeus-python-dev \ 46 | --platform=emscripten-wasm32 \ 47 | -c https://repo.prefix.dev/emscripten-forge-dev \ 48 | -c https://repo.prefix.dev/conda-forge \ 49 | --yes \ 50 | "python>=3.11" pybind11 nlohmann_json pybind11_json numpy pytest \ 51 | bzip2 sqlite zlib libffi xtl pyjs \ 52 | xeus xeus-lite 53 | ``` 54 | 55 | #### Build the kernel 56 | 57 | This depends on your kernel, but it will look something like this: 58 | 59 | ```bash 60 | # path to your emscripten emsdk 61 | source $EMSDK_DIR/emsdk_env.sh 62 | 63 | WASM_ENV_NAME=xeus-python-dev 64 | WASM_ENV_PREFIX=$MAMBA_ROOT_PREFIX/envs/$WASM_ENV_NAME 65 | 66 | # let cmake know where the env is 67 | export PREFIX=$WASM_ENV_PREFIX 68 | export CMAKE_PREFIX_PATH=$PREFIX 69 | export CMAKE_SYSTEM_PREFIX_PATH=$PREFIX 70 | 71 | cd /path/to/your/kernel/src 72 | mkdir build_wasm 73 | cd build_wasm 74 | emcmake cmake \ 75 | -DCMAKE_BUILD_TYPE=Release \ 76 | -DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=ON \ 77 | -DCMAKE_INSTALL_PREFIX=$PREFIX \ 78 | .. 79 | emmake make -j8 install 80 | ``` 81 | 82 | #### Build the JupyterLite site 83 | 84 | You will need to create a new environment with the dependencies to build the JupyterLite site. 85 | 86 | ```bash 87 | # create new environment 88 | micromamba create -n xeus-lite-host \ 89 | jupyterlite-core 90 | 91 | # activate the environment 92 | micromamba activate xeus-lite-host 93 | 94 | # install jupyterlite_xeus via pip 95 | python -m pip install jupyterlite-xeus 96 | ``` 97 | 98 | When running `jupyter lite build`, we pass the `prefix` option and point it to the local environment / prefix we just created: 99 | 100 | ```bash 101 | jupyter lite build --XeusAddon.prefix=$WASM_ENV_PREFIX 102 | ``` 103 | -------------------------------------------------------------------------------- /docs/build-environment.yml: -------------------------------------------------------------------------------- 1 | name: xeus-python-kernel-docs 2 | 3 | channels: 4 | - conda-forge 5 | - conda-forge/label/jupyterlite_core_alpha 6 | 7 | dependencies: 8 | - micromamba=2.0.5 9 | - pip 10 | - nodejs=20 11 | - click 12 | - typer 13 | - linkify-it-py 14 | - myst-parser 15 | - sphinx-design 16 | - pydata-sphinx-theme 17 | - jupyterlite-sphinx >=0.13.1 18 | - jupyterlab >=4.4,<5 19 | - empack >=5.1.1 20 | - pip: 21 | - jupyterlite-core >=0.6,<0.7 22 | - .. 23 | -------------------------------------------------------------------------------- /docs/changelog.md: -------------------------------------------------------------------------------- 1 | ```{include} ../CHANGELOG.md 2 | 3 | ``` 4 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | extensions = [ 2 | "jupyterlite_sphinx", 3 | "myst_parser", 4 | "sphinx_design", 5 | ] 6 | 7 | myst_enable_extensions = [ 8 | "linkify", 9 | "colon_fence", 10 | ] 11 | 12 | master_doc = "index" 13 | source_suffix = ".rst" 14 | 15 | project = "jupyterlite-xeus" 16 | copyright = "JupyterLite Team" 17 | author = "JupyterLite Team" 18 | 19 | exclude_patterns = [] 20 | 21 | html_theme = "pydata_sphinx_theme" 22 | 23 | html_static_path = ['_static'] 24 | 25 | html_css_files = [ 26 | 'css/custom.css', 27 | ] 28 | 29 | jupyterlite_dir = "." 30 | jupyterlite_silence = False 31 | 32 | html_theme_options = { 33 | "logo": { 34 | "image_light": "jupyterlite.svg", 35 | "image_dark": "jupyterlite.svg", 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /docs/deploy.md: -------------------------------------------------------------------------------- 1 | (deploy)= 2 | 3 | # Deploy your own 4 | 5 | ## On Github Pages 6 | 7 | In order to make your own JupyterLite deployment, you can use the [xeus-python-demo repository template](https://github.com/jupyterlite/xeus-python-demo) 8 | that allows you to easily make a JupyteLite deployment on Github pages with xeus-python as default kernel. 9 | 10 | This template repository contains an `environment.yml` file where you can specify the packages you need. You can also add Notebooks to the `content` folder. 11 | 12 | ## In Sphinx documentation 13 | 14 | You can make a JupyterLite deployment with xeus-python installed using the [jupyterlite-sphinx extension](https://github.com/jupyterlite/jupyterlite-sphinx) 15 | -------------------------------------------------------------------------------- /docs/environment-cpp.yml: -------------------------------------------------------------------------------- 1 | name: xeus-cpp-kernel-docs 2 | channels: 3 | - https://repo.prefix.dev/emscripten-forge-dev 4 | - conda-forge 5 | dependencies: 6 | - xeus-cpp 7 | -------------------------------------------------------------------------------- /docs/environment-other.yml: -------------------------------------------------------------------------------- 1 | name: xeus-kernel-docs 2 | channels: 3 | - https://repo.mamba.pm/emscripten-forge 4 | - conda-forge 5 | dependencies: 6 | - xeus-lua 7 | - xeus-javascript 8 | - xeus-nelson 9 | -------------------------------------------------------------------------------- /docs/environment-python.yml: -------------------------------------------------------------------------------- 1 | name: xeus-python-kernel-docs 2 | channels: 3 | - https://repo.prefix.dev/emscripten-forge-dev 4 | - conda-forge 5 | dependencies: 6 | - xeus-python 7 | - numpy 8 | - matplotlib 9 | - pillow 10 | - ipywidgets>=8.1.3 11 | - ipyleaflet 12 | - pip: 13 | - ipycanvas 14 | -------------------------------------------------------------------------------- /docs/environment-r.yml: -------------------------------------------------------------------------------- 1 | name: xeus-r-kernel-docs 2 | channels: 3 | - https://repo.mamba.pm/emscripten-forge 4 | - conda-forge 5 | dependencies: 6 | - xeus-r 7 | -------------------------------------------------------------------------------- /docs/environment.md: -------------------------------------------------------------------------------- 1 | (environment)= 2 | 3 | # Emscripten Environment 4 | 5 | ## Pre-installed packages 6 | 7 | `jupyterlite-xeus` allows you to pre-install packages in the runtime. You can pre-install packages by adding an `environment.yml` file in the JupyterLite build directory, this file will be found automatically by jupyterlite-xeus which will pre-build the environment when running `jupyter lite build`. 8 | 9 | Furthermore, this automatically installs any labextension that it founds, for example installing ipyleaflet will make ipyleaflet work without the need to manually install the jupyter-leaflet labextension. 10 | 11 | Say you want to install `NumPy`, `Matplotlib` and `ipycanvas`, it can be done by creating the `environment.yml` file with the following content: 12 | 13 | ```yaml 14 | name: xeus-python-kernel 15 | channels: 16 | - https://repo.prefix.dev/emscripten-forge-dev 17 | - https://repo.prefix.dev/conda-forge 18 | dependencies: 19 | - xeus-python 20 | - numpy 21 | - matplotlib 22 | - ipycanvas 23 | ``` 24 | 25 | Then you only need to build JupyterLite: 26 | 27 | ``` 28 | jupyter lite build 29 | ``` 30 | 31 | You can also pick another name for that environment file (_e.g._ `custom.yml`), by doing so, you will need to specify that name to xeus-python: 32 | 33 | ``` 34 | jupyter lite build --XeusAddon.environment_file=custom.yml 35 | ``` 36 | 37 | ```{warning} 38 | It is common to provide `pip` dependencies in a conda environment file. This is currently **partially supported** by jupyterlite-xeus. See "pip packages" section. 39 | ``` 40 | 41 | Then those packages are usable directly: 42 | 43 | ```{eval-rst} 44 | .. replite:: 45 | :kernel: xeus-python 46 | :height: 600px 47 | :prompt: Try it! 48 | 49 | %matplotlib inline 50 | 51 | import matplotlib.pyplot as plt 52 | import numpy as np 53 | 54 | fig = plt.figure() 55 | plt.plot(np.sin(np.linspace(0, 20, 100))) 56 | plt.show(); 57 | ``` 58 | 59 | ### Multi environment support 60 | 61 | Starting with jupyterlite-xeus v4.0.0a0, you can now pass multiple environment files or prefixes. 62 | 63 | ``` 64 | jupyter lite build --XeusAddon.environment_file=environment-python.yml --XeusAddon.environment_file=environment-r.yml 65 | ``` 66 | 67 | This allows e.g. to make multiple xeus-python kernels available, with a different set of packages. 68 | 69 | ### pip packages 70 | 71 | ⚠ This feature is experimental. You won't have the same user-experience as when using conda/mamba in a "normal" setup ⚠ 72 | 73 | `jupyterlite-xeus` provides a way to install packages with pip. 74 | 75 | There are a couple of limitations that you should be aware of: 76 | 77 | - it can **only** install **pure Python packages** (Python code + data files) 78 | - it **does not install the package dependencies**, you should make sure to install them yourself using conda-forge/emscripten-forge. 79 | - it does not work (yet?) using `-r requirements.txt` in your environment file 80 | 81 | For example, if you were to install `ipycanvas` from PyPI, you would need to install the ipycanvas dependencies for it to work (`pillow`, `numpy` and `ipywidgets`): 82 | 83 | ```yaml 84 | name: xeus-python-kernel 85 | channels: 86 | - https://repo.prefix.dev/emscripten-forge-dev 87 | - https://repo.prefix.dev/conda-forge 88 | dependencies: 89 | - xeus-python 90 | - numpy 91 | - pillow 92 | - ipywidgets 93 | - pip: 94 | - ipycanvas 95 | ``` 96 | 97 | You can also install a local Python package, this is very practical if you want to embed 98 | a jupyterlite deployment in your Package documentation, allowing to test the very latest dev version: 99 | 100 | ```yaml 101 | name: xeus-python-kernel 102 | channels: 103 | - https://repo.prefix.dev/emscripten-forge-dev 104 | - https://repo.prefix.dev/conda-forge 105 | dependencies: 106 | - xeus-python 107 | - pip: 108 | - .. 109 | ``` 110 | -------------------------------------------------------------------------------- /docs/files.md: -------------------------------------------------------------------------------- 1 | (files)= 2 | 3 | # Accessing and managing files 4 | 5 | Using jupyterlite-xeus, you can mount files and directories into the kernel runtime. You have multiple approaches for this: 6 | 7 | ## JupyterLite content 8 | 9 | ### Using the default JupyterLite setup 10 | 11 | ⚠ This feature is very experimental and may fail in weird ways. ⚠ 12 | 13 | xeus kernels will automatically have access to files served by JupyterLite. 14 | 15 | This feature depends on either the service worker ([which may not be available in some browser setups](https://jupyterlite.readthedocs.io/en/stable/howto/configure/advanced/service-worker.html#limitations)) or SharedArrayBuffers if the proper flags are set. 16 | 17 | See [accessing files from a kernel](https://jupyterlite.readthedocs.io/en/stable/howto/content/python.html) for more information. 18 | 19 | ### Making it more robust 20 | 21 | To make things more robust, you can embed the JupyterLite content into the xeus kernel. 22 | 23 | You can enable this feature using the `--XeusAddon.mount_jupyterlite_content=True` CLI option: 24 | 25 | ```bash 26 | jupyter lite build --XeusAddon.mount_jupyterlite_content=True 27 | ``` 28 | 29 | This approach has behavior differences with the service worker approach: 30 | 31 | - This makes file access more robust, not depending on the service worker. 32 | - Kernels will automatically start from the `/files` directory, where the jupyterlite content is mounted. 33 | - If your kernel changes the content (creates files, updates files content _etc_), changes **will not** reflect in the JupyterLite served content. This means that if you open the updated files from the filebrowser UI by double clicking on them, you will see the initial content of the files. It also means that restarting the kernel will reinitialize the `/files` directory content, and it will not be shared between kernels. 34 | 35 | ```{note} 36 | This option is set to True by default when generating a [Voici dashboard](https://github.com/voila-dashboards/voici) 37 | ``` 38 | 39 | ## Extra mount points 40 | 41 | You can mount extra directories into the kernel using the mounts option: 42 | 43 | ```bash 44 | jupyter lite build \ 45 | --XeusAddon.mounts="mypackage:/lib/python3.11/site-packages/mypackage" 46 | --XeusAddon.mounts="hispackage:/lib/python3.11/site-packages/hispackage" 47 | ``` 48 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | # xeus kernels in JupyterLite 🚀🪐 2 | 3 | ![Xeus logo](./xeus.svg) 4 | 5 | `jupyterlite-xeus` is an extension for JupyterLite that enables fully client-side Jupyter environments powered by xeus kernels compiled to WebAssembly (Wasm). It allows users to create statically-served Jupyter deployments with custom pre-built environments — no server required. 6 | 7 | The core feature of `jupyterlite-xeus` is its integration with [emscripten-forge](https://github.com/emscripten-forge), a conda package distribution tailored for WebAssembly. This makes it possible to bundle your favorite scientific or data analysis packages directly into the browser-based environment, delivering a reproducible computing experience with zero backend dependencies. 8 | 9 | Ideal for demos, educational resources, and offline computing. Use it in combination with [Voici](https://github.com/voila-dashboards/voici)! 10 | 11 | Currently supported kernels are: 12 | 13 | - [xeus-python](https://github.com/jupyter-xeus/xeus-python) 14 | - [xeus-lua](https://github.com/jupyter-xeus/xeus-lua) 15 | - [xeus-r](https://github.com/jupyter-xeus/xeus-r) 16 | - [xeus-cpp](https://github.com/compiler-research/xeus-cpp) 17 | - [xeus-nelson](https://github.com/jupyter-xeus/xeus-nelson) 18 | - [xeus-javascript](https://github.com/jupyter-xeus/xeus-javascript) 19 | 20 | Try it here! 21 | 22 | ::::{tab-set} 23 | :::{tab-item} Python 24 | 25 | ```{eval-rst} 26 | .. replite:: 27 | :kernel: xpython 28 | :height: 600px 29 | 30 | print("Hello from xeus-python!") 31 | 32 | from ipyleaflet import Map, Marker 33 | 34 | center = (52.204793, 360.121558) 35 | 36 | m = Map(center=center, zoom=15) 37 | 38 | marker = Marker(location=center, draggable=False) 39 | m.add(marker); 40 | 41 | m 42 | ``` 43 | 44 | ::: 45 | :::{tab-item} Lua 46 | 47 | ```{eval-rst} 48 | .. replite:: 49 | :kernel: xlua 50 | :height: 600px 51 | :prompt: Try Lua! 52 | 53 | print("Hello from xeus-lua!") 54 | ``` 55 | 56 | ::: 57 | :::{tab-item} R 58 | 59 | ```{eval-rst} 60 | .. replite:: 61 | :kernel: xr 62 | :height: 600px 63 | :prompt: Try R! 64 | 65 | print("Hello from R!") 66 | 67 | A <- matrix(c(4, 1, 1, 3), nrow = 2, byrow = TRUE) 68 | 69 | # Eigen decomposition 70 | eigen_result <- eigen(A) 71 | 72 | print(eigen_result$values) 73 | 74 | print(eigen_result$vectors) 75 | ``` 76 | 77 | ::: 78 | :::{tab-item} C++ 79 | 80 | ```{eval-rst} 81 | .. replite:: 82 | :kernel: xcpp20 83 | :height: 600px 84 | :prompt: Try C++! 85 | 86 | #include 87 | #include 88 | 89 | void funky_sin_wave(int length) { 90 | for (int y = 0; y < 20; y++) { 91 | for (int x = 0; x < length; x++) { 92 | double wave = sin(x * 0.1); 93 | if ((int)(10 + 10 * wave) == y) { 94 | printf("*"); 95 | } else { 96 | printf(" "); 97 | } 98 | } 99 | printf("\n"); 100 | } 101 | } 102 | 103 | funky_sin_wave(80); 104 | ``` 105 | 106 | ::: 107 | :::: 108 | 109 | ## Installation 110 | 111 | You can install `jupyterlite-xeus` with conda/mamba 112 | 113 | ``` 114 | mamba install -c conda-forge jupyterlite-xeus 115 | ``` 116 | 117 | Or with `pip` (you must install micromamba 2.0.5): 118 | 119 | ``` 120 | pip install jupyterlite-xeus 121 | ``` 122 | 123 | ## Usage 124 | 125 | Once installed, you can create an `environment.yml` file at the root of your jupyterlite build directory containing the following: 126 | 127 | ```yaml 128 | name: xeus-kernels 129 | channels: 130 | - https://repo.prefix.dev/emscripten-forge-dev 131 | - https://repo.prefix.dev/conda-forge 132 | dependencies: 133 | - xeus-python 134 | - xeus-lua 135 | - xeus-nelson 136 | - numpy 137 | - matplotlib 138 | - pillow 139 | - ipywidgets 140 | - pip: 141 | - ipycanvas 142 | ``` 143 | 144 | You can then run the usual `jupyter lite build` or `voici my-notebook.ipynb`. The `environment.yml` file will be picked-up automatically by `jupyterlite-xeus`, installing `xeus-python`, `xeus-lua`, `xeus-nelson` and some useful Python packages into the user environment. 145 | 146 | ## Features 147 | 148 | ### Dynamic install of packages 149 | 150 | Starting with jupyterlite-xeus v4.0.0a11, you can use the `%pip` magic or the `%mamba` magics to install packages dynamically once the kernel started: 151 | 152 | ``` 153 | %pip install my_package 154 | ``` 155 | 156 | or 157 | 158 | ``` 159 | %mamba install my_package 160 | ``` 161 | 162 | ### stdin 163 | 164 | Starting with jupyterlite-xeus v4.0.0a8, latest jupyterlite 0.6.0, and latest xeus kernels (tested in Python, C++, lua), blocking stdin is now supported: 165 | 166 | ```python 167 | name = input("what's your name") 168 | ``` 169 | 170 | ### Multiple kernels 171 | 172 | To create a deployment with multiple kernels, you can simply add them to the `environment.yml` file: 173 | 174 | ```yaml 175 | name: xeus-lite-wasm 176 | channels: 177 | - https://repo.prefix.dev/emscripten-forge-dev 178 | - https://repo.prefix.dev/conda-forge 179 | dependencies: 180 | - xeus-python 181 | - xeus-lua 182 | - xeus-sqlite 183 | - numpy 184 | ``` 185 | 186 | Learn more in [](./environment.md) 187 | 188 | ### Mounting additional files 189 | 190 | To copy additional files and directories into the virtual filesystem of the xeus-lite kernels you can use the `--XeusAddon.mount` option. 191 | Each mount is specified as a pair of paths separated by a colon `:`. The first path is the path to the file or directory on the host machine, the second path is the path to the file or directory in the virtual filesystem of the kernel. 192 | 193 | ```bash 194 | jupyter lite build \ 195 | --XeusAddon.environment_file=environment.yml \ 196 | --XeusAddon.mounts=/some/path/on/host_machine:/some/path/in/virtual/filesystem 197 | ``` 198 | 199 | Learn more in [](./files.md) 200 | 201 | ## Learn more 202 | 203 | ```{toctree} 204 | :maxdepth: 2 205 | 206 | deploy 207 | environment 208 | files 209 | advanced 210 | changelog 211 | ``` 212 | -------------------------------------------------------------------------------- /docs/jupyter_lite_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "XeusAddon": { 3 | "environment_file": [ 4 | "environment-python.yml", 5 | "environment-other.yml", 6 | "environment-r.yml", 7 | "environment-cpp.yml" 8 | ] 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /docs/jupyterlite.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 19 | 21 | 25 | 29 | 33 | 37 | 41 | 45 | 49 | 53 | 57 | 61 | 65 | 69 | 70 | 95 | 100 | 101 | 103 | 104 | 106 | image/svg+xml 107 | 109 | 110 | 111 | 112 | 113 | 119 | 124 | 125 | 130 | 133 | 140 | 150 | 160 | 170 | 174 | 178 | 183 | 184 | 189 | 193 | 198 | 199 | 200 | 201 | 202 | 203 | -------------------------------------------------------------------------------- /docs/xeus.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 9 | 10 | 18 | 23 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /environment-dev.yaml: -------------------------------------------------------------------------------- 1 | name: xeus-lite-dev 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - black 6 | - python-build 7 | - hatch-jupyter-builder 8 | - hatch-nodejs-version 9 | - jupyterlab >=4.0 10 | - jupyterlite-core 11 | - mamba <2 12 | - nodejs 13 | - pip 14 | - pytest 15 | - python 16 | - requests 17 | - ruff 18 | - traitlets 19 | - typer 20 | - yarn=3 21 | - pip: 22 | - empack >=5.1.1 23 | -------------------------------------------------------------------------------- /example_envs/env_with_extension.yaml: -------------------------------------------------------------------------------- 1 | name: xeus-lite-wasm 2 | channels: 3 | - https://repo.mamba.pm/emscripten-forge 4 | - conda-forge 5 | dependencies: 6 | - xeus-python 7 | - ipycanvas -------------------------------------------------------------------------------- /example_envs/env_with_many_kernels.yaml: -------------------------------------------------------------------------------- 1 | name: xeus-lite-wasm 2 | channels: 3 | - https://repo.mamba.pm/emscripten-forge 4 | - conda-forge 5 | dependencies: 6 | - xeus-python 7 | - xeus-sqlite 8 | - xeus-lua -------------------------------------------------------------------------------- /example_envs/env_with_pip.yaml: -------------------------------------------------------------------------------- 1 | name: xeus-lite-wasm 2 | channels: 3 | - https://repo.mamba.pm/emscripten-forge 4 | - conda-forge 5 | dependencies: 6 | - xeus-python 7 | - pip: 8 | - python-random-name-generator -------------------------------------------------------------------------------- /install.json: -------------------------------------------------------------------------------- 1 | { 2 | "packageManager": "python", 3 | "packageName": "jupyterlite_xeus", 4 | "uninstallInstructions": "Use your Python package manager (pip, conda, etc.) to uninstall the package jupyterlite_xeus" 5 | } 6 | -------------------------------------------------------------------------------- /jupyterlite_xeus/__init__.py: -------------------------------------------------------------------------------- 1 | try: 2 | from ._version import __version__ 3 | except ImportError: 4 | # Fallback when using the package in dev mode without installing 5 | # in editable mode with pip. It is highly recommended to install 6 | # the package from a stable release or in editable mode: https://pip.pypa.io/en/stable/topics/local-project-installs/#editable-installs 7 | import warnings 8 | 9 | warnings.warn("Importing 'jupyterlite_xeus' outside a proper installation.") 10 | __version__ = "dev" 11 | 12 | 13 | def _jupyter_labextension_paths(): 14 | return [{"src": "labextension", "dest": "@jupyterlite/xeus"}] 15 | -------------------------------------------------------------------------------- /jupyterlite_xeus/_pip.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import shutil 3 | import os 4 | from subprocess import run as subprocess_run 5 | from tempfile import TemporaryDirectory 6 | from pathlib import Path 7 | import csv 8 | import json 9 | import glob 10 | 11 | 12 | def _get_python_version(prefix_path): 13 | path = glob.glob(f"{prefix_path}/conda-meta/python-3.*.json") 14 | 15 | if not path: 16 | raise RuntimeError("Python needs to be installed for installing pip dependencies") 17 | 18 | version = json.load(open(path[0]))["version"].split(".") 19 | return f"{version[0]}.{version[1]}" 20 | 21 | 22 | def _install_pip_dependencies(prefix_path, dependencies, log=None): 23 | # Why is this so damn complicated? 24 | # Isn't it easier to download the .whl ourselves? pip is hell 25 | 26 | if log is not None: 27 | log.warning( 28 | """ 29 | Installing pip dependencies. This is very much experimental so use 30 | this feature at your own risks. 31 | Note that you can only install pure-python packages. 32 | pip is being run with the --no-deps option to not pull undesired 33 | system-specific dependencies, so please install your package dependencies 34 | from emscripten-forge or conda-forge. 35 | """ 36 | ) 37 | 38 | # Installing with pip in another prefix that has a different Python version IS NOT POSSIBLE 39 | # So we need to do this whole mess "manually" 40 | pkg_dir = TemporaryDirectory() 41 | 42 | python_version = _get_python_version(prefix_path) 43 | 44 | subprocess_run( 45 | [ 46 | sys.executable, 47 | "-m", 48 | "pip", 49 | "install", 50 | *dependencies, 51 | # Install in a tmp directory while we process it 52 | "--target", 53 | pkg_dir.name, 54 | # Specify the right Python version 55 | "--python-version", 56 | python_version, 57 | # No dependency installed 58 | "--no-deps", 59 | "--no-input", 60 | "--verbose", 61 | ], 62 | check=True, 63 | ) 64 | 65 | # We need to read the RECORD and try to be smart about what goes 66 | # under site-packages and what goes where 67 | packages_dist_info = Path(pkg_dir.name).glob("*.dist-info") 68 | 69 | for package_dist_info in packages_dist_info: 70 | with open(package_dist_info / "RECORD") as record: 71 | record_content = record.read() 72 | record_csv = csv.reader(record_content.splitlines()) 73 | all_files = [_file[0] for _file in record_csv] 74 | 75 | # List of tuples: (path: str, inside_site_packages: bool) 76 | files = [(_file, not _file.startswith("../../")) for _file in all_files] 77 | 78 | # Why? 79 | fixed_record_data = record_content.replace("../../", "../../../") 80 | 81 | # OVERWRITE RECORD file 82 | with open(package_dist_info / "RECORD", "w") as record: 83 | record.write(fixed_record_data) 84 | 85 | non_supported_files = [".so", ".a", ".dylib", ".lib", ".exe.dll"] 86 | 87 | # COPY files under `prefix_path` 88 | for _file, inside_site_packages in files: 89 | path = Path(_file) 90 | 91 | # FAIL if .so / .a / .dylib / .lib / .exe / .dll 92 | if path.suffix in non_supported_files: 93 | raise RuntimeError( 94 | "Cannot install binary PyPI package, only pure Python packages are supported" 95 | ) 96 | 97 | file_path = _file[6:] if not inside_site_packages else _file 98 | install_path = ( 99 | prefix_path 100 | if not inside_site_packages 101 | else prefix_path / "lib" / f"python{python_version}" / "site-packages" 102 | ) 103 | 104 | src_path = Path(pkg_dir.name) / file_path 105 | dest_path = install_path / file_path 106 | 107 | os.makedirs(dest_path.parent, exist_ok=True) 108 | 109 | shutil.copy(src_path, dest_path) 110 | -------------------------------------------------------------------------------- /jupyterlite_xeus/add_on.py: -------------------------------------------------------------------------------- 1 | """a JupyterLite addon for creating the env for xeus kernels""" 2 | 3 | import json 4 | import os 5 | from pathlib import Path 6 | import shutil 7 | from tempfile import TemporaryDirectory 8 | from urllib.parse import urlparse 9 | import warnings 10 | 11 | import yaml 12 | 13 | import requests 14 | 15 | from jupyterlite_core.addons.federated_extensions import FederatedExtensionAddon 16 | from jupyterlite_core.constants import ( 17 | ALL_FEDERATED_JSON, 18 | FEDERATED_EXTENSIONS, 19 | JUPYTERLITE_JSON, 20 | LAB_EXTENSIONS, 21 | SHARE_LABEXTENSIONS, 22 | UTF8, 23 | ) 24 | from traitlets import Bool, Callable, List, Unicode 25 | 26 | from .create_conda_env import ( 27 | create_conda_env_from_env_file, 28 | create_conda_env_from_specs, 29 | ) 30 | from .constants import EXTENSION_NAME 31 | from .constants import EXTENSION_NAME 32 | 33 | from empack.pack import ( 34 | DEFAULT_CONFIG_PATH, 35 | pack_env, 36 | pack_directory, 37 | pack_file, 38 | add_tarfile_to_env_meta, 39 | ) 40 | from empack.file_patterns import PkgFileFilter, pkg_file_filter_from_yaml 41 | 42 | EMPACK_ENV_META = "empack_env_meta.json" 43 | 44 | 45 | def get_kernel_binaries(path): 46 | """Return paths to the kernel binaries (js, wasm, and optionally data) if they exist, else None.""" 47 | json_file = path / "kernel.json" 48 | if json_file.exists(): 49 | kernel_spec = json.loads(json_file.read_text(**UTF8)) 50 | argv = kernel_spec.get("argv") 51 | kernel_binary = argv[0] 52 | 53 | kernel_binary_js = Path(kernel_binary + ".js") 54 | kernel_binary_wasm = Path(kernel_binary + ".wasm") 55 | kernel_binary_data = Path(kernel_binary + ".data") 56 | 57 | if kernel_binary_js.exists() and kernel_binary_wasm.exists(): 58 | # Return all three, with None for .data if it doesn't exist 59 | # as this might not be neccessary for all kernels. 60 | return ( 61 | kernel_binary_js, 62 | kernel_binary_wasm, 63 | kernel_binary_data if kernel_binary_data.exists() else None, 64 | ) 65 | else: 66 | warnings.warn(f"kernel binaries not found for {path.name}") 67 | 68 | else: 69 | warnings.warn(f"kernel.json not found for {path.name}") 70 | 71 | return None 72 | 73 | 74 | class ListLike(List): 75 | def from_string(self, s): 76 | return [s] 77 | 78 | 79 | class XeusAddon(FederatedExtensionAddon): 80 | __all__ = ["post_build"] 81 | 82 | empack_config = Unicode( 83 | None, 84 | config=True, 85 | allow_none=True, 86 | description="The path or URL to the empack config file", 87 | ) 88 | 89 | environment_file = ListLike( 90 | [], 91 | config=True, 92 | description='The path to the environment file. Defaults to looking for "environment.yml" or "environment.yaml"', 93 | ) 94 | 95 | prefix = ListLike( 96 | [], 97 | config=True, 98 | description="The path to the wasm prefix", 99 | ) 100 | 101 | mount_jupyterlite_content = Bool( 102 | None, 103 | allow_none=True, 104 | config=True, 105 | description="Whether or not to mount the jupyterlite content into the kernel. This would make the jupyterlite content available under the '/files' directory, and the kernels will automatically be started from there.", 106 | ) 107 | 108 | mounts = ListLike( 109 | [], 110 | config=True, 111 | description="A list of mount points, in the form : to mount in the wasm prefix", 112 | ) 113 | 114 | package_url_factory = Callable( 115 | None, 116 | allow_none=True, 117 | config=True, 118 | description="Factory to generate package download URL from package metadata. This is used to load python packages from external host", 119 | ) 120 | 121 | def __init__(self, *args, **kwargs): 122 | super().__init__(*args, **kwargs) 123 | self.xeus_output_dir = Path(self.manager.output_dir) / "xeus" 124 | self.cwd = TemporaryDirectory() 125 | # TODO Make this configurable 126 | # You can provide another cwd_name if you want 127 | self.cwd_name = self.cwd.name 128 | 129 | def post_build(self, manager): 130 | if not self.environment_file: 131 | if (Path(self.manager.lite_dir) / "environment.yml").exists(): 132 | self.environment_file = ["environment.yml"] 133 | 134 | if (Path(self.manager.lite_dir) / "environment.yaml").exists(): 135 | self.environment_file = ["environment.yaml"] 136 | 137 | # check that either prefix or environment_file is set 138 | if not self.prefix and not self.environment_file: 139 | raise ValueError("Either prefix or environment_file must be set") 140 | 141 | # create the prefixes if it does not exist 142 | self.prefixes = {} 143 | if not self.prefix: 144 | for environment_file in self.environment_file: 145 | env_name, prefix = self.create_prefix(Path(self.manager.lite_dir) / environment_file) 146 | if env_name in self.prefixes: 147 | raise ValueError(f"Environment name '{env_name}' used more than once") 148 | self.prefixes[env_name] = prefix 149 | else: 150 | for prefix in self.prefix: 151 | path = Path(prefix) 152 | if not path.is_dir(): 153 | raise ValueError(f"Prefix '{prefix}' is not a directory") 154 | env_name = path.name # Take the final path component and the env_name 155 | if env_name in self.prefixes: 156 | raise ValueError(f"Environment name '{env_name}' used more than once") 157 | self.prefixes[env_name] = prefix 158 | 159 | all_kernels = [] 160 | for env_name, prefix in self.prefixes.items(): 161 | # copy the kernels from the prefix 162 | kernels = yield from self.copy_kernels_from_prefix(env_name, prefix) 163 | all_kernels.extend(kernels) 164 | 165 | # copy the jupyterlab extensions 166 | yield from self.copy_jupyterlab_extensions_from_prefix(prefix) 167 | 168 | # write the kernels.json file 169 | kernel_file = Path(self.cwd_name) / "kernels.json" 170 | kernel_file.write_text(json.dumps(all_kernels), **UTF8) 171 | yield dict( 172 | name=f"copy:{kernel_file}", 173 | actions=[ 174 | (self.copy_one, [kernel_file, self.xeus_output_dir / "kernels.json"]) 175 | ], 176 | ) 177 | 178 | def create_prefix(self, env_file: Path): 179 | # read the environment file 180 | root_prefix = Path(self.cwd_name) / "_env" 181 | 182 | with open(env_file, "r") as file: 183 | yaml_content = yaml.safe_load(file) 184 | 185 | env_name = yaml_content["name"] 186 | env_prefix = root_prefix / "envs" / env_name 187 | 188 | create_conda_env_from_env_file(root_prefix, yaml_content, env_file.parent) 189 | 190 | return env_name, env_prefix 191 | 192 | def copy_kernels_from_prefix(self, env_name, prefix): 193 | kernel_spec_path = Path(prefix) / "share" / "jupyter" / "kernels" 194 | 195 | if not kernel_spec_path.exists(): 196 | warnings.warn( 197 | f"No kernels are installed in the prefix {prefix}. Try adding e.g. xeus-python in your environment.yml file." 198 | ) 199 | return 200 | 201 | all_kernels = [] 202 | # find all folders in the kernelspec path 203 | for kernel_dir in kernel_spec_path.iterdir(): 204 | kernel_binaries = get_kernel_binaries(kernel_dir) 205 | if kernel_binaries: 206 | kernel_js, kernel_wasm, kernel_data = kernel_binaries 207 | all_kernels.append(dict(kernel=kernel_dir.name, env_name=env_name)) 208 | # take care of each kernel 209 | yield from self.copy_kernel(env_name, prefix, kernel_dir, kernel_wasm, kernel_js, kernel_data) 210 | 211 | # pack prefix packages 212 | yield from self.pack_prefix(env_name, prefix) 213 | 214 | return all_kernels 215 | 216 | def copy_kernel(self, env_name, prefix, kernel_dir, kernel_wasm, kernel_js, kernel_data): 217 | kernel_spec = json.loads((kernel_dir / "kernel.json").read_text(**UTF8)) 218 | 219 | # update kernel_executable path in kernel.json 220 | kernel_spec["argv"][0] = f"xeus/{env_name}/bin/{kernel_js.name}" 221 | 222 | # find logos in the directory 223 | image_files = [] 224 | for file_type in ["*.jpg", "*.png", "*.svg"]: 225 | image_files.extend(kernel_dir.glob(file_type)) 226 | 227 | kernel_spec["resources"] = {} 228 | for image in image_files: 229 | output_image = ( 230 | self.xeus_output_dir / env_name / kernel_dir.name / image.name 231 | ) 232 | kernel_spec["resources"][image.stem] = str( 233 | output_image.relative_to(self.manager.output_dir) 234 | ) 235 | 236 | # copy the logo file 237 | yield dict( 238 | name=f"copy:{env_name}:{kernel_dir.name}:{image.name}", 239 | actions=[ 240 | ( 241 | self.copy_one, 242 | [ 243 | kernel_dir / image.name, 244 | output_image, 245 | ], 246 | ), 247 | ], 248 | ) 249 | 250 | if kernel_spec.get("metadata", {}).get("shared", None) is not None: 251 | for filename, location in kernel_spec["metadata"]["shared"].items(): 252 | # Copy shared lib file in the output 253 | yield dict( 254 | name=f"copy:{env_name}:{kernel_dir.name}:{filename}", 255 | actions=[ 256 | ( 257 | self.copy_one, 258 | [ 259 | Path(prefix) / location, 260 | self.xeus_output_dir / env_name / kernel_dir.name / filename, 261 | ], 262 | ), 263 | ], 264 | ) 265 | 266 | # Copy libxeus shared lib file in the output 267 | filename = "libxeus.so" 268 | location = "lib/libxeus.so" 269 | target_location = self.xeus_output_dir / env_name / kernel_dir.name / filename 270 | if (Path(prefix) / location).exists() and not target_location.exists(): 271 | yield dict( 272 | name=f"copy:{env_name}:{kernel_dir.name}:{filename}", 273 | actions=[ 274 | ( 275 | self.copy_one, 276 | [ 277 | Path(prefix) / location, 278 | target_location, 279 | ], 280 | ), 281 | ], 282 | ) 283 | 284 | # write to temp file 285 | kernel_json = Path(self.cwd_name) / env_name / f"{kernel_dir.name}_kernel.json" 286 | kernel_json.parent.mkdir(parents=True, exist_ok=True) 287 | kernel_json.write_text(json.dumps(kernel_spec), **UTF8) 288 | 289 | # copy the kernel binary files to the bin dir 290 | yield dict( 291 | name=f"copy:{env_name}:{kernel_dir.name}:binaries", 292 | actions=[ 293 | ( 294 | self.copy_one, 295 | [kernel_js, self.xeus_output_dir / env_name / "bin" / kernel_js.name], 296 | ), 297 | ( 298 | self.copy_one, 299 | [kernel_wasm, self.xeus_output_dir / env_name / "bin" / kernel_wasm.name], 300 | ), 301 | ], 302 | ) 303 | 304 | # copy the kernel.data file to the bin dir if present 305 | if kernel_data: 306 | yield dict( 307 | name=f"copy:{env_name}:{kernel_dir.name}:data", 308 | actions=[ 309 | ( 310 | self.copy_one, 311 | [kernel_data, self.xeus_output_dir / env_name / "bin" / kernel_data.name], 312 | ), 313 | ], 314 | ) 315 | 316 | # copy the kernel.json file 317 | yield dict( 318 | name=f"copy:{env_name}:{kernel_dir.name}:kernel.json", 319 | actions=[ 320 | ( 321 | self.copy_one, 322 | [ 323 | kernel_json, 324 | self.xeus_output_dir 325 | / env_name 326 | / kernel_dir.name 327 | / "kernel.json", 328 | ], 329 | ) 330 | ], 331 | ) 332 | 333 | def pack_prefix(self, env_name, prefix): 334 | env_dir = self.xeus_output_dir / env_name 335 | packages_dir = env_dir / "kernel_packages" 336 | 337 | out_path = Path(self.cwd_name) / "packed_env" / env_name 338 | out_path.mkdir(parents=True, exist_ok=True) 339 | 340 | pack_kwargs = {} 341 | 342 | empack_config = self.empack_config 343 | 344 | # Download env filter config 345 | if empack_config: 346 | empack_config_is_url = urlparse(empack_config).scheme in ("http", "https") 347 | if empack_config_is_url: 348 | empack_config_content = requests.get(empack_config).content 349 | pack_kwargs["file_filters"] = PkgFileFilter( 350 | **yaml.safe_load(empack_config_content) 351 | ) 352 | else: 353 | pack_kwargs["file_filters"] = pkg_file_filter_from_yaml(empack_config) 354 | else: 355 | pack_kwargs["file_filters"] = pkg_file_filter_from_yaml(DEFAULT_CONFIG_PATH) 356 | 357 | if self.package_url_factory is not None: 358 | pack_kwargs["package_url_factory"] = self.package_url_factory 359 | 360 | pack_env( 361 | env_prefix=prefix, 362 | relocate_prefix="/", 363 | outdir=out_path, 364 | use_cache=False, 365 | **pack_kwargs, 366 | ) 367 | 368 | # Pack user defined mount points 369 | for mount_index, mount in enumerate(self.mounts): 370 | if mount.count(":") != 1: 371 | raise ValueError( 372 | f"invalid mount {mount}, must be :" 373 | ) 374 | 375 | host_path, mount_path = mount.split(":") 376 | host_path = Path(host_path) 377 | mount_path = Path(mount_path) 378 | 379 | if not mount_path.is_absolute() or ( 380 | os.name == "nt" and mount_path.anchor != "\\" 381 | ): 382 | raise ValueError(f"mount_path {mount_path} needs to be absolute") 383 | 384 | if str(mount_path).startswith("/files"): 385 | raise ValueError( 386 | f"Mount point '/files' is reserved for jupyterlite content. Cannot mount {mount}" 387 | ) 388 | 389 | outname = f"mount_{mount_index}.tar.gz" 390 | 391 | if host_path.is_dir(): 392 | pack_directory( 393 | host_dir=host_path, 394 | mount_dir=mount_path, 395 | outname=outname, 396 | outdir=out_path, 397 | ) 398 | elif host_path.is_file(): 399 | pack_file( 400 | host_file=host_path, 401 | mount_dir=mount_path, 402 | outname=outname, 403 | outdir=out_path, 404 | ) 405 | else: 406 | raise ValueError( 407 | f"host_path {host_path} needs to be a file or a directory" 408 | ) 409 | 410 | add_tarfile_to_env_meta( 411 | env_meta_filename=out_path / EMPACK_ENV_META, tarfile=out_path / outname 412 | ) 413 | 414 | # Pack JupyterLite content if enabled 415 | # If we only build a voici output, mount jupyterlite content into the kernel by default 416 | if self.mount_jupyterlite_content or ( 417 | list(self.manager.apps) == ["voici"] 418 | and self.mount_jupyterlite_content is None 419 | ): 420 | contents_dir = self.manager.output_dir / "files" 421 | 422 | outname = f"mount_{len(self.mounts)}.tar.gz" 423 | 424 | pack_directory( 425 | host_dir=contents_dir, 426 | mount_dir="/files", 427 | outname=outname, 428 | outdir=out_path, 429 | ) 430 | 431 | add_tarfile_to_env_meta( 432 | env_meta_filename=out_path / EMPACK_ENV_META, tarfile=out_path / outname 433 | ) 434 | 435 | # copy all the packages to the packages dir 436 | # (this is shared between multiple kernels in the same environment) 437 | for pkg_path in out_path.iterdir(): 438 | if pkg_path.name.endswith(".tar.gz"): 439 | yield dict( 440 | name=f"xeus:{env_name}:copy:{pkg_path.name}", 441 | actions=[(self.copy_one, [pkg_path, packages_dir / pkg_path.name])], 442 | ) 443 | 444 | # copy the empack_env_meta.json 445 | # (this is shared between multiple kernels in the same environment) 446 | yield dict( 447 | name=f"xeus:{env_name}:copy_env_file:{EMPACK_ENV_META}", 448 | actions=[ 449 | ( 450 | self.copy_one, 451 | [ 452 | out_path / EMPACK_ENV_META, 453 | env_dir / EMPACK_ENV_META, 454 | ], 455 | ) 456 | ], 457 | ) 458 | 459 | def copy_jupyterlab_extensions_from_prefix(self, prefix): 460 | federated_extensions = self.env_extensions(Path(prefix) / SHARE_LABEXTENSIONS) 461 | 462 | # Find the federated extensions in the emscripten-env and install them 463 | for pkg_json in federated_extensions: 464 | yield from self.safe_copy_jupyterlab_extension(pkg_json) 465 | 466 | jupyterlite_json = self.manager.output_dir / JUPYTERLITE_JSON 467 | 468 | yield dict( 469 | name=f"patch:xeus:{prefix}", 470 | doc=f"ensure {JUPYTERLITE_JSON} includes the federated_extensions", 471 | file_dep=[*federated_extensions, jupyterlite_json], 472 | actions=[(self.patch_jupyterlite_json, [jupyterlite_json])], 473 | ) 474 | 475 | app_schemas = self.manager.output_dir / "build" / "schemas" 476 | all_federated_json = app_schemas / ALL_FEDERATED_JSON 477 | 478 | if app_schemas.is_dir(): 479 | yield self.task( 480 | name=f"patch:xeus:federated_settings:{prefix}", 481 | doc=f"ensure {ALL_FEDERATED_JSON} includes the settings of federated extensions", 482 | file_dep=[*federated_extensions], 483 | actions=[ 484 | (self.patch_federated_settings, [self.manager, federated_extensions, all_federated_json]) 485 | ], 486 | ) 487 | 488 | def patch_federated_settings(self, manager, lab_extensions, all_federated_json): 489 | """ensure settings from federated extensions are aggregated in a single file""" 490 | federated_settings = [ 491 | setting for p in lab_extensions for setting in self.get_federated_settings(p.parent) 492 | ] 493 | current_json = json.loads(all_federated_json.read_text()) 494 | current_json = current_json + federated_settings 495 | all_federated_json.write_text(json.dumps(current_json), **UTF8) 496 | 497 | def safe_copy_jupyterlab_extension(self, pkg_json): 498 | """Copy a labextension, and overwrite it 499 | if it's already in the output 500 | """ 501 | pkg_path = pkg_json.parent 502 | stem = json.loads(pkg_json.read_text(**UTF8))["name"] 503 | dest = self.output_extensions / stem 504 | file_dep = [ 505 | p 506 | for p in pkg_path.rglob("*") 507 | if not (p.is_dir() or self.is_ignored_sourcemap(p.name)) 508 | ] 509 | 510 | yield dict( 511 | name=f"xeus:copy:ext:{stem}", 512 | file_dep=file_dep, 513 | actions=[(self.copy_one, [pkg_path, dest])], 514 | ) 515 | 516 | def dedupe_federated_extensions(self, config): 517 | if FEDERATED_EXTENSIONS not in config: 518 | return 519 | 520 | named = {} 521 | 522 | # Making sure to dedupe extensions by keeping the most recent ones 523 | for ext in config[FEDERATED_EXTENSIONS]: 524 | if os.path.exists(self.output_extensions / ext["name"] / ext["load"]): 525 | named[ext["name"]] = ext 526 | 527 | config[FEDERATED_EXTENSIONS] = sorted(named.values(), key=lambda x: x["name"]) 528 | -------------------------------------------------------------------------------- /jupyterlite_xeus/constants.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | 4 | EXTENSION_NAME = "xeus" 5 | STATIC_DIR = Path("@jupyterlite") / EXTENSION_NAME / "static" 6 | -------------------------------------------------------------------------------- /jupyterlite_xeus/create_conda_env.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | import sys 3 | from pathlib import Path 4 | from subprocess import run as subprocess_run 5 | import os 6 | 7 | from ._pip import _install_pip_dependencies 8 | 9 | MICROMAMBA_COMMAND = shutil.which("micromamba") 10 | PLATFORM = "emscripten-wasm32" 11 | 12 | 13 | def _extract_specs(env_location, env_data): 14 | specs = [] 15 | pip_dependencies = [] 16 | 17 | # iterate dependencies 18 | for dependency in env_data.get("dependencies", []): 19 | if isinstance(dependency, str): 20 | specs.append(dependency) 21 | elif isinstance(dependency, dict) and "pip" in dependency: 22 | for pip_dependency in dependency["pip"]: 23 | # If it's a local Python package, make its path relative to the environment file 24 | if (env_location / pip_dependency).is_dir(): 25 | pip_dependencies.append((env_location / pip_dependency).resolve()) 26 | else: 27 | pip_dependencies.append(pip_dependency) 28 | 29 | return specs, pip_dependencies 30 | 31 | 32 | def create_conda_env_from_env_file(root_prefix, env_file_content, env_file_location): 33 | # get the name of the environment 34 | env_name = env_file_content.get("name", "xeus-env") 35 | 36 | # get the channels 37 | channels = env_file_content.get( 38 | "channels", ["https://repo.prefix.dev/emscripten-forge-dev", "https://repo.prefix.dev/conda-forge"] 39 | ) 40 | 41 | # get the specs 42 | specs, pip_dependencies = _extract_specs(env_file_location, env_file_content) 43 | 44 | create_conda_env_from_specs( 45 | env_name=env_name, 46 | root_prefix=root_prefix, 47 | specs=specs, 48 | channels=channels, 49 | pip_dependencies=pip_dependencies, 50 | ) 51 | 52 | 53 | def create_conda_env_from_specs( 54 | env_name, 55 | root_prefix, 56 | specs, 57 | channels, 58 | pip_dependencies=None, 59 | ): 60 | _create_conda_env_from_specs_impl( 61 | env_name=env_name, 62 | root_prefix=root_prefix, 63 | specs=specs, 64 | channels=channels, 65 | ) 66 | if pip_dependencies: 67 | _install_pip_dependencies( 68 | prefix_path=Path(root_prefix) / "envs" / env_name, 69 | dependencies=pip_dependencies, 70 | ) 71 | 72 | 73 | def _create_conda_env_from_specs_impl(env_name, root_prefix, specs, channels): 74 | """Create the emscripten environment with the given specs.""" 75 | prefix_path = Path(root_prefix) / "envs" / env_name 76 | 77 | Path(root_prefix).mkdir(parents=True, exist_ok=True) 78 | 79 | channels_args = [] 80 | for channel in channels: 81 | channels_args.extend(["-c", channel]) 82 | 83 | if not MICROMAMBA_COMMAND: 84 | raise RuntimeError(""" 85 | micromamba is needed for creating the emscripten environment. 86 | Please install it using conda `conda install micromamba -c conda-forge` or 87 | from https://mamba.readthedocs.io/en/latest/installation/micromamba-installation.html 88 | """) 89 | 90 | subprocess_run( 91 | [ 92 | MICROMAMBA_COMMAND, 93 | "create", 94 | "--yes", 95 | "--no-pyc", 96 | "--prefix", 97 | prefix_path, 98 | "--relocate-prefix", 99 | "", 100 | "--root-prefix", 101 | root_prefix, 102 | f"--platform={PLATFORM}", 103 | *channels_args, 104 | *specs, 105 | ], 106 | check=True, 107 | ) 108 | -------------------------------------------------------------------------------- /lerna.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "node_modules/lerna/schemas/lerna-schema.json", 3 | "version": "independent", 4 | "npmClient": "yarn" 5 | } 6 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@jupyterlite/xeus-root", 3 | "version": "4.0.2", 4 | "private": true, 5 | "description": "JupyterLite loader for Xeus kernels", 6 | "keywords": [ 7 | "jupyter", 8 | "jupyterlab", 9 | "jupyterlab-extension" 10 | ], 11 | "homepage": "https://github.com/jupyterlite/xeus", 12 | "bugs": { 13 | "url": "https://github.com/jupyterlite/xeus/issues" 14 | }, 15 | "license": "BSD-3-Clause", 16 | "author": "JupyterLite Contributors", 17 | "workspaces": { 18 | "packages": [ 19 | "packages/*" 20 | ] 21 | }, 22 | "repository": { 23 | "type": "git", 24 | "url": "https://github.com/jupyterlite/xeus.git" 25 | }, 26 | "scripts": { 27 | "build": "lerna run build", 28 | "build:prod": "lerna run build:prod", 29 | "clean": "lerna run clean", 30 | "clean:all": "lerna run clean:all", 31 | "eslint": "jlpm eslint:check --fix", 32 | "eslint:check": "eslint . --cache --ext .ts,.tsx", 33 | "lint": "jlpm prettier && jlpm eslint", 34 | "lint:check": "jlpm prettier:check && jlpm eslint:check", 35 | "prettier": "jlpm prettier:base --write --list-different", 36 | "prettier:base": "prettier \"**/*{.ts,.tsx,.js,.jsx,.css,.json,.md}\"", 37 | "prettier:check": "jlpm prettier:base --check", 38 | "watch": "lerna run watch" 39 | }, 40 | "devDependencies": { 41 | "@typescript-eslint/eslint-plugin": "^6.1.0", 42 | "@typescript-eslint/parser": "^6.1.0", 43 | "eslint": "^8.36.0", 44 | "eslint-config-prettier": "^8.8.0", 45 | "eslint-plugin-prettier": "^5.0.0", 46 | "lerna": "^8.1.9", 47 | "npm-run-all": "^4.1.5", 48 | "prettier": "^3.0.0", 49 | "rimraf": "^5.0.1" 50 | }, 51 | "eslintIgnore": [ 52 | "node_modules", 53 | "dist", 54 | "coverage", 55 | "**/*.d.ts", 56 | "tests", 57 | "**/__tests__", 58 | "ui-tests" 59 | ], 60 | "eslintConfig": { 61 | "extends": [ 62 | "eslint:recommended", 63 | "plugin:@typescript-eslint/eslint-recommended", 64 | "plugin:@typescript-eslint/recommended", 65 | "plugin:prettier/recommended" 66 | ], 67 | "parser": "@typescript-eslint/parser", 68 | "parserOptions": { 69 | "project": "tsconfig.json", 70 | "sourceType": "module" 71 | }, 72 | "plugins": [ 73 | "@typescript-eslint" 74 | ], 75 | "rules": { 76 | "@typescript-eslint/naming-convention": [ 77 | "error", 78 | { 79 | "selector": "interface", 80 | "format": [ 81 | "PascalCase" 82 | ], 83 | "custom": { 84 | "regex": "^I[A-Z]", 85 | "match": true 86 | } 87 | } 88 | ], 89 | "@typescript-eslint/no-unused-vars": [ 90 | "warn", 91 | { 92 | "args": "none" 93 | } 94 | ], 95 | "@typescript-eslint/no-explicit-any": "off", 96 | "@typescript-eslint/no-namespace": "off", 97 | "@typescript-eslint/no-use-before-define": "off", 98 | "@typescript-eslint/quotes": [ 99 | "error", 100 | "single", 101 | { 102 | "avoidEscape": true, 103 | "allowTemplateLiterals": false 104 | } 105 | ], 106 | "curly": [ 107 | "error", 108 | "all" 109 | ], 110 | "eqeqeq": "error", 111 | "prefer-arrow-callback": "error" 112 | } 113 | }, 114 | "prettier": { 115 | "singleQuote": true, 116 | "trailingComma": "none", 117 | "arrowParens": "avoid", 118 | "endOfLine": "auto", 119 | "overrides": [ 120 | { 121 | "files": "package.json", 122 | "options": { 123 | "tabWidth": 4 124 | } 125 | } 126 | ] 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /packages/xeus-core/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@jupyterlite/xeus-core", 3 | "version": "4.0.2", 4 | "description": "JupyterLite Xeus core library", 5 | "keywords": [ 6 | "jupyter", 7 | "jupyterlab" 8 | ], 9 | "homepage": "https://github.com/jupyterlite/xeus", 10 | "bugs": { 11 | "url": "https://github.com/jupyterlite/xeus/issues" 12 | }, 13 | "license": "BSD-3-Clause", 14 | "author": "JupyterLite Contributors", 15 | "files": [ 16 | "lib/**/*.{d.ts,eot,gif,html,jpg,js,js.map,json,png,svg,woff2,ttf}" 17 | ], 18 | "main": "lib/index.js", 19 | "types": "lib/index.d.ts", 20 | "repository": { 21 | "type": "git", 22 | "url": "https://github.com/jupyterlite/xeus.git" 23 | }, 24 | "scripts": { 25 | "build": "jlpm build:lib", 26 | "build:prod": "jlpm clean && jlpm build:lib:prod", 27 | "build:lib": "tsc --sourceMap", 28 | "build:lib:prod": "tsc", 29 | "clean": "jlpm clean:lib", 30 | "clean:lib": "rimraf lib tsconfig.tsbuildinfo", 31 | "clean:all": "jlpm clean:lib", 32 | "watch": "run-p watch:src", 33 | "watch:src": "tsc -w --sourceMap" 34 | }, 35 | "dependencies": { 36 | "@emscripten-forge/mambajs-core": "^0.13.0", 37 | "@jupyterlab/coreutils": "^6.4.2", 38 | "@jupyterlab/services": "^7.4.2", 39 | "@jupyterlite/contents": "^0.6.0", 40 | "@jupyterlite/kernel": "^0.6.0", 41 | "@jupyterlite/server": "^0.6.0", 42 | "@lumino/coreutils": "^2", 43 | "@lumino/signaling": "^2" 44 | }, 45 | "devDependencies": { 46 | "@types/json-schema": "^7.0.11", 47 | "@types/react": "^18.0.26", 48 | "@types/react-addons-linked-state-mixin": "^0.14.22", 49 | "npm-run-all": "^4.1.5", 50 | "rimraf": "^5.0.1", 51 | "source-map-loader": "^1.0.2", 52 | "ts-loader": "^9.2.6", 53 | "typescript": "^5.5", 54 | "webpack": "^5.87.0", 55 | "yjs": "^13.5.0" 56 | }, 57 | "publishConfig": { 58 | "access": "public" 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /packages/xeus-core/src/index.ts: -------------------------------------------------------------------------------- 1 | export * from './interfaces'; 2 | export * from './web.worker.kernel.base'; 3 | export * from './worker.base'; 4 | -------------------------------------------------------------------------------- /packages/xeus-core/src/interfaces.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Jupyter Development Team. 2 | // Distributed under the terms of the Modified BSD License. 3 | 4 | /** 5 | * Definitions for the Xeus kernel. 6 | */ 7 | 8 | import type { KernelMessage } from '@jupyterlab/services'; 9 | 10 | import { 11 | TDriveMethod, 12 | TDriveRequest, 13 | TDriveResponse 14 | } from '@jupyterlite/contents'; 15 | 16 | import { IWorkerKernel } from '@jupyterlite/kernel'; 17 | 18 | /** 19 | * An interface for Xeus workers. 20 | */ 21 | export interface IXeusWorkerKernel extends IWorkerKernel { 22 | /** 23 | * Handle any lazy initialization activities. 24 | */ 25 | initialize(options: IXeusWorkerKernel.IOptions): Promise; 26 | 27 | /** 28 | * Process drive request 29 | * @param data 30 | */ 31 | processDriveRequest( 32 | data: TDriveRequest 33 | ): TDriveResponse; 34 | 35 | /** 36 | * Process a message sent from the main thread to the worker. 37 | * @param msg 38 | */ 39 | processMessage(msg: any): void; 40 | 41 | /** 42 | * Process stdin request, blocking until the reply is received. 43 | * This is sync for the web worker, async for the UI thread. 44 | * @param inputRequest 45 | */ 46 | processStdinRequest( 47 | inputRequest: KernelMessage.IInputRequestMsg 48 | ): KernelMessage.IInputReplyMsg; 49 | 50 | /** 51 | * Process worker message 52 | * @param msg 53 | */ 54 | processWorkerMessage(msg: any): void; 55 | 56 | /** 57 | * Whether the kernel is ready. 58 | * @returns a promise that resolves when the kernel is ready. 59 | */ 60 | ready(): Promise; 61 | 62 | /** 63 | * Mount a drive 64 | * @param driveName The name of the drive 65 | * @param mountpoint The mountpoint of the drive 66 | * @param baseUrl The base URL of the server 67 | * @param browsingContextId The current page id 68 | */ 69 | mount( 70 | driveName: string, 71 | mountpoint: string, 72 | baseUrl: string, 73 | browsingContextId: string 74 | ): Promise; 75 | 76 | /** 77 | * Change the current working directory 78 | * @param path The path to change to 79 | */ 80 | cd(path: string): Promise; 81 | 82 | /** 83 | * Check if a path is a directory 84 | * @param path The path to check 85 | */ 86 | isDir(path: string): Promise; 87 | } 88 | 89 | /** 90 | * An namespace for Xeus workers. 91 | */ 92 | export namespace IXeusWorkerKernel { 93 | /** 94 | * Initialization options for a worker. 95 | */ 96 | export interface IOptions extends IWorkerKernel.IOptions { 97 | baseUrl: string; 98 | kernelId: string; 99 | kernelSpec: any; 100 | mountDrive: boolean; 101 | browsingContextId: string; 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /packages/xeus-core/src/web.worker.kernel.base.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) JupyterLite Contributors 2 | // Distributed under the terms of the Modified BSD License. 3 | 4 | import type { Remote } from 'comlink'; 5 | 6 | import { ISignal, Signal } from '@lumino/signaling'; 7 | import { PromiseDelegate } from '@lumino/coreutils'; 8 | 9 | import { PageConfig } from '@jupyterlab/coreutils'; 10 | import { Contents, KernelMessage } from '@jupyterlab/services'; 11 | 12 | import { IKernel } from '@jupyterlite/kernel'; 13 | import { DriveContentsProcessor } from '@jupyterlite/contents'; 14 | 15 | import { IXeusWorkerKernel } from './interfaces'; 16 | 17 | export abstract class WebWorkerKernelBase implements IKernel { 18 | /** 19 | * Instantiate a new WebWorkerKernelBase 20 | * 21 | * @param options The instantiation options for a new WebWorkerKernelBase 22 | */ 23 | constructor(options: WebWorkerKernelBase.IOptions) { 24 | const { id, name, sendMessage, location, contentsManager } = options; 25 | this._id = id; 26 | this._name = name; 27 | this._location = location; 28 | this.contentsManager = contentsManager; 29 | this.contentsProcessor = new DriveContentsProcessor({ 30 | contentsManager: this.contentsManager 31 | }); 32 | this.sendMessage = sendMessage; 33 | this.worker = this.initWorker(options); 34 | this.remoteKernel = this.createRemote(options); 35 | this.initRemote(options).then(this._ready.resolve.bind(this._ready)); 36 | this.initFileSystem(options); 37 | } 38 | 39 | /** 40 | * Load the worker. 41 | */ 42 | abstract initWorker(options: WebWorkerKernelBase.IOptions): Worker; 43 | 44 | /** 45 | * Create the remote kernel. 46 | */ 47 | abstract createRemote( 48 | options: WebWorkerKernelBase.IOptions 49 | ): IXeusWorkerKernel | Remote; 50 | 51 | /** 52 | * Initialize the remote kernel 53 | * @param options 54 | */ 55 | protected async initRemote(options: WebWorkerKernelBase.IOptions) { 56 | return this.remoteKernel.initialize({ 57 | baseUrl: PageConfig.getBaseUrl(), 58 | kernelId: this.id, 59 | mountDrive: options.mountDrive, 60 | kernelSpec: options.kernelSpec, 61 | browsingContextId: options.browsingContextId 62 | }); 63 | } 64 | 65 | async handleMessage(msg: KernelMessage.IMessage): Promise { 66 | this._parent = msg; 67 | this._parentHeader = msg.header; 68 | await this._sendMessageToWorker(msg); 69 | } 70 | 71 | protected processWorkerMessage(msg: any) { 72 | if (!msg.header) { 73 | // Custom msg bypassing comlink/coincident protocol 74 | if (msg._stream) { 75 | const parentHeaderValue = this.parentHeader; 76 | const { name, text } = msg._stream; 77 | if (name === 'stderr') { 78 | const errorMessage = 79 | KernelMessage.createMessage({ 80 | msgType: 'execute_reply', 81 | channel: 'shell', 82 | parentHeader: 83 | parentHeaderValue as KernelMessage.IHeader<'execute_request'>, 84 | session: parentHeaderValue?.session ?? '', 85 | content: { 86 | execution_count: msg._stream.executionCount, 87 | status: 'error', 88 | ename: msg._stream.ename, 89 | evalue: msg._stream.evalue, 90 | traceback: msg._stream.traceback.join('') 91 | } 92 | }); 93 | this.sendMessage(errorMessage); 94 | } 95 | 96 | const message = KernelMessage.createMessage({ 97 | channel: 'iopub', 98 | msgType: 'stream', 99 | session: parentHeaderValue?.session ?? '', 100 | parentHeader: parentHeaderValue, 101 | content: { 102 | name, 103 | text 104 | } 105 | }); 106 | 107 | this.sendMessage(message); 108 | return; 109 | } else { 110 | return; 111 | } 112 | } 113 | 114 | msg.header.session = this.parentHeader?.session ?? ''; 115 | msg.session = this.parentHeader?.session ?? ''; 116 | this.sendMessage(msg); 117 | 118 | // resolve promise 119 | if ( 120 | msg.header.msg_type === 'status' && 121 | msg.content.execution_state === 'idle' 122 | ) { 123 | this.executeDelegate.resolve(); 124 | } 125 | } 126 | 127 | private async _sendMessageToWorker(msg: any): Promise { 128 | if (msg.header.msg_type === 'input_reply') { 129 | this.inputDelegate.resolve(msg); 130 | } else { 131 | this.executeDelegate = new PromiseDelegate(); 132 | await this.remoteKernel.processMessage({ msg, parent: this.parent }); 133 | return await this.executeDelegate.promise; 134 | } 135 | } 136 | 137 | /** 138 | * Get the last parent header 139 | */ 140 | get parentHeader(): 141 | | KernelMessage.IHeader 142 | | undefined { 143 | return this._parentHeader; 144 | } 145 | 146 | /** 147 | * Get the last parent message (mimick ipykernel's get_parent) 148 | */ 149 | get parent(): KernelMessage.IMessage | undefined { 150 | return this._parent; 151 | } 152 | 153 | /** 154 | * Get the kernel location 155 | */ 156 | get location(): string { 157 | return this._location; 158 | } 159 | 160 | /** 161 | * A promise that is fulfilled when the kernel is ready. 162 | */ 163 | get ready(): Promise { 164 | return this._ready.promise; 165 | } 166 | 167 | /** 168 | * Return whether the kernel is disposed. 169 | */ 170 | get isDisposed(): boolean { 171 | return this._isDisposed; 172 | } 173 | 174 | /** 175 | * A signal emitted when the kernel is disposed. 176 | */ 177 | get disposed(): ISignal { 178 | return this._disposed; 179 | } 180 | 181 | /** 182 | * Dispose the kernel. 183 | */ 184 | dispose(): void { 185 | if (this.isDisposed) { 186 | return; 187 | } 188 | this.worker.terminate(); 189 | (this.worker as any) = null; 190 | (this.remoteKernel as any) = null; 191 | this._isDisposed = true; 192 | this._disposed.emit(void 0); 193 | } 194 | 195 | /** 196 | * Get the kernel id 197 | */ 198 | get id(): string { 199 | return this._id; 200 | } 201 | 202 | /** 203 | * Get the name of the kernel 204 | */ 205 | get name(): string { 206 | return this._name; 207 | } 208 | 209 | private async initFileSystem(options: WebWorkerKernelBase.IOptions) { 210 | let driveName: string; 211 | let localPath: string; 212 | 213 | if (options.location.includes(':')) { 214 | const parts = options.location.split(':'); 215 | driveName = parts[0]; 216 | localPath = parts[1]; 217 | } else { 218 | driveName = ''; 219 | localPath = options.location; 220 | } 221 | 222 | await this.remoteKernel.ready(); 223 | 224 | await this.remoteKernel.mount( 225 | driveName, 226 | '/drive', 227 | PageConfig.getBaseUrl(), 228 | options.browsingContextId 229 | ); 230 | 231 | if (await this.remoteKernel.isDir('/files')) { 232 | await this.remoteKernel.cd('/files'); 233 | } else { 234 | await this.remoteKernel.cd(`/drive/${localPath}`); 235 | } 236 | } 237 | 238 | private _id: string; 239 | private _name: string; 240 | private _location: string; 241 | private _isDisposed = false; 242 | private _disposed = new Signal(this); 243 | 244 | protected contentsManager: Contents.IManager; 245 | protected contentsProcessor: DriveContentsProcessor; 246 | protected remoteKernel: IXeusWorkerKernel | Remote; 247 | protected worker: Worker; 248 | protected sendMessage: IKernel.SendMessage; 249 | protected executeDelegate = new PromiseDelegate(); 250 | protected inputDelegate = new PromiseDelegate(); 251 | 252 | private _parentHeader: 253 | | KernelMessage.IHeader 254 | | undefined = undefined; 255 | private _parent: KernelMessage.IMessage | undefined = undefined; 256 | private _ready = new PromiseDelegate(); 257 | } 258 | 259 | /** 260 | * A namespace for WebWorkerKernelBase statics. 261 | */ 262 | export namespace WebWorkerKernelBase { 263 | /** 264 | * The instantiation options for a Pyodide kernel 265 | */ 266 | export interface IOptions extends IKernel.IOptions { 267 | contentsManager: Contents.IManager; 268 | mountDrive: boolean; 269 | kernelSpec: any; 270 | browsingContextId: string; 271 | } 272 | } 273 | -------------------------------------------------------------------------------- /packages/xeus-core/src/worker.base.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Thorsten Beier 2 | // Copyright (c) JupyterLite Contributors 3 | // Distributed under the terms of the Modified BSD License. 4 | 5 | import { IXeusWorkerKernel } from './interfaces'; 6 | import { 7 | waitRunDependencies, 8 | ILogger, 9 | parse 10 | } from '@emscripten-forge/mambajs-core'; 11 | 12 | declare function createXeusModule(options: any): any; 13 | 14 | const STREAM = { log: 'stdout', warn: 'stdout', error: 'stderr' }; 15 | 16 | export class XeusWorkerLoggerBase implements ILogger { 17 | constructor(kernelId: string) { 18 | this._id = kernelId; 19 | this._channel = new BroadcastChannel('/xeus-kernel-logs-broadcast'); 20 | } 21 | 22 | log(...msg: any[]): void { 23 | postMessage({ 24 | _stream: { 25 | name: STREAM['log'], 26 | text: msg.join(' ') + '\n' 27 | } 28 | }); 29 | 30 | this._channel.postMessage({ 31 | kernelId: this._id, 32 | payload: { type: 'text', level: 'info', data: msg.join(' ') } 33 | }); 34 | } 35 | 36 | warn(...msg: any[]): void { 37 | postMessage({ 38 | _stream: { 39 | name: STREAM['warn'], 40 | text: '\x1b[38;5;208m' + msg.join(' ') + '\x1b[0m' + '\n' 41 | } 42 | }); 43 | 44 | this._channel.postMessage({ 45 | kernelId: this._id, 46 | payload: { type: 'text', level: 'warning', data: msg.join(' ') } 47 | }); 48 | } 49 | 50 | error(...msg: any[]): void { 51 | postMessage({ 52 | _stream: { 53 | name: STREAM['error'], 54 | evalue: msg.join(''), 55 | traceback: [], 56 | executionCount: this.executionCount, 57 | text: msg.join('') 58 | } 59 | }); 60 | this._channel.postMessage({ 61 | kernelId: this._id, 62 | payload: { type: 'text', level: 'critical', data: msg.join(' ') } 63 | }); 64 | } 65 | 66 | private _id: string; 67 | private _channel: BroadcastChannel; 68 | executionCount: number = 0; 69 | } 70 | 71 | /** 72 | * The base class for the worker kernel 73 | * 74 | * Meant to be extended in order to load kernels from other sources, implement custom magics etc. 75 | */ 76 | export abstract class XeusRemoteKernelBase { 77 | constructor(options: XeusRemoteKernelBase.IOptions = {}) { 78 | this._ready = new Promise(resolve => { 79 | this.setKernelReady = resolve; 80 | }); 81 | } 82 | 83 | async ready(): Promise { 84 | return await this._ready; 85 | } 86 | 87 | async cd(path: string): Promise { 88 | if (!path || !this.Module.FS) { 89 | return; 90 | } 91 | 92 | this.Module.FS.chdir(path); 93 | } 94 | 95 | async isDir(path: string): Promise { 96 | try { 97 | const lookup = this.Module.FS.lookupPath(path); 98 | return this.Module.FS.isDir(lookup.node.mode); 99 | } catch (e) { 100 | return false; 101 | } 102 | } 103 | 104 | async processMessage(event: any): Promise { 105 | const msg_type = event.msg.header.msg_type; 106 | 107 | await globalThis.ready; 108 | 109 | if ( 110 | globalThis.toplevel_promise !== null && 111 | globalThis.toplevel_promise_py_proxy !== null 112 | ) { 113 | await globalThis.toplevel_promise; 114 | globalThis.toplevel_promise_py_proxy.delete(); 115 | globalThis.toplevel_promise_py_proxy = null; 116 | globalThis.toplevel_promise = null; 117 | } 118 | 119 | if (msg_type === 'input_reply') { 120 | // Should never be called as input_reply messages are handled by get_stdin 121 | // via SharedArrayBuffer or service worker. 122 | } else if (msg_type === 'execute_request') { 123 | this.logger.executionCount += 1; 124 | event.msg.content.code = await this.processMagics(event.msg.content.code); 125 | this.xserver.notify_listener(event.msg); 126 | } else { 127 | this.xserver.notify_listener(event.msg); 128 | } 129 | } 130 | 131 | protected get Module() { 132 | return globalThis.Module; 133 | } 134 | 135 | protected set Module(value: any) { 136 | globalThis.Module = value; 137 | } 138 | 139 | async initialize(options: IXeusWorkerKernel.IOptions): Promise { 140 | const { baseUrl, browsingContextId, kernelSpec } = options; 141 | 142 | this.logger = this.initializeLogger(options); 143 | 144 | // when a toplevel cell uses an await, the cell is implicitly 145 | // wrapped in a async function. Since the webloop - eventloop 146 | // implementation does not support `eventloop.run_until_complete(f)` 147 | // we need to convert the toplevel future in a javascript Promise 148 | // this `toplevel` promise is then awaited before we 149 | // execute the next cell. After the promise is awaited we need 150 | // to do some cleanup and delete the python proxy 151 | // (ie a js-wrapped python object) to avoid memory leaks 152 | globalThis.toplevel_promise = null; 153 | globalThis.toplevel_promise_py_proxy = null; 154 | 155 | this.Module = await this.initializeModule(options); 156 | this.Module = await createXeusModule(this.Module); 157 | 158 | try { 159 | await waitRunDependencies(this.Module); 160 | 161 | await this.initializeFileSystem(options); 162 | await this.initializeInterpreter(options); 163 | this.initializeStdin(baseUrl, browsingContextId); 164 | 165 | try { 166 | this.xkernel = new this.Module.xkernel(kernelSpec.argv); 167 | } catch (e) { 168 | this.xkernel = new this.Module.xkernel(); 169 | } 170 | this.xserver = this.xkernel.get_server(); 171 | if (!this.xserver) { 172 | this.logger.error('Failed to start kernel!'); 173 | } 174 | this.xkernel.start(); 175 | } catch (e) { 176 | if (typeof e === 'number') { 177 | const msg = this.Module.get_exception_message(e); 178 | this.logger.error(msg); 179 | throw new Error(msg); 180 | } else { 181 | this.logger.error(e); 182 | throw e; 183 | } 184 | } 185 | 186 | this.logger.log('Kernel successfuly started!'); 187 | 188 | this.setKernelReady(); 189 | } 190 | 191 | protected initializeLogger( 192 | options: IXeusWorkerKernel.IOptions 193 | ): XeusWorkerLoggerBase { 194 | return new XeusWorkerLoggerBase(options.kernelId); 195 | } 196 | 197 | /** 198 | * Initialize the emscripten Module 199 | * @param options 200 | */ 201 | protected abstract initializeModule(options: IXeusWorkerKernel.IOptions): any; 202 | 203 | /** 204 | * Initialize the this.Module.FS as needed 205 | * @param options 206 | */ 207 | protected abstract initializeFileSystem( 208 | options: IXeusWorkerKernel.IOptions 209 | ): Promise; 210 | 211 | /** 212 | * Initialize the interpreter if needed 213 | * @param options 214 | */ 215 | protected abstract initializeInterpreter( 216 | options: IXeusWorkerKernel.IOptions 217 | ): Promise; 218 | 219 | /** 220 | * Add get_stdin function to globalThis that takes an input_request message, blocks 221 | * until the corresponding input_reply is received and returns the input_reply message. 222 | * If an error occurs return an object of the form { error: "Error explanation" } 223 | * This function is called by xeus-lite's get_stdin. 224 | */ 225 | protected abstract initializeStdin( 226 | baseUrl: string, 227 | browsingContextId: string 228 | ): void; 229 | 230 | /** 231 | * Setup custom Emscripten FileSystem 232 | */ 233 | abstract mount( 234 | driveName: string, 235 | mountpoint: string, 236 | baseUrl: string, 237 | browsingContextId: string 238 | ): Promise; 239 | 240 | /** 241 | * Implements dynamic installation of packages 242 | */ 243 | protected abstract install( 244 | channels: string[], 245 | specs: string[], 246 | pipSpecs: string[] 247 | ): Promise; 248 | 249 | /** 250 | * Implements dynamic installation of packages 251 | */ 252 | protected abstract listInstalledPackages(): Promise; 253 | 254 | /** 255 | * Process magics prior to executing code 256 | * @returns the runnable code without magics 257 | */ 258 | protected async processMagics(code: string) { 259 | const { commands, run } = parse(code); 260 | for (const command of commands) { 261 | switch (command.type) { 262 | case 'install': 263 | if (command.data) { 264 | const { channels, specs, pipSpecs } = command.data; 265 | await this.install( 266 | channels, 267 | specs as string[], 268 | pipSpecs as string[] 269 | ); 270 | } 271 | break; 272 | case 'list': 273 | await this.listInstalledPackages(); 274 | break; 275 | default: 276 | break; 277 | } 278 | } 279 | return run; 280 | } 281 | 282 | protected xkernel: any; 283 | protected xserver: any; 284 | 285 | protected setKernelReady: (value: void) => void; 286 | 287 | private _ready: Promise; 288 | 289 | protected logger: XeusWorkerLoggerBase; 290 | } 291 | 292 | export namespace XeusRemoteKernelBase { 293 | export interface IOptions {} 294 | } 295 | -------------------------------------------------------------------------------- /packages/xeus-core/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "lib", 5 | "strictPropertyInitialization": false, 6 | "rootDir": "src" 7 | }, 8 | "include": ["src/**/*"] 9 | } 10 | -------------------------------------------------------------------------------- /packages/xeus-extension/lab.webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const CopyPlugin = require('copy-webpack-plugin'); 3 | 4 | const wasmPath = [ 5 | __dirname, 6 | '..', 7 | '..', 8 | 'node_modules', 9 | '@emscripten-forge', 10 | 'mambajs-core', 11 | 'lib', 12 | '*.wasm' 13 | ]; 14 | const staticPath = [ 15 | __dirname, 16 | '..', 17 | '..', 18 | 'jupyterlite_xeus', 19 | 'labextension', 20 | 'static', 21 | '[name].wasm' 22 | ]; 23 | 24 | module.exports = { 25 | plugins: [ 26 | new CopyPlugin({ 27 | patterns: [ 28 | { 29 | from: path.resolve(...wasmPath), 30 | to: path.join(...staticPath) 31 | } 32 | ] 33 | }) 34 | ] 35 | }; 36 | -------------------------------------------------------------------------------- /packages/xeus-extension/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@jupyterlite/xeus-extension", 3 | "version": "4.0.2", 4 | "description": "JupyterLite loader for Xeus kernels", 5 | "keywords": [ 6 | "jupyter", 7 | "jupyterlab", 8 | "jupyterlab-extension" 9 | ], 10 | "homepage": "https://github.com/jupyterlite/xeus", 11 | "bugs": { 12 | "url": "https://github.com/jupyterlite/xeus/issues" 13 | }, 14 | "license": "BSD-3-Clause", 15 | "author": "JupyterLite Contributors", 16 | "files": [ 17 | "lib/**/*.{d.ts,eot,gif,html,jpg,js,js.map,json,png,svg,woff2,ttf}", 18 | "style/**/*.{css,js,eot,gif,html,jpg,json,png,svg,woff2,ttf}", 19 | "schema/*.json" 20 | ], 21 | "main": "lib/index.js", 22 | "types": "lib/index.d.ts", 23 | "style": "style/index.css", 24 | "repository": { 25 | "type": "git", 26 | "url": "https://github.com/jupyterlite/xeus.git" 27 | }, 28 | "scripts": { 29 | "build": "jlpm build:lib && jlpm build:labextension:dev", 30 | "build:prod": "jlpm clean && jlpm build:lib:prod && jlpm build:labextension", 31 | "build:labextension": "jupyter labextension build .", 32 | "build:labextension:dev": "jupyter labextension build --development True .", 33 | "build:lib": "tsc --sourceMap", 34 | "build:lib:prod": "tsc", 35 | "clean": "jlpm clean:lib", 36 | "clean:lib": "rimraf lib tsconfig.tsbuildinfo", 37 | "clean:labextension": "rimraf jupyterlite_xeus/labextension jupyterlite_xeus/_version.py", 38 | "clean:all": "jlpm clean:lib && jlpm clean:labextension", 39 | "watch": "run-p watch:src watch:labextension", 40 | "watch:src": "tsc -w --sourceMap", 41 | "watch:labextension": "jupyter labextension watch ." 42 | }, 43 | "dependencies": { 44 | "@jupyterlab/application": "^4.4.2", 45 | "@jupyterlab/coreutils": "^6.4.2", 46 | "@jupyterlab/logconsole": "^4.4.2", 47 | "@jupyterlab/notebook": "^4.4.2", 48 | "@jupyterlite/contents": "^0.6.0", 49 | "@jupyterlite/kernel": "^0.6.0", 50 | "@jupyterlite/server": "^0.6.0", 51 | "@jupyterlite/xeus": "^4.0.2", 52 | "@lumino/coreutils": "^2" 53 | }, 54 | "devDependencies": { 55 | "@jupyterlab/builder": "^4.4.2", 56 | "@types/json-schema": "^7.0.11", 57 | "@types/react": "^18.0.26", 58 | "@types/react-addons-linked-state-mixin": "^0.14.22", 59 | "copy-webpack-plugin": "^12.0.2", 60 | "css-loader": "^6.7.1", 61 | "npm-run-all": "^4.1.5", 62 | "rimraf": "^5.0.1", 63 | "source-map-loader": "^1.0.2", 64 | "ts-loader": "^9.2.6", 65 | "typescript": "^5.5", 66 | "webpack": "^5.87.0", 67 | "yjs": "^13.5.0" 68 | }, 69 | "publishConfig": { 70 | "access": "public" 71 | }, 72 | "sideEffects": [ 73 | "style/*.css", 74 | "style/index.js" 75 | ], 76 | "styleModule": "style/index.js", 77 | "jupyterlab": { 78 | "extension": true, 79 | "schemaDir": "schema", 80 | "outputDir": "../../jupyterlite_xeus/labextension", 81 | "webpackConfig": "lab.webpack.config.js", 82 | "sharedPackages": { 83 | "@jupyterlite/kernel": { 84 | "bundled": false, 85 | "singleton": true 86 | }, 87 | "@jupyterlite/server": { 88 | "bundled": false, 89 | "singleton": true 90 | }, 91 | "@jupyterlite/contents": { 92 | "bundled": false, 93 | "singleton": true 94 | } 95 | } 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /packages/xeus-extension/schema/xeus-kernel-status.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Xeus Kernels Settings", 3 | "description": "Xeus Kernels Settings.", 4 | "jupyter.lab.toolbars": { 5 | "Notebook": [{ "name": "xeusKernelLogs", "rank": 1002.5 }] 6 | }, 7 | "properties": {}, 8 | "additionalProperties": false, 9 | "type": "object" 10 | } 11 | -------------------------------------------------------------------------------- /packages/xeus-extension/src/index.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Thorsten Beier 2 | // Copyright (c) JupyterLite Contributors 3 | // Distributed under the terms of the Modified BSD License. 4 | 5 | import { 6 | JupyterFrontEndPlugin, 7 | JupyterFrontEnd 8 | } from '@jupyterlab/application'; 9 | import { ILoggerRegistry, ILogPayload } from '@jupyterlab/logconsole'; 10 | import { PageConfig, URLExt } from '@jupyterlab/coreutils'; 11 | 12 | import { IServiceWorkerManager } from '@jupyterlite/server'; 13 | import { IKernel, IKernelSpecs } from '@jupyterlite/kernel'; 14 | 15 | import { WebWorkerKernel } from '@jupyterlite/xeus'; 16 | 17 | import { IEmpackEnvMetaFile } from './tokens'; 18 | 19 | /** 20 | * Interface for items in the kernel list (kernels.json file), created in XeusAddon. 21 | */ 22 | interface IKernelListItem { 23 | env_name: string; 24 | kernel: string; 25 | } 26 | 27 | /** 28 | * Fetches JSON data from the specified URL asynchronously. 29 | * 30 | * This function constructs the full URL using the base URL from the PageConfig and 31 | * the provided relative URL. It then performs a GET request using the Fetch API 32 | * and returns the parsed JSON data. 33 | * 34 | * @param {string} url - The relative URL to fetch the JSON data from. 35 | * @returns {Promise} - A promise that resolves to the parsed JSON data. 36 | * @throws {Error} - Throws an error if the HTTP request fails. 37 | * 38 | */ 39 | async function getJson(url: string) { 40 | const jsonUrl = URLExt.join(PageConfig.getBaseUrl(), url); 41 | const response = await fetch(jsonUrl, { method: 'GET' }); 42 | 43 | if (!response.ok) { 44 | throw new Error(`HTTP error! status: ${response.status}`); 45 | } 46 | 47 | const data = await response.json(); 48 | return data; 49 | } 50 | 51 | const kernelPlugin: JupyterFrontEndPlugin = { 52 | id: '@jupyterlite/xeus-kernel:register', 53 | autoStart: true, 54 | requires: [IKernelSpecs], 55 | optional: [IServiceWorkerManager, IEmpackEnvMetaFile, ILoggerRegistry], 56 | activate: async ( 57 | app: JupyterFrontEnd, 58 | kernelspecs: IKernelSpecs, 59 | serviceWorker?: IServiceWorkerManager, 60 | empackEnvMetaFile?: IEmpackEnvMetaFile, 61 | loggerRegistry?: ILoggerRegistry 62 | ) => { 63 | // Fetch kernel list 64 | let kernelList: IKernelListItem[] = []; 65 | try { 66 | kernelList = await getJson('xeus/kernels.json'); 67 | } catch (err) { 68 | console.log(`Could not fetch xeus/kernels.json: ${err}`); 69 | throw err; 70 | } 71 | const contentsManager = app.serviceManager.contents; 72 | 73 | const kernelNames = kernelList.map(item => item.kernel); 74 | const duplicateNames = kernelNames.filter( 75 | (item, index) => kernelNames.indexOf(item) !== index 76 | ); 77 | 78 | for (const kernelItem of kernelList) { 79 | const { env_name, kernel } = kernelItem; 80 | // Fetch kernel spec 81 | const kernelspec = await getJson( 82 | `xeus/${env_name}/${kernel}/kernel.json` 83 | ); 84 | kernelspec.name = kernel; 85 | kernelspec.dir = kernel; 86 | kernelspec.envName = env_name; 87 | 88 | if (duplicateNames.includes(kernel)) { 89 | // Ensure kernelspec.name and display_name are unique. 90 | kernelspec.name = `${kernel} (${env_name})`; 91 | kernelspec.display_name = `${kernelspec.display_name} [${env_name}]`; 92 | } 93 | 94 | for (const [key, value] of Object.entries(kernelspec.resources)) { 95 | kernelspec.resources[key] = URLExt.join( 96 | PageConfig.getBaseUrl(), 97 | value as string 98 | ); 99 | } 100 | kernelspecs.register({ 101 | spec: kernelspec, 102 | create: async (options: IKernel.IOptions): Promise => { 103 | // If kernelspec.name contains a space then the actual name of the executable 104 | // is only the part before the space. 105 | const index = kernelspec.name.indexOf(' '); 106 | if (index > 0) { 107 | kernelspec.name = kernelspec.name.slice(0, index); 108 | } 109 | 110 | const mountDrive = !!(serviceWorker?.enabled || crossOriginIsolated); 111 | if (mountDrive) { 112 | console.info( 113 | `${kernelspec.name} contents will be synced with Jupyter Contents` 114 | ); 115 | } else { 116 | console.warn( 117 | `${kernelspec.name} contents will NOT be synced with Jupyter Contents` 118 | ); 119 | } 120 | const link = empackEnvMetaFile 121 | ? await empackEnvMetaFile.getLink(kernelspec) 122 | : ''; 123 | 124 | return new WebWorkerKernel({ 125 | ...options, 126 | contentsManager, 127 | mountDrive, 128 | kernelSpec: kernelspec, 129 | empackEnvMetaLink: link, 130 | browsingContextId: serviceWorker?.browsingContextId || '' 131 | }); 132 | } 133 | }); 134 | } 135 | 136 | // @ts-expect-error: refreshSpecs() is not doing what it says it does, so we don't use it 137 | await app.serviceManager.kernelspecs._specsChanged.emit( 138 | app.serviceManager.kernelspecs.specs 139 | ); 140 | 141 | // Kernel logs 142 | if (loggerRegistry) { 143 | const channel = new BroadcastChannel('/xeus-kernel-logs-broadcast'); 144 | 145 | channel.onmessage = event => { 146 | const { kernelId, payload } = event.data as { 147 | kernelId: string; 148 | payload: ILogPayload; 149 | }; 150 | 151 | const { sessions } = app.serviceManager; 152 | 153 | // Find the session path that corresponds to the kernel ID 154 | let sessionPath = ''; 155 | for (const session of sessions.running()) { 156 | if (session.kernel?.id === kernelId) { 157 | sessionPath = session.path; 158 | break; 159 | } 160 | } 161 | 162 | const logger = loggerRegistry.getLogger(sessionPath); 163 | logger.log(payload); 164 | }; 165 | } 166 | } 167 | }; 168 | 169 | const empackEnvMetaPlugin: JupyterFrontEndPlugin = { 170 | id: '@jupyterlite/xeus:empack-env-meta', 171 | autoStart: true, 172 | provides: IEmpackEnvMetaFile, 173 | activate: (): IEmpackEnvMetaFile => { 174 | return { 175 | getLink: async (kernelspec: Record) => { 176 | const { envName } = kernelspec; 177 | const kernel_root_url = URLExt.join( 178 | PageConfig.getBaseUrl(), 179 | `xeus/${envName}` 180 | ); 181 | return `${kernel_root_url}`; 182 | } 183 | }; 184 | } 185 | }; 186 | 187 | export default [empackEnvMetaPlugin, kernelPlugin]; 188 | export { IEmpackEnvMetaFile }; 189 | -------------------------------------------------------------------------------- /packages/xeus-extension/src/tokens.ts: -------------------------------------------------------------------------------- 1 | import { Token } from '@lumino/coreutils'; 2 | 3 | export interface IEmpackEnvMetaFile { 4 | /** 5 | * Get empack_env_meta link. 6 | */ 7 | getLink: (kernelspec: Record) => Promise; 8 | } 9 | 10 | export const IEmpackEnvMetaFile = new Token( 11 | '@jupyterlite/xeus:IEmpackEnvMetaFile' 12 | ); 13 | -------------------------------------------------------------------------------- /packages/xeus-extension/style/index.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyterlite/xeus/506c0c818b94b5097c65b76456b01308e5e6248c/packages/xeus-extension/style/index.css -------------------------------------------------------------------------------- /packages/xeus-extension/style/index.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyterlite/xeus/506c0c818b94b5097c65b76456b01308e5e6248c/packages/xeus-extension/style/index.js -------------------------------------------------------------------------------- /packages/xeus-extension/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "lib", 5 | "rootDir": "src" 6 | }, 7 | "include": ["src/**/*"] 8 | } 9 | -------------------------------------------------------------------------------- /packages/xeus/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@jupyterlite/xeus", 3 | "version": "4.0.2", 4 | "description": "JupyterLite Xeus kernels", 5 | "keywords": [ 6 | "jupyter", 7 | "jupyterlab", 8 | "jupyterlab-extension" 9 | ], 10 | "homepage": "https://github.com/jupyterlite/xeus", 11 | "bugs": { 12 | "url": "https://github.com/jupyterlite/xeus/issues" 13 | }, 14 | "license": "BSD-3-Clause", 15 | "author": "JupyterLite Contributors", 16 | "files": [ 17 | "lib/**/*.{d.ts,eot,gif,html,jpg,js,js.map,json,png,svg,woff2,ttf}" 18 | ], 19 | "main": "lib/index.js", 20 | "types": "lib/index.d.ts", 21 | "repository": { 22 | "type": "git", 23 | "url": "https://github.com/jupyterlite/xeus.git" 24 | }, 25 | "scripts": { 26 | "build": "jlpm build:lib && jlpm build:worker", 27 | "build:prod": "jlpm clean && jlpm build:lib:prod && jlpm build:worker:prod", 28 | "build:worker": "webpack --config worker.webpack.config.js --mode=development", 29 | "build:worker:prod": "webpack --config worker.webpack.config.js --mode=production", 30 | "build:lib": "tsc --sourceMap", 31 | "build:lib:prod": "tsc", 32 | "clean": "jlpm clean:lib", 33 | "clean:lib": "rimraf lib tsconfig.tsbuildinfo", 34 | "clean:all": "jlpm clean:lib", 35 | "watch": "run-p watch:src", 36 | "watch:src": "tsc -w --sourceMap" 37 | }, 38 | "dependencies": { 39 | "@emscripten-forge/mambajs": "^0.13.1", 40 | "@jupyterlab/coreutils": "^6.4.2", 41 | "@jupyterlab/services": "^7.4.2", 42 | "@jupyterlite/contents": "^0.6.0", 43 | "@jupyterlite/kernel": "^0.6.0", 44 | "@jupyterlite/server": "^0.6.0", 45 | "@jupyterlite/xeus-core": "^4.0.2", 46 | "@lumino/coreutils": "^2", 47 | "@lumino/signaling": "^2", 48 | "coincident": "^1.2.3", 49 | "comlink": "^4.4.1" 50 | }, 51 | "devDependencies": { 52 | "@types/json-schema": "^7.0.11", 53 | "@types/react": "^18.0.26", 54 | "@types/react-addons-linked-state-mixin": "^0.14.22", 55 | "npm-run-all": "^4.1.5", 56 | "rimraf": "^5.0.1", 57 | "source-map-loader": "^1.0.2", 58 | "ts-loader": "^9.2.6", 59 | "typescript": "^5.5", 60 | "webpack": "^5.87.0", 61 | "yjs": "^13.5.0" 62 | }, 63 | "publishConfig": { 64 | "access": "public" 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /packages/xeus/src/coincident.worker.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Thorsten Beier 2 | // Copyright (c) JupyterLite Contributors 3 | // Distributed under the terms of the Modified BSD License. 4 | 5 | import coincident from 'coincident'; 6 | 7 | import type { KernelMessage } from '@jupyterlab/services'; 8 | 9 | import { 10 | ContentsAPI, 11 | DriveFS, 12 | TDriveRequest, 13 | TDriveMethod, 14 | TDriveResponse 15 | } from '@jupyterlite/contents'; 16 | 17 | import { IXeusWorkerKernel } from '@jupyterlite/xeus-core'; 18 | import { EmpackedXeusRemoteKernel } from './worker'; 19 | 20 | const workerAPI = coincident(self) as IXeusWorkerKernel; 21 | 22 | /** 23 | * An Emscripten-compatible synchronous Contents API using shared array buffers. 24 | */ 25 | export class SharedBufferContentsAPI extends ContentsAPI { 26 | request(data: TDriveRequest): TDriveResponse { 27 | return workerAPI.processDriveRequest(data); 28 | } 29 | } 30 | 31 | /** 32 | * A custom drive implementation which uses shared array buffers (via coincident) if available 33 | */ 34 | class XeusDriveFS extends DriveFS { 35 | createAPI(options: DriveFS.IOptions): ContentsAPI { 36 | return new SharedBufferContentsAPI(options); 37 | } 38 | } 39 | 40 | export class XeusCoincidentKernel extends EmpackedXeusRemoteKernel { 41 | /** 42 | * Setup custom Emscripten FileSystem 43 | */ 44 | async mount( 45 | driveName: string, 46 | mountpoint: string, 47 | baseUrl: string, 48 | browsingContextId: string 49 | ): Promise { 50 | const { FS, PATH, ERRNO_CODES } = globalThis.Module; 51 | 52 | if (!FS) { 53 | return; 54 | } 55 | 56 | const drive = new XeusDriveFS({ 57 | FS, 58 | PATH, 59 | ERRNO_CODES, 60 | baseUrl, 61 | driveName, 62 | mountpoint, 63 | browsingContextId 64 | }); 65 | 66 | FS.mkdir(mountpoint); 67 | FS.mount(drive, {}, mountpoint); 68 | FS.chdir(mountpoint); 69 | } 70 | 71 | protected initializeStdin(baseUrl: string, browsingContextId: string): void { 72 | globalThis.get_stdin = ( 73 | inputRequest: KernelMessage.IInputRequestMsg 74 | ): KernelMessage.IInputReplyMsg => 75 | workerAPI.processStdinRequest(inputRequest); 76 | } 77 | } 78 | 79 | const worker = new XeusCoincidentKernel(); 80 | 81 | workerAPI.initialize = worker.initialize.bind(worker); 82 | workerAPI.mount = worker.mount.bind(worker); 83 | workerAPI.ready = worker.ready.bind(worker); 84 | workerAPI.cd = worker.cd.bind(worker); 85 | workerAPI.isDir = worker.isDir.bind(worker); 86 | workerAPI.processMessage = worker.processMessage.bind(worker); 87 | -------------------------------------------------------------------------------- /packages/xeus/src/comlink.worker.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Jupyter Development Team. 2 | // Distributed under the terms of the Modified BSD License. 3 | 4 | /** 5 | * A WebWorker entrypoint that uses comlink to handle postMessage details 6 | */ 7 | 8 | import { expose } from 'comlink'; 9 | 10 | import { URLExt } from '@jupyterlab/coreutils'; 11 | 12 | import { DriveFS } from '@jupyterlite/contents'; 13 | 14 | import { EmpackedXeusRemoteKernel } from './worker'; 15 | 16 | export class XeusComlinkKernel extends EmpackedXeusRemoteKernel { 17 | /** 18 | * Setup custom Emscripten FileSystem 19 | */ 20 | async mount( 21 | driveName: string, 22 | mountpoint: string, 23 | baseUrl: string, 24 | browsingContextId: string 25 | ): Promise { 26 | const { FS, PATH, ERRNO_CODES } = globalThis.Module; 27 | 28 | if (!FS) { 29 | return; 30 | } 31 | 32 | const drive = new DriveFS({ 33 | FS, 34 | PATH, 35 | ERRNO_CODES, 36 | baseUrl, 37 | driveName, 38 | mountpoint, 39 | browsingContextId 40 | }); 41 | 42 | FS.mkdir(mountpoint); 43 | FS.mount(drive, {}, mountpoint); 44 | FS.chdir(mountpoint); 45 | } 46 | 47 | protected initializeStdin(baseUrl: string, browsingContextId: string): void { 48 | globalThis.get_stdin = (inputRequest: any): any => { 49 | // Send a input request to the front-end via the service worker and block until 50 | // the reply is received. 51 | try { 52 | const xhr = new XMLHttpRequest(); 53 | const url = URLExt.join(baseUrl, '/api/stdin/kernel'); 54 | xhr.open('POST', url, false); // Synchronous XMLHttpRequest 55 | const msg = JSON.stringify({ 56 | browsingContextId, 57 | data: inputRequest 58 | }); 59 | // Send input request, this blocks until the input reply is received. 60 | xhr.send(msg); 61 | const inputReply = JSON.parse(xhr.response as string); 62 | 63 | if ('error' in inputReply) { 64 | // Service worker may return an error instead of an input reply message. 65 | throw new Error(inputReply['error']); 66 | } 67 | 68 | return inputReply; 69 | } catch (err) { 70 | return { error: `Failed to request stdin via service worker: ${err}` }; 71 | } 72 | }; 73 | } 74 | } 75 | 76 | const worker = new XeusComlinkKernel(); 77 | 78 | expose(worker); 79 | -------------------------------------------------------------------------------- /packages/xeus/src/index.ts: -------------------------------------------------------------------------------- 1 | export * from './coincident.worker'; 2 | export * from './comlink.worker'; 3 | export * from './web.worker.kernel'; 4 | export * from './worker'; 5 | -------------------------------------------------------------------------------- /packages/xeus/src/interfaces.ts: -------------------------------------------------------------------------------- 1 | import { IXeusWorkerKernel } from '@jupyterlite/xeus-core'; 2 | 3 | export interface IEmpackXeusWorkerKernel extends IXeusWorkerKernel { 4 | initialize(options: IEmpackXeusWorkerKernel.IOptions): Promise; 5 | } 6 | 7 | export namespace IEmpackXeusWorkerKernel { 8 | export interface IOptions extends IXeusWorkerKernel.IOptions { 9 | empackEnvMetaLink?: string | undefined; 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /packages/xeus/src/web.worker.kernel.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Thorsten Beier 2 | // Copyright (c) JupyterLite Contributors 3 | // Distributed under the terms of the Modified BSD License. 4 | 5 | import coincident from 'coincident'; 6 | 7 | import { wrap } from 'comlink'; 8 | import type { Remote } from 'comlink'; 9 | 10 | import { PromiseDelegate } from '@lumino/coreutils'; 11 | 12 | import { KernelMessage } from '@jupyterlab/services'; 13 | 14 | import { 15 | DriveContentsProcessor, 16 | TDriveMethod, 17 | TDriveRequest 18 | } from '@jupyterlite/contents'; 19 | 20 | import { WebWorkerKernelBase } from '@jupyterlite/xeus-core'; 21 | import { IEmpackXeusWorkerKernel } from './interfaces'; 22 | import { PageConfig } from '@jupyterlab/coreutils'; 23 | 24 | export class WebWorkerKernel extends WebWorkerKernelBase { 25 | /** 26 | * Instantiate a new WebWorkerKernel 27 | * 28 | * @param options The instantiation options for a new WebWorkerKernel 29 | */ 30 | constructor(options: WebWorkerKernel.IOptions) { 31 | super(options); 32 | } 33 | 34 | /** 35 | * Load the worker. 36 | */ 37 | initWorker(options: WebWorkerKernel.IOptions): Worker { 38 | if (crossOriginIsolated) { 39 | return new Worker(new URL('./coincident.worker.js', import.meta.url), { 40 | type: 'module' 41 | }); 42 | } else { 43 | return new Worker(new URL('./comlink.worker.js', import.meta.url), { 44 | type: 'module' 45 | }); 46 | } 47 | } 48 | 49 | /** 50 | * Initialize the remote kernel. 51 | * Use coincident if crossOriginIsolated, comlink otherwise 52 | * See the two following issues for more context: 53 | * - https://github.com/jupyterlite/jupyterlite/issues/1424 54 | * - https://github.com/jupyterlite/xeus/issues/102 55 | */ 56 | createRemote( 57 | options: WebWorkerKernel.IOptions 58 | ): IEmpackXeusWorkerKernel | Remote { 59 | let remote: IEmpackXeusWorkerKernel | Remote; 60 | 61 | // We directly forward messages to xeus, which will dispatch them properly 62 | // See discussion in https://github.com/jupyterlite/xeus/pull/108#discussion_r1750143661 63 | this.worker.onmessage = e => { 64 | this.processWorkerMessage(e.data); 65 | }; 66 | 67 | if (crossOriginIsolated) { 68 | remote = coincident(this.worker) as IEmpackXeusWorkerKernel; 69 | // The coincident worker uses its own filesystem API: 70 | (remote.processDriveRequest as any) = async ( 71 | data: TDriveRequest 72 | ) => { 73 | if (!DriveContentsProcessor) { 74 | throw new Error( 75 | 'File system calls over Atomics.wait is only supported with jupyterlite>=0.4.0a3' 76 | ); 77 | } 78 | 79 | return await this.contentsProcessor.processDriveRequest(data); 80 | }; 81 | 82 | // Stdin request is synchronous from the web worker's point of view, blocking 83 | // until the reply is received. From the UI thread's point of view it is async. 84 | (remote.processStdinRequest as any) = async ( 85 | inputRequest: KernelMessage.IInputRequestMsg 86 | ): Promise => { 87 | this.processWorkerMessage(inputRequest); 88 | this.inputDelegate = 89 | new PromiseDelegate(); 90 | return await this.inputDelegate.promise; 91 | }; 92 | } else { 93 | remote = wrap(this.worker) as Remote; 94 | } 95 | 96 | return remote; 97 | } 98 | 99 | /** 100 | * Initialize the remote kernel 101 | * @param options 102 | */ 103 | protected async initRemote(options: WebWorkerKernel.IOptions) { 104 | return (this.remoteKernel as IEmpackXeusWorkerKernel).initialize({ 105 | baseUrl: PageConfig.getBaseUrl(), 106 | kernelId: this.id, 107 | mountDrive: options.mountDrive, 108 | kernelSpec: options.kernelSpec, 109 | browsingContextId: options.browsingContextId, 110 | empackEnvMetaLink: options.empackEnvMetaLink 111 | }); 112 | } 113 | } 114 | 115 | /** 116 | * A namespace for WebWorkerKernel statics. 117 | */ 118 | export namespace WebWorkerKernel { 119 | /** 120 | * The instantiation options for a Pyodide kernel 121 | */ 122 | export interface IOptions extends WebWorkerKernelBase.IOptions { 123 | empackEnvMetaLink?: string | undefined; 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /packages/xeus/src/worker.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Thorsten Beier 2 | // Copyright (c) JupyterLite Contributors 3 | // Distributed under the terms of the Modified BSD License. 4 | 5 | import { URLExt } from '@jupyterlab/coreutils'; 6 | 7 | import { 8 | IEmpackEnvMeta, 9 | bootstrapEmpackPackedEnvironment, 10 | bootstrapPython, 11 | getPythonVersion, 12 | loadShareLibs, 13 | ISolvedPackages, 14 | solve, 15 | installPackagesToEmscriptenFS, 16 | removePackagesFromEmscriptenFS, 17 | showPackagesList, 18 | TSharedLibsMap, 19 | ISolvedPackage 20 | } from '@emscripten-forge/mambajs'; 21 | import { IUnpackJSAPI } from '@emscripten-forge/untarjs'; 22 | import { XeusRemoteKernelBase } from '@jupyterlite/xeus-core'; 23 | import { IEmpackXeusWorkerKernel } from './interfaces'; 24 | 25 | async function fetchJson(url: string): Promise { 26 | const response = await fetch(url); 27 | if (!response.ok) { 28 | throw new Error(`HTTP error! status: ${response.status}`); 29 | } 30 | const json = await response.json(); 31 | return json; 32 | } 33 | 34 | /** 35 | * A worker kernel that is backed by an empack environment 36 | */ 37 | export abstract class EmpackedXeusRemoteKernel extends XeusRemoteKernelBase { 38 | protected async initializeModule( 39 | options: IEmpackXeusWorkerKernel.IOptions 40 | ): Promise { 41 | const { baseUrl, kernelSpec } = options; 42 | 43 | // location of the kernel binary on the server 44 | const binaryJS = URLExt.join(baseUrl, kernelSpec.argv[0]); 45 | const binaryWASM = binaryJS.replace('.js', '.wasm'); 46 | const binaryDATA = binaryJS.replace('.js', '.data'); 47 | const kernelRootUrl = URLExt.join(baseUrl, 'xeus', kernelSpec.envName); 48 | 49 | const sharedLibs = 50 | kernelSpec.metadata && kernelSpec.metadata.shared 51 | ? kernelSpec.metadata.shared 52 | : {}; 53 | 54 | importScripts(binaryJS); 55 | return { 56 | locateFile: (file: string) => { 57 | if (file in sharedLibs) { 58 | return URLExt.join(kernelRootUrl, kernelSpec.name, file); 59 | } 60 | 61 | // Special case for libxeus 62 | if (['libxeus.so'].includes(file)) { 63 | return URLExt.join(kernelRootUrl, kernelSpec.name, file); 64 | } 65 | 66 | if (file.endsWith('.wasm')) { 67 | return binaryWASM; 68 | } else if (file.endsWith('.data')) { 69 | // Handle the .data file if it exists 70 | return binaryDATA; 71 | } 72 | 73 | return file; 74 | } 75 | }; 76 | } 77 | 78 | protected async initializeFileSystem( 79 | options: IEmpackXeusWorkerKernel.IOptions 80 | ): Promise { 81 | const { baseUrl, kernelSpec, empackEnvMetaLink } = options; 82 | 83 | if ( 84 | this.Module.FS === undefined || 85 | this.Module.loadDynamicLibrary === undefined 86 | ) { 87 | throw new Error('Cannot load kernel without a valid FS'); 88 | } 89 | 90 | // location of the kernel binary on the server 91 | const kernelRootUrl = URLExt.join( 92 | baseUrl, 93 | 'xeus', 94 | 'kernels', 95 | kernelSpec.dir 96 | ); 97 | 98 | const empackEnvMetaLocation = empackEnvMetaLink || kernelRootUrl; 99 | const packagesJsonUrl = `${empackEnvMetaLocation}/empack_env_meta.json`; 100 | this._pkgRootUrl = URLExt.join( 101 | baseUrl, 102 | `xeus/${kernelSpec.envName}/kernel_packages` 103 | ); 104 | const empackEnvMeta = (await fetchJson(packagesJsonUrl)) as IEmpackEnvMeta; 105 | 106 | // Initialize installed packages from empack env meta 107 | this._installedPackages = {}; 108 | empackEnvMeta.packages.map(pkg => { 109 | this._installedPackages[pkg.filename] = { 110 | name: pkg.name, 111 | version: pkg.version, 112 | repo_url: pkg.channel ? pkg.channel : '', 113 | url: pkg.url ? pkg.url : '', 114 | repo_name: pkg.channel ? pkg.channel : '', 115 | build_string: pkg.build 116 | }; 117 | }); 118 | 119 | this._pythonVersion = getPythonVersion(empackEnvMeta.packages); 120 | this._prefix = empackEnvMeta.prefix; 121 | 122 | const bootstrapped = await bootstrapEmpackPackedEnvironment({ 123 | empackEnvMeta, 124 | pkgRootUrl: this._pkgRootUrl, 125 | Module: this.Module, 126 | logger: this.logger, 127 | pythonVersion: this._pythonVersion 128 | }); 129 | 130 | this._paths = bootstrapped.paths; 131 | this._untarjs = bootstrapped.untarjs; 132 | this._sharedLibs = bootstrapped.sharedLibs; 133 | } 134 | 135 | /** 136 | * Initialize the interpreter if needed 137 | * @param options 138 | */ 139 | protected async initializeInterpreter( 140 | options: IEmpackXeusWorkerKernel.IOptions 141 | ) { 142 | // Bootstrap Python, if it's xeus-python 143 | if (options.kernelSpec.name === 'xpython') { 144 | if (!this._pythonVersion) { 145 | throw new Error('Python is not installed, cannot start Python!'); 146 | } 147 | 148 | this.logger.log('Starting Python'); 149 | 150 | await bootstrapPython({ 151 | prefix: this._prefix, 152 | pythonVersion: this._pythonVersion, 153 | Module: this.Module 154 | }); 155 | } 156 | 157 | // Load shared libs 158 | await loadShareLibs({ 159 | sharedLibs: this._sharedLibs, 160 | prefix: this._prefix, 161 | Module: this.Module, 162 | logger: this.logger 163 | }); 164 | } 165 | 166 | protected async install( 167 | channels: string[], 168 | specs: string[], 169 | pipSpecs: string[] 170 | ) { 171 | if (specs.length || pipSpecs.length) { 172 | try { 173 | const newPackages = await solve({ 174 | ymlOrSpecs: specs, 175 | installedPackages: this._installedPackages, 176 | pipSpecs, 177 | channels, 178 | logger: this.logger 179 | }); 180 | 181 | await this._reloadPackagesInFS({ 182 | ...newPackages.condaPackages, 183 | ...newPackages.pipPackages 184 | }); 185 | } catch (error: any) { 186 | this.logger?.error(error.stack); 187 | } 188 | } 189 | } 190 | 191 | /** 192 | * Implements dynamic installation of packages 193 | */ 194 | protected listInstalledPackages(): Promise { 195 | showPackagesList(this._installedPackages, this.logger); 196 | return Promise.resolve(); 197 | } 198 | 199 | private async _reloadPackagesInFS(newInstalledPackages: ISolvedPackages) { 200 | const removedPackages: ISolvedPackages = {}; 201 | const newPackages: ISolvedPackages = {}; 202 | 203 | // First create structures we can quickly inspect 204 | const newInstalledPackagesMap: { [name: string]: ISolvedPackage } = {}; 205 | for (const newInstalledPkg of Object.values(newInstalledPackages)) { 206 | newInstalledPackagesMap[newInstalledPkg.name] = newInstalledPkg; 207 | } 208 | const oldInstalledPackagesMap: { [name: string]: ISolvedPackage } = {}; 209 | for (const oldInstalledPkg of Object.values(this._installedPackages)) { 210 | oldInstalledPackagesMap[oldInstalledPkg.name] = oldInstalledPkg; 211 | } 212 | 213 | // Compare old installed packages with new ones 214 | for (const filename of Object.keys(this._installedPackages)) { 215 | const installedPkg = this._installedPackages[filename]; 216 | 217 | // Exact same build of the package already installed 218 | if ( 219 | installedPkg.name in newInstalledPackagesMap && 220 | installedPkg.build_string === 221 | newInstalledPackagesMap[installedPkg.name].build_string 222 | ) { 223 | continue; 224 | } 225 | 226 | removedPackages[filename] = installedPkg; 227 | } 228 | 229 | // Compare new installed packages with old ones 230 | for (const filename of Object.keys(newInstalledPackages)) { 231 | const newPkg = newInstalledPackages[filename]; 232 | 233 | // Exact same build of the package already installed 234 | if ( 235 | newPkg.name in oldInstalledPackagesMap && 236 | newPkg.build_string === 237 | oldInstalledPackagesMap[newPkg.name].build_string 238 | ) { 239 | continue; 240 | } 241 | 242 | newPackages[filename] = newPkg; 243 | } 244 | 245 | await removePackagesFromEmscriptenFS({ 246 | removedPackages, 247 | Module: this.Module, 248 | paths: this._paths, 249 | logger: this.logger 250 | }); 251 | 252 | const { sharedLibs, paths } = await installPackagesToEmscriptenFS({ 253 | packages: newPackages, 254 | pkgRootUrl: this._pkgRootUrl, 255 | pythonVersion: this._pythonVersion, 256 | Module: this.Module, 257 | untarjs: this._untarjs, 258 | logger: this.logger 259 | }); 260 | this._paths = { ...this._paths, ...paths }; 261 | 262 | await loadShareLibs({ 263 | sharedLibs, 264 | prefix: this._prefix, 265 | Module: this.Module, 266 | logger: this.logger 267 | }); 268 | 269 | this._installedPackages = newInstalledPackages; 270 | } 271 | 272 | private _pythonVersion: number[] | undefined; 273 | private _prefix = ''; 274 | 275 | private _pkgRootUrl = ''; 276 | 277 | private _sharedLibs: TSharedLibsMap; 278 | private _installedPackages: ISolvedPackages = {}; 279 | private _paths = {}; 280 | 281 | private _untarjs: IUnpackJSAPI | undefined; 282 | } 283 | 284 | export namespace XeusRemoteKernel { 285 | export interface IOptions {} 286 | } 287 | -------------------------------------------------------------------------------- /packages/xeus/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "lib", 5 | "strictPropertyInitialization": false, 6 | "rootDir": "src" 7 | }, 8 | "include": ["src/**/*"] 9 | } 10 | -------------------------------------------------------------------------------- /packages/xeus/worker.webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const rules = [ 3 | { 4 | test: /\.js$/, 5 | exclude: /node_modules/, 6 | loader: 'source-map-loader' 7 | } 8 | ]; 9 | 10 | const resolve = { 11 | fallback: { 12 | fs: false, 13 | child_process: false, 14 | crypto: false 15 | }, 16 | extensions: ['.js'] 17 | }; 18 | 19 | module.exports = [ 20 | { 21 | entry: { 22 | ['coincident.worker']: './lib/coincident.worker.js', 23 | ['comlink.worker']: './lib/comlink.worker.js' 24 | }, 25 | output: { 26 | filename: '[name].js', 27 | path: path.resolve(__dirname, 'lib'), 28 | libraryTarget: 'amd' 29 | }, 30 | module: { 31 | rules 32 | }, 33 | devtool: 'source-map', 34 | resolve 35 | } 36 | ]; 37 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["hatchling>=1.5.0", "jupyterlab>=4.4.0.b0,<5", "hatch-nodejs-version>=0.3.2"] 3 | build-backend = "hatchling.build" 4 | 5 | [project] 6 | name = "jupyterlite_xeus" 7 | readme = "README.md" 8 | license = { file = "LICENSE" } 9 | requires-python = ">=3.8" 10 | classifiers = [ 11 | "Framework :: Jupyter", 12 | "Framework :: Jupyter :: JupyterLab", 13 | "Framework :: Jupyter :: JupyterLab :: 4", 14 | "Framework :: Jupyter :: JupyterLab :: Extensions", 15 | "Framework :: Jupyter :: JupyterLab :: Extensions :: Prebuilt", 16 | "License :: OSI Approved :: BSD License", 17 | "Programming Language :: Python", 18 | "Programming Language :: Python :: 3", 19 | "Programming Language :: Python :: 3.8", 20 | "Programming Language :: Python :: 3.9", 21 | "Programming Language :: Python :: 3.10", 22 | "Programming Language :: Python :: 3.11", 23 | "Programming Language :: Python :: 3.12", 24 | ] 25 | dependencies = [ 26 | "empack>=5.1.1,<6", 27 | "traitlets", 28 | "jupyterlite-core>=0.6,<0.7", 29 | "pyyaml", 30 | "requests", 31 | ] 32 | dynamic = ["version", "description", "authors", "urls", "keywords"] 33 | 34 | [project.entry-points."jupyterlite.addon.v0"] 35 | jupyterlite-xeus = "jupyterlite_xeus.add_on:XeusAddon" 36 | 37 | [tool.hatch.version] 38 | source = "nodejs" 39 | 40 | [tool.hatch.metadata.hooks.nodejs] 41 | fields = ["description", "authors", "urls"] 42 | 43 | [tool.hatch.build.targets.sdist] 44 | artifacts = ["jupyterlite_xeus/labextension"] 45 | exclude = [".github", "binder"] 46 | 47 | [tool.hatch.build.targets.wheel.shared-data] 48 | "jupyterlite_xeus/labextension" = "share/jupyter/labextensions/@jupyterlite/xeus" 49 | "install.json" = "share/jupyter/labextensions/@jupyterlite/xeus/install.json" 50 | 51 | [tool.hatch.build.hooks.version] 52 | path = "jupyterlite_xeus/_version.py" 53 | 54 | [tool.hatch.build.hooks.jupyter-builder] 55 | dependencies = ["hatch-jupyter-builder>=0.5"] 56 | build-function = "hatch_jupyter_builder.npm_builder" 57 | ensured-targets = [ 58 | "jupyterlite_xeus/labextension/static/style.js", 59 | "jupyterlite_xeus/labextension/package.json", 60 | ] 61 | skip-if-exists = ["jupyterlite_xeus/labextension/static/style.js"] 62 | 63 | [tool.hatch.build.hooks.jupyter-builder.build-kwargs] 64 | build_cmd = "build:prod" 65 | npm = ["jlpm"] 66 | 67 | [tool.hatch.build.hooks.jupyter-builder.editable-build-kwargs] 68 | build_cmd = "build" 69 | npm = ["jlpm"] 70 | source_dir = "src" 71 | build_dir = "jupyterlite_xeus/labextension" 72 | 73 | [tool.jupyter-releaser.options] 74 | version_cmd = "python ./scripts/bump_version.py" 75 | 76 | [tool.jupyter-releaser.hooks] 77 | before-bump-version = ["python -m pip install 'jupyterlab>=4.0.0,<5'", "jlpm"] 78 | before-build-npm = [ 79 | "python -m pip install 'jupyterlab>=4.0.0,<4.3'", 80 | "YARN_ENABLE_IMMUTABLE_INSTALLS=0 jlpm", 81 | "jlpm build:prod" 82 | ] 83 | before-build-python = ["jlpm clean:all"] 84 | 85 | [tool.check-wheel-contents] 86 | ignore = ["W002"] 87 | 88 | [tool.ruff] 89 | extend-include = ["*.ipynb"] 90 | 91 | [tool.ruff.lint] 92 | select = [ 93 | # pycodestyle 94 | "E", 95 | ] 96 | ignore = ["E501", "E731"] 97 | -------------------------------------------------------------------------------- /scripts/bump_version.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | from pathlib import Path 4 | from subprocess import run 5 | 6 | BUMP_VERSION_CMD = "npx lerna version --no-push --force-publish --no-git-tag-version --yes" 7 | 8 | 9 | def main(): 10 | parser = argparse.ArgumentParser() 11 | parser.add_argument("version") 12 | args = parser.parse_args() 13 | version = args.version 14 | 15 | run(f"{BUMP_VERSION_CMD} {version}", shell=True, check=True) 16 | 17 | root = Path(__file__).parent.parent 18 | version_file = root / "packages" / "xeus-extension" / "package.json" 19 | package_file = root / "package.json" 20 | 21 | version_json = json.loads(version_file.read_text()) 22 | version = version_json["version"].replace("-alpha.", "-a").replace("-beta.", "-b").replace("-rc.", "-rc") 23 | 24 | package_json = json.loads(package_file.read_text()) 25 | package_json["version"] = version 26 | package_file.write_text(json.dumps(package_json, indent=4)) 27 | 28 | 29 | if __name__ == "__main__": 30 | main() 31 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | __import__("setuptools").setup() 2 | -------------------------------------------------------------------------------- /tests/environment-1.yml: -------------------------------------------------------------------------------- 1 | name: xeus-python-kernel-1 2 | dependencies: 3 | - xeus-python 4 | - xeus-lua 5 | - numpy 6 | - matplotlib 7 | - pillow 8 | - ipywidgets 9 | - pip: 10 | # Installing a python package that ships a lab extension under share/ 11 | - ipycanvas 12 | # Installing a pure python package 13 | - py2vega 14 | -------------------------------------------------------------------------------- /tests/environment-2.yml: -------------------------------------------------------------------------------- 1 | name: xeus-python-kernel-2 2 | dependencies: 3 | - xeus-python 4 | - pip: 5 | # Installing NumPy with pip should fail 6 | - numpy 7 | -------------------------------------------------------------------------------- /tests/environment-3.yml: -------------------------------------------------------------------------------- 1 | name: xeus-cpp-env 2 | dependencies: 3 | - xeus-cpp 4 | -------------------------------------------------------------------------------- /tests/test_package/environment-3.yml: -------------------------------------------------------------------------------- 1 | name: xeus-python-kernel-3 2 | dependencies: 3 | - xeus-python 4 | - numpy 5 | - pip: 6 | - . 7 | -------------------------------------------------------------------------------- /tests/test_package/pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "test-package" 3 | version = "0.1.0" 4 | -------------------------------------------------------------------------------- /tests/test_package/test_package/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyterlite/xeus/506c0c818b94b5097c65b76456b01308e5e6248c/tests/test_package/test_package/__init__.py -------------------------------------------------------------------------------- /tests/test_package/test_package/hey.py: -------------------------------------------------------------------------------- 1 | print("Hey") 2 | -------------------------------------------------------------------------------- /tests/test_xeus.py: -------------------------------------------------------------------------------- 1 | """Test creating Python envs for jupyterlite-xeus-python.""" 2 | 3 | import os 4 | from tempfile import TemporaryDirectory 5 | from pathlib import Path 6 | import tarfile 7 | 8 | import pytest 9 | 10 | from jupyterlite_core.app import LiteStatusApp 11 | 12 | from jupyterlite_xeus.add_on import XeusAddon 13 | 14 | 15 | def test_python_env_from_file_1(): 16 | app = LiteStatusApp(log_level="DEBUG") 17 | app.initialize() 18 | manager = app.lite_manager 19 | 20 | addon = XeusAddon(manager) 21 | addon.environment_file = "environment-1.yml" 22 | 23 | for step in addon.post_build(manager): 24 | pass 25 | 26 | # Check env 27 | env_name = "xeus-python-kernel-1" 28 | assert env_name in addon.prefixes 29 | env_path = Path(addon.prefixes[env_name]) 30 | assert env_path.is_dir() 31 | 32 | assert os.path.isfile(env_path / "bin/xpython.js") 33 | assert os.path.isfile(env_path / "bin/xpython.wasm") 34 | 35 | assert os.path.isfile(env_path / "bin/xlua.js") 36 | assert os.path.isfile(env_path / "bin/xlua.wasm") 37 | 38 | # Checking pip packages 39 | assert os.path.isdir(env_path / "lib/python3.13") 40 | assert os.path.isdir(env_path / "lib/python3.13/site-packages") 41 | assert os.path.isdir(env_path / "lib/python3.13/site-packages/ipywidgets") 42 | assert os.path.isdir(env_path / "lib/python3.13/site-packages/ipycanvas") 43 | assert os.path.isdir(env_path / "lib/python3.13/site-packages/py2vega") 44 | 45 | # Checking labextensions 46 | assert os.path.isdir( 47 | env_path 48 | / "share/jupyter/labextensions/@jupyter-widgets/jupyterlab-manager" 49 | ) 50 | assert os.path.isdir(env_path / "share/jupyter/labextensions/ipycanvas") 51 | 52 | 53 | def test_python_env_from_file_3(): 54 | app = LiteStatusApp(log_level="DEBUG") 55 | app.initialize() 56 | manager = app.lite_manager 57 | 58 | addon = XeusAddon(manager) 59 | addon.environment_file = "test_package/environment-3.yml" 60 | 61 | for step in addon.post_build(manager): 62 | pass 63 | 64 | # Test 65 | env_name = "xeus-python-kernel-3" 66 | assert env_name in addon.prefixes 67 | env_path = Path(addon.prefixes[env_name]) 68 | assert env_path.is_dir() 69 | 70 | assert os.path.isdir( 71 | env_path / "lib/python3.13/site-packages/test_package" 72 | ) 73 | assert os.path.isfile( 74 | env_path / "lib/python3.13/site-packages/test_package/hey.py" 75 | ) 76 | 77 | 78 | def test_python_env_from_file_2(): 79 | app = LiteStatusApp(log_level="DEBUG") 80 | app.initialize() 81 | manager = app.lite_manager 82 | 83 | addon = XeusAddon(manager) 84 | addon.environment_file = "environment-2.yml" 85 | 86 | with pytest.raises(RuntimeError, match="Cannot install binary PyPI package"): 87 | for step in addon.post_build(manager): 88 | pass 89 | 90 | 91 | def test_mount_point(): 92 | app = LiteStatusApp(log_level="DEBUG") 93 | app.initialize() 94 | manager = app.lite_manager 95 | 96 | addon = XeusAddon(manager) 97 | addon.environment_file = "environment-1.yml" 98 | addon.mounts = [ 99 | f"{(Path(__file__).parent / "environment-1.yml").resolve()}:/share", 100 | f"{(Path(__file__).parent / "test_package").resolve()}:/share/test_package", 101 | ] 102 | 103 | for step in addon.post_build(manager): 104 | pass 105 | 106 | env_name = "xeus-python-kernel-1" 107 | assert env_name in addon.prefixes 108 | 109 | outpath = Path(addon.cwd_name) / "packed_env" / env_name 110 | 111 | with tarfile.open(outpath / "mount_0.tar.gz", "r") as fobj: 112 | names = fobj.getnames() 113 | assert "share/environment-1.yml" in names 114 | 115 | with tarfile.open(outpath / "mount_1.tar.gz", "r") as fobj: 116 | names = fobj.getnames() 117 | assert "share/test_package/environment-3.yml" in names 118 | 119 | 120 | def test_multiple_envs_with_same_name_raises(): 121 | app = LiteStatusApp(log_level="DEBUG") 122 | app.initialize() 123 | manager = app.lite_manager 124 | 125 | addon = XeusAddon(manager) 126 | addon.environment_file = ["environment-3.yml", "environment-3.yml"] 127 | 128 | with pytest.raises(ValueError): 129 | for step in addon.post_build(manager): 130 | pass 131 | 132 | 133 | def test_multiple_envs(): 134 | app = LiteStatusApp(log_level="DEBUG") 135 | app.initialize() 136 | manager = app.lite_manager 137 | 138 | addon = XeusAddon(manager) 139 | addon.environment_file = ["environment-1.yml", "environment-3.yml"] 140 | 141 | # Store steps by name so can check locations copied to without actually copying. 142 | steps = {} 143 | for step in addon.post_build(manager): 144 | steps[step["name"]] = step 145 | 146 | env_name_py_lua = "xeus-python-kernel-1" 147 | env_name_cpp = "xeus-cpp-env" 148 | assert env_name_py_lua in addon.prefixes 149 | assert env_name_cpp in addon.prefixes 150 | 151 | env_path_py_lua = Path(addon.prefixes[env_name_py_lua]) 152 | env_path_cpp = Path(addon.prefixes[env_name_cpp]) 153 | assert env_path_py_lua.is_dir() 154 | assert env_path_cpp.is_dir() 155 | 156 | target_path = Path(__file__).parent / "_output" / "xeus" 157 | keys = list(steps.keys()) 158 | 159 | # kernel.json (one per kernel) 160 | action = steps[f"copy:{env_name_py_lua}:xpython:kernel.json"]["actions"][0][1] 161 | assert action[1] == target_path / env_name_py_lua / "xpython" / "kernel.json" 162 | 163 | action = steps[f"copy:{env_name_py_lua}:xlua:kernel.json"]["actions"][0][1] 164 | assert action[1] == target_path / env_name_py_lua / "xlua" / "kernel.json" 165 | 166 | action = steps[f"copy:{env_name_cpp}:xcpp20:kernel.json"]["actions"][0][1] 167 | assert action[1] == target_path / env_name_cpp / "xcpp20" / "kernel.json" 168 | 169 | # shared libraries 170 | action = steps[f"copy:{env_name_cpp}:xcpp20:libclangCppInterOp.so"]["actions"][0][1] 171 | assert action[1] == target_path / env_name_cpp / "xcpp20" / "libclangCppInterOp.so" 172 | 173 | # binaries (one set per kernel) 174 | actions = steps[f"copy:{env_name_py_lua}:xlua:binaries"]["actions"] 175 | assert actions[0][1][1] == target_path / env_name_py_lua / "bin" / "xlua.js" 176 | assert actions[1][1][1] == target_path / env_name_py_lua / "bin" / "xlua.wasm" 177 | 178 | actions = steps[f"copy:{env_name_py_lua}:xpython:binaries"]["actions"] 179 | assert actions[0][1][1] == target_path / env_name_py_lua / "bin" / "xpython.js" 180 | assert actions[1][1][1] == target_path / env_name_py_lua / "bin" / "xpython.wasm" 181 | 182 | actions = steps[f"copy:{env_name_cpp}:xcpp20:binaries"]["actions"] 183 | assert actions[0][1][1] == target_path / env_name_cpp / "bin" / "xcpp.js" 184 | assert actions[1][1][1] == target_path / env_name_cpp / "bin" / "xcpp.wasm" 185 | 186 | # data (potentially one per kernel but only for xcpp here) 187 | action = steps[f"copy:{env_name_cpp}:xcpp20:data"]["actions"][0][1] 188 | assert action[1] == target_path / env_name_cpp / "bin" / "xcpp.data" 189 | 190 | # empack_env_meta.json (one per env) 191 | action = steps[f"xeus:{env_name_py_lua}:copy_env_file:empack_env_meta.json"]["actions"][0][1] 192 | assert action[1] == target_path / env_name_py_lua / "empack_env_meta.json" 193 | 194 | action = steps[f"xeus:{env_name_cpp}:copy_env_file:empack_env_meta.json"]["actions"][0][1] 195 | assert action[1] == target_path / env_name_cpp / "empack_env_meta.json" 196 | 197 | # kernel_packages (one directory per env) 198 | ## packages that are in py_lua env but not cpp env 199 | for pkg in ["xeus-lua-", "ipycanvas-"]: 200 | match = list(filter(lambda k: k.startswith(f"xeus:{env_name_py_lua}:copy:{pkg}"), keys)) 201 | assert len(match) == 1 202 | filename = match[0].split(':')[-1] # e.g. xeus-lua-0.7.4-he62be5e_0.tar.gz 203 | action = steps[match[0]]["actions"][0][1] 204 | assert action[1] == target_path / env_name_py_lua / "kernel_packages" / filename 205 | 206 | match = list(filter(lambda k: k.startswith(f"xeus:{env_name_cpp}:copy:{pkg}"), keys)) 207 | assert len(match) == 0 208 | 209 | ## packages that are in cpp env but not py_lua env 210 | for pkg in ["xeus-cpp-", "cppinterop-"]: 211 | match = list(filter(lambda k: k.startswith(f"xeus:{env_name_py_lua}:copy:{pkg}"), keys)) 212 | assert len(match) == 0 213 | 214 | match = list(filter(lambda k: k.startswith(f"xeus:{env_name_cpp}:copy:{pkg}"), keys)) 215 | assert len(match) == 1 216 | filename = match[0].split(':')[-1] # e.g. xeus-cpp-0.6.0-h18da88b_1.tar.gz 217 | action = steps[match[0]]["actions"][0][1] 218 | assert action[1] == target_path / env_name_cpp / "kernel_packages" / filename 219 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "allowSyntheticDefaultImports": true, 4 | "composite": true, 5 | "declaration": true, 6 | "esModuleInterop": true, 7 | "incremental": true, 8 | "jsx": "react", 9 | "module": "esnext", 10 | "moduleResolution": "node", 11 | "noEmitOnError": true, 12 | "noImplicitAny": false, 13 | "noUnusedLocals": true, 14 | "preserveWatchOutput": true, 15 | "resolveJsonModule": true, 16 | "strict": true, 17 | "strictNullChecks": true, 18 | "target": "ES2019" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /ui-tests/README.md: -------------------------------------------------------------------------------- 1 | # Integration Testing 2 | 3 | This folder contains the integration tests of the extension. 4 | 5 | They are defined using [Playwright](https://playwright.dev/docs/intro) test runner 6 | and [Galata](https://github.com/jupyterlab/jupyterlab/tree/main/galata) helper. 7 | 8 | The Playwright configuration is defined in [playwright.config.js](./playwright.config.js). 9 | 10 | The default configuration will produce video for failing tests and an HTML report. 11 | 12 | > There is a new experimental UI mode that you may fall in love with; see [that video](https://www.youtube.com/watch?v=jF0yA-JLQW0). 13 | 14 | ## Run the tests 15 | 16 | > All commands are assumed to be executed from the root directory 17 | 18 | To run the tests, you need to: 19 | 20 | 1. Compile the extension: 21 | 22 | ```sh 23 | jlpm install 24 | jlpm build:prod 25 | ``` 26 | 27 | > Check the extension is installed in JupyterLab. 28 | 29 | 2. Install test dependencies (needed only once): 30 | 31 | ```sh 32 | cd ./ui-tests 33 | jlpm install 34 | jlpm playwright install 35 | cd .. 36 | ``` 37 | 38 | 3. Execute the [Playwright](https://playwright.dev/docs/intro) tests: 39 | 40 | ```sh 41 | cd ./ui-tests 42 | jlpm playwright test 43 | ``` 44 | 45 | Test results will be shown in the terminal. In case of any test failures, the test report 46 | will be opened in your browser at the end of the tests execution; see 47 | [Playwright documentation](https://playwright.dev/docs/test-reporters#html-reporter) 48 | for configuring that behavior. 49 | 50 | ## Update the tests snapshots 51 | 52 | > All commands are assumed to be executed from the root directory 53 | 54 | If you are comparing snapshots to validate your tests, you may need to update 55 | the reference snapshots stored in the repository. To do that, you need to: 56 | 57 | 1. Compile the extension: 58 | 59 | ```sh 60 | jlpm install 61 | jlpm build:prod 62 | ``` 63 | 64 | > Check the extension is installed in JupyterLab. 65 | 66 | 2. Install test dependencies (needed only once): 67 | 68 | ```sh 69 | cd ./ui-tests 70 | jlpm install 71 | jlpm playwright install 72 | cd .. 73 | ``` 74 | 75 | 3. Execute the [Playwright](https://playwright.dev/docs/intro) command: 76 | 77 | ```sh 78 | cd ./ui-tests 79 | jlpm playwright test -u 80 | ``` 81 | 82 | > Some discrepancy may occurs between the snapshots generated on your computer and 83 | > the one generated on the CI. To ease updating the snapshots on a PR, you can 84 | > type `please update playwright snapshots` to trigger the update by a bot on the CI. 85 | > Once the bot has computed new snapshots, it will commit them to the PR branch. 86 | 87 | ## Create tests 88 | 89 | > All commands are assumed to be executed from the root directory 90 | 91 | To create tests, the easiest way is to use the code generator tool of playwright: 92 | 93 | 1. Compile the extension: 94 | 95 | ```sh 96 | jlpm install 97 | jlpm build:prod 98 | ``` 99 | 100 | > Check the extension is installed in JupyterLab. 101 | 102 | 2. Install test dependencies (needed only once): 103 | 104 | ```sh 105 | cd ./ui-tests 106 | jlpm install 107 | jlpm playwright install 108 | cd .. 109 | ``` 110 | 111 | 3. Start the server: 112 | 113 | ```sh 114 | cd ./ui-tests 115 | jlpm start 116 | ``` 117 | 118 | 4. Execute the [Playwright code generator](https://playwright.dev/docs/codegen) in **another terminal**: 119 | 120 | ```sh 121 | cd ./ui-tests 122 | jlpm playwright codegen localhost:8888 123 | ``` 124 | 125 | ## Debug tests 126 | 127 | > All commands are assumed to be executed from the root directory 128 | 129 | To debug tests, a good way is to use the inspector tool of playwright: 130 | 131 | 1. Compile the extension: 132 | 133 | ```sh 134 | jlpm install 135 | jlpm build:prod 136 | ``` 137 | 138 | > Check the extension is installed in JupyterLab. 139 | 140 | 2. Install test dependencies (needed only once): 141 | 142 | ```sh 143 | cd ./ui-tests 144 | jlpm install 145 | jlpm playwright install 146 | cd .. 147 | ``` 148 | 149 | 3. Execute the Playwright tests in [debug mode](https://playwright.dev/docs/debug): 150 | 151 | ```sh 152 | cd ./ui-tests 153 | jlpm playwright test --debug 154 | ``` 155 | 156 | ## Upgrade Playwright and the browsers 157 | 158 | To update the web browser versions, you must update the package `@playwright/test`: 159 | 160 | ```sh 161 | cd ./ui-tests 162 | jlpm up "@playwright/test" 163 | jlpm playwright install 164 | ``` 165 | -------------------------------------------------------------------------------- /ui-tests/build.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from subprocess import run 3 | 4 | import jupyterlab 5 | 6 | extra_labextensions_path = str(Path(jupyterlab.__file__).parent / "galata") 7 | cmd = f"jupyter lite build --FederatedExtensionAddon.extra_labextensions_path={extra_labextensions_path}" 8 | 9 | run( 10 | cmd, 11 | check=True, 12 | shell=True, 13 | ) 14 | -------------------------------------------------------------------------------- /ui-tests/env1.yml: -------------------------------------------------------------------------------- 1 | name: env1 2 | channels: 3 | - https://repo.prefix.dev/emscripten-forge-dev 4 | - conda-forge 5 | dependencies: 6 | - xeus-python 7 | - xeus-lua 8 | - xeus-r 9 | - xeus-cpp 10 | - pandas 11 | - bqplot 12 | -------------------------------------------------------------------------------- /ui-tests/env2.yml: -------------------------------------------------------------------------------- 1 | name: env2 2 | channels: 3 | - https://repo.prefix.dev/emscripten-forge-dev 4 | - conda-forge 5 | dependencies: 6 | - xeus-python 7 | -------------------------------------------------------------------------------- /ui-tests/jupyter-lite.json: -------------------------------------------------------------------------------- 1 | { 2 | "jupyter-lite-schema-version": 0, 3 | "jupyter-config-data": { 4 | "appName": "JupyterLite UI Tests", 5 | "defaultKernelName": "xlua", 6 | "exposeAppInBrowser": true 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /ui-tests/jupyter_lite_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "LiteBuildConfig": { 3 | "output_dir": "dist" 4 | }, 5 | "XeusAddon": { 6 | "environment_file": ["env1.yml", "env2.yml"] 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /ui-tests/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@jupyterlite/xeus-ui-tests", 3 | "version": "1.0.0", 4 | "description": "JupyterLab @jupyterlite/xeus Integration Tests", 5 | "private": true, 6 | "scripts": { 7 | "build": "yarn run clean && python build.py", 8 | "clean": "rimraf dist", 9 | "start": "python -m http.server -b 127.0.0.1 8000 --directory dist", 10 | "start:crossoriginisolated": "npx static-handler --cors --coop --coep --corp ./dist", 11 | "start:detached": "yarn run start&", 12 | "test": "playwright test", 13 | "test:debug": "PWDEBUG=1 playwright test", 14 | "test:report": "http-server ./playwright-report -a localhost -o", 15 | "test:update": "playwright test --update-snapshots" 16 | }, 17 | "devDependencies": { 18 | "rimraf": "^5.0.5" 19 | }, 20 | "dependencies": { 21 | "@jupyterlab/galata": "~5.4.0", 22 | "@playwright/test": "^1.48.0" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /ui-tests/playwright.config.js: -------------------------------------------------------------------------------- 1 | const baseConfig = require('@jupyterlab/galata/lib/playwright-config'); 2 | 3 | module.exports = { 4 | ...baseConfig, 5 | reporter: [[process.env.CI ? 'dot' : 'list'], ['html']], 6 | use: { 7 | acceptDownloads: true, 8 | appPath: '', 9 | autoGoto: false, 10 | baseURL: 'http://localhost:8000', 11 | trace: 'on-first-retry', 12 | video: 'retain-on-failure' 13 | }, 14 | projects: [ 15 | { 16 | name: 'default', 17 | use: { 18 | baseURL: 'http://localhost:8000' 19 | } 20 | }, 21 | { 22 | name: 'crossoriginisolated', 23 | use: { 24 | baseURL: 'http://localhost:8080' 25 | } 26 | } 27 | ], 28 | retries: 1, 29 | webServer: [ 30 | { 31 | command: 'yarn start', 32 | port: 8000, 33 | timeout: 120 * 1000, 34 | reuseExistingServer: true 35 | }, 36 | { 37 | command: 'yarn start:crossoriginisolated', 38 | port: 8080, 39 | timeout: 120 * 1000, 40 | reuseExistingServer: true 41 | } 42 | ] 43 | }; 44 | -------------------------------------------------------------------------------- /ui-tests/tests/jupyterlite_xeus.spec.ts: -------------------------------------------------------------------------------- 1 | import { test } from '@jupyterlab/galata'; 2 | 3 | import { expect } from '@playwright/test'; 4 | 5 | test.describe('General Tests', () => { 6 | test.beforeEach(({ page }) => { 7 | page.setDefaultTimeout(600000); 8 | 9 | page.on('console', message => { 10 | console.log('CONSOLE MSG ---', message.text()); 11 | }); 12 | }); 13 | 14 | test('Launcher should contain xeus-python and xeus-lua kernels', async ({ 15 | page 16 | }) => { 17 | await page.goto('lab/index.html'); 18 | 19 | const notebookSection = page.locator('.jp-Launcher-section').first(); 20 | expect(await notebookSection.screenshot()).toMatchSnapshot( 21 | 'jupyter-xeus-launcher.png' 22 | ); 23 | }); 24 | 25 | test('xeus-python should execute some code', async ({ page }) => { 26 | await page.goto('lab/index.html'); 27 | 28 | const xpython = page 29 | .locator('[title="Python 3.13 (XPython) [env1]"]') 30 | .first(); 31 | await xpython.click(); 32 | 33 | // Wait for kernel to be idle 34 | await page.locator('#jp-main-statusbar').getByText('Idle').waitFor(); 35 | 36 | await page.notebook.addCell('code', 'import bqplot; print("ok")'); 37 | await page.notebook.runCell(1); 38 | 39 | // Wait for kernel to be idle 40 | await page.locator('#jp-main-statusbar').getByText('Idle').waitFor(); 41 | 42 | const cell = await page.notebook.getCellOutput(1); 43 | 44 | expect(await cell?.screenshot()).toMatchSnapshot( 45 | 'jupyter-xeus-execute.png' 46 | ); 47 | }); 48 | 49 | test('should support the same kernel from a second environment', async ({ 50 | page 51 | }) => { 52 | await page.goto('lab/index.html'); 53 | 54 | const xpython = page 55 | .locator('[title="Python 3.13 (XPython) [env2]"]') 56 | .first(); 57 | await xpython.click(); 58 | 59 | // Wait for kernel to be idle 60 | await page.locator('#jp-main-statusbar').getByText('Idle').waitFor(); 61 | 62 | // xeus-python from env2 does not have bqplot installed. 63 | await page.notebook.addCell('code', 'import bqplot'); 64 | await page.notebook.runCell(1); 65 | 66 | // Wait for kernel to be idle 67 | await page.locator('#jp-main-statusbar').getByText('Idle').waitFor(); 68 | 69 | const cell = await page.notebook.getCellOutput(1); 70 | 71 | expect(await cell?.screenshot()).toMatchSnapshot( 72 | 'jupyter-xeus-execute-env2.png' 73 | ); 74 | }); 75 | 76 | test('the kernel should have access to the file system', async ({ page }) => { 77 | await page.goto('lab/index.html'); 78 | 79 | // Create a Python notebook 80 | const xpython = page 81 | .locator('[title="Python 3.13 (XPython) [env1]"]') 82 | .first(); 83 | await xpython.click(); 84 | 85 | await page.notebook.save(); 86 | 87 | await page.notebook.setCell(0, 'code', 'import os; os.listdir()'); 88 | await page.notebook.runCell(0); 89 | 90 | const cell = await page.notebook.getCellOutput(0); 91 | const cellContent = await cell?.textContent(); 92 | const name = 'Untitled.ipynb'; 93 | expect(cellContent).toContain(name); 94 | }); 95 | 96 | test('Stdin using python kernel', async ({ page }) => { 97 | await page.goto('lab/index.html'); 98 | 99 | // Create a Python notebook 100 | const xpython = page 101 | .locator('[title="Python 3.13 (XPython) [env2]"]') 102 | .first(); 103 | await xpython.click(); 104 | 105 | await page.notebook.save(); 106 | 107 | await page.notebook.setCell(0, 'code', 'name = input("Prompt:")'); 108 | let cell0 = page.notebook.runCell(0); // Do not await yet. 109 | 110 | // Run cell containing `input`. 111 | await page.locator('.jp-Stdin >> text=Prompt:').waitFor(); 112 | await page.keyboard.insertText('My Name'); 113 | await page.keyboard.press('Enter'); 114 | await cell0; // await end of cell. 115 | 116 | let output = await page.notebook.getCellTextOutput(0); 117 | expect(output![0]).toEqual('Prompt: My Name\n'); 118 | 119 | await page.notebook.setCell( 120 | 0, 121 | 'code', 122 | 'import getpass; pw = getpass.getpass("Password:")' 123 | ); 124 | cell0 = page.notebook.runCell(0); // Do not await yet. 125 | 126 | // Run cell containing `input`. 127 | await page.locator('.jp-Stdin >> text=Password:').waitFor(); 128 | await page.keyboard.insertText('hidden123'); 129 | await page.keyboard.press('Enter'); 130 | await cell0; // await end of cell. 131 | 132 | output = await page.notebook.getCellTextOutput(0); 133 | expect(output![0]).toEqual('Password: ········\n'); 134 | }); 135 | 136 | test('pip install using python kernel', async ({ page }) => { 137 | await page.goto('lab/index.html'); 138 | 139 | // Create a Python notebook 140 | const xpython = page 141 | .locator('[title="Python 3.13 (XPython) [env2]"]') 142 | .first(); 143 | await xpython.click(); 144 | 145 | await page.notebook.save(); 146 | 147 | await page.notebook.setCell(0, 'code', 'import py2vega'); 148 | await page.notebook.runCell(0); 149 | 150 | let output = await page.notebook.getCellTextOutput(0); 151 | expect(output![0]).toContain('ModuleNotFoundError'); 152 | 153 | await page.notebook.setCell(1, 'code', '%pip install py2vega'); 154 | await page.notebook.runCell(1); 155 | 156 | await page.notebook.setCell(2, 'code', 'import py2vega; print("ok")'); 157 | await page.notebook.runCell(2); 158 | 159 | output = await page.notebook.getCellTextOutput(2); 160 | expect(output![0]).not.toContain('ModuleNotFoundError'); 161 | }); 162 | 163 | test('conda install using python kernel', async ({ page }) => { 164 | await page.goto('lab/index.html'); 165 | 166 | // Create a Python notebook 167 | const xpython = page 168 | .locator('[title="Python 3.13 (XPython) [env2]"]') 169 | .first(); 170 | await xpython.click(); 171 | 172 | await page.notebook.save(); 173 | 174 | await page.notebook.setCell(0, 'code', 'import ipycanvas'); 175 | await page.notebook.runCell(0); 176 | 177 | let output = await page.notebook.getCellTextOutput(0); 178 | expect(output![0]).toContain('ModuleNotFoundError'); 179 | 180 | await page.notebook.setCell(1, 'code', '%conda install ipycanvas'); 181 | await page.notebook.runCell(1); 182 | 183 | await page.notebook.setCell(2, 'code', 'import ipycanvas; print("ok")'); 184 | await page.notebook.runCell(2); 185 | 186 | output = await page.notebook.getCellTextOutput(2); 187 | expect(output![0]).not.toContain('ModuleNotFoundError'); 188 | }); 189 | }); 190 | -------------------------------------------------------------------------------- /ui-tests/tests/jupyterlite_xeus.spec.ts-snapshots/jupyter-xeus-execute-crossoriginisolated-linux.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyterlite/xeus/506c0c818b94b5097c65b76456b01308e5e6248c/ui-tests/tests/jupyterlite_xeus.spec.ts-snapshots/jupyter-xeus-execute-crossoriginisolated-linux.png -------------------------------------------------------------------------------- /ui-tests/tests/jupyterlite_xeus.spec.ts-snapshots/jupyter-xeus-execute-default-linux.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyterlite/xeus/506c0c818b94b5097c65b76456b01308e5e6248c/ui-tests/tests/jupyterlite_xeus.spec.ts-snapshots/jupyter-xeus-execute-default-linux.png -------------------------------------------------------------------------------- /ui-tests/tests/jupyterlite_xeus.spec.ts-snapshots/jupyter-xeus-execute-env2-crossoriginisolated-linux.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyterlite/xeus/506c0c818b94b5097c65b76456b01308e5e6248c/ui-tests/tests/jupyterlite_xeus.spec.ts-snapshots/jupyter-xeus-execute-env2-crossoriginisolated-linux.png -------------------------------------------------------------------------------- /ui-tests/tests/jupyterlite_xeus.spec.ts-snapshots/jupyter-xeus-execute-env2-default-linux.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyterlite/xeus/506c0c818b94b5097c65b76456b01308e5e6248c/ui-tests/tests/jupyterlite_xeus.spec.ts-snapshots/jupyter-xeus-execute-env2-default-linux.png -------------------------------------------------------------------------------- /ui-tests/tests/jupyterlite_xeus.spec.ts-snapshots/jupyter-xeus-launcher-crossoriginisolated-linux.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyterlite/xeus/506c0c818b94b5097c65b76456b01308e5e6248c/ui-tests/tests/jupyterlite_xeus.spec.ts-snapshots/jupyter-xeus-launcher-crossoriginisolated-linux.png -------------------------------------------------------------------------------- /ui-tests/tests/jupyterlite_xeus.spec.ts-snapshots/jupyter-xeus-launcher-default-linux.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyterlite/xeus/506c0c818b94b5097c65b76456b01308e5e6248c/ui-tests/tests/jupyterlite_xeus.spec.ts-snapshots/jupyter-xeus-launcher-default-linux.png --------------------------------------------------------------------------------