├── .github └── workflows │ └── release.yaml ├── .gitignore ├── LICENSE ├── MANIFEST.in ├── README.md ├── allthekernels.py ├── atk └── kernel.json.tpl ├── img └── allthekernels.png ├── pyproject.toml ├── setup.cfg ├── setup.py └── test_atk.ipynb /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | # Build releases and (on tags) publish to PyPI 2 | name: Release 3 | 4 | # always build releases (to make sure wheel-building works) 5 | # but only publish to PyPI on tags 6 | on: 7 | push: 8 | pull_request: 9 | 10 | jobs: 11 | build-release: 12 | runs-on: ubuntu-20.04 13 | steps: 14 | - uses: actions/checkout@v2 15 | - uses: actions/setup-python@v2 16 | with: 17 | python-version: "3.10" 18 | 19 | - name: install build package 20 | run: | 21 | pip install --upgrade pip 22 | pip install build 23 | pip freeze 24 | 25 | - name: build release 26 | run: | 27 | python -m build --sdist --wheel . 28 | ls -l dist 29 | 30 | - uses: actions/upload-artifact@v3 31 | with: 32 | name: allthekernels-${{ github.sha }} 33 | path: "dist/*" 34 | if-no-files-found: error 35 | 36 | - name: publish to pypi 37 | uses: pypa/gh-action-pypi-publish@v1.5.1 38 | if: startsWith(github.ref, 'refs/tags/') 39 | with: 40 | user: __token__ 41 | password: ${{ secrets.pypi_password }} 42 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Generated kernelspec 2 | atk/kernel.json 3 | 4 | # Byte-compiled / optimized / DLL files 5 | __pycache__/ 6 | *.py[cod] 7 | *$py.class 8 | 9 | # C extensions 10 | *.so 11 | 12 | # Distribution / packaging 13 | MANIFEST 14 | .Python 15 | env/ 16 | build/ 17 | develop-eggs/ 18 | dist/ 19 | downloads/ 20 | eggs/ 21 | .eggs/ 22 | lib/ 23 | lib64/ 24 | parts/ 25 | sdist/ 26 | var/ 27 | *.egg-info/ 28 | .installed.cfg 29 | *.egg 30 | 31 | # PyInstaller 32 | # Usually these files are written by a python script from a template 33 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 34 | *.manifest 35 | *.spec 36 | 37 | # Installer logs 38 | pip-log.txt 39 | pip-delete-this-directory.txt 40 | 41 | # Unit test / coverage reports 42 | htmlcov/ 43 | .tox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *,cover 50 | .hypothesis/ 51 | 52 | # Translations 53 | *.mo 54 | *.pot 55 | 56 | # Django stuff: 57 | *.log 58 | 59 | # intellij idea 60 | .idea 61 | 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | #Ipython Notebook 70 | .ipynb_checkpoints 71 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Min RK 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | include LICENSE 3 | include img 4 | include atk/kernel.json.tpl 5 | 6 | prune htmlcov 7 | prune build 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # All the Kernels! 2 | 3 | A Jupyter kernel that multiplexes all the kernels you have installed. 4 | 5 | ## Installation 6 | 7 | 1. Download and extract the contents of this repository or clone it with git clone https://github.com/minrk/allthekernels.git 8 | 2. Install with: 9 | 10 | ``` 11 | python3 -m pip install allthekernels 12 | ``` 13 | Or to do a dev install from source: 14 | ``` 15 | python3 -m pip install -e . 16 | # manually install kernelspec in your environment, e.g.: 17 | # mkdir ~/mambaforge/envs/allthekernels/share/jupyter/kernels/atk 18 | # cp atk/kernel.json ~/mambaforge/envs/allthekernels/share/jupyter/kernels/atk 19 | ``` 20 | 21 | ## Usage 22 | 23 | Specify which kernel a cell should use with `>kernelname`. 24 | If no kernel is specified, IPython will be used. 25 | 26 | ![atk](img/allthekernels.png) 27 | 28 | ([All the things source](http://hyperboleandahalf.blogspot.no/2010/06/this-is-why-ill-never-be-adult.html)) 29 | 30 | ## Making a release 31 | 32 | Anyone with push access to this repo can make a release. 33 | We use [tbump][] to publish releases. 34 | 35 | tbump updates version numbers and publishes the `git tag` of the version. 36 | [Our GitHub Actions](https://github.com/minrk/allthekernels/actions) 37 | then build the releases and publish them to PyPI. 38 | 39 | The steps involved: 40 | 41 | 1. install tbump: `pip install tbump` 42 | 2. tag and publish the release `tbump $NEW_VERSION`. 43 | 44 | That's it! 45 | 46 | [tbump]: https://github.com/your-tools/tbump 47 | -------------------------------------------------------------------------------- /allthekernels.py: -------------------------------------------------------------------------------- 1 | """All the Kernels: all your kernels in one kernel. 2 | 3 | Like magic! 4 | """ 5 | 6 | import os 7 | import sys 8 | import asyncio 9 | from tornado.ioloop import IOLoop 10 | 11 | import zmq 12 | from zmq.eventloop import ioloop 13 | ioloop.install() 14 | from zmq.eventloop.future import Context 15 | 16 | from traitlets import Dict 17 | 18 | from jupyter_client import KernelManager 19 | from ipykernel.kernelbase import Kernel 20 | from ipykernel.kernelapp import IPKernelApp 21 | 22 | 23 | banner = """\ 24 | All The Kernels: A single Jupyter kernel that multiplexes. 25 | 26 | Per default, all cells will be executed in the default python kernel. If the 27 | first line of a cell starts with `>`, the line will be parsed as kernel name and 28 | the rest of the cell will be executed in that kernel. 29 | 30 | For instance, 31 | 32 | >python2 33 | def foo(): 34 | ... 35 | 36 | will run the cell in a Python 2 kernel, and 37 | 38 | >julia-0.4 39 | 40 | will run in Julia 0.4, etc. 41 | 42 | You can also set a new default kernel by prefixing the kernel name with `!`: 43 | 44 | >!ir 45 | 46 | In this case the current cell and all further cells without a kernel name will 47 | be executed in an R kernel. 48 | """ 49 | 50 | 51 | __version__ = '1.3.0.dev' 52 | 53 | 54 | class KernelProxy(object): 55 | """A proxy for a single kernel 56 | 57 | 58 | Hooks up relay of messages on the shell channel. 59 | """ 60 | def __init__(self, manager, shell_upstream, iopub_upstream, context): 61 | self.manager = manager 62 | self.session = manager.session 63 | self.shell = self.manager.connect_shell() 64 | self.shell_upstream = shell_upstream 65 | self.iopub_upstream = iopub_upstream 66 | self.iosub = context.socket(zmq.SUB) 67 | self.iosub.subscribe = b'' 68 | self.iosub.connect(self.manager._make_url('iopub')) 69 | IOLoop.current().add_callback(self.relay_shell) 70 | IOLoop.current().add_callback(self.relay_iopub) 71 | self.shell_reply_event = asyncio.Event() # to track if on shell channel the reply message has been received 72 | 73 | async def relay_shell(self): 74 | """Coroutine for relaying any shell replies""" 75 | while True: 76 | msg = await self.shell.recv_multipart() 77 | self.shell_reply_event.set() # the status of shell_reply_event is changed to set when the reply is received 78 | self.shell_upstream.send_multipart(msg) 79 | 80 | async def relay_iopub(self): 81 | """Coroutine for relaying IOPub messages from all of our kernels""" 82 | while True: 83 | raw_msg = await self.iosub.recv_multipart() 84 | ident, msg = self.session.feed_identities(raw_msg) 85 | msg = self.session.deserialize(msg) 86 | if (msg["msg_type"] != "status"): 87 | self.iopub_upstream.send_multipart(raw_msg) 88 | # our status is published when replying to the request. 89 | 90 | 91 | class AllTheKernels(Kernel): 92 | """Kernel class for proxying ALL THE KERNELS YOU HAVE""" 93 | implementation = 'AllTheKernels' 94 | implementation_version = __version__ 95 | language_info = { 96 | 'name': 'all-of-them', 97 | 'mimetype': 'text/plain', 98 | } 99 | banner = banner 100 | 101 | kernels = Dict() 102 | default_kernel = os.environ.get('ATK_DEFAULT_KERNEL') or 'python%i' % (sys.version_info[0]) 103 | _atk_parent = None 104 | 105 | def __init__(self, *args, **kwargs): 106 | super().__init__(*args, **kwargs) 107 | self.future_context = Context() 108 | self.shell_stream = self.shell_streams[0] 109 | 110 | def start_kernel(self, name): 111 | """Start a new kernel""" 112 | base, ext = os.path.splitext(self.parent.connection_file) 113 | cf = '{base}-{name}{ext}'.format( 114 | base=base, 115 | name=name, 116 | ext=ext, 117 | ) 118 | manager = KernelManager( 119 | kernel_name=name, 120 | session=self.session, 121 | context=self.future_context, 122 | connection_file=cf, 123 | ) 124 | manager.start_kernel() 125 | self.kernels[name] = KernelProxy( 126 | manager=manager, 127 | shell_upstream=self.shell_stream, 128 | iopub_upstream=self.iopub_socket, 129 | context=self.future_context 130 | ) 131 | 132 | return self.kernels[name] 133 | 134 | def get_kernel(self, name): 135 | """Get a kernel, start it if it doesn't exist""" 136 | if name not in self.kernels: 137 | self.start_kernel(name) 138 | return self.kernels[name] 139 | 140 | def set_parent(self, ident, parent, channel="shell"): 141 | # record the parent message 142 | self._atk_parent = parent 143 | return super().set_parent(ident, parent, channel) 144 | 145 | def split_cell(self, cell): 146 | """Return the kernel name and remaining cell contents 147 | 148 | If no kernel name is specified, use the default kernel. 149 | """ 150 | if not cell.startswith('>'): 151 | # no kernel magic, use default kernel 152 | return self.default_kernel, cell 153 | split = cell.split('\n', 1) 154 | if len(split) == 2: 155 | first_line, cell = split 156 | else: 157 | first_line = cell 158 | cell = '' 159 | kernel_name = first_line[1:].strip() 160 | if kernel_name[0] == "!": 161 | # >!kernelname sets it as the new default 162 | kernel_name = kernel_name[1:].strip() 163 | self.default_kernel = kernel_name 164 | return kernel_name, cell 165 | 166 | async def relay_to_kernel(self, stream, ident, parent): 167 | """Relay a message to the kernel 168 | 169 | Gets the `>kernel` line off of the cell, 170 | finds the kernel (starts it if necessary), 171 | then relays the request. 172 | """ 173 | content = parent['content'] 174 | cell = content['code'] 175 | kernel_name, cell = self.split_cell(cell) 176 | content['code'] = cell 177 | kernel_client = self.get_kernel(kernel_name) 178 | self.log.debug("Relaying %s to %s", parent['header']['msg_type'], kernel_name) 179 | self.session.send(kernel_client.shell, parent, ident=ident) 180 | await kernel_client.shell_reply_event.wait() # waiting till shell_reply event status is 'set' 181 | kernel_client.shell_reply_event.clear() # then the event's status is changed to 'unset' 182 | 183 | execute_request = relay_to_kernel 184 | inspect_request = relay_to_kernel 185 | complete_request = relay_to_kernel 186 | 187 | def do_shutdown(self, restart): 188 | for kernel in self.kernels.values(): 189 | kernel.manager.shutdown_kernel(False, restart) 190 | return super().do_shutdown(restart) 191 | 192 | 193 | class AllTheKernelsApp(IPKernelApp): 194 | 195 | kernel_class = AllTheKernels 196 | # disable IO capture 197 | outstream_class = None 198 | 199 | def _log_level_default(self): 200 | return 10 201 | 202 | 203 | main = AllTheKernelsApp.launch_instance 204 | 205 | 206 | if __name__ == '__main__': 207 | main() 208 | -------------------------------------------------------------------------------- /atk/kernel.json.tpl: -------------------------------------------------------------------------------- 1 | { 2 | "display_name": "All the Kernels", 3 | "argv": [ 4 | "$python", 5 | "-m", 6 | "allthekernels", 7 | "-f", 8 | "{connection_file}" 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /img/allthekernels.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/minrk/allthekernels/16f70d11727c4c81dc5591b2f9f840d271d624c6/img/allthekernels.png -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.tbump] 2 | # Uncomment this if your project is hosted on GitHub: 3 | github_url = "https://github.com/minrk/allthekernels" 4 | 5 | [tool.tbump.version] 6 | current = "1.3.0.dev" 7 | 8 | # Example of a semver regexp. 9 | # Make sure this matches current_version before 10 | # using tbump 11 | regex = ''' 12 | (?P\d+) 13 | \. 14 | (?P\d+) 15 | \. 16 | (?P\d+) 17 | (?P(\.dev|a|b|rc)\d*) 18 | ''' 19 | 20 | [tool.tbump.git] 21 | message_template = "Bump to {new_version}" 22 | tag_template = "{new_version}" 23 | 24 | [[tool.tbump.file]] 25 | src = "allthekernels.py" 26 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal=1 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | import string 5 | 6 | from setuptools import setup 7 | from setuptools.command.bdist_egg import bdist_egg 8 | 9 | 10 | class bdist_egg_disabled(bdist_egg): 11 | """Disabled version of bdist_egg 12 | 13 | Prevents setup.py install from performing setuptools' default easy_install, 14 | which it should never ever do. 15 | """ 16 | def run(self): 17 | sys.exit("Aborting implicit building of eggs. Use `pip install .` to install from source.") 18 | 19 | 20 | with open("README.md") as f: 21 | long_description = f.read() 22 | 23 | 24 | version_ns = {} 25 | with open('allthekernels.py') as f: 26 | for line in f: 27 | if line.startswith('__version__'): 28 | exec(line, version_ns) 29 | 30 | 31 | here = os.path.dirname(__file__) 32 | 33 | 34 | setup_args = dict( 35 | name='allthekernels', 36 | version=version_ns['__version__'], 37 | author="Min RK", 38 | author_email="benjaminrk@gmail.com", 39 | description="Multiplexing kernel for Jupyter", 40 | long_description=long_description, 41 | long_description_content_type="text/markdown", 42 | url="https://github.com/minrk/allthekernels", 43 | py_modules=['allthekernels'], 44 | data_files=[('share/jupyter/kernels/atk', ['atk/kernel.json'])], 45 | license="MIT", 46 | cmdclass={"bdist_egg": bdist_egg_disabled}, 47 | python_requires=">=3.5", 48 | classifiers=[ 49 | "Development Status :: 3 - Alpha", 50 | "Intended Audience :: Developers", 51 | "License :: OSI Approved :: MIT License", 52 | "Programming Language :: Python :: 3", 53 | ], 54 | install_requires=[ 55 | 'ipykernel>=6.0', 56 | 'jupyter-client>=5.0', 57 | 'pyzmq>=15.2', 58 | 'tornado>=5', 59 | ], 60 | ) 61 | 62 | 63 | def kernelspec(executable): 64 | with open(os.path.join(here, 'atk', 'kernel.json.tpl')) as input_file: 65 | template = string.Template(input_file.read()) 66 | text = template.safe_substitute({ 'python': executable }) 67 | with open(os.path.join(here, 'atk', 'kernel.json'), 'w') as output_file: 68 | output_file.write(text) 69 | 70 | 71 | # When building a wheel, the executable specified in the kernelspec is simply 'python'. 72 | if any(a.startswith('bdist') for a in sys.argv): 73 | kernelspec(executable='python') 74 | 75 | # When installing, the kernel executable path is set to `sys.executable`. 76 | if any(a.startswith(("install", "develop")) for a in sys.argv): 77 | kernelspec(executable=sys.executable) 78 | 79 | 80 | if __name__ == "__main__": 81 | setup(**setup_args) 82 | -------------------------------------------------------------------------------- /test_atk.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "91db32cc-d897-4d2c-b66b-0fe720f3f148", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | ">xpython\n", 11 | "import time\n", 12 | "for i in range(5):\n", 13 | " time.sleep(1.0)\n", 14 | " print('i:',i)" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": null, 20 | "id": "2b0a0592-62f4-4ac0-a91f-7b1e8c3689a6", 21 | "metadata": {}, 22 | "outputs": [], 23 | "source": [ 24 | ">python3\n", 25 | "import time\n", 26 | "for i in range(10):\n", 27 | " time.sleep(0.25)\n", 28 | " print('i:',i)" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": null, 34 | "id": "80802388-060a-4e75-96c1-32de4b0b8fed", 35 | "metadata": {}, 36 | "outputs": [], 37 | "source": [ 38 | ">xpython\n", 39 | "import time\n", 40 | "for i in range(5):\n", 41 | " time.sleep(1.0)\n", 42 | " print('i:',i)" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": null, 48 | "id": "61771bc9-1361-4347-b27c-be7f24837bfc", 49 | "metadata": {}, 50 | "outputs": [], 51 | "source": [ 52 | ">python3\n", 53 | "import time\n", 54 | "for i in range(10):\n", 55 | " time.sleep(0.25)\n", 56 | " print('i:',i)" 57 | ] 58 | } 59 | ], 60 | "metadata": { 61 | "kernelspec": { 62 | "display_name": "All the Kernels", 63 | "language": "", 64 | "name": "atk" 65 | }, 66 | "language_info": { 67 | "mimetype": "text/plain", 68 | "name": "all-of-them" 69 | } 70 | }, 71 | "nbformat": 4, 72 | "nbformat_minor": 5 73 | } 74 | --------------------------------------------------------------------------------