├── .github └── workflows │ └── ci.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .readthedocs.yaml ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── binder ├── environment.yml └── postBuild ├── docs ├── Makefile ├── _static │ ├── GitHub-Mark-32px.png │ ├── download-solid.svg │ └── jupyter_rfb.svg ├── conf.py ├── contributing.rst ├── events.rst ├── examples │ └── index.rst ├── guide.rst ├── index.rst ├── make.bat ├── reference.rst └── widget.rst ├── examples ├── grid.ipynb ├── hello_world.ipynb ├── interaction1.ipynb ├── interaction2.ipynb ├── ipywidgets_embed.ipynb ├── performance.ipynb ├── push.ipynb ├── pygfx1.ipynb ├── pygfx2.ipynb ├── pygfx3.ipynb ├── video.ipynb ├── visibility_check.ipynb ├── vispy1.ipynb └── wgpu1.ipynb ├── install.json ├── js ├── .eslintrc.json ├── README.md ├── amd-public-path.js ├── lib │ ├── extension.js │ ├── index.js │ ├── labplugin.js │ └── widget.js ├── package.json └── webpack.config.js ├── jupyter_rfb.json ├── jupyter_rfb ├── __init__.py ├── _jpg.py ├── _png.py ├── _utils.py ├── _version.py ├── events.py └── widget.py ├── pyproject.toml ├── release.py └── tests ├── test_jpg.py ├── test_png.py ├── test_utils.py └── test_widget.py /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | 10 | jobs: 11 | 12 | lint: 13 | name: Lint 14 | runs-on: ubuntu-latest 15 | strategy: 16 | fail-fast: false 17 | steps: 18 | - uses: actions/checkout@v4 19 | - name: Set up Python 20 | uses: actions/setup-python@v5 21 | with: 22 | python-version: 3.13 23 | - name: Install dependencies 24 | run: | 25 | python -m pip install --upgrade pip 26 | pip install ruff 27 | - name: Ruff lint 28 | run: | 29 | ruff check --output-format=github . 30 | - name: Ruff format 31 | run: | 32 | ruff format --check . 33 | 34 | tests: 35 | name: ${{ matrix.name }} 36 | runs-on: ${{ matrix.os }} 37 | strategy: 38 | fail-fast: false 39 | matrix: 40 | include: 41 | # Python versions 42 | - name: Linux py39 43 | os: ubuntu-latest 44 | pyversion: '3.9' 45 | - name: Linux py310 46 | os: ubuntu-latest 47 | pyversion: '3.10' 48 | - name: Linux py311 49 | os: ubuntu-latest 50 | pyversion: '3.11' 51 | - name: Linux py312 52 | os: ubuntu-latest 53 | pyversion: '3.12' 54 | - name: Linux py313 55 | os: ubuntu-latest 56 | pyversion: '3.13' 57 | # Other systems / interpreters 58 | - name: Linux pypy3 59 | os: ubuntu-latest 60 | pyversion: 'pypy3.9' 61 | - name: Windows py311 62 | os: windows-latest 63 | pyversion: '3.11' 64 | - name: MacOS py311 65 | os: macos-latest 66 | pyversion: '3.11' 67 | - name: Linux py313 complete 68 | os: ubuntu-latest 69 | pyversion: '3.13' 70 | alltests: true 71 | 72 | steps: 73 | 74 | - uses: actions/checkout@v4 75 | - name: Set up Python ${{ matrix.pyversion }} 76 | uses: actions/setup-python@v5 77 | with: 78 | python-version: ${{ matrix.pyversion }} 79 | - name: Setup node 80 | uses: actions/setup-node@v4 81 | with: 82 | node-version: 18 83 | - name: Install dependencies 84 | shell: bash 85 | run: | 86 | python -m pip install --upgrade pip 87 | pip install .[tests] 88 | rm -rf ./jupyter_rfb ./build ./egg-info 89 | - name: Install more dependencies 90 | if: ${{ matrix.alltests }} 91 | shell: bash 92 | run: | 93 | pip install -U pillow opencv-python-headless 94 | - name: Test with pytest 95 | shell: bash 96 | run: | 97 | python -c "import sys; print(sys.version, '\n', sys.prefix)"; 98 | pytest -v . 99 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | js/package-lock.json 2 | *.egg-info/ 3 | .ipynb_checkpoints/ 4 | dist/ 5 | build/ 6 | *.py[cod] 7 | node_modules/ 8 | docs/site/ 9 | docs/_build 10 | docs/examples/*.ipynb 11 | .coverage 12 | htmlcov/ 13 | 14 | # Compiled javascript 15 | jupyter_rfb/nbextension/ 16 | jupyter_rfb/labextension/ 17 | js/yarn.lock 18 | 19 | # OS X 20 | .DS_Store 21 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/astral-sh/ruff-pre-commit 3 | rev: v0.7.0 4 | hooks: 5 | - id: ruff 6 | args: [ --fix ] 7 | - id: ruff-format 8 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | build: 3 | os: ubuntu-22.04 4 | tools: 5 | nodejs: '18' 6 | python: '3.13' 7 | apt_packages: 8 | - freeglut3-dev 9 | - xvfb 10 | - x11-utils 11 | jobs: 12 | post_system_dependencies: 13 | - nohup Xvfb $DISPLAY -screen 0 1400x900x24 -dpi 96 +extension RANDR +render & 14 | pre_install: 15 | - npm install -g yarn 16 | sphinx: 17 | builder: html 18 | configuration: docs/conf.py 19 | fail_on_warning: true 20 | python: 21 | install: 22 | - method: pip 23 | path: . 24 | extra_requirements: 25 | - docs 26 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributor's Guide 2 | 3 | See `the contributor guide in our docs `_. 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021-2025 - jupyter_rfb contributors 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # jupyter_rfb 2 | 3 | Remote Frame Buffer for Jupyter 4 | 5 | [![PyPI version](https://badge.fury.io/py/jupyter-rfb.svg)](https://badge.fury.io/py/jupyter-rfb) 6 | [![CI](https://github.com/vispy/jupyter_rfb/actions/workflows/ci.yml/badge.svg)](https://github.com/vispy/jupyter_rfb/actions) 7 | [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/vispy/jupyter_rfb/main?urlpath=lab/tree/examples/hello_world.ipynb) 8 | 9 | ## Introduction 10 | 11 | The `jupyter_rfb` library provides a widget (an `ipywidgets` subclass) 12 | that can be used in the Jupyter notebook and in JupyterLab to implement 13 | a remote frame-buffer. 14 | 15 | Images that are generated at the server are streamed to the client 16 | (Jupyter) where they are shown. Events (such as mouse interactions) are 17 | streamed in the other direction, where the server can react by 18 | generating new images. 19 | 20 | This *remote-frame-buffer* approach can be an effective method for 21 | server-generated visualizations to be dispayed in Jupyter notebook/lab. For 22 | example visualization created by tools like vispy, datoviz or pygfx. 23 | 24 | 25 | ## Scope 26 | 27 | The above defines the full scope of this library; it's a base widget 28 | that other libraries can extend for different purposes. Consequently, 29 | these libraries don't have to each invent a Jupyter widget, and in 30 | *this* library we can focus on doing that one task really well. 31 | 32 | 33 | ## Installation 34 | 35 | To install use pip: 36 | 37 | $ pip install jupyter_rfb 38 | 39 | For better performance, also ``pip install simplejpeg`` or ``pip install pillow``. 40 | On older versions of Jupyter notebook/lab an extra step might be needed 41 | to enable the widget. 42 | 43 | To install into an existing conda environment: 44 | 45 | $ conda install -c conda-forge jupyter-rfb 46 | 47 | 48 | ## Developer notes 49 | 50 | See [the contributor guide](https://jupyter-rfb.readthedocs.io/en/stable/contributing.html) on how to install ``jupyter_rfb`` 51 | in a dev environment, and on how to contribute. 52 | 53 | 54 | ## License 55 | 56 | MIT 57 | -------------------------------------------------------------------------------- /binder/environment.yml: -------------------------------------------------------------------------------- 1 | name: jupyter_rfb 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - pip: 6 | - jupyter_rfb 7 | - numpy 8 | - Pillow 9 | - imageio 10 | - imageio-ffmpeg 11 | - vispy 12 | - glfw 13 | - wgpu 14 | -------------------------------------------------------------------------------- /binder/postBuild: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/_static/GitHub-Mark-32px.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vispy/jupyter_rfb/f84110f2f53eca9f7a99484039bced49a4309ebc/docs/_static/GitHub-Mark-32px.png -------------------------------------------------------------------------------- /docs/_static/download-solid.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | """Configuration script for Sphinx.""" 2 | 3 | import os 4 | import sys 5 | import json 6 | 7 | 8 | ROOT_DIR = os.path.abspath(os.path.join(__file__, "..", "..")) 9 | sys.path.insert(0, ROOT_DIR) 10 | 11 | import ipywidgets # noqa: F401, E402 12 | 13 | import jupyter_rfb # noqa: E402 14 | 15 | 16 | def insert_examples(): 17 | """Copy notebooks from examples dir to docs and create an index.""" 18 | dir1 = os.path.join(ROOT_DIR, "examples") 19 | dir2 = os.path.join(ROOT_DIR, "docs", "examples") 20 | # Collect examples 21 | examples_names = [] 22 | for fname in os.listdir(dir1): 23 | if fname.endswith(".ipynb") and not fname.startswith("_"): 24 | examples_names.append(fname) 25 | examples_names.sort(key=lambda f: "0" + f if f.startswith("hello") else f) 26 | # Clear current example notebooks 27 | for fname in os.listdir(dir2): 28 | if fname.endswith(".ipynb"): 29 | os.remove(os.path.join(dir2, fname)) 30 | # Copy fresh examples over 31 | for fname in examples_names: 32 | # shutil.copy(os.path.join(dir1, fname), os.path.join(dir2, fname)) 33 | with open(os.path.join(dir1, fname), "rb") as f: 34 | d = json.loads(f.read().decode()) 35 | jupyter_rfb.remove_rfb_models_from_nb(d) 36 | with open(os.path.join(dir2, fname), "wb") as f: 37 | f.write(json.dumps(d, indent=2).encode()) 38 | 39 | 40 | insert_examples() 41 | 42 | 43 | # -- Project information ----------------------------------------------------- 44 | 45 | project = "jupyter_rfb" 46 | copyright = "2021-2025, jupyter_rfb contributors" 47 | author = "Almar Klein" 48 | 49 | 50 | # -- General configuration --------------------------------------------------- 51 | 52 | # Add any Sphinx extension module names here, as strings. They can be 53 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 54 | # ones. 55 | extensions = [ 56 | "sphinx.ext.autodoc", 57 | "sphinx.ext.napoleon", 58 | "nbsphinx", 59 | ] 60 | 61 | nbsphinx_execute = "never" 62 | nbsphinx_prolog = """ 63 | .. raw:: html 64 | 65 | 84 | 85 |
86 | 89 | 90 | View notebook on Github 91 | 92 | 93 | 96 | 97 | Download 98 | 99 | 100 | 103 | 104 | 105 | 106 |
107 |
108 | 109 | """ 110 | 111 | # Add any paths that contain templates here, relative to this directory. 112 | templates_path = ["_templates"] 113 | 114 | # List of patterns, relative to source directory, that match files and 115 | # directories to ignore when looking for source files. 116 | # This pattern also affects html_static_path and html_extra_path. 117 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] 118 | 119 | 120 | # -- Options for HTML output ------------------------------------------------- 121 | 122 | # The theme to use for HTML and HTML Help pages. See the documentation for 123 | # a list of builtin themes. 124 | # 125 | # html_theme = 'alabaster' 126 | 127 | # Add any paths that contain custom static files (such as style sheets) here, 128 | # relative to this directory. They are copied after the builtin static files, 129 | # so a file named "default.css" will overwrite the builtin "default.css". 130 | html_static_path = ["_static"] 131 | -------------------------------------------------------------------------------- /docs/contributing.rst: -------------------------------------------------------------------------------- 1 | The jupyter_rfb contributor guide 2 | ================================= 3 | 4 | This page is for those who plan to hack on ``jupyter_rfb`` or make other contributions. 5 | 6 | 7 | How can I contribute? 8 | --------------------- 9 | 10 | Anyone can contribute to ``jupyter_rfb``. We strive for a welcoming environment - 11 | see our `code of conduct `_. 12 | 13 | Contribution can vary from reporting bugs, suggesting improvements, 14 | help improving documentations. We also welcome improvements in the code and tests. 15 | We uphold high standards for the code, and we'll help you achieve that. 16 | 17 | 18 | Install jupyter_rfb in development mode 19 | --------------------------------------- 20 | 21 | For a development installation (requires Node.js and Yarn): 22 | 23 | .. code-block:: 24 | 25 | $ git clone https://github.com/vispy/jupyter_rfb.git 26 | $ cd jupyter_rfb 27 | $ pip install -e .[dev] 28 | $ jupyter nbextension install --py --symlink --overwrite --sys-prefix jupyter_rfb 29 | $ jupyter nbextension enable --py --sys-prefix jupyter_rfb 30 | 31 | When actively developing the JavaScript code, run the command: 32 | 33 | .. code-block:: 34 | 35 | $ jupyter labextension develop --overwrite jupyter_rfb 36 | 37 | Then you need to rebuild the JS when you make a code change: 38 | 39 | .. code-block:: 40 | 41 | $ cd js 42 | $ yarn run build 43 | 44 | You then need to refresh the JupyterLab page when your javascript changes. 45 | 46 | 47 | Automated tools 48 | --------------- 49 | 50 | To make it easier to keep the code valid and clean, we use the following tools: 51 | 52 | * Run ``ruff format`` to autoformat the code. 53 | * Run ``ruff check`` for linting and formatting checks. 54 | * Run ``python release.py`` to do a release (for maintainers only). 55 | 56 | 57 | Autocommit hook 58 | --------------- 59 | 60 | Optionally, you can setup an autocommit hook to automatically run these on each commit: 61 | 62 | .. code-block:: 63 | 64 | $ pip install pre-commit 65 | $ pre-commit install 66 | 67 | 68 | Tips to test changes made to code 69 | --------------------------------- 70 | 71 | In general you should not have to restart the server when working on the code of jupyter_rfb: 72 | 73 | * When Python code has changed: restart and clear all outputs. 74 | * When the JavaScript code has changed: rebuild with yarn, and then refresh (F5) the page. 75 | -------------------------------------------------------------------------------- /docs/events.rst: -------------------------------------------------------------------------------- 1 | The jupyter_rfb event spec 2 | -------------------------- 3 | 4 | .. automodule:: jupyter_rfb.events 5 | -------------------------------------------------------------------------------- /docs/examples/index.rst: -------------------------------------------------------------------------------- 1 | The jupyter_rfb examples 2 | ======================== 3 | 4 | This is a list of example notebooks demonstrating the use of jupyter_rfb: 5 | 6 | .. toctree:: 7 | :maxdepth: 2 8 | :glob: 9 | 10 | * 11 | -------------------------------------------------------------------------------- /docs/guide.rst: -------------------------------------------------------------------------------- 1 | The jupyter_rfb guide 2 | ===================== 3 | 4 | Installation 5 | ------------ 6 | 7 | Install with pip: 8 | 9 | .. code-block:: 10 | 11 | pip install -U jupyter_rfb 12 | 13 | Or to install into a conda environment: 14 | 15 | .. code-block:: 16 | 17 | conda install -c conda-forge jupyter-rfb 18 | 19 | For better performance, also install ``simplejpeg`` or ``pillow``. 20 | 21 | If you plan to hack on this library, see the :doc:`contributor guide ` 22 | for a dev installation and more. 23 | 24 | 25 | Subclassing the widget 26 | ---------------------- 27 | 28 | The provided :class:`RemoteFrameBuffer ` class cannot do much by itself, but it provides 29 | a basis for widgets that want to generate images at the server, and be informed 30 | of user events. The way to use ``jupyter_rfb`` is therefore to create a subclass 31 | and implement two specific methods. 32 | 33 | The first method to implement is :func:`.get_frame() `, which should return a uint8 numpy array. For example: 34 | 35 | .. code-block:: py 36 | 37 | class MyRemoteFrameBuffer(jupyter_rfb.RemoteFrameBuffer): 38 | 39 | def get_frame(self): 40 | return np.random.uniform(0, 255, (100,100)).astype(np.uint8) 41 | 42 | The second method to implement is :func:`.handle_event() `, 43 | which accepts an event object. This is where you can react to changes 44 | and user interactions. The most important one may be the resize event, 45 | so that you can match the array size to the region on screen. For 46 | example: 47 | 48 | .. code-block:: py 49 | 50 | class MyRemoteFrameBuffer(jupyter_rfb.RemoteFrameBuffer): 51 | 52 | def handle_event(self, event): 53 | event_type = event["event_type"] 54 | if event_type == "resize": 55 | self.logical_size = event["width"], event["height"] 56 | self.pixel_ratio = event["pixel_ratio"] 57 | elif event_type == "pointer_down": 58 | pass # ... 59 | 60 | 61 | Logical vs physical pixels 62 | -------------------------- 63 | 64 | The size of e.g. the resize event is expressed in logical pixels. This 65 | is a unit of measurement that changes as the user changes the browser 66 | zoom level. 67 | 68 | By multiplying the logical size with the pixel-ratio, you obtain the 69 | physical size, which represents the actual pixels of the screen. With 70 | a zoom level of 100% the pixel-ratio is 1 on a regular display and 2 71 | on a HiDPI display, although the actual values may also be affected by 72 | the OS's zoom level. 73 | 74 | 75 | Scheduling draws 76 | ---------------- 77 | 78 | The :func:`.get_frame() ` 79 | method is called automatically when a new draw is 80 | performed. There are cases when the widget knows that a redraw is 81 | (probably) required, such as when the widget is resized. 82 | 83 | If you want to trigger a redraw (e.g. because certain state has 84 | changed in reaction to user interaction), you can call 85 | :func:`.request_draw() ` to schedule a new draw. 86 | 87 | The scheduling of draws is done in such a way to avoid images being 88 | produced faster than the client can consume them - a process known as 89 | throttling. In more detail: the client sends a confirmation for each 90 | frame that it receives, and the server waits with producing a new frame 91 | until the client has confirmed receiving the nth latest frame. This 92 | mechanism causes the calls to :func:`.get_frame() ` 93 | to match the speed by which 94 | the frames can be communicated and displayed. This helps minimize the 95 | lag and optimize the FPS. 96 | 97 | 98 | Event throttling 99 | ---------------- 100 | 101 | Events go from the client (browser) to the server (Python). Some of 102 | these are throttled so they are emitted a maximum number of times per 103 | second. This is to avoid spamming the communication channel and server 104 | process. The throttling applies to the resize, scroll, and pointer_move 105 | events. 106 | 107 | 108 | Taking snapshots 109 | ---------------- 110 | 111 | In a notebook, the :meth:`.snapshot() ` 112 | method can be used to create a picture of the current state of the 113 | widget. This image remains visible when the notebook is in off-line 114 | mode (e.g. in nbviewer). This functionality can be convenient if you're 115 | using a notebook to tell a story, and you want to display a certain 116 | result that is still visible in off-line mode. 117 | 118 | When a widget is first displayed, it automatically creates a 119 | snapshot, which is hidden by default, but becomes visible when the 120 | widget itself is not loaded. In other words, example notebooks 121 | have pretty pictures! 122 | 123 | 124 | Exceptions and logging 125 | ---------------------- 126 | 127 | The :func:`.handle_event() ` 128 | and :func:`.get_frame() ` 129 | methods are called from a Jupyter 130 | COM event and in an asyncio task, respectively. Under these circumstances, 131 | Jupyter Lab/Notebook may swallow exceptions as well as writes to stdout/stderr. 132 | See `issue #35 `_ for details. 133 | These are limitation of Jupyter, and we should expect these to be fixed/improved in the future. 134 | 135 | In jupyter_rfb we take measures to make exceptions raised in 136 | either of these methods result in a traceback shown right above the 137 | widget. To ensure that calls to ``print()`` in these methods are also 138 | shown, use ``self.print()`` instead. 139 | 140 | Note that any other streaming to stdout and stderr (e.g. logging) may 141 | not become visible anywhere. 142 | 143 | 144 | Measuring statistics 145 | -------------------- 146 | 147 | The :class:`RemoteFrameBuffer ` class has a 148 | method :func:`.get_stats() ` that 149 | returns a dict with performance metrics: 150 | 151 | .. code-block:: py 152 | 153 | >>> w.reset_stats() # start measuring 154 | ... interact or run a test 155 | >>> w.get_stats() 156 | { 157 | ... 158 | } 159 | 160 | 161 | Performance tips 162 | ---------------- 163 | 164 | The framerate that can be obtained depends on a number of factors: 165 | 166 | * The size of a frame: smaller frames generally encode faster and result 167 | in smaller blobs, causing less strain on both CPU and IO. 168 | * How many widgets are drawing simultaneously: they use the same communication channel. 169 | * The ``widget.quality`` trait: lower quality results in faster encoding and smaller blobs. 170 | * When using lossless images (``widget.quality == 100``), the entropy 171 | (information density) of a frame also matters, because for PNG, high entropy 172 | data takes longer to compress and results in larger blobs. 173 | 174 | For more details about performance considerations in the implementation of ``jupyter_rfb``, 175 | see `issue #3 `_. 176 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | 2 | Welcome to the jupyter_rfb docs! 3 | ================================ 4 | 5 | The `jupyter_rfb` library provides a widget (an `ipywidgets` subclass) 6 | that can be used in the Jupyter notebook and in JupyterLab to realize 7 | a *remote frame buffer*. 8 | 9 | .. image:: _static/jupyter_rfb.svg 10 | :width: 500 11 | :alt: Remote Frame Buffer explained 12 | 13 | Images that are generated at the server are streamed to the client 14 | (Jupyter) where they are shown. Events (such as mouse interactions) are 15 | streamed in the other direction, where the server can react by 16 | generating new images. 17 | 18 | This *remote-frame-buffer* approach can be an effective method for 19 | server-generated visualizations to be dispayed in Jupyter notebook/lab. For 20 | example visualization created by tools like vispy, datoviz or pygfx. 21 | 22 | 23 | Contents 24 | -------- 25 | 26 | .. toctree:: 27 | :maxdepth: 2 28 | 29 | Guide 30 | Reference 31 | Examples 32 | Contributor's guide 33 | 34 | 35 | Indices and tables 36 | ------------------ 37 | 38 | * :ref:`genindex` 39 | * :ref:`modindex` 40 | * :ref:`search` 41 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/reference.rst: -------------------------------------------------------------------------------- 1 | The jupyter_rfb reference 2 | ========================= 3 | 4 | 5 | .. toctree:: 6 | :maxdepth: 1 7 | 8 | widget.rst 9 | events.rst 10 | -------------------------------------------------------------------------------- /docs/widget.rst: -------------------------------------------------------------------------------- 1 | RemoteFrameBuffer class 2 | ----------------------- 3 | 4 | .. autoclass:: jupyter_rfb.RemoteFrameBuffer 5 | :members: 6 | :member-order: bysource 7 | -------------------------------------------------------------------------------- /examples/grid.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "4b310a3a", 6 | "metadata": {}, 7 | "source": [ 8 | "# Grid Example\n", 9 | "\n", 10 | "Show a grid, to check that it's aligned correctly in terms of physical pixels. One can see how during the resizing, when the data has not updated to the new size, the Moiré effect temporarily occurs. " 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": 1, 16 | "id": "d5327490", 17 | "metadata": {}, 18 | "outputs": [ 19 | { 20 | "data": { 21 | "application/vnd.jupyter.widget-view+json": { 22 | "model_id": "0f32c2c9f5cc4ae182c9319c50b44540", 23 | "version_major": 2, 24 | "version_minor": 0 25 | }, 26 | "text/plain": [ 27 | "RFBOutputContext()" 28 | ] 29 | }, 30 | "metadata": {}, 31 | "output_type": "display_data" 32 | }, 33 | { 34 | "data": { 35 | "application/vnd.jupyter.widget-view+json": { 36 | "model_id": "da51a5e8e63f47c28f7c26c269e81136", 37 | "version_major": 2, 38 | "version_minor": 0 39 | }, 40 | "text/html": [ 41 | "
snapshot
" 42 | ], 43 | "text/plain": [ 44 | "Grid()" 45 | ] 46 | }, 47 | "execution_count": 1, 48 | "metadata": {}, 49 | "output_type": "execute_result" 50 | } 51 | ], 52 | "source": [ 53 | "import numpy as np\n", 54 | "import jupyter_rfb\n", 55 | "\n", 56 | "\n", 57 | "class Grid(jupyter_rfb.RemoteFrameBuffer):\n", 58 | " def handle_event(self, event):\n", 59 | " if event[\"event_type\"] == \"resize\":\n", 60 | " self._size = event\n", 61 | " # self.print(event) # uncomment to display the event\n", 62 | "\n", 63 | " def get_frame(self):\n", 64 | " w, h, r = self._size[\"width\"], self._size[\"height\"], self._size[\"pixel_ratio\"]\n", 65 | " physical_size = int(h * r), int(w * r)\n", 66 | " a = np.zeros((physical_size[0], physical_size[1], 3), np.uint8)\n", 67 | " self.draw_grid(a)\n", 68 | " return a\n", 69 | "\n", 70 | " def draw_grid(self, a):\n", 71 | " a[::2, : a.shape[1] // 2] = 255\n", 72 | " a[: a.shape[0] // 2, ::2] = 255\n", 73 | "\n", 74 | "\n", 75 | "w = Grid()\n", 76 | "w" 77 | ] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "execution_count": 2, 82 | "id": "99036999", 83 | "metadata": {}, 84 | "outputs": [ 85 | { 86 | "data": { 87 | "application/vnd.jupyter.widget-view+json": { 88 | "model_id": "b28caefcd2224e4f94e3a78a961077cf", 89 | "version_major": 2, 90 | "version_minor": 0 91 | }, 92 | "text/plain": [ 93 | "RFBOutputContext()" 94 | ] 95 | }, 96 | "metadata": {}, 97 | "output_type": "display_data" 98 | }, 99 | { 100 | "data": { 101 | "application/vnd.jupyter.widget-view+json": { 102 | "model_id": "867783e4be764337a24bdae7c9a9494e", 103 | "version_major": 2, 104 | "version_minor": 0 105 | }, 106 | "text/html": [ 107 | "
snapshot
" 108 | ], 109 | "text/plain": [ 110 | "Grid2()" 111 | ] 112 | }, 113 | "execution_count": 2, 114 | "metadata": {}, 115 | "output_type": "execute_result" 116 | } 117 | ], 118 | "source": [ 119 | "class Grid2(Grid):\n", 120 | " def draw_grid(self, a):\n", 121 | " a[::4, : a.shape[1] // 2] = 255\n", 122 | " a[1::4, : a.shape[1] // 2] = 255\n", 123 | " a[: a.shape[0] // 2, ::4] = 255\n", 124 | " a[: a.shape[0] // 2, 1::4] = 255\n", 125 | "\n", 126 | "\n", 127 | "w = Grid2()\n", 128 | "w" 129 | ] 130 | } 131 | ], 132 | "metadata": { 133 | "kernelspec": { 134 | "display_name": "Python 3 (ipykernel)", 135 | "language": "python", 136 | "name": "python3" 137 | }, 138 | "language_info": { 139 | "codemirror_mode": { 140 | "name": "ipython", 141 | "version": 3 142 | }, 143 | "file_extension": ".py", 144 | "mimetype": "text/x-python", 145 | "name": "python", 146 | "nbconvert_exporter": "python", 147 | "pygments_lexer": "ipython3", 148 | "version": "3.9.9" 149 | } 150 | }, 151 | "nbformat": 4, 152 | "nbformat_minor": 5 153 | } 154 | -------------------------------------------------------------------------------- /examples/hello_world.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "f6315c1a", 6 | "metadata": {}, 7 | "source": [ 8 | "# Hello world example\n", 9 | "\n", 10 | "In this example we demonstrate the very basics of the ``RemoteFrameBuffer`` class." 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": 1, 16 | "id": "eb328d8f", 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "import numpy as np\n", 21 | "import jupyter_rfb" 22 | ] 23 | }, 24 | { 25 | "cell_type": "markdown", 26 | "id": "627cd505", 27 | "metadata": {}, 28 | "source": [ 29 | "We start by implementing ``get_frame()`` to produce an image." 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": 2, 35 | "id": "7691480a", 36 | "metadata": {}, 37 | "outputs": [ 38 | { 39 | "data": { 40 | "application/vnd.jupyter.widget-view+json": { 41 | "model_id": "c6b1dd169b524a58af707fc23e8691aa", 42 | "version_major": 2, 43 | "version_minor": 0 44 | }, 45 | "text/plain": [ 46 | "RFBOutputContext()" 47 | ] 48 | }, 49 | "metadata": {}, 50 | "output_type": "display_data" 51 | }, 52 | { 53 | "data": { 54 | "application/vnd.jupyter.widget-view+json": { 55 | "model_id": "d41f4c8a51974d56ac50551e4ae1edc9", 56 | "version_major": 2, 57 | "version_minor": 0 58 | }, 59 | "text/html": [ 60 | "
snapshot
" 61 | ], 62 | "text/plain": [ 63 | "HelloWorld1()" 64 | ] 65 | }, 66 | "execution_count": 2, 67 | "metadata": {}, 68 | "output_type": "execute_result" 69 | } 70 | ], 71 | "source": [ 72 | "class HelloWorld1(jupyter_rfb.RemoteFrameBuffer):\n", 73 | " def get_frame(self):\n", 74 | " a = np.zeros((100, 100, 3), np.uint8)\n", 75 | " a[20:-20, 20:-20, 1] = 255\n", 76 | " return a\n", 77 | "\n", 78 | "\n", 79 | "w = HelloWorld1()\n", 80 | "w" 81 | ] 82 | }, 83 | { 84 | "cell_type": "markdown", 85 | "id": "91b9bcd3", 86 | "metadata": {}, 87 | "source": [ 88 | "Let's make it a bit more advanced. By keeping track of the widget size, we can provide an array with matching shape. We also take pixel_ratio into account, in case this is a hidpi display, or when the user has used the browser's zoom." 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": 3, 94 | "id": "45578bd6", 95 | "metadata": {}, 96 | "outputs": [ 97 | { 98 | "data": { 99 | "application/vnd.jupyter.widget-view+json": { 100 | "model_id": "0bf1c8a39d1348d6a9ad1206b22e7f31", 101 | "version_major": 2, 102 | "version_minor": 0 103 | }, 104 | "text/plain": [ 105 | "RFBOutputContext()" 106 | ] 107 | }, 108 | "metadata": {}, 109 | "output_type": "display_data" 110 | }, 111 | { 112 | "data": { 113 | "application/vnd.jupyter.widget-view+json": { 114 | "model_id": "e5c53bbaf1df4464bcf9fa39bfc8f605", 115 | "version_major": 2, 116 | "version_minor": 0 117 | }, 118 | "text/html": [ 119 | "
snapshot
" 120 | ], 121 | "text/plain": [ 122 | "HelloWorld2()" 123 | ] 124 | }, 125 | "execution_count": 3, 126 | "metadata": {}, 127 | "output_type": "execute_result" 128 | } 129 | ], 130 | "source": [ 131 | "class HelloWorld2(jupyter_rfb.RemoteFrameBuffer):\n", 132 | " def handle_event(self, event):\n", 133 | " if event[\"event_type\"] == \"resize\":\n", 134 | " self._size = event\n", 135 | " # self.print(event) # uncomment to display the event\n", 136 | "\n", 137 | " def get_frame(self):\n", 138 | " w, h, r = self._size[\"width\"], self._size[\"height\"], self._size[\"pixel_ratio\"]\n", 139 | " physical_size = int(h * r), int(w * r)\n", 140 | " a = np.zeros((physical_size[0], physical_size[1], 3), np.uint8)\n", 141 | " margin = int(20 * r)\n", 142 | " a[margin:-margin, margin:-margin, 1] = 255\n", 143 | " return a\n", 144 | "\n", 145 | "\n", 146 | "w = HelloWorld2()\n", 147 | "w" 148 | ] 149 | }, 150 | { 151 | "cell_type": "markdown", 152 | "id": "a9cfd8f7", 153 | "metadata": {}, 154 | "source": [ 155 | "If this is a live session, try resizing the widget to see how it adjusts." 156 | ] 157 | }, 158 | { 159 | "cell_type": "code", 160 | "execution_count": null, 161 | "id": "00c53e7e", 162 | "metadata": {}, 163 | "outputs": [], 164 | "source": [] 165 | } 166 | ], 167 | "metadata": { 168 | "kernelspec": { 169 | "display_name": "Python 3 (ipykernel)", 170 | "language": "python", 171 | "name": "python3" 172 | }, 173 | "language_info": { 174 | "codemirror_mode": { 175 | "name": "ipython", 176 | "version": 3 177 | }, 178 | "file_extension": ".py", 179 | "mimetype": "text/x-python", 180 | "name": "python", 181 | "nbconvert_exporter": "python", 182 | "pygments_lexer": "ipython3", 183 | "version": "3.9.9" 184 | } 185 | }, 186 | "nbformat": 4, 187 | "nbformat_minor": 5 188 | } 189 | -------------------------------------------------------------------------------- /examples/interaction1.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "5c8c5b03", 6 | "metadata": {}, 7 | "source": [ 8 | "# Interaction example\n", 9 | "\n", 10 | "In this example we implement a simple interaction use-case. This lets you get a feel for the performance (FPS, lag). Note that the snappyness will depend on where the server is (e.g. localhost will work better than MyBinder).\n", 11 | "\n", 12 | "The app presents a dark background with cyan square that can be dragged around." 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": 1, 18 | "id": "a1297418", 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "import numpy as np\n", 23 | "import jupyter_rfb" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": 2, 29 | "id": "c7d172bd", 30 | "metadata": {}, 31 | "outputs": [ 32 | { 33 | "data": { 34 | "application/vnd.jupyter.widget-view+json": { 35 | "model_id": "216a3a9b96cf4516b5672cd5ada95a48", 36 | "version_major": 2, 37 | "version_minor": 0 38 | }, 39 | "text/plain": [ 40 | "RFBOutputContext()" 41 | ] 42 | }, 43 | "metadata": {}, 44 | "output_type": "display_data" 45 | }, 46 | { 47 | "data": { 48 | "application/vnd.jupyter.widget-view+json": { 49 | "model_id": "fe50a182a0d342bfad7639c071c30f3c", 50 | "version_major": 2, 51 | "version_minor": 0 52 | }, 53 | "text/html": [ 54 | "
snapshot
" 55 | ], 56 | "text/plain": [ 57 | "InteractionApp()" 58 | ] 59 | }, 60 | "execution_count": 2, 61 | "metadata": {}, 62 | "output_type": "execute_result" 63 | } 64 | ], 65 | "source": [ 66 | "class InteractionApp(jupyter_rfb.RemoteFrameBuffer):\n", 67 | " def __init__(self):\n", 68 | " super().__init__()\n", 69 | " self._size = (1, 1, 1)\n", 70 | " self._pos = 100, 100\n", 71 | " self._radius = 20\n", 72 | " self._drag_pos = None\n", 73 | "\n", 74 | " def handle_event(self, event):\n", 75 | " event_type = event.get(\"event_type\", None)\n", 76 | " if event_type == \"resize\":\n", 77 | " self._size = event[\"width\"], event[\"height\"], event[\"pixel_ratio\"]\n", 78 | " elif event_type == \"pointer_down\" and event[\"button\"] == 1:\n", 79 | " x, y = event[\"x\"], event[\"y\"]\n", 80 | " if (\n", 81 | " abs(x - self._pos[0]) < self._radius\n", 82 | " and abs(y - self._pos[1]) < self._radius\n", 83 | " ):\n", 84 | " self._drag_pos = self._pos[0] - x, self._pos[1] - y\n", 85 | " self.request_draw()\n", 86 | " elif event_type == \"pointer_up\":\n", 87 | " self._drag_pos = None\n", 88 | " self.request_draw()\n", 89 | " elif event_type == \"pointer_move\" and self._drag_pos is not None:\n", 90 | " self._pos = self._drag_pos[0] + event[\"x\"], self._drag_pos[1] + event[\"y\"]\n", 91 | " self.request_draw()\n", 92 | "\n", 93 | " def get_frame(self):\n", 94 | " ratio = self._size[2]\n", 95 | " radius = self._radius\n", 96 | " w, h = int(self._size[0] * ratio), int(self._size[1] * ratio)\n", 97 | " array = np.zeros((h, w, 3), np.uint8)\n", 98 | " array[:, :, 2] = np.linspace(50, 200, h).reshape(-1, 1) # bg gradient\n", 99 | " array[\n", 100 | " int(ratio * (self._pos[1] - radius)) : int(ratio * (self._pos[1] + radius)),\n", 101 | " int(ratio * (self._pos[0] - radius)) : int(ratio * (self._pos[0] + radius)),\n", 102 | " 1,\n", 103 | " ] = 250 if self._drag_pos else 200\n", 104 | " return array\n", 105 | "\n", 106 | "\n", 107 | "w = InteractionApp()\n", 108 | "w.max_buffered_frames = 2\n", 109 | "w" 110 | ] 111 | }, 112 | { 113 | "cell_type": "markdown", 114 | "id": "b7010f8e", 115 | "metadata": {}, 116 | "source": [ 117 | "You can now interact with the figure by dragging the square to another position." 118 | ] 119 | }, 120 | { 121 | "cell_type": "code", 122 | "execution_count": 3, 123 | "id": "75112087", 124 | "metadata": {}, 125 | "outputs": [], 126 | "source": [ 127 | "# Or we can do that programatically :)\n", 128 | "w.handle_event({\"event_type\": \"pointer_down\", \"button\": 1, \"x\": 100, \"y\": 100})\n", 129 | "w.handle_event({\"event_type\": \"pointer_move\", \"button\": 1, \"x\": 200, \"y\": 200})\n", 130 | "w.handle_event({\"event_type\": \"pointer_up\", \"button\": 1, \"x\": 200, \"y\": 200})" 131 | ] 132 | }, 133 | { 134 | "cell_type": "code", 135 | "execution_count": 4, 136 | "id": "7cd81a00", 137 | "metadata": {}, 138 | "outputs": [ 139 | { 140 | "data": { 141 | "text/html": [ 142 | "
snapshot
" 143 | ], 144 | "text/plain": [ 145 | "" 146 | ] 147 | }, 148 | "execution_count": 4, 149 | "metadata": {}, 150 | "output_type": "execute_result" 151 | } 152 | ], 153 | "source": [ 154 | "w.snapshot()" 155 | ] 156 | }, 157 | { 158 | "cell_type": "markdown", 159 | "id": "5fd9316e", 160 | "metadata": {}, 161 | "source": [ 162 | "To get some quantative resuls, run ``reset_stats()``, interact, and then call ``get_stats()``." 163 | ] 164 | }, 165 | { 166 | "cell_type": "code", 167 | "execution_count": 5, 168 | "id": "2dd857a7", 169 | "metadata": {}, 170 | "outputs": [], 171 | "source": [ 172 | "w.reset_stats()" 173 | ] 174 | }, 175 | { 176 | "cell_type": "code", 177 | "execution_count": 6, 178 | "id": "082fa326", 179 | "metadata": {}, 180 | "outputs": [ 181 | { 182 | "data": { 183 | "text/plain": [ 184 | "{'sent_frames': 0,\n", 185 | " 'confirmed_frames': 0,\n", 186 | " 'roundtrip': 0.0,\n", 187 | " 'delivery': 0.0,\n", 188 | " 'img_encoding': 0.0,\n", 189 | " 'b64_encoding': 0.0,\n", 190 | " 'fps': 0.0}" 191 | ] 192 | }, 193 | "execution_count": 6, 194 | "metadata": {}, 195 | "output_type": "execute_result" 196 | } 197 | ], 198 | "source": [ 199 | "w.get_stats()" 200 | ] 201 | }, 202 | { 203 | "cell_type": "code", 204 | "execution_count": null, 205 | "id": "b366c263", 206 | "metadata": {}, 207 | "outputs": [], 208 | "source": [] 209 | } 210 | ], 211 | "metadata": { 212 | "kernelspec": { 213 | "display_name": "Python 3 (ipykernel)", 214 | "language": "python", 215 | "name": "python3" 216 | }, 217 | "language_info": { 218 | "codemirror_mode": { 219 | "name": "ipython", 220 | "version": 3 221 | }, 222 | "file_extension": ".py", 223 | "mimetype": "text/x-python", 224 | "name": "python", 225 | "nbconvert_exporter": "python", 226 | "pygments_lexer": "ipython3", 227 | "version": "3.9.9" 228 | } 229 | }, 230 | "nbformat": 4, 231 | "nbformat_minor": 5 232 | } 233 | -------------------------------------------------------------------------------- /examples/interaction2.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "28b38e38", 6 | "metadata": {}, 7 | "source": [ 8 | "# Interactive drawing example\n", 9 | "\n", 10 | "A simple drawing app:\n", 11 | "\n", 12 | "* Draw dots by clicking with LMB.\n", 13 | "* Toggle color by clicking RMB." 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": 1, 19 | "id": "47315bf0", 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "import numpy as np\n", 24 | "import jupyter_rfb" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": 2, 30 | "id": "fb037e0b", 31 | "metadata": {}, 32 | "outputs": [ 33 | { 34 | "data": { 35 | "application/vnd.jupyter.widget-view+json": { 36 | "model_id": "", 37 | "version_major": 2, 38 | "version_minor": 0 39 | }, 40 | "text/plain": [ 41 | "RFBOutputContext()" 42 | ] 43 | }, 44 | "metadata": {}, 45 | "output_type": "display_data" 46 | }, 47 | { 48 | "data": { 49 | "application/vnd.jupyter.widget-view+json": { 50 | "model_id": "", 51 | "version_major": 2, 52 | "version_minor": 0 53 | }, 54 | "text/html": [ 55 | "
snapshot
" 56 | ], 57 | "text/plain": [ 58 | "Drawingapp(css_height='400px', css_width='600px', resizable=False)" 59 | ] 60 | }, 61 | "execution_count": 2, 62 | "metadata": {}, 63 | "output_type": "execute_result" 64 | } 65 | ], 66 | "source": [ 67 | "class Drawingapp(jupyter_rfb.RemoteFrameBuffer):\n", 68 | " def __init__(self):\n", 69 | " super().__init__()\n", 70 | " self.pixel_ratio = 1 / 8\n", 71 | " w, h = 600, 400\n", 72 | " self.css_width = f\"{w}px\"\n", 73 | " self.css_height = f\"{h}px\"\n", 74 | " self.resizable = False\n", 75 | " self.array = (\n", 76 | " np.ones((int(h * self.pixel_ratio), int(w * self.pixel_ratio), 4), np.uint8)\n", 77 | " * 5\n", 78 | " )\n", 79 | " self.pen_colors = [(1, 0.2, 0, 1), (0, 1, 0.2, 1), (0.2, 0, 1, 1)]\n", 80 | " self.pen_index = 0\n", 81 | "\n", 82 | " def handle_event(self, event):\n", 83 | " event_type = event.get(\"event_type\", None)\n", 84 | " if event_type == \"pointer_down\":\n", 85 | " if event[\"button\"] == 1:\n", 86 | " # Draw\n", 87 | " x = int(event[\"x\"] * self.pixel_ratio)\n", 88 | " y = int(event[\"y\"] * self.pixel_ratio)\n", 89 | " self.array[y, x] = 255 * np.array(self.pen_colors[self.pen_index])\n", 90 | " self.request_draw()\n", 91 | " elif event[\"button\"] == 2:\n", 92 | " # Toggle color\n", 93 | " self.pen_index = (self.pen_index + 1) % len(self.pen_colors)\n", 94 | "\n", 95 | " def get_frame(self):\n", 96 | " return self.array\n", 97 | "\n", 98 | "\n", 99 | "app = Drawingapp()\n", 100 | "app" 101 | ] 102 | }, 103 | { 104 | "cell_type": "markdown", 105 | "id": "4efdc5dc", 106 | "metadata": {}, 107 | "source": [ 108 | "After some clicking ..." 109 | ] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "execution_count": 3, 114 | "id": "1f6c0ab9", 115 | "metadata": {}, 116 | "outputs": [], 117 | "source": [ 118 | "# We can also generate some clicks programatically :)\n", 119 | "for x, y in [\n", 120 | " (503, 37),\n", 121 | " (27, 182),\n", 122 | " (182, 383),\n", 123 | " (396, 235),\n", 124 | " (477, 151),\n", 125 | " (328, 308),\n", 126 | " (281, 16),\n", 127 | "]:\n", 128 | " app.handle_event({\"event_type\": \"pointer_down\", \"button\": 1, \"x\": x, \"y\": y})\n", 129 | "app.handle_event({\"event_type\": \"pointer_down\", \"button\": 2, \"x\": 0, \"y\": 0})\n", 130 | "for x, y in [\n", 131 | " (226, 115),\n", 132 | " (135, 253),\n", 133 | " (351, 220),\n", 134 | " (57, 11),\n", 135 | " (345, 87),\n", 136 | " (67, 175),\n", 137 | " (559, 227),\n", 138 | "]:\n", 139 | " app.handle_event({\"event_type\": \"pointer_down\", \"button\": 1, \"x\": x, \"y\": y})" 140 | ] 141 | }, 142 | { 143 | "cell_type": "code", 144 | "execution_count": 4, 145 | "id": "c9b1a03d", 146 | "metadata": {}, 147 | "outputs": [ 148 | { 149 | "data": { 150 | "text/html": [ 151 | "
snapshot
" 152 | ], 153 | "text/plain": [ 154 | "" 155 | ] 156 | }, 157 | "execution_count": 4, 158 | "metadata": {}, 159 | "output_type": "execute_result" 160 | } 161 | ], 162 | "source": [ 163 | "app.snapshot()" 164 | ] 165 | }, 166 | { 167 | "cell_type": "code", 168 | "execution_count": 5, 169 | "id": "f48342b8", 170 | "metadata": {}, 171 | "outputs": [], 172 | "source": [ 173 | "app.request_draw()" 174 | ] 175 | }, 176 | { 177 | "cell_type": "code", 178 | "execution_count": null, 179 | "id": "94b13fdc", 180 | "metadata": {}, 181 | "outputs": [], 182 | "source": [] 183 | } 184 | ], 185 | "metadata": { 186 | "kernelspec": { 187 | "display_name": "Python 3 (ipykernel)", 188 | "language": "python", 189 | "name": "python3" 190 | }, 191 | "language_info": { 192 | "codemirror_mode": { 193 | "name": "ipython", 194 | "version": 3 195 | }, 196 | "file_extension": ".py", 197 | "mimetype": "text/x-python", 198 | "name": "python", 199 | "nbconvert_exporter": "python", 200 | "pygments_lexer": "ipython3", 201 | "version": "3.9.9" 202 | } 203 | }, 204 | "nbformat": 4, 205 | "nbformat_minor": 5 206 | } 207 | -------------------------------------------------------------------------------- /examples/ipywidgets_embed.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "f6315c1a", 6 | "metadata": {}, 7 | "source": [ 8 | "# Embedding in an ipywidgets app\n", 9 | "\n", 10 | "In this example we demonstrate embedding the ``RemoteFrameBuffer`` class inside a larger ipywidgets app." 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": null, 16 | "id": "eb328d8f", 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "import numpy as np\n", 21 | "import ipywidgets\n", 22 | "import jupyter_rfb" 23 | ] 24 | }, 25 | { 26 | "cell_type": "markdown", 27 | "id": "627cd505", 28 | "metadata": {}, 29 | "source": [ 30 | "Implement a simple RFB class, for the sake of the example:" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": null, 36 | "id": "7691480a", 37 | "metadata": {}, 38 | "outputs": [], 39 | "source": [ 40 | "class SimpleRFB(jupyter_rfb.RemoteFrameBuffer):\n", 41 | " green_value = 200\n", 42 | "\n", 43 | " def get_frame(self):\n", 44 | " a = np.zeros((100, 100, 3), np.uint8)\n", 45 | " a[20:-20, 20:-20, 1] = self.green_value\n", 46 | " return a" 47 | ] 48 | }, 49 | { 50 | "cell_type": "markdown", 51 | "id": "91b9bcd3", 52 | "metadata": {}, 53 | "source": [ 54 | "Compose a simple app:" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": null, 60 | "id": "45578bd6", 61 | "metadata": {}, 62 | "outputs": [], 63 | "source": [ 64 | "slider = ipywidgets.IntSlider(min=50, max=255, value=200)\n", 65 | "rfb = SimpleRFB()\n", 66 | "\n", 67 | "\n", 68 | "def on_slider_change(change):\n", 69 | " rfb.green_value = change[\"new\"]\n", 70 | " rfb.request_draw()\n", 71 | "\n", 72 | "\n", 73 | "slider.observe(on_slider_change, names=\"value\")\n", 74 | "ipywidgets.HBox([rfb, slider])" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": null, 80 | "id": "e7ee8d86-ab48-4e1d-8279-6bfbd53e5056", 81 | "metadata": {}, 82 | "outputs": [], 83 | "source": [] 84 | } 85 | ], 86 | "metadata": { 87 | "kernelspec": { 88 | "display_name": "Python 3 (ipykernel)", 89 | "language": "python", 90 | "name": "python3" 91 | }, 92 | "language_info": { 93 | "codemirror_mode": { 94 | "name": "ipython", 95 | "version": 3 96 | }, 97 | "file_extension": ".py", 98 | "mimetype": "text/x-python", 99 | "name": "python", 100 | "nbconvert_exporter": "python", 101 | "pygments_lexer": "ipython3", 102 | "version": "3.9.9" 103 | } 104 | }, 105 | "nbformat": 4, 106 | "nbformat_minor": 5 107 | } 108 | -------------------------------------------------------------------------------- /examples/performance.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "c40c06a2", 6 | "metadata": {}, 7 | "source": [ 8 | "# Performance measurements\n", 9 | "This notebook tries to push a number of frames to the browser as fast as posisble, using different parameters, and measures how fast it goes." 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "id": "fd3aa015", 16 | "metadata": {}, 17 | "outputs": [], 18 | "source": [ 19 | "import numpy as np\n", 20 | "from jupyter_rfb import RemoteFrameBuffer\n", 21 | "\n", 22 | "\n", 23 | "class PerformanceTester(RemoteFrameBuffer):\n", 24 | " i = 0\n", 25 | " n = 0\n", 26 | "\n", 27 | " def get_frame(self):\n", 28 | " if self.i >= self.n:\n", 29 | " return None\n", 30 | " array = np.zeros((640, 480), np.uint8)\n", 31 | " # array = np.random.uniform(0, 255, (640, 480)).astype(np.uint8)\n", 32 | " array[:20, : int(array.shape[1] * (self.i + 1) / self.n)] = 255\n", 33 | " self.i += 1\n", 34 | " self.request_draw() # keep going\n", 35 | " return array\n", 36 | "\n", 37 | " def run(self, n=100):\n", 38 | " self.i = 0\n", 39 | " self.n = n\n", 40 | " self.request_draw()\n", 41 | "\n", 42 | "\n", 43 | "w = PerformanceTester(css_height=\"100px\")" 44 | ] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "execution_count": null, 49 | "id": "6ea1d044", 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "w" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": null, 59 | "id": "cbc00754", 60 | "metadata": {}, 61 | "outputs": [], 62 | "source": [ 63 | "n = 50\n", 64 | "w.max_buffered_frames = 2\n", 65 | "w.reset_stats()\n", 66 | "w.run(n)" 67 | ] 68 | }, 69 | { 70 | "cell_type": "code", 71 | "execution_count": null, 72 | "id": "4b7c7dd0", 73 | "metadata": {}, 74 | "outputs": [], 75 | "source": [ 76 | "# Call this when it's done\n", 77 | "w.get_stats()" 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": null, 83 | "id": "c77e31a0", 84 | "metadata": {}, 85 | "outputs": [], 86 | "source": [] 87 | } 88 | ], 89 | "metadata": { 90 | "kernelspec": { 91 | "display_name": "Python 3 (ipykernel)", 92 | "language": "python", 93 | "name": "python3" 94 | }, 95 | "language_info": { 96 | "codemirror_mode": { 97 | "name": "ipython", 98 | "version": 3 99 | }, 100 | "file_extension": ".py", 101 | "mimetype": "text/x-python", 102 | "name": "python", 103 | "nbconvert_exporter": "python", 104 | "pygments_lexer": "ipython3", 105 | "version": "3.9.9" 106 | } 107 | }, 108 | "nbformat": 4, 109 | "nbformat_minor": 5 110 | } 111 | -------------------------------------------------------------------------------- /examples/push.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "f011941f", 6 | "metadata": {}, 7 | "source": [ 8 | "# Push example\n", 9 | "The default behavior of `jupyter_rfb` is to automatically call `get_frame()` when a new draw is requested and when the widget is ready for it. In use-cases where you want to push frames to the widget, you may prefer a different approach. Here is an example solution." 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "id": "c278a5e8", 16 | "metadata": {}, 17 | "outputs": [], 18 | "source": [ 19 | "import numpy as np\n", 20 | "from jupyter_rfb import RemoteFrameBuffer\n", 21 | "\n", 22 | "\n", 23 | "class FramePusher(RemoteFrameBuffer):\n", 24 | " def __init__(self, **kwargs):\n", 25 | " super().__init__(**kwargs)\n", 26 | " self._queue = []\n", 27 | "\n", 28 | " def push_frame(self, frame):\n", 29 | " self._queue.append(frame)\n", 30 | " self._queue[:-10] = [] # drop older frames if len > 10\n", 31 | " self.request_draw()\n", 32 | "\n", 33 | " def get_frame(self):\n", 34 | " if not self._queue:\n", 35 | " return\n", 36 | " self.request_draw()\n", 37 | " return self._queue.pop(0)\n", 38 | "\n", 39 | "\n", 40 | "w = FramePusher(css_width=\"100px\", css_height=\"100px\")\n", 41 | "w" 42 | ] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "execution_count": null, 47 | "id": "0bade53a", 48 | "metadata": {}, 49 | "outputs": [], 50 | "source": [ 51 | "w.push_frame(np.random.uniform(0, 255, (100, 100)).astype(np.uint8))" 52 | ] 53 | }, 54 | { 55 | "cell_type": "code", 56 | "execution_count": null, 57 | "id": "2156c042", 58 | "metadata": {}, 59 | "outputs": [], 60 | "source": [ 61 | "# Push 20 frames. Note that only the latest 10 will be shown\n", 62 | "for _ in range(20):\n", 63 | " w.push_frame(np.random.uniform(0, 255, (100, 100)).astype(np.uint8))\n", 64 | "len(w._queue)" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": null, 70 | "id": "1ed1d6a9", 71 | "metadata": {}, 72 | "outputs": [], 73 | "source": [] 74 | } 75 | ], 76 | "metadata": { 77 | "kernelspec": { 78 | "display_name": "Python 3 (ipykernel)", 79 | "language": "python", 80 | "name": "python3" 81 | }, 82 | "language_info": { 83 | "codemirror_mode": { 84 | "name": "ipython", 85 | "version": 3 86 | }, 87 | "file_extension": ".py", 88 | "mimetype": "text/x-python", 89 | "name": "python", 90 | "nbconvert_exporter": "python", 91 | "pygments_lexer": "ipython3", 92 | "version": "3.9.9" 93 | } 94 | }, 95 | "nbformat": 4, 96 | "nbformat_minor": 5 97 | } 98 | -------------------------------------------------------------------------------- /examples/pygfx1.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "f4ce92ce", 6 | "metadata": {}, 7 | "source": [ 8 | "# PyGfx cube example\n", 9 | "\n", 10 | "**Note that this example depends on pygfx (`pip install -U pygfx`).**\n", 11 | "\n", 12 | "An example showing a lit, textured, rotating cube. Tested against pygfx v0.7.0." 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": 1, 18 | "id": "f00886ec", 19 | "metadata": {}, 20 | "outputs": [ 21 | { 22 | "data": { 23 | "application/vnd.jupyter.widget-view+json": { 24 | "model_id": "646a74752fb7499497029e2c015c6012", 25 | "version_major": 2, 26 | "version_minor": 0 27 | }, 28 | "text/plain": [ 29 | "RFBOutputContext()" 30 | ] 31 | }, 32 | "metadata": {}, 33 | "output_type": "display_data" 34 | }, 35 | { 36 | "data": { 37 | "application/vnd.jupyter.widget-view+json": { 38 | "model_id": "bad2a2bbd04a4619a8274609b8509d56", 39 | "version_major": 2, 40 | "version_minor": 0 41 | }, 42 | "text/html": [ 43 | "
snapshot
" 44 | ], 45 | "text/plain": [ 46 | "JupyterWgpuCanvas()" 47 | ] 48 | }, 49 | "metadata": {}, 50 | "output_type": "display_data" 51 | } 52 | ], 53 | "source": [ 54 | "import pygfx as gfx\n", 55 | "import pylinalg as la\n", 56 | "\n", 57 | "cube = gfx.Mesh(\n", 58 | " gfx.box_geometry(200, 200, 200),\n", 59 | " gfx.MeshPhongMaterial(color=\"#336699\"),\n", 60 | ")\n", 61 | "\n", 62 | "\n", 63 | "def animate():\n", 64 | " rot = la.quat_from_euler((0.005, 0.01), order=\"XY\")\n", 65 | " cube.local.rotation = la.quat_mul(rot, cube.local.rotation)\n", 66 | "\n", 67 | "\n", 68 | "disp = gfx.Display()\n", 69 | "disp.before_render = animate\n", 70 | "disp.stats = True\n", 71 | "disp.show(cube)\n", 72 | "disp.canvas" 73 | ] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "execution_count": null, 78 | "id": "cc0ca8b1", 79 | "metadata": {}, 80 | "outputs": [], 81 | "source": [] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "execution_count": null, 86 | "id": "c964ed64", 87 | "metadata": {}, 88 | "outputs": [], 89 | "source": [] 90 | } 91 | ], 92 | "metadata": { 93 | "kernelspec": { 94 | "display_name": "Python 3 (ipykernel)", 95 | "language": "python", 96 | "name": "python3" 97 | }, 98 | "language_info": { 99 | "codemirror_mode": { 100 | "name": "ipython", 101 | "version": 3 102 | }, 103 | "file_extension": ".py", 104 | "mimetype": "text/x-python", 105 | "name": "python", 106 | "nbconvert_exporter": "python", 107 | "pygments_lexer": "ipython3", 108 | "version": "3.9.9" 109 | } 110 | }, 111 | "nbformat": 4, 112 | "nbformat_minor": 5 113 | } 114 | -------------------------------------------------------------------------------- /examples/pygfx3.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "3d571a3d", 6 | "metadata": {}, 7 | "source": [ 8 | "# PyGfx picking example\n", 9 | "\n", 10 | "**Note that this example depends on pygfx (`pip install -U pygfx`).**\n", 11 | "\n", 12 | "An example demonstrating pickable points." 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": 1, 18 | "id": "25eb8174", 19 | "metadata": {}, 20 | "outputs": [ 21 | { 22 | "data": { 23 | "application/vnd.jupyter.widget-view+json": { 24 | "model_id": "230eee3142eb4aaea283797ec340240f", 25 | "version_major": 2, 26 | "version_minor": 0 27 | }, 28 | "text/plain": [ 29 | "RFBOutputContext()" 30 | ] 31 | }, 32 | "metadata": {}, 33 | "output_type": "display_data" 34 | }, 35 | { 36 | "data": { 37 | "application/vnd.jupyter.widget-view+json": { 38 | "model_id": "8297e2f5538643dc8082c5133392dabb", 39 | "version_major": 2, 40 | "version_minor": 0 41 | }, 42 | "text/html": [ 43 | "
snapshot
" 44 | ], 45 | "text/plain": [ 46 | "PickingWgpuCanvas()" 47 | ] 48 | }, 49 | "execution_count": 1, 50 | "metadata": {}, 51 | "output_type": "execute_result" 52 | } 53 | ], 54 | "source": [ 55 | "import numpy as np\n", 56 | "import pygfx as gfx\n", 57 | "from wgpu.gui.jupyter import WgpuCanvas\n", 58 | "\n", 59 | "\n", 60 | "class PickingWgpuCanvas(WgpuCanvas):\n", 61 | " def handle_event(self, event):\n", 62 | " super().handle_event(event)\n", 63 | " # Get a dict with info about the clicked location\n", 64 | " if event[\"event_type\"] == \"pointer_down\":\n", 65 | " xy = event[\"x\"], event[\"y\"]\n", 66 | " info = renderer.get_pick_info(xy)\n", 67 | " wobject = info[\"world_object\"]\n", 68 | " # If a point was clicked ..\n", 69 | " if wobject and \"vertex_index\" in info:\n", 70 | " i = round(info[\"vertex_index\"])\n", 71 | " geometry.positions.data[i, 1] *= -1\n", 72 | " geometry.positions.update_range(i)\n", 73 | " canvas.request_draw()\n", 74 | "\n", 75 | "\n", 76 | "canvas = PickingWgpuCanvas()\n", 77 | "renderer = gfx.renderers.WgpuRenderer(canvas)\n", 78 | "scene = gfx.Scene()\n", 79 | "\n", 80 | "xx = np.linspace(-50, 50, 10)\n", 81 | "yy = np.random.uniform(20, 50, 10)\n", 82 | "geometry = gfx.Geometry(positions=[(x, y, 0) for x, y in zip(xx, yy)])\n", 83 | "if True: # Set to False to try this for a line\n", 84 | " ob = gfx.Points(geometry, gfx.PointsMaterial(color=(0, 1, 1, 1), size=16))\n", 85 | "else:\n", 86 | " ob = gfx.Line(geometry, gfx.LineMaterial(color=(0, 1, 1, 1), thickness=10))\n", 87 | "scene.add(ob)\n", 88 | "\n", 89 | "camera = gfx.OrthographicCamera(120, 120)\n", 90 | "\n", 91 | "canvas.request_draw(lambda: renderer.render(scene, camera))\n", 92 | "canvas" 93 | ] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "execution_count": null, 98 | "id": "21419674", 99 | "metadata": {}, 100 | "outputs": [], 101 | "source": [] 102 | } 103 | ], 104 | "metadata": { 105 | "kernelspec": { 106 | "display_name": "Python 3 (ipykernel)", 107 | "language": "python", 108 | "name": "python3" 109 | }, 110 | "language_info": { 111 | "codemirror_mode": { 112 | "name": "ipython", 113 | "version": 3 114 | }, 115 | "file_extension": ".py", 116 | "mimetype": "text/x-python", 117 | "name": "python", 118 | "nbconvert_exporter": "python", 119 | "pygments_lexer": "ipython3", 120 | "version": "3.9.9" 121 | } 122 | }, 123 | "nbformat": 4, 124 | "nbformat_minor": 5 125 | } 126 | -------------------------------------------------------------------------------- /examples/visibility_check.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "3a254e70", 6 | "metadata": {}, 7 | "source": [ 8 | "# Visibility check\n", 9 | "\n", 10 | "This notebook is to verify that widgets that are not visible do not perform any draws.\n" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": null, 16 | "id": "367f4c89", 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "import asyncio\n", 21 | "import numpy as np\n", 22 | "from jupyter_rfb import RemoteFrameBuffer" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": null, 28 | "id": "df8d3680", 29 | "metadata": {}, 30 | "outputs": [], 31 | "source": [ 32 | "class ProgressBar(RemoteFrameBuffer):\n", 33 | " i = 0\n", 34 | " n = 32\n", 35 | " channel = 0\n", 36 | " callback = lambda *args: None\n", 37 | "\n", 38 | " def get_frame(self):\n", 39 | " self.callback()\n", 40 | " self.i += 1\n", 41 | " if self.i >= self.n:\n", 42 | " self.i = 0\n", 43 | " array = np.zeros((100, 600, 3), np.uint8)\n", 44 | " array[:, : int(array.shape[1] * (self.i + 1) / self.n), self.channel] = 255\n", 45 | " return array\n", 46 | "\n", 47 | "\n", 48 | "class AutoDrawWidget(ProgressBar):\n", 49 | " channel = 2 # blue\n", 50 | "\n", 51 | " def __init__(self, **kwargs):\n", 52 | " super().__init__(**kwargs)\n", 53 | " loop = asyncio.get_event_loop()\n", 54 | " loop.create_task(self._keep_at_it())\n", 55 | "\n", 56 | " async def _keep_at_it(self):\n", 57 | " while True:\n", 58 | " await asyncio.sleep(0.5)\n", 59 | " self.request_draw()\n", 60 | "\n", 61 | "\n", 62 | "class IndicatorWidget(ProgressBar):\n", 63 | " channel = 1 # green\n", 64 | "\n", 65 | "\n", 66 | "indicator = IndicatorWidget(css_width=\"600px\", css_height=\"100px\")\n", 67 | "autodraw = AutoDrawWidget(css_width=\"600px\", css_height=\"100px\")\n", 68 | "autodraw.callback = lambda *args: indicator.request_draw()" 69 | ] 70 | }, 71 | { 72 | "cell_type": "markdown", 73 | "id": "9bc5e9ba", 74 | "metadata": {}, 75 | "source": [ 76 | "We display a widget that automatically keeps progressing. Actually, we create two views of that widgets to make sure that this works for multiple views." 77 | ] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "execution_count": null, 82 | "id": "47cc2c4d", 83 | "metadata": {}, 84 | "outputs": [], 85 | "source": [ 86 | "autodraw" 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": null, 92 | "id": "b6e695ef", 93 | "metadata": {}, 94 | "outputs": [], 95 | "source": [ 96 | "autodraw" 97 | ] 98 | }, 99 | { 100 | "cell_type": "markdown", 101 | "id": "f5120414", 102 | "metadata": {}, 103 | "source": [ 104 | "Some empty space ...\n", 105 | "\n", 106 | ".\n", 107 | "\n", 108 | ".\n", 109 | "\n", 110 | ".\n", 111 | "\n", 112 | ".\n", 113 | "\n", 114 | ".\n", 115 | "\n", 116 | ".\n", 117 | "\n", 118 | ".\n", 119 | "\n", 120 | ".\n", 121 | "\n", 122 | ".\n", 123 | "\n", 124 | ".\n", 125 | "\n", 126 | ".\n", 127 | "Then we display an indicator widget, that only progresses when the widget above is drawing.\n" 128 | ] 129 | }, 130 | { 131 | "cell_type": "code", 132 | "execution_count": null, 133 | "id": "45a05eb5", 134 | "metadata": {}, 135 | "outputs": [], 136 | "source": [ 137 | "indicator" 138 | ] 139 | }, 140 | { 141 | "cell_type": "markdown", 142 | "id": "8183275f", 143 | "metadata": {}, 144 | "source": [ 145 | "More empty space so there is something to scroll down to ...\n", 146 | "\n", 147 | ".\n", 148 | "\n", 149 | ".\n", 150 | "\n", 151 | ".\n", 152 | "\n", 153 | ".\n", 154 | "\n", 155 | ".\n", 156 | "\n", 157 | ".\n", 158 | "\n", 159 | ".\n", 160 | "\n", 161 | ".\n", 162 | "\n", 163 | ".\n", 164 | "\n", 165 | ".\n", 166 | "\n", 167 | ".\n", 168 | "\n", 169 | "." 170 | ] 171 | } 172 | ], 173 | "metadata": { 174 | "kernelspec": { 175 | "display_name": "Python 3 (ipykernel)", 176 | "language": "python", 177 | "name": "python3" 178 | }, 179 | "language_info": { 180 | "codemirror_mode": { 181 | "name": "ipython", 182 | "version": 3 183 | }, 184 | "file_extension": ".py", 185 | "mimetype": "text/x-python", 186 | "name": "python", 187 | "nbconvert_exporter": "python", 188 | "pygments_lexer": "ipython3", 189 | "version": "3.9.9" 190 | } 191 | }, 192 | "nbformat": 4, 193 | "nbformat_minor": 5 194 | } 195 | -------------------------------------------------------------------------------- /examples/vispy1.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "f3a10a5c", 6 | "metadata": {}, 7 | "source": [ 8 | "# Vispy example\n", 9 | "\n", 10 | "**Note that this example depends on the Vispy library, and that you need a bleeding edge version of Vispy to run this.**\n", 11 | "\n", 12 | "An example showing how jupyter_rfb is used in Vispy. Note that Vispy implements a subclass of `RemoteFrameBuffer` for this to work." 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": 1, 18 | "id": "be51c953", 19 | "metadata": {}, 20 | "outputs": [ 21 | { 22 | "data": { 23 | "application/vnd.jupyter.widget-view+json": { 24 | "model_id": "0522fb062b3544a8a782a9a7a0511ec8", 25 | "version_major": 2, 26 | "version_minor": 0 27 | }, 28 | "text/plain": [ 29 | "RFBOutputContext()" 30 | ] 31 | }, 32 | "metadata": {}, 33 | "output_type": "display_data" 34 | }, 35 | { 36 | "data": { 37 | "application/vnd.jupyter.widget-view+json": { 38 | "model_id": "6b5c6862cae9421594551dd8ec206d86", 39 | "version_major": 2, 40 | "version_minor": 0 41 | }, 42 | "text/html": [ 43 | "
snapshot
" 44 | ], 45 | "text/plain": [ 46 | "CanvasBackend(css_height='400px')" 47 | ] 48 | }, 49 | "execution_count": 1, 50 | "metadata": {}, 51 | "output_type": "execute_result" 52 | } 53 | ], 54 | "source": [ 55 | "from vispy import scene\n", 56 | "from vispy.visuals.transforms import STTransform\n", 57 | "\n", 58 | "canvas = scene.SceneCanvas(\n", 59 | " keys=\"interactive\", bgcolor=\"white\", size=(500, 400), show=True, resizable=True\n", 60 | ")\n", 61 | "\n", 62 | "view = canvas.central_widget.add_view()\n", 63 | "view.camera = \"arcball\"\n", 64 | "\n", 65 | "sphere1 = scene.visuals.Sphere(\n", 66 | " radius=1, method=\"latitude\", parent=view.scene, edge_color=\"black\"\n", 67 | ")\n", 68 | "\n", 69 | "sphere2 = scene.visuals.Sphere(\n", 70 | " radius=1, method=\"ico\", parent=view.scene, edge_color=\"black\"\n", 71 | ")\n", 72 | "\n", 73 | "sphere3 = scene.visuals.Sphere(\n", 74 | " radius=1,\n", 75 | " rows=10,\n", 76 | " cols=10,\n", 77 | " depth=10,\n", 78 | " method=\"cube\",\n", 79 | " parent=view.scene,\n", 80 | " edge_color=\"black\",\n", 81 | ")\n", 82 | "\n", 83 | "sphere1.transform = STTransform(translate=[-2.5, 0, 0])\n", 84 | "sphere3.transform = STTransform(translate=[2.5, 0, 0])\n", 85 | "\n", 86 | "view.camera.set_range(x=[-3, 3])\n", 87 | "canvas" 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "execution_count": null, 93 | "id": "8601aff2", 94 | "metadata": {}, 95 | "outputs": [], 96 | "source": [] 97 | } 98 | ], 99 | "metadata": { 100 | "kernelspec": { 101 | "display_name": "Python 3 (ipykernel)", 102 | "language": "python", 103 | "name": "python3" 104 | }, 105 | "language_info": { 106 | "codemirror_mode": { 107 | "name": "ipython", 108 | "version": 3 109 | }, 110 | "file_extension": ".py", 111 | "mimetype": "text/x-python", 112 | "name": "python", 113 | "nbconvert_exporter": "python", 114 | "pygments_lexer": "ipython3", 115 | "version": "3.9.9" 116 | } 117 | }, 118 | "nbformat": 4, 119 | "nbformat_minor": 5 120 | } 121 | -------------------------------------------------------------------------------- /install.json: -------------------------------------------------------------------------------- 1 | { 2 | "packageManager": "python", 3 | "packageName": "jupyter_rfb", 4 | "uninstallInstructions": "Use your Python package manager (pip, conda, etc.) to uninstall the package jupyter_rfb" 5 | } -------------------------------------------------------------------------------- /js/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "browser": true, 4 | "es2021": true 5 | }, 6 | "extends": [ 7 | "standard" 8 | ], 9 | "parserOptions": { 10 | "ecmaVersion": 12 11 | }, 12 | "rules": { 13 | "no-var": "off", 14 | "camelcase": "off", 15 | "prefer-const": "off", 16 | "indent": ["error", 4], 17 | "semi": ["error", "always"], 18 | "comma-dangle": ["error", "only-multiline"], 19 | "no-multiple-empty-lines": ["error", {"max": 2}] 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /js/README.md: -------------------------------------------------------------------------------- 1 | Remote Frame Buffer for Jupyter 2 | 3 | Package Install 4 | --------------- 5 | 6 | **Prerequisites** 7 | - [node](http://nodejs.org/) 8 | 9 | ```bash 10 | npm install --save jupyter_rfb 11 | ``` 12 | -------------------------------------------------------------------------------- /js/amd-public-path.js: -------------------------------------------------------------------------------- 1 | // In an AMD module, we set the public path using the magic requirejs 'module' dependency 2 | // See https://github.com/requirejs/requirejs/wiki/Differences-between-the-simplified-CommonJS-wrapper-and-standard-AMD-define#module 3 | // Since 'module' is a requirejs magic module, we must include 'module' in the webpack externals configuration. 4 | import * as module from 'module'; 5 | const url = new URL(module.uri, document.location) 6 | // Using lastIndexOf('/')+1 gives us the empty string if there is no '/', so pathname becomes '/' 7 | url.pathname = url.pathname.slice(0,url.pathname.lastIndexOf('/')+1); 8 | __webpack_public_path__ = `${url.origin}${url.pathname}`; 9 | -------------------------------------------------------------------------------- /js/lib/extension.js: -------------------------------------------------------------------------------- 1 | // This file contains the javascript that is run when the notebook is loaded. 2 | // It contains some requirejs configuration and the `load_ipython_extension` 3 | // which is required for any notebook extension. 4 | 5 | // Configure requirejs 6 | if (window.require) { 7 | window.require.config({ 8 | map: { 9 | "*" : { 10 | "jupyter_rfb": "nbextensions/jupyter_rfb/index", 11 | } 12 | } 13 | }); 14 | } 15 | 16 | export function load_ipython_extension() { }; 17 | -------------------------------------------------------------------------------- /js/lib/index.js: -------------------------------------------------------------------------------- 1 | // Export widget models and views, and the npm package version number. 2 | 3 | export { RemoteFrameBufferModel, RemoteFrameBufferView, version } from './widget'; 4 | -------------------------------------------------------------------------------- /js/lib/labplugin.js: -------------------------------------------------------------------------------- 1 | import {RemoteFrameBufferModel, RemoteFrameBufferView, version} from './index'; 2 | import {IJupyterWidgetRegistry} from '@jupyter-widgets/base'; 3 | 4 | export const remoteFrameBufferPlugin = { 5 | id: 'jupyter_rfb:plugin', 6 | requires: [IJupyterWidgetRegistry], 7 | activate: function(app, widgets) { 8 | widgets.registerWidget({ 9 | name: 'jupyter_rfb', 10 | version: version, 11 | exports: { RemoteFrameBufferModel, RemoteFrameBufferView } 12 | }); 13 | }, 14 | autoStart: true 15 | }; 16 | 17 | export default remoteFrameBufferPlugin; -------------------------------------------------------------------------------- /js/lib/widget.js: -------------------------------------------------------------------------------- 1 | import { DOMWidgetModel, DOMWidgetView } from '@jupyter-widgets/base'; 2 | 3 | /* 4 | * For the kernel counterpart to this file, see widget.py 5 | * For the base class, see https://github.com/jupyter-widgets/ipywidgets/blob/master/packages/base/src/widget.ts 6 | * 7 | * The server sends frames to the client, and the client sends back 8 | * a confirmation when it has processed the frame. 9 | * 10 | * The client queues the frames it receives and processes them one-by-one 11 | * at the browser's pace, using requestAnimationFrame. We send back a 12 | * confirmation when the frame is processed (not when it is technically received). 13 | * It is the responsibility of the server to not send too many frames beyond the 14 | * last confirmed one. 15 | * 16 | * When setting the img.src attribute, the browser still needs to actually render 17 | * the image. We wait for this before requesting a new animation. If we don't do 18 | * this on FF, the animation is not smooth because the image "gets stuck". 19 | */ 20 | 21 | export const version = "0.5.3"; 22 | 23 | export class RemoteFrameBufferModel extends DOMWidgetModel { 24 | defaults() { 25 | return { 26 | ...super.defaults(), 27 | _model_name: 'RemoteFrameBufferModel', 28 | _view_name: 'RemoteFrameBufferView', 29 | _model_module: 'jupyter_rfb', 30 | _view_module: 'jupyter_rfb', 31 | _model_module_version: version, 32 | _view_module_version: version, 33 | // For the frames 34 | frame_feedback: {}, 35 | // For the widget 36 | css_width: '500px', 37 | css_height: '300px', 38 | resizable: true, 39 | has_visible_views: false, 40 | cursor: 'default' 41 | }; 42 | } 43 | initialize() { 44 | super.initialize.apply(this, arguments); 45 | // Keep a list if img elements. 46 | this.img_elements = []; 47 | // Observer that will check whether the img elements are within the viewport. 48 | this._intersection_observer = new IntersectionObserver(this._intersection_callback.bind(this)); 49 | // We update the img element list (and the intersection observer) automatically when 50 | // a new view is added or removed. But this (especially the latter) may not work in 51 | // all possible cases. So let's also call it on a low interval. 52 | window.setInterval(this.collect_view_img_elements.bind(this), 5000); 53 | // Keep a list of frames to render. 54 | this.frames = []; 55 | // We populate the above list from this callback. 56 | this.on('msg:custom', this.on_msg, this); 57 | // Initialize a stub frame. 58 | this.last_frame = { 59 | src: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR42mOor68HAAL+AX6E2KOJAAAAAElFTkSuQmCC', 60 | index: 0, 61 | timestamp: 0, 62 | }; 63 | // Start the animation loop 64 | this._img_update_pending = false; 65 | this._request_animation_frame(); 66 | } 67 | 68 | async collect_view_img_elements() { 69 | // Here we collect img elements corresponding to the current views. 70 | // We also set their onload methods which we use to schedule new draws. 71 | // Plus we reset out visibility obserer. 72 | this._intersection_observer.disconnect(); 73 | // Reset 74 | for (let img of this.img_elements) { 75 | img.onload = null; 76 | } 77 | this.img_elements = []; 78 | // Collect 79 | for (let view_id in this.views) { 80 | let view = await this.views[view_id]; 81 | this._intersection_observer.observe(view.img); 82 | this.img_elements.push(view.img); 83 | view.img.onload = this._request_animation_frame.bind(this); 84 | } 85 | // Just in case we lose the animation loop somehow because of images dropping out. 86 | this._request_animation_frame(); 87 | } 88 | 89 | /** 90 | * @param {Object} msg 91 | * @param {DataView[]} buffers 92 | */ 93 | on_msg(msg, buffers) { 94 | if (msg.type === 'framebufferdata') { 95 | this.frames.push({ ...msg, buffers: buffers }); 96 | } 97 | } 98 | 99 | _intersection_callback(entries, observer) { 100 | // This gets called when one of the views becomes visible/invisible. 101 | // Note that entries only contains the *changed* elements. 102 | 103 | // Set visibility of changed img elements. 104 | for (let entry of entries) { 105 | entry.target._is_visible = entry.isIntersecting; 106 | } 107 | // Now count how many are visible 108 | let count = 0; 109 | for (let img of this.img_elements) { 110 | if (img._is_visible) { count += 1; } 111 | } 112 | // If the state changed, update our flag 113 | let has_visible_views = count > 0; 114 | if (has_visible_views != this.get("has_visible_views")) { 115 | this.set('has_visible_views', has_visible_views); 116 | this.save_changes(); 117 | } 118 | } 119 | 120 | _send_response() { 121 | // Let Python know what we have at the model. This prop is a dict, making it "atomic". 122 | let frame = this.last_frame; 123 | let frame_feedback = { index: frame.index, timestamp: frame.timestamp, localtime: Date.now() / 1000 }; 124 | this.set('frame_feedback', frame_feedback); 125 | this.save_changes(); 126 | } 127 | 128 | _request_animation_frame() { 129 | // Request an animation frame, but with a tiny delay, just to avoid 130 | // straining the browser. This seems to actually make things more smooth. 131 | if (!this._img_update_pending) { 132 | this._img_update_pending = true; 133 | let func = this._animate.bind(this); 134 | window.setTimeout(window.requestAnimationFrame, 5, func); 135 | } 136 | } 137 | 138 | _animate() { 139 | this._img_update_pending = false; 140 | if (!this.frames.length) { 141 | this._request_animation_frame(); 142 | return; 143 | } 144 | // Pick the oldest frame from the stack 145 | let frame = this.frames.shift(); 146 | let new_src; 147 | if (frame.buffers.length > 0) { 148 | let blob = new Blob([frame.buffers[0].buffer], { type: frame.mimetype }); 149 | new_src = URL.createObjectURL(blob); 150 | } else { 151 | new_src = frame.data_b64; 152 | } 153 | let old_src = this.img_elements?.[0]?.src; 154 | if (old_src.startsWith('blob:')) { URL.revokeObjectURL(old_src); } 155 | // Update the image sources 156 | for (let img of this.img_elements) { 157 | img.src = new_src; 158 | } 159 | // Let the server know we processed the image (even if it's not shown yet) 160 | this.last_frame = frame; 161 | this._send_response(); 162 | // Request a new frame. If we have images, a frame is requested *after* they load. 163 | if (this.img_elements.length === 0) { 164 | this._request_animation_frame(); 165 | } 166 | } 167 | 168 | close() { 169 | // This gets called when model is closed and the comm is removed. Notify Py just in time! 170 | this.send({ event_type: 'close', time_stamp: get_time_stamp() }); // does nothing if this.comm is already gone 171 | super.close.apply(this, arguments); 172 | } 173 | } 174 | 175 | 176 | export class RemoteFrameBufferView extends DOMWidgetView { 177 | // Defines how the widget gets rendered into the DOM 178 | render() { 179 | var that = this; 180 | 181 | // Create a stub element that can grab focus 182 | this.focus_el = document.createElement("a"); 183 | this.focus_el.href = "#"; 184 | this.focus_el.style.position = "absolute"; 185 | this.focus_el.style.width = "1px"; 186 | this.focus_el.style.height = "1px"; 187 | this.focus_el.style.padding = "0"; 188 | this.focus_el.style.zIndex = "-99"; 189 | this.el.appendChild(this.focus_el); 190 | 191 | // Create image element 192 | this.img = new Image(); 193 | // Tweak loading behavior. These should be the defaults, but we set them just in case. 194 | this.img.decoding = 'sync'; 195 | this.img.loading = 'eager'; 196 | // Tweak mouse/touch/pointer/key behavior 197 | this.img.style.touchAction = 'none'; // prevent default pan/zoom behavior 198 | this.img.ondragstart = () => false; // prevent browser's built-in image drag 199 | this.img.tabIndex = -1; 200 | // Prevent context menu on RMB. Firefox still shows it when shift is pressed. It seems 201 | // impossible to override this (tips welcome!), so let's make this the actual behavior. 202 | this.img.oncontextmenu = function (e) { if (!e.shiftKey) { e.preventDefault(); e.stopPropagation(); return false; } }; 203 | 204 | // Initialize the image 205 | this.img.src = this.model.last_frame.src; 206 | this.el.appendChild(this.img); 207 | this.model.collect_view_img_elements(); 208 | 209 | // Cursor 210 | this.el.style.cursor = this.model.get('cursor'); 211 | this.model.on('change:cursor', function () { this.el.style.cursor = this.model.get('cursor'); }, this); 212 | 213 | // Set of throttler functions to send events at a friendly pace 214 | this._throttlers = {}; 215 | 216 | // Initialize sizing. 217 | // Setting the this.el's size right now has no effect. We also set it in _check_size() below. 218 | this.img.style.width = '100%'; 219 | this.img.style.height = '100%'; 220 | this.el.style.width = this.model.get('css_width'); 221 | this.el.style.height = this.model.get('css_height'); 222 | this.el.style.resize = this.model.get('resizable') ? 'both' : 'none'; 223 | this.el.style.overflow = 'hidden'; // resize does not work if overflow is 'visible' 224 | 225 | // Keep track of size changes from the server 226 | this.model.on('change:css_width', function () { this.el.style.width = this.model.get('css_width'); }, this); 227 | this.model.on('change:css_height', function () { this.el.style.height = this.model.get('css_height'); }, this); 228 | this.model.on('change:resizable', function () { this.el.style.resize = this.model.get('resizable') ? 'both' : 'none'; }, this); 229 | 230 | // Keep track of size changes in JS, so we can notify the server 231 | this._current_size = [0, 0, 1]; 232 | this._resizeObserver = new ResizeObserver(this._check_resize.bind(that)); 233 | this._resizeObserver.observe(this.img); 234 | window.addEventListener('resize', this._check_resize.bind(this)); 235 | 236 | // Pointer events 237 | this._pointers = {}; 238 | this._last_buttons = []; 239 | this.img.addEventListener('pointerdown', function (e) { 240 | // This is what makes the JS PointerEvent so great. We can enable mouse capturing 241 | // and we will receive mouse-move and mouse-up even when the pointer moves outside 242 | // the element. Best of all, the capturing is disabled automatically! 243 | that.focus_el.focus({ preventScroll: true, focusVisble: false }); 244 | that.img.setPointerCapture(e.pointerId); 245 | that._pointers[e.pointerId] = e; 246 | let event = create_pointer_event(that.img, e, that._pointers, 'pointer_down'); 247 | that._last_buttons = event.buttons; 248 | that.send(event); 249 | if (!e.altKey) { e.preventDefault(); } 250 | }); 251 | this.img.addEventListener('lostpointercapture', function (e) { 252 | // This happens on pointer-up or pointer-cancel. We threat them the same. 253 | // The event we emit will still include the touch hat goes up. 254 | let event = create_pointer_event(that.img, e, that._pointers, 'pointer_up'); 255 | delete that._pointers[e.pointerId]; 256 | that._last_buttons = event.buttons; 257 | that.send(event); 258 | }); 259 | this.img.addEventListener('pointermove', function (e) { 260 | // If this pointer is not down, but other pointers are, don't emit an event. 261 | if (that._pointers[e.pointerId] === undefined) { 262 | if (Object.keys(that._pointers).length > 0) { return; } 263 | } 264 | let event = create_pointer_event(that.img, e, that._pointers, 'pointer_move'); 265 | that.send_throttled(event, 20); 266 | }); 267 | this.img.addEventListener('pointerenter', function (e) { 268 | // If this pointer is not down, but other pointers are, don't emit an event. 269 | if (that._pointers[e.pointerId] === undefined) { 270 | if (Object.keys(that._pointers).length > 0) { return; } 271 | } 272 | let event = create_pointer_event(that.img, e, {[e.pointerId]: e}, 'pointer_enter'); 273 | that.send(event); 274 | }); 275 | this.img.addEventListener('pointerleave', function (e) { 276 | // If this pointer is not down, but other pointers are, don't emit an event. 277 | if (that._pointers[e.pointerId] === undefined) { 278 | if (Object.keys(that._pointers).length > 0) { return; } 279 | } 280 | let event = create_pointer_event(that.img, e, {[e.pointerId]: e}, 'pointer_leave'); 281 | that.send(event); 282 | }); 283 | 284 | // Click events are not pointer events. Not sure if we need click events. It seems to make 285 | // less sense, because the img is just a single element. Only double-click for now. 286 | this.img.addEventListener('dblclick', function (e) { 287 | let event = create_pointer_event(that.img, e, {}, 'double_click'); 288 | delete event.touches; 289 | delete event.ntouches; 290 | that.send(event); 291 | if (!e.altKey) { e.preventDefault(); } 292 | }); 293 | 294 | // Scrolling. Need a special throttling that accumulates the deltas. 295 | // Also, only consume the wheel event when we have focus. 296 | // On Firefox, e.buttons is always 0 for wheel events, so we use a cached value for the buttons. 297 | this._wheel_state = { dx: 0, dy: 0, e: null, pending: false }; 298 | function send_wheel_event() { 299 | let e = that._wheel_state.e; 300 | let rect = that.img.getBoundingClientRect(); 301 | let event = { 302 | event_type: 'wheel', 303 | x: Number(e.clientX - rect.left), 304 | y: Number(e.clientY - rect.top), 305 | dx: that._wheel_state.dx, 306 | dy: that._wheel_state.dy, 307 | buttons: that._last_buttons, 308 | modifiers: get_modifiers(e), 309 | time_stamp: get_time_stamp(), 310 | }; 311 | that._wheel_state.dx = 0; 312 | that._wheel_state.dy = 0; 313 | that._wheel_state.pending = false; 314 | that.send(event); 315 | } 316 | this.img.addEventListener('wheel', function (e) { 317 | if (window.document.activeElement !== that.focus_el) { return; } 318 | let scales = [1 / window.devicePixelRatio, 16, 600]; // pixel, line, page 319 | let scale = scales[e.deltaMode]; 320 | that._wheel_state.dx += e.deltaX * scale; 321 | that._wheel_state.dy += e.deltaY * scale; 322 | if (!that._wheel_state.pending) { 323 | that._wheel_state.pending = true; 324 | that._wheel_state.e = e; 325 | window.setTimeout(send_wheel_event, 20); 326 | } 327 | if (!e.altKey) { e.preventDefault(); } 328 | }); 329 | 330 | // Key events - approach inspired from ipyevents 331 | function key_event_handler(e) { 332 | // Failsafe in case the element is deleted or detached. 333 | if (that.el.offsetParent === null) { return; } 334 | let event = { 335 | event_type: 'key_' + e.type.slice(3), 336 | key: KEYMAP[e.key] || e.key, 337 | modifiers: get_modifiers(e), 338 | time_stamp: get_time_stamp(), 339 | }; 340 | if (!e.repeat) { that.send(event); } // dont do the sticky key thing 341 | e.stopPropagation(); 342 | e.preventDefault(); 343 | } 344 | this.focus_el.addEventListener('keydown', key_event_handler, true); 345 | this.focus_el.addEventListener('keyup', key_event_handler, true); 346 | } 347 | 348 | remove() { 349 | // This gets called when the view is removed from the DOM. There can still be other views though! 350 | super.remove.apply(this, arguments); 351 | window.setTimeout(this.model.collect_view_img_elements.bind(this.model), 10); 352 | } 353 | 354 | _check_resize() { 355 | // Called when the widget resizes. 356 | // During initialization Jupyter sets .el.style.width and .height to the empty string. 357 | // It looks like VS Code tries harder to do this than the notebook, 358 | // so we need to check for this pretty aggressively. 359 | if (!this.el.style.width && this.model.get('css_width')) { 360 | this.el.style.width = this.model.get('css_width'); 361 | this.el.style.height = this.model.get('css_height'); 362 | // prevent massive size due to auto-scroll (issue #62) 363 | this.el.style.maxWidth = Math.max(1024, window.innerWidth) + 'px'; 364 | this.el.style.maxHeight = Math.max(1024, window.innerHeight) + 'px'; 365 | this.el.style.overflow = 'hidden'; 366 | return; // Don't send a resize event now 367 | } 368 | // Width and height are in logical pixels. 369 | let w = this.img.clientWidth; 370 | let h = this.img.clientHeight; 371 | let r = window.devicePixelRatio; 372 | if (w === 0 && h === 0) { return; } 373 | if (this._current_size[0] !== w || this._current_size[1] !== h || this._current_size[2] !== r) { 374 | this._current_size = [w, h, r]; 375 | this.send_throttled({ event_type: 'resize', width: w, height: h, pixel_ratio: r, time_stamp: get_time_stamp() }, 200); 376 | } 377 | } 378 | 379 | send_throttled(msg, wait) { 380 | // Like .send(), but throttled 381 | let event_type = msg.event_type || ''; 382 | let func = this._throttlers[event_type]; 383 | if (func === undefined) { 384 | func = throttled(this.send, wait || 50); 385 | this._throttlers[event_type] = func; 386 | } 387 | func.call(this, msg); 388 | } 389 | } 390 | 391 | 392 | 393 | var KEYMAP = { 394 | Ctrl: 'Control', 395 | Del: 'Delete', 396 | Esc: 'Escape', 397 | }; 398 | 399 | 400 | function get_modifiers(e) { 401 | let modifiers = ['Alt', 'Shift', 'Ctrl', 'Meta'].filter((n) => e[n.toLowerCase() + 'Key']); 402 | return modifiers.map((m) => KEYMAP[m] || m); 403 | } 404 | 405 | 406 | function throttled(func, wait) { 407 | var context, args, result; 408 | var timeout = null; 409 | var previous = 0; 410 | var later = function () { 411 | previous = Date.now(); 412 | timeout = null; 413 | result = func.apply(context, args); 414 | if (!timeout) context = args = null; 415 | }; 416 | return function () { 417 | var now = Date.now(); 418 | var remaining = wait - (now - previous); 419 | context = this; 420 | args = arguments; 421 | if (remaining <= 0 || remaining > wait) { 422 | if (timeout) { clearTimeout(timeout); timeout = null; } 423 | previous = now; 424 | result = func.apply(context, args); 425 | if (!timeout) context = args = null; 426 | } else if (!timeout) { 427 | timeout = setTimeout(later, remaining); 428 | } 429 | return result; 430 | }; 431 | } 432 | 433 | 434 | function create_pointer_event(el, e, pointers, event_type) { 435 | let rect = el.getBoundingClientRect(); 436 | let offset = [rect.left, rect.top]; 437 | let main_x = Number(e.clientX - offset[0]); 438 | let main_y = Number(e.clientY - offset[1]); 439 | 440 | // Collect touches (we may add more fields later) 441 | let touches = {}; 442 | let ntouches = 0; 443 | for (let pointer_id in pointers) { 444 | let pe = pointers[pointer_id]; // pointer event 445 | let x = Number(pe.clientX - offset[0]); 446 | let y = Number(pe.clientY - offset[1]); 447 | let touch = { x: x, y: y, pressure: pe.pressure }; 448 | touches[pe.pointerId] = touch; 449 | ntouches += 1; 450 | } 451 | 452 | // Get button that changed, and the button state 453 | var button = { 0: 1, 1: 3, 2: 2, 3: 4, 4: 5, 5: 6 }[e.button] || 0; 454 | var buttons = []; 455 | for (let b of [0, 1, 2, 3, 4, 5]) { if ((1 << b) & e.buttons) { buttons.push(b + 1); } } 456 | 457 | return { 458 | event_type: event_type, 459 | x: main_x, 460 | y: main_y, 461 | button: button, 462 | buttons: buttons, 463 | modifiers: get_modifiers(e), 464 | ntouches: ntouches, 465 | touches: touches, 466 | time_stamp: get_time_stamp(), 467 | }; 468 | } 469 | 470 | function get_time_stamp() { 471 | return Date.now() / 1000; 472 | } 473 | -------------------------------------------------------------------------------- /js/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "jupyter_rfb", 3 | "version": "0.5.3", 4 | "description": "Remote Frame Buffer for Jupyter", 5 | "license": "MIT", 6 | "author": "Almar Klein", 7 | "main": "lib/index.js", 8 | "repository": { 9 | "type": "git", 10 | "url": "https://github.com/vispy/jupyter_rfb.git" 11 | }, 12 | "keywords": [ 13 | "jupyter", 14 | "widgets", 15 | "ipython", 16 | "ipywidgets", 17 | "jupyterlab-extension" 18 | ], 19 | "files": [ 20 | "lib/**/*.js", 21 | "dist/*.js" 22 | ], 23 | "scripts": { 24 | "clean": "rimraf dist/ && rimraf ../jupyter_rfb/labextension/ && rimraf ../jupyter_rfb/nbextension", 25 | "prepublish": "yarn run clean && yarn run build:prod", 26 | "build": "webpack --mode=development && yarn run build:labextension:dev", 27 | "build:prod": "webpack --mode=production && yarn run build:labextension", 28 | "build:labextension": "jupyter labextension build .", 29 | "build:labextension:dev": "jupyter labextension build --development True .", 30 | "watch": "webpack --watch --mode=development", 31 | "test": "echo \"Error: no test specified\" && exit 1" 32 | }, 33 | "devDependencies": { 34 | "@jupyterlab/builder": "^3.6 || ^4", 35 | "rimraf": "^2.6.1", 36 | "webpack": "^5", 37 | "webpack-cli": "^5.1.4" 38 | }, 39 | "dependencies": { 40 | "@jupyter-widgets/base": "^1.1 || ^2 || ^3 || ^4 || ^5 || ^6", 41 | "lodash": "^4.17.4" 42 | }, 43 | "jupyterlab": { 44 | "extension": "lib/labplugin", 45 | "outputDir": "../jupyter_rfb/labextension", 46 | "sharedPackages": { 47 | "@jupyter-widgets/base": { 48 | "bundled": false, 49 | "singleton": true 50 | } 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /js/webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const version = require('./package.json').version; 3 | 4 | // Custom webpack rules are generally the same for all webpack bundles, hence 5 | // stored in a separate local variable. 6 | const rules = [ 7 | { test: /\.css$/, use: ['style-loader', 'css-loader']} 8 | ] 9 | 10 | 11 | module.exports = (env, argv) => { 12 | const devtool = argv.mode === 'development' ? 'source-map' : false; 13 | return [ 14 | {// Notebook extension 15 | // 16 | // This bundle only contains the part of the JavaScript that is run on 17 | // load of the notebook. This section generally only performs 18 | // some configuration for requirejs, and provides the legacy 19 | // "load_ipython_extension" function which is required for any notebook 20 | // extension. 21 | entry: './lib/extension.js', 22 | output: { 23 | filename: 'extension.js', 24 | path: path.resolve(__dirname, '..', 'jupyter_rfb', 'nbextension'), 25 | libraryTarget: 'amd', 26 | }, 27 | devtool 28 | }, 29 | {// Bundle for the notebook containing the custom widget views and models 30 | // 31 | // This bundle contains the implementation for the custom widget views and 32 | // custom widget. 33 | // It must be an amd module 34 | entry: ['./amd-public-path.js', './lib/index.js'], 35 | output: { 36 | filename: 'index.js', 37 | path: path.resolve(__dirname, '..', 'jupyter_rfb', 'nbextension'), 38 | libraryTarget: 'amd', 39 | publicPath: '', // Set in amd-public-path.js 40 | }, 41 | devtool, 42 | module: { 43 | rules: rules 44 | }, 45 | // 'module' is the magic requirejs dependency used to set the publicPath 46 | externals: ['@jupyter-widgets/base', 'module'] 47 | }, 48 | {// Embeddable jupyter_rfb bundle 49 | // 50 | // This bundle is identical to the notebook bundle containing the custom 51 | // widget views and models. The only difference is it is placed in the 52 | // dist/ directory and shipped with the npm package for use from a CDN 53 | // like jsdelivr. 54 | // 55 | // The target bundle is always `dist/index.js`, which is the path 56 | // required by the custom widget embedder. 57 | entry: ['./amd-public-path.js', './lib/index.js'], 58 | output: { 59 | filename: 'index.js', 60 | path: path.resolve(__dirname, 'dist'), 61 | libraryTarget: 'amd', 62 | publicPath: '', // Set in amd-public-path.js 63 | }, 64 | devtool, 65 | module: { 66 | rules: rules 67 | }, 68 | // 'module' is the magic requirejs dependency used to set the publicPath 69 | externals: ['@jupyter-widgets/base', 'module'] 70 | } 71 | ]; 72 | } 73 | -------------------------------------------------------------------------------- /jupyter_rfb.json: -------------------------------------------------------------------------------- 1 | { 2 | "load_extensions": { 3 | "jupyter_rfb/extension": true 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /jupyter_rfb/__init__.py: -------------------------------------------------------------------------------- 1 | # ruff: noqa: F401 2 | from . import events 3 | from ._version import __version__, version_info 4 | from .widget import RemoteFrameBuffer 5 | from ._utils import remove_rfb_models_from_nb 6 | 7 | 8 | def _jupyter_labextension_paths(): 9 | """Called by Jupyter Lab Server to detect if it is a valid labextension and 10 | to install the widget 11 | 12 | Returns 13 | ======= 14 | src: Source directory name to copy files from. Webpack outputs generated files 15 | into this directory and Jupyter Lab copies from this directory during 16 | widget installation 17 | dest: Destination directory name to install widget files to. Jupyter Lab copies 18 | from `src` directory into /labextensions/ directory 19 | during widget installation 20 | """ 21 | return [ 22 | { 23 | "src": "labextension", 24 | "dest": "jupyter_rfb", 25 | } 26 | ] 27 | 28 | 29 | def _jupyter_nbextension_paths(): 30 | """Called by Jupyter Notebook Server to detect if it is a valid nbextension and 31 | to install the widget 32 | 33 | Returns 34 | ======= 35 | section: The section of the Jupyter Notebook Server to change. 36 | Must be 'notebook' for widget extensions 37 | src: Source directory name to copy files from. Webpack outputs generated files 38 | into this directory and Jupyter Notebook copies from this directory during 39 | widget installation 40 | dest: Destination directory name to install widget files to. Jupyter Notebook copies 41 | from `src` directory into /nbextensions/ directory 42 | during widget installation 43 | require: Path to importable AMD Javascript module inside the 44 | /nbextensions/ directory 45 | """ 46 | return [ 47 | { 48 | "section": "notebook", 49 | "src": "nbextension", 50 | "dest": "jupyter_rfb", 51 | "require": "jupyter_rfb/extension", 52 | } 53 | ] 54 | -------------------------------------------------------------------------------- /jupyter_rfb/_jpg.py: -------------------------------------------------------------------------------- 1 | import io 2 | 3 | 4 | class JpegEncoder: 5 | """Base JPEG encoder class. 6 | 7 | Subclasses must import their dependencies in their __init__, 8 | and implement the _encode() method. 9 | """ 10 | 11 | def encode(self, array, quality): 12 | """Encode the array, returning bytes.""" 13 | 14 | quality = int(quality) 15 | 16 | # Check types 17 | if hasattr(array, "shape") and hasattr(array, "dtype"): 18 | if array.dtype != "uint8": 19 | raise ValueError("Image array to convert to JPEG must be uint8") 20 | original_shape = shape = array.shape 21 | else: 22 | raise ValueError( 23 | f"Invalid type for array, need ndarray-like, got {type(array)}" 24 | ) 25 | 26 | # Check shape 27 | if len(shape) == 2: 28 | shape = (*shape, 1) 29 | array = array.reshape(shape) 30 | if not (len(shape) == 3 and shape[2] in (1, 3)): 31 | raise ValueError(f"Unexpected image shape: {original_shape}") 32 | 33 | return self._encode(array, quality) 34 | 35 | def _encode(self, array, quality): 36 | raise NotImplementedError() 37 | 38 | 39 | class StubJpegEncoder(JpegEncoder): 40 | """A stub encoder that returns None.""" 41 | 42 | def _encode(self, array, quality): 43 | return None 44 | 45 | 46 | class SimpleJpegEncoder(JpegEncoder): 47 | """A JPEG encoder using the simplejpeg library.""" 48 | 49 | def __init__(self): 50 | import simplejpeg 51 | 52 | self.simplejpeg = simplejpeg 53 | 54 | def _encode(self, array, quality): 55 | # Simplejpeg requires contiguous data 56 | if not array.flags.c_contiguous: 57 | array = array.copy() 58 | 59 | # Get appropriate colorspace 60 | nchannels = array.shape[2] 61 | if nchannels == 1: 62 | colorspace = "GRAY" 63 | colorsubsampling = "Gray" 64 | elif nchannels == 3: 65 | colorspace = "RGB" 66 | colorsubsampling = "444" 67 | elif nchannels == 4: # no-cover 68 | # No alpha in JPEG - transparent pixels become black 69 | colorspace = "RGBA" 70 | colorsubsampling = "444" 71 | 72 | # Encode! 73 | return self.simplejpeg.encode_jpeg( 74 | array, 75 | quality=quality, 76 | colorspace=colorspace, 77 | colorsubsampling=colorsubsampling, 78 | fastdct=True, 79 | ) 80 | 81 | 82 | class PillowJpegEncoder(JpegEncoder): 83 | """A JPEG encoder using the Pillow library.""" 84 | 85 | def __init__(self): 86 | import PIL.Image 87 | 88 | self.pillow = PIL.Image 89 | 90 | def _encode(self, array, quality): 91 | # Pillow likes grayscale as an NxM array (not NxMx1) 92 | if len(array.shape) == 3 and array.shape[2] == 1: 93 | array = array.reshape(array.shape[:-1]) 94 | 95 | # Encode! 96 | img_pil = self.pillow.fromarray(array) 97 | f = io.BytesIO() 98 | img_pil.save(f, format="JPEG", quality=quality) 99 | return f.getvalue() 100 | 101 | 102 | class OpenCVJpegEncoder(JpegEncoder): 103 | """A JPEG encoder using the OpenCV library.""" 104 | 105 | def __init__(self): 106 | import cv2 107 | 108 | self.cv2 = cv2 109 | 110 | def _encode(self, array, quality): 111 | if len(array.shape) == 3 and array.shape[2] == 3: 112 | # Convert RGB to BGR if needed (assume input is RGB) 113 | array = self.cv2.cvtColor(array, self.cv2.COLOR_RGB2BGR) 114 | 115 | # Encode with the specified quality 116 | encode_param = [self.cv2.IMWRITE_JPEG_QUALITY, quality] 117 | success, encoded_image = self.cv2.imencode(".jpg", array, encode_param) 118 | if not success: 119 | raise RuntimeError("OpenCV failed to encode image") 120 | 121 | return encoded_image.tobytes() 122 | 123 | 124 | def select_encoder(): 125 | """Select an encoder.""" 126 | 127 | for cls in [ 128 | SimpleJpegEncoder, # simplejpeg is fast and lean 129 | PillowJpegEncoder, # pillow is commonly available 130 | OpenCVJpegEncoder, # opencv is readily installed in conda environments 131 | ]: 132 | try: 133 | return cls() 134 | except ImportError: 135 | continue 136 | else: 137 | return StubJpegEncoder() # if all else fails 138 | 139 | 140 | encoder = select_encoder() 141 | 142 | 143 | def array2jpg(array, quality=90): 144 | """Create a JPEG image from a numpy array, with the given quality (percentage). 145 | 146 | The provided array's shape must be either NxM (grayscale), NxMx1 (grayscale), 147 | or NxMx3 (RGB). 148 | 149 | The encoding is performed be one of multiple possible backends. If 150 | no backend is available, None is returned. 151 | """ 152 | return encoder.encode(array, quality) 153 | -------------------------------------------------------------------------------- /jupyter_rfb/_png.py: -------------------------------------------------------------------------------- 1 | import io 2 | import struct 3 | import zlib 4 | 5 | import numpy as np 6 | 7 | 8 | def array2png(array, file=None): 9 | """Create a PNG image from a numpy array. 10 | 11 | The written image is in RGB or RGBA format, with 8 bit precision, 12 | zlib-compressed, without interlacing. 13 | 14 | The provided array's shape must be either NxM (grayscale), NxMx1 (grayscale), 15 | NxMx3 (RGB) or NxNx4 (RGBA). 16 | """ 17 | 18 | # Check types 19 | if hasattr(array, "shape") and hasattr(array, "dtype"): 20 | if array.dtype != "uint8": 21 | raise ValueError("Image array to convert to PNG must be uint8") 22 | original_shape = shape = array.shape 23 | else: 24 | raise ValueError( 25 | f"Invalid type for array, need ndarray-like, got {type(array)}" 26 | ) 27 | 28 | # Allow grayscale: convert to RGB 29 | if len(shape) == 2 or (len(shape) == 3 and shape[2] == 1): 30 | shape = shape[0], shape[1], 3 31 | array = array.reshape(shape[:2]) 32 | array3 = np.empty(shape, np.uint8) 33 | array3[..., 0] = array 34 | array3[..., 1] = array 35 | array3[..., 2] = array 36 | array = array3 37 | elif not array.flags.c_contiguous: 38 | array = array.copy() 39 | 40 | # Check shape 41 | if not (len(shape) == 3 and shape[2] in (3, 4)): 42 | raise ValueError(f"Unexpected image shape: {original_shape}") 43 | 44 | # Get file object 45 | f = io.BytesIO() if file is None else file 46 | 47 | def add_chunk(data, name): 48 | name = name.encode("ASCII") 49 | crc = zlib.crc32(data, zlib.crc32(name)) 50 | f.write(struct.pack(">I", len(data))) 51 | f.write(name) 52 | f.write(data) 53 | f.write(struct.pack(">I", crc & 0xFFFFFFFF)) 54 | 55 | # Write ... 56 | 57 | # Header 58 | f.write(b"\x89PNG\x0d\x0a\x1a\x0a") 59 | 60 | # First chunk 61 | w, h = shape[1], shape[0] 62 | depth = 8 63 | ctyp = 0b0110 if shape[2] == 4 else 0b0010 64 | ihdr = struct.pack(">IIBBBBB", w, h, depth, ctyp, 0, 0, 0) 65 | add_chunk(ihdr, "IHDR") 66 | 67 | # Chunk with pixels. Just one chunk, no fancy filters. 68 | compressor = zlib.compressobj(level=7) 69 | compressed_data = [] 70 | for row_index in range(shape[0]): 71 | row = array[row_index] 72 | compressed_data.append(compressor.compress(b"\x00")) # prepend filter byter 73 | compressed_data.append(compressor.compress(row)) 74 | compressed_data.append(compressor.flush()) 75 | add_chunk(b"".join(compressed_data), "IDAT") 76 | 77 | # Closing chunk 78 | add_chunk(b"", "IEND") 79 | 80 | if file is None: 81 | return f.getvalue() 82 | -------------------------------------------------------------------------------- /jupyter_rfb/_utils.py: -------------------------------------------------------------------------------- 1 | import io 2 | import builtins 3 | import traceback 4 | from base64 import encodebytes 5 | 6 | from IPython.display import DisplayObject 7 | import ipywidgets 8 | 9 | from ._png import array2png 10 | from ._jpg import array2jpg 11 | 12 | 13 | _original_print = builtins.print 14 | 15 | 16 | def array2compressed(array, quality=90): 17 | """Convert the given image (a numpy array) as a compressed array. 18 | 19 | If the quality is 100, a PNG is returned. Otherwise, JPEG is 20 | preferred and PNG is used as a fallback. Returns (mimetype, bytes). 21 | """ 22 | 23 | # Drop alpha channel if there is one 24 | if len(array.shape) == 3 and array.shape[2] == 4: 25 | array = array[:, :, :3] 26 | 27 | if quality >= 100: 28 | mimetype = "image/png" 29 | result = array2png(array) 30 | else: 31 | mimetype = "image/jpeg" 32 | result = array2jpg(array, quality) 33 | if result is None: 34 | mimetype = "image/png" 35 | result = array2png(array) 36 | 37 | return mimetype, result 38 | 39 | 40 | class RFBOutputContext(ipywidgets.Output): 41 | """An output widget with a different implementation of the context manager. 42 | 43 | Handles prints and errors in a more reliable way, that is also 44 | lightweight (i.e. no peformance cost). 45 | 46 | See https://github.com/vispy/jupyter_rfb/issues/35 47 | """ 48 | 49 | capture_print = False 50 | _prev_print = None 51 | 52 | def print(self, *args, **kwargs): 53 | """Print function that show up in the output.""" 54 | f = io.StringIO() 55 | kwargs.pop("file", None) 56 | _original_print(*args, file=f, flush=True, **kwargs) 57 | text = f.getvalue() 58 | self.append_stdout(text) 59 | 60 | def __enter__(self): 61 | """Enter context, replace print function.""" 62 | if self.capture_print: 63 | self._prev_print = builtins.print 64 | builtins.print = self.print 65 | return self 66 | 67 | def __exit__(self, etype, value, tb): 68 | """Exit context, restore print function and show any errors.""" 69 | if self.capture_print and self._prev_print is not None: 70 | builtins.print = self._prev_print 71 | self._prev_print = None 72 | if etype: 73 | err = "".join(traceback.format_exception(etype, value, tb)) 74 | self.append_stderr(err) 75 | return True # declare that we handled the exception 76 | 77 | 78 | class Snapshot(DisplayObject): 79 | """An IPython DisplayObject representing an image snapshot. 80 | 81 | The ``data`` attribute is the image array object. One could use 82 | this to process the data further, e.g. storing it to disk. 83 | """ 84 | 85 | # Not an IPython.display.Image, because we want to use some HTML to 86 | # give it a custom css class and a title. 87 | 88 | def __init__(self, data, width, height, title="snapshot", class_name=None): 89 | super().__init__(data) 90 | self.width = width 91 | self.height = height 92 | self.title = title 93 | self.class_name = class_name 94 | 95 | def _check_data(self): 96 | assert hasattr(self.data, "shape") and hasattr(self.data, "dtype") 97 | 98 | def _repr_mimebundle_(self, **kwargs): 99 | return {"text/html": self._repr_html_()} 100 | 101 | def _repr_html_(self): 102 | # Convert to PNG 103 | png_data = array2png(self.data) 104 | preamble = "data:image/png;base64," 105 | src = preamble + encodebytes(png_data).decode() 106 | # Create html repr 107 | class_str = f"class='{self.class_name}'" if self.class_name else "" 108 | img_style = f"width:{self.width}px;height:{self.height}px;" 109 | tt_style = "position: absolute; top:0; left:0; padding:1px 3px; " 110 | tt_style += ( 111 | "background: #777; color:#fff; font-size: 90%; font-family:sans-serif; " 112 | ) 113 | html = f""" 114 |
115 | 116 |
{self.title}
117 |
118 | """ 119 | return html.replace("\n", "").replace(" ", "").strip() 120 | 121 | 122 | def remove_rfb_models_from_nb(d): 123 | """Remove the widget model output from a notebook dict. 124 | 125 | Given a notebook as a dict (loaded using json), remove the widget 126 | model output if there is also a text/html snapshot output. 127 | 128 | This is to work around the fact that nbsphinx favors the model over 129 | the text/html output. Which is sad, because that's where we put the 130 | initial screenshot for offline viewing. 131 | """ 132 | 133 | to_remove = set() 134 | for key, val in d.items(): 135 | if key == "cells" and isinstance(val, list): 136 | for v in val: 137 | remove_rfb_models_from_nb(v) 138 | elif key == "outputs" and isinstance(val, list): 139 | for v in val: 140 | data = v.get("data", None) 141 | if data: 142 | remove_rfb_models_from_nb(data) 143 | elif key == "application/vnd.jupyter.widget-view+json": 144 | html_sibling = d.get("text/html", []) 145 | if html_sibling and "
` method. 4 | Events are simple dict objects containing at least the key `event_type`. 5 | Additional keys provide more information regarding the event. 6 | 7 | *Last update: 09-05-2025* 8 | 9 | Event types 10 | ----------- 11 | 12 | * **resize**: emitted when the widget changes size. 13 | This event is throttled. 14 | 15 | * *width*: in logical pixels. 16 | * *height*: in logical pixels. 17 | * *pixel_ratio*: the pixel ratio between logical and physical pixels. 18 | * *time_stamp*: a timestamp in seconds. 19 | 20 | * **close**: emitted when the widget is closed (i.e. destroyed). 21 | This event has no additional keys. 22 | 23 | * **pointer_down**: emitted when the user interacts with mouse, 24 | touch or other pointer devices, by pressing it down. 25 | 26 | * *x*: horizontal position of the pointer within the widget. 27 | * *y*: vertical position of the pointer within the widget. 28 | * *button*: the button to which this event applies. See section below for details. 29 | * *buttons*: a tuple of buttons being pressed down. 30 | * *modifiers*: a tuple of modifier keys being pressed down. See section below for details. 31 | * *ntouches*: the number of simultaneous pointers being down. 32 | * *touches*: a dict with int keys (pointer id's), and values that are dicts 33 | that contain "x", "y", and "pressure". 34 | * *time_stamp*: a timestamp in seconds. 35 | 36 | * **pointer_up**: emitted when the user releases a pointer. 37 | This event has the same keys as the pointer down event. 38 | 39 | * **pointer_move**: emitted when the user moves a pointer. 40 | This event has the same keys as the pointer down event. 41 | This event is throttled. 42 | 43 | * **pointer_enter**: emitted when the user moves a pointer into the 44 | boundary of the widget. 45 | This event has no additional keys. 46 | 47 | * **pointer_leave**: emitted when the user moves a pointer out of the 48 | boundary of the widget. 49 | This event has no additional keys. 50 | 51 | * **double_click**: emitted on a double-click. 52 | This event looks like a pointer event, but without the touches. 53 | 54 | * **wheel**: emitted when the mouse-wheel is used (scrolling), 55 | or when scrolling/pinching on the touchpad/touchscreen. 56 | 57 | Similar to the JS wheel event, the values of the deltas depend on the 58 | platform and whether the mouse-wheel, trackpad or a touch-gesture is 59 | used. Also, scrolling can be linear or have inertia. As a rule of 60 | thumb, one "wheel action" results in a cumulative ``dy`` of around 61 | 100. Positive values of ``dy`` are associated with scrolling down and 62 | zooming out. Positive values of ``dx`` are associated with scrolling 63 | to the right. (A note for Qt users: the sign of the deltas is (usually) 64 | reversed compared to the QWheelEvent.) 65 | 66 | On MacOS, using the mouse-wheel while holding shift results in horizontal 67 | scrolling. In applications where the scroll dimension does not matter, 68 | it is therefore recommended to use `delta = event['dy'] or event['dx']`. 69 | 70 | * *dx*: the horizontal scroll delta (positive means scroll right). 71 | * *dy*: the vertical scroll delta (positive means scroll down or zoom out). 72 | * *x*: the mouse horizontal position during the scroll. 73 | * *y*: the mouse vertical position during the scroll. 74 | * *buttons*: a tuple of buttons being pressed down. 75 | * *modifiers*: a tuple of modifier keys being pressed down. 76 | * *time_stamp*: a timestamp in seconds. 77 | 78 | * **key_down**: emitted when a key is pressed down. 79 | 80 | * *key*: the key being pressed as a string. See section below for details. 81 | * *modifiers*: a tuple of modifier keys being pressed down. 82 | * *time_stamp*: a timestamp in seconds. 83 | 84 | * **key_up**: emitted when a key is released. 85 | This event has the same keys as the key down event. 86 | 87 | 88 | Time stamps 89 | ----------- 90 | 91 | Since the time origin of ``time_stamp`` values is undefined, 92 | time stamp values only make sense in relation to other time stamps. 93 | 94 | 95 | Mouse buttons 96 | ------------- 97 | 98 | * 0: No button. 99 | * 1: Left button. 100 | * 2: Right button. 101 | * 3: Middle button 102 | * 4-9: etc. 103 | 104 | 105 | Keys 106 | ---- 107 | 108 | The key names follow the `browser spec `_. 109 | 110 | * Keys that represent a character are simply denoted as such. For these the case matters: 111 | "a", "A", "z", "Z" "3", "7", "&", " " (space), etc. 112 | * The modifier keys are: 113 | "Shift", "Control", "Alt", "Meta". 114 | * Some example keys that do not represent a character: 115 | "ArrowDown", "ArrowUp", "ArrowLeft", "ArrowRight", "F1", "Backspace", etc. 116 | 117 | 118 | Coordinate frame 119 | ---------------- 120 | 121 | The coordinate frame is defined with the origin in the top-left corner. 122 | Positive `x` moves to the right, positive `y` moves down. 123 | 124 | 125 | Event capturing 126 | --------------- 127 | 128 | The *pointer_move* event only occurs when the pointer is over the widget, 129 | unless a button is down (i.e. dragging). The *pointer_down* event can only 130 | occur inside the widget, the *pointer_up* can occur outside of the widget. 131 | 132 | Some events only work when the widget has focus within the application 133 | (i.e. having received a pointer down). 134 | This applies to the *key_down*, *key_up*, and *wheel* events. 135 | 136 | 137 | Application focus 138 | ----------------- 139 | 140 | (In the case of ``jupyter_rfb``, the 'application' typically means the browser.) 141 | 142 | * When the application does not have focus, it does not emit any pointer events. 143 | * When the application loses focus, a *pointer_leave* event is emitted, and 144 | also a *pointer_up* event if a button is currently down. 145 | * When the application regains focus, an enter event is emitted if the pointer 146 | if over the canvas. This not may happen until the pointer is moved. 147 | * If the application regained focus by clicking on the canvas, that click does 148 | not result in pointer events (down, move, nor up). 149 | 150 | 151 | Event throttling 152 | ---------------- 153 | 154 | To avoid straining the IO, certain events can be throttled. Their effect 155 | is accumulated if this makes sense (e.g. wheel event). The consumer of 156 | the events should take this into account. The events that are throttled 157 | in jupyte_rfb widgets are *resize*, *pointer_move* and *wheel*. 158 | 159 | """ 160 | 161 | # The only purpose of this module is to document the events. 162 | # In Sphinx autodoc we can use automodule. 163 | # In Pyzo/Spyder/Jupyter a user can do ``jupyter_rfb.events?``. 164 | -------------------------------------------------------------------------------- /jupyter_rfb/widget.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | See widget.js for the client counterpart to this file. 4 | 5 | ## Developer notes 6 | 7 | The server sends frames to the client, and the client sends back 8 | a confirmation when it has processed the frame. 9 | 10 | The server will not send more than *max_buffered_frames* beyond the 11 | last confirmed frame. As such, if the client processes frames slower, 12 | the server will slow down too. 13 | """ 14 | 15 | import asyncio 16 | import time 17 | from base64 import encodebytes 18 | 19 | import ipywidgets 20 | import numpy as np 21 | from IPython.display import display 22 | from traitlets import Bool, Dict, Int, Unicode 23 | 24 | from ._utils import array2compressed, RFBOutputContext, Snapshot 25 | from ._version import ref_version 26 | 27 | 28 | @ipywidgets.register 29 | class RemoteFrameBuffer(ipywidgets.DOMWidget): 30 | """A widget implementing a remote frame buffer. 31 | 32 | This is a subclass of `ipywidgets.DOMWidget `_. 33 | To use this class, it should be subclassed, and its 34 | :func:`.get_frame() ` and 35 | :func:`.handle_event() ` 36 | methods should be implemented. 37 | 38 | This widget has the following traits: 39 | 40 | * *css_width*: the logical width of the frame as a CSS string. Default '500px'. 41 | * *css_height*: the logical height of the frame as a CSS string. Default '300px'. 42 | * *resizable*: whether the frame can be manually resized. Default True. 43 | * *quality*: the quality of the JPEG encoding during interaction/animation 44 | as a number between 1 and 100. Default 80. Set to lower numbers for more 45 | performance on slow connections. Note that each interaction is ended with a 46 | lossless image (PNG). If set to 100 or if JPEG encoding isn't possible (missing 47 | pillow or simplejpeg dependencies), then lossless PNGs will always be sent. 48 | * *max_buffered_frames*: the number of frames that is allowed to be "in-flight", 49 | i.e. sent, but not yet confirmed by the client. Default 2. Higher values 50 | may result in a higher FPS at the cost of introducing lag. 51 | * *cursor*: the cursor style, ex: "crosshair", "grab". Valid cursors: 52 | https://developer.mozilla.org/en-US/docs/Web/CSS/cursor#keyword 53 | 54 | """ 55 | 56 | # Name of the widget view class in front-end 57 | _view_name = Unicode("RemoteFrameBufferView").tag(sync=True) 58 | 59 | # Name of the widget model class in front-end 60 | _model_name = Unicode("RemoteFrameBufferModel").tag(sync=True) 61 | 62 | # Name of the front-end module containing widget view 63 | _view_module = Unicode("jupyter_rfb").tag(sync=True) 64 | 65 | # Name of the front-end module containing widget model 66 | _model_module = Unicode("jupyter_rfb").tag(sync=True) 67 | 68 | # Version of the front-end module containing widget view 69 | _view_module_version = Unicode(f"^{ref_version}").tag(sync=True) 70 | # Version of the front-end module containing widget model 71 | _model_module_version = Unicode(f"^{ref_version}").tag(sync=True) 72 | 73 | # Widget specific traits 74 | frame_feedback = Dict({}).tag(sync=True) 75 | has_visible_views = Bool(False).tag(sync=True) 76 | max_buffered_frames = Int(2, min=1) 77 | quality = Int(80, min=1, max=100) 78 | css_width = Unicode("500px").tag(sync=True) 79 | css_height = Unicode("300px").tag(sync=True) 80 | resizable = Bool(True).tag(sync=True) 81 | cursor = Unicode("default").tag(sync=True) 82 | 83 | def __init__(self, *args, **kwargs): 84 | super().__init__(*args, **kwargs) 85 | self._ipython_display_ = None # we use _repr_mimebundle_ instread 86 | # Setup an output widget, so that any errors in our callbacks 87 | # are actually shown. We display the output in the cell-output 88 | # corresponding to the cell that instantiates the widget. 89 | self._output_context = RFBOutputContext() 90 | display(self._output_context) 91 | # Init attributes for drawing 92 | self._rfb_draw_requested = False 93 | self._rfb_frame_index = 0 94 | self._rfb_last_confirmed_index = 0 95 | self._rfb_last_resize_event = None 96 | self._rfb_warned_png = False 97 | self._rfb_lossless_draw_info = None 98 | self._use_websocket = True # Could be a prop, private for now 99 | # Init stats 100 | self.reset_stats() 101 | # Setup events 102 | self.on_msg(self._rfb_handle_msg) 103 | self.observe( 104 | self._rfb_schedule_maybe_draw, names=["frame_feedback", "has_visible_views"] 105 | ) 106 | 107 | def _repr_mimebundle_(self, **kwargs): 108 | data = {} 109 | 110 | # Always add plain text 111 | plaintext = repr(self) 112 | if len(plaintext) > 110: 113 | plaintext = plaintext[:110] + "…" 114 | data["text/plain"] = plaintext 115 | 116 | # Get the actual representation 117 | try: 118 | data.update(super()._repr_mimebundle_(**kwargs)) 119 | except Exception: 120 | # On 7.6.3 and below, _ipython_display_ is used instead of _repr_mimebundle_. 121 | # We fill in the widget representation that has been in use for 5+ years. 122 | data["application/vnd.jupyter.widget-view+json"] = { 123 | "version_major": 2, 124 | "version_minor": 0, 125 | "model_id": self._model_id, 126 | } 127 | 128 | # Add initial snapshot. 129 | if self._view_name is not None: 130 | data["text/html"] = self.snapshot()._repr_html_() 131 | 132 | return data 133 | 134 | def print(self, *args, **kwargs): 135 | """Print to the widget's output area (for debugging purposes). 136 | 137 | In Jupyter, print calls that occur in a callback or an asyncio task 138 | may (depending on your version of the notebook/lab) not be shown. 139 | Inside :func:`.get_frame() ` 140 | and :func:`.handle_event() ` 141 | you can use this method instead. The signature of this method 142 | is fully compatible with the builtin print function (except for 143 | the ``file`` argument). 144 | """ 145 | self._output_context.print(*args, **kwargs) 146 | 147 | def close(self, *args, **kwargs): 148 | """Close all views of the widget and emit a close event.""" 149 | # When the widget is closed, we notify by creating a close event. The 150 | # same event is emitted from JS when the model is closed in the client. 151 | super().close(*args, **kwargs) 152 | self._rfb_handle_msg(self, {"event_type": "close"}, []) 153 | 154 | def _rfb_handle_msg(self, widget, content, buffers): 155 | """Receive custom messages and filter our events.""" 156 | if "event_type" in content: 157 | # We have some builtin handling 158 | if content["event_type"] == "resize": 159 | self._rfb_last_resize_event = content 160 | self.request_draw() 161 | elif content["event_type"] == "close": 162 | self._repr_mimebundle_ = None 163 | # Turn lists into tuples (js/json does not have tuples) 164 | if "buttons" in content: 165 | content["buttons"] = tuple(content["buttons"]) 166 | if "modifiers" in content: 167 | content["modifiers"] = tuple(content["modifiers"]) 168 | # Let the subclass handle the event 169 | with self._output_context: 170 | self.handle_event(content) 171 | 172 | # ---- drawing 173 | 174 | def snapshot(self, pixel_ratio=None, _initial=False): 175 | """Create a snapshot of the current state of the widget. 176 | 177 | Returns an ``IPython DisplayObject`` that can simply be used as 178 | a cell output. The display object has a ``data`` attribute that holds 179 | the image array data (typically a numpy array). 180 | 181 | The ``pixel_ratio`` can optionally be set to influence the resolution. 182 | By default the widgets' "native" pixel-ratio is used. 183 | """ 184 | # Get the current size 185 | ref_resize_event = self._rfb_last_resize_event 186 | new_pixel_ratio = None 187 | if ref_resize_event: 188 | # We know the size from the last resize event 189 | w = ref_resize_event["width"] 190 | h = ref_resize_event["height"] 191 | if pixel_ratio and pixel_ratio != ref_resize_event["pixel_ratio"]: 192 | new_pixel_ratio = pixel_ratio 193 | else: 194 | # There has not been a resize event yet -> guess the size from our traits 195 | new_pixel_ratio = pixel_ratio or 1 196 | css_width, css_height = self.css_width, self.css_height 197 | w = float(css_width[:-2]) if css_width.endswith("px") else 500 198 | h = float(css_height[:-2]) if css_height.endswith("px") else 300 199 | # If the new pixel ratio is different from "native", we need to resize first 200 | if new_pixel_ratio: 201 | evt = { 202 | "event_type": "resize", 203 | "width": w, 204 | "height": h, 205 | "pixel_ratio": new_pixel_ratio, 206 | } 207 | self.handle_event(evt) 208 | # Render a frame 209 | array = self.get_frame() 210 | # Reset pixel ratio 211 | if new_pixel_ratio and ref_resize_event: 212 | self.handle_event(ref_resize_event) 213 | # Create snapshot object 214 | if array is None: 215 | array = np.ones((1, 1), np.uint8) * 127 216 | if _initial: 217 | title = "initial snapshot" 218 | class_name = "initial-snapshot-" + self._model_id 219 | else: 220 | title = "snapshot" 221 | class_name = "snapshot-" + self._model_id 222 | return Snapshot(array, w, h, title, class_name) 223 | 224 | def request_draw(self): 225 | """Schedule a new draw. This method itself returns immediately. 226 | 227 | This method is automatically called on each resize event. During 228 | a draw, the :func:`.get_frame() ` 229 | method is called, and the resulting array is sent to the client. 230 | See the docs for details about scheduling. 231 | """ 232 | # Technically, _maybe_draw() may not perform a draw if there are too 233 | # many frames in-flight. But in this case, we'll eventually get 234 | # new frame_feedback, which will then trigger a draw. 235 | if not self._rfb_draw_requested: 236 | self._rfb_draw_requested = True 237 | self._rfb_cancel_lossless_draw() 238 | self._rfb_schedule_maybe_draw() 239 | 240 | def _rfb_schedule_maybe_draw(self, *args): 241 | """Schedule _maybe_draw() to be called in a fresh event loop iteration.""" 242 | loop = asyncio.get_event_loop() 243 | loop.call_soon(self._rfb_maybe_draw) 244 | # or 245 | # ioloop = tornado.ioloop.IOLoop.current() 246 | # ioloop.add_callback(self._rfb_maybe_draw) 247 | 248 | def _rfb_maybe_draw(self): 249 | """Perform a draw, if we can and should.""" 250 | feedback = self.frame_feedback 251 | # Update stats 252 | self._rfb_update_stats(feedback) 253 | # Determine whether we should perform a draw: a draw was requested, and 254 | # the client is ready for a new frame, and the client widget is visible. 255 | frames_in_flight = self._rfb_frame_index - feedback.get("index", 0) 256 | should_draw = ( 257 | self._rfb_draw_requested 258 | and frames_in_flight < self.max_buffered_frames 259 | and self.has_visible_views 260 | ) 261 | # Do the draw if we should. 262 | if should_draw: 263 | self._rfb_draw_requested = False 264 | with self._output_context: 265 | array = self.get_frame() 266 | if array is not None: 267 | self._rfb_send_frame(array) 268 | 269 | def _rfb_schedule_lossless_draw(self, array, delay=0.3): 270 | self._rfb_cancel_lossless_draw() 271 | loop = asyncio.get_event_loop() 272 | handle = loop.call_later(delay, self._rfb_lossless_draw) 273 | self._rfb_lossless_draw_info = array, handle 274 | 275 | def _rfb_cancel_lossless_draw(self): 276 | if self._rfb_lossless_draw_info: 277 | _, handle = self._rfb_lossless_draw_info 278 | self._rfb_lossless_draw_info = None 279 | handle.cancel() 280 | 281 | def _rfb_lossless_draw(self): 282 | array, _ = self._rfb_lossless_draw_info 283 | self._rfb_send_frame(array, True) 284 | 285 | def _rfb_send_frame(self, array, is_lossless_redraw=False): 286 | """Actually send a frame over to the client.""" 287 | 288 | # For considerations about performance, 289 | # see https://github.com/vispy/jupyter_rfb/issues/3 290 | 291 | quality = 100 if is_lossless_redraw else self.quality 292 | 293 | self._rfb_frame_index += 1 294 | timestamp = time.time() 295 | 296 | # Turn array into a based64-encoded JPEG or PNG 297 | t1 = time.perf_counter() 298 | mimetype, data = array2compressed(array, quality) 299 | if self._use_websocket: 300 | datas = [data] 301 | data_b64 = None 302 | else: 303 | datas = [] 304 | data_b64 = f"data:{mimetype};base64," + encodebytes(data).decode() 305 | t2 = time.perf_counter() 306 | 307 | if "jpeg" in mimetype: 308 | self._rfb_schedule_lossless_draw(array) 309 | else: 310 | self._rfb_cancel_lossless_draw() 311 | # Issue png warning? 312 | if quality < 100 and not self._rfb_warned_png: 313 | self._rfb_warned_png = True 314 | self.print( 315 | "Warning: No JPEG encoder found, using PNG instead. " 316 | + "Install simplejpeg or pillow for better performance." 317 | ) 318 | 319 | if is_lossless_redraw: 320 | # No stats, also not on the confirmation of this frame 321 | self._rfb_last_confirmed_index = self._rfb_frame_index 322 | else: 323 | # Stats 324 | self._rfb_stats["img_encoding_sum"] += t2 - t1 325 | self._rfb_stats["sent_frames"] += 1 326 | if self._rfb_stats["start_time"] <= 0: # Start measuring 327 | self._rfb_stats["start_time"] = timestamp 328 | self._rfb_last_confirmed_index = self._rfb_frame_index - 1 329 | 330 | # Compose message and send 331 | msg = dict( 332 | type="framebufferdata", 333 | mimetype=mimetype, 334 | data_b64=data_b64, 335 | index=self._rfb_frame_index, 336 | timestamp=timestamp, 337 | ) 338 | self.send(msg, datas) 339 | 340 | # ----- related to stats 341 | 342 | def reset_stats(self): 343 | """Restart measuring statistics from the next sent frame.""" 344 | self._rfb_stats = { 345 | "start_time": 0, 346 | "last_time": 1, 347 | "sent_frames": 0, 348 | "confirmed_frames": 0, 349 | "roundtrip_count": 0, 350 | "roundtrip_sum": 0, 351 | "delivery_sum": 0, 352 | "img_encoding_sum": 0, 353 | } 354 | 355 | def get_stats(self): 356 | """Get the current stats since the last time ``.reset_stats()`` was called. 357 | 358 | Stats is a dict with the following fields: 359 | 360 | * *sent_frames*: the number of frames sent. 361 | * *confirmed_frames*: number of frames confirmed by the client. 362 | * *roundtrip*: avererage time for processing a frame, including receiver confirmation. 363 | * *delivery*: average time for processing a frame until it's received by the client. 364 | This measure assumes that the clock of the server and client are precisely synced. 365 | * *img_encoding*: the average time spent on encoding the array into an image. 366 | * *b64_encoding*: the average time spent on base64 encoding the data. 367 | * *fps*: the average FPS, measured from the first frame sent since ``.reset_stats()`` 368 | was called, until the last confirmed frame. 369 | """ 370 | d = self._rfb_stats 371 | roundtrip_count_div = d["roundtrip_count"] or 1 372 | sent_frames_div = d["sent_frames"] or 1 373 | fps_div = (d["last_time"] - d["start_time"]) or 0.001 374 | return { 375 | "sent_frames": d["sent_frames"], 376 | "confirmed_frames": d["confirmed_frames"], 377 | "roundtrip": d["roundtrip_sum"] / roundtrip_count_div, 378 | "delivery": d["delivery_sum"] / roundtrip_count_div, 379 | "img_encoding": d["img_encoding_sum"] / sent_frames_div, 380 | "fps": d["confirmed_frames"] / fps_div, 381 | } 382 | 383 | def _rfb_update_stats(self, feedback): 384 | """Update the stats when a new frame feedback has arrived.""" 385 | last_index = feedback.get("index", 0) 386 | if last_index > self._rfb_last_confirmed_index: 387 | timestamp = feedback["timestamp"] 388 | nframes = last_index - self._rfb_last_confirmed_index 389 | self._rfb_last_confirmed_index = last_index 390 | self._rfb_stats["confirmed_frames"] += nframes 391 | self._rfb_stats["roundtrip_count"] += 1 392 | self._rfb_stats["roundtrip_sum"] += time.time() - timestamp 393 | self._rfb_stats["delivery_sum"] += feedback["localtime"] - timestamp 394 | self._rfb_stats["last_time"] = time.time() 395 | 396 | # ----- for the subclass to implement 397 | 398 | def get_frame(self): 399 | """Return image array for the next frame. 400 | 401 | Subclasses should overload this method. It is automatically called during a draw. 402 | The returned numpy array must be NxM (grayscale), NxMx3 (RGB) or NxMx4 (RGBA). 403 | May also return ``None`` to cancel the draw. 404 | """ 405 | return np.ones((1, 1), np.uint8) * 127 406 | 407 | def handle_event(self, event): 408 | """Handle an incoming event. 409 | 410 | Subclasses should overload this method. Events include widget resize, 411 | mouse/touch interaction, key events, and more. An event is a dict with at least 412 | the key *event_type*. See :mod:`jupyter_rfb.events` for details. 413 | """ 414 | pass 415 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | # -- Project info 2 | 3 | [project] 4 | version = "0.5.3" 5 | name = "jupyter-rfb" 6 | description = "Remote Frame Buffer for Jupyter" 7 | readme = "README.md" 8 | license = { file = "LICENSE" } 9 | requires-python = ">=3.9" 10 | authors = [{ name = "Almar Klein" }] 11 | keywords = [ 12 | "ipython", 13 | "jupyter", 14 | "remote frame buffer", 15 | "visualization", 16 | "widgets", 17 | ] 18 | classifiers = [ 19 | "Development Status :: 4 - Beta", 20 | "Framework :: IPython", 21 | "Intended Audience :: Developers", 22 | "Intended Audience :: Science/Research", 23 | "Topic :: Multimedia :: Graphics", 24 | ] 25 | dependencies = ["ipywidgets>=7.6.0,<9", "jupyterlab-widgets", "numpy"] 26 | [project.optional-dependencies] 27 | build = ["build", "hatchling", "hatch-jupyter-builder", "twine"] 28 | lint = ["ruff", "pre-commit"] 29 | tests = ["pytest", "simplejpeg"] 30 | docs = ["numpy", "ipywidgets", "sphinx", "nbsphinx"] 31 | dev = ["jupyter_rfb[build,lint,tests,docs]"] 32 | 33 | [project.urls] 34 | Homepage = "https://github.com/vispy/jupyter_rfb" 35 | Documentation = "https://jupyter-rfb.readthedocs.io/en/stable/" 36 | Repository = "https://github.com/vispy/jupyter_rfb" 37 | 38 | 39 | # --- Build system 40 | # To do a release, run `python release.py` 41 | 42 | [build-system] 43 | requires = ["hatchling", "jupyterlab>=3.0.0,<5"] 44 | build-backend = "hatchling.build" 45 | 46 | [tool.hatch.build.targets.wheel.shared-data] 47 | "jupyter_rfb/nbextension/*.*" = "share/jupyter/nbextensions/jupyter_rfb/*.*" 48 | "jupyter_rfb/labextension" = "share/jupyter/labextensions/jupyter_rfb" 49 | "./install.json" = "share/jupyter/labextensions/jupyter_rfb/install.json" 50 | "./jupyter_rfb.json" = "etc/jupyter/nbconfig/notebook.d/jupyter_rfb.json" 51 | 52 | [tool.hatch.build.targets.sdist] 53 | exclude = [".github", ".git"] 54 | 55 | [tool.hatch.build.hooks.jupyter-builder] 56 | ensured-targets = ["js/dist/index.js"] 57 | dependencies = ["hatch-jupyter-builder"] 58 | build-function = "hatch_jupyter_builder.npm_builder" 59 | 60 | [tool.hatch.build.hooks.jupyter-builder.build-kwargs] 61 | path = "js" 62 | build_cmd = "build:prod" 63 | npm = ["yarn"] 64 | 65 | # --- Tooling 66 | 67 | [tool.ruff] 68 | line-length = 88 69 | 70 | [tool.ruff.lint] 71 | select = ["F", "E", "W", "N", "B", "RUF"] 72 | ignore = [ 73 | "E501", # Line too long 74 | "E731", # Do not assign a `lambda` expression, use a `def` 75 | "RUF006", # Store a reference to the return value of `loop.create_task` 76 | ] 77 | 78 | 79 | [tool.coverage.report] 80 | exclude_also = [ 81 | # Have to re-enable the standard pragma, plus a less-ugly flavor 82 | "pragma: no cover", 83 | "no-cover", 84 | "raise NotImplementedError", 85 | ] 86 | -------------------------------------------------------------------------------- /release.py: -------------------------------------------------------------------------------- 1 | """Release script for jupyter_rfb. 2 | 3 | Usage: 4 | 5 | python release.py 1.2.3 6 | 7 | This script will then: 8 | 9 | * Update all files that contain the version number. 10 | * Show a diff and ask for confirmation. 11 | * Commit the change, and tag that commit. 12 | * Git push main and the new tag. 13 | * Ask for confirmation to push to Pypi. 14 | * Create an sdist and bdist_wheel build. 15 | * Push these to Pypi. 16 | """ 17 | 18 | import os 19 | import re 20 | import sys 21 | import shutil 22 | import importlib 23 | import subprocess 24 | 25 | 26 | NAME = "jupyter_rfb" 27 | LIBNAME = NAME.replace("-", "_") 28 | ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) 29 | if not os.path.isdir(os.path.join(ROOT_DIR, LIBNAME)): 30 | sys.exit("package NAME seems to be incorrect.") 31 | 32 | 33 | finder = re.compile( 34 | r"^ *((__version__)|(version)|(\"version\")|(export const version))\s*(\=|\:)\s*[\"\']([\d\.]+)[\"\']", 35 | re.MULTILINE, 36 | ) 37 | 38 | 39 | def release(version): 40 | """Bump the version and create a release. If no version is specified, show the current version.""" 41 | 42 | version = version.lstrip("v") 43 | version_info = tuple( 44 | int(part) if part.isnumeric() else part for part in version.split(".") 45 | ) 46 | if len(version_info) == 3: 47 | version_info = (*version_info, "final", 0) 48 | if len(version_info) > 3 and version_info[3] == "final": 49 | tag_name = ".".join(str(part) for part in version_info[:3]) 50 | else: 51 | tag_name = ".".join(str(part) for part in version_info) 52 | # Check that we're not missing any libraries 53 | for x in ["build", "hatchling", "twine"]: 54 | try: 55 | importlib.import_module(x) 56 | except ImportError: 57 | sys.exit(f"You need to ``pip install {x}`` to do a version bump") 58 | # Check that there are no outstanding changes 59 | lines = ( 60 | subprocess.check_output(["git", "status", "--porcelain"]).decode().splitlines() 61 | ) 62 | lines = [line for line in lines if not line.startswith("?? ")] 63 | if lines and version: 64 | print("Cannot bump version because there are outstanding changes:") 65 | print("\n".join(lines)) 66 | return 67 | # Get the version definition 68 | only_show_version = False 69 | if not version.strip("x-"): 70 | print(__doc__) 71 | only_show_version = True 72 | 73 | for filename in [ 74 | os.path.join(ROOT_DIR, "pyproject.toml"), 75 | os.path.join(ROOT_DIR, LIBNAME, "_version.py"), 76 | os.path.join(ROOT_DIR, "js", "package.json"), 77 | os.path.join(ROOT_DIR, "js", "lib", "widget.js"), 78 | ]: 79 | fname = os.path.basename(filename) 80 | with open(filename, "rb") as f: 81 | text = f.read().decode() 82 | m = finder.search(text) 83 | if not m: 84 | raise ValueError(f"Could not find version definition in {filename}") 85 | lastgroup = len(m.groups()) 86 | if only_show_version: 87 | print( 88 | f"The current version in {fname.ljust(16)}: {m.group(lastgroup)} -> {m.group(0)}" 89 | ) 90 | else: 91 | # Apply changse 92 | i1, i2 = m.start(lastgroup), m.end(lastgroup) 93 | text = text[:i1] + version + text[i2:] 94 | with open(filename, "wb") as f: 95 | f.write(text.encode()) 96 | if only_show_version: 97 | return 98 | # Ask confirmation 99 | subprocess.run(["git", "diff"]) 100 | while True: 101 | x = input("Is this diff correct? [Y/N]: ") 102 | if x.lower() == "y": 103 | break 104 | elif x.lower() == "n": 105 | print("Cancelling (git checkout)") 106 | subprocess.run(["git", "checkout", filename]) 107 | return 108 | # Git 109 | print("Git commit and tag") 110 | subprocess.run(["git", "add", "."]) 111 | subprocess.run(["git", "commit", "-m", f"Bump version to {tag_name}"]) 112 | subprocess.run(["git", "tag", f"v{tag_name}"]) 113 | print(f"git push origin main v{tag_name}") 114 | subprocess.check_call(["git", "push", "origin", "main", f"v{tag_name}"]) 115 | # Pypi 116 | input("\nHit enter to upload to pypi: ") 117 | dist_dir = os.path.join(ROOT_DIR, "dist") 118 | if os.path.isdir(dist_dir): 119 | shutil.rmtree(dist_dir) 120 | subprocess.check_call([sys.executable, "-m", "build", "-n", "-w"]) 121 | subprocess.check_call([sys.executable, "-m", "build", "-n", "-s"]) 122 | 123 | subprocess.check_call([sys.executable, "-m", "twine", "upload", dist_dir + "/*"]) 124 | # Bye bye 125 | print("Done!") 126 | print("Don't forget to write release notes, and check pypi!") 127 | 128 | 129 | if __name__ == "__main__": 130 | version = ".".join(sys.argv[1:]) 131 | release(version) 132 | -------------------------------------------------------------------------------- /tests/test_jpg.py: -------------------------------------------------------------------------------- 1 | """Test jpg module.""" 2 | 3 | import numpy as np 4 | import pytest 5 | from pytest import raises 6 | 7 | from jupyter_rfb._jpg import ( 8 | array2jpg, 9 | select_encoder, 10 | SimpleJpegEncoder, 11 | PillowJpegEncoder, 12 | OpenCVJpegEncoder, 13 | ) 14 | 15 | 16 | def get_random_im(*shape): 17 | """Get a random image.""" 18 | return np.random.randint(0, 255, shape).astype(np.uint8) 19 | 20 | 21 | def test_array2jpg(): 22 | """Tests for array2jpg function.""" 23 | 24 | im = get_random_im(100, 100, 3) 25 | bb1 = array2jpg(im) # has default quality 26 | bb2 = array2jpg(im, 20) 27 | assert isinstance(bb1, bytes) 28 | assert len(bb2) < len(bb1) 29 | 30 | 31 | def test_simplejpeg_jpeg_encoder(): 32 | """Test the simplejpeg encoder.""" 33 | pytest.importorskip("simplejpeg") 34 | encoder = SimpleJpegEncoder() 35 | _perform_checks(encoder) 36 | _perform_error_checks(encoder) 37 | 38 | 39 | def test_pillow_jpeg_encoder(): 40 | """Test the pillow encoder.""" 41 | pytest.importorskip("PIL") 42 | encoder = PillowJpegEncoder() 43 | _perform_checks(encoder) 44 | _perform_error_checks(encoder) 45 | 46 | 47 | def test_opencv_jpeg_encoder(): 48 | """Test the opencv encoder.""" 49 | pytest.importorskip("cv2") 50 | encoder = OpenCVJpegEncoder() 51 | _perform_checks(encoder) 52 | _perform_error_checks(encoder) 53 | 54 | 55 | def _perform_checks(encoder): 56 | # RGB 57 | im = get_random_im(100, 100, 3) 58 | bb1 = encoder.encode(im, 90) 59 | bb2 = encoder.encode(im, 20) 60 | assert isinstance(bb1, bytes) 61 | assert len(bb2) < len(bb1) 62 | 63 | # RGB non-contiguous 64 | im = get_random_im(100, 100, 3) 65 | bb1 = encoder.encode(im[20:-20, 20:-20, :], 90) 66 | bb2 = encoder.encode(im[20:-20, 20:-20, :], 20) 67 | assert isinstance(bb1, bytes) 68 | assert len(bb2) < len(bb1) 69 | 70 | # Gray1 71 | im = get_random_im(100, 100) 72 | bb1 = encoder.encode(im, 90) 73 | bb2 = encoder.encode(im, 20) 74 | assert isinstance(bb1, bytes) 75 | assert len(bb2) < len(bb1) 76 | 77 | # Gray2 78 | im = get_random_im(100, 100, 1) 79 | bb1 = encoder.encode(im, 90) 80 | bb2 = encoder.encode(im, 20) 81 | assert isinstance(bb1, bytes) 82 | assert len(bb2) < len(bb1) 83 | 84 | # Gray non-contiguous 85 | im = get_random_im(100, 100) 86 | bb1 = encoder.encode(im[20:-20, 20:-20], 90) 87 | bb2 = encoder.encode(im[20:-20, 20:-20], 20) 88 | assert isinstance(bb1, bytes) 89 | assert len(bb2) < len(bb1) 90 | 91 | 92 | def _perform_error_checks(encoder): 93 | # JUst to verify that this is ok 94 | encoder.encode(get_random_im(10, 10, 3), 90) 95 | 96 | with raises(ValueError): # not a numpy array 97 | encoder.encode([1, 2, 3, 4], 90) 98 | 99 | with raises(ValueError): # not a numpy array 100 | encoder.encode(b"1234", 90) 101 | 102 | with raises(ValueError): # NxMx2? 103 | encoder.encode(get_random_im(10, 10, 2), 90) 104 | 105 | with raises(ValueError): # NxMx4? 106 | encoder.encode(get_random_im(10, 10, 4), 90) 107 | 108 | with raises(ValueError): 109 | encoder.encode(get_random_im(10, 10, 3).astype(np.float32), 90) 110 | 111 | 112 | def raise_importerror(): 113 | """Raise an import error.""" 114 | raise ImportError() 115 | 116 | 117 | def test_select_encoder(): 118 | """Test the JPEG encoder selection mechanism.""" 119 | 120 | encoder = select_encoder() 121 | assert isinstance(encoder, (SimpleJpegEncoder, PillowJpegEncoder)) 122 | 123 | # Sabotage 124 | simple_init = SimpleJpegEncoder.__init__ 125 | pillow_init = PillowJpegEncoder.__init__ 126 | cv2_init = OpenCVJpegEncoder.__init__ 127 | try: 128 | SimpleJpegEncoder.__init__ = lambda self: raise_importerror() 129 | PillowJpegEncoder.__init__ = lambda self: raise_importerror() 130 | OpenCVJpegEncoder.__init__ = lambda self: raise_importerror() 131 | 132 | encoder = select_encoder() 133 | assert not isinstance(encoder, (SimpleJpegEncoder, PillowJpegEncoder)) 134 | 135 | # Without a valid encoder, we get the stub encoder 136 | result = encoder.encode(get_random_im(10, 10, 3), 90) 137 | assert result is None 138 | 139 | finally: 140 | SimpleJpegEncoder.__init__ = simple_init 141 | PillowJpegEncoder.__init__ = pillow_init 142 | OpenCVJpegEncoder.__init__ = cv2_init 143 | -------------------------------------------------------------------------------- /tests/test_png.py: -------------------------------------------------------------------------------- 1 | """Test png module.""" 2 | 3 | import os 4 | import tempfile 5 | 6 | import numpy as np 7 | from pytest import raises 8 | 9 | from jupyter_rfb._png import array2png 10 | 11 | tempdir = tempfile.gettempdir() 12 | 13 | shape0 = 100, 100 14 | im0 = b"\x77" * 10000 15 | 16 | shape1 = 5, 5 17 | im1 = b"" 18 | im1 += b"\x00\x00\x99\x00\x00" 19 | im1 += b"\x00\x00\xff\x00\x00" 20 | im1 += b"\x99\xff\xff\xff\x99" 21 | im1 += b"\x00\x00\xff\x00\x00" 22 | im1 += b"\x00\x00\x99\x00\x00" 23 | 24 | shape2 = 6, 6 25 | im2 = b"" 26 | im2 += b"\x00\x00\x00\x88\x88\x88" 27 | im2 += b"\x00\x00\x00\x88\x88\x88" 28 | im2 += b"\x00\x00\x00\x88\x88\x88" 29 | im2 += b"\x44\x44\x44\xbb\xbb\xbb" 30 | im2 += b"\x44\x44\x44\xbb\xbb\xbb" 31 | im2 += b"\x44\x44\x44\xbb\xbb\xbb" 32 | 33 | shape3 = 5, 5, 3 34 | im3 = bytearray(5 * 5 * 3) 35 | im3[0::3] = im0[:25] 36 | im3[1::3] = im1[:25] 37 | im3[2::3] = im2[:25] 38 | 39 | shape4 = 5, 5, 4 40 | im4 = bytearray(5 * 5 * 4) 41 | im4[0::4] = im0[:25] 42 | im4[1::4] = im1[:25] 43 | im4[2::4] = im2[:25] 44 | im4[3::4] = im0[:25] 45 | 46 | im0 = np.frombuffer(im0, np.uint8).reshape(shape0) 47 | im1 = np.frombuffer(im1, np.uint8).reshape(shape1) 48 | im2 = np.frombuffer(im2, np.uint8).reshape(shape2) 49 | im3 = np.frombuffer(im3, np.uint8).reshape(shape3) 50 | im4 = np.frombuffer(im4, np.uint8).reshape(shape4) 51 | 52 | ims = im0, im1, im2, im3, im4 53 | shapes = shape0, shape1, shape2, shape3, shape4 54 | 55 | 56 | def test_writing(): 57 | """Test writing png.""" 58 | 59 | # Get bytes 60 | b0 = array2png(im0) 61 | b1 = array2png(im1) 62 | b2 = array2png(im2) 63 | b3 = array2png(im3) 64 | b4 = array2png(im4) 65 | 66 | blobs = b0, b1, b2, b3, b4 67 | 68 | # Write to disk (also for visual inspection) 69 | for i in range(5): 70 | filename = os.path.join(tempdir, "test%i.png" % i) 71 | with open(filename, "wb") as f: 72 | f.write(blobs[i]) 73 | print("wrote PNG test images to", tempdir) 74 | 75 | assert len(b1) < len(b4) # because all zeros are easier to compress 76 | 77 | # Check that providing file object yields same result 78 | with open(filename + ".check", "wb") as f: 79 | array2png(im4, f) 80 | bb1 = open(filename, "rb").read() 81 | bb2 = open(filename + ".check", "rb").read() 82 | assert len(bb1) == len(bb2) 83 | assert bb1 == bb2 84 | 85 | # Test shape with singleton dim 86 | b1_check = array2png(im1.reshape(shape1[0], shape1[1], 1)) 87 | assert b1_check == b1 88 | 89 | # Test noncontiguus data 90 | array2png(im3[1:-1, 1:-1, :]) 91 | 92 | 93 | def test_writing_failures(): 94 | """Test that errors are raised when needed.""" 95 | 96 | with raises(ValueError): 97 | array2png([1, 2, 3, 4]) 98 | 99 | with raises(ValueError): 100 | array2png(b"x" * 10) 101 | 102 | with raises(ValueError): 103 | array2png(im0.reshape(-1)) 104 | 105 | with raises(ValueError): 106 | array2png(im4.reshape(-1, -1, 8)) 107 | 108 | with raises(ValueError): 109 | array2png(im4.astype(np.float32)) 110 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | """Test the things in the utils module.""" 2 | 3 | import numpy as np 4 | from jupyter_rfb._utils import array2compressed, RFBOutputContext, Snapshot 5 | from jupyter_rfb import _jpg 6 | 7 | 8 | def test_array2compressed(): 9 | """Test the array2compressed function.""" 10 | 11 | # This test assumes that a JPEG encoder is available 12 | 13 | im = np.random.randint(0, 255, (100, 100, 3)).astype(np.uint8) 14 | 15 | # Basic check 16 | preamble, bb = array2compressed(im) 17 | assert isinstance(preamble, str) 18 | assert isinstance(bb, bytes) 19 | assert "jpeg" in preamble and "png" not in preamble 20 | 21 | # Check compression 22 | preamble1, bb1 = array2compressed(im, 90) 23 | preamble2, bb2 = array2compressed(im, 30) 24 | assert len(bb2) < len(bb1) 25 | 26 | # Check quality 100 27 | preamble3, bb3 = array2compressed(im, 100) 28 | assert len(bb3) > len(bb1) 29 | 30 | assert "jpeg" in preamble1 and "png" not in preamble1 31 | assert "jpeg" in preamble2 and "png" not in preamble1 32 | assert "png" in preamble3 and "jpeg" not in preamble3 33 | 34 | # Check that RGBA is made RGB 35 | im4 = np.random.randint(0, 255, (100, 100, 4)).astype(np.uint8) 36 | im3 = im4[:, :, :3] 37 | _, bb1 = array2compressed(im4, 90) 38 | _, bb2 = array2compressed(im3, 90) 39 | assert bb1 == bb2 40 | 41 | # Also for PNG mode 42 | _, bb1 = array2compressed(im4, 100) 43 | _, bb2 = array2compressed(im3, 100) 44 | assert bb1 == bb2 45 | 46 | # Check fallback - disable JPEG encoding, we get PNG 47 | _jpg.encoder = _jpg.StubJpegEncoder() 48 | try: 49 | preamble, bb = array2compressed(im) 50 | assert isinstance(preamble, str) 51 | assert isinstance(bb, bytes) 52 | assert "png" in preamble and "jpeg" not in preamble 53 | 54 | finally: 55 | _jpg.encoder = _jpg.select_encoder() 56 | 57 | # Should be back to normal now 58 | preamble, bb = array2compressed(im) 59 | assert "jpeg" in preamble and "png" not in preamble 60 | 61 | 62 | class StubRFBOutputContext(RFBOutputContext): 63 | """A helper class for these tests.""" 64 | 65 | def __init__(self): 66 | super().__init__() 67 | self.stdouts = [] 68 | self.stderrs = [] 69 | 70 | def append_stdout(self, msg): 71 | """Overloaded method.""" 72 | self.stdouts.append(msg) 73 | 74 | def append_stderr(self, msg): 75 | """Overloaded method.""" 76 | self.stderrs.append(msg) 77 | 78 | 79 | def test_output_context(): 80 | """Test the RFBOutputContext class.""" 81 | 82 | c = StubRFBOutputContext() 83 | 84 | # The context captures errors and sends tracebacks to its "stdout stream" 85 | with c: 86 | 1 / 0 # noqa 87 | assert len(c.stderrs) == 1 88 | assert "Traceback" in c.stderrs[0] 89 | assert "ZeroDivisionError" in c.stderrs[0] 90 | 91 | # By default it does not capture prints 92 | with c: 93 | print("aa") 94 | assert len(c.stdouts) == 0 95 | 96 | # But we can turn that on 97 | c.capture_print = True 98 | print("aa") 99 | with c: 100 | print("bb") 101 | print("cc") 102 | print("dd") 103 | c.capture_print = False 104 | with c: 105 | print("ee") 106 | assert len(c.stdouts) == 2 107 | assert c.stdouts[0] == "bb\n" 108 | assert c.stdouts[1] == "cc\n" 109 | 110 | # The print is a proper print 111 | c.print("foo", "bar", sep="-", end=".") 112 | assert c.stdouts[-1] == "foo-bar." 113 | 114 | 115 | def test_snapshot(): 116 | """Test the Snapshot class.""" 117 | 118 | a = np.zeros((10, 10), np.uint8) 119 | 120 | s = Snapshot(a, 5, 5, "footitle", "KLS") 121 | 122 | # The get_array method returns the raw data 123 | assert s.data is a 124 | 125 | # Most importantly, it has a Jupyter mime data! 126 | data = s._repr_mimebundle_() 127 | assert "text/html" in data 128 | html = data["text/html"] 129 | assert "data:image/png;base64" in html # looks like the png is in there 130 | assert "width:5px" in html and "height:5px" in html # logical size 131 | assert "class='KLS'" in html # css class name 132 | assert "footitle" in html # the title 133 | -------------------------------------------------------------------------------- /tests/test_widget.py: -------------------------------------------------------------------------------- 1 | """Tests the RemoteFrameBuffer widget class. 2 | 3 | We don't test it live (in a notebook) here, but other than that these 4 | tests are pretty complete to test the Python-side logic. 5 | """ 6 | 7 | import time 8 | 9 | import numpy as np 10 | from pytest import raises 11 | from jupyter_rfb import RemoteFrameBuffer 12 | from jupyter_rfb._utils import Snapshot 13 | from traitlets import TraitError 14 | 15 | 16 | class MyRFB(RemoteFrameBuffer): 17 | """RFB class to use in the tests.""" 18 | 19 | max_buffered_frames = 1 20 | 21 | _rfb_draw_requested = False 22 | 23 | def __init__(self): 24 | super().__init__() 25 | self.frame_feedback = {} 26 | self.has_visible_views = True 27 | self.msgs = [] 28 | 29 | def send(self, msg, buffers): 30 | """Overload the send method so we can check what was sent.""" 31 | msg = msg.copy() 32 | msg["buffers"] = buffers 33 | self.msgs.append(msg) 34 | 35 | def get_frame(self): 36 | """Return a stub array.""" 37 | return np.array([[1, 2], [3, 4]], np.uint8) 38 | 39 | def handle_event(self, event): 40 | """Implement to do nothing. 41 | 42 | Just to make sure that some events that are automatically sent 43 | dont rely on the super to be called. 44 | """ 45 | pass 46 | 47 | def trigger(self, request): 48 | """Simulate an "event loop iteration", optionally request a new draw.""" 49 | if request: 50 | self._rfb_draw_requested = True 51 | self._rfb_maybe_draw() 52 | 53 | def flush(self): 54 | """Prentend to flush a frame by setting the widget's frame feedback.""" 55 | if not len(self.msgs): 56 | return 57 | self.frame_feedback["index"] = len(self.msgs) 58 | self.frame_feedback["timestamp"] = self.msgs[-1]["timestamp"] 59 | self.frame_feedback["localtime"] = time.time() 60 | 61 | 62 | def test_widget_frames_and_stats_1(): 63 | """Test sending frames with max 1 in-flight, and how it affects stats.""" 64 | 65 | fs = MyRFB() 66 | fs.max_buffered_frames = 1 67 | 68 | assert len(fs.msgs) == 0 69 | 70 | # Request a draw 71 | fs.trigger(True) 72 | assert len(fs.msgs) == 1 73 | 74 | # Request another, no draw yet, because previous one is not yet confirmed 75 | fs.trigger(True) 76 | fs.trigger(True) 77 | fs.trigger(True) 78 | assert len(fs.msgs) == 1 79 | 80 | assert fs.get_stats()["sent_frames"] == 1 81 | assert fs.get_stats()["confirmed_frames"] == 0 82 | 83 | fs.flush() 84 | 85 | # Trigger, the previous request is still open 86 | fs.trigger(False) 87 | assert len(fs.msgs) == 2 88 | 89 | assert fs.get_stats()["sent_frames"] == 2 90 | assert fs.get_stats()["confirmed_frames"] == 1 91 | 92 | fs.flush() 93 | 94 | fs.trigger(False) 95 | assert len(fs.msgs) == 2 96 | 97 | assert fs.get_stats()["sent_frames"] == 2 98 | assert fs.get_stats()["confirmed_frames"] == 2 99 | 100 | # Trigger with no request do not trigger a draw 101 | # We *can* draw but *should* not. 102 | fs.trigger(False) 103 | assert len(fs.msgs) == 2 104 | 105 | assert fs.get_stats()["sent_frames"] == 2 106 | assert fs.get_stats()["confirmed_frames"] == 2 107 | 108 | # One more draw 109 | fs.trigger(True) 110 | assert len(fs.msgs) == 3 111 | 112 | assert fs.get_stats()["sent_frames"] == 3 113 | assert fs.get_stats()["confirmed_frames"] == 2 114 | 115 | 116 | def test_widget_frames_and_stats_3(): 117 | """Test sending frames with max 3 in-flight, and how it affects stats.""" 118 | 119 | fs = MyRFB() 120 | fs.max_buffered_frames = 3 121 | 122 | assert len(fs.msgs) == 0 123 | 124 | # 1 Send a frame 125 | fs.trigger(True) 126 | assert len(fs.msgs) == 1 127 | 128 | # 2 Send a frame 129 | fs.trigger(True) 130 | assert len(fs.msgs) == 2 131 | 132 | # 3 Send a frame 133 | fs.trigger(True) 134 | assert len(fs.msgs) == 3 135 | 136 | # 4 Send a frame - no draw, because first (3 frames ago) is no confirmed 137 | fs.trigger(True) 138 | assert len(fs.msgs) == 3 139 | 140 | assert fs.get_stats()["sent_frames"] == 3 141 | assert fs.get_stats()["confirmed_frames"] == 0 142 | 143 | fs.flush() 144 | 145 | # Trigger with True. We request a new frame, but there was a request open 146 | fs.trigger(True) 147 | assert len(fs.msgs) == 4 148 | 149 | assert fs.get_stats()["sent_frames"] == 4 150 | assert fs.get_stats()["confirmed_frames"] == 3 151 | 152 | fs.flush() 153 | 154 | # Trigger, but nothing to send (no frame pending) 155 | fs.trigger(False) 156 | assert len(fs.msgs) == 4 157 | 158 | assert fs.get_stats()["sent_frames"] == 4 159 | assert fs.get_stats()["confirmed_frames"] == 4 160 | 161 | # We can but should not do a draw 162 | fs.trigger(False) 163 | assert len(fs.msgs) == 4 164 | 165 | # Do three more draws 166 | fs.trigger(True) 167 | fs.trigger(True) 168 | fs.trigger(True) 169 | assert len(fs.msgs) == 7 170 | 171 | # And request another (but this one will have to wait) 172 | fs.trigger(True) 173 | assert len(fs.msgs) == 7 174 | 175 | fs.flush() 176 | 177 | # Trigger with False. no new request, but there was a request open 178 | fs.trigger(False) 179 | assert len(fs.msgs) == 8 180 | 181 | 182 | def test_get_frame_can_be_none(): 183 | """Test that the frame can be None to cancel a draw.""" 184 | w = MyRFB() 185 | w.max_buffered_frames = 1 186 | 187 | # Make get_frame() return None 188 | w.get_frame = lambda: None 189 | 190 | assert len(w.msgs) == 0 191 | 192 | # Request a draw 193 | w.trigger(True) 194 | assert len(w.msgs) == 0 195 | 196 | # Request another 197 | w.trigger(True) 198 | w.trigger(True) 199 | assert len(w.msgs) == 0 200 | 201 | assert w.get_stats()["sent_frames"] == 0 202 | assert w.get_stats()["confirmed_frames"] == 0 203 | 204 | 205 | def test_widget_traits(): 206 | """Test the widget's traits default values.""" 207 | 208 | w = RemoteFrameBuffer() 209 | 210 | assert w.frame_feedback == {} 211 | 212 | assert w.max_buffered_frames == 2 213 | w.max_buffered_frames = 99 214 | w.max_buffered_frames = 1 215 | with raises(TraitError): # TraitError -> min 1 216 | w.max_buffered_frames = 0 217 | 218 | assert w.css_width.endswith("px") 219 | assert w.css_height.endswith("px") 220 | 221 | assert w.resizable 222 | 223 | 224 | def test_widget_default_get_frame(): 225 | """Test default return value of get_frame().""" 226 | 227 | w = RemoteFrameBuffer() 228 | frame = w.get_frame() 229 | assert (frame is None) or frame.shape == (1, 1) 230 | 231 | 232 | def test_requesting_draws(): 233 | """Test that requesting draws works as intended.""" 234 | 235 | # By default no frame is requested 236 | w = RemoteFrameBuffer() 237 | assert not w._rfb_draw_requested 238 | 239 | # Call request_draw to request a draw 240 | w._rfb_draw_requested = False 241 | w.request_draw() 242 | assert w._rfb_draw_requested 243 | 244 | # On a resize event, a frame is requested too 245 | w._rfb_draw_requested = False 246 | w._rfb_handle_msg(None, {"event_type": "resize"}, []) 247 | assert w._rfb_draw_requested 248 | 249 | 250 | def test_has_visible_views(): 251 | """Test that no draws are performed if there are no visible views.""" 252 | 253 | fs = MyRFB() 254 | 255 | fs.trigger(True) 256 | assert len(fs.msgs) == 1 257 | 258 | fs.flush() 259 | 260 | fs.has_visible_views = False 261 | for _ in range(3): 262 | fs.trigger(True) 263 | assert len(fs.msgs) == 1 264 | 265 | fs.has_visible_views = True 266 | fs.trigger(True) 267 | assert len(fs.msgs) == 2 268 | 269 | 270 | def test_automatic_events(): 271 | """Test that some events are indeed emitted automatically.""" 272 | 273 | w = MyRFB() 274 | events = [] 275 | w.handle_event = lambda event: events.append(event) 276 | 277 | # On closing, an event is emitted 278 | # Note that when the model is closed from JS, we emit a close event from there. 279 | w.close() 280 | assert len(events) == 1 and events[0]["event_type"] == "close" 281 | 282 | 283 | def test_print(): 284 | """Test that the widget has a fully featured print method.""" 285 | w = MyRFB() 286 | w.print("foo bar", sep="-", end=".") 287 | # mmm, a bit hard to see where the data has ended up, 288 | # but if it did not error, that's something! 289 | # We test the printing itself in test_utils.py 290 | 291 | 292 | def test_snapshot(): 293 | """Test that the widget has a snapshot method that produces a Snapshot.""" 294 | w = MyRFB() 295 | s = w.snapshot() 296 | assert isinstance(s, Snapshot) 297 | assert np.all(s.data == w.get_frame()) 298 | 299 | 300 | def test_use_websocket(): 301 | """Test the use of websocket and base64.""" 302 | 303 | w = MyRFB() 304 | 305 | # The default uses a websocket 306 | w.flush() 307 | w.trigger(True) 308 | msg = w.msgs[-1] 309 | assert len(msg["buffers"]) == 1 310 | assert isinstance(msg["buffers"][0], bytes) 311 | assert msg["data_b64"] is None 312 | 313 | # Websocket use can be turned off, falling back to base64 encoded images instead 314 | w._use_websocket = False 315 | w.flush() 316 | w.trigger(True) 317 | msg = w.msgs[-1] 318 | assert len(msg["buffers"]) == 0 319 | assert isinstance(msg["data_b64"], str) 320 | assert msg["data_b64"].startswith("data:image/jpeg;base64,") 321 | 322 | # Turn it back on 323 | w._use_websocket = True 324 | w.flush() 325 | w.trigger(True) 326 | msg = w.msgs[-1] 327 | assert len(msg["buffers"]) == 1 328 | assert isinstance(msg["buffers"][0], bytes) 329 | assert msg["data_b64"] is None 330 | --------------------------------------------------------------------------------