├── .dvc ├── .gitignore └── config ├── .github ├── dependabot.yml └── workflows │ ├── docs.yml │ └── pre-commit.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .python-version ├── .readthedocs.yaml ├── .vale.ini ├── CHANGELOG.rst ├── CITATION.cff ├── LICENSE ├── README.rst ├── codemeta.json ├── data ├── .gitignore ├── fortune500.csv.dvc └── iris.csv.dvc ├── docs ├── Makefile ├── _build │ └── .gitignore ├── _static │ ├── css │ │ └── alerts.css │ └── images │ │ └── logo │ │ ├── favicon.ico │ │ └── logo.svg ├── _templates │ └── base.html ├── binder.rst ├── conf.py ├── dashboards │ ├── index.rst │ ├── panel │ │ ├── deploy.ipynb │ │ ├── fastAPI │ │ │ ├── main.py │ │ │ ├── sliders │ │ │ │ ├── pn_app.py │ │ │ │ └── sinewave.py │ │ │ └── templates │ │ │ │ └── base.html │ │ ├── fastapi.rst │ │ ├── index.rst │ │ ├── install.rst │ │ ├── interact.ipynb │ │ ├── overview.ipynb │ │ ├── panel-app.png │ │ ├── panel-fastapi.png │ │ ├── params.ipynb │ │ ├── pipelines.ipynb │ │ ├── pyodide-example.png │ │ ├── pyodide.rst │ │ ├── pyodide │ │ │ └── overview.html │ │ ├── sample_template.html │ │ ├── style.ipynb │ │ ├── templates.ipynb │ │ └── widgets.ipynb │ ├── voila-panel.rst │ └── voila │ │ ├── bqplot_vuetify_example.ipynb │ │ ├── debug.ipynb │ │ ├── index.rst │ │ ├── install.rst │ │ ├── templating.rst │ │ ├── voila-debug.png │ │ ├── voila-example-1.png │ │ ├── voila-example-2.png │ │ ├── voila-example-3.png │ │ ├── voila-gridstack.png │ │ ├── voila-vuetify-iphone.png │ │ └── voila-vuetify-laptop.png ├── genindex.rst ├── hub │ ├── Pipfile │ ├── Pipfile.lock │ ├── config.rst │ ├── index.rst │ ├── install.rst │ ├── ipyparallel │ │ ├── asyncresult.ipynb │ │ ├── check.ipynb │ │ ├── config.rst │ │ ├── direct.ipynb │ │ ├── index.rst │ │ ├── intro.rst │ │ ├── magics.ipynb │ │ ├── mpi.ipynb │ │ ├── psum.py │ │ └── task.ipynb │ ├── jupyterhub_config.py │ ├── nbviewer-service.rst │ └── systemdspawner.rst ├── index.rst ├── intro.rst ├── ipywidgets │ ├── custom-widget.ipynb │ ├── embedding.rst │ ├── examples.ipynb │ ├── index.rst │ ├── libs │ │ ├── Big.Buck.Bunny.mp4.dvc │ │ ├── index.rst │ │ ├── ipycanvas.ipynb │ │ ├── ipympl.ipynb │ │ ├── ipysheet.ipynb │ │ ├── ipyvuetify.ipynb │ │ └── ipywebrtc.ipynb │ ├── smiley.gif │ ├── widget-events.ipynb │ └── widget-list.ipynb ├── jupyterlab │ ├── collaboration.rst │ ├── extensions.rst │ ├── index.rst │ ├── initial-jupyterlab-dashboard.png │ ├── install.rst │ ├── jupyterhub.rst │ └── scheduler.rst ├── kernels │ ├── index.rst │ ├── install.rst │ ├── python310.ipynb │ ├── python38.ipynb │ ├── python39.ipynb │ └── r.rst ├── make.bat ├── nbconvert.rst ├── nbextensions │ ├── code-folding.png │ ├── configure-nbextensions.png │ ├── create-plugin.rst │ ├── index.rst │ ├── install.rst │ ├── ipylayout.ipynb │ ├── latex-env.png │ ├── list.rst │ ├── setup.ipynb │ ├── snippets-menu.png │ └── voila-ipylayout.png ├── nbviewer.rst ├── notebook │ ├── config.rst │ ├── create-notebook.rst │ ├── example.ipynb │ ├── index.rst │ ├── initial-jupyter-dashboard.png │ ├── initial-notebook.png │ ├── install.rst │ ├── parameterise │ │ ├── index.rst │ │ ├── input.ipynb │ │ ├── output.ipynb │ │ ├── output_2023-06-26_15:57:33.ipynb │ │ └── params.yaml │ ├── rename-notebook.png │ ├── shortcuts.rst │ └── testing │ │ ├── doctest.ipynb │ │ ├── hypothesis.ipynb │ │ ├── index.rst │ │ ├── ipytest.ipynb │ │ ├── mock.ipynb │ │ └── unittest.ipynb ├── sphinx │ ├── executablebooks.rst │ ├── index.rst │ └── nbsphinx.rst ├── use-cases.rst ├── viz │ └── index.rst └── whatsnew.rst ├── fastapi └── main.py ├── pipenvs ├── python-311 │ ├── Pipfile │ └── Pipfile.lock └── python-38 │ ├── Pipfile │ └── Pipfile.lock ├── pyproject.toml ├── spackenvs └── python-38 │ ├── spack.lock │ └── spack.yaml └── styles ├── .gitignore └── cusy ├── Polite.yml ├── Spelling.yml └── ignore.txt /.dvc/.gitignore: -------------------------------------------------------------------------------- 1 | /lock 2 | /config.local 3 | /updater 4 | /updater.lock 5 | /state-journal 6 | /state-wal 7 | /state 8 | /cache 9 | /tmp 10 | -------------------------------------------------------------------------------- /.dvc/config: -------------------------------------------------------------------------------- 1 | [core] 2 | analytics = false 3 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: github-actions 4 | directory: / 5 | schedule: 6 | interval: monthly 7 | -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | name: ci 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: [main] 7 | 8 | jobs: 9 | docs: 10 | name: Build docs and check links 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v4 14 | - uses: pandoc/actions/setup@v1 15 | - uses: ts-graphviz/setup-graphviz@v2 16 | - uses: actions/setup-python@v5 17 | with: 18 | cache: pip 19 | # Keep in sync with .readthedocs.yaml 20 | python-version-file: .python-version 21 | - run: python -m pip install -e ".[docs]" 22 | - run: python -m sphinx -nb html docs/ docs/_build/html 23 | - run: python -m sphinx -b linkcheck docs/ docs/_build/html 24 | -------------------------------------------------------------------------------- /.github/workflows/pre-commit.yml: -------------------------------------------------------------------------------- 1 | name: pre-commit 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: [main] 7 | 8 | jobs: 9 | pre-commit: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v4 13 | - uses: actions/setup-python@v5 14 | - uses: actions/cache@v4 15 | with: 16 | path: ~/.cache/pre-commit 17 | key: pre-commit|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }} 18 | - uses: pre-commit/action@v3.0.0 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Mac OS X 2 | .DS_Store 3 | 4 | # IPython 5 | .ipynb_checkpoints 6 | 7 | # JupyterHub 8 | jupyterhub.sqlite 9 | jupyterhub_cookie_secret 10 | 11 | # Intersphinx objects 12 | _intersphinx 13 | 14 | # spack 15 | .spack-env 16 | 17 | # Node 18 | node_modules 19 | 20 | # Examples 21 | fortune500.csv 22 | mprun_demo.py 23 | panel-examples 24 | deploy-panel.html 25 | test.png 26 | pyviz.pkl 27 | 28 | # vale 29 | /styles/* 30 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # See https://pre-commit.com for more information 2 | # See https://pre-commit.com/hooks.html for more hooks 3 | 4 | ci: 5 | autoupdate_schedule: monthly 6 | 7 | repos: 8 | - repo: https://github.com/pre-commit/pre-commit-hooks 9 | rev: v5.0.0 10 | hooks: 11 | - id: trailing-whitespace 12 | - id: end-of-file-fixer 13 | - id: check-yaml 14 | types: [file] 15 | files: \.(yml|yaml|cff)$ 16 | - id: check-added-large-files 17 | - id: check-json 18 | types: [file] # override `types: [json]` 19 | files: \.(json|ipynb)$ 20 | - repo: https://github.com/sphinx-contrib/sphinx-lint 21 | rev: v1.0.0 22 | hooks: 23 | - id: sphinx-lint 24 | types: [rst] 25 | - repo: https://github.com/pycqa/isort 26 | rev: 6.0.1 27 | hooks: 28 | - id: isort 29 | additional_dependencies: ["toml"] 30 | entry: isort --profile=black 31 | name: isort (python) 32 | - repo: https://github.com/psf/black 33 | rev: 25.1.0 34 | hooks: 35 | - id: black 36 | - repo: https://github.com/adamchainz/blacken-docs 37 | rev: "1.19.1" 38 | hooks: 39 | - id: blacken-docs 40 | args: [--line-length=79] 41 | additional_dependencies: 42 | - black 43 | - repo: https://github.com/codespell-project/codespell 44 | rev: v2.4.1 45 | hooks: 46 | - id: codespell 47 | - repo: local 48 | hooks: 49 | - id: linkcheck 50 | entry: sphinx-build -b linkcheck docs _build/html 51 | name: linkcheck 52 | language: system 53 | files: $(git diff --staged --name-only "*") 54 | -------------------------------------------------------------------------------- /.python-version: -------------------------------------------------------------------------------- 1 | 3.12 2 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yaml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | # Set the version of Python and other tools you might need 9 | build: 10 | os: ubuntu-22.04 11 | apt_packages: 12 | # graphviz is required for sphinx.ext.graphviz 13 | - graphviz 14 | tools: 15 | python: "3.12" 16 | 17 | # Build documentation in the docs/ directory with Sphinx 18 | sphinx: 19 | configuration: docs/conf.py 20 | 21 | # If using Sphinx, optionally build your docs in additional formats such as PDF 22 | formats: 23 | - epub 24 | - pdf 25 | 26 | # Optionally declare the Python requirements required to build your docs 27 | python: 28 | install: 29 | - method: pip 30 | path: . 31 | extra_requirements: 32 | - docs 33 | -------------------------------------------------------------------------------- /.vale.ini: -------------------------------------------------------------------------------- 1 | StylesPath = styles 2 | MinAlertLevel = suggestion 3 | 4 | Packages = https://github.com/cusyio/cusy-vale/archive/refs/tags/v0.1.0.zip 5 | 6 | [*.{md,rst}] 7 | TokenIgnores = (:linenos:) 8 | BasedOnStyles = cusy-en 9 | -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | Changelog 2 | ========= 3 | 4 | The versions follow `Semantic Versioning `_: 5 | ``MAJOR.MINOR.PATCH``. 6 | 7 | ``MAJOR`` 8 | is increased when incompatible changes are published. 9 | ``MINOR`` 10 | is increased when new compatible functionalities are released. 11 | ``PATCH`` 12 | is increased if the changes include only compatible bug fixes. 13 | 14 | .. _changelog 15 | 16 | 24.1.0 17 | 18 | * 🌱 Add matplotlib for social cards 19 | * 🔧 Use git tag for versioning the docs 20 | * 📝 Switch voila example to bqplot vueitfy 21 | * 📝 Switch to panel sampledata 22 | * 🔧 Add sphinx-lint 23 | * 📝 Add more alert boxes 24 | * 🔥 Remove node env 25 | * 🔥 Remove nbviewer env 26 | * 📝 Remove qgrid as it is not being developed further 27 | * 📝 Update MacTex install 28 | * 🔧 Add JupyterHub env 29 | * 🔧 Add Python 3.11 kernel config 30 | 31 | 1.1.0 32 | 33 | * 🔖 Jupyter-Tutorial 1.1.0 34 | * ✏️ Fix PDF structure 35 | * 📝 Add ‘What’s new’ section 36 | * 📝 Add Executable Books 37 | * 💄 Beautify the Jupyter overview 38 | * 📝 Add JupyterLab documentation 39 | 40 | 1.0.0 41 | 42 | * 🔧 Moving the Data Science content into Python4DataScience 43 | 44 | * /first-steps/index.html -> /notebook/index.html 45 | * /first-steps/create-notebook.html -> /notebook/create-notebook.html 46 | * /first-steps/install.html -> /notebook/install.html 47 | * /workspace/jupyter/$rest -> / 48 | * /workspace/first-steps/$rest -> /notebook/ 49 | * /workspace/ipython/$rest -> Python4DataScience:/workspace/ipython/ 50 | * /workspace/numpy/$rest -> Python4DataScience:/workspace/numpy/ 51 | * /workspace/pandas/$rest -> Python4DataScience:/workspace/pandas/ 52 | * /data-processing/$rest -> Python4DataScience:/data-processing/ 53 | * /clean-prep/$rest -> Python4DataScience:/clean-prep/ 54 | * /parameterise/$rest -> /notebook/parameterise/ 55 | * /performance/ipyparallel//$rest -> /hub/ipyparallel/ 56 | * /performance/ -> Python4DataScience:/performance/ 57 | * /productive/ -> Python4DataScience:/productive/ 58 | * /testing/$rest -> /notebook/testing/ 59 | * /web/dashboards/$rest -> /dashboards/ 60 | -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | cff-version: 1.2.0 2 | title: Jupyter Tutorial 3 | message: >- 4 | If you use this tutorial, please cite it using the 5 | metadata from this file. 6 | type: software 7 | authors: 8 | - given-names: Veit 9 | family-names: Schiele 10 | email: veit@cusy.io 11 | identifiers: 12 | - type: doi 13 | value: 10.5281/zenodo.10961038 14 | - type: url 15 | value: "https://github.com/veit/jupyter-tutorial/tree/24.1.0" 16 | repository-code: "https://github.com/veit/jupyter-tutorial/" 17 | url: "https://jupyter-tutorial.readthedocs.io/en/24.1.0" 18 | abstract: >- 19 | Training materials for setting up and using a research 20 | infrastructure based on Jupyter notebooks. 21 | keywords: 22 | - Jupyter 23 | - JupyterLab 24 | - Notebook 25 | - JupyterHub 26 | - Binder 27 | - ipywidgets 28 | - Jupyter Widgets 29 | - Dashboard 30 | - nbviewer 31 | - Panel 32 | - Voilà 33 | - Sphinx 34 | - Executable Books 35 | - MyST 36 | - nbsphinx 37 | license: BSD-3-Clause 38 | version: 24.1.0 39 | date-released: "2024-04-11" 40 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) Veit Schiele. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, 5 | are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | 14 | 3. Neither the name of Veit Schiele nor the names of its contributors 15 | may be used to endorse or promote products derived from this software 16 | without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 19 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 22 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 25 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Quick start 2 | =========== 3 | 4 | .. _badges: 5 | 6 | Status 7 | ------ 8 | 9 | .. image:: https://img.shields.io/github/contributors/veit/jupyter-tutorial.svg 10 | :alt: Contributors 11 | :target: https://github.com/veit/jupyter-tutorial/graphs/contributors 12 | .. image:: https://img.shields.io/github/license/veit/jupyter-tutorial.svg 13 | :alt: License 14 | :target: https://github.com/veit/jupyter-tutorial/blob/master/LICENSE 15 | .. image:: https://results.pre-commit.ci/badge/github/veit/jupyter-tutorial/main.svg 16 | :alt: pre-commit.ci status 17 | :target: https://results.pre-commit.ci/latest/github/veit/jupyter-tutorial/main 18 | .. image:: https://readthedocs.org/projects/jupyter-tutorial/badge/?version=latest 19 | :alt: Docs 20 | :target: https://jupyter-tutorial.readthedocs.io/en/latest/ 21 | .. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.10961038.svg 22 | :alt: DOI 23 | :target: https://doi.org/10.5281/zenodo.10961038 24 | .. image:: https://img.shields.io/badge/dynamic/json?label=Mastodon&query=totalItems&url=https%3A%2F%2Fmastodon.social%2F@JupyterTutorial%2Ffollowers.json&logo=mastodon 25 | :alt: Mastodon 26 | :target: https://mastodon.social/@JupyterTutorial 27 | 28 | .. _first-steps: 29 | 30 | Installation 31 | ------------ 32 | 33 | #. Download and unpack: 34 | 35 | .. code-block:: console 36 | 37 | $ curl -O https://codeload.github.com/veit/jupyter-tutorial/zip/main 38 | $ unzip main 39 | Archive: main 40 | … 41 | creating: jupyter-tutorial-main/ 42 | … 43 | 44 | #. Install Python packages: 45 | 46 | .. code-block:: console 47 | 48 | $ cd jupyter-tutorial-main 49 | $ python3 -m venv .venv 50 | $ . .venv/bin/activate 51 | $ python -m pip install --upgrade pip 52 | $ python -m pip install -e ".[dev]" 53 | 54 | #. Install the `Jupyter Notebook Extensions 55 | `_ Javascript and CSS 56 | files: 57 | 58 | .. code-block:: console 59 | 60 | $ jupyter contrib nbextension install --user 61 | jupyter contrib nbextension install --user 62 | Installing jupyter_contrib_nbextensions nbextension files to jupyter data directory 63 | … 64 | Successfully installed jupyter-contrib-core-0.3.3 jupyter-contrib-nbextensions-0.5.1 65 | jupyter-highlight-selected-word-0.2.0 jupyter-latex-envs-1.4.6 66 | jupyter-nbextensions-configurator-0.4.1 67 | … 68 | $ jupyter nbextension enable latex_envs --user --py 69 | Enabling notebook extension latex_envs/latex_envs... 70 | - Validating: OK 71 | 72 | #. Create HTML documentation: 73 | 74 | Note that pandoc has to be installed. On Debian/Ubuntu you can just run 75 | 76 | .. code-block:: console 77 | 78 | $ sudo apt install pandoc 79 | 80 | To create the HTML documentation run these commands: 81 | 82 | .. code-block:: console 83 | 84 | $ cd docs/ 85 | $ make html 86 | 87 | #. Create a PDF: 88 | 89 | For the creation of a PDF file you need additional packages. 90 | 91 | For Debian/Ubuntu you get them with the following command: 92 | 93 | .. code-block:: console 94 | 95 | $ sudo apt install texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended latexmk 96 | 97 | or for macOS with: 98 | 99 | .. code-block:: console 100 | 101 | $ brew cask install mactex 102 | … 103 | 🍺 mactex was successfully installed! 104 | $ curl --remote-name https://www.tug.org/fonts/getnonfreefonts/install-getnonfreefonts 105 | $ sudo texlua install-getnonfreefonts 106 | … 107 | mktexlsr: Updating /usr/local/texlive/2020/texmf-dist/ls-R... 108 | mktexlsr: Done. 109 | 110 | Then you can generate a PDF with: 111 | 112 | .. code-block:: console 113 | 114 | $ make latexpdf 115 | … 116 | The LaTeX files are in _build/latex. 117 | Run 'make' in that directory to run these through (pdf)latex 118 | … 119 | 120 | You can find the PDF at ``docs/_build/latex/jupytertutorial.pdf``. 121 | 122 | #. Run Vale to check spelling 123 | 124 | You can install download cusy-vale with: 125 | 126 | .. code-block:: console 127 | 128 | $ vale sync 129 | Syncing cusy-vale [1/1] ██████████████████████████████████████████████ 100% | 0s 130 | SUCCESS Synced 1 package(s) to '/Users/veit/cusy/trn/jupyter-tutorial/styles'. 131 | 132 | .. seealso:: 133 | * `Vale installation `_ 134 | * `Vale formats `_ 135 | 136 | Now you can check the RestructuredText files with: 137 | 138 | .. code-block:: console 139 | 140 | $ vale . 141 | ✔ 0 errors, 0 warnings and 0 suggestions in 201 files. 142 | 143 | .. _follow-us: 144 | 145 | Follow us 146 | --------- 147 | 148 | * `GitHub `_ 149 | * `Twitter `_ 150 | * `Mastodon `_ 151 | 152 | Pull-Requests 153 | ------------- 154 | 155 | If you have suggestions for improvements and additions, I recommend that you 156 | create a `Fork `_ of my `GitHub 157 | Repository `_ and make your changes 158 | there. . You are also welcome to make a *pull request*. If the changes 159 | contained therein are small and atomic, I’ll be happy to look at your 160 | suggestions. 161 | -------------------------------------------------------------------------------- /codemeta.json: -------------------------------------------------------------------------------- 1 | { 2 | "@context": "https://doi.org/10.5063/schema/codemeta-2.0", 3 | "@type": "SoftwareSourceCode", 4 | "codeRepository": "https://github.com/veit/jupyter-tutorial", 5 | "dateCreated": "2019-06-27", 6 | "datePublished": "2019-06-27", 7 | "dateModified": "2024-04-11", 8 | "issueTracker": "https://github.com/veit/jupyter-tutorial/issues", 9 | "name": "Jupyter Tutorial", 10 | "version": "24.1.0", 11 | "description": "Training materials for setting up and using a research infrastructure based on Jupyter notebooks.\n", 12 | "developmentStatus": "active", 13 | "isPartOf": "https://www.python4data.science/", 14 | "referencePublication": "https://doi.org/10.5281/zenodo.10961038", 15 | "keywords": [ 16 | "Jupyter", 17 | "JupyterLab", 18 | "Notebook", 19 | "JupyterHub", 20 | "ipywidgets", 21 | "Jupyter Widgets", 22 | "Dashboard", 23 | "Binder", 24 | "Dashboard", 25 | "nbviewer", 26 | "Panel", 27 | "Voilà", 28 | "Sphinx", 29 | "Executable Books", 30 | "MyST", 31 | "nbsphinx" 32 | ], 33 | "programmingLanguage": [ 34 | "Python 3" 35 | ], 36 | "operatingSystem": [ 37 | "Linux", 38 | "Windows", 39 | "macOS" 40 | ], 41 | "author": [ 42 | { 43 | "@type": "Person", 44 | "givenName": "Veit", 45 | "familyName": "Schiele", 46 | "email": "veit@cusy.io", 47 | "affiliation": { 48 | "@type": "Organization", 49 | "name": "cusy GmbH" 50 | } 51 | } 52 | ] 53 | } 54 | -------------------------------------------------------------------------------- /data/.gitignore: -------------------------------------------------------------------------------- 1 | /fortune500.csv 2 | /Big.Buck.Bunny.mp4 3 | /example.webm 4 | /iris.csv 5 | -------------------------------------------------------------------------------- /data/fortune500.csv.dvc: -------------------------------------------------------------------------------- 1 | outs: 2 | - path: fortune500.csv 3 | metric: false 4 | cache: true 5 | persist: false 6 | md5: 2e48c4204c3a74bab06bb93c8e86da74 7 | md5: df4b8a006484f97ac35cc6b5fe8a82cc 8 | -------------------------------------------------------------------------------- /data/iris.csv.dvc: -------------------------------------------------------------------------------- 1 | md5: b79de48ff285c75387327c481eec8f42 2 | outs: 3 | - md5: 4bb1a88c88be36dccdeb2cc60f51bcff 4 | path: iris.csv 5 | cache: true 6 | metric: false 7 | persist: false 8 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/_build/.gitignore: -------------------------------------------------------------------------------- 1 | # ignore everything except .gitignore 2 | * 3 | !.gitignore 4 | -------------------------------------------------------------------------------- /docs/_static/css/alerts.css: -------------------------------------------------------------------------------- 1 | .alert { 2 | padding: 15px; 3 | margin-bottom: 18px; 4 | border: 1px solid transparent; 5 | border-radius: 2px; 6 | } 7 | .alert h4 { 8 | margin-top: 0; 9 | color: inherit; 10 | } 11 | .alert .alert-link { 12 | font-weight: bold; 13 | } 14 | .alert > p, 15 | .alert > ul { 16 | margin-bottom: 0; 17 | } 18 | .alert > p + p { 19 | margin-top: 5px; 20 | } 21 | .alert-dismissable, 22 | .alert-dismissible { 23 | padding-right: 35px; 24 | } 25 | .alert-dismissable .close, 26 | .alert-dismissible .close { 27 | position: relative; 28 | top: -2px; 29 | right: -21px; 30 | color: inherit; 31 | } 32 | .alert-success { 33 | color: #3c763d; 34 | background-color: #dff0d8; 35 | border-color: #d6e9c6; 36 | } 37 | .alert-success hr { 38 | border-top-color: #c9e2b3; 39 | } 40 | .alert-success .alert-link { 41 | color: #2b542c; 42 | } 43 | .alert-info { 44 | color: #31708f; 45 | background-color: #d9edf7; 46 | border-color: #bce8f1; 47 | } 48 | .alert-info hr { 49 | border-top-color: #a6e1ec; 50 | } 51 | .alert-info .alert-link { 52 | color: #245269; 53 | } 54 | .alert-warning { 55 | color: #8a6d3b; 56 | background-color: #fcf8e3; 57 | border-color: #faebcc; 58 | } 59 | .alert-warning hr { 60 | border-top-color: #f7e1b5; 61 | } 62 | .alert-warning .alert-link { 63 | color: #66512c; 64 | } 65 | .alert-danger { 66 | color: #a94442; 67 | background-color: #f2dede; 68 | border-color: #ebccd1; 69 | } 70 | .alert-danger hr { 71 | border-top-color: #e4b9c0; 72 | } 73 | .alert-danger .alert-link { 74 | color: #843534; 75 | } 76 | 77 | .rendered_html .alert { 78 | margin-bottom: initial; 79 | } 80 | .rendered_html * + .alert { 81 | margin-top: 1em; 82 | } 83 | -------------------------------------------------------------------------------- /docs/_static/images/logo/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veit/jupyter-tutorial/24fcd76f43fc46ffd02e3bd98479a41ad6565197/docs/_static/images/logo/favicon.ico -------------------------------------------------------------------------------- /docs/_templates/base.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {%- block site_meta -%} 5 | 6 | 7 | 8 | 9 | 10 | {%- if metatags %}{{ metatags }}{% endif -%} 11 | 12 | {%- block linktags %} 13 | {%- if hasdoc('about') -%} 14 | 15 | {%- endif -%} 16 | {%- if hasdoc('genindex') -%} 17 | 18 | {%- endif -%} 19 | {%- if hasdoc('search') -%} 20 | 21 | {%- endif -%} 22 | {%- if hasdoc('copyright') -%} 23 | 24 | {%- endif -%} 25 | {%- if next -%} 26 | 27 | {%- endif -%} 28 | {%- if prev -%} 29 | 30 | {%- endif -%} 31 | {#- rel="canonical" (set by html_baseurl) -#} 32 | {%- if pageurl %} 33 | 34 | {%- endif %} 35 | {%- endblock linktags %} 36 | 37 | {# Favicon #} 38 | {%- if favicon_url -%} 39 | 40 | {%- endif -%} 41 | 42 | 43 | 44 | {%- endblock site_meta -%} 45 | 46 | {#- Site title -#} 47 | {%- block htmltitle -%} 48 | {% if not docstitle %} 49 | {{ title|striptags|e }} 50 | {% elif pagename == master_doc %} 51 | {{ docstitle|striptags|e }} 52 | {% else %} 53 | {{ title|striptags|e }} - {{ docstitle|striptags|e }} 54 | {% endif %} 55 | {%- endblock -%} 56 | 57 | {%- block styles -%} 58 | 59 | {# Custom stylesheets #} 60 | {%- block regular_styles -%} 61 | {%- for css in css_files -%} 62 | {% if css|attr("filename") -%} 63 | {{ css_tag(css) }} 64 | {%- else -%} 65 | 66 | {%- endif %} 67 | {% endfor -%} 68 | {%- endblock regular_styles -%} 69 | 70 | {#- Theme-related stylesheets -#} 71 | {%- block theme_styles %} 72 | {% include "partials/_head_css_variables.html" with context %} 73 | {%- endblock -%} 74 | 75 | {%- block extra_styles %} 76 | {%- endblock -%} 77 | 78 | {%- endblock styles -%} 79 | 80 | {#- Custom front matter #} 81 | {%- block extrahead -%}{%- endblock -%} 82 | 83 | 84 | {% block body %} 85 | 88 | {% endblock %} 89 | 90 | {%- block scripts -%} 91 | 92 | {# Custom JS #} 93 | {%- block regular_scripts -%} 94 | {% for path in script_files -%} 95 | {{ js_tag(path) }} 96 | {% endfor -%} 97 | {%- endblock regular_scripts -%} 98 | 99 | {# Theme-related JavaScript code #} 100 | {%- block theme_scripts -%} 101 | {%- endblock -%} 102 | 103 | {%- endblock scripts -%} 104 | 105 | 106 | -------------------------------------------------------------------------------- /docs/binder.rst: -------------------------------------------------------------------------------- 1 | Binder 2 | ====== 3 | 4 | `Binder `_ provides an easy way to share computer 5 | environments with everyone. Binder is used for 6 | 7 | Teaching and training 8 | Binder can be used to share links to interactive data analysis environments. 9 | This is great for workshops, tutorials and courses and allows you to get 10 | students up to speed with the code much more quickly. 11 | Technical documentation 12 | Binder tools can be used to make documentation and demonstrations of tools 13 | interactive. 14 | Open educational resources 15 | Binder can provide publicly available interactive educational materials, 16 | enabling richer experiences. 17 | Reproducible scientific analysis 18 | Binder allows you to share an interactive environment along with your code 19 | and analysis. You can share a link where others can reproduce and interact 20 | with your work. For example, the `Neurolibre `_ 21 | project uses Binder to reproduce neuroscience analyses. 22 | 23 | Binder provides a full open-source infrastructure stack. The main tools are 24 | 25 | ``BinderHub`` 26 | provides the Binder service in the cloud. 27 | 28 | .. seealso:: 29 | * `Repository `_ 30 | * `Docs `_ 31 | * `Examples `_ 32 | 33 | ``repo2docker`` 34 | creates reproducible Docker images from a Git repository. 35 | 36 | .. seealso:: 37 | * `Repository `__ 38 | * `Docs `__ 39 | 40 | `mybinder.org `_ 41 | public BinderHub deployment. 42 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # http://www.sphinx-doc.org/en/master/config 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | # sys.path.insert(0, os.path.abspath('.')) 14 | import os 15 | import re 16 | 17 | 18 | # Set canonical URL from the Read the Docs Domain 19 | html_baseurl = os.environ.get("READTHEDOCS_CANONICAL_URL", "") 20 | 21 | html_context = {} 22 | # Tell Jinja2 templates the build is running on Read the Docs 23 | if os.environ.get("READTHEDOCS", "") == "True": 24 | html_context["READTHEDOCS"] = True 25 | 26 | 27 | # -- Project information ----------------------------------------------------- 28 | 29 | project = "Jupyter Tutorial" 30 | author = "Veit Schiele" 31 | copyright = f"2019–2024, {author}" 32 | 33 | # The full version, including alpha/beta/rc tags 34 | release = re.sub("^v", "", os.popen("git describe --abbrev=0").read().strip()) 35 | 36 | 37 | # -- General configuration --------------------------------------------------- 38 | 39 | # Add any Sphinx extension module names here, as strings. They can be 40 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 41 | # ones. 42 | extensions = [ 43 | "nbsphinx", 44 | "IPython.sphinxext.ipython_console_highlighting", 45 | # 'jupyter_sphinx.execute', 46 | "sphinx.ext.autodoc", 47 | "sphinx.ext.intersphinx", 48 | "sphinx.ext.graphviz", 49 | "sphinx.ext.todo", 50 | "sphinxcontrib.cairosvgconverter", 51 | "sphinxext.opengraph", 52 | "sphinx_copybutton", 53 | "sphinx_inline_tabs", 54 | ] 55 | 56 | # Add any paths that contain templates here, relative to this directory. 57 | templates_path = ["_templates"] 58 | 59 | # Fix for Sphinx < 2.0 60 | master_doc = "index" 61 | 62 | # The language for content autogenerated by Sphinx. Refer to documentation 63 | # for a list of supported languages. 64 | # 65 | # This is also used if you do content translation via gettext catalogs. 66 | # Usually you set "language" from the command line for these cases. 67 | language = "en" 68 | 69 | # List of patterns, relative to source directory, that match files and 70 | # directories to ignore when looking for source files. 71 | # This pattern also affects html_static_path and html_extra_path. 72 | exclude_patterns = [ 73 | "_build", 74 | "Thumbs.db", 75 | ".DS_Store", 76 | "**/.ipynb_checkpoints", 77 | "**/.*.ipynb", 78 | "**/input.ipynb", 79 | "**/output*.ipynb", 80 | ] 81 | 82 | 83 | # -- Options for HTML output ------------------------------------------------- 84 | 85 | # The theme to use for HTML and HTML Help pages. See the documentation for 86 | # a list of builtin themes. 87 | html_theme = "furo" 88 | 89 | # Theme options are theme-specific and customize the look and feel of a theme 90 | # further. For a list of options available for each theme, see the 91 | # documentation. 92 | # 93 | # Change default HTML title 94 | html_title = f"{project} {release}" 95 | # 96 | # html_theme_options = {} 97 | # html_sidebars = {} 98 | 99 | # Add any paths that contain custom static files (such as style sheets) here, 100 | # relative to this directory. They are copied after the builtin static files, 101 | # so a file named "default.css" will overwrite the builtin "default.css". 102 | html_static_path = ["_static"] 103 | 104 | html_logo = "_static/images/logo/logo.svg" 105 | html_favicon = "_static/images/logo/favicon.ico" 106 | 107 | html_css_files = [ 108 | "css/alerts.css", 109 | ] 110 | 111 | # Add any paths that contain templates here, relative to this directory. 112 | templates_path = ["_templates"] 113 | 114 | # -- Options for LaTeX output --------------------------------------------- 115 | 116 | latex_elements = { 117 | # The paper size ('letterpaper' or 'a4paper'). 118 | # 'papersize': 'a4paper', 119 | # The font size ('10pt', '11pt' or '12pt'). 120 | # 'pointsize': '9pt', 121 | # Additional stuff for the LaTeX preamble. 122 | "preamble": r"\usepackage{pmboxdraw}", 123 | # Latex figure (float) alignment 124 | # 'figure_align': 'htbp', 125 | } 126 | 127 | # Grouping the document tree into LaTeX files. List of tuples 128 | # (source start file, target name, title, 129 | # author, documentclass [howto, manual, or own class]). 130 | latex_documents = [ 131 | ( 132 | master_doc, 133 | "jupyter-tutorial.tex", 134 | "Jupyter Tutorial", 135 | "Veit Schiele", 136 | "manual", 137 | ), 138 | ] 139 | 140 | # -- nbsphinx configuration -------------------------------------------------- 141 | 142 | nbsphinx_allow_errors = True 143 | # nbsphinx_execute = 'always' 144 | 145 | # -- intersphinx configuration ----------------------------------------------- 146 | 147 | intersphinx_mapping = { 148 | "python": ("https://docs.python.org/3", None), 149 | "ipython": ("https://ipython.readthedocs.io/en/latest/", None), 150 | "pytest": ("https://docs.pytest.org/en/latest/", None), 151 | "jupyter-notebook": ( 152 | "https://jupyter-notebook.readthedocs.io/en/stable/", 153 | None, 154 | ), 155 | "jupyterhub": ("https://jupyterhub.readthedocs.io/en/stable/", None), 156 | "nbconvert": ("https://nbconvert.readthedocs.io/en/latest/", None), 157 | "jupyter-contrib-nbextensions": ( 158 | "https://jupyter-contrib-nbextensions.readthedocs.io/en/latest/", 159 | None, 160 | ), 161 | "sphinx": ("https://www.sphinx-doc.org/en/master/", None), 162 | "nbsphinx": ("https://nbsphinx.readthedocs.io/en/0.4.2/", None), 163 | "spack": ("https://spack-tutorial.readthedocs.io/en/latest/", None), 164 | "ipyparallel": ("https://ipyparallel.readthedocs.io/en/latest/", None), 165 | "bokeh": ("https://docs.bokeh.org/en/latest", None), 166 | "pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None), 167 | "pyviz": ("https://pyviz-tutorial.readthedocs.io/de/latest/", None), 168 | "python-basics": ( 169 | "https://python-basics-tutorial.readthedocs.io/en/latest/", 170 | None, 171 | ), 172 | "python4datascience": ( 173 | "https://www.python4data.science/en/latest/", 174 | None, 175 | ), 176 | } 177 | 178 | # All HTTP redirections from the source URI to the canonical URI will be treated# as "working". 179 | linkcheck_allowed_redirects = { 180 | "ttps://docs.pyscript.net/latest/reference/faq.html": r"https://docs.pyscript.net/latest/faq/", 181 | } 182 | linkcheck_ignore = [ 183 | # Anchor not found 184 | r"https://github.com/jupyter/nbviewer#extending-the-notebook-viewer", 185 | # 403 Client Error: Forbidden for url 186 | r"https://jupyter-flex.danielfrg.com", 187 | r"https://sourceforge.net/projects/*", 188 | ] 189 | 190 | 191 | def setup(app): 192 | # from sphinx.ext.autodoc import cut_lines 193 | # app.connect('autodoc-process-docstring', cut_lines(4, what=['module'])) 194 | app.add_object_type( 195 | "label", 196 | "label", 197 | objname="label value", 198 | indextemplate="pair: %s; label value", 199 | ) 200 | 201 | 202 | # -- graphviz configuration -------------------------------------------------- 203 | 204 | graphviz_output_format = "svg" 205 | -------------------------------------------------------------------------------- /docs/dashboards/index.rst: -------------------------------------------------------------------------------- 1 | Dashboards 2 | ========== 3 | 4 | :doc:`voila/index` 5 | was developed by `QuantStack `_ as a dashboard 6 | solution based on Jupyter Notebooks and :doc:`/ipywidgets/index`. It 7 | displays the output of all notebook cells. 8 | :doc:`panel/index` 9 | was developed on the basis of `Bokeh `_ 10 | and `Param `_ and offers a Python toolkit 11 | specifically for creating apps and dashboards, but it does not only support 12 | Bokeh plots. It allows multi-level and multi-page workflows. 13 | 14 | With this tabular overview you can quickly compare the activities and licenses 15 | of the various libraries. 16 | 17 | .. csv-table:: GitHub-Insights: Dashboards 18 | :header: "Name", "Stars", "Contributors", "Commit activity", "License" 19 | 20 | "`Voilà `_",".. image:: https://raster.shields.io/github/stars/voila-dashboards/voila",".. image:: https://raster.shields.io/github/contributors/voila-dashboards/voila",".. image:: https://raster.shields.io/github/commit-activity/y/voila-dashboards/voila",".. image:: https://raster.shields.io/github/license/voila-dashboards/voila" 21 | "`Panel `_",".. image:: https://raster.shields.io/github/stars/holoviz/panel",".. image:: https://raster.shields.io/github/contributors/holoviz/panel",".. image:: https://raster.shields.io/github/commit-activity/y/holoviz/panel",".. image:: https://raster.shields.io/github/license/holoviz/panel" 22 | 23 | .. tip:: 24 | cusy seminars: 25 | 26 | * `Visualising data with Python 27 | `_ 28 | * `Designing data visualisations 29 | `_ 30 | * `Create dashboards 31 | `_ 32 | 33 | .. toctree:: 34 | :titlesonly: 35 | :hidden: 36 | :maxdepth: 0 37 | 38 | voila-panel 39 | voila/index 40 | panel/index 41 | -------------------------------------------------------------------------------- /docs/dashboards/panel/fastAPI/main.py: -------------------------------------------------------------------------------- 1 | import panel as pn 2 | 3 | from bokeh.embed import server_document 4 | from sliders.pn_app import createApp 5 | 6 | from fastapi import FastAPI, Request 7 | from fastapi.templating import Jinja2Templates 8 | 9 | 10 | app = FastAPI() 11 | templates = Jinja2Templates(directory="templates") 12 | 13 | 14 | @app.get("/") 15 | async def bkapp_page(request: Request): 16 | script = server_document("http://127.0.0.1:5000/app") 17 | return templates.TemplateResponse( 18 | "base.html", {"request": request, "script": script} 19 | ) 20 | 21 | 22 | pn.serve( 23 | {"/app": createApp}, 24 | port=5000, 25 | allow_websocket_origin=["127.0.0.1:8000"], 26 | address="127.0.0.1", 27 | show=False, 28 | ) 29 | -------------------------------------------------------------------------------- /docs/dashboards/panel/fastAPI/sliders/pn_app.py: -------------------------------------------------------------------------------- 1 | import panel as pn 2 | 3 | from .sinewave import SineWave 4 | 5 | 6 | def createApp(): 7 | sw = SineWave() 8 | return pn.Row(sw.param, sw.plot).servable() 9 | -------------------------------------------------------------------------------- /docs/dashboards/panel/fastAPI/sliders/sinewave.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import param 3 | 4 | from bokeh.models import ColumnDataSource 5 | from bokeh.plotting import figure 6 | 7 | 8 | class SineWave(param.Parameterized): 9 | offset = param.Number(default=0.0, bounds=(-5.0, 5.0)) 10 | amplitude = param.Number(default=1.0, bounds=(-5.0, 5.0)) 11 | phase = param.Number(default=0.0, bounds=(0.0, 2 * np.pi)) 12 | frequency = param.Number(default=1.0, bounds=(0.1, 5.1)) 13 | N = param.Integer(default=200, bounds=(0, None)) 14 | x_range = param.Range(default=(0, 4 * np.pi), bounds=(0, 4 * np.pi)) 15 | y_range = param.Range(default=(-2.5, 2.5), bounds=(-10, 10)) 16 | 17 | def __init__(self, **params): 18 | super().__init__(**params) 19 | x, y = self.sine() 20 | self.cds = ColumnDataSource(data=dict(x=x, y=y)) 21 | self.plot = figure( 22 | height=400, 23 | width=400, 24 | tools="crosshair, pan, reset, save, wheel_zoom", 25 | x_range=self.x_range, 26 | y_range=self.y_range, 27 | ) 28 | self.plot.line("x", "y", source=self.cds, line_width=3, line_alpha=0.6) 29 | 30 | @param.depends( 31 | "N", 32 | "frequency", 33 | "amplitude", 34 | "offset", 35 | "phase", 36 | "x_range", 37 | "y_range", 38 | watch=True, 39 | ) 40 | def update_plot(self): 41 | x, y = self.sine() 42 | self.cds.data = dict(x=x, y=y) 43 | self.plot.x_range.start, self.plot.x_range.end = self.x_range 44 | self.plot.y_range.start, self.plot.y_range.end = self.y_range 45 | 46 | def sine(self): 47 | x = np.linspace(0, 4 * np.pi, self.N) 48 | y = ( 49 | self.amplitude * np.sin(self.frequency * x + self.phase) 50 | + self.offset 51 | ) 52 | return x, y 53 | -------------------------------------------------------------------------------- /docs/dashboards/panel/fastAPI/templates/base.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Panel in FastAPI: sliders 5 | 6 | 7 | {{ script|safe }} 8 | 9 | 10 | -------------------------------------------------------------------------------- /docs/dashboards/panel/fastapi.rst: -------------------------------------------------------------------------------- 1 | FastAPI integration 2 | =================== 3 | 4 | Panel usually runs on a :doc:`pyviz:bokeh/bokeh-server`, which in turn runs on 5 | `Tornado `_. However, it is also often 6 | useful to embed a Panel app into a large web application, such as a FastAPI web 7 | server. Integration with FastAPI is easier compared to others such as 8 | :doc:`Flask `, as it is a more lightweight 9 | framework. Using Panel with FastAPI requires only a little more effort than 10 | notebooks and Bokeh servers. 11 | 12 | Configuration 13 | ------------- 14 | 15 | Before we start adding a Bokeh application to our FastApi server, we need to set 16 | up some of the basic configuration in :download:`fastAPI/main.py`: 17 | 18 | #. First, we import all the necessary elements: 19 | 20 | .. literalinclude:: fastAPI/main.py 21 | :caption: fastAPI/main.py 22 | :linenos: 23 | :lines: 1-3, 5-7 24 | 25 | #. Next, we define ``app`` as an instance of ``FastAPI`` and define the path to 26 | the template directory: 27 | 28 | .. literalinclude:: fastAPI/main.py 29 | :lineno-start: 10 30 | :lines: 10-11 31 | 32 | #. Now we create our first routine via an asynchronous function and refer it to 33 | our BokehServer: 34 | 35 | .. literalinclude:: fastAPI/main.py 36 | :lineno-start: 14 37 | :lines: 14-19 38 | 39 | #. As you can see from the code, a `Jinja2 40 | `_ 41 | template :download:`fastAPI/templates/base.html` is expected. This can have 42 | the following content, for example: 43 | 44 | .. literalinclude:: fastAPI/templates/base.html 45 | :caption: fastAPI/templates/base.html 46 | :language: html 47 | :linenos: 48 | :lines: 1- 49 | 50 | #. Let’s now return to our :download:`fastAPI/main.py` file to start our bokeh 51 | server with ``pn.serve()``: 52 | 53 | .. literalinclude:: fastAPI/main.py 54 | :lineno-start: 22 55 | :lines: 22- 56 | 57 | ``createApp`` 58 | calls up our panel app in this example, but this is not covered until the 59 | next section. 60 | ``address``, ``port`` 61 | Address and port at which the server listens for requests; in our case 62 | ``http://127.0.0.1:5000``. 63 | 64 | If necessary, the environment variable ``BOKEH_ALLOW_WS_ORIGIN`` must be 65 | set with : 66 | 67 | .. code-block:: console 68 | 69 | $ export BOKEH_ALLOW_WS_ORIGIN=127.0.0.1:5000 70 | 71 | ``show=False`` 72 | ensures that the Bokeh server is started but is not immediately displayed 73 | in the browser. 74 | ``allow_websocket_origin`` 75 | lists the hosts that can connect to the websocket. In our example, this 76 | should be ``fastApi``, so we use ``127.0.0.1:8000``. 77 | 78 | #. Now we define the ``sliders`` app based on a standard template for FastAPI 79 | apps, which shows how Panel and FastAPI can be integrated: 80 | 81 | :download:`fastAPI/sliders/sinewave.py` 82 | a parameterised object that represents your existing code: 83 | 84 | .. literalinclude:: fastAPI/sliders/sinewave.py 85 | :caption: fastAPI/sliders/sinewave.py 86 | :name: fastAPI/sliders/sinewave.py 87 | :linenos: 88 | :lines: 1- 89 | 90 | :download:`fastAPI/sliders/pn_app.py` 91 | creates an app function from the ``SineWave`` class: 92 | 93 | .. literalinclude:: fastAPI/sliders/pn_app.py 94 | :caption: fastAPI/sliders/pn_app.py 95 | :name: fastAPI/sliders/pn_app.py 96 | :linenos: 97 | :lines: 1- 98 | 99 | #. Finally, we return to our :download:`fastAPI/main.py` and import the 100 | ``createApp`` function: 101 | 102 | .. literalinclude:: fastAPI/main.py 103 | :caption: fastAPI/main.py 104 | :name: fastAPI/main.py 105 | :lineno-start: 4 106 | :lines: 4 107 | 108 | The file structure should now look like this: 109 | 110 | .. code-block:: console 111 | 112 | fastAPI 113 | ├── main.py 114 | ├── sliders 115 | │   ├── pn_app.py 116 | │   └── sinewave.py 117 | └── templates 118 | └── base.html 119 | 120 | You can now start the server with: 121 | 122 | .. code-block:: console 123 | 124 | $ uv run uvicorn main:app --reload 125 | INFO: Will watch for changes in these directories: ['/srv/jupyter/jupyter-tutorial/docs/web/dashboards/panel/fastAPI'] 126 | INFO: Uvicorn running on http://127.0.0.1:8000 (Press CTRL+C to quit) 127 | INFO: Started reloader process [218214] using StatReload 128 | Launching server at http://127.0.0.1:5000 129 | INFO: Started server process [218216] 130 | INFO: Waiting for application startup. 131 | INFO: Application startup complete. 132 | 133 | You should then see the following in your web browser under the URL 134 | ``http://127.0.0.1:8000``: 135 | 136 | .. figure:: panel-fastapi.png 137 | :alt: Widgets and sine curve in bokeh plot 138 | -------------------------------------------------------------------------------- /docs/dashboards/panel/index.rst: -------------------------------------------------------------------------------- 1 | Panel 2 | ===== 3 | 4 | `Panel `_ as developed on the basis of `Bokeh 5 | `_ and `Param 6 | `_ and offers a Python toolkit specifically 7 | for creating apps and dashboards, but it does not only support Bokeh plots. It 8 | allows multi-level and multi-page workflows. 9 | 10 | .. seealso:: 11 | * `Panel Announcement `_ 12 | * `Panel: A high-level app and dashboarding solution for the PyData ecosystem 13 | `_. 14 | 15 | .. toctree:: 16 | :hidden: 17 | 18 | install 19 | overview.ipynb 20 | interact.ipynb 21 | widgets.ipynb 22 | params.ipynb 23 | style.ipynb 24 | deploy.ipynb 25 | pipelines.ipynb 26 | templates.ipynb 27 | pyodide 28 | fastapi 29 | -------------------------------------------------------------------------------- /docs/dashboards/panel/install.rst: -------------------------------------------------------------------------------- 1 | Installation 2 | ============ 3 | 4 | You can install Panel in the virtual environment of your Jupyter kernel with: 5 | 6 | .. code-block:: console 7 | 8 | $ uv add panel 9 | 10 | .. tip:: 11 | `watchfiles 12 | `_ supports the autoreload functions of 13 | Panel if the ``--dev`` mode is activated: 14 | 15 | .. code-block:: console 16 | 17 | $ uv add --dev watchfiles 18 | 19 | .. tip:: 20 | For syntax highlighting, `pygments `_ should also be 21 | installed: 22 | 23 | .. code-block:: console 24 | 25 | $ uv add pygments 26 | 27 | .. seealso:: 28 | If you want to use Panel for example in VSCode or Google Colab, have a look 29 | at `Develop in other notebook environments 30 | `_. 31 | -------------------------------------------------------------------------------- /docs/dashboards/panel/panel-app.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veit/jupyter-tutorial/24fcd76f43fc46ffd02e3bd98479a41ad6565197/docs/dashboards/panel/panel-app.png -------------------------------------------------------------------------------- /docs/dashboards/panel/panel-fastapi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veit/jupyter-tutorial/24fcd76f43fc46ffd02e3bd98479a41ad6565197/docs/dashboards/panel/panel-fastapi.png -------------------------------------------------------------------------------- /docs/dashboards/panel/pyodide-example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veit/jupyter-tutorial/24fcd76f43fc46ffd02e3bd98479a41ad6565197/docs/dashboards/panel/pyodide-example.png -------------------------------------------------------------------------------- /docs/dashboards/panel/pyodide.rst: -------------------------------------------------------------------------------- 1 | Running Panel in the browser with WASM 2 | ====================================== 3 | 4 | Panel lets you write dashboards and other applications in Python that are 5 | accessed through a web browser. Normally, the Python interpreter runs as a 6 | separate Jupyter or Bokeh server process and communicates with the JavaScript 7 | code running in the client browser. However, Python can also be run directly in 8 | the browser using :abbr:`WASM (WebAssembly)`, without the need for a separate 9 | server. 10 | 11 | Panel uses `Pyodide `_ for this and `PyScript 12 | `_ for rendering. 13 | 14 | Converting panel applications 15 | ----------------------------- 16 | 17 | Future versions of Panel can convert your Panel application from one or more 18 | Python scripts or Notebook files, including :doc:`templates`, into an HTML file 19 | using ``panel convert``. The only requirements are: 20 | 21 | * they only import global modules and packages and no relative imports from 22 | other scripts or modules 23 | * the libraries have been `compiled for Pyodide 24 | `_ or are available as 25 | `Python wheels ` on the :term:`Python Package Index` (:term:`PyPI`). 26 | 27 | Example 28 | ------- 29 | 30 | In the following example we will convert the :doc:`deploy` notebook into a 31 | standalone HTML page with 32 | 33 | .. code-block:: console 34 | 35 | $ uv run panel convert .ipynb --out pyodide 36 | Column 37 | [0] Column 38 | [0] FloatSlider(end=3.0, name='frequency', start=-1.0, value=1.0) 39 | [1] FloatSlider(end=3.0, name='amplitude', start=-1.0, value=1.0) 40 | [2] IntSlider(end=100, name='n', start=5, value=200) 41 | [1] Row 42 | [0] Matplotlib(Figure, name='interactive00114') 43 | Launching server at http://localhost:40405 44 | 45 | Now you can open ``http://localhost:40405`` in your browser and try out the app: 46 | 47 | 48 | .. figure:: pyodide-example.png 49 | :alt: Pyodide example 50 | 51 | You can now add the :download:`pyodide/deploy.html` file to your GitHub pages or 52 | similar – no separate server is required. 53 | 54 | .. seealso:: 55 | * `Awesome Panel/Webassembly Apps 56 | `_ 57 | 58 | Options 59 | ------- 60 | 61 | In the following I explain some of the options of ``panel convert``. 62 | 63 | ``--to`` 64 | The format to convert to. There are three options, each with different 65 | advantages and disadvantages: 66 | 67 | ``pyodide`` (default) 68 | The application is run with pyodide in the main thread. This option is 69 | less performant than ``pyodide-worker``, but produces a fully 70 | self-contained HTML file that does not need to be hosted on a static 71 | file server, such as Github Pages. 72 | ``pyodide-worker`` 73 | generates HTML and JS files, but includes a web worker that runs in a 74 | separate thread. This is the most powerful option, but the files must be 75 | hosted on a static file server. 76 | ``pyscript`` 77 | creates an HTML file that uses `PyScript `_. This 78 | creates standalone HTML files with ```` and ```` tags 79 | containing the dependencies and application code. This output is the 80 | most readable and should have the same performance as the ``pyodide`` 81 | option. 82 | ``-out`` 83 | The directory to write the files to. 84 | ``--pwa`` 85 | adds files that make the application a Progressive Web App. 86 | 87 | `Progressive Web Apps (PWAs) 88 | `_ provide a way for your 89 | web apps to behave almost like a native app, both on mobile devices and on 90 | the desktop. ``panel convert`` has a ``--pwa`` option that generates the 91 | files necessary to turn your panel and pyodide app into a PWA. 92 | 93 | ``--skip-embed`` 94 | skips embedding pre-rendered content in the converted file. 95 | 96 | Panel embeds pre-rendered content in the HTML page and replaces it with live 97 | components once the page is loaded. However, this can take a long time. If 98 | you want to disable this behaviour and render a blank page first, use the 99 | ``--skip-embed`` option. 100 | 101 | ``--index`` 102 | creates an index when you convert several applications at once, so you can 103 | easily navigate between them. 104 | ``--requirements`` 105 | Explicit requirements to add to the converted file or to a 106 | ``requirements.txt`` file. 107 | 108 | By default, requirements are derived from code. 109 | 110 | If a library uses an optional import that cannot be derived from your 111 | application’s list of imports, you must specify an explicit list of 112 | dependencies. 113 | 114 | .. note:: 115 | panel and its dependencies, including NumPy and Bokeh, are loaded 116 | automatically, which means that the explicit requirements for the above 117 | application would be as follows: 118 | 119 | .. code-block:: console 120 | 121 | $ uv run panel convert deploy.ipynb --out pyodide --requirements pandas matplotlib 122 | 123 | Alternatively, you can provide a ``requirements.txt`` file: 124 | 125 | .. code-block:: console 126 | 127 | $ uv run panel convert deploy.ipynb --out pyodide --requirements requirements.txt 128 | 129 | ``--watch`` 130 | Observe the source files. 131 | 132 | You can get a complete overview with ``uv run panel convert -u``. 133 | 134 | .. tip:: 135 | 136 | If the converted application does not work as expected, you can usually find 137 | the errors in the browser console, see `Finding Your Browser's Developer 138 | Console `_. 139 | 140 | .. seealso:: 141 | Answers to the most frequently asked questions about Python in the browser 142 | can be found in the 143 | 144 | * `Pyodide FAQ `_ 145 | * `PyScript FAQ `_ 146 | -------------------------------------------------------------------------------- /docs/dashboards/panel/sample_template.html: -------------------------------------------------------------------------------- 1 | {% extends base %} 2 | 3 | 4 | {% block postamble %} 5 | 6 | {% endblock %} 7 | 8 | 9 | {% block contents %} 10 |

Custom template for multiple panels

11 |

This is a Panel app with a custom template allowing us to compose multiple Panel objects into a single HTML document.

12 |
13 |
14 |
15 |
16 | {{ embed(roots.A) }} 17 |
18 |
19 | {{ embed(roots.B) }} 20 |
21 |
22 |
23 | {% endblock %} 24 | -------------------------------------------------------------------------------- /docs/dashboards/voila-panel.rst: -------------------------------------------------------------------------------- 1 | Voilà vs. Panel 2 | =============== 3 | 4 | A major difference between Panel and Voilà lies in the processing of the 5 | notebooks: Voilà is based directly on the notebook format and transfers the 6 | entire output to the Voilà dashboard, whereas in Panel the output of a notebook 7 | cell must be explicitly declared as a panel object. Voilà therefore has the 8 | advantage that existing notebooks can be used unchanged. However, if not all 9 | notebook cells are to be transferred to the dashboard, two similar notebooks 10 | must be maintained – one for `Literate Programming 11 | `_ and one for the 12 | dashboard. However, if `the inverted pyramid 13 | `_ is to be used for 14 | storytelling in a dashboard, two notebooks are required in both cases. 15 | 16 | Scalability 17 | ----------- 18 | 19 | Voilà and Panel are based on `Tornado `_, 20 | but they differ in that Voilà starts a new Jupyter kernel for each person, while 21 | the Bokeh server serves multiple people with the same Python process. This 22 | difference has two main effects: 23 | 24 | * The overhead per person for a dashboard is much lower with the Bokeh server 25 | than with Voilà: After the libraries are imported, there is only a tiny 26 | overhead for creating each new session. For a session that imports pandas and 27 | Matplotlib, the overhead per user is about 75 MB, and the number of people a 28 | Voilà server can handle for a given application is reduced. Also, startup and 29 | data access times are usually slower. 30 | * Since a Bokeh server shares a single process for multiple sessions, data or 31 | processing can also be shared between the different sessions if necessary. 32 | 33 | Multi-page app 34 | -------------- 35 | 36 | Voilà is not designed for multi-page applications while Panel offers several 37 | options for creating multi-page apps, including :doc:`panel/pipelines` and 38 | overview pages with a collection of independent dashboards and apps. 39 | 40 | Authorisation and authentication 41 | -------------------------------- 42 | 43 | Typically, the dashboard should not be used for authentication and 44 | authorisation, but delegated to a service. `ContainDS Dashboards 45 | `_ is an example of a 46 | :doc:`/hub/index` extension that does this dashboard-independently. 47 | Authentication and authorisation can also be performed via a web server using 48 | one of the following tools: 49 | 50 | * `Traefik Forward Auth `_ 51 | * `OAuth2 Proxy `_ 52 | * `PyCasbin `_ 53 | 54 | If you still want to let the dashboarding library do the authentication, there 55 | are a few options of varying maturity: 56 | 57 | * Panel is based on Bokeh, which provides authentication, and Panel ships with a 58 | number of OAuth providers, such as GitHub, GitLab, and Azure. 59 | 60 | .. seealso:: 61 | `Configuring Authentication 62 | `_ 63 | 64 | * Voilà can reuse authentication from :doc:`/hub/index`. 65 | 66 | BI tool 67 | ------- 68 | 69 | Both Voilà and Panel require programming knowledge in order to create 70 | dashboards. However, `Lumen `_, which is based 71 | on Panel, offers a promising BI-like user interface. 72 | -------------------------------------------------------------------------------- /docs/dashboards/voila/index.rst: -------------------------------------------------------------------------------- 1 | Voilà 2 | ===== 3 | 4 | `Voilà `_ was developed by 5 | `QuantStack `_. 6 | 7 | Features 8 | -------- 9 | 10 | * Voilà supports :doc:`interactive Jupyter widgets `, 11 | including round trips to the kernel. 12 | Custom widgets like :doc:`pyviz:d3js/bqplot/index`, 13 | :doc:`pyviz:js/ipyleaflet`, 14 | :doc:`pyviz:js/ipyvolume`, 15 | `ipympl `_, 16 | :doc:`ipysheet `, 17 | `plotly `_, 18 | `ipywebrtc `_ etc. are also 19 | supported. 20 | * Voilà does not allow arbitrary code execution by dashboard users. 21 | * Voilà is based on Jupyter standard protocols and file formats and works with 22 | any :doc:`Jupyter-Kernel `: C++, Python, Julia. This makes it 23 | a language-independent dashboard system. 24 | * Voilà is expandable. It contains a flexible :doc:`Template ` 25 | system for creating extensive layouts. 26 | 27 | Execution model 28 | --------------- 29 | 30 | .. graphviz:: 31 | 32 | digraph execution_model { 33 | graph [fontname = "Calibri", fontsize="12"]; 34 | node [fontname = "Calibri", fontsize="12"]; 35 | edge [fontname = "Consolas", fontsize="12"]; 36 | tooltip="Ausführungsmodell"; 37 | subgraph cluster_browser { 38 | notebook_url [ 39 | shape=box, 40 | label="Notebook url"; 41 | style=filled; 42 | fillcolor="#edf8fb"; 43 | color="#b2e2e2"]; 44 | html_rendering [ 45 | shape=box, 46 | label="HTML rendering"; 47 | style=filled; 48 | fillcolor="#edf8fb"; 49 | color="#b2e2e2"]; 50 | init_websocket [ 51 | shape=box, 52 | label="Javascript initiates Websocket\nwith the Jupyter kernel", 53 | style=filled; 54 | fillcolor="#edf8fb"; 55 | color="#b2e2e2"]; 56 | label="Browser"; 57 | labeljust="l"; 58 | } 59 | subgraph cluster_server { 60 | html_open_notebook [ 61 | shape=box, 62 | label="Open the notebook and start\nthe associated kernel"; 63 | style=filled; 64 | fillcolor="#bdd7e7"; 65 | color="#2171b5"]; 66 | html_run_notebook [ 67 | shape=box, 68 | label="Run notebook cells\nOutput results\nAssign Voilà template\nConvert notebook"; 69 | style=filled; 70 | fillcolor="#bdd7e7"; 71 | color="#2171b5"]; 72 | ws_open_notebook [ 73 | shape=box, 74 | label="Open the notebook and\nstart the associated kernel"; 75 | style=filled; 76 | fillcolor="#bdd7e7"; 77 | color="#2171b5"]; 78 | label="Server"; 79 | labeljust="l"; 80 | } 81 | // Edges 82 | notebook_url -> html_open_notebook [label="GET Request"] 83 | html_open_notebook -> html_run_notebook 84 | html_run_notebook -> html_rendering [label="HTTP Responses\lwith HTML or\lHTML fragments\l"] 85 | html_rendering -> init_websocket 86 | init_websocket -> ws_open_notebook [dir=both label="Web-Socket:\l- comm_msg\l- comm_info_request\l- kernel_info_request\l- shutdown_request\l"] 87 | // Arrangement 88 | {rank = same; notebook_url; html_rendering; init_websocket;} 89 | {rank = same; html_open_notebook; html_run_notebook; ws_open_notebook;} 90 | } 91 | 92 | An important aspect of this execution model is that the frontend cannot specify 93 | which code is executed by the backend. Unless otherwise specified with the option 94 | ``--strip-sources=False``, the source code of the rendered notebook does not 95 | even reach the frontend. The Voilà instance of ``jupyter_server`` does not allow 96 | execution requests by default. 97 | 98 | .. warning:: 99 | The current version of Voilà does not respond to the first ``GET`` request 100 | until all cells have been executed. This can take longer. However, work is 101 | being done to enable progressive rendering, see `feat: progressive cell 102 | rendering 103 | `_. 104 | 105 | .. seealso:: 106 | 107 | * `Voilà Gallery `_ 108 | * `And voilà! `_ 109 | 110 | .. toctree:: 111 | :hidden: 112 | :titlesonly: 113 | :maxdepth: 0 114 | 115 | install 116 | templating 117 | bqplot_vuetify_example.ipynb 118 | debug.ipynb 119 | -------------------------------------------------------------------------------- /docs/dashboards/voila/install.rst: -------------------------------------------------------------------------------- 1 | Installation and use 2 | ==================== 3 | 4 | Installation 5 | ------------ 6 | 7 | voilà can be installed with: 8 | 9 | .. code-block:: console 10 | 11 | $ uv add voila 12 | 13 | Start 14 | ----- 15 | 16 | … as a stand-alone application 17 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 18 | 19 | You can check the installation, e.g. with: 20 | 21 | .. code-block:: console 22 | 23 | $ uv run voila docs/dashboards/voila/bqplot_vuetify_example.ipynb 24 | ... 25 | [Voila] Voilà is running at: 26 | http://localhost:8866/ 27 | 28 | Your standard browser should open and display the ``voila`` examples from 29 | our tutorial: 30 | 31 | .. image:: voila-example-1.png 32 | :alt: Voilà example 33 | 34 | Alternatively, you can also display a directory with all the notebooks it 35 | contains: 36 | 37 | .. code-block:: console 38 | 39 | $ uv run voila docs/dashboards/voila 40 | 41 | .. image:: voila-example-2.png 42 | :alt: Voilà example of a directory view 43 | 44 | It is also possible to display the source code with: 45 | 46 | .. code-block:: console 47 | 48 | $ uv run voila --strip_sources=False docs/dashboards/voila/bqplot_vuetify_example.ipynb 49 | 50 | .. note:: 51 | Note that the code is only displayed. Voilà does not allow users to edit or 52 | run the code. 53 | 54 | .. image:: voila-example-3.png 55 | :alt: Voilà example with source code 56 | 57 | Usually the ``light`` theme is used; however, you can also choose the ``dark`` 58 | theme: 59 | 60 | .. code-block:: console 61 | 62 | $ uv run voila --theme=dark docs/dashboards/voila/bqplot_vuetify_example.ipynb 63 | 64 | … as an extension of the Jupyter server 65 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 66 | 67 | Alternatively you can start voilà as an extension of the Jupyter server: 68 | 69 | .. code-block:: console 70 | 71 | $ uv run jupyter notebook 72 | 73 | Then you can call up voilà, e.g. under the URL 74 | ``http://localhost:8888/voila``. 75 | -------------------------------------------------------------------------------- /docs/dashboards/voila/voila-debug.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veit/jupyter-tutorial/24fcd76f43fc46ffd02e3bd98479a41ad6565197/docs/dashboards/voila/voila-debug.png -------------------------------------------------------------------------------- /docs/dashboards/voila/voila-example-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veit/jupyter-tutorial/24fcd76f43fc46ffd02e3bd98479a41ad6565197/docs/dashboards/voila/voila-example-1.png -------------------------------------------------------------------------------- /docs/dashboards/voila/voila-example-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veit/jupyter-tutorial/24fcd76f43fc46ffd02e3bd98479a41ad6565197/docs/dashboards/voila/voila-example-2.png -------------------------------------------------------------------------------- /docs/dashboards/voila/voila-example-3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veit/jupyter-tutorial/24fcd76f43fc46ffd02e3bd98479a41ad6565197/docs/dashboards/voila/voila-example-3.png -------------------------------------------------------------------------------- /docs/dashboards/voila/voila-gridstack.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veit/jupyter-tutorial/24fcd76f43fc46ffd02e3bd98479a41ad6565197/docs/dashboards/voila/voila-gridstack.png -------------------------------------------------------------------------------- /docs/dashboards/voila/voila-vuetify-iphone.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veit/jupyter-tutorial/24fcd76f43fc46ffd02e3bd98479a41ad6565197/docs/dashboards/voila/voila-vuetify-iphone.png -------------------------------------------------------------------------------- /docs/dashboards/voila/voila-vuetify-laptop.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veit/jupyter-tutorial/24fcd76f43fc46ffd02e3bd98479a41ad6565197/docs/dashboards/voila/voila-vuetify-laptop.png -------------------------------------------------------------------------------- /docs/genindex.rst: -------------------------------------------------------------------------------- 1 | .. Workaround for displaying the index in the toc 2 | 3 | Index 4 | ===== 5 | -------------------------------------------------------------------------------- /docs/hub/Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | url = "https://pypi.org/simple" 3 | verify_ssl = true 4 | name = "pypi" 5 | 6 | [packages] 7 | jupyter = "*" 8 | jupyterlab = "*" 9 | jupyterhub = "*" 10 | jupyterhub-systemdspawner = "*" 11 | ipython = {extras = ["all"], version = "*"} 12 | nbconvert = "*" 13 | nbviewer = "*" 14 | nb2xls = "*" 15 | jupyter-contrib-nbextensions = "*" 16 | jupyter-latex-envs = "*" 17 | yapf = "*" 18 | autopep8 = "*" 19 | jupyter-nbextensions-configurator = "*" 20 | 21 | [dev-packages] 22 | 23 | [requires] 24 | python_version = "3.11" 25 | python_full_version = "3.11.4" 26 | -------------------------------------------------------------------------------- /docs/hub/config.rst: -------------------------------------------------------------------------------- 1 | Configuration 2 | ============= 3 | 4 | JupyterHub configuration 5 | ------------------------ 6 | 7 | Create configuration file: 8 | 9 | .. code-block:: console 10 | 11 | $ uv run jupyterhub --generate-config 12 | Writing default config to: jupyterhub_config.py 13 | 14 | .. seealso:: 15 | 16 | * :doc:`JupyterHub Configuration Basics 17 | ` 18 | * :doc:`JupyterHub Networking basics 19 | ` 20 | 21 | System Service for JupyterHub 22 | ----------------------------- 23 | 24 | #. Determine the Python virtual environment: 25 | 26 | .. code-block:: console 27 | 28 | $ cd jupyterhub_env 29 | $ pwd 30 | /srv/jupyter/jupyterhub_env 31 | 32 | #. Configure the absolute path to :file:`jupyterhub-singleuser` in the 33 | :file:`jupyterhub_config.py` file: 34 | 35 | .. code-block:: python 36 | 37 | c.Spawner.cmd = ["/srv/jupyter/jupyterhub_env/.venv/bin/jupyterhub-singleuser"] 38 | 39 | #. Add a new systemd unit file :file:`/etc/systemd/system/jupyterhub.service` 40 | with the command: 41 | 42 | .. code-block:: console 43 | 44 | $ sudo systemctl edit --force --full jupyterhub.service 45 | 46 | Add your corresponding Python environment. 47 | 48 | .. code-block:: ini 49 | 50 | [Unit] 51 | Description=Jupyterhub 52 | 53 | [Service] 54 | User=root 55 | Environment="PATH=/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/srv/jupyter/.local/share/virtualenvs/jupyterhub-aFv4x91W/bin" 56 | ExecStart=/srv/jupyter//srv/jupyter/jupyterhub_env/.venv/bin/jupyterhub -f /srv/jupyter/jupyterhub_env/jupyterhub_config.py 57 | 58 | [Install] 59 | WantedBy=multi-user.target 60 | 61 | #. Loading the configuration with: 62 | 63 | .. code-block:: console 64 | 65 | $ sudo systemctl daemon-reload 66 | 67 | #. The JupyterHub can be managed with: 68 | 69 | .. code-block:: console 70 | 71 | $ sudo systemctl jupyterhub 72 | 73 | #. To ensure that the service is also loaded during a system start, the 74 | following is called: 75 | 76 | .. code-block:: console 77 | 78 | $ sudo systemctl enable jupyterhub.service 79 | Created symlink /etc/systemd/system/multi-user.target.wants/jupyterhub.service → /etc/systemd/system/jupyterhub.service. 80 | 81 | #. To be able to use the ``jupyterhub-singleuser`` and start your own server, 82 | the ix users must be entered in the ``jupyter`` group, for example with 83 | :samp:`usermod -aG jupyter {VEIT}`. 84 | 85 | 86 | Since JupyterHub includes authentication and allows the execution of any code, 87 | it should not be executed without SSL (HTTPS). To do this, an official, 88 | trustworthy SSL certificate must be created. After you have received and 89 | installed a key and a certificate, you don’t configure the JupyterHub itself, 90 | but the upstream Apache web server. 91 | 92 | #. For this purpose, the additional modules are first activated with 93 | 94 | .. code-block:: apacheconf 95 | 96 | # a2enmod ssl rewrite proxy proxy_http proxy_wstunnel 97 | 98 | #. Then the VirtualHost can be configured in 99 | ``/etc/apache2/sites-available/jupyter.cusy.io.conf`` 100 | 101 | .. code-block:: console 102 | 103 | # redirect HTTP to HTTPS 104 | 105 | ServerName jupyter.cusy.io 106 | ServerAdmin webmaster@cusy.io 107 | 108 | ErrorLog ${APACHE_LOG_DIR}/jupyter.cusy.io_error.log 109 | CustomLog ${APACHE_LOG_DIR}/jupyter.cusy.io_access.log combined 110 | 111 | Redirect / https://jupyter.cusy.io/ 112 | 113 | 114 | 115 | ServerName jupyter.cusy.io 116 | ServerAdmin webmaster@cusy.io 117 | 118 | # configure SSL 119 | SSLEngine On 120 | SSLCertificateFile /etc/ssl/certs/jupyter.cusy.io_cert.pem 121 | SSLCertificateKeyFile /etc/ssl/private/jupyter.cusy.io_sec_key.pem 122 | # for an up-to-date SSL configuration see e.g. 123 | # https://ssl-config.mozilla.org/ 124 | 125 | # Use RewriteEngine to handle websocket connection upgrades 126 | RewriteEngine On 127 | RewriteCond %{HTTP:Connection} Upgrade [NC] 128 | RewriteCond %{HTTP:Upgrade} websocket [NC] 129 | RewriteRule /(.*) ws://127.0.0.1:8000/$1 [P,L] 130 | 131 | 132 | # preserve Host header to avoid cross-origin problems 133 | ProxyPreserveHost on 134 | # proxy to JupyterHub 135 | ProxyPass http://127.0.0.1:8000/ 136 | ProxyPassReverse http://127.0.0.1:8000/ 137 | 138 | 139 | ErrorLog ${APACHE_LOG_DIR}/jupyter.cusy.io_error.log 140 | CustomLog ${APACHE_LOG_DIR}/jupyter.cusy.io_access.log combined 141 | 142 | 143 | #. This VirtualHost is activated with 144 | 145 | .. code-block:: console 146 | 147 | # a2ensite jupyter.cusy.io.conf 148 | 149 | #. Finally, the status of the Apache web server is checked with 150 | 151 | .. code-block:: console 152 | 153 | # systemctl status apache2 154 | ● apache2.service - The Apache HTTP Server 155 | Loaded: loaded (/lib/systemd/system/apache2.service; enabled; vendor preset: enabled) 156 | Active: active (running) (Result: exit-code) since Mon 2019-03-25 16:50:26 CET; 1 day 22h ago 157 | Process: 31773 ExecReload=/usr/sbin/apachectl graceful (code=exited, status=0/SUCCESS) 158 | Main PID: 20273 (apache2) 159 | Tasks: 55 (limit: 4915) 160 | CGroup: /system.slice/apache2.service 161 | ├─20273 /usr/sbin/apache2 -k start 162 | ├─31779 /usr/sbin/apache2 -k start 163 | └─31780 /usr/sbin/apache2 -k start 164 | 165 | Mar 27 06:25:01 jupyter.cusy.io systemd[1]: Reloaded The Apache HTTP Server. 166 | 167 | Cookie Secret 168 | ------------- 169 | 170 | The cookie secret is used to encrypt the browser cookies that are used for 171 | authentication. 172 | 173 | #. The cookie secret can e.g. be created with 174 | 175 | .. code-block:: console 176 | 177 | $ openssl rand -hex 32 > /srv/jupyterhub/venv/jupyterhub_cookie_secret 178 | 179 | #. The file should not be readable by either ``group`` or ``anonymous``: 180 | 181 | .. code-block:: console 182 | 183 | $ chmod 600 /srv/jupyterhub/venv/jupyterhub_cookie_secret 184 | 185 | #. Finally it will be entered in the :file:`jupyterhub_config.py` file: 186 | 187 | .. code-block:: python 188 | 189 | c.JupyterHub.cookie_secret_file = "jupyterhub_cookie_secret" 190 | 191 | Proxy authentication token 192 | -------------------------- 193 | 194 | The hub authenticates its requests to the proxy using a secret token that the 195 | hub and proxy agree on. Usually, the proxy authentication token does not need to 196 | be set, as the hub itself generates a random key. This means that the proxy has 197 | to be restarted every time unless the proxy is a subprocess of the hub. 198 | 199 | #. Alternatively, the value can e.g. can be generated with 200 | 201 | .. code-block:: console 202 | 203 | $ openssl rand -hex 32 204 | 205 | #. It can then be entered in the configuration file, for example with 206 | 207 | .. code-block:: python 208 | 209 | c.JupyterHub.proxy_auth_token = ( 210 | "18a0335b7c2e7edeaf7466894a32bea8d1c3cff4b07860298dbe353ecb179fc6" 211 | ) 212 | -------------------------------------------------------------------------------- /docs/hub/index.rst: -------------------------------------------------------------------------------- 1 | JupyterHub 2 | ========== 3 | 4 | `JupyterHub `_ is a multi-user 5 | server for Jupyter Notebooks, which can create and manage many different 6 | instances of Jupyter Notebooks and which acts as a proxy. 7 | 8 | .. toctree:: 9 | :hidden: 10 | :titlesonly: 11 | :maxdepth: 0 12 | 13 | install 14 | config 15 | systemdspawner 16 | nbviewer-service 17 | ipyparallel/index 18 | -------------------------------------------------------------------------------- /docs/hub/install.rst: -------------------------------------------------------------------------------- 1 | Installation 2 | ============ 3 | 4 | #. Install Python≥3.6 and :term:`pip`: 5 | 6 | .. code-block:: console 7 | 8 | $ sudo apt update 9 | $ sudo apt install python3 10 | $ python3 -V 11 | Python 3.10.6 12 | $ sudo apt install python3-pip 13 | 14 | #. Create service user ``jupyter``: 15 | 16 | .. code-block:: console 17 | 18 | $ sudo useradd -s /bin/bash -rmd /srv/jupyter jupyter 19 | 20 | #. Switch to the service user ``jupyter``: 21 | 22 | .. code-block:: console 23 | 24 | $ sudo -u jupyter -i 25 | 26 | #. Install :term:`uv`: 27 | 28 | .. code-block:: console 29 | 30 | $ curl -LsSf https://astral.sh/uv/install.sh | sh 31 | 32 | #. Activate automatic shell completion: 33 | 34 | .. code-block:: console 35 | 36 | $ echo 'eval "$(uv generate-shell-completion bash)"' >> ~/.bashrc 37 | 38 | #. Create a virtual environment and install JupyterHub: 39 | 40 | .. code-block:: console 41 | 42 | $ uv init --package jupyterhub_env 43 | $ cd jupyterhub_env 44 | $ uv add jupyterhub 45 | 46 | #. Install ``nodejs`` and ``npm``: 47 | 48 | .. code-block:: console 49 | 50 | $ sudo apt install nodejs npm 51 | $ node -v 52 | v23.3.0 53 | $ npm -v 54 | 10.9.0 55 | 56 | #. Install the HTTP proxy: 57 | 58 | .. code-block:: console 59 | 60 | $ sudo npm install -g configurable-http-proxy 61 | 62 | #. If JupyterLab and Notebook are to run in the same environment, they must also 63 | be installed here: 64 | 65 | .. code-block:: console 66 | 67 | $ uv add jupyterlab notebook 68 | 69 | #. Testing the installation: 70 | 71 | .. code-block:: console 72 | 73 | $ uv run jupyterhub -h 74 | $ configurable-http-proxy -h 75 | 76 | #. Starting the JupyterHub: 77 | 78 | .. code-block:: console 79 | 80 | $ uv run jupyterhub 81 | ... 82 | [I 2025-01-10 18:07:29.993 JupyterHub app:3770] JupyterHub is now running at http://:8000 83 | 84 | You can end the process again with :kbd:`ctrl-c`. 85 | -------------------------------------------------------------------------------- /docs/hub/ipyparallel/check.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Check the installation" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": {}, 14 | "outputs": [ 15 | { 16 | "data": { 17 | "text/plain": [ 18 | "[0, 1, 2, 3]" 19 | ] 20 | }, 21 | "execution_count": 1, 22 | "metadata": {}, 23 | "output_type": "execute_result" 24 | } 25 | ], 26 | "source": [ 27 | "import ipyparallel as ipp\n", 28 | "\n", 29 | "\n", 30 | "c = ipp.Client()\n", 31 | "c.ids" 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": 2, 37 | "metadata": {}, 38 | "outputs": [ 39 | { 40 | "data": { 41 | "text/plain": [ 42 | "['Hello, World', 'Hello, World', 'Hello, World', 'Hello, World']" 43 | ] 44 | }, 45 | "execution_count": 2, 46 | "metadata": {}, 47 | "output_type": "execute_result" 48 | } 49 | ], 50 | "source": [ 51 | "c[:].apply_sync(lambda : \"Hello, World\")" 52 | ] 53 | } 54 | ], 55 | "metadata": { 56 | "kernelspec": { 57 | "display_name": "Python 3.11 Kernel", 58 | "language": "python", 59 | "name": "python311" 60 | }, 61 | "language_info": { 62 | "codemirror_mode": { 63 | "name": "ipython", 64 | "version": 3 65 | }, 66 | "file_extension": ".py", 67 | "mimetype": "text/x-python", 68 | "name": "python", 69 | "nbconvert_exporter": "python", 70 | "pygments_lexer": "ipython3", 71 | "version": "3.11.4" 72 | }, 73 | "latex_envs": { 74 | "LaTeX_envs_menu_present": true, 75 | "autoclose": false, 76 | "autocomplete": true, 77 | "bibliofile": "biblio.bib", 78 | "cite_by": "apalike", 79 | "current_citInitial": 1, 80 | "eqLabelWithNumbers": true, 81 | "eqNumInitial": 1, 82 | "hotkeys": { 83 | "equation": "Ctrl-E", 84 | "itemize": "Ctrl-I" 85 | }, 86 | "labels_anchors": false, 87 | "latex_user_defs": false, 88 | "report_style_numbering": false, 89 | "user_envs_cfg": false 90 | }, 91 | "varInspector": { 92 | "cols": { 93 | "lenName": 16, 94 | "lenType": 16, 95 | "lenVar": 40 96 | }, 97 | "kernels_config": { 98 | "python": { 99 | "delete_cmd_postfix": "", 100 | "delete_cmd_prefix": "del ", 101 | "library": "var_list.py", 102 | "varRefreshCmd": "print(var_dic_list())" 103 | }, 104 | "r": { 105 | "delete_cmd_postfix": ") ", 106 | "delete_cmd_prefix": "rm(", 107 | "library": "var_list.r", 108 | "varRefreshCmd": "cat(var_dic_list()) " 109 | } 110 | }, 111 | "types_to_exclude": [ 112 | "module", 113 | "function", 114 | "builtin_function_or_method", 115 | "instance", 116 | "_Feature" 117 | ], 118 | "window_display": false 119 | }, 120 | "widgets": { 121 | "application/vnd.jupyter.widget-state+json": { 122 | "state": {}, 123 | "version_major": 2, 124 | "version_minor": 0 125 | } 126 | } 127 | }, 128 | "nbformat": 4, 129 | "nbformat_minor": 4 130 | } 131 | -------------------------------------------------------------------------------- /docs/hub/ipyparallel/config.rst: -------------------------------------------------------------------------------- 1 | Configuration 2 | ============= 3 | 4 | For the configuration, a configuration file is created for the client and engine 5 | when the IPython hub is started, usually in 6 | :file:`~/.ipython/profile_default/security/`. 7 | 8 | #. If we don’t want to use the ``default`` profile, we should first create a new 9 | IPython profile with: 10 | 11 | .. code-block:: console 12 | 13 | $ uv run ipython profile create --parallel --profile=local 14 | [ProfileCreate] Generating default config file: '/Users/veit/.ipython/profile_parallel/ipython_config.py' 15 | [ProfileCreate] Generating default config file: '/Users/veit/.ipython/profile_parallel/ipython_kernel_config.py' 16 | [ProfileCreate] Generating default config file: '/Users/veit/.ipython/profile_parallel/ipcontroller_config.py' 17 | [ProfileCreate] Generating default config file: '/Users/veit/.ipython/profile_parallel/ipengine_config.py' 18 | [ProfileCreate] Generating default config file: '/Users/veit/.ipython/profile_parallel/ipcluster_config.py 19 | 20 | ``--parallel`` 21 | includes the configuration files for *Parallel Computing* (``ipengine``, 22 | ``ipcontroller`` :abbr:`etc. (et cetera)`). 23 | 24 | #. When the IPython controller and the engines are started, the files 25 | :file:`ipcontroller-engine.json` and :file:`ipcontroller-client.json` are 26 | generated in :file:`~/.ipython/profile_default/security/`. 27 | 28 | ``ipcluster`` in ``mpiexec``/``mpirun`` mode 29 | -------------------------------------------- 30 | 31 | #. Creating the profile: 32 | 33 | .. code-block:: console 34 | 35 | $ uv run ipython profile create --parallel --profile=mpi 36 | [ProfileCreate] Generating default config file: '/Users/veit/.ipython/profile_mpi/ipython_config.py' 37 | [ProfileCreate] Generating default config file: '/Users/veit/.ipython/profile_mpi/ipython_kernel_config.py' 38 | [ProfileCreate] Generating default config file: '/Users/veit/.ipython/profile_mpi/ipcontroller_config.py' 39 | [ProfileCreate] Generating default config file: '/Users/veit/.ipython/profile_mpi/ipengine_config.py' 40 | [ProfileCreate] Generating default config file: '/Users/veit/.ipython/profile_mpi/ipcluster_config.py' 41 | 42 | #. Editing of :file:`ipcluster_config.py`: 43 | 44 | #. so that the MPI launcher can be used: 45 | 46 | .. code-block:: python 47 | 48 | c.IPClusterEngines.engine_launcher_class = "MPIEngineSetLauncher" 49 | 50 | #. The cluster can then be started with: 51 | 52 | .. code-block:: console 53 | 54 | $ uv run ipcluster start -n 4 --profile=mpi 55 | [IPClusterStart] Starting ipcluster with [daemon=False] 56 | [IPClusterStart] Creating pid file: /Users/veit/.ipython/profile_mpi/pid/ipcluster.pid 57 | [IPClusterStart] Starting Controller with LocalControllerLauncher 58 | [IPClusterStart] Starting 4 Engines with LocalEngineSetLauncher 59 | [IPClusterStart] Engines appear to have started successfully 60 | -------------------------------------------------------------------------------- /docs/hub/ipyparallel/direct.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# IPython’s `Direct` interface" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "## Create a `DirectView`" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 1, 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "import ipyparallel as ipp\n", 24 | "\n", 25 | "\n", 26 | "rc = ipp.Client()" 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "execution_count": 2, 32 | "metadata": {}, 33 | "outputs": [], 34 | "source": [ 35 | "rc = ipp.Client(profile=\"default\")" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": 3, 41 | "metadata": {}, 42 | "outputs": [ 43 | { 44 | "data": { 45 | "text/plain": [ 46 | "[0, 1, 2, 3]" 47 | ] 48 | }, 49 | "execution_count": 3, 50 | "metadata": {}, 51 | "output_type": "execute_result" 52 | } 53 | ], 54 | "source": [ 55 | "rc.ids" 56 | ] 57 | }, 58 | { 59 | "cell_type": "markdown", 60 | "metadata": {}, 61 | "source": [ 62 | "Use all engines:" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": 4, 68 | "metadata": {}, 69 | "outputs": [], 70 | "source": [ 71 | "dview = rc[:]" 72 | ] 73 | }, 74 | { 75 | "cell_type": "markdown", 76 | "metadata": {}, 77 | "source": [ 78 | "## `map()` function\n", 79 | "\n", 80 | "Python’s builtin `map()` function can be applied to a sequence of elements and is usually easy to parallelise.\n", 81 | "\n", 82 | "Please note that the `DirectView` version of `map()` does not do automatic load balancing. You may have to use `LoadBalancedView` for this." 83 | ] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "execution_count": 5, 88 | "metadata": {}, 89 | "outputs": [], 90 | "source": [ 91 | "serial_result = list(map(lambda x: x**10, range(32)))" 92 | ] 93 | }, 94 | { 95 | "cell_type": "code", 96 | "execution_count": 6, 97 | "metadata": {}, 98 | "outputs": [], 99 | "source": [ 100 | "parallel_result = dview.map_sync(lambda x: x**10, range(32))" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": 7, 106 | "metadata": {}, 107 | "outputs": [ 108 | { 109 | "data": { 110 | "text/plain": [ 111 | "True" 112 | ] 113 | }, 114 | "execution_count": 7, 115 | "metadata": {}, 116 | "output_type": "execute_result" 117 | } 118 | ], 119 | "source": [ 120 | "serial_result == parallel_result" 121 | ] 122 | } 123 | ], 124 | "metadata": { 125 | "kernelspec": { 126 | "display_name": "Python 3.11 Kernel", 127 | "language": "python", 128 | "name": "python311" 129 | }, 130 | "language_info": { 131 | "codemirror_mode": { 132 | "name": "ipython", 133 | "version": 3 134 | }, 135 | "file_extension": ".py", 136 | "mimetype": "text/x-python", 137 | "name": "python", 138 | "nbconvert_exporter": "python", 139 | "pygments_lexer": "ipython3", 140 | "version": "3.11.4" 141 | }, 142 | "latex_envs": { 143 | "LaTeX_envs_menu_present": true, 144 | "autoclose": false, 145 | "autocomplete": true, 146 | "bibliofile": "biblio.bib", 147 | "cite_by": "apalike", 148 | "current_citInitial": 1, 149 | "eqLabelWithNumbers": true, 150 | "eqNumInitial": 1, 151 | "hotkeys": { 152 | "equation": "Ctrl-E", 153 | "itemize": "Ctrl-I" 154 | }, 155 | "labels_anchors": false, 156 | "latex_user_defs": false, 157 | "report_style_numbering": false, 158 | "user_envs_cfg": false 159 | }, 160 | "varInspector": { 161 | "cols": { 162 | "lenName": 16, 163 | "lenType": 16, 164 | "lenVar": 40 165 | }, 166 | "kernels_config": { 167 | "python": { 168 | "delete_cmd_postfix": "", 169 | "delete_cmd_prefix": "del ", 170 | "library": "var_list.py", 171 | "varRefreshCmd": "print(var_dic_list())" 172 | }, 173 | "r": { 174 | "delete_cmd_postfix": ") ", 175 | "delete_cmd_prefix": "rm(", 176 | "library": "var_list.r", 177 | "varRefreshCmd": "cat(var_dic_list()) " 178 | } 179 | }, 180 | "types_to_exclude": [ 181 | "module", 182 | "function", 183 | "builtin_function_or_method", 184 | "instance", 185 | "_Feature" 186 | ], 187 | "window_display": false 188 | }, 189 | "widgets": { 190 | "application/vnd.jupyter.widget-state+json": { 191 | "state": {}, 192 | "version_major": 2, 193 | "version_minor": 0 194 | } 195 | } 196 | }, 197 | "nbformat": 4, 198 | "nbformat_minor": 4 199 | } 200 | -------------------------------------------------------------------------------- /docs/hub/ipyparallel/index.rst: -------------------------------------------------------------------------------- 1 | ``ipyparallel`` 2 | =============== 3 | 4 | This section provides an overview of `ipyparallel 5 | `_ which supports different types 6 | of parallelisation, including: 7 | 8 | - Single Program, Multiple Data (SPMD) 9 | - Multiple program, multiple data (MPMD) 10 | - Message Passing Interface (MPI) 11 | 12 | .. toctree:: 13 | :hidden: 14 | :titlesonly: 15 | :maxdepth: 0 16 | 17 | install 18 | intro 19 | check.ipynb 20 | config 21 | direct.ipynb 22 | magics.ipynb 23 | task.ipynb 24 | asyncresult 25 | mpi.ipynb 26 | -------------------------------------------------------------------------------- /docs/hub/ipyparallel/intro.rst: -------------------------------------------------------------------------------- 1 | Overview 2 | ======== 3 | 4 | Architecture 5 | ------------ 6 | 7 | The ``IPython.parallel`` architecture consists of four components: 8 | 9 | .. graphviz:: 10 | 11 | digraph IPython_parallel { 12 | graph [fontname = "Calibri", fontsize="16"]; 13 | node [fontname = "Calibri", fontsize="16"]; 14 | edge [fontname = "Calibri", fontsize="16"]; 15 | // Nodes 16 | hub [ 17 | label="Hub" 18 | target="_top", 19 | href="../parallel/ipyparallel/intro.html#ipython-hub"] 20 | engine [ 21 | label="Engine" 22 | target="_top", 23 | href="../parallel/ipyparallel/intro.html#ipython-engine"] 24 | schedulers [ 25 | label="Schedulers" 26 | target="_top", 27 | href="../parallel/ipyparallel/intro.html#ipython-schedulers"] 28 | client [ 29 | label="Client" 30 | target="_top", 31 | href="../parallel/ipyparallel/intro.html#ipython-client"] 32 | // Edges 33 | engine -> hub 34 | client -> hub 35 | schedulers -> hub 36 | engine -> schedulers 37 | client -> schedulers 38 | } 39 | 40 | IPython-Engine 41 | ~~~~~~~~~~~~~~ 42 | 43 | The IPython engine is an extension of the IPython kernel for Jupyter. The module 44 | waits for requests from the network, executes code and returns the results. 45 | IPython parallel extends the Jupyter messaging protocol with native Python 46 | object serialisation and adds some additional commands. Several engines are 47 | started for parallel and distributed computing. 48 | 49 | IPython-Hub 50 | ~~~~~~~~~~~ 51 | 52 | The main job of the hub is to establish and monitor connections to clients and 53 | engines. 54 | 55 | IPython-Schedulers 56 | ~~~~~~~~~~~~~~~~~~ 57 | 58 | All actions that can be carried out on the engine go through a scheduler. While 59 | the engines themselves block when user code is executed, the schedulers hide 60 | this from the user to provide a fully asynchronous interface for a number of 61 | engines. 62 | 63 | IPython-Client 64 | ~~~~~~~~~~~~~~ 65 | 66 | There is a main object ``Client`` to connect to the cluster. Then there is a 67 | corresponding ``View`` for each execution model. These ``Views`` allow users to 68 | interact with a number of engines. The two standard views are: 69 | 70 | - :py:class:`ipyparallel.DirectView` class for explicit addressing 71 | - :py:class:`ipyparallel.LoadBalancedView` class for target-independent scheduling 72 | 73 | 74 | Installation 75 | ------------ 76 | 77 | .. code-block:: console 78 | 79 | $ uv add ipyparallel 80 | 81 | Start 82 | ----- 83 | 84 | #. Starting the IPython Hub: 85 | 86 | .. code-block:: console 87 | 88 | $ uv run ipcontroller 89 | [IPControllerApp] Hub listening on tcp://127.0.0.1:53847 for registration. 90 | [IPControllerApp] Hub using DB backend: 'DictDB' 91 | [IPControllerApp] hub::created hub 92 | [IPControllerApp] writing connection info to /Users/veit/.ipython/profile_default/security/ipcontroller-client.json 93 | [IPControllerApp] writing connection info to /Users/veit/.ipython/profile_default/security/ipcontroller-engine.json 94 | [IPControllerApp] task::using Python leastload Task scheduler 95 | … 96 | 97 | DB backend 98 | The database in which the IPython tasks are managed. In addition to the 99 | in-memory database ``DictDB``, ``MongoDB`` and ``SQLite`` are further 100 | options. 101 | ``ipcontroller-client.json`` 102 | Configuration file for the IPython client 103 | ``ipcontroller-engine.json`` 104 | Configuration file for the IPython engine 105 | Task-Schedulers 106 | The possible routing scheme. ``leastload`` always assigns tasks to the 107 | engine with the fewest open tasks. Alternatively, ``lru`` (Least Recently 108 | Used), ``plainrandom``, ``twobin`` and ``weighted`` can be selected, the 109 | latter two also need NumPy. 110 | 111 | This can be configured in ``ipcontroller_config.py``, for example with 112 | ``c.TaskScheduler.scheme_name = 'leastload'`` or with 113 | 114 | .. code-block:: console 115 | 116 | $ uv run ipcontroller --scheme=pure 117 | 118 | #. Starting the IPython controller and the engines: 119 | 120 | .. code-block:: console 121 | 122 | $ uv run ipcluster start 123 | [IPClusterStart] Starting ipcluster with [daemon=False] 124 | [IPClusterStart] Creating pid file: /Users/veit/.ipython/profile_default/pid/ipcluster.pid 125 | [IPClusterStart] Starting Controller with LocalControllerLauncher 126 | [IPClusterStart] Starting 4 Engines with LocalEngineSetLauncher 127 | 128 | Batch systems 129 | Besides the possibility to start ``ipcontroller`` and ``ipengine`` locally, 130 | see `Starting a cluster with SSH 131 | `_, 132 | there are also the profiles for ``MPI``, ``PBS``, ``SGE``, ``LSF``, 133 | ``HTCondor``, ``Slurm``, ``SSH`` and ``WindowsHPC``. 134 | 135 | This can be configured in ``ipcluster_config.py`` for example with 136 | ``c.IPClusterEngines.engine_launcher_class = 'SSH'`` or with 137 | 138 | .. code-block:: console 139 | 140 | $ uv run ipcluster start --engines=MPI 141 | 142 | .. seealso:: :doc:`mpi` 143 | 144 | #. Starting the Jupyter Notebook and loading the IPython-Parallel-Extension: 145 | 146 | .. code-block:: console 147 | 148 | $ uv run jupyter notebook 149 | [I NotebookApp] Loading IPython parallel extension 150 | [I NotebookApp] [jupyter_nbextensions_configurator] enabled 0.4.1 151 | [I NotebookApp] Serving notebooks from local directory: /Users/veit//jupyter-tutorial 152 | [I NotebookApp] The Jupyter Notebook is running at: 153 | [I NotebookApp] http://localhost:8888/?token=4e9acb8993758c2e7f3bda3b1957614c6f3528ee5e3343b3 154 | 155 | #. Finally the cluster with the ``default`` profile can be started in the 156 | browser at the URL 157 | ``http://localhost:8888/tree/docs/parallel/ipyparallel#ipyclusters``. 158 | -------------------------------------------------------------------------------- /docs/hub/ipyparallel/mpi.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# MPI\n", 8 | "\n", 9 | "Often, a parallel algorithm requires moving data between the engines. One way is to push and pull over the `DirectView`. However, this is slow because all of the data has to get through the controller to the client and then back to the final destination.\n", 10 | "\n", 11 | "A much better option is to use the [Message Passing Interface (MPI)](https://de.wikipedia.org/wiki/Message_Passing_Interface). IPython's parallel computing architecture was designed from the ground up to integrate with MPI. This notebook gives a brief introduction to using MPI with IPython." 12 | ] 13 | }, 14 | { 15 | "cell_type": "markdown", 16 | "metadata": {}, 17 | "source": [ 18 | "## Requirements\n", 19 | "\n", 20 | "* A standard MPI implementation like [OpenMPI](https://www.open-mpi.org/) or [MPICH](https://www.mpich.org/). \n", 21 | "\n", 22 | " For Debian/Ubuntu these can be installed with\n", 23 | " \n", 24 | " ```bash\n", 25 | " $ sudo apt install openmpi-bin\n", 26 | " ```\n", 27 | " \n", 28 | " or\n", 29 | " \n", 30 | " ```bash\n", 31 | " $ sudo apt install mpich\n", 32 | " ```\n", 33 | "\n", 34 | " Alternatively, OpenMPI or MPICH can also be installed with [Spack](https://www.python4data.science/en/latest/productive/envs/spack/use.html): the packages are `openmpi` or `mpich`. \n", 35 | "\n", 36 | "\n", 37 | "* [mpi4py](https://mpi4py.readthedocs.io/en/stable/)" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "metadata": {}, 43 | "source": [ 44 | "## Starting the engines with activated MPI\n", 45 | "\n", 46 | "### Automatic start with `mpiexec` and `ipcluster`\n", 47 | "\n", 48 | "This can be done with, for example\n", 49 | "\n", 50 | "```bash\n", 51 | "$ uv run ipcluster start -n 4 --profile=mpi\n", 52 | "```\n", 53 | "\n", 54 | "For this, however, a corresponding profile must first be created; see [configuration](config.rst).\n", 55 | "\n", 56 | "### Automatic start with PBS and `ipcluster`\n", 57 | "\n", 58 | "The `ipcluster` command also offers integration in [PBS](https://www.openpbs.org/). You can find more information about this in [Starting IPython Parallel on a traditional cluster](https://ipyparallel.readthedocs.io/en/latest/tutorial/process.html#starting-ipython-parallel-on-a-traditional-cluster)." 59 | ] 60 | }, 61 | { 62 | "cell_type": "markdown", 63 | "metadata": {}, 64 | "source": [ 65 | "## Example\n", 66 | "\n", 67 | "The following notebook cell calls `psum.py` with the following content:\n", 68 | "\n", 69 | "```Python\n", 70 | "from mpi4py import MPI\n", 71 | "import numpy as np\n", 72 | "\n", 73 | "def psum(a):\n", 74 | " locsum = np.sum(a)\n", 75 | " rcvBuf = np.array(0.0,'d')\n", 76 | " MPI.COMM_WORLD.Allreduce([locsum, MPI.DOUBLE],\n", 77 | " [rcvBuf, MPI.DOUBLE],\n", 78 | " op=MPI.SUM)\n", 79 | " return rcvBuf\n", 80 | "```" 81 | ] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "execution_count": 1, 86 | "metadata": {}, 87 | "outputs": [ 88 | { 89 | "data": { 90 | "text/plain": [ 91 | "[array([0., 1., 2., 3.]),\n", 92 | " array([4., 5., 6., 7.]),\n", 93 | " array([ 8., 9., 10., 11.]),\n", 94 | " array([12., 13., 14., 15.])]" 95 | ] 96 | }, 97 | "execution_count": 1, 98 | "metadata": {}, 99 | "output_type": "execute_result" 100 | } 101 | ], 102 | "source": [ 103 | "import ipyparallel as ipp\n", 104 | "\n", 105 | "c = ipp.Client(profile='mpi')\n", 106 | "view = c[:]\n", 107 | "view.activate()\n", 108 | "view.run('psum.py')\n", 109 | "view.scatter('a',np.arange(16,dtype='float'))\n", 110 | "view['a']" 111 | ] 112 | }, 113 | { 114 | "cell_type": "code", 115 | "execution_count": 2, 116 | "metadata": {}, 117 | "outputs": [ 118 | { 119 | "data": { 120 | "text/plain": [ 121 | "Parallel execution on engines: [0,1,2,3]" 122 | ] 123 | }, 124 | "execution_count": 2, 125 | "metadata": {}, 126 | "output_type": "execute_result" 127 | } 128 | ], 129 | "source": [ 130 | "%px totalsum = psum(a)" 131 | ] 132 | }, 133 | { 134 | "cell_type": "code", 135 | "execution_count": 3, 136 | "metadata": {}, 137 | "outputs": [ 138 | { 139 | "data": { 140 | "text/plain": [ 141 | "[120.0, 120.0, 120.0, 120.0]" 142 | ] 143 | }, 144 | "execution_count": 3, 145 | "metadata": {}, 146 | "output_type": "execute_result" 147 | } 148 | ], 149 | "source": [ 150 | "view['totalsum']" 151 | ] 152 | } 153 | ], 154 | "metadata": { 155 | "kernelspec": { 156 | "display_name": "Python 3.11 Kernel", 157 | "language": "python", 158 | "name": "python311" 159 | }, 160 | "language_info": { 161 | "codemirror_mode": { 162 | "name": "ipython", 163 | "version": 3 164 | }, 165 | "file_extension": ".py", 166 | "mimetype": "text/x-python", 167 | "name": "python", 168 | "nbconvert_exporter": "python", 169 | "pygments_lexer": "ipython3", 170 | "version": "3.11.4" 171 | }, 172 | "latex_envs": { 173 | "LaTeX_envs_menu_present": true, 174 | "autoclose": false, 175 | "autocomplete": true, 176 | "bibliofile": "biblio.bib", 177 | "cite_by": "apalike", 178 | "current_citInitial": 1, 179 | "eqLabelWithNumbers": true, 180 | "eqNumInitial": 1, 181 | "hotkeys": { 182 | "equation": "Ctrl-E", 183 | "itemize": "Ctrl-I" 184 | }, 185 | "labels_anchors": false, 186 | "latex_user_defs": false, 187 | "report_style_numbering": false, 188 | "user_envs_cfg": false 189 | }, 190 | "varInspector": { 191 | "cols": { 192 | "lenName": 16, 193 | "lenType": 16, 194 | "lenVar": 40 195 | }, 196 | "kernels_config": { 197 | "python": { 198 | "delete_cmd_postfix": "", 199 | "delete_cmd_prefix": "del ", 200 | "library": "var_list.py", 201 | "varRefreshCmd": "print(var_dic_list())" 202 | }, 203 | "r": { 204 | "delete_cmd_postfix": ") ", 205 | "delete_cmd_prefix": "rm(", 206 | "library": "var_list.r", 207 | "varRefreshCmd": "cat(var_dic_list()) " 208 | } 209 | }, 210 | "types_to_exclude": [ 211 | "module", 212 | "function", 213 | "builtin_function_or_method", 214 | "instance", 215 | "_Feature" 216 | ], 217 | "window_display": false 218 | }, 219 | "widgets": { 220 | "application/vnd.jupyter.widget-state+json": { 221 | "state": {}, 222 | "version_major": 2, 223 | "version_minor": 0 224 | } 225 | } 226 | }, 227 | "nbformat": 4, 228 | "nbformat_minor": 2 229 | } 230 | -------------------------------------------------------------------------------- /docs/hub/ipyparallel/psum.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from mpi4py import MPI 4 | 5 | 6 | def psum(a): 7 | locsum = np.sum(a) 8 | rcvBuf = np.array(0.0, "d") 9 | MPI.COMM_WORLD.Allreduce( 10 | [locsum, MPI.DOUBLE], [rcvBuf, MPI.DOUBLE], op=MPI.SUM 11 | ) 12 | return rcvBuf 13 | -------------------------------------------------------------------------------- /docs/hub/nbviewer-service.rst: -------------------------------------------------------------------------------- 1 | Create service ``nbviewer`` 2 | =========================== 3 | 4 | #. Configuring the notebook viewer as a JupyterHub service has the advantage 5 | that only users who have previously logged on to JupyterHub can call up the 6 | ``nbviewer`` instance. This can be used to protect access to notebooks as a 7 | JupyterHub service in ``/srv/jupyter/jupyter-tutorial/jupyterhub_config.py``: 8 | 9 | .. code-block:: javascript 10 | 11 | c.JupyterHub.services = [ 12 | { 13 | 'name': 'nbviewer', 14 | 'url': 'http://127.0.0.1:9000', 15 | 'cwd': '/srv/jupyterhub/nbviewer-repo', 16 | 'command': ['/srv/jupyter/.local/share/virtualenvs/jupyter-tutorial--q5BvmfG/bin/python', '-m', 'nbviewer'] 17 | } 18 | ] 19 | 20 | ``name`` 21 | The path name under which the notebook viewer can be reached: :samp:`/services/{NAME}` 22 | ``url`` 23 | Protocol, address and port used by ``nbviewer`` 24 | ``cwd`` 25 | The path to the ``nbviewer`` repository 26 | ``command`` 27 | Command to start ``nbviewer`` 28 | -------------------------------------------------------------------------------- /docs/hub/systemdspawner.rst: -------------------------------------------------------------------------------- 1 | systemdspawner 2 | ============== 3 | 4 | The systemdspawner allows JupyterHub to create single user notebook servers with 5 | `systemd `_. You get isolation and 6 | security without having to use Docker, rkt or similar. In addition, 7 | systemdspawner offers other features: 8 | 9 | * the maximum allowed memory and CPU per person can be limited via cgroups and 10 | checked with ``systemd-cgtop``. 11 | * all get their own `/tmp` directory to increase isolation 12 | * notebook servers can be started as specific local users on the system 13 | * the use of sudo in notebooks can be restricted 14 | * the paths that can be read and written to can be restricted 15 | * logs for each notebook can be managed 16 | 17 | Requirements 18 | ------------ 19 | 20 | systemdspawner requires systemd ≥ v211; the security-related functions require 21 | systemd ≥ v228. You can check which version of systemd is available on your 22 | system with 23 | 24 | .. code-block:: console 25 | 26 | $ systemctl --version | head -1 27 | systemd 249 (249.11-0ubuntu3.7) 28 | 29 | To limit memory and CPU allocations, certain kernel options must also be 30 | available. This can be checked with `check-kernel.bash 31 | `_. 32 | 33 | If the default setting :samp:`c.SystemdSpawner.dynamic_users = False` is used, 34 | the server is started with the local Unix user account. Therefore, this spawner 35 | requires that all users, already have a local account on the machine. With 36 | :samp:`c.SystemdSpawner.dynamic_users = True`, on the other hand, no local user 37 | accounts are required; they are dynamically created by systemd as needed. 38 | 39 | Installation and Configuration 40 | ------------------------------ 41 | 42 | You can install systemdspawner with 43 | 44 | .. code-block:: console 45 | 46 | $ uv add jupyterhub-systemdspawner 47 | 48 | Then you can activate it in :file:`jupyterhub_config.py` with 49 | 50 | .. code-block:: python 51 | 52 | c.JupyterHub.spawner_class = "systemdspawner.SystemdSpawner" 53 | 54 | There are many other configuration options open to you, for example 55 | 56 | :samp:`c.SystemdSpawner.mem_limit = '4G'` 57 | specifies the maximum amount of memory that can be used by each user. The 58 | ``None`` setting disables the memory limit. 59 | 60 | Although individual users should be able to use as much memory as possible, 61 | it is still useful to set a memory limit of 80–90% of the total physical 62 | memory. This prevents one user from accidentally crippling the machine 63 | single-handedly. 64 | 65 | :samp:`c.SystemdSpawner.cpu_limit = 4.0` 66 | A floating point number that specifies the number of CPU cores that each 67 | user can use. 68 | :samp:`c.SystemdSpawner.user_workingdir = '/home/{USERNAME}'` 69 | The directory where each user’s notebook server is started. This directory 70 | is also what users see when they open their notebook servers. Normally this 71 | is the user’s home directory. 72 | :samp:`c.SystemdSpawner.username_template = 'jupyter-{USERNAME}'` 73 | Template for the Unix user name under which each user should be created. 74 | :samp:`c.SystemdSpawner.default_shell = '/bin/bash'` 75 | The default shell used for the terminal in the notebook. Sets the 76 | environment variable ``SHELL`` to this value. 77 | :samp:`c.SystemdSpawner.extra_paths = ['/home/{USERNAME}/conda/bin']` 78 | List of paths to prepend to the ``PATH`` environment variable for the 79 | spawned notebook server. This is easier than setting the ``env`` property 80 | because you want to add ``PATH`` and not replace it completely. This is very 81 | useful if you want to include a virtualenv or conda installation in the 82 | user’s ``PATH`` by default. 83 | :samp:`c.SystemdSpawner.unit_name_template = 'jupyter-{USERNAME}-singleuser'` 84 | Systemd service unit name template for each user notebook server. This 85 | allows differentiation between multiple JupyterHubs with systemd spawners on 86 | the same machine. Should only contain ``[a-zA-Z0-9_-]``. 87 | :samp:`c.SystemdSpawner.unit_extra_properties = {'LimitNOFILE': '16384'}` 88 | Dict of key-value pairs used to add arbitrary properties to spawned 89 | JupyerHub units – see also ``man systemd-run`` for details on properties. 90 | :samp:`c.SystemdSpawner.isolate_tmp = True` 91 | Setting this to ``True`` will create a separate, private :file:`/tmp` 92 | directory for each user. This is very useful to protect against accidental 93 | leakage of otherwise private information. 94 | 95 | This requires systemd version > 227. If you enable this in earlier versions, 96 | spawning will fail. 97 | 98 | :samp:`c.SystemdSpawner.isolate_devices = True` 99 | If you set this option to ``True``, a separate, private :file:`/dev` 100 | directory will be created for each user. This prevents users from accessing 101 | hardware devices directly, which could be a potential source of security 102 | problems. ``/dev/null``, ``/dev/zero``, ``/dev/random`` and the ``ttyp`` 103 | pseudo devices are already mounted, so most users should not notice any 104 | change if this is enabled. 105 | 106 | :samp:`c.SystemdSpawner.disable_user_sudo = True` 107 | Setting this option to ``True`` will prevent users from being able to use 108 | sudo or other means to become other users. This helps limit the damage done 109 | by compromising a user’s credentials if they also have sudo privileges on 110 | the machine – a web-based exploit can now only damage the user’s own data. 111 | 112 | This requires systemd version > 228. If you enable this in earlier versions, 113 | spawning will fail. 114 | 115 | :samp:`c.SystemdSpawner.readonly_paths = ['/']` 116 | List of file system paths to be mounted read-only for the user’s notebook 117 | server. This overrides any existing file system permissions. Subpaths of 118 | paths that are mounted readonly can be marked as readwrite with 119 | ``readwrite_paths``. This is useful for marking ``/`` as read-only and 120 | listing only those paths to which notebook users are allowed to write. If 121 | the paths listed here do not exist, you will get an error message. 122 | 123 | This requires systemd version > 228. If you enable this feature in earlier 124 | versions, spawning will fail. Up to systemd version 231 it can also only 125 | contain directories and no files. 126 | 127 | :samp:`c.SystemdSpawner.readwrite_paths = ['/home/{USERNAME}']` 128 | List of file system paths to be mounted read-only for the user’s notebook 129 | server. This only makes sense if ``readonly_paths`` is used to make some 130 | paths read-only. This does not override the file system permissions – the 131 | user must have the appropriate permissions to write to these paths. 132 | 133 | This requires systemd version > 228. If you enable this feature in earlier 134 | versions, spawning will fail. Up to systemd version 231, it can also only 135 | contain directories and not files. 136 | 137 | .. seealso:: 138 | * `systemdspawner `_ 139 | -------------------------------------------------------------------------------- /docs/intro.rst: -------------------------------------------------------------------------------- 1 | Introduction 2 | ============ 3 | 4 | Target group 5 | ------------ 6 | 7 | The users of Jupyter notebooks are diverse, from data scientists to data 8 | engineers and analysts to system engineers. Their skills and workflows are very 9 | different. However, one of the great strengths of Jupyter notebooks is that they 10 | allow these different experts to work closely together in cross-functional 11 | teams. 12 | 13 | Data scientists 14 | explore data with different parameters and summarise the results. 15 | Data engineers 16 | check the quality of the code and make it more robust, efficient and 17 | scalable. 18 | Data analysts 19 | use the code provided by data engineers to systematically analyse the data. 20 | System engineers 21 | provide the research platform based on the :doc:`hub/index` on which the 22 | other roles can perform their work. 23 | 24 | In this tutorial we address system engineers who want to build and run a 25 | platform based on Jupyter notebooks. We then explain how this platform can be 26 | used effectively by data scientists, data engineers and analysts. 27 | 28 | Why Jupyter? 29 | ------------ 30 | 31 | How can these diverse tasks be simplified? You will hardly find a tool that 32 | covers all of these tasks, and several tools are often required even for 33 | individual tasks. Therefore, on a more abstract level, we are looking for more 34 | general patterns for tools and languages with which data can be analysed and 35 | visualised and a project can be documented and presented. This is exactly what 36 | we are aiming for with `Project Jupyter `_. 37 | 38 | The Jupyter project started in 2014 with the aim of creating a consistent set of 39 | open source tools for scientific research, reproducible workflows, 40 | `computational narratives 41 | `_ 42 | and data analysis. In 2017, Jupyter received the `ACM Software Systems Award 43 | `_ 44 | – a prestigious award which, among other things, shares with Unix and the web. 45 | 46 | To understand why Jupyter notebooks are so successful, let’s take a closer look 47 | at the core functions: 48 | 49 | `Jupyter Notebook Format `_ 50 | Jupyter Notebooks are an open, JSON-based document format with full records 51 | of the user’s sessions and the code they contain. 52 | Interactive Computing Protocol 53 | The notebook communicates with the computing kernel via the *Interactive 54 | Computing Protocol*, an open network protocol based on JSON data via `ZMQ 55 | `_ and `WebSockets 56 | `_. 57 | :doc:`/kernels/index` 58 | Kernels are processes that execute interactive code in a specific 59 | programming language and return the output to the user. 60 | 61 | .. seealso:: 62 | * `Jupyter celebrates 20 years 63 | `_ 64 | 65 | Jupyter infrastructure 66 | ---------------------- 67 | 68 | A platform for the above-mentioned use cases requires an extensive 69 | infrastructure that not only allows the provision of the kernel and the 70 | parameterisation, time control and parallelisation of notebooks, but also the 71 | uniform provision of resources. 72 | 73 | This tutorial provides a platform that enables fast, flexible and comprehensive 74 | data analysis beyond Jupyter notebooks. At the moment, however, we are not yet 75 | going into how it can be expanded to include streaming pipelines and 76 | domain-driven data stores. 77 | 78 | However, you can also create and run the examples in the Jupyter tutorial 79 | locally. 80 | 81 | Workspace 82 | --------- 83 | 84 | Setting up the workspace includes installing and configuring 85 | :doc:`python4datascience:workspace/ipython/index` and :doc:`Jupyter notebooks 86 | `, :doc:`nbextensions/index` and :doc:`ipywidgets/index`. 87 | -------------------------------------------------------------------------------- /docs/ipywidgets/embedding.rst: -------------------------------------------------------------------------------- 1 | Embed Jupyter widgets 2 | ===================== 3 | 4 | Jupyter widgets can be serialised and then embedded in other contexts: 5 | 6 | * static web pages 7 | * Sphinx documentation 8 | * HTML converted notebooks on Nbviewer 9 | 10 | 11 | The npm package ``@jupyter-widgets/html-manager`` allows embedding in two 12 | different ways: 13 | 14 | 15 | * embedding the standard elements that can be used on any website 16 | * embedding with `RequireJS `_ also for custom widgets. 17 | 18 | Embed widgets in HTML pages 19 | --------------------------- 20 | 21 | The widgets menu provides several options for this: 22 | 23 | *Save Notebook Widget State* 24 | A notebook file is saved with the current widget status as metadata. This 25 | allows to be rendered with the widgets in the browser. 26 | *Clear Notebook Widget State* 27 | The widget status metadata is deleted from the notebook file. 28 | *Embed widgets* 29 | The menu item offers a dialog box with an HTML page on which the current 30 | widgets are embedded. The RequireJS embedder is used to support custom 31 | widgets. 32 | 33 | .. note:: 34 | The first script tag loads RequireJS from a CDN. However, RequireJS 35 | should be made available on the site itself and this script tag should 36 | be deleted. 37 | 38 | .. note:: 39 | The second script tag loads the RequireJS widget embedder. This defines 40 | suitable modules and then sets up a function for rendering all widget 41 | views contained on the page. 42 | 43 | If you only embed standard widgets and don’t use RequireJS, you can 44 | replace the first two script tags with a script tag that loads the 45 | standard script. 46 | 47 | *Download Widget State* 48 | The option downloads a JSON file that contains the serialized status of all 49 | widget models currently in use in the 50 | ``application/vnd.jupyter.widget-state+json`` format specified in the 51 | ``@jupyter-widgets/schema`` npm package. 52 | 53 | Sphinx integration 54 | ------------------ 55 | 56 | Jupyter Sphinx 57 | ~~~~~~~~~~~~~~ 58 | 59 | `jupyter_sphinx `_ enables 60 | jupyter-specific functions in Sphinx. It can be installed with ``pip``. 61 | 62 | Configuration 63 | ::::::::::::: 64 | 65 | Adds ``jupyter_sphinx.embed_widgets`` to the list of extensions in the 66 | :file:`conf.py` file. 67 | 68 | Then you can use the following directives in reStructuredText: 69 | 70 | ``ipywidgets-setup`` 71 | .. code-block:: python 72 | 73 | from ipywidgets import Button, IntSlider, VBox, jsdlink 74 | 75 | ``ipywidgets-display`` 76 | .. code-block:: python 77 | 78 | s1, s2 = IntSlider(max=200, value=100), IntSlider(value=40) 79 | b = Button(icon="legal") 80 | jsdlink((s1, "value"), (s2, "max")) 81 | VBox([s1, s2, b]) 82 | 83 | 84 | Example 85 | ::::::: 86 | 87 | .. code-block:: rest 88 | 89 | .. ipywidgets-setup:: 90 | 91 | from ipywidgets import VBox, jsdlink, IntSlider, Button 92 | 93 | .. ipywidgets-display:: 94 | :hide-code: 95 | 96 | s1, s2 = IntSlider(max=200, value=100), IntSlider(value=40) 97 | b = Button(icon="legal") 98 | jsdlink((s1, "value"), (s2, "max")) 99 | VBox([s1, s2, b]) 100 | 101 | Options 102 | ::::::: 103 | 104 | The ``ipywidgets-setup`` and ``ipywidgets-display`` directives have the 105 | following options: 106 | 107 | ``ipywidgets-setup`` 108 | with the option ``:show:`` to display the setup code as a code block 109 | ``ipywidgets-display`` 110 | with the following options: 111 | ``:hide-code:`` 112 | doesn’t show the code, only the widget 113 | 114 | Widget 115 | 116 | ``:code-below:`` 117 | shows the code after the widget 118 | ``:alt:`` 119 | Alternate text if the widget cannot be rendered 120 | 121 | .. seealso:: 122 | `Options `_ 123 | -------------------------------------------------------------------------------- /docs/ipywidgets/index.rst: -------------------------------------------------------------------------------- 1 | ``ipywidgets`` 2 | ============== 3 | 4 | `ipywidgets `_ are interactive 5 | widgets for Jupyter notebooks. They extend notebooks by the possibility that 6 | users can enter data themselves, manipulate data and see the changed results. 7 | 8 | .. toctree:: 9 | :hidden: 10 | :titlesonly: 11 | :maxdepth: 0 12 | 13 | examples.ipynb 14 | widget-list.ipynb 15 | widget-events.ipynb 16 | custom-widget.ipynb 17 | libs/index 18 | embedding 19 | -------------------------------------------------------------------------------- /docs/ipywidgets/libs/Big.Buck.Bunny.mp4.dvc: -------------------------------------------------------------------------------- 1 | md5: f5afed0a3388659637766e6de41f4afc 2 | locked: true 3 | deps: 4 | - etag: '"b068c21dac9145f3b986727d14281b93d7952918"' 5 | path: https://github.com/maartenbreddels/ipywebrtc/raw/master/docs/source/Big.Buck.Bunny.mp4 6 | outs: 7 | - md5: 614617b0c19b4558cbabadcdd2b1dbde 8 | path: Big.Buck.Bunny.mp4 9 | cache: true 10 | metric: false 11 | persist: false 12 | -------------------------------------------------------------------------------- /docs/ipywidgets/libs/index.rst: -------------------------------------------------------------------------------- 1 | ipywidgets libraries 2 | ==================== 3 | 4 | Popular widget libraries are 5 | 6 | ``qplot`` 7 | 2-D plotting library for Jupyter notebooks 8 | 9 | * :doc:`pyviz:d3js/bqplot/index` 10 | 11 | ``ipycanvas`` 12 | Interactive canvas elements in Jupyter notebooks 13 | 14 | .. toctree:: 15 | :maxdepth: 1 16 | 17 | ipycanvas.ipynb 18 | 19 | ``pythreejs`` 20 | Jupyter `Three.js `_ bridge 21 | 22 | * :doc:`pyviz:js/pythreejs` 23 | 24 | ``ipyvolume`` 25 | IPyvolume is a Python library for visualizing 3D volumes and glyphs (for 26 | example 3D scatter plots). 27 | 28 | * :doc:`pyviz:js/ipyvolume` 29 | 30 | ``ipyleaflet`` 31 | Jupyter-`Leaflet.js `_ bridge 32 | 33 | * :doc:`pyviz:js/ipyleaflet` 34 | 35 | ``ipywebrtc`` 36 | `WebRTC `_ and `MediaStream 37 | `_ API for 38 | Jupyter notebooks 39 | 40 | .. toctree:: 41 | :maxdepth: 1 42 | 43 | ipywebrtc.ipynb 44 | 45 | ``ipysheet`` 46 | Interactive tables to use IPython widgets in tables of Jupyter notebooks. 47 | 48 | .. toctree:: 49 | :maxdepth: 1 50 | 51 | ipysheet.ipynb 52 | 53 | ``ipydatagrid`` 54 | `ipydatagrid `_ is a fast 55 | and versatile datagrid widget. 56 | 57 | ``ipyvuetify`` 58 | `Vuetify `_ widgets in Jupyter notebooks 59 | 60 | .. toctree:: 61 | :maxdepth: 1 62 | 63 | ipyvuetify.ipynb 64 | 65 | ``ipympl`` 66 | ``ipympl`` or `jupyter-matplotlib 67 | `_ offer interactive 68 | widgets for matplotlib. 69 | 70 | .. toctree:: 71 | :maxdepth: 1 72 | 73 | ipympl.ipynb 74 | -------------------------------------------------------------------------------- /docs/ipywidgets/smiley.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veit/jupyter-tutorial/24fcd76f43fc46ffd02e3bd98479a41ad6565197/docs/ipywidgets/smiley.gif -------------------------------------------------------------------------------- /docs/jupyterlab/collaboration.rst: -------------------------------------------------------------------------------- 1 | Real-time collaboration 2 | ======================= 3 | 4 | From JupyterLab ≥ 4 it is possible to activate Real-Time Collaboration by 5 | installing the extension ``jupyter_collaboration``. This enables real-time 6 | collaboration between multiple clients. In addition, you can see the cursors of 7 | others. 8 | 9 | .. seealso:: 10 | * `jupyter_collaboration documentation 11 | `_ 12 | 13 | Installation 14 | ------------ 15 | 16 | .. code-block:: console 17 | 18 | $ python -m pip install jupyter-collaboration 19 | 20 | To share a document with others, you can copy the URL and send it, or you can 21 | additionally install `jupyterlab-link-share 22 | `_, which allows 23 | you to share the link including the token. 24 | -------------------------------------------------------------------------------- /docs/jupyterlab/extensions.rst: -------------------------------------------------------------------------------- 1 | JupyterLab extensions 2 | ===================== 3 | 4 | JupyterLab is designed as an extensible environment. In doing so, JupyterLab 5 | extensions can customise any part of JupyterLab. They can provide new themes, 6 | file viewers and editors, or renderers for rich outputs in 7 | :doc:`../notebook/index`. 8 | 9 | .. seealso:: 10 | `JupyterLab Extensions by Examples 11 | `_ 12 | 13 | Installing extensions 14 | --------------------- 15 | 16 | A JupyterLab extension contains JavaScript that is installed in JupyterLab and 17 | executed in the browser. Most JupyterLab extensions can be installed with 18 | :term:`pip`. These packages can also contain server-side components that are 19 | required for the extension to function. 20 | 21 | Since JupyterLab ≥ 4, the default extension manager uses :term:`PyPI` as a 22 | source for the available extensions and :term:`pip` to install them. An 23 | extension is listed if the Python package has the :term:`trove classifier 24 | ` ``Framework :: Jupyter :: JupyterLab :: Extensions :: 25 | Prebuilt``. 26 | 27 | .. warning:: 28 | It does not check if the extension is compatible with the current JupyterLab 29 | version. 30 | 31 | .. danger:: 32 | Installing an extension allows arbitrary code to be executed on the server, 33 | kernel and browser. Therefore, avoid installing extensions that you do not 34 | trust. 35 | 36 | Configuring the Extension Manager 37 | --------------------------------- 38 | 39 | By default, there are two extension managers provided by JupyterLab: 40 | 41 | ``pypi`` 42 | Default setting that allows the installation from :term:`pypi.org`. 43 | ``readonly`` 44 | shows the installed extensions with the possibility to disable or enable 45 | them. 46 | 47 | You can specify the manager with the command line option 48 | ``--LabApp.extension_manager``, for example :samp:`jupyter lab 49 | --LabApp.extension_manager={readonly}`. 50 | 51 | When searching for extensions in the extension manager, JupyterLab usually shows 52 | all search results and any source extension can be installed. However, to 53 | increase security, JupyterLab can be configured so that extensions can only be 54 | activated using the block or allow lists. 55 | 56 | You can define the loading of the lists with ``blocked_extensions_uris`` or 57 | ``allowed_extensions_uris``, which contain a list of comma-separated URIs, for 58 | example 59 | :samp:`--LabServerApp.blocked_extensions_uris=http://example.com/blocklist.json` 60 | with the following :file:`blocklist.json` file: 61 | 62 | .. code-block:: json 63 | 64 | { 65 | "blocked_extensions": [ 66 | { 67 | "name": "@jupyterlab-examples/launcher", 68 | "type": "jupyterlab", 69 | "reason": "@jupyterlab-examples/launcher is blocklisted for test purpose - Do NOT take this for granted!!!", 70 | "creation_date": "2020-03-11T03:28:56.782Z", 71 | "last_update_date": "2020-03-11T03:28:56.782Z" 72 | } 73 | ] 74 | } 75 | 76 | Another example shows an :file:`allowlist.json` file that allows all extensions 77 | of the `JupyterLab organisation `_: 78 | 79 | .. code-block:: json 80 | 81 | { 82 | "allowed_extensions": [ 83 | { 84 | "name": "@jupyterlab/*", 85 | "type": "jupyterlab", 86 | "reason": "All @jupyterlab org extensions are allowed, of course…", 87 | "creation_date": "2020-03-11T03:28:56.782Z", 88 | "last_update_date": "2020-03-11T03:28:56.782Z" 89 | } 90 | ] 91 | } 92 | -------------------------------------------------------------------------------- /docs/jupyterlab/index.rst: -------------------------------------------------------------------------------- 1 | JupyterLab 2 | ========== 3 | 4 | `JupyterLab `_ is an 5 | extensible, feature-rich editor for creating and editing :doc:`Jupyter Notebooks 6 | <../notebook/index>`: 7 | 8 | * You can arrange multiple documents and activities side by side in your 9 | workspace using tabs. 10 | * Code consoles provide temporary scratchpads for running code interactively, 11 | which can also be linked to a :doc:`notebook kernel <../kernels/index>`. 12 | * There is also preview of 13 | :doc:`python4datascience:data-processing/serialisation-formats/csv/index` and 14 | :doc:`pyviz:vega/index` files. 15 | * :doc:`extensions` can customise or enhance any part of JupyterLab. 16 | 17 | JupyterLab currently uses the same Notebook document format as the classic 18 | :doc:`Jupyter Notebook <../notebook/index>`. Notebook 7 will replace the classic 19 | Jupyter Notebook format. 20 | 21 | .. seealso:: 22 | `Migrating to Notebook 7 23 | `_ 24 | 25 | .. toctree:: 26 | :hidden: 27 | :titlesonly: 28 | :maxdepth: 0 29 | 30 | install 31 | extensions 32 | jupyterhub 33 | collaboration 34 | scheduler 35 | -------------------------------------------------------------------------------- /docs/jupyterlab/initial-jupyterlab-dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veit/jupyter-tutorial/24fcd76f43fc46ffd02e3bd98479a41ad6565197/docs/jupyterlab/initial-jupyterlab-dashboard.png -------------------------------------------------------------------------------- /docs/jupyterlab/install.rst: -------------------------------------------------------------------------------- 1 | Install JupyterLab 2 | ================== 3 | 4 | Creating a virtual environment with JupyterLab 5 | ---------------------------------------------- 6 | 7 | .. code-block:: console 8 | 9 | $ python3 -m venv myproject 10 | $ cd myproject 11 | $ . bin/activate 12 | $ python -m pip install jupyterlab 13 | 14 | Start JupyterLab 15 | ---------------- 16 | 17 | .. code-block:: console 18 | 19 | $ jupyter lab 20 | [I 2023-06-16 13:01:43.205 ServerApp] Package jupyterlab took 0.0000s to import 21 | ... 22 | To access the server, open this file in a browser: 23 | file:///Users/veit/Library/Jupyter/runtime/jpserver-48904-open.html 24 | Or copy and paste one of these URLs: 25 | http://localhost:8888/lab?token=72d33027f130e602f43ef0cdfbff7471c8406ffafd94f075 26 | http://127.0.0.1:8888/lab?token=72d33027f130e602f43ef0cdfbff7471c8406ffafd94f075 27 | 28 | Your default web browser will then open with this URL. 29 | 30 | .. image:: initial-jupyterlab-dashboard.png 31 | 32 | Localisation 33 | ------------ 34 | 35 | Since version 3.0, JupyterLab offers the possibility to set the display language 36 | of the interface. To do this, the appropriate language packages must be 37 | installed, for example: 38 | 39 | .. code-block:: console 40 | 41 | $ python -m pip install jupyterlab-language-pack-de-DE 42 | 43 | In the `language-packs `_ 44 | repository you will find a list of the available language packs. 45 | 46 | Then you can select the newly installed language in :menuselection:`Settings --> 47 | Language`. 48 | -------------------------------------------------------------------------------- /docs/jupyterlab/jupyterhub.rst: -------------------------------------------------------------------------------- 1 | JupyterLab on JupyterHub 2 | ======================== 3 | 4 | JupyterLab works by default with :doc:`../hub/index` ≥ 1.0 and can even run 5 | alongside classic :doc:`notebooks <../notebook/index>`. 6 | 7 | When JupyterLab is used with JupyterHub, additional menu items are displayed in 8 | the :menuselection:`File` menu to :menuselection:`Log Out` or to call up the 9 | :menuselection:`Hub Control Panel`. 10 | 11 | If JupyterLab is not yet the default, you can change the configuration in 12 | :file:`jupyterhub_config.py`: 13 | 14 | .. code-block:: python 15 | 16 | c.Spawner.default_url = "/lab" 17 | -------------------------------------------------------------------------------- /docs/jupyterlab/scheduler.rst: -------------------------------------------------------------------------------- 1 | Scheduler 2 | ========= 3 | 4 | `Jupyter Scheduler 5 | `_ is a 6 | collection of extensions for programming jobs to run immediately or on a 7 | schedule. It has a Lab (client) and a Server extension. Both are needed to 8 | schedule and run notebooks. If you install Jupyter Scheduler via the JupyterLab 9 | extension manager, you may only install the client extension and not the server 10 | extension. Therefore, install Jupyter Scheduler with :term:`pip`: 11 | 12 | .. code-block:: console 13 | 14 | $ python -m pip install jupyter_scheduler 15 | 16 | This will automatically activate the lab and server extensions. You can check 17 | this with 18 | 19 | .. code-block:: console 20 | 21 | $ jupyter server extension list 22 | ... 23 | jupyter_scheduler enabled 24 | - Validating jupyter_scheduler... 25 | Package jupyter_scheduler took 0.0785s to import 26 | jupyter_scheduler 1.3.2 OK 27 | ... 28 | 29 | and 30 | 31 | .. code-block:: console 32 | 33 | $ jupyter labextension list 34 | ... 35 | @jupyterlab/scheduler v1.3.2 enabled X 36 | ... 37 | 38 | #. To create a jog from an open notebook, click the :guilabel:`Create a notebook 39 | job` button in the top toolbar of the open notebook. 40 | #. Give your notebook job a name, select the output formats and specify 41 | parameters that will be set as local variables when your notebook is 42 | executed. This parameterised execution is similar to that used in 43 | :doc:`Papermill `. 44 | #. To create a job that will run once, select :guilabel:`Run now` and click 45 | :guilabel:`Create`. 46 | #. To create a job definition that will run repeatedly on a schedule, select 47 | :guilabel:`Run on a schedule`. 48 | -------------------------------------------------------------------------------- /docs/kernels/index.rst: -------------------------------------------------------------------------------- 1 | Kernels 2 | ======= 3 | 4 | The Jupyter team manages the `IPython-Kernel 5 | `_. In addition to Python, many other 6 | languages can be used in notebooks. The following Jupyter kernels are widely 7 | used: 8 | 9 | * R 10 | 11 | * IRKernel: `Docs `__ | `GitHub `__ 12 | * IRdisplay: `GitHub `__ 13 | * Repr: `GitHub `__ 14 | 15 | * Julia 16 | 17 | * IJulia: `GitHub `__ 18 | * Interact.jl: `GitHub `__ 19 | 20 | A list of available kernels can be found in `Jupyter kernels 21 | `_. 22 | 23 | .. seealso:: 24 | * `Using Wolfram Language in Jupyter: A free alternative to Mathematica 25 | `_ 26 | 27 | .. toctree:: 28 | :hidden: 29 | :titlesonly: 30 | :maxdepth: 0 31 | 32 | install 33 | python38.ipynb 34 | python39.ipynb 35 | python310.ipynb 36 | r 37 | -------------------------------------------------------------------------------- /docs/kernels/install.rst: -------------------------------------------------------------------------------- 1 | Install, view and start the kernel 2 | ================================== 3 | 4 | Install a kernel 5 | ---------------- 6 | 7 | Kernels are searched for in the following directories, for example: 8 | 9 | * :samp:`/srv/jupyter/.local/share/jupyter/kernels` 10 | * :samp:`/usr/local/share/jupyter/kernels` 11 | * :samp:`/usr/share/jupyter/kernels` 12 | * :samp:`/srv/jupyter/.ipython/kernels` 13 | 14 | To make your new environment available as a Jupyter kernel in one of the 15 | directories, you should install ipykernel: 16 | 17 | .. code-block:: console 18 | 19 | $ uv add ipykernel 20 | 21 | You can then register your kernel, for example with 22 | 23 | .. code-block:: console 24 | 25 | $ uv run python -m ipykernel install --prefix=/srv/jupyter/.ipython/kernels --name python311 --display-name 'Python 3.11 Kernel' 26 | 27 | :samp:`--prefix={/PATH/TO/KERNEL}` 28 | specifies the path where the Jupyter kernel is to be installed. 29 | :samp:`--user` 30 | installs the kernel for the current user and not system-wide 31 | :samp:`name {NAME}` 32 | gives a name for the ``kernelspec``. This is required in order to be able to 33 | use several IPython kernels at the same time. 34 | 35 | ``ipykernel install`` creates a ``kernelspec`` file in JSON format for the 36 | current Python environment, for example: 37 | 38 | .. code-block:: json 39 | 40 | { 41 | "display_name": "My Kernel", 42 | "language": "python" 43 | "argv": [ 44 | "/srv/jupyter/.ipython/kernels/python311_kernel-7y9G693U/bin/python", 45 | "-m", 46 | "ipykernel_launcher", 47 | "-f", 48 | "{connection_file}" 49 | ], 50 | } 51 | 52 | :samp:`display_name` 53 | The name of the kernel as it should be displayed in the browser. In contrast 54 | to the kernel name used in the API, it can contain any Unicode characters. 55 | :samp:`language` 56 | The name of the language of the kernel. If no suitable ``kernelspec`` key is 57 | found when loading notebooks, a kernel with a suitable language is used. In 58 | this way, a notebook written for a Python or Julia kernel can be linked to 59 | the user’s Python or Julia kernel, even if it does not have the same name as 60 | the author’s. 61 | :samp:`argv` 62 | A list of command line arguments used to start the kernel. 63 | ``{connection_file}`` refers to a file that contains the IP address, ports, 64 | and authentication key required for the connection. Usually this JSON file 65 | is saved in a safe place of the current profile: 66 | 67 | :samp:`{connection_file}` refers to a file containing the IP address, ports 68 | and authentication key needed for the connection. Typically, this JSON file 69 | is stored in a secure location of the current profile: 70 | 71 | .. code-block:: javascript 72 | 73 | { 74 | "shell_port": 61656, 75 | "iopub_port": 61657, 76 | "stdin_port": 61658, 77 | "control_port": 61659, 78 | "hb_port": 61660, 79 | "ip": "127.0.0.1", 80 | "key": "a0436f6c-1916-498b-8eb9-e81ab9368e84" 81 | "transport": "tcp", 82 | "signature_scheme": "hmac-sha256", 83 | "kernel_name": "" 84 | } 85 | 86 | :samp:`interrupt_mode` 87 | can be either ``signal`` or ``message`` and specifies how a client should 88 | interrupt the execution of a cell on this kernel. 89 | 90 | ``signal`` 91 | sends an interrupt, e.g. ``SIGINT`` on *POSIX* systems 92 | ``message`` 93 | sends an ``interrupt_request``, see also `Kernel Interrupt 94 | `_. 95 | 96 | :samp:`env` 97 | ``dict`` with environment variables to be set for the kernel. These are 98 | added to the current environment variables before the kernel starts. 99 | :samp:`metadata` 100 | ``dict`` with additional attributes for this kernel. Used by clients to 101 | support the kernel selection. Metadata added here should have a namespace 102 | for the tool to read and write that metadata. 103 | 104 | You can edit this ``kernelspec`` file at a later time. 105 | 106 | Show available kernels 107 | ---------------------- 108 | 109 | .. code-block:: console 110 | 111 | $ uv run jupyter kernelspec list 112 | Available kernels: 113 | mykernel /Users/veit/Library/Jupyter/kernels/mykernel 114 | python311 /Users/veit/Library/Jupyter/kernels/python311 115 | python313 /Users/veit/Library/Jupyter/kernels/python313 116 | 117 | Start kernel 118 | ------------ 119 | 120 | .. code-block:: console 121 | 122 | $ uv run jupyter console --kernel mykernel 123 | Jupyter console 6.0.0 124 | Python 2.7.15 (default, Oct 22 2018, 19:33:46) 125 | ... 126 | 127 | In [1]: 128 | 129 | With :kbd:`ctrl-d` you can exit the kernel again. 130 | 131 | Delete kernel 132 | ------------- 133 | 134 | .. code-block:: console 135 | 136 | $ uv run jupyter kernelspec uninstall mykernel 137 | 138 | Uninstall the Standard kernel 139 | ----------------------------- 140 | 141 | If not already done, a configuration file can be created, for example with 142 | 143 | .. code-block:: console 144 | 145 | $ uv run jupyter lab --generate-config 146 | 147 | Then you can add the following line to this configuration file: 148 | 149 | .. code-block:: python 150 | 151 | c.KernelSpecManager.ensure_native_kernel = False 152 | -------------------------------------------------------------------------------- /docs/kernels/r.rst: -------------------------------------------------------------------------------- 1 | R-Kernel 2 | ======== 3 | 4 | #. ZMQ 5 | 6 | For Ubuntu & Debian: 7 | 8 | .. code-block:: console 9 | 10 | $ sudo apt install libzmq3-dev libcurl4-openssl-dev libssl-dev jupyter-core jupyter-client 11 | 12 | #. R packages 13 | 14 | .. code-block:: console 15 | 16 | $ R 17 | > install.packages(c('crayon', 'pbdZMQ', 'devtools')) 18 | ... 19 | --- Please select a CRAN mirror for use in this session --- 20 | ... 21 | 33: Germany (Münster) [https] 22 | ... 23 | Selection: 33 24 | > devtools::install_github(paste0('IRkernel/', c('repr', 'IRdisplay', 'IRkernel'))) 25 | Downloading GitHub repo IRkernel/repr@master 26 | from URL https://api.github.com/repos/IRkernel/repr/zipball/master 27 | ... 28 | 29 | #. Deploy the kernel 30 | 31 | .. code-block:: console 32 | 33 | > IRkernel::installspec() 34 | ... 35 | [InstallKernelSpec] Installed kernelspec ir in /Users/veit/Library/Jupyter/kernels/ir3.3.3/share/jupyter/kernels/ir 36 | 37 | You can also deploy the kernel system-wide: 38 | 39 | .. code-block:: console 40 | 41 | > IRkernel::installspec(user = FALSE) 42 | 43 | .. seealso:: 44 | * `IRkernel Installation `_ 45 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/nbconvert.rst: -------------------------------------------------------------------------------- 1 | ``nbconvert`` 2 | ============= 3 | 4 | `nbconvert `_ 5 | converts notebooks to other formats 6 | 7 | Installation 8 | ------------ 9 | 10 | .. code-block:: console 11 | 12 | $ uv add nbconvert 13 | 14 | .. important:: 15 | To be able to use all functions of ``nbconvert``, Pandoc and TeX 16 | (especially XeLaTeX) are required. These must be installed separately. 17 | 18 | Install Pandoc 19 | ~~~~~~~~~~~~~~ 20 | 21 | ``nbconvert`` uses `Pandoc `_ to convert Markdown to 22 | formats other than HTML. 23 | 24 | .. tab:: Debian/Ubuntu 25 | 26 | .. code-block:: console 27 | 28 | $ sudo apt install pandoc 29 | 30 | .. tab:: macOS 31 | 32 | .. code-block:: console 33 | 34 | $ brew install pandoc 35 | 36 | Install Tex 37 | ~~~~~~~~~~~ 38 | 39 | For the conversion to PDF, ``nbconvert`` uses the Tex ecosystem in preparation: 40 | A ``.tex`` file is created, which is converted into a PDF by the XeTeX engine. 41 | 42 | .. tab:: Debian/Ubuntu 43 | 44 | .. code-block:: console 45 | 46 | $ sudo apt install texlive-xetex 47 | 48 | .. tab:: macOS 49 | 50 | .. code-block:: console 51 | 52 | $ eval "$(/usr/libexec/path_helper)" 53 | $ brew install --cask mactex 54 | 55 | .. seealso:: 56 | 57 | `MacTeX `_ 58 | 59 | Use on the command line 60 | ----------------------- 61 | 62 | .. code-block:: console 63 | 64 | $ jupyter nbconvert --to FORMAT mynotebook.ipynb 65 | 66 | ``latex`` 67 | creates a ``NOTEBOOK_NAME.tex`` file and possibly images as PNG files in a 68 | folder. With ``--template`` you can choose between one of two templates: 69 | 70 | ``--template article`` 71 | default 72 | 73 | Latex article, derived from the Sphinx how-to 74 | 75 | ``--template report`` 76 | Latex report with table of contents and chapters 77 | 78 | ``pdf`` 79 | creates a PDF over latex. Supports the same templates as ``latex``. 80 | 81 | ``slides`` 82 | creates `Reveal.js `_ slides. 83 | 84 | ``script`` 85 | converts the notebook into an executable script. This is the easiest way to 86 | create a Python script or a script in another language. 87 | 88 | .. note:: 89 | If a notebook contains *Magics*, then this can possibly only be carried 90 | out in one Jupyter session. 91 | 92 | We can for example convert 93 | :doc:`python4datascience:workspace/ipython/mypackage/foo` into a Python 94 | script with: 95 | 96 | .. code-block:: console 97 | 98 | $ uv run --with jupyter jupyter nbconvert --to script workspace/ipython/mypackage/foo.ipynb 99 | [NbConvertApp] Converting notebook docs/basics/ipython/mypackage/foo.ipynb to script 100 | [NbConvertApp] Writing 245 bytes to docs/basics/ipython/mypackage/foo.py 101 | 102 | The result is then :file:`foo.py` with: 103 | 104 | .. code-block:: python 105 | 106 | #!/usr/bin/env python 107 | # coding: utf-8 108 | 109 | # # `foo.ipynb` 110 | 111 | 112 | # In[1]: 113 | def bar(): 114 | return "bar" 115 | 116 | 117 | # In[2]: 118 | def has_ip_syntax(): 119 | listing = get_ipython().getoutput("ls") 120 | return listing 121 | 122 | 123 | # In[3]: 124 | def whatsmyname(): 125 | return __name__ 126 | 127 | .. note:: 128 | In order to assign notebook cells to slides, you should select 129 | :menuselection:`View --> Cell Toolbar --> Slideshow`. Then a menu is 130 | displayed in each cell at the top right with the options: 131 | :menuselection:`Slide, Sub-Slide, Fragment, Skip, Notes`. 132 | 133 | .. note:: 134 | Lecture notes require a local copy of ``reveal.js``. The following option 135 | can be specified so that ``nbconvert`` can find this: ``--reveal-prefix 136 | /path/to/reveal.js``. 137 | 138 | Further details for ``FORMAT`` are ``asciidoc``, ``custom``, ``html``, 139 | ``markdown``, ``notebook``, and ``rst``. 140 | 141 | nb2xls 142 | ------ 143 | 144 | `nb2xls `_ converts Jupyter notebooks into 145 | Excel files (``.xlsx``) taking into account pandas DataFrames and Matplotlib 146 | outputs. However, the input cells are not converted and only part of the 147 | Markdown is converted. 148 | 149 | Own exporters 150 | ------------- 151 | 152 | .. seealso:: 153 | `Customizing exporters 154 | `_ 155 | allows you to write your own exporters. 156 | -------------------------------------------------------------------------------- /docs/nbextensions/code-folding.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veit/jupyter-tutorial/24fcd76f43fc46ffd02e3bd98479a41ad6565197/docs/nbextensions/code-folding.png -------------------------------------------------------------------------------- /docs/nbextensions/configure-nbextensions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veit/jupyter-tutorial/24fcd76f43fc46ffd02e3bd98479a41ad6565197/docs/nbextensions/configure-nbextensions.png -------------------------------------------------------------------------------- /docs/nbextensions/create-plugin.rst: -------------------------------------------------------------------------------- 1 | Create plugin 2 | ============= 3 | 4 | In addition to the existing notebook extensions, other plugins can also be 5 | added. The directory in which ``jupyter_contrib_nbextensions/nbextensions`` is 6 | located can be found with ``pip show``: 7 | 8 | .. code-block:: console 9 | 10 | $ uv pip show jupyter_contrib_nbextensions 11 | Name: jupyter-contrib-nbextensions 12 | Version: 0.7.0 13 | Location: /Users/veit/cusy/trn/jupyter-tutorial/.venv/lib/python3.13/site-packages 14 | Requires: ipython-genutils, jupyter-contrib-core, jupyter-core, jupyter-highlight-selected-word, jupyter-nbextensions-configurator, lxml, nbconvert, notebook, tornado, traitlets 15 | Required-by: jupyter-tutorial 16 | 17 | This directory contains the individual notebook extensions, e.g. with the 18 | following structure: 19 | 20 | .. code-block:: console 21 | 22 | $ tree 23 | . 24 | ├── main.js 25 | ├── main.yaml 26 | └── readme.md 27 | 28 | ``main.js`` 29 | contains the actual logic of the extension, e.g .: 30 | 31 | .. code-block:: javascript 32 | 33 | define([ 34 | 'require', 35 | 'base/js/namespace', 36 | ], function ( 37 | requirejs 38 | $, 39 | Jupyter, 40 | ) { 41 | "use strict"; 42 | 43 | // define default values for config parameters 44 | var params = { 45 | my_config_value : 100 46 | }; 47 | 48 | var initialize = function () { 49 | $.extend(true, params, Jupyter.notebook.config.myextension); 50 | 51 | $('') 52 | .attr({ 53 | rel: 'stylesheet', 54 | type: 'text/css', 55 | href: requirejs.toUrl('./myextension.css') 56 | }) 57 | .appendTo('head'); 58 | }; 59 | 60 | var load_ipython_extension = function () { 61 | return Jupyter.notebook.config.loaded.then(initialize); 62 | }; 63 | 64 | return { 65 | load_ipython_extension : load_ipython_extension 66 | }; 67 | }); 68 | 69 | ``main.yaml`` 70 | `yaml `_ file that describes the 71 | extension for the Jupyter Extensions Configurator. 72 | 73 | .. code-block:: yaml 74 | 75 | Compatibility: 3.x, 4.x, 5.x, 6.x 76 | Name: My notebook extensions 77 | Main: main.js 78 | Link: README.md 79 | Description: | 80 | My notebook extension helps with the use of Jupyter notebooks. 81 | Parameters: 82 | - none 83 | 84 | More information about the options supported by the configurator can be 85 | found on GitHub: `jupyter_nbextensions_configurator 86 | `_. 87 | 88 | ``readme.md`` 89 | Markdown file that describes the extension and how it can be used. This is 90 | also displayed in the :menuselection:`Nbextensions` tab. 91 | 92 | .. seealso:: 93 | * :doc:`jupyter-contrib-nbextensions:internals` 94 | 95 | Setup Jupyter Notebook Extension 96 | -------------------------------- 97 | 98 | This is an extension that fixes some problems when working with notebooks that 99 | Joel Grus presented at JupyterCon 2018: `I Don’t Like Notebooks 100 | `_: 101 | 102 | * it asks you to name the notebook 103 | * it creates a template to improve the documentation 104 | * it imports and configures frequently used libraries 105 | 106 | Installation 107 | ~~~~~~~~~~~~ 108 | 109 | #. Find out where the notebook extensions are installed: 110 | 111 | .. code-block:: console 112 | 113 | $ uv pip show jupyter_contrib_nbextensions 114 | Name: jupyter-contrib-nbextensions 115 | Version: 0.7.0 116 | Location: /Users/veit/cusy/trn/jupyter-tutorial/.venv/lib/python3.13/site-packages 117 | Requires: ipython-genutils, jupyter-contrib-core, jupyter-core, jupyter-highlight-selected-word, jupyter-nbextensions-configurator, lxml, nbconvert, notebook, tornado, traitlets 118 | Required-by: jupyter-tutorial 119 | 120 | #. Download the `Setup 121 | `_ directory in 122 | ``jupyter_contrib_nbextensions/nbextensions/``. 123 | 124 | #. Install the extension with 125 | 126 | .. code-block:: console 127 | 128 | $ uv run --with jupyter jupyter contrib nbextensions install --user 129 | … 130 | [I 10:54:46 InstallContribNbextensionsApp] Installing /Users/veit/.local/share/virtualenvs/jupyter-tutorial--q5BvmfG/lib/python3.7/site-packages/jupyter_contrib_nbextensions/nbextensions/setup -> setup 131 | [I 10:54:46 InstallContribNbextensionsApp] Making directory: /Users/veit/Library/Jupyter/nbextensions/setup/ 132 | [I 10:54:46 InstallContribNbextensionsApp] Copying: /Users/veit/.local/share/virtualenvs/jupyter-tutorial--q5BvmfG/lib/python3.7/site-packages/jupyter_contrib_nbextensions/nbextensions/setup/setup.yaml -> /Users/veit/Library/Jupyter/nbextensions/setup/setup.yaml 133 | [I 10:54:46 InstallContribNbextensionsApp] Copying: /Users/veit/.local/share/virtualenvs/jupyter-tutorial--q5BvmfG/lib/python3.7/site-packages/jupyter_contrib_nbextensions/nbextensions/setup/README.md -> /Users/veit/Library/Jupyter/nbextensions/setup/README.md 134 | [I 10:54:46 InstallContribNbextensionsApp] Copying: /Users/veit/.local/share/virtualenvs/jupyter-tutorial--q5BvmfG/lib/python3.7/site-packages/jupyter_contrib_nbextensions/nbextensions/setup/main.js -> /Users/veit/Library/Jupyter/nbextensions/setup/main.js 135 | [I 10:54:46 InstallContribNbextensionsApp] - Validating: OK 136 | … 137 | 138 | #. Activate the *Setup* extension in :menuselection:`Nbextensions`. 139 | 140 | Finally you can create a new notebook, which then has the following structure: 141 | `setup.ipynb `_. 142 | 143 | .. seealso:: 144 | * `Set Your Jupyter Notebook up Right with this Extension 145 | `_ 146 | * `GitHub `_ 147 | -------------------------------------------------------------------------------- /docs/nbextensions/index.rst: -------------------------------------------------------------------------------- 1 | ``nbextensions`` 2 | ================ 3 | 4 | :doc:`Jupyter Notebook Extensions ` contains 5 | a collection of extensions. These are mostly written in Javascript and are 6 | loaded locally in your browser. 7 | 8 | .. seealso:: 9 | 10 | * :doc:`Docs ` 11 | * `Github `_ 12 | 13 | .. toctree:: 14 | :hidden: 15 | :titlesonly: 16 | :maxdepth: 0 17 | 18 | install 19 | list 20 | create-plugin 21 | setup.ipynb 22 | ipylayout.ipynb 23 | -------------------------------------------------------------------------------- /docs/nbextensions/install.rst: -------------------------------------------------------------------------------- 1 | Installation 2 | ============ 3 | 4 | #. Installation with Pipenv: 5 | 6 | .. code-block:: console 7 | 8 | $ uv add jupyter_contrib_nbextensions 9 | 10 | #. Installation of the associated Javascript and CSS files: 11 | 12 | .. code-block:: console 13 | 14 | $ uv run --with jupyter jupyter contrib nbextension install --user 15 | [I 20:57:19 InstallContribNbextensionsApp] jupyter contrib nbextension install --user 16 | [I 20:57:19 InstallContribNbextensionsApp] Installing jupyter_contrib_nbextensions nbextension files to jupyter data directory 17 | … 18 | [I 20:57:20 InstallContribNbextensionsApp] - Writing config: /Users/veit/.jupyter/jupyter_nbconvert_config.json 19 | [I 20:57:20 InstallContribNbextensionsApp] -- Writing updated config file /Users/veit/.jupyter/jupyter_nbconvert_config.json 20 | 21 | #. Check the installation: 22 | 23 | .. code-block:: console 24 | 25 | $ uv run --with jupyter jupyter nbextension list 26 | Known nbextensions: 27 | config dir: /Users/veit/.jupyter/nbconfig 28 | notebook section 29 | nbextensions_configurator/config_menu/main enabled 30 | - Validating: problems found: 31 | - require? X nbextensions_configurator/config_menu/main 32 | contrib_nbextensions_help_item/main enabled 33 | - Validating: OK 34 | tree section 35 | nbextensions_configurator/tree_tab/main enabled 36 | - Validating: problems found: 37 | - require? X nbextensions_configurator/tree_tab/main 38 | config dir: /Users/veit/.local/share/virtualenvs/jupyter-tutorial--q5BvmfG/bin/../etc/jupyter/nbconfig 39 | notebook section 40 | jupyter-js-widgets/extension enabled 41 | - Validating: OK 42 | 43 | #. Latex environments 44 | 45 | .. code-block:: console 46 | 47 | $ uv run --with jupyter jupyter nbextension install --py latex_envs --user 48 | Installing /Users/veit/.local/share/virtualenvs/jupyter-tutorial--q5BvmfG/lib/python3.7/site-packages/latex_envs/static -> latex_envs 49 | … 50 | - Validating: OK 51 | To initialize this nbextension in the browser every time the notebook (or other app) loads: 52 | jupyter nbextension enable latex_envs --user --py 53 | … 54 | $ uv run --with jupyter jupyter nbextension enable --py latex_envs --user 55 | Enabling notebook extension latex_envs/latex_envs… 56 | - Validating: OK 57 | 58 | #. `yapf `_ Code Prettyfier 59 | 60 | … for Python: 61 | 62 | .. code-block:: console 63 | 64 | $ uv add yapf 65 | 66 | … for Javascript: 67 | 68 | .. code-block:: console 69 | 70 | $ npm install js-beautify 71 | … 72 | + js-beautify@1.10.0 73 | added 29 packages from 21 contributors and audited 32 packages in 2.632s 74 | found 0 vulnerabilities 75 | 76 | … for R: 77 | 78 | .. code-block:: console 79 | 80 | $ Rscript -e 'install.packages(c("formatR", "jsonlite"), repos="http://cran.rstudio.com")' 81 | Installiere Pakete nach ‘/usr/local/lib/R/3.6/site-library’ 82 | … 83 | 84 | #. Highlighter 85 | 86 | .. code-block:: console 87 | 88 | $ uv run --with jupyter jupyter nbextension install https://rawgit.com/jfbercher/small_nbextensions/master/highlighter.zip --user 89 | Downloading: https://rawgit.com/jfbercher/small_nbextensions/master/highlighter.zip -> /var/folders/_4/cs4t3m8d4ys8lcs67r3lghtw0000gn/T/tmpn9qrcrdz/highlighter.zip 90 | Extracting: /var/folders/_4/cs4t3m8d4ys8lcs67r3lghtw0000gn/T/tmpn9qrcrdz/highlighter.zip -> /Users/veit/Library/Jupyter/nbextensions 91 | $ uv run --with jupyter jupyter nbextension enable highlighter/highlighter 92 | Enabling notebook extension highlighter/highlighter… 93 | - Validating: OK 94 | 95 | #. nbTranslate 96 | 97 | .. code-block:: console 98 | 99 | $ uv add jupyter_latex_envs --upgrade --user 100 | Installing jupyter_latex_envs… 101 | … 102 | $ uv run --with jupyter jupyter nbextension install --py latex_envs --user 103 | Installing /srv/jupyter/.local/share/virtualenvs/jupyterhub-aFv4x91W/lib/python3.5/site-packages/latex_envs/static -> latex_envs 104 | … 105 | $ uv run --with jupyter jupyter nbextension enable --py latex_envs 106 | -------------------------------------------------------------------------------- /docs/nbextensions/latex-env.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veit/jupyter-tutorial/24fcd76f43fc46ffd02e3bd98479a41ad6565197/docs/nbextensions/latex-env.png -------------------------------------------------------------------------------- /docs/nbextensions/list.rst: -------------------------------------------------------------------------------- 1 | List of extensions 2 | ================== 3 | 4 | You can activate and configure the notebook extensions by clicking on the 5 | :menuselection:`Nbextensions` tab. There you have access to the extensions, 6 | which can be activated/deactivated via checkboxes. In addition, documentation 7 | and configuration options are displayed for each extension. 8 | 9 | .. image:: configure-nbextensions.png 10 | :alt: Configuration of the notebook extensions 11 | 12 | Hereinafter I will give a brief overview of some of the notebook extensions. 13 | 14 | (some) LaTeX environments for Jupyter notebook 15 | enables the use of Markdown cells for LaTeX commands and environments. In 16 | addition, two menus are added: :menuselection:`LaTeX_envs` for quick 17 | selection of the suitable LaTeX environment and :menuselection:`Some 18 | configuration options` for further options: 19 | 20 | .. image:: latex-env.png 21 | :alt: LaTeX environment 22 | 23 | The notebook can then be exported as an HTML or LaTeX document. 24 | 25 | The configuration of the LaTeX environments is done in ``user_envs.json`` 26 | and for the styles in ``latex_env.css``. Additional environments can be 27 | added in ``user_envs.json`` or in ``thmsInNb4.js`` (→ `LaTeX-Environments 28 | doc 29 | `_). 30 | 31 | :doc:`jupyter-contrib-nbextensions:nbextensions/code_prettify/README_autopep8` 32 | formats/beautifies Python code in cells. The extension uses `autopep8 33 | `_ and can therefore only be used with 34 | a Python kernel. 35 | 36 | :doc:`jupyter-contrib-nbextensions:nbextensions/code_prettify/README_code_prettify` 37 | formats/beautifies code notebook code cells. The current notebook kernel is 38 | used, which is why the Prettifier package used must be available in this 39 | kernel. Sample implementations are provided for ipython, R, and Javascript 40 | kernels. 41 | 42 | :doc:`jupyter-contrib-nbextensions:nbextensions/limit_output/readme` 43 | limits the number of characters that a code cell outputs as text or HTML. 44 | This also breaks endless loops. You can set the number of characters with 45 | the ``ConfigManager``: 46 | 47 | .. code-block:: 48 | 49 | from notebook.services.config import ConfigManager 50 | cm = ConfigManager().update('notebook', {'limit_output': 1000}) 51 | 52 | `Nbextensions edit menu item `_ 53 | adds an edit menu to open the configuration page of ``nbextensions``. 54 | 55 | :doc:`jupyter-contrib-nbextensions:nbextensions/printview/readme` 56 | adds an icon to display the print preview of the current notebook in a new 57 | browser tab. 58 | 59 | :doc:`jupyter-contrib-nbextensions:nbextensions/ruler/readme` 60 | adds a ruler after a certain number of characters. The number of characters 61 | can be specified with the ``ConfigManager``: 62 | 63 | .. code-block:: 64 | 65 | from notebook.services.config import ConfigManager 66 | ip = get_ipython() 67 | cm = ConfigManager(parent=ip) 68 | cm.update('notebook', {"ruler_column": [80]}) 69 | 70 | :doc:`jupyter-contrib-nbextensions:nbextensions/scratchpad/README` 71 | adds a note cell to the notebook. In this cell you can run code from the 72 | current kernel without changing the document. 73 | 74 | :doc:`jupyter-contrib-nbextensions:nbextensions/snippets/README` 75 | adds a configurable menu item to notebooks to insert snippets, 76 | boilerplate and code examples. 77 | 78 | .. image:: snippets-menu.png 79 | :alt: Snippets menu 80 | 81 | You can also define your own menu items, see also 82 | :doc:`jupyter-contrib-nbextensions:nbextensions/snippets/README`. 83 | 84 | :doc:`jupyter-contrib-nbextensions:nbextensions/toc2/README` 85 | makes it possible to collect all headings and display them in a floating 86 | window, as a sidebar or in a navigation menu. 87 | 88 | If headings shouldn’t be displayed in the table of contents, you can do this 89 | with: 90 | 91 | .. code-block:: HTML 92 | 93 | ## My title 94 | 95 | The table of contents can also be exported by specifying a corresponding 96 | template, e.g. 97 | 98 | .. code-block:: console 99 | 100 | $ jupyter nbconvert mynotebook.ipynb --template toc2 101 | 102 | General documentation on templates can be found in 103 | :label:`nbconvert:external_exporters`. 104 | 105 | :doc:`jupyter-contrib-nbextensions:nbextensions/tree-filter/readme` 106 | filters in the Jupyter dashboard by file name. 107 | 108 | :doc:`jupyter-contrib-nbextensions:nbextensions/code_prettify/README_2to3` 109 | converts Python2 to Python3 code in a code cell using the `lib2to3 110 | `_ library. 111 | 112 | :doc:`jupyter-contrib-nbextensions:nbextensions/codefolding/readme` 113 | enables code folding in code cells. 114 | 115 | .. image:: code-folding.png 116 | :alt: Codefolding 117 | 118 | Usually code folding is retained when exporting with nbconvert. This can 119 | either be changed in ``jupyter_nbconvert_config.py`` with: 120 | 121 | .. code-block:: python 122 | 123 | c.CodeFoldingPreprocessor.remove_folded_code = True 124 | 125 | or on the command line with: 126 | 127 | .. code-block:: console 128 | 129 | $ jupyter nbconvert --to html --CodeFoldingPreprocessor.remove_folded_code=True mynotebook.ipynb 130 | 131 | :doc:`jupyter-contrib-nbextensions:nbextensions/collapsible_headings/readme` 132 | enables notebooks to have collapsible sections separated by headings. 133 | 134 | :doc:`jupyter-contrib-nbextensions:nbextensions/datestamper/readme` 135 | inserts the current time and date in one cell. 136 | 137 | :doc:`jupyter-contrib-nbextensions:nbextensions/hinterland/README` 138 | enables autocompletion. 139 | 140 | :doc:`jupyter-contrib-nbextensions:nbextensions/varInspector/README` 141 | collects all defined variables and displays them in a floating window. 142 | 143 | :doc:`jupyter-contrib-nbextensions:nbextensions/load_tex_macros/readme` 144 | automatically loads a series of latex commands from the ``latexdefs.tex`` 145 | file when a notebook is opened. 146 | -------------------------------------------------------------------------------- /docs/nbextensions/setup.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# `setup.ipynb`" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "## Introduction\n", 15 | "State notebook purpose here" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": {}, 21 | "source": [ 22 | "### Imports\n", 23 | "Import libraries and write settings here." 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": 1, 29 | "metadata": {}, 30 | "outputs": [ 31 | { 32 | "data": { 33 | "text/html": [ 34 | " \n", 49 | " " 50 | ] 51 | }, 52 | "metadata": {}, 53 | "output_type": "display_data" 54 | }, 55 | { 56 | "data": { 57 | "text/html": [ 58 | " \n", 73 | " " 74 | ] 75 | }, 76 | "metadata": {}, 77 | "output_type": "display_data" 78 | } 79 | ], 80 | "source": [ 81 | "# Data manipulation\n", 82 | "import pandas as pd\n", 83 | "import numpy as np\n", 84 | "\n", 85 | "# Options for pandas\n", 86 | "pd.options.display.max_columns = 50\n", 87 | "pd.options.display.max_rows = 30\n", 88 | "\n", 89 | "# Display all cell outputs\n", 90 | "from IPython.core.interactiveshell import InteractiveShell\n", 91 | "\n", 92 | "InteractiveShell.ast_node_interactivity = \"all\"\n", 93 | "\n", 94 | "from IPython import get_ipython\n", 95 | "\n", 96 | "ipython = get_ipython()\n", 97 | "\n", 98 | "# autoreload extension\n", 99 | "if \"autoreload\" not in ipython.extension_manager.loaded:\n", 100 | " %load_ext autoreload\n", 101 | "\n", 102 | "%autoreload 2\n", 103 | "\n", 104 | "# Visualizations\n", 105 | "import chart_studio.plotly as py\n", 106 | "import plotly.graph_objs as go\n", 107 | "from plotly.offline import iplot, init_notebook_mode\n", 108 | "\n", 109 | "init_notebook_mode(connected=True)\n", 110 | "\n", 111 | "import cufflinks as cf\n", 112 | "\n", 113 | "cf.go_offline(connected=True)\n", 114 | "cf.set_config_file(theme=\"white\")" 115 | ] 116 | }, 117 | { 118 | "cell_type": "markdown", 119 | "metadata": {}, 120 | "source": [ 121 | "## Analysis/Modeling\n", 122 | "Do work here" 123 | ] 124 | }, 125 | { 126 | "cell_type": "markdown", 127 | "metadata": {}, 128 | "source": [ 129 | "## Results\n", 130 | "Show graphs and stats here" 131 | ] 132 | }, 133 | { 134 | "cell_type": "markdown", 135 | "metadata": {}, 136 | "source": [ 137 | "## Conclusions and Next Steps\n", 138 | "Summarize findings here" 139 | ] 140 | } 141 | ], 142 | "metadata": { 143 | "kernelspec": { 144 | "display_name": "Python 3.11 Kernel", 145 | "language": "python", 146 | "name": "python311" 147 | }, 148 | "language_info": { 149 | "codemirror_mode": { 150 | "name": "ipython", 151 | "version": 3 152 | }, 153 | "file_extension": ".py", 154 | "mimetype": "text/x-python", 155 | "name": "python", 156 | "nbconvert_exporter": "python", 157 | "pygments_lexer": "ipython3", 158 | "version": "3.11.4" 159 | }, 160 | "latex_envs": { 161 | "LaTeX_envs_menu_present": true, 162 | "autoclose": false, 163 | "autocomplete": true, 164 | "bibliofile": "biblio.bib", 165 | "cite_by": "apalike", 166 | "current_citInitial": 1, 167 | "eqLabelWithNumbers": true, 168 | "eqNumInitial": 1, 169 | "hotkeys": { 170 | "equation": "Ctrl-E", 171 | "itemize": "Ctrl-I" 172 | }, 173 | "labels_anchors": false, 174 | "latex_user_defs": false, 175 | "report_style_numbering": false, 176 | "user_envs_cfg": false 177 | }, 178 | "varInspector": { 179 | "cols": { 180 | "lenName": 16, 181 | "lenType": 16, 182 | "lenVar": 40 183 | }, 184 | "kernels_config": { 185 | "python": { 186 | "delete_cmd_postfix": "", 187 | "delete_cmd_prefix": "del ", 188 | "library": "var_list.py", 189 | "varRefreshCmd": "print(var_dic_list())" 190 | }, 191 | "r": { 192 | "delete_cmd_postfix": ") ", 193 | "delete_cmd_prefix": "rm(", 194 | "library": "var_list.r", 195 | "varRefreshCmd": "cat(var_dic_list()) " 196 | } 197 | }, 198 | "types_to_exclude": [ 199 | "module", 200 | "function", 201 | "builtin_function_or_method", 202 | "instance", 203 | "_Feature" 204 | ], 205 | "window_display": false 206 | }, 207 | "widgets": { 208 | "application/vnd.jupyter.widget-state+json": { 209 | "state": {}, 210 | "version_major": 2, 211 | "version_minor": 0 212 | } 213 | } 214 | }, 215 | "nbformat": 4, 216 | "nbformat_minor": 2 217 | } 218 | -------------------------------------------------------------------------------- /docs/nbextensions/snippets-menu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veit/jupyter-tutorial/24fcd76f43fc46ffd02e3bd98479a41ad6565197/docs/nbextensions/snippets-menu.png -------------------------------------------------------------------------------- /docs/nbextensions/voila-ipylayout.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veit/jupyter-tutorial/24fcd76f43fc46ffd02e3bd98479a41ad6565197/docs/nbextensions/voila-ipylayout.png -------------------------------------------------------------------------------- /docs/nbviewer.rst: -------------------------------------------------------------------------------- 1 | ``nbviewer`` 2 | ============ 3 | 4 | `nbviewer `_ 5 | :doc:`nbconvert` as web service: Renders Jupyter notebooks as static web 6 | pages. 7 | 8 | Installation 9 | ------------ 10 | 11 | #. The Notebook Viewer requires several binary packages that have to be 12 | installed on our system, for 13 | 14 | .. tab:: Debian/Ubuntu 15 | 16 | .. code-block:: console 17 | 18 | $ sudo apt install libmemcached-dev libcurl4-openssl-dev pandoc libevent-dev 19 | 20 | .. tab:: macOS 21 | 22 | .. code-block:: console 23 | 24 | $ brew install libmemcached openssl pandoc libevent 25 | 26 | #. The Jupyter Notebook Viewer can then be installed in a new virtual 27 | environment with: 28 | 29 | .. code-block:: console 30 | 31 | $ mkdir nbviewer 32 | $ cd !$ 33 | cd nbviewer 34 | 35 | .. note:: 36 | The notebook app outputs the error ``AttributeError: module 37 | 'tornado.gen' has no attribute 'Task'`` with current versions of 38 | `Tornado `_. This error does not 39 | occur with ``tornado<6.0``, , see also `Delete Terminal Not Working 40 | with Tornado version 6.0.1 41 | `_: 42 | 43 | .. code-block:: console 44 | 45 | $ uv add "tornado<6.0" 46 | 47 | Now ``nbviewer`` can also be installed: 48 | 49 | .. code-block:: console 50 | 51 | $ uv add nbviewer 52 | 53 | #. For testing, the server can be started with: 54 | 55 | .. code-block:: console 56 | 57 | $ uv run python -m nbviewer --debug --no-cache 58 | 59 | Extending the Notebook Viewer 60 | ----------------------------- 61 | 62 | The notebook viewer can be extended to include providers, see 63 | `Extending the Notebook Viewer 64 | `_. 65 | 66 | 67 | Access control 68 | -------------- 69 | 70 | If the viewer is run as :doc:`hub/nbviewer-service`, only users who have 71 | authenticated themselves on the JupyterHub can access the nbviewer’s notebooks. 72 | -------------------------------------------------------------------------------- /docs/notebook/config.rst: -------------------------------------------------------------------------------- 1 | Jupyter paths and configuration 2 | =============================== 3 | 4 | Configuration files are usually stored in the :file:`~/.jupyter` directory. 5 | However, another directory can be specified with the environment variable 6 | :envvar:`JUPYTER_CONFIG_DIR`. If Jupyter cannot find a configuration in 7 | :envvar:`JUPYTER_CONFIG_DIR`, Jupyter runs through the search path with 8 | :file:`/{SYS.PREFIX}/etc/jupyter/` and then for Unix 9 | :file:`/usr/local/etc/jupyter/` and :file:`/etc/jupyter/`, for Windows 10 | :file:`%PROGRAMDATA%\\jupyter\\`. 11 | 12 | You can have the currently used configuration directories listed with: 13 | 14 | .. code-block:: console 15 | 16 | $ jupyter --paths 17 | config: 18 | /Users/veit/.jupyter 19 | /Users/veit/.local/share/virtualenvs/jupyter-tutorial--q5BvmfG/bin/../etc/jupyter 20 | /usr/local/etc/jupyter 21 | /etc/jupyter 22 | ... 23 | 24 | Create the configuration files 25 | ------------------------------ 26 | 27 | You can create a standard configuration with: 28 | 29 | .. code-block:: console 30 | 31 | $ jupyter notebook --generate-config 32 | Writing default config to: /Users/veit/.jupyter/jupyter_notebook_config.py 33 | 34 | More generally, configuration files can be created for all Jupyter applications 35 | with :samp:`jupyter {APPLICATION} --generate-config`. 36 | 37 | Change the configuration 38 | ------------------------ 39 | 40 | … by editing the configuration file 41 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 42 | 43 | e.g. in ``jupyter_notebook_config.py``: 44 | 45 | .. code-block:: python 46 | 47 | c.NotebookApp.port = 8754 48 | 49 | If the values are saved as ``list``, ``dict`` or ``set``, they can also be 50 | supplemented with ``append``, ``extend``, ``prepend``, ``add`` and 51 | ``update``, e.g.: 52 | 53 | .. code-block:: python 54 | 55 | c.TemplateExporter.template_path.append("./templates") 56 | 57 | … with the command line 58 | ~~~~~~~~~~~~~~~~~~~~~~~ 59 | 60 | for example: 61 | 62 | .. code-block:: console 63 | 64 | $ jupyter notebook --NotebookApp.port=8754 65 | 66 | There are aliases for frequently used options such as for ``--port`` or 67 | ``--no-browser``. 68 | 69 | The command line options override options set in a configuration file. 70 | 71 | .. seealso:: 72 | `traitlets.config 73 | `_ 74 | -------------------------------------------------------------------------------- /docs/notebook/create-notebook.rst: -------------------------------------------------------------------------------- 1 | Create notebook 2 | =============== 3 | 4 | After the notebook server has started, we can create our first notebook. 5 | 6 | Create a notebook 7 | ----------------- 8 | 9 | In your standard browser you should see the notebook dashboard with the *New* 10 | menu on the right. All notebook kernels are listed in this menu, but initially 11 | probably only *Python 3*. 12 | 13 | After you have selected :menuselection:`New --> Python 3`, a new notebook 14 | ``Untitled.ipynb`` will be created and displayed in a new tab: 15 | 16 | .. image:: initial-notebook.png 17 | 18 | Renaming the notebook 19 | --------------------- 20 | 21 | Next you should rename this notebook by clicking on the title *Untitled*: 22 | 23 | .. image:: rename-notebook.png 24 | 25 | The notebook user interface 26 | --------------------------- 27 | 28 | There are two important terms used to describe Jupyter Notebooks: *cell* and 29 | *kernel*: 30 | 31 | .. glossary:: 32 | 33 | *Notebook kernel* 34 | *Computational engine* that executes the code contained in a notebook. 35 | 36 | *Notebook cell* 37 | Container for text to be displayed in a notebook or for code to be 38 | executed by the notebook’s kernel. 39 | 40 | *Code* 41 | contains code to be executed in the kernel, and the output which is 42 | shown below. 43 | 44 | In front of the code cells are brackets that indicate the order in 45 | which the code was executed. 46 | 47 | ``In [ ]:`` 48 | indicates that the code has not yet been executed. 49 | ``In [*]:`` 50 | indicates that the execution has not yet been completed. 51 | 52 | .. warning:: 53 | The output of cells can be used in other cells later. Therefore, 54 | the result depends on the order. If you choose a different order 55 | than the one from top to bottom, you may get different results 56 | later when you e.g. select :menuselection:`Cell --> Run All`. 57 | 58 | *Markdown* 59 | contains text formatted with `Markdown 60 | `_, which is 61 | interpreted as soon as :menuselection:`Run` is pressed. 62 | 63 | .. _whats-an-ipynb-file: 64 | 65 | What’s an ``ipynb`` file? 66 | ------------------------- 67 | 68 | This file describes a notebook in 69 | :doc:`python4datascience:data-processing/serialisation-formats/json/index` 70 | format. Each cell and its contents including pictures are listed there along 71 | with some metadata. You can have a look at them if you select the notebook in 72 | the dashboard and then click on :menuselection:`edit`. For example the JSON file 73 | for :file:`my-first-notebook.ipynb` looks like this: 74 | 75 | .. code-block:: json 76 | 77 | { 78 | "cells": [ 79 | { 80 | "cell_type": "markdown", 81 | "metadata": {}, 82 | "source": [ 83 | "# My first notebook" 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": 1, 89 | "metadata": {}, 90 | "outputs": [ 91 | { 92 | "name": "stdout", 93 | "output_type": "stream", 94 | "text": [ 95 | "Hello World!\n" 96 | ] 97 | } 98 | ], 99 | "source": [ 100 | "print('Hello World!')" 101 | ] 102 | } 103 | ], 104 | "metadata": { 105 | "kernelspec": { 106 | "display_name": "Python 3", 107 | "language": "python", 108 | "name": "python3" 109 | }, 110 | "language_info": { 111 | "codemirror_mode": { 112 | "name": "ipython", 113 | "version": 3 114 | }, 115 | "file_extension": ".py", 116 | "mimetype": "text/x-python", 117 | "name": "python", 118 | "nbconvert_exporter": "python", 119 | "pygments_lexer": "ipython3", 120 | "version": "3.7.0" 121 | } 122 | }, 123 | "nbformat": 4, 124 | "nbformat_minor": 2 125 | } 126 | 127 | Save and checkpoints 128 | -------------------- 129 | 130 | When you click on :menuselection:`Save and Checkpoint`, your :file:`*.ipynb` 131 | file will be saved. But what is the checkpoint all about? 132 | 133 | Every time you create a new notebook, a file is also created, which usually 134 | automatically saves your changes every 120 seconds. This checkpoint is usually 135 | located in a hidden directory called :file:`.ipynb_checkpoints/`. This 136 | checkpoint file therefore enables you to restore your unsaved data in the event 137 | of an unexpected problem. You can go back to one of the last checkpoints in 138 | :menuselection:`File --> Revert to Checkpoint`. 139 | 140 | Tips and tricks 141 | --------------- 142 | 143 | #. Give the notebook a title (:samp:`# {MY TITLE}`) and a meaningful foreword to 144 | describe the content and purpose of the notebook. 145 | #. Create headings and documentation in Markdown cells to structure your 146 | notebook and explain your workflow steps. It doesn’t matter whether you do 147 | this for your colleagues or for yourself in the future. 148 | #. Use *Table of Contents (2)* from the :doc:`../nbextensions/list` to 149 | create a table of contents. 150 | #. Use the notebook extension :doc:`setup <../nbextensions/setup>`. 151 | #. Use snippets from the list of extensions to add more frequently used code 152 | blocks, for example typical import instructions, easy to insert. 153 | -------------------------------------------------------------------------------- /docs/notebook/index.rst: -------------------------------------------------------------------------------- 1 | Notebook 2 | ======== 3 | 4 | Jupyter Notebooks extend the console-based approach to interactive computing 5 | with a web-based application, with which the entire process can be recorded: 6 | from developing and executing the code to documenting and presenting the 7 | results. 8 | 9 | .. toctree:: 10 | :hidden: 11 | :titlesonly: 12 | :maxdepth: 0 13 | 14 | install 15 | create-notebook 16 | shortcuts 17 | config 18 | parameterise/index 19 | testing/index 20 | -------------------------------------------------------------------------------- /docs/notebook/initial-jupyter-dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veit/jupyter-tutorial/24fcd76f43fc46ffd02e3bd98479a41ad6565197/docs/notebook/initial-jupyter-dashboard.png -------------------------------------------------------------------------------- /docs/notebook/initial-notebook.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veit/jupyter-tutorial/24fcd76f43fc46ffd02e3bd98479a41ad6565197/docs/notebook/initial-notebook.png -------------------------------------------------------------------------------- /docs/notebook/install.rst: -------------------------------------------------------------------------------- 1 | Install Jupyter Notebook 2 | ======================== 3 | 4 | Create a virtual environment with ``jupyter`` 5 | --------------------------------------------- 6 | 7 | :term:`Python virtual environments ` allow Python packages 8 | to be installed in an isolated location for a specific application, rather than 9 | installing them globally. So you have your own installation directories and do 10 | not share libraries with other virtual environments: 11 | 12 | .. code-block:: console 13 | 14 | $ python3 -m venv myproject 15 | $ cd myproject 16 | $ . bin/activate 17 | $ python -m pip install jupyter 18 | 19 | Start ``jupyter notebook`` 20 | -------------------------- 21 | 22 | .. code-block:: console 23 | 24 | $ jupyter notebook 25 | ... 26 | [I 12:46:53.852 NotebookApp] The Jupyter Notebook is running at: 27 | [I 12:46:53.852 NotebookApp] http://localhost:8888/?token=53abd45a3002329de77f66886e4ca02539d664c2f5e6072e 28 | [I 12:46:53.852 NotebookApp] Use Control-C to stop this server and shut down all kernels (twice to skip confirmation). 29 | [C 12:46:53.858 NotebookApp] 30 | 31 | To access the notebook, open this file in a browser: 32 | file:///Users/veit/Library/Jupyter/runtime/nbserver-7372-open.html 33 | Or copy and paste one of these URLs: 34 | http://localhost:8888/?token=53abd45a3002329de77f66886e4ca02539d664c2f5e6072e 35 | 36 | Your standard web browser will then open with this URL. 37 | 38 | When the notebook opens in your browser, the notebook dashboard is displayed 39 | with a list of the notebooks, files and subdirectories in the directory in which 40 | the notebook server was started. In most cases you want to start a notebook 41 | server in your project directory. 42 | 43 | .. image:: initial-jupyter-dashboard.png 44 | -------------------------------------------------------------------------------- /docs/notebook/parameterise/index.rst: -------------------------------------------------------------------------------- 1 | Parameterisation and scheduling 2 | =============================== 3 | 4 | With :doc:`../../jupyterlab/index` you can use the :doc:`Jupyter Scheduler 5 | <../../jupyterlab/scheduler>` for parameterisation and time-controlled 6 | execution. For Jupyter Notebooks, `papermill 7 | `_ is available. 8 | 9 | Install 10 | ------- 11 | 12 | .. code-block:: console 13 | 14 | $ uv add papermill 15 | 16 | Use 17 | --- 18 | 19 | #. Parameterise 20 | 21 | The first step is to parameterise the notebook. For this purpose the cells are 22 | tagged as `parameters` in :menuselection:`View --> Cell Toolbar --> Tags`. 23 | 24 | #. Inspect 25 | 26 | You can inspect a notebook for example with: 27 | 28 | .. code-block:: console 29 | 30 | $ uv run papermill --help-notebook docs/notebook/parameterise/input.ipynb 31 | Usage: papermill [OPTIONS] NOTEBOOK_PATH [OUTPUT_PATH] 32 | 33 | Parameters inferred for notebook 'docs/notebook/parameterise/input.ipynb': 34 | msg: Unknown type (default None) 35 | 36 | #. Execute 37 | 38 | There are two ways to run a notebook with parameters: 39 | 40 | * … via the Python API 41 | 42 | The ``execute_notebook`` function can be called to execute a notebook with 43 | a dict of parameters: 44 | 45 | .. code-block:: python 46 | 47 | execute_notebook(INPUT_NOTEBOOK, OUTPUT_NOTEBOOK, DICTIONARY_OF_PARAMETERS) 48 | 49 | for example for :file:`input.ipynb`: 50 | 51 | .. code-block:: ipython 52 | 53 | In [1]: import papermill as pm 54 | 55 | In [2]: pm.execute_notebook( 56 | "PATH/TO/INPUT_NOTEBOOK.ipynb", 57 | "PATH/TO/OUTPUT_NOTEBOOK.ipynb", 58 | parameters=dict(salutation="Hello", name="pythonistas"), 59 | ) 60 | 61 | The result is :file:`output.ipynb`: 62 | 63 | .. code-block:: ipython 64 | 65 | In [1]: salutation = None 66 | name = None 67 | 68 | In [2]: # Parameters 69 | salutation = "Hello" 70 | name = "pythonistas" 71 | 72 | In [3]: from datetime import date 73 | 74 | 75 | today = date.today() 76 | print( 77 | salutation, 78 | name, 79 | "– welcome to our event on this " + today.strftime("%A, %d %B %Y"), 80 | ) 81 | 82 | Out[3]: Hello pythonistas – welcome to our event on this Monday, 26 June 2023 83 | 84 | .. seealso:: 85 | * `Workflow reference 86 | `_ 87 | 88 | * … via CLI 89 | 90 | .. code-block:: console 91 | 92 | $ uv run papermill input.ipynb output.ipynb -p salutation 'Hello' -p name 'pythonistas' 93 | 94 | Alternatively, a YAML file can be specified with the parameters, for 95 | example :file:`params.yaml`: 96 | 97 | .. literalinclude:: params.yaml 98 | :caption: params.yaml 99 | :name: params.yaml 100 | 101 | .. code-block:: console 102 | 103 | $ uv run papermill input.ipynb output.ipynb -f params.yaml 104 | 105 | With ``-b``, a base64-encoded YAML string can be provided, containing the 106 | parameter values: 107 | 108 | .. code-block:: console 109 | 110 | $ uv run papermill input.ipynb output.ipynb -b c2FsdXRhdGlvbjogIkhlbGxvIgpuYW1lOiAiUHl0aG9uaXN0YXMi 111 | 112 | .. seealso:: 113 | * `CLI reference 114 | `_ 115 | 116 | You can also add a timestamp to the file name: 117 | 118 | .. code-block:: console 119 | 120 | $ dt=$(date '+%Y-%m-%d_%H:%M:%S') 121 | $ uv run papermill input.ipynb output_$(date '+%Y-%m-%d_%H:%M:%S').ipynb -f params.yaml 122 | 123 | This creates an output file whose file name contains a timestamp, for 124 | example :download:`output_2023-06-26_15:57:33.ipynb`. 125 | 126 | Finally, you can use ``crontab -e`` to execute the two commands 127 | automatically at certain times, for example on the first day of every 128 | month: 129 | 130 | .. code-block:: 131 | 132 | dt=$(date '+%Y-%m-%d_%H:%M:%S') 133 | 0 0 1 * * cd ~/jupyter-notebook && uv run papermill input.ipynb output_$(date '+%Y-%m-%d_%H:%M:%S').ipynb -f params.yaml 134 | 135 | #. Store 136 | 137 | Papermill can store notebooks in a number of locations including S3, Azure 138 | data blobs, and Azure data lakes. Papermill allows new data stores to be 139 | added over time. 140 | 141 | .. seealso:: 142 | * `papermill Storage 143 | `_ 144 | * `Extending papermill through entry points 145 | `_ 146 | -------------------------------------------------------------------------------- /docs/notebook/parameterise/input.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Notebook with parameters" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": { 14 | "tags": [ 15 | "parameters" 16 | ] 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "salutation = None\n", 21 | "name = None" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 2, 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "from datetime import date\n", 31 | "\n", 32 | "\n", 33 | "today = date.today()" 34 | ] 35 | }, 36 | { 37 | "cell_type": "code", 38 | "execution_count": 3, 39 | "metadata": {}, 40 | "outputs": [ 41 | { 42 | "name": "stdout", 43 | "output_type": "stream", 44 | "text": [ 45 | "None None – welcome to our event on this Monday, 26 June 2023\n" 46 | ] 47 | } 48 | ], 49 | "source": [ 50 | "print(\n", 51 | " salutation,\n", 52 | " name,\n", 53 | " \"– welcome to our event on this \" + today.strftime(\"%A, %d %B %Y\"),\n", 54 | ")" 55 | ] 56 | } 57 | ], 58 | "metadata": { 59 | "kernelspec": { 60 | "display_name": "Python 3.11 Kernel", 61 | "language": "python", 62 | "name": "python311" 63 | }, 64 | "language_info": { 65 | "codemirror_mode": { 66 | "name": "ipython", 67 | "version": 3 68 | }, 69 | "file_extension": ".py", 70 | "mimetype": "text/x-python", 71 | "name": "python", 72 | "nbconvert_exporter": "python", 73 | "pygments_lexer": "ipython3", 74 | "version": "3.11.4" 75 | }, 76 | "latex_envs": { 77 | "LaTeX_envs_menu_present": true, 78 | "autoclose": false, 79 | "autocomplete": true, 80 | "bibliofile": "biblio.bib", 81 | "cite_by": "apalike", 82 | "current_citInitial": 1, 83 | "eqLabelWithNumbers": true, 84 | "eqNumInitial": 1, 85 | "hotkeys": { 86 | "equation": "Ctrl-E", 87 | "itemize": "Ctrl-I" 88 | }, 89 | "labels_anchors": false, 90 | "latex_user_defs": false, 91 | "report_style_numbering": false, 92 | "user_envs_cfg": false 93 | } 94 | }, 95 | "nbformat": 4, 96 | "nbformat_minor": 4 97 | } 98 | -------------------------------------------------------------------------------- /docs/notebook/parameterise/output.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "31de0394", 6 | "metadata": { 7 | "papermill": { 8 | "duration": 0.00946, 9 | "end_time": "2021-09-13T08:28:28.753607", 10 | "exception": false, 11 | "start_time": "2021-09-13T08:28:28.744147", 12 | "status": "completed" 13 | }, 14 | "tags": [] 15 | }, 16 | "source": [ 17 | "# Notebook with parameters" 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": 1, 23 | "id": "de513186", 24 | "metadata": { 25 | "papermill": { 26 | "duration": 0.026798, 27 | "end_time": "2021-09-13T08:28:28.787199", 28 | "exception": false, 29 | "start_time": "2021-09-13T08:28:28.760401", 30 | "status": "completed" 31 | }, 32 | "tags": [ 33 | "parameters" 34 | ] 35 | }, 36 | "outputs": [], 37 | "source": [ 38 | "salutation = None\n", 39 | "name = None" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": 2, 45 | "id": "42dba07c", 46 | "metadata": { 47 | "papermill": { 48 | "duration": 0.014807, 49 | "end_time": "2021-09-13T08:28:28.809717", 50 | "exception": false, 51 | "start_time": "2021-09-13T08:28:28.794910", 52 | "status": "completed" 53 | }, 54 | "tags": [ 55 | "injected-parameters" 56 | ] 57 | }, 58 | "outputs": [], 59 | "source": [ 60 | "# Parameters\n", 61 | "salutation = \"Hello\"\n", 62 | "name = \"pythonistas\"" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": 3, 68 | "id": "0695aab7", 69 | "metadata": { 70 | "papermill": { 71 | "duration": 0.012479, 72 | "end_time": "2021-09-13T08:28:28.827883", 73 | "exception": false, 74 | "start_time": "2021-09-13T08:28:28.815404", 75 | "status": "completed" 76 | }, 77 | "tags": [] 78 | }, 79 | "outputs": [], 80 | "source": [ 81 | "from datetime import date\n", 82 | "\n", 83 | "\n", 84 | "today = date.today()" 85 | ] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "execution_count": 4, 90 | "id": "2989bbe3", 91 | "metadata": { 92 | "papermill": { 93 | "duration": 0.011712, 94 | "end_time": "2021-09-13T08:28:28.844012", 95 | "exception": false, 96 | "start_time": "2021-09-13T08:28:28.832300", 97 | "status": "completed" 98 | }, 99 | "tags": [] 100 | }, 101 | "outputs": [ 102 | { 103 | "name": "stdout", 104 | "output_type": "stream", 105 | "text": [ 106 | "Hello pythonistas – welcome to our event on this Monday, 26 June 2023\n" 107 | ] 108 | } 109 | ], 110 | "source": [ 111 | "print(\n", 112 | " salutation,\n", 113 | " name,\n", 114 | " \"– welcome to our event on this \" + today.strftime(\"%A, %d %B %Y\"),\n", 115 | ")" 116 | ] 117 | } 118 | ], 119 | "metadata": { 120 | "kernelspec": { 121 | "display_name": "Python 3.11 Kernel", 122 | "language": "python", 123 | "name": "python311" 124 | }, 125 | "language_info": { 126 | "codemirror_mode": { 127 | "name": "ipython", 128 | "version": 3 129 | }, 130 | "file_extension": ".py", 131 | "mimetype": "text/x-python", 132 | "name": "python", 133 | "nbconvert_exporter": "python", 134 | "pygments_lexer": "ipython3", 135 | "version": "3.11.4" 136 | }, 137 | "latex_envs": { 138 | "LaTeX_envs_menu_present": true, 139 | "autoclose": false, 140 | "autocomplete": true, 141 | "bibliofile": "biblio.bib", 142 | "cite_by": "apalike", 143 | "current_citInitial": 1, 144 | "eqLabelWithNumbers": true, 145 | "eqNumInitial": 1, 146 | "hotkeys": { 147 | "equation": "Ctrl-E", 148 | "itemize": "Ctrl-I" 149 | }, 150 | "labels_anchors": false, 151 | "latex_user_defs": false, 152 | "report_style_numbering": false, 153 | "user_envs_cfg": false 154 | }, 155 | "papermill": { 156 | "default_parameters": {}, 157 | "duration": 2.06338, 158 | "end_time": "2021-09-13T08:28:28.958340", 159 | "environment_variables": {}, 160 | "exception": null, 161 | "input_path": "docs/refactoring/parameterise/input.ipynb", 162 | "output_path": "docs/refactoring/parameterise/output.ipynb", 163 | "parameters": { 164 | "name": "pythonistas", 165 | "salutation": "Hello" 166 | }, 167 | "start_time": "2021-09-13T08:28:26.894960", 168 | "version": "2.3.3" 169 | } 170 | }, 171 | "nbformat": 4, 172 | "nbformat_minor": 5 173 | } 174 | -------------------------------------------------------------------------------- /docs/notebook/parameterise/output_2023-06-26_15:57:33.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "77487f8d", 6 | "metadata": { 7 | "papermill": { 8 | "duration": 0.010352, 9 | "end_time": "2021-09-13T08:42:38.495270", 10 | "exception": false, 11 | "start_time": "2021-09-13T08:42:38.484918", 12 | "status": "completed" 13 | }, 14 | "tags": [] 15 | }, 16 | "source": [ 17 | "# Notebook with parameters" 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": 1, 23 | "id": "5ff2a94a", 24 | "metadata": { 25 | "papermill": { 26 | "duration": 0.029402, 27 | "end_time": "2021-09-13T08:42:38.533478", 28 | "exception": false, 29 | "start_time": "2021-09-13T08:42:38.504076", 30 | "status": "completed" 31 | }, 32 | "tags": [ 33 | "parameters" 34 | ] 35 | }, 36 | "outputs": [], 37 | "source": [ 38 | "salutation = None\n", 39 | "name = None" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": 2, 45 | "id": "71e802dd", 46 | "metadata": { 47 | "papermill": { 48 | "duration": 0.013999, 49 | "end_time": "2021-09-13T08:42:38.555226", 50 | "exception": false, 51 | "start_time": "2021-09-13T08:42:38.541227", 52 | "status": "completed" 53 | }, 54 | "tags": [ 55 | "injected-parameters" 56 | ] 57 | }, 58 | "outputs": [], 59 | "source": [ 60 | "# Parameters\n", 61 | "salutation = \"Hello\"\n", 62 | "name = \"pythonistas\"" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": 3, 68 | "id": "ef853d4a", 69 | "metadata": { 70 | "papermill": { 71 | "duration": 0.011214, 72 | "end_time": "2021-09-13T08:42:38.571083", 73 | "exception": false, 74 | "start_time": "2021-09-13T08:42:38.559869", 75 | "status": "completed" 76 | }, 77 | "tags": [] 78 | }, 79 | "outputs": [], 80 | "source": [ 81 | "from datetime import date\n", 82 | "\n", 83 | "\n", 84 | "today = date.today()" 85 | ] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "execution_count": 4, 90 | "id": "c023d477", 91 | "metadata": { 92 | "papermill": { 93 | "duration": 0.012967, 94 | "end_time": "2021-09-13T08:42:38.588113", 95 | "exception": false, 96 | "start_time": "2021-09-13T08:42:38.575146", 97 | "status": "completed" 98 | }, 99 | "tags": [] 100 | }, 101 | "outputs": [ 102 | { 103 | "name": "stdout", 104 | "output_type": "stream", 105 | "text": [ 106 | "Hello pythonistas – welcome to our event on this Monday, 26 June 2023\n" 107 | ] 108 | } 109 | ], 110 | "source": [ 111 | "print(\n", 112 | " salutation,\n", 113 | " name,\n", 114 | " \"– welcome to our event on this \" + today.strftime(\"%A, %d %B %Y\"),\n", 115 | ")" 116 | ] 117 | } 118 | ], 119 | "metadata": { 120 | "kernelspec": { 121 | "display_name": "Python 3.11 Kernel", 122 | "language": "python", 123 | "name": "python311" 124 | }, 125 | "language_info": { 126 | "codemirror_mode": { 127 | "name": "ipython", 128 | "version": 3 129 | }, 130 | "file_extension": ".py", 131 | "mimetype": "text/x-python", 132 | "name": "python", 133 | "nbconvert_exporter": "python", 134 | "pygments_lexer": "ipython3", 135 | "version": "3.11.4" 136 | }, 137 | "latex_envs": { 138 | "LaTeX_envs_menu_present": true, 139 | "autoclose": false, 140 | "autocomplete": true, 141 | "bibliofile": "biblio.bib", 142 | "cite_by": "apalike", 143 | "current_citInitial": 1, 144 | "eqLabelWithNumbers": true, 145 | "eqNumInitial": 1, 146 | "hotkeys": { 147 | "equation": "Ctrl-E", 148 | "itemize": "Ctrl-I" 149 | }, 150 | "labels_anchors": false, 151 | "latex_user_defs": false, 152 | "report_style_numbering": false, 153 | "user_envs_cfg": false 154 | }, 155 | "papermill": { 156 | "default_parameters": {}, 157 | "duration": 2.635495, 158 | "end_time": "2021-09-13T08:42:38.705856", 159 | "environment_variables": {}, 160 | "exception": null, 161 | "input_path": "docs/refactoring/parameterise/input.ipynb", 162 | "output_path": "docs/refactoring/parameterise/output_2021-09-13_10:42:33.ipynb", 163 | "parameters": { 164 | "name": "Pythonistas", 165 | "salutation": "Hello" 166 | }, 167 | "start_time": "2021-09-13T08:42:36.070361", 168 | "version": "2.3.3" 169 | } 170 | }, 171 | "nbformat": 4, 172 | "nbformat_minor": 5 173 | } 174 | -------------------------------------------------------------------------------- /docs/notebook/parameterise/params.yaml: -------------------------------------------------------------------------------- 1 | salutation: "Hello" 2 | name: "Pythonistas" 3 | -------------------------------------------------------------------------------- /docs/notebook/rename-notebook.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veit/jupyter-tutorial/24fcd76f43fc46ffd02e3bd98479a41ad6565197/docs/notebook/rename-notebook.png -------------------------------------------------------------------------------- /docs/notebook/testing/doctest.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Doctests" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": {}, 14 | "outputs": [ 15 | { 16 | "name": "stdout", 17 | "output_type": "stream", 18 | "text": [ 19 | "Trying:\n", 20 | " add(7,6)\n", 21 | "Expecting:\n", 22 | " 13\n", 23 | "ok\n", 24 | "1 items had no tests:\n", 25 | " __main__\n", 26 | "1 items passed all tests:\n", 27 | " 1 tests in __main__.add\n", 28 | "1 tests in 2 items.\n", 29 | "1 passed and 0 failed.\n", 30 | "Test passed.\n" 31 | ] 32 | }, 33 | { 34 | "data": { 35 | "text/plain": [ 36 | "TestResults(failed=0, attempted=1)" 37 | ] 38 | }, 39 | "execution_count": 1, 40 | "metadata": {}, 41 | "output_type": "execute_result" 42 | } 43 | ], 44 | "source": [ 45 | "import doctest\n", 46 | "\n", 47 | "\n", 48 | "def add(a, b):\n", 49 | " \"\"\"\n", 50 | " This is a test:\n", 51 | " >>> add(7,6)\n", 52 | " 13\n", 53 | " \"\"\"\n", 54 | " return a + b\n", 55 | "\n", 56 | "\n", 57 | "doctest.testmod(verbose=True)" 58 | ] 59 | }, 60 | { 61 | "cell_type": "markdown", 62 | "metadata": {}, 63 | "source": [ 64 | "## Debugging" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": 2, 70 | "metadata": {}, 71 | "outputs": [], 72 | "source": [ 73 | "doctest.testmod()\n", 74 | "\n", 75 | "\n", 76 | "def multiply(a, b):\n", 77 | " \"\"\"\n", 78 | " This is a test:\n", 79 | " >>> multiply(2, 2)\n", 80 | " 5\n", 81 | " \"\"\"\n", 82 | " import pdb\n", 83 | "\n", 84 | " pdb.set_trace()\n", 85 | " return a * b" 86 | ] 87 | }, 88 | { 89 | "cell_type": "markdown", 90 | "metadata": {}, 91 | "source": [ 92 | "1. `import pdb` imports the Python debugger\n", 93 | "2. `pdb.set_trace()` creates a breakpoint that starts the Python debugger.\n", 94 | "\n", 95 | "
\n", 96 | "\n", 97 | "**See also:**\n", 98 | "\n", 99 | "* [pdb – The Python Debugger](https://docs.python.org/3/library/pdb.html)\n", 100 | "
" 101 | ] 102 | } 103 | ], 104 | "metadata": { 105 | "kernelspec": { 106 | "display_name": "Python 3.11 Kernel", 107 | "language": "python", 108 | "name": "python311" 109 | }, 110 | "language_info": { 111 | "codemirror_mode": { 112 | "name": "ipython", 113 | "version": 3 114 | }, 115 | "file_extension": ".py", 116 | "mimetype": "text/x-python", 117 | "name": "python", 118 | "nbconvert_exporter": "python", 119 | "pygments_lexer": "ipython3", 120 | "version": "3.11.4" 121 | }, 122 | "latex_envs": { 123 | "LaTeX_envs_menu_present": true, 124 | "autoclose": false, 125 | "autocomplete": true, 126 | "bibliofile": "biblio.bib", 127 | "cite_by": "apalike", 128 | "current_citInitial": 1, 129 | "eqLabelWithNumbers": true, 130 | "eqNumInitial": 1, 131 | "hotkeys": { 132 | "equation": "Ctrl-E", 133 | "itemize": "Ctrl-I" 134 | }, 135 | "labels_anchors": false, 136 | "latex_user_defs": false, 137 | "report_style_numbering": false, 138 | "user_envs_cfg": false 139 | }, 140 | "widgets": { 141 | "application/vnd.jupyter.widget-state+json": { 142 | "state": {}, 143 | "version_major": 2, 144 | "version_minor": 0 145 | } 146 | } 147 | }, 148 | "nbformat": 4, 149 | "nbformat_minor": 4 150 | } 151 | -------------------------------------------------------------------------------- /docs/notebook/testing/index.rst: -------------------------------------------------------------------------------- 1 | Testing 2 | ======= 3 | 4 | Concepts 5 | -------- 6 | 7 | .. glossary:: 8 | 9 | Test Case 10 | tests a single scenario. 11 | 12 | .. seealso:: 13 | * `pytest fixtures 14 | `_ 15 | 16 | Test Fixture 17 | is a consistent test environment. 18 | 19 | Test Suite 20 | is a collection of several test cases. 21 | 22 | Test Runner 23 | runs through a test suite and presents the results. 24 | 25 | Notebooks 26 | --------- 27 | 28 | .. toctree:: 29 | :maxdepth: 1 30 | 31 | unittest.ipynb 32 | doctest.ipynb 33 | mock.ipynb 34 | 35 | Tools 36 | ----- 37 | 38 | .. toctree:: 39 | :titlesonly: 40 | :maxdepth: 3 41 | 42 | ipytest.ipynb 43 | hypothesis.ipynb 44 | -------------------------------------------------------------------------------- /docs/sphinx/executablebooks.rst: -------------------------------------------------------------------------------- 1 | Executable Books 2 | ================ 3 | 4 | `Executable Books `_ is a collection of 5 | open-source tools that facilitate the publication of computational narratives 6 | using the Jupyter ecosystem, primarily: 7 | 8 | Jupyter Book 9 | :doc:`index` distribution that allows you to write content in Markdown and 10 | Jupyter Notebooks, execute content and insert it into your book. 11 | 12 | .. seealso:: 13 | * `jupyterbook.org `_ is 14 | the landing page of the project. 15 | * `gallery.jupyterbook.org 16 | `_ is a gallery of 17 | Jupyter books. 18 | * `github.com/jupyter-book/jupyter-book 19 | `_ is the project’s 20 | repository. 21 | 22 | MyST 23 | is an extensible semantic variant of Markdown designed for scientific and 24 | computational narratives. MyST-Markdown is a language- and 25 | implementation-independent variant of Markdown supported by several tools. 26 | 27 | .. seealso:: 28 | * `mystmd.org `_ is the landing page of the project. 29 | * `spec.mystmd.org `_ describes the MyST 30 | specification. 31 | * `MyST Enhancement Proposals 32 | `_ is a 33 | process for proposing and deciding on changes to the MyST 34 | specification. 35 | 36 | JupyterLab MyST Extension 37 | renders Markdown cells in :doc:`/jupyterlab/index` using MyST Markdown, 38 | including interactive references, notes, figure numbering, tabs, cards and 39 | grids. 40 | 41 | .. seealso:: 42 | * `github.com/jupyter-book/jupyterlab-myst 43 | `_ 44 | -------------------------------------------------------------------------------- /docs/sphinx/index.rst: -------------------------------------------------------------------------------- 1 | Sphinx 2 | ====== 3 | 4 | `Sphinx `_, a documentation tool that 5 | converts :doc:`python-basics:document/sphinx/rest` into HTML or PDF, EPub and 6 | man pages. The Jupyter tutorial is also created with Sphinx. 7 | 8 | Originally developed for Python documentation, Sphinx is now used in almost all 9 | Python projects, including `NumPy `_ and `SciPy 10 | `_, `Matplotlib 11 | `_, `pandas 12 | `_ and `SQLAlchemy 13 | `_. 14 | 15 | With :doc:`nbsphinx`, Jupyter Notebooks can also be integrated into Sphinx. 16 | :doc:`executablebooks`, on the other hand, is a collection of open-source tools 17 | that allow you to write Markdown and Jupyter Notebooks, execute content and 18 | insert it into your book, among other things. 19 | 20 | .. tip:: 21 | cusy seminars: 22 | 23 | * `Software documentation with Sphinx 24 | `_ 25 | * `Technical writing 26 | `_ 27 | 28 | .. toctree:: 29 | :hidden: 30 | :titlesonly: 31 | :maxdepth: 0 32 | 33 | nbsphinx 34 | executablebooks 35 | -------------------------------------------------------------------------------- /docs/use-cases.rst: -------------------------------------------------------------------------------- 1 | Use cases 2 | ========= 3 | 4 | In some companies, Jupyter notebooks are used to explore the ever-increasing 5 | amounts of data. These include: 6 | 7 | * Netflix 8 | 9 | * `Beyond Interactive: Notebook Innovation at Netflix 10 | `_ 11 | * `Part 2: Scheduling Notebooks at Netflix 12 | `_ 13 | 14 | * Bloomberg BQuant platform 15 | 16 | * `Bloomberg BQuant (BQNT) `_ 17 | 18 | * PayPal 19 | 20 | * `PayPal Notebooks: Data science and machine learning at scale, powered by 21 | Jupyter 22 | `_ 23 | 24 | * Société Générale 25 | 26 | * `Jupyter & Python in the corporate LAN 27 | `_ 28 | -------------------------------------------------------------------------------- /docs/viz/index.rst: -------------------------------------------------------------------------------- 1 | Visualise data 2 | ============== 3 | 4 | We have outsourced the visualisation of data to a separate tutorial: 5 | :doc:`PyViz Tutorial `. 6 | -------------------------------------------------------------------------------- /docs/whatsnew.rst: -------------------------------------------------------------------------------- 1 | What’s new? 2 | =========== 3 | 4 | .. include:: ../CHANGELOG.rst 5 | :start-after: changelog 6 | -------------------------------------------------------------------------------- /fastapi/main.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from pydantic import BaseModel 4 | 5 | from fastapi import FastAPI 6 | 7 | 8 | app = FastAPI() 9 | 10 | 11 | class Item(BaseModel): 12 | name: str 13 | price: float 14 | is_offer: Optional[bool] = None 15 | 16 | 17 | @app.get("/") 18 | def read_root(): 19 | return {"Hello": "World"} 20 | 21 | 22 | @app.get("/items/{item_id}") 23 | def read_item(item_id: int, q: Optional[str] = None): 24 | return {"item_id": item_id, "q": q} 25 | 26 | 27 | @app.put("/items/{item_id}") 28 | def update_item(item_id: int, item: Item): 29 | return {"item_name": item.name, "item_id": item_id} 30 | -------------------------------------------------------------------------------- /pipenvs/python-311/Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | url = "https://pypi.org/simple" 3 | verify_ssl = true 4 | name = "pypi" 5 | 6 | [packages] 7 | jupyter = "*" 8 | papermill = "*" 9 | ipython-unittest = "*" 10 | mock = "*" 11 | ipytest = "*" 12 | hypothesis = {extras = ["numpy", "pandas"], version = "*"} 13 | ipykernel = "*" 14 | ipywidgets = "*" 15 | bqplot = "*" 16 | ipycanvas = "*" 17 | pythreejs = "*" 18 | ipyvolume = "*" 19 | ipyleaflet = "*" 20 | ipywebrtc = "*" 21 | ipysheet = "*" 22 | ipyvuetify = "*" 23 | ipympl = "*" 24 | qgrid = "*" 25 | ipylayout = "*" 26 | mpl-scatter-density = "*" 27 | fiona = "*" 28 | descartes = "*" 29 | geopandas = "*" 30 | geodatasets = "*" 31 | folium = "*" 32 | seaborn = "*" 33 | plotnine = {extras = ["all"], version = "*"} 34 | networkx = "*" 35 | graphviz = "*" 36 | cartopy = "*" 37 | rtree = "*" 38 | geopy = "*" 39 | mapclassify = "*" 40 | yt = "*" 41 | altair = "*" 42 | vega-datasets = "*" 43 | pdvega = "*" 44 | bokeh = "*" 45 | flask = "*" 46 | hvplot = "*" 47 | intake = "*" 48 | intake-parquet = "*" 49 | s3fs = "*" 50 | pyviz-comms = "*" 51 | datashader = "*" 52 | # datashader requires Numba 53 | # Numba requires NumPy ≤ 1.24 54 | numpy = "<=1.24" 55 | geoviews = "*" 56 | plotly = "*" 57 | toyplot = "*" 58 | xarray-leaflet = "*" 59 | jupyter-dashboards = "*" 60 | appmode = "*" 61 | panel = "*" 62 | holoviews = {extras = ["recommended"], version = "*"} 63 | fastapi = "*" 64 | voila = "*" 65 | voila-vuetify = "*" 66 | voila-reveal = "*" 67 | jupyter-flex = "*" 68 | 69 | [dev-packages] 70 | 71 | [requires] 72 | python_version = "3.11" 73 | python_full_version = "3.11.4" 74 | -------------------------------------------------------------------------------- /pipenvs/python-38/Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | url = "https://pypi.org/simple" 3 | verify_ssl = true 4 | name = "pypi" 5 | 6 | [packages] 7 | qgrid = "*" 8 | pandas-datareader = "*" 9 | dedupe = "*" 10 | pythreejs = "*" 11 | ipyvolume = "*" 12 | importlib-metadata = "*" 13 | shot-scraper = "*" 14 | py-heat-magic = "*" 15 | pandas-profiling = {extras = ["notebook"], version = "*"} 16 | panel = {editable = true, git = "https://github.com/holoviz/panel.git"} 17 | fastapi = "*" 18 | uvicorn = "*" 19 | 20 | [dev-packages] 21 | 22 | [requires] 23 | python_version = "3.8" 24 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "jupyter-tutorial" 3 | version = "24.1.0" 4 | authors = [ 5 | { name="Veit Schiele", email="veit@cusy.io" }, 6 | ] 7 | description = "Training materials for setting up and using a research infrastructure based on Jupyter notebooks: https://cusy.io/en/seminars" 8 | readme = "README.rst" 9 | requires-python = ">=3.9" 10 | classifiers = [ 11 | "Programming Language :: Python :: 3", 12 | "License :: OSI Approved :: BSD License", 13 | "Operating System :: OS Independent", 14 | ] 15 | dependencies = [] 16 | 17 | [project.optional-dependencies] 18 | docs = [ 19 | "sphinx<8.2", 20 | "furo", 21 | "ipython", 22 | "ipywidgets", 23 | "nbsphinx", 24 | "sphinxext.opengraph", # matplotlib is required for social cards 25 | "matplotlib", 26 | "sphinx_copybutton", 27 | "sphinx-inline-tabs", 28 | "sphinxcontrib-svg2pdfconverter", 29 | "sphinx-lint", 30 | ] 31 | 32 | dev = [ 33 | "jupyter-tutorial[docs]", 34 | "pre-commit", 35 | "codespell", 36 | "vale", 37 | ] 38 | 39 | [project.urls] 40 | "Homepage" = "https://github.com/veit/jupyter-tutorial/" 41 | "Bug Tracker" = "https://github.com/veit/jupyter-tutorial/issues" 42 | 43 | [tool.setuptools] 44 | packages = [] 45 | 46 | [tool.black] 47 | line-length = 79 48 | 49 | [tool.isort] 50 | atomic=true 51 | force_grid_wrap=0 52 | include_trailing_comma=true 53 | lines_after_imports=2 54 | lines_between_types=1 55 | multi_line_output=3 56 | not_skip="__init__.py" 57 | use_parentheses=true 58 | 59 | known_first_party="jupyter-tutorial-de" 60 | known_third_party=["Cython", "accounts_pb2", "accounts_pb2_grpc", "dataprep", "google", "grpc", "mpi4py", "numpy", "pandas", "pydantic", "pytest", "requests", "setuptools"] 61 | 62 | [tool.codespell] 63 | skip = "*.html, *.ipynb" 64 | -------------------------------------------------------------------------------- /spackenvs/python-38/spack.yaml: -------------------------------------------------------------------------------- 1 | # This is a Spack Environment file. 2 | # 3 | # It describes a set of packages to be installed, along with 4 | # configuration settings. 5 | spack: 6 | # add package specs to the `specs` list 7 | specs: 8 | - python@3.8.12%gcc@11.2.0+bz2+ctypes+dbm~debug+libxml2+lzma~nis~optimizations+pic+pyexpat+pythoncmd+readline+shared+sqlite3+ssl~tix~tkinter~ucs4+uuid+zlib 9 | - py-jupyter@1.0.0%gcc@11.2.0 10 | - py-numpy@1.21.3%gcc@11.2.0+blas+lapack 11 | - py-pandas@1.3.4%gcc@11.2.0 12 | - py-scipy@1.7.1%gcc@11.2.0 13 | - py-lxml@4.6.3%gcc@11.2.0 14 | - py-html5lib@1.1%gcc@11.2.0 15 | - py-beautifulsoup4@4.10.0%gcc@11.2.0~html5lib~lxml 16 | - py-requests@2.25.1%gcc@11.2.0~socks 17 | - py-fuzzywuzzy@0.18.0%gcc@11.2.0+speedup 18 | - py-dask@2021.6.2%gcc@11.2.0+array+bag+dataframe+delayed~diagnostics+distributed+yaml 19 | - py-sphinx@4.1.2%gcc@11.2.0 20 | - py-nbsphinx@0.8.7%gcc@11.2.0 21 | - py-sphinxcontrib-bibtex@2.2.1%gcc@11.2.0 22 | - py-sphinxcontrib-programoutput@0.15%gcc@11.2.0 23 | - py-graphviz@0.13.2%gcc@11.2.0~dev~docs 24 | - py-pydot@1.4.2%gcc@11.2.0 25 | - py-networkx@2.5.1%gcc@11.2.0 26 | - cufflinks@2.2.1%gcc@11.2.0 27 | - py-matplotlib@3.4.3%gcc@11.2.0~animation~fonts+image~latex~movies 28 | - py-widgetsnbextension@3.5.1%gcc@11.2.0 29 | - py-yapf@0.30.0%gcc@11.2.0 30 | - py-ipycanvas@0.9.0%gcc@11.2.0 31 | - py-geopandas@0.9.0%gcc@11.2.0 32 | - py-seaborn@0.11.2%gcc@11.2.0 33 | - py-cartopy@0.18.0%gcc@11.2.0~epsg~ows~plotting 34 | - py-bokeh@2.4.1%gcc@11.2.0 35 | - py-selenium@3.141.0%gcc@11.2.0 36 | - snappy@1.1.8%gcc@11.2.0~ipo+pic+shared 37 | - py-s3fs@0.5.2%gcc@11.2.0 38 | - py-xarray@0.18.2%gcc@11.2.0~io 39 | - py-pre-commit@1.20.0%gcc@11.2.0 40 | - py-dvc@2.1.0%gcc@11.2.0~s3~ssh 41 | - py-autopep8@1.6.0%gcc@11.2.0 42 | - py-pytest@6.2.4%gcc@11.2.0 43 | - py-line-profiler@2.1.2%gcc@11.2.0 44 | - py-memory-profiler@0.57.0%gcc@11.2.0 45 | - py-ipyparallel@7.1.0%gcc@11.2.0 46 | - py-netcdf4@1.5.3%gcc@11.2.0+mpi 47 | - py-nest-asyncio@1.4.0%gcc@11.2.0 48 | - py-xlrd@2.0.1%gcc@11.2.0 49 | - py-openpyxl@3.0.3%gcc@11.2.0 50 | concretization: together 51 | view: true 52 | -------------------------------------------------------------------------------- /styles/.gitignore: -------------------------------------------------------------------------------- 1 | # ignore everything except .gitignore 2 | * 3 | !.gitignore 4 | -------------------------------------------------------------------------------- /styles/cusy/Polite.yml: -------------------------------------------------------------------------------- 1 | extends: existence 2 | message: 'Do not use “%s” in technical documentation.' 3 | level: warning 4 | ignorecase: true 5 | tokens: 6 | - please 7 | - thank you 8 | -------------------------------------------------------------------------------- /styles/cusy/Spelling.yml: -------------------------------------------------------------------------------- 1 | extends: spelling 2 | message: "Spelling check: '%s'?" 3 | dicpath: /Library/Spelling 4 | dictionaries: 5 | - en_GB 6 | level: warning 7 | ignore: styles/cusy/ignore.txt 8 | -------------------------------------------------------------------------------- /styles/cusy/ignore.txt: -------------------------------------------------------------------------------- 1 | Ansible 2 | Apache 3 | Appmode 4 | Arcpy 5 | Atlassian 6 | attrs 7 | Bitbucket 8 | Bullard 9 | California 10 | Cassandra 11 | cffi 12 | Codecov 13 | cookiecutter 14 | Codacy 15 | concretisation 16 | crobarcro 17 | Cusy 18 | cyclomatic 19 | Cython 20 | Dask 21 | DataCite 22 | Dataverse 23 | Distrowatch 24 | DOI 25 | dotfiles 26 | dvc 27 | Ecography 28 | Esri 29 | FastAPI 30 | Gigascience 31 | GitHub 32 | GitLab 33 | Gitleaks 34 | Grafana 35 | Graphviz 36 | Hadoop 37 | HoloViz 38 | hotfix 39 | Homebrew 40 | Howison 41 | Hstore 42 | Hypertable 43 | Intersphinx 44 | Javascript 45 | Jinja 46 | Jupyter 47 | Jython 48 | Kaggle 49 | Kotlin 50 | Kwalify 51 | Lua 52 | Manylinux 53 | Mapnik 54 | mathjax 55 | Matplotlib 56 | Modin 57 | multiscale 58 | neurocomputing 59 | neuroinformatics 60 | monorepo 61 | Magics 62 | Mypy 63 | Numba 64 | Numpy 65 | Pandoc 66 | Pipenv 67 | Plotly 68 | Prettyfier 69 | Protobuf 70 | psychonomic 71 | Psycopg 72 | Pydantic 73 | Pysa 74 | Pytype 75 | Redis 76 | Riak 77 | Roboflow 78 | Rossum 79 | sharding 80 | slideshow 81 | Spack 82 | Starlette 83 | Stata 84 | Vue 85 | Vuetify 86 | Wireshark 87 | Zenodo 88 | --------------------------------------------------------------------------------