├── .git-blame-ignore-revs ├── .gitattributes ├── .github ├── ISSUE_TEMPLATE.md ├── codeql │ └── codeql-config.yml ├── dependabot.yml └── workflows │ ├── build.yml │ └── codeql-analysis.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .readthedocs.yaml ├── LICENSE.md ├── Makefile ├── README.md ├── codecov.yml ├── conftest.py ├── docs ├── Makefile ├── doc-requirements.txt ├── environment.yml ├── make.bat └── source │ ├── _static │ ├── custom.css │ └── jupyter-logo.png │ ├── conf.py │ ├── contributors │ ├── contrib.md │ ├── debug.md │ ├── devinstall.md │ ├── docker.md │ ├── index.rst │ ├── roadmap.md │ ├── sequence-diagrams.md │ └── system-architecture.md │ ├── developers │ ├── custom-images.md │ ├── dev-process-proxy.md │ ├── index.rst │ ├── kernel-launcher.md │ ├── kernel-library.md │ ├── kernel-manager.md │ ├── kernel-specification.md │ └── rest-api.rst │ ├── images │ ├── Scalability-After-JEG.gif │ ├── Scalability-Before-JEG.gif │ ├── debug_configuration.png │ ├── deployment.png │ ├── process_proxy_hierarchy.png │ └── yarnui.jpg │ ├── index.rst │ ├── operators │ ├── config-add-env.md │ ├── config-availability.md │ ├── config-cli.md │ ├── config-culling.md │ ├── config-dynamic.md │ ├── config-env-debug.md │ ├── config-file.md │ ├── config-kernel-override.md │ ├── config-security.md │ ├── config-sys-env.md │ ├── deploy-conductor.md │ ├── deploy-distributed.md │ ├── deploy-docker.md │ ├── deploy-kubernetes.md │ ├── deploy-single.md │ ├── deploy-yarn-cluster.md │ ├── index.rst │ ├── installing-eg.md │ ├── installing-kernels.md │ └── launching-eg.md │ ├── other │ ├── index.rst │ ├── related-resources.md │ └── troubleshooting.md │ └── users │ ├── client-config.md │ ├── connecting-to-eg.md │ ├── index.rst │ ├── installation.md │ └── kernel-envs.md ├── enterprise_gateway ├── __init__.py ├── __main__.py ├── _version.py ├── base │ ├── __init__.py │ └── handlers.py ├── client │ ├── __init__.py │ └── gateway_client.py ├── enterprisegatewayapp.py ├── itests │ ├── __init__.py │ ├── kernels │ │ └── authorization_test │ │ │ └── kernel.json │ ├── test_authorization.py │ ├── test_base.py │ ├── test_python_kernel.py │ ├── test_r_kernel.py │ └── test_scala_kernel.py ├── mixins.py ├── services │ ├── __init__.py │ ├── api │ │ ├── __init__.py │ │ ├── handlers.py │ │ ├── swagger.json │ │ └── swagger.yaml │ ├── kernels │ │ ├── __init__.py │ │ ├── handlers.py │ │ └── remotemanager.py │ ├── kernelspecs │ │ ├── __init__.py │ │ ├── handlers.py │ │ └── kernelspec_cache.py │ ├── processproxies │ │ ├── __init__.py │ │ ├── conductor.py │ │ ├── container.py │ │ ├── crd.py │ │ ├── distributed.py │ │ ├── docker_swarm.py │ │ ├── k8s.py │ │ ├── processproxy.py │ │ ├── spark_operator.py │ │ └── yarn.py │ └── sessions │ │ ├── __init__.py │ │ ├── handlers.py │ │ ├── kernelsessionmanager.py │ │ └── sessionmanager.py └── tests │ ├── __init__.py │ ├── resources │ ├── failing_code2.ipynb │ ├── failing_code3.ipynb │ ├── kernel_api2.ipynb │ ├── kernel_api3.ipynb │ ├── kernels │ │ └── kernel_defaults_test │ │ │ └── kernel.json │ ├── public │ │ └── index.html │ ├── responses_2.ipynb │ ├── responses_3.ipynb │ ├── simple_api2.ipynb │ ├── simple_api3.ipynb │ ├── unknown_kernel.ipynb │ ├── zen2.ipynb │ └── zen3.ipynb │ ├── test_enterprise_gateway.py │ ├── test_gatewayapp.py │ ├── test_handlers.py │ ├── test_kernelspec_cache.py │ └── test_mixins.py ├── etc ├── Makefile ├── docker │ ├── demo-base │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── bootstrap-yarn-spark.sh │ │ ├── core-site.xml.template │ │ ├── fix-permissions │ │ ├── hdfs-site.xml │ │ ├── mapred-site.xml │ │ ├── ssh_config │ │ └── yarn-site.xml.template │ ├── docker-compose.yml │ ├── enterprise-gateway-demo │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── bootstrap-enterprise-gateway.sh │ │ └── start-enterprise-gateway.sh.template │ ├── enterprise-gateway │ │ ├── Dockerfile │ │ ├── README.md │ │ └── start-enterprise-gateway.sh │ ├── kernel-image-puller │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── image_fetcher.py │ │ ├── kernel_image_puller.py │ │ └── requirements.txt │ ├── kernel-py │ │ ├── Dockerfile │ │ └── README.md │ ├── kernel-r │ │ ├── Dockerfile │ │ └── README.md │ ├── kernel-scala │ │ ├── Dockerfile │ │ └── README.md │ ├── kernel-spark-py │ │ ├── Dockerfile │ │ └── README.md │ ├── kernel-spark-r │ │ ├── Dockerfile │ │ └── README.md │ ├── kernel-tf-gpu-py │ │ ├── Dockerfile │ │ └── README.md │ └── kernel-tf-py │ │ ├── Dockerfile │ │ └── README.md ├── kernel-launchers │ ├── R │ │ └── scripts │ │ │ ├── launch_IRkernel.R │ │ │ └── server_listener.py │ ├── bootstrap │ │ └── bootstrap-kernel.sh │ ├── docker │ │ └── scripts │ │ │ └── launch_docker.py │ ├── kubernetes │ │ └── scripts │ │ │ ├── kernel-pod.yaml.j2 │ │ │ └── launch_kubernetes.py │ ├── operators │ │ └── scripts │ │ │ ├── launch_custom_resource.py │ │ │ └── sparkoperator.k8s.io-v1beta2.yaml.j2 │ ├── python │ │ └── scripts │ │ │ └── launch_ipykernel.py │ └── scala │ │ └── toree-launcher │ │ ├── build.sbt │ │ ├── project │ │ ├── build.properties │ │ ├── plugins.sbt │ │ └── scalastyle-config.xml │ │ └── src │ │ └── main │ │ └── scala │ │ └── launcher │ │ ├── KernelProfile.scala │ │ ├── ToreeLauncher.scala │ │ └── utils │ │ ├── SecurityUtils.scala │ │ └── SocketUtils.scala ├── kernel-resources │ ├── apache_toree │ │ └── logo-64x64.png │ ├── ir │ │ ├── kernel.js │ │ └── logo-64x64.png │ ├── python │ │ └── logo-64x64.png │ └── tensorflow │ │ └── logo-64x64.png ├── kernelspecs │ ├── R_docker │ │ └── kernel.json │ ├── R_kubernetes │ │ └── kernel.json │ ├── dask_python_yarn_remote │ │ ├── bin │ │ │ └── run.sh │ │ └── kernel.json │ ├── python_distributed │ │ └── kernel.json │ ├── python_docker │ │ └── kernel.json │ ├── python_kubernetes │ │ └── kernel.json │ ├── python_tf_docker │ │ └── kernel.json │ ├── python_tf_gpu_docker │ │ └── kernel.json │ ├── python_tf_gpu_kubernetes │ │ └── kernel.json │ ├── python_tf_kubernetes │ │ └── kernel.json │ ├── scala_docker │ │ └── kernel.json │ ├── scala_kubernetes │ │ └── kernel.json │ ├── spark_R_conductor_cluster │ │ ├── bin │ │ │ └── run.sh │ │ └── kernel.json │ ├── spark_R_kubernetes │ │ ├── bin │ │ │ └── run.sh │ │ └── kernel.json │ ├── spark_R_yarn_client │ │ ├── bin │ │ │ └── run.sh │ │ └── kernel.json │ ├── spark_R_yarn_cluster │ │ ├── bin │ │ │ └── run.sh │ │ └── kernel.json │ ├── spark_python_conductor_cluster │ │ ├── bin │ │ │ └── run.sh │ │ └── kernel.json │ ├── spark_python_kubernetes │ │ ├── bin │ │ │ └── run.sh │ │ └── kernel.json │ ├── spark_python_operator │ │ └── kernel.json │ ├── spark_python_yarn_client │ │ ├── bin │ │ │ └── run.sh │ │ └── kernel.json │ ├── spark_python_yarn_cluster │ │ ├── bin │ │ │ └── run.sh │ │ └── kernel.json │ ├── spark_scala_conductor_cluster │ │ ├── bin │ │ │ └── run.sh │ │ └── kernel.json │ ├── spark_scala_kubernetes │ │ ├── bin │ │ │ └── run.sh │ │ └── kernel.json │ ├── spark_scala_yarn_client │ │ ├── bin │ │ │ └── run.sh │ │ └── kernel.json │ └── spark_scala_yarn_cluster │ │ ├── bin │ │ └── run.sh │ │ └── kernel.json └── kubernetes │ └── helm │ └── enterprise-gateway │ ├── Chart.yaml │ ├── templates │ ├── daemonset.yaml │ ├── deployment.yaml │ ├── eg-clusterrole.yaml │ ├── eg-clusterrolebinding.yaml │ ├── eg-serviceaccount.yaml │ ├── imagepullSecret.yaml │ ├── ingress.yaml │ ├── kip-clusterrole.yaml │ ├── kip-clusterrolebinding.yaml │ ├── kip-serviceaccount.yaml │ ├── psp.yaml │ └── service.yaml │ └── values.yaml ├── pyproject.toml ├── release.sh ├── requirements.yml └── website ├── .gitignore ├── README.md ├── _config.yml ├── _data └── navigation.yml ├── _includes ├── call-to-action.html ├── contact.html ├── features.html ├── head.html ├── header.html ├── nav.html ├── platforms.html └── scripts.html ├── _layouts ├── home.html └── page.html ├── _sass ├── _base.scss └── _mixins.scss ├── css ├── animate.min.css ├── bootstrap.css ├── bootstrap.min.css └── main.scss ├── favicon.ico ├── font-awesome ├── css │ ├── font-awesome.css │ └── font-awesome.min.css ├── fonts │ ├── FontAwesome.otf │ ├── fontawesome-webfont.eot │ ├── fontawesome-webfont.svg │ ├── fontawesome-webfont.ttf │ ├── fontawesome-webfont.woff │ └── fontawesome-webfont.woff2 ├── less │ ├── animated.less │ ├── bordered-pulled.less │ ├── core.less │ ├── fixed-width.less │ ├── font-awesome.less │ ├── icons.less │ ├── larger.less │ ├── list.less │ ├── mixins.less │ ├── path.less │ ├── rotated-flipped.less │ ├── stacked.less │ └── variables.less └── scss │ ├── _animated.scss │ ├── _bordered-pulled.scss │ ├── _core.scss │ ├── _fixed-width.scss │ ├── _icons.scss │ ├── _larger.scss │ ├── _list.scss │ ├── _mixins.scss │ ├── _path.scss │ ├── _rotated-flipped.scss │ ├── _stacked.scss │ ├── _variables.scss │ └── font-awesome.scss ├── fonts ├── glyphicons-halflings-regular.eot ├── glyphicons-halflings-regular.svg ├── glyphicons-halflings-regular.ttf ├── glyphicons-halflings-regular.woff └── glyphicons-halflings-regular.woff2 ├── img ├── dask-logo.png ├── docker-swarm-logo.png ├── header.jpg ├── kubernetes-logo.png ├── platform-kubernetes-jupyterhub.png ├── platform-kubernetes.png ├── platform-spark-hdp.png ├── platform-spark-yarn.png ├── spark-logo-trademark.png ├── spectrum-conductor-logo.jpg └── spectrum-conductor-logo.png ├── index.md ├── js ├── bootstrap.js ├── bootstrap.min.js ├── cbpAnimatedHeader.js ├── classie.js ├── creative.js ├── jquery.easing.min.js ├── jquery.fittext.js ├── jquery.js └── wow.min.js ├── platform-kubernetes.md ├── platform-spark.md ├── privacy-policy.md └── publish.sh /.git-blame-ignore-revs: -------------------------------------------------------------------------------- 1 | # Initial pre-commit reformat 2 | df811d0deacebfd6cc77e8bf501d9b87ff006fb5 3 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Set the default behavior to have all files normalized to Unix-style 2 | # line endings upon check-in. 3 | * text=auto 4 | 5 | # Declare files that will always have CRLF line endings on checkout. 6 | *.bat text eol=crlf 7 | 8 | # Denote all files that are truly binary and should not be modified. 9 | *.dll binary 10 | *.exp binary 11 | *.lib binary 12 | *.pdb binary 13 | *.exe binary 14 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | Help us improve the Jupyter Enterprise Gateway project by reporting issues 2 | or asking questions. 3 | 4 | ## Description 5 | 6 | ## Screenshots / Logs 7 | 8 | If applicable, add screenshots and/or logs to help explain your problem. 9 | To generate better logs, please run the gateway with `--debug` command line parameter. 10 | 11 | ## Environment 12 | 13 | - Enterprise Gateway Version \[e.g. 1.x, 2.x, ...\] 14 | - Platform: \[e.g. YARN, Kubernetes ...\] 15 | - Others \[e.g. Jupyter Server 5.7, JupyterHub 1.0, etc\] 16 | -------------------------------------------------------------------------------- /.github/codeql/codeql-config.yml: -------------------------------------------------------------------------------- 1 | name: "Enterprise Gateway CodeQL config" 2 | 3 | queries: 4 | - uses: security-and-quality 5 | 6 | paths-ignore: 7 | - enterprise_gateway/tests 8 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | # Set update schedule for GitHub Actions 4 | - package-ecosystem: "github-actions" 5 | directory: "/" 6 | schedule: 7 | # Check for updates to GitHub Actions once a week (Mondays by default) 8 | interval: "weekly" 9 | # Set update schedule for pip 10 | - package-ecosystem: "pip" 11 | directory: "/" 12 | schedule: 13 | # Check for updates to Python deps once a week (Mondays by default) 14 | interval: "weekly" 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | *.egg-info/ 23 | .installed.cfg 24 | *.egg 25 | 26 | # PyInstaller 27 | # Usually these files are written by a python script from a template 28 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 29 | *.manifest 30 | *.spec 31 | 32 | # Installer logs 33 | pip-log.txt 34 | pip-delete-this-directory.txt 35 | 36 | # Unit test / coverage reports 37 | htmlcov/ 38 | .tox/ 39 | .coverage 40 | .coverage.* 41 | .cache 42 | nosetests.xml 43 | coverage.xml 44 | *,cover 45 | .pytest_cache/ 46 | 47 | # Translations 48 | *.mo 49 | *.pot 50 | 51 | # Django stuff: 52 | *.log 53 | 54 | # Sphinx documentation 55 | docs/_build/ 56 | 57 | # PyBuilder 58 | target/ 59 | 60 | .DS_Store 61 | .ipynb_checkpoints/ 62 | 63 | # PyCharm 64 | .idea/ 65 | *.iml 66 | 67 | # Build-related 68 | .image-* 69 | 70 | # Jekyll 71 | _site/ 72 | .sass-cache/ 73 | 74 | # Debug-related 75 | .kube/ 76 | 77 | # vscode ide stuff 78 | *.code-workspace 79 | .history/ 80 | .vscode/ 81 | 82 | # jetbrains ide stuff 83 | *.iml 84 | .idea/ 85 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | ci: 2 | autoupdate_schedule: monthly 3 | 4 | repos: 5 | - repo: https://github.com/pre-commit/pre-commit-hooks 6 | rev: v4.5.0 7 | hooks: 8 | - id: check-case-conflict 9 | - id: check-ast 10 | - id: check-docstring-first 11 | - id: check-executables-have-shebangs 12 | - id: check-added-large-files 13 | - id: check-case-conflict 14 | - id: check-merge-conflict 15 | - id: check-json 16 | - id: check-toml 17 | - id: check-yaml 18 | exclude: etc/kubernetes/.*.yaml 19 | - id: end-of-file-fixer 20 | - id: trailing-whitespace 21 | 22 | - repo: https://github.com/python-jsonschema/check-jsonschema 23 | rev: 0.27.4 24 | hooks: 25 | - id: check-github-workflows 26 | 27 | - repo: https://github.com/executablebooks/mdformat 28 | rev: 0.7.17 29 | hooks: 30 | - id: mdformat 31 | additional_dependencies: 32 | [mdformat-gfm, mdformat-frontmatter, mdformat-footnote] 33 | 34 | - repo: https://github.com/psf/black 35 | rev: 24.2.0 36 | hooks: 37 | - id: black 38 | 39 | - repo: https://github.com/charliermarsh/ruff-pre-commit 40 | rev: v0.3.0 41 | hooks: 42 | - id: ruff 43 | args: ["--fix"] 44 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | build: 3 | os: "ubuntu-22.04" 4 | tools: 5 | python: "mambaforge-22.9" 6 | sphinx: 7 | configuration: docs/source/conf.py 8 | conda: 9 | environment: docs/environment.yml 10 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | codecov: 2 | notify: 3 | require_ci_to_pass: yes 4 | 5 | coverage: 6 | precision: 2 7 | round: down 8 | range: "70...100" 9 | 10 | status: 11 | project: no 12 | patch: no 13 | changes: no 14 | 15 | parsers: 16 | gcov: 17 | branch_detection: 18 | conditional: yes 19 | loop: yes 20 | method: no 21 | macro: no 22 | 23 | comment: off 24 | -------------------------------------------------------------------------------- /conftest.py: -------------------------------------------------------------------------------- 1 | def pytest_addoption(parser): 2 | parser.addoption("--host", action="store", default="localhost:8888") 3 | parser.addoption("--username", action="store", default="elyra") 4 | parser.addoption("--impersonation", action="store", default="false") 5 | 6 | 7 | def pytest_generate_tests(metafunc): 8 | # This is called for every test. Only get/set command line arguments 9 | # if the argument is specified in the list of test "fixturenames". 10 | if "host" in metafunc.fixturenames: 11 | metafunc.parametrize("host", [metafunc.config.option.host]) 12 | if "username" in metafunc.fixturenames: 13 | metafunc.parametrize("username", [metafunc.config.option.username]) 14 | if "impersonation" in metafunc.fixturenames: 15 | metafunc.parametrize("impersonation", [metafunc.config.option.impersonation]) 16 | -------------------------------------------------------------------------------- /docs/doc-requirements.txt: -------------------------------------------------------------------------------- 1 | # https://github.com/miyakogi/m2r/issues/66 2 | mistune<4 3 | myst-parser 4 | pydata_sphinx_theme 5 | sphinx 6 | sphinx-markdown-tables 7 | sphinx_book_theme 8 | sphinxcontrib-mermaid 9 | sphinxcontrib-openapi 10 | sphinxcontrib_github_alt 11 | sphinxcontrib_spelling 12 | sphinxemoji 13 | tornado 14 | -------------------------------------------------------------------------------- /docs/environment.yml: -------------------------------------------------------------------------------- 1 | name: enterprise_gateway_docs 2 | channels: 3 | - conda-forge 4 | - defaults 5 | - free 6 | dependencies: 7 | - pip 8 | - python=3.8 9 | - pip: 10 | - -r doc-requirements.txt 11 | -------------------------------------------------------------------------------- /docs/source/_static/custom.css: -------------------------------------------------------------------------------- 1 | body div.sphinxsidebarwrapper p.logo { 2 | text-align: left; 3 | } 4 | .mermaid svg { 5 | height: 100%; 6 | } 7 | -------------------------------------------------------------------------------- /docs/source/_static/jupyter-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/docs/source/_static/jupyter-logo.png -------------------------------------------------------------------------------- /docs/source/contributors/contrib.md: -------------------------------------------------------------------------------- 1 | # Contributing to Jupyter Enterprise Gateway 2 | 3 | Thank you for your interest in Jupyter Enterprise Gateway! If you would like to contribute to the 4 | project please first take a look at the 5 | [Project Jupyter Contributor Documentation](https://jupyter.readthedocs.io/en/latest/contributing/content-contributor.html). 6 | 7 | Enterprise Gateway has recently joined the [Jupyter Server organization](https://github.com/jupyter-server). Please check out our [team compass page](https://github.com/jupyter-server/team-compass#jupyter-server-team-compass) and try to attend our weekly dev meeting as we have a common goal of making all Jupyter server-side applications better! 8 | 9 | Prior to your contribution, we strongly recommend getting acquainted with Enterprise Gateway by checking 10 | out the [System Architecture](system-architecture.md) and [Development Workflow](devinstall.md) pages. 11 | -------------------------------------------------------------------------------- /docs/source/contributors/debug.md: -------------------------------------------------------------------------------- 1 | # Debugging Jupyter Enterprise Gateway 2 | 3 | This page discusses how to go about debugging Enterprise Gateway. We also provide troubleshooting information 4 | in our [Troubleshooting Guide](../other/troubleshooting.md). 5 | 6 | ## Configuring your IDE 7 | 8 | While your mileage may vary depending on which IDE you are using, the steps below (using PyCharm as an example) should be useful for configuring a debugging session for Enterprise Gateway with minimum 9 | adjustments for different IDEs. 10 | 11 | ### Creating a new Debug Configuration 12 | 13 | Go to Run->Edit Configuration and create a new python configuration with the following settings: 14 | 15 | ![Enterprise Gateway debug configuration](../images/debug_configuration.png) 16 | 17 | **Script Path:** 18 | 19 | ```bash 20 | /Users/jovyan/opensource/jupyter/elyra/scripts/jupyter-enterprisegateway 21 | ``` 22 | 23 | **Parameters:** 24 | 25 | ```bash 26 | --ip=0.0.0.0 27 | --log-level=DEBUG 28 | --EnterpriseGatewayApp.yarn_endpoint=“http://elyra-fyi-node-1.fyre.ibm.com:8088/ws/v1/cluster” 29 | --EnterpriseGatewayApp.remote_hosts=['localhost'] 30 | ``` 31 | 32 | **Environment Variables:** 33 | 34 | ```bash 35 | EG_ENABLE_TUNNELING=False 36 | ``` 37 | 38 | **Working Directory:** 39 | 40 | ```bash 41 | /Users/jovyan/opensource/jupyter/elyra/scripts 42 | ``` 43 | 44 | ### Running in debug mode 45 | 46 | Now that you have handled the necessary configuration, use Run-Debug and select the debug configuration 47 | you just created and happy debugging! 48 | -------------------------------------------------------------------------------- /docs/source/contributors/index.rst: -------------------------------------------------------------------------------- 1 | Contributors Guide 2 | ================== 3 | 4 | These pages target people who are interested in contributing directly to the Jupyter Enterprise Gateway Project. 5 | 6 | .. admonition:: Use cases 7 | 8 | - *As a contributor, I want to learn more about kernel management within the Jupyter ecosystem.* 9 | - *As a contributor, I want to make Enterprise Gateway a more stable service for my organization and the community as a whole.* 10 | - *As a contributor, I'm interested in adding the ability for Enterprise Gateway to be highly available and fault tolerant.* 11 | 12 | .. note:: 13 | As a *contributor*, we encourage you to be familiar with all of the guides (Users, Developers, Operators) to best support Enterprise Gateway. This guide provides an overview of Enterprise Gateway along with instructions on how to get set up. 14 | 15 | 16 | .. toctree:: 17 | :maxdepth: 1 18 | :name: contributors 19 | 20 | contrib 21 | system-architecture 22 | docker 23 | devinstall 24 | sequence-diagrams 25 | debug 26 | roadmap 27 | -------------------------------------------------------------------------------- /docs/source/contributors/roadmap.md: -------------------------------------------------------------------------------- 1 | # Project Roadmap 2 | 3 | We have plenty to do, now and in the future. Here's where we're headed: 4 | 5 | ## Planned for 3.0 6 | 7 | - Spark 3.0 support 8 | - Includes pod template files 9 | 10 | ## Planned for 4.0 11 | 12 | - Kernel Provisioners 13 | - Provisioners will replace process proxies and enable Enterprise Gateway to remove its cap on `jupyter_client < 7`. 14 | - Parameterized Kernels 15 | - Enable the ability to prompt for parameters 16 | - These will likely be based on kernel provisioners (4.0) 17 | 18 | ## Wish list 19 | 20 | - High Availability 21 | - Session persistence using a shared location (NoSQL DB) (File persistence has been implemented) 22 | - Active/active support 23 | - Multi-gateway support on client-side 24 | - Enables the ability for a single Jupyter Server to be configured against multiple Gateway servers simultaneously. This work will primarily be in Jupyter Server. 25 | - Pluggable load-balancers into `DistributedProcessProxy` (currently uses simple round-robin) 26 | - Support for other resource managers 27 | - Slurm? 28 | - Mesos? 29 | - User Environments 30 | - Improve the way user files are made available to remote kernels 31 | - Administration UI 32 | - Dashboard with running kernels 33 | - Lifecycle management 34 | - Time running, stop/kill, Profile Management, etc 35 | 36 | We'd love to hear any other use cases you might have and look forward to your contributions to Jupyter Enterprise Gateway! 37 | -------------------------------------------------------------------------------- /docs/source/contributors/sequence-diagrams.md: -------------------------------------------------------------------------------- 1 | # Sequence Diagrams 2 | 3 | The following consists of various sequence diagrams you might find helpful. We plan to add 4 | diagrams based on demand and contributions. 5 | 6 | ## Kernel launch: Jupyter Lab to Enterprise Gateway 7 | 8 | This diagram depicts the interactions between components when a kernel start request 9 | is submitted from Jupyter Lab running against [Jupyter Server configured to use 10 | Enterprise Gateway](../users/connecting-to-eg.md). The diagram also includes the 11 | retrieval of kernel specifications (kernelspecs) prior to the kernel's initialization. 12 | 13 | ```{mermaid} 14 | sequenceDiagram 15 | participant JupyterLab 16 | participant JupyterServer 17 | participant EnterpriseGateway 18 | participant ProcessProxy 19 | participant Kernel 20 | participant ResourceManager 21 | Note left of JupyterLab: fetch kernelspecs 22 | JupyterLab->>JupyterServer: https GET api/kernelspecs 23 | JupyterServer->>EnterpriseGateway: https GET api/kernelspecs 24 | EnterpriseGateway-->>JupyterServer: api/kernelspecs response 25 | JupyterServer-->>JupyterLab: api/kernelspecs response 26 | 27 | Note left of JupyterLab: kernel initialization 28 | JupyterLab->>JupyterServer: https POST api/sessions 29 | JupyterServer->>EnterpriseGateway: https POST api/kernels 30 | EnterpriseGateway->>ProcessProxy: launch_process() 31 | ProcessProxy->>Kernel: launch kernel 32 | ProcessProxy->>ResourceManager: confirm startup 33 | Kernel-->>ProcessProxy: connection info 34 | ResourceManager-->>ProcessProxy: state & host info 35 | ProcessProxy-->>EnterpriseGateway: complete connection info 36 | EnterpriseGateway->>Kernel: TCP socket requests 37 | Kernel-->>EnterpriseGateway: TCP socket handshakes 38 | EnterpriseGateway-->>JupyterServer: api/kernels response 39 | JupyterServer-->>JupyterLab: api/sessions response 40 | 41 | JupyterLab->>JupyterServer: ws GET api/kernels 42 | JupyterServer->>EnterpriseGateway: ws GET api/kernels 43 | EnterpriseGateway->>Kernel: kernel_info_request message 44 | Kernel-->>EnterpriseGateway: kernel_info_reply message 45 | EnterpriseGateway-->>JupyterServer: websocket upgrade response 46 | JupyterServer-->>JupyterLab: websocket upgrade response 47 | ``` 48 | -------------------------------------------------------------------------------- /docs/source/developers/index.rst: -------------------------------------------------------------------------------- 1 | Developers Guide 2 | ================ 3 | 4 | These pages target *developers* writing applications against the REST API, authoring process proxies for other resource managers, or integrating applications with remote kernel functionality. 5 | 6 | .. admonition:: Use cases 7 | 8 | - *As a developer, I want to explore supporting a different resource manager with Enterprise Gateway, by implementing a new `ProcessProxy` class such that I can easily take advantage of specific functionality provided by the resource manager.* 9 | - *As a developer, I want to extend the `nbclient` application to use a `KernelManager` that can leverage remote kernels spawned from Enterprise Gateway.* 10 | - *As a developer, I want to easily integrate the ability to launch remote kernels with existing platforms, so I can leverage my compute cluster in a customizable way.* 11 | - *As a developer, I am currently using Golang and need to implement a kernel launcher to allow the Go kernel I use to run remotely in my Kubernetes cluster.* 12 | - *As a developer, I'd like to extend some of the kernel container images and, eventually, create my own to better enable the data scientists I support.* 13 | - *As a developer, I need want to author my own Kernel-as-a-Service application.* 14 | 15 | .. toctree:: 16 | :maxdepth: 1 17 | :name: developers 18 | 19 | dev-process-proxy 20 | kernel-launcher 21 | kernel-specification 22 | custom-images 23 | kernel-library 24 | kernel-manager 25 | rest-api 26 | -------------------------------------------------------------------------------- /docs/source/developers/kernel-library.md: -------------------------------------------------------------------------------- 1 | # Standalone Remote Kernel Execution 2 | 3 | Remote kernels can be executed by using the `RemoteKernelManager` class directly. This enables running kernels using `ProcessProxy`s without requiring deployment of the Enterprise Gateway web application. This approach is also known as _Library Mode_. 4 | 5 | This can be useful in niche situations, for example, using [nbconvert](https://nbconvert.readthedocs.io/) or [nbclient](https://nbclient.readthedocs.io/) to execute a kernel on a remote cluster. 6 | 7 | Sample code using nbclient 0.2.0: 8 | 9 | ```python 10 | import nbformat 11 | from nbclient import NotebookClient 12 | from enterprise_gateway.services.kernels.remotemanager import RemoteKernelManager 13 | 14 | with open("my_notebook.ipynb") as fp: 15 | test_notebook = nbformat.read(fp, as_version=4) 16 | 17 | client = NotebookClient(nb=test_notebook, kernel_manager_class=RemoteKernelManager) 18 | client.execute(kernel_name='my_remote_kernel') 19 | ``` 20 | 21 | The above code will execute the notebook on a kernel named `my_remote_kernel` using its configured `ProcessProxy`. 22 | 23 | Depending on the process proxy, the _hosting application_ (e.g., `nbclient`) will likely need to be configured to run on the same network as the remote kernel. So, for example, with Kubernetes, `nbclient` would need to be configured as a Kubernetes POD. 24 | -------------------------------------------------------------------------------- /docs/source/developers/kernel-manager.md: -------------------------------------------------------------------------------- 1 | # Using Jupyter Server's `GatewayKernelManager` 2 | 3 | Another way to expose other Jupyter applications like `nbclient` or `papermill` to remote kernels is to use the [`GatewayKernelManager`](https://github.com/jupyter-server/jupyter_server/blob/745f5ba3f00280c1e1900326a7e08463d48a3912/jupyter_server/gateway/managers.py#L317) (and, implicitly, [`GatewayKernelClient`](https://github.com/jupyter-server/jupyter_server/blob/745f5ba3f00280c1e1900326a7e08463d48a3912/jupyter_server/gateway/managers.py#L562)) classes that are embedded in Jupyter Server. 4 | 5 | These classes essentially emulate the lower level [`KernelManager`](https://github.com/jupyter/jupyter_client/blob/10decd25308c306b6005cbf271b96493824a83e8/jupyter_client/manager.py#L84) and [`KernelClient`](https://github.com/jupyter/jupyter_client/blob/10decd25308c306b6005cbf271b96493824a83e8/jupyter_client/client.py#L75) classes but _forward_ their requests to/from a configured gateway server. Their necessary configuration for interacting with the gateway server is set on the [`GatewayClient` configurable](../users/client-config.md#gateway-client-configuration). 6 | 7 | This allows for the _hosting application_ to remain **outside** the resource-managed cluster since the kernel is actually being managed by the target gateway server. 8 | 9 | So, using the previous example, one my have... 10 | 11 | ```python 12 | import nbformat 13 | from nbclient import NotebookClient 14 | from jupyter_server.gateway.gateway_client import GatewayClient 15 | from jupyter_server.gateway.managers import GatewayKernelManager 16 | 17 | with open("my_notebook.ipynb") as fp: 18 | test_notebook = nbformat.read(fp, as_version=4) 19 | 20 | # Set any other gateway-specific parameters on the GatewayClient (singleton) instance 21 | gw_client = GatewayClient.instance() 22 | gw_client.url = "http://my-gateway-server.com:8888" 23 | 24 | client = NotebookClient(nb=test_notebook, kernel_manager_class=GatewayKernelManager) 25 | client.execute(kernel_name='my_remote_kernel') 26 | ``` 27 | 28 | In this case, `my_remote_kernel`'s kernel specification file actually resides on the Gateway server. `NotebookClient` will _think_ its talking to local `KernelManager` and `KernelClient` instances, when, in actuality, they are forwarding requests to (and getting response from) the Gateway server at 'http://my-gateway-server.com:8888'. 29 | -------------------------------------------------------------------------------- /docs/source/images/Scalability-After-JEG.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/docs/source/images/Scalability-After-JEG.gif -------------------------------------------------------------------------------- /docs/source/images/Scalability-Before-JEG.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/docs/source/images/Scalability-Before-JEG.gif -------------------------------------------------------------------------------- /docs/source/images/debug_configuration.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/docs/source/images/debug_configuration.png -------------------------------------------------------------------------------- /docs/source/images/deployment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/docs/source/images/deployment.png -------------------------------------------------------------------------------- /docs/source/images/process_proxy_hierarchy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/docs/source/images/process_proxy_hierarchy.png -------------------------------------------------------------------------------- /docs/source/images/yarnui.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/docs/source/images/yarnui.jpg -------------------------------------------------------------------------------- /docs/source/operators/config-culling.md: -------------------------------------------------------------------------------- 1 | # Culling idle kernels 2 | 3 | With the adoption of notebooks and interactive development for data science, a new "resource utilization" pattern has arisen, where kernel resources are locked for a given notebook, but due to interactive development processes it might be idle for a long period of time causing the cluster resources to starve. One way to workaround this problem is to enable the culling of idle kernels after a specific timeout period. 4 | 5 | Idle kernel culling is set to “off” by default. It’s enabled by setting `--RemoteKernelManager.cull_idle_timeout` to a positive value representing the number of seconds a kernel must remain idle to be culled (default: 0, recommended: 43200, 12 hours). 6 | 7 | ```{tip} 8 | When managing large clusters with limited resources, we recommend enabling the culling of idle kernels. 9 | ``` 10 | 11 | You can also configure the interval that the kernels are checked for their idle timeouts by adjusting the setting `--RemoteKernelManager.cull_interval` to a positive value. If the interval is not set or set to a non-positive value, the system uses 300 seconds as the default value: (default: 300 seconds). 12 | 13 | There are use-cases where we would like to enable only culling of idle kernels that have no connections (e.g. the notebook browser was closed without stopping the kernel first), this can be configured by adjusting the setting `--RemoteKernelManager.cull_connected` (default: False). 14 | 15 | Here's an updated start script that provides some default configuration to enable the culling of idle kernels: 16 | 17 | ```bash 18 | #!/bin/bash 19 | 20 | LOG=/var/log/enterprise_gateway.log 21 | PIDFILE=/var/run/enterprise_gateway.pid 22 | 23 | jupyter enterprisegateway --ip=0.0.0.0 --port_retries=0 --log-level=DEBUG \ 24 | --RemoteKernelManager.cull_idle_timeout=43200 --MappingKernelManager.cull_interval=60 > $LOG 2>&1 & 25 | 26 | if [ "$?" -eq 0 ]; then 27 | echo $! > $PIDFILE 28 | else 29 | exit 1 30 | fi 31 | ``` 32 | -------------------------------------------------------------------------------- /docs/source/operators/config-dynamic.md: -------------------------------------------------------------------------------- 1 | # Dynamic configurables 2 | 3 | Enterprise Gateway also supports the ability to update configuration variables without having to 4 | restart Enterprise Gateway. This enables the ability to do things like enable debug logging or 5 | adjust the maximum number of kernels per user, all without having to restart Enterprise Gateway. 6 | 7 | To enable dynamic configurables configure `EnterpriseGatewayApp.dynamic_config_interval` to a 8 | positive value (default is 0 or disabled). Since this is the number of seconds to poll Enterprise Gateway's configuration files, 9 | a value greater than 60 (1 minute) is recommended. This functionality works for most configuration 10 | values, but does have the following caveats: 11 | 12 | 1. Any configuration variables set on the command line (CLI) or via environment variables are 13 | NOT eligible for dynamic updates. This is because Jupyter gives those values priority over 14 | file-based configuration variables. 15 | 1. Any configuration variables tied to background processing may not reflect their update if 16 | the variable is not _observed_ for changes. For example, the code behind 17 | `RemoteKernelManager.cull_idle_timeout` may not reflect changes to the timeout period if 18 | that variable is not monitored (i.e., observed) for changes. 19 | 1. Only `Configurables` registered by Enterprise Gateway are eligible for dynamic updates. 20 | Currently, that list consists of the following (and their subclasses): EnterpriseGatewayApp, 21 | RemoteKernelManager, KernelSpecManager, and KernelSessionManager. 22 | 23 | As a result, operators and adminstrators are encouraged to configure Enterprise Gateway via configuration files with only static values configured via the command line or environment. 24 | 25 | Note that if `EnterpriseGatewayApp.dynamic_config_interval` is configured with a positive value 26 | via the configuration file (i.e., is eligible for updates) and is subsequently set to 0, then 27 | dynamic configuration updates will be disabled until Enterprise Gateway is restarted with a 28 | positive value. Therefore, we recommend `EnterpriseGatewayApp.dynamic_config_interval` be 29 | configured via the command line or environment. 30 | -------------------------------------------------------------------------------- /docs/source/operators/config-file.md: -------------------------------------------------------------------------------- 1 | # Configuration file options 2 | 3 | Placing configuration options into the configuration file `jupyter_enterprise_gateway_config.py` is recommended because this will enabled the use of the [_dynamic configurables_](config-dynamic.md/#dynamic-configurables) functionality. To generate a template configuration file, run the following: 4 | 5 | ```bash 6 | jupyter enterprisegateway --generate-config 7 | ``` 8 | 9 | This command will produce a `jupyter_enterprise_gateway_config.py` file, typically located in the invoking user's `$HOME/.jupyter` directory. The file contains python code, including comments, relative to each available configuration option. The actual option itself will also be commented out. To enable that option, set its value and uncomment the code. 10 | 11 | ```{Note} 12 | Some options may appear duplicated. For example, the `remote_hosts` trait appears on both `c.EnterpriseGatewayConfigMixin` and `c.EnterpriseGatewayApp`. This is due to how configurable traits appear in the class hierarchy. Since `EnterpriseGatewayApp` derives from `EnterpriseGatewayConfigMixin` and both are configurable classes, the output contains duplicated values. If both values are set, the value _closest_ to the derived class will be used (in this case, `EnterpriseGatewayApp`). 13 | ``` 14 | 15 | Here's an example entry. Note that its default value, when defined, is also displayed, along with the corresponding environment variable name: 16 | 17 | ```python 18 | ## Bracketed comma-separated list of hosts on which DistributedProcessProxy 19 | # kernels will be launched e.g., ['host1','host2']. 20 | # (EG_REMOTE_HOSTS env var - non-bracketed, just comma-separated) 21 | # Default: ['localhost'] 22 | # c.EnterpriseGatewayConfigMixin.remote_hosts = ['localhost'] 23 | ``` 24 | -------------------------------------------------------------------------------- /docs/source/operators/config-sys-env.md: -------------------------------------------------------------------------------- 1 | # System-owned environment variables 2 | 3 | The following environment variables are managed by Enterprise Gateway and listed here for completeness. 4 | 5 | ```{warning} 6 | Manually setting these variables could adversely affect operations. 7 | ``` 8 | 9 | ```text 10 | EG_DOCKER_MODE 11 | Docker only. Used by launch_docker.py to determine if the kernel container 12 | should be created using the swarm service API or the regular docker container 13 | API. Enterprise Gateway sets this value depending on whether the kernel is 14 | using the DockerSwarmProcessProxy or DockerProcessProxy. 15 | 16 | EG_RESPONSE_ADDRESS 17 | This value is set during each kernel launch and resides in the environment of 18 | the kernel launch process. Its value represents the address to which the remote 19 | kernel's connection information should be sent. Enterprise Gateway is listening 20 | on that socket and will associate that connnection information with the responding 21 | kernel. 22 | ``` 23 | -------------------------------------------------------------------------------- /docs/source/operators/deploy-conductor.md: -------------------------------------------------------------------------------- 1 | # IBM Spectrum Conductor deployments 2 | 3 | This information will be added shortly. The configuration is similar to that of [Hadoop YARN deployments](deploy-yarn-cluster.md) with the `ConductorClusterProcessProxy` used in place of `YARNClusterProcessProxy`. 4 | 5 | The following sample kernel specifications are currently available on IBM Spectrum Conductor: 6 | 7 | - spark_R_conductor_cluster 8 | - spark_python_conductor_cluster 9 | - spark_scala_conductor_cluster 10 | -------------------------------------------------------------------------------- /docs/source/operators/deploy-single.md: -------------------------------------------------------------------------------- 1 | # Single-server deployments 2 | 3 | Single-server deployment can be useful for development and is not meant to be run in production environments as it subjects the gateway server to resource exhaustion. 4 | 5 | Steps to deploy a single server are: 6 | 7 | 1. [Install Enterprise Gateway](installing-eg.md) 8 | 1. [Install the desired kernels](installing-kernels.md) 9 | 1. Install and configure the server and desired kernel specifications (see below) 10 | 1. [Launch Enterprise Gateway](launching-eg.md) 11 | 12 | If you just want to try Enterprise Gateway in a single-server setup, you can use the following kernels specification (no need for a kernel launcher since the kernel runs locally): 13 | 14 | ```json 15 | { 16 | "display_name": "Python 3 Local", 17 | "language": "python", 18 | "metadata": { 19 | "process_proxy": { 20 | "class_name": "enterprise_gateway.services.processproxies.processproxy.LocalProcessProxy" 21 | } 22 | }, 23 | "argv": ["python", "-m", "ipykernel_launcher", "-f", "{connection_file}"] 24 | } 25 | ``` 26 | 27 | `process_proxy` is optional (if Enterprise Gateway encounters a kernel specification without the `process_proxy` stanza, it will treat that specification as if it contained `LocalProcessProxy`). 28 | 29 | ```{tip} 30 | You can run a local kernel in [Distributed mode](./deploy-distributed.md) by setting `remote_hosts` to the localhost. Why would you do that? 31 | 32 | 1. One reason is that it decreases the window in which a port conflict can occur since the 5 kernel ports are created by the launcher (within the same process and therefore closer to the actual invocation of the kernel) rather than by the server prior to the launch of the kernel process. 33 | 2. The second reason is that auto-restarted kernels - when an issue occurs - say due to a port conflict - will create a new set of ports rather than try to re-use the same set that produced the failure in the first place. In this case, you'd want to use the [per-kernel configuration](config-kernel-override.md#per-kernel-configuration-overrides) approach and set `remote_hosts` in the config stanza of the `process_proxy` stanza (using the stanza instead of the global `EG_REMOTE_HOSTS` allows you to not interfere with the other resource managers configuration, e.g. Spark Standalone or YARN Client kernels - Those other kernels need to be able to continue leveraging the full cluster nodes). 34 | 35 | ``` 36 | -------------------------------------------------------------------------------- /docs/source/operators/installing-eg.md: -------------------------------------------------------------------------------- 1 | # Installing Enterprise Gateway (common) 2 | 3 | For new users, we **highly recommend** [installing Anaconda](https://www.anaconda.com/download). 4 | Anaconda conveniently installs Python, the [Jupyter Notebook](https://jupyter.readthedocs.io/en/latest/install.html), the [IPython kernel](http://ipython.readthedocs.io/en/stable/install/kernel_install.html) and other commonly used 5 | packages for scientific computing and data science. 6 | 7 | Use the following installation steps: 8 | 9 | - Download [Anaconda](https://www.anaconda.com/download). We recommend downloading Anaconda’s 10 | latest Python version (currently Python 3.9). 11 | 12 | - Install the version of Anaconda which you downloaded, following the instructions on the download page. 13 | 14 | - Install the latest version of Jupyter Enterprise Gateway from [PyPI](https://pypi.python.org/pypi/jupyter_enterprise_gateway/) 15 | or [conda forge](https://conda-forge.org/) along with its dependencies. 16 | 17 | ```{warning} 18 | Enterprise Gateway is currently incompatible with `jupyter_client >= 7.0`. As a result, you should **not** install Enterprise Gateway into the same Python environment in which you intend to run Jupyter Notebook or Jupyter Lab since they will likely be using `jupyter_client >= 7.0`. Since Enterprise Gateway is tupically installed on servers remote from the notebook users, this is usually not an issue. 19 | ``` 20 | 21 | ```bash 22 | # install using pip from pypi 23 | pip install --upgrade jupyter_enterprise_gateway 24 | ``` 25 | 26 | ```bash 27 | # install using conda from conda forge 28 | conda install -c conda-forge jupyter_enterprise_gateway 29 | ``` 30 | 31 | At this point, the Jupyter Enterprise Gateway deployment provides local kernel support which is fully compatible with Jupyter Kernel Gateway. 32 | 33 | To uninstall Jupyter Enterprise Gateway... 34 | 35 | ```bash 36 | #uninstall using pip 37 | pip uninstall jupyter_enterprise_gateway 38 | ``` 39 | 40 | ```bash 41 | #uninstall using conda 42 | conda uninstall jupyter_enterprise_gateway 43 | ``` 44 | -------------------------------------------------------------------------------- /docs/source/operators/launching-eg.md: -------------------------------------------------------------------------------- 1 | # Launching Enterprise Gateway (common) 2 | 3 | Very few arguments are necessary to minimally start Enterprise Gateway. The following command could be considered a minimal command: 4 | 5 | ```bash 6 | jupyter enterprisegateway --ip=0.0.0.0 --port_retries=0 7 | ``` 8 | 9 | where `--ip=0.0.0.0` exposes Enterprise Gateway on the public network and `--port_retries=0` ensures that a single instance will be started. 10 | 11 | ```{note} 12 | The ability to target resource-managed clusters (and use remote kernels) will require additional configuration settings depending on the resource manager. For additional information see the appropriate server-based deployment topic of our Operators Guide. 13 | ``` 14 | 15 | We recommend starting Enterprise Gateway as a background task. As a result, you might find it best to create a start script to maintain options, file redirection, etc. 16 | 17 | The following script starts Enterprise Gateway with `DEBUG` tracing enabled (default is `INFO`) and idle kernel culling for any kernels idle for 12 hours with idle check intervals occurring every 60 seconds. The Enterprise Gateway log can then be monitored via `tail -F enterprise_gateway.log` and it can be stopped via `kill $(cat enterprise_gateway.pid)` 18 | 19 | ```bash 20 | #!/bin/bash 21 | 22 | LOG=/var/log/enterprise_gateway.log 23 | PIDFILE=/var/run/enterprise_gateway.pid 24 | 25 | jupyter enterprisegateway --ip=0.0.0.0 --port_retries=0 --log-level=DEBUG --RemoteKernelManager.cull_idle_timeout=43200 --MappingKernelManager.cull_interval=60 > $LOG 2>&1 & 26 | if [ "$?" -eq 0 ]; then 27 | echo $! > $PIDFILE 28 | else 29 | exit 1 30 | fi 31 | ``` 32 | 33 | ```{tip} 34 | Remember that any options set via the command-line will not be available for [dynamic configuration funtionality](config-dynamic.md#dynamic-configurables). 35 | ``` 36 | -------------------------------------------------------------------------------- /docs/source/other/index.rst: -------------------------------------------------------------------------------- 1 | Other helpful information 2 | =========================== 3 | This section includes some additional information you might find helpful and that spans the various *guides*, like troubleshooting and related resources. 4 | 5 | .. toctree:: 6 | :maxdepth: 1 7 | :name: other 8 | 9 | troubleshooting 10 | related-resources 11 | -------------------------------------------------------------------------------- /docs/source/other/related-resources.md: -------------------------------------------------------------------------------- 1 | # Related Resources 2 | 3 | Here are some resources related to the Jupyter Enterprise Gateway project. 4 | 5 | - [Jupyter.org](https://jupyter.org) 6 | - [Jupyter Server Team Compass](https://github.com/jupyter-server/team-compass#jupyter-server-team-compass) 7 | - [Jupyter Calendar - Community Meetings](https://docs.jupyter.org/en/latest/community/content-community.html#jupyter-community-meetings) 8 | - [Jupyter Community Discourse Forum](https://discourse.jupyter.org/) 9 | - [Jupyter Kernel Gateway Github Repo](https://github.com/jupyter-server/kernel_gateway) - the source code for Kernel Gateway - which supports local kernels and notebook-hosted end-points. 10 | - [Jupyter Server Github Repo](https://github.com/jupyter-server/jupyter_server) - the source code for the Jupyter Server. Many of the Enterprise Gateway's handlers and kernel management classes either _are_ or are derived from the Jupyter Server classes. 11 | - [Jupyter Notebook Github Repo](https://github.com/jupyter/notebook) - the source code for the classic Notebook from which the gateways and Jupyter Server were derived. 12 | - [Jupyter Client Github Repo](https://github.com/jupyter/jupyter_client) - the source code for the base kernel lifecycle management and message classes. Enterprise Gateway extends the `KernelManager` classes of `jupyter_client`. 13 | -------------------------------------------------------------------------------- /docs/source/users/index.rst: -------------------------------------------------------------------------------- 1 | Users Guide 2 | =========== 3 | 4 | Because Enterprise Gateway is a headless web server, it is typically accessed from other applications like JupyterLab and Jupyter Notebook. 5 | 6 | .. admonition:: Use cases 7 | 8 | - *As a data scientist, I want to run my notebook using the Enterprise Gateway such that I can free up resources on my own laptop and leverage my company's large Hadoop YARN cluster to run my compute-intensive operations.* 9 | 10 | - *As a student, my Data Science 101 course is leveraging GPUs in our experiments. Since GPUs are expensive, we must share resources within the university's compute cluster and configure our Notebooks to leverage the department's Enterprise Gateway server, which can then spawn container-based kernels that have access to a GPU on Kubernetes.* 11 | 12 | The following assumes an Enterprise Gateway server has been configured and deployed. Please consult the `operators <../operators/index.html>`_ documentation to deploy and configure the Enterprise Gateway server. 13 | 14 | .. note:: 15 | There are two primary client applications that can use Enterprise Gateway, JupyterLab running on Jupyter Server and Jupyter Notebook. When a reference to a *Jupyter server* (lowercase 'server') or *the server* is made, the reference applies to both Jupyter Server and Jupyter Notebook. Generally speaking, the client-side behaviors are identical between the two, although references to Jupyter Server are preferred since it's more current. If anything is different, that difference will be noted, otherwise, please assume discussion of the two are interchangeable. 16 | 17 | .. toctree:: 18 | :maxdepth: 1 19 | :name: users 20 | 21 | installation 22 | connecting-to-eg 23 | client-config 24 | kernel-envs 25 | .. 26 | other clients (nbclient, papermill) 27 | -------------------------------------------------------------------------------- /docs/source/users/installation.md: -------------------------------------------------------------------------------- 1 | # Installing the client 2 | 3 | In terms of Enterprise Gateway, the client application is typically Jupyter Server (hosting JupyterLab) or Jupyter Notebook. These applications are then configured to connect to Enterprise Gateway. 4 | 5 | To install Jupyter Server via `pip`: 6 | 7 | ```bash 8 | pip install jupyter_server 9 | ``` 10 | 11 | or via `conda`: 12 | 13 | ```bash 14 | conda install -c conda-forge jupyter_server 15 | ``` 16 | 17 | Likewise, for Jupyter Notebook via `pip`: 18 | 19 | ```bash 20 | pip install notebook 21 | ``` 22 | 23 | or via `conda`: 24 | 25 | ```bash 26 | conda install -c conda-forge notebook 27 | ``` 28 | 29 | For additional information regarding the installation of [Jupyter Server](https://jupyter-server.readthedocs.io/en/latest/index.html) or [Jupyter Notebook](https://jupyter-notebook.readthedocs.io/en/latest/), please refer to their respective documentation (see embedded links). 30 | -------------------------------------------------------------------------------- /enterprise_gateway/__init__.py: -------------------------------------------------------------------------------- 1 | """Lazy-loading entrypoint for the enterprise gateway package.""" 2 | 3 | # Copyright (c) Jupyter Development Team. 4 | # Distributed under the terms of the Modified BSD License. 5 | from ._version import __version__ # noqa 6 | 7 | 8 | def launch_instance(*args, **kwargs): 9 | from enterprise_gateway.enterprisegatewayapp import launch_instance 10 | 11 | launch_instance(*args, **kwargs) 12 | -------------------------------------------------------------------------------- /enterprise_gateway/__main__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Jupyter Development Team. 2 | # Distributed under the terms of the Modified BSD License. 3 | """CLI entrypoint for the enterprise gateway package.""" 4 | 5 | if __name__ == "__main__": 6 | import enterprise_gateway.enterprisegatewayapp as app 7 | 8 | app.launch_instance() 9 | -------------------------------------------------------------------------------- /enterprise_gateway/_version.py: -------------------------------------------------------------------------------- 1 | """enterprise_gateway version info""" 2 | 3 | # Copyright (c) Jupyter Development Team. 4 | # Distributed under the terms of the Modified BSD License. 5 | 6 | __version__ = "3.3.0.dev0" 7 | -------------------------------------------------------------------------------- /enterprise_gateway/base/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/enterprise_gateway/base/__init__.py -------------------------------------------------------------------------------- /enterprise_gateway/base/handlers.py: -------------------------------------------------------------------------------- 1 | """Tornado handlers for the base of the API.""" 2 | 3 | # Copyright (c) Jupyter Development Team. 4 | # Distributed under the terms of the Modified BSD License. 5 | 6 | 7 | import json 8 | from typing import List 9 | 10 | import jupyter_server._version 11 | from jupyter_server.base.handlers import APIHandler 12 | from tornado import web 13 | 14 | from .._version import __version__ 15 | from ..mixins import CORSMixin, JSONErrorsMixin, TokenAuthorizationMixin 16 | 17 | 18 | class APIVersionHandler(TokenAuthorizationMixin, CORSMixin, JSONErrorsMixin, APIHandler): 19 | """ " 20 | Extends the jupyter_server base API handler with token auth, CORS, and 21 | JSON errors to produce version information for jupyter_server and gateway. 22 | """ 23 | 24 | def get(self): 25 | """Get the API version.""" 26 | # not authenticated, so give as few info as possible 27 | # to be backwards compatibile, use only 'version' for the jupyter_server version 28 | # and be more specific for gateway_version 29 | self.finish( 30 | json.dumps({"version": jupyter_server.__version__, "gateway_version": __version__}) 31 | ) 32 | 33 | 34 | class NotFoundHandler(JSONErrorsMixin, web.RequestHandler): 35 | """ 36 | Catches all requests and responds with 404 JSON messages. 37 | 38 | Installed as the fallback error for all unhandled requests. 39 | 40 | Raises 41 | ------ 42 | tornado.web.HTTPError 43 | Always 404 Not Found 44 | """ 45 | 46 | def prepare(self): 47 | """Prepare the response.""" 48 | raise web.HTTPError(404) 49 | 50 | 51 | default_handlers: List[tuple] = [(r"/api", APIVersionHandler), (r"/(.*)", NotFoundHandler)] 52 | -------------------------------------------------------------------------------- /enterprise_gateway/client/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/enterprise_gateway/client/__init__.py -------------------------------------------------------------------------------- /enterprise_gateway/itests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Jupyter Development Team. 2 | # Distributed under the terms of the Modified BSD License. 3 | from tornado import ioloop 4 | 5 | 6 | def teardown(): 7 | """The test fixture appears to leak something on certain platforms that 8 | endlessly tries an async socket connect and fails after the tests end. 9 | As a stopgap, force a cleanup here. 10 | """ 11 | ioloop.IOLoop.current().stop() 12 | ioloop.IOLoop.current().close(True) 13 | -------------------------------------------------------------------------------- /enterprise_gateway/itests/kernels/authorization_test/kernel.json: -------------------------------------------------------------------------------- 1 | { 2 | "display_name": "Authorization Testing", 3 | "language": "python", 4 | "metadata": { 5 | "process_proxy": { 6 | "class_name": "enterprise_gateway.services.processproxies.processproxy.LocalProcessProxy", 7 | "config": { 8 | "authorized_users": "bob,alice,bad_guy", 9 | "unauthorized_users": "bad_guy" 10 | } 11 | } 12 | }, 13 | "env": {}, 14 | "argv": ["python", "-m", "ipykernel_launcher", "-f", "{connection_file}"] 15 | } 16 | -------------------------------------------------------------------------------- /enterprise_gateway/itests/test_authorization.py: -------------------------------------------------------------------------------- 1 | import os 2 | import unittest 3 | 4 | from enterprise_gateway.client.gateway_client import GatewayClient 5 | 6 | 7 | class TestAuthorization(unittest.TestCase): 8 | KERNELSPEC = os.getenv("AUTHORIZATION_KERNEL_NAME", "authorization_test") 9 | 10 | @classmethod 11 | def setUpClass(cls): 12 | super().setUpClass() 13 | 14 | # initialize environment 15 | cls.gateway_client = GatewayClient() 16 | 17 | def setUp(self): 18 | pass 19 | 20 | def tearDown(self): 21 | pass 22 | 23 | def test_authorized_users(self): 24 | kernel = None 25 | try: 26 | kernel = self.gateway_client.start_kernel(TestAuthorization.KERNELSPEC, username="bob") 27 | result, has_error = kernel.execute("print('The cow jumped over the moon.')") 28 | self.assertEqual(result, "The cow jumped over the moon.\n") 29 | self.assertEqual(has_error, False) 30 | finally: 31 | if kernel: 32 | self.gateway_client.shutdown_kernel(kernel) 33 | 34 | def test_unauthorized_users(self): 35 | kernel = None 36 | try: 37 | kernel = self.gateway_client.start_kernel( 38 | TestAuthorization.KERNELSPEC, username="bad_guy" 39 | ) 40 | self.assertTrue(False, msg="Unauthorization exception expected!") 41 | except Exception as be: 42 | self.assertRegex(be.args[0], "403") 43 | finally: 44 | if kernel: 45 | self.gateway_client.shutdown_kernel(kernel) 46 | 47 | 48 | if __name__ == "__main__": 49 | unittest.main() 50 | -------------------------------------------------------------------------------- /enterprise_gateway/itests/test_base.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | expected_hostname = os.getenv("ITEST_HOSTNAME_PREFIX", "") + "*" # use ${KERNEL_USERNAME} on k8s 4 | expected_application_id = os.getenv( 5 | "EXPECTED_APPLICATION_ID", "application_*" 6 | ) # use 'spark-application-*' on k8s 7 | expected_spark_version = os.getenv("EXPECTED_SPARK_VERSION", "3.2.*") # use '2.4.*' on k8s 8 | expected_spark_master = os.getenv("EXPECTED_SPARK_MASTER", "yarn") # use 'k8s:*' on k8s 9 | expected_deploy_mode = os.getenv("EXPECTED_DEPLOY_MODE", "(cluster|client)") # use 'client' on k8s 10 | 11 | 12 | class TestBase: 13 | def get_expected_application_id(self): 14 | return expected_application_id 15 | 16 | def get_expected_spark_version(self): 17 | return expected_spark_version 18 | 19 | def get_expected_spark_master(self): 20 | return expected_spark_master 21 | 22 | def get_expected_deploy_mode(self): 23 | return expected_deploy_mode 24 | 25 | def get_expected_hostname(self): 26 | return expected_hostname 27 | -------------------------------------------------------------------------------- /enterprise_gateway/services/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/enterprise_gateway/services/__init__.py -------------------------------------------------------------------------------- /enterprise_gateway/services/api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/enterprise_gateway/services/api/__init__.py -------------------------------------------------------------------------------- /enterprise_gateway/services/api/handlers.py: -------------------------------------------------------------------------------- 1 | """Tornado handlers for kernel specs.""" 2 | 3 | # Copyright (c) Jupyter Development Team. 4 | # Distributed under the terms of the Modified BSD License. 5 | 6 | import os 7 | from typing import List 8 | 9 | from jupyter_server.utils import ensure_async 10 | from tornado import web 11 | 12 | from ...mixins import CORSMixin 13 | 14 | 15 | class BaseSpecHandler(CORSMixin, web.StaticFileHandler): 16 | """Exposes the ability to return specifications from static files""" 17 | 18 | @staticmethod 19 | def get_resource_metadata() -> tuple: 20 | """Returns the (resource, mime-type) for the handlers spec.""" 21 | pass 22 | 23 | def initialize(self) -> None: 24 | """Initializes the instance of this class to serve files. 25 | 26 | The handler is initialized to serve files from the directory 27 | where this module is defined. `path` parameter will be overridden. 28 | """ 29 | web.StaticFileHandler.initialize(self, path=os.path.dirname(__file__)) 30 | 31 | async def get(self) -> None: 32 | """Handler for a get on a specific handler""" 33 | resource_name, content_type = self.get_resource_metadata() 34 | self.set_header("Content-Type", content_type) 35 | res = web.StaticFileHandler.get(self, resource_name) 36 | await ensure_async(res) 37 | 38 | def options(self, **kwargs) -> None: 39 | """Method for properly handling CORS pre-flight""" 40 | self.finish() 41 | 42 | 43 | class SpecJsonHandler(BaseSpecHandler): 44 | """Exposes a JSON swagger specification""" 45 | 46 | @staticmethod 47 | def get_resource_metadata() -> tuple: 48 | """Get the resource metadata.""" 49 | return "swagger.json", "application/json" 50 | 51 | 52 | class APIYamlHandler(BaseSpecHandler): 53 | """Exposes a YAML swagger specification""" 54 | 55 | @staticmethod 56 | def get_resource_metadata() -> tuple: 57 | """Get the resource metadata.""" 58 | return "swagger.yaml", "text/x-yaml" 59 | 60 | 61 | default_handlers: List[tuple] = [ 62 | (f"/api/{SpecJsonHandler.get_resource_metadata()[0]}", SpecJsonHandler), 63 | (f"/api/{APIYamlHandler.get_resource_metadata()[0]}", APIYamlHandler), 64 | ] 65 | -------------------------------------------------------------------------------- /enterprise_gateway/services/kernels/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/enterprise_gateway/services/kernels/__init__.py -------------------------------------------------------------------------------- /enterprise_gateway/services/kernelspecs/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Jupyter Development Team. 2 | # Distributed under the terms of the Modified BSD License. 3 | 4 | from .kernelspec_cache import KernelSpecCache # noqa 5 | -------------------------------------------------------------------------------- /enterprise_gateway/services/processproxies/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/enterprise_gateway/services/processproxies/__init__.py -------------------------------------------------------------------------------- /enterprise_gateway/services/processproxies/spark_operator.py: -------------------------------------------------------------------------------- 1 | """A spark operator process proxy.""" 2 | 3 | # Copyright (c) Jupyter Development Team. 4 | # Distributed under the terms of the Modified BSD License. 5 | from __future__ import annotations 6 | 7 | from ..kernels.remotemanager import RemoteKernelManager 8 | from .crd import CustomResourceProcessProxy 9 | 10 | 11 | class SparkOperatorProcessProxy(CustomResourceProcessProxy): 12 | """Spark operator process proxy.""" 13 | 14 | # Identifies the kind of object being managed by this process proxy. 15 | # For these values we will prefer the values found in the 'kind' field 16 | # of the object's metadata. This attribute is strictly used to provide 17 | # context to log messages. 18 | object_kind = "SparkApplication" 19 | 20 | def __init__(self, kernel_manager: RemoteKernelManager, proxy_config: dict): 21 | """Initialize the proxy.""" 22 | super().__init__(kernel_manager, proxy_config) 23 | self.group = "sparkoperator.k8s.io" 24 | self.version = "v1beta2" 25 | self.plural = "sparkapplications" 26 | -------------------------------------------------------------------------------- /enterprise_gateway/services/sessions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/enterprise_gateway/services/sessions/__init__.py -------------------------------------------------------------------------------- /enterprise_gateway/services/sessions/handlers.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Jupyter Development Team. 2 | # Distributed under the terms of the Modified BSD License. 3 | """Tornado handlers for session CRUD.""" 4 | from typing import List 5 | 6 | import jupyter_server.services.sessions.handlers as jupyter_server_handlers 7 | import tornado 8 | from jupyter_server.utils import ensure_async 9 | 10 | from ...mixins import CORSMixin, JSONErrorsMixin, TokenAuthorizationMixin 11 | 12 | 13 | class SessionRootHandler( 14 | TokenAuthorizationMixin, CORSMixin, JSONErrorsMixin, jupyter_server_handlers.SessionRootHandler 15 | ): 16 | """Extends the jupyter_server root session handler with token auth, CORS, and 17 | JSON errors. 18 | """ 19 | 20 | async def get(self) -> None: 21 | """Overrides the super class method to honor the kernel listing 22 | configuration setting. 23 | 24 | Raises 25 | ------ 26 | tornado.web.HTTPError 27 | If eg_list_kernels is False, respond with 403 Forbidden 28 | """ 29 | if "eg_list_kernels" not in self.settings or not self.settings["eg_list_kernels"]: 30 | raise tornado.web.HTTPError(403, "Forbidden") 31 | else: 32 | await ensure_async(super().get()) 33 | 34 | 35 | default_handlers: List[tuple] = [] 36 | for path, cls in jupyter_server_handlers.default_handlers: 37 | if cls.__name__ in globals(): 38 | # Use the same named class from here if it exists 39 | default_handlers.append((path, globals()[cls.__name__])) 40 | else: 41 | # Everything should have CORS and token auth 42 | bases = (TokenAuthorizationMixin, CORSMixin, cls) 43 | default_handlers.append((path, type(cls.__name__, bases, {}))) 44 | -------------------------------------------------------------------------------- /enterprise_gateway/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Jupyter Development Team. 2 | # Distributed under the terms of the Modified BSD License. 3 | from tornado import ioloop 4 | 5 | 6 | def teardown(): 7 | """The test fixture appears to leak something on certain platforms that 8 | endlessly tries an async socket connect and fails after the tests end. 9 | As a stopgap, force a cleanup here. 10 | """ 11 | ioloop.IOLoop.current().stop() 12 | # Close is not necessary since process termination closes the loop. This was causing intermittent 13 | # `Event loop is closed` exceptions. These didn't affect the test resutls, but produced output that 14 | # was otherwise misleading noise. 15 | # ioloop.IOLoop.current().close(True) 16 | -------------------------------------------------------------------------------- /enterprise_gateway/tests/resources/failing_code2.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": { 7 | "collapsed": false 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import not-a-real-module" 12 | ] 13 | } 14 | ], 15 | "metadata": { 16 | "kernelspec": { 17 | "display_name": "Python 2", 18 | "language": "python", 19 | "name": "python2" 20 | }, 21 | "language_info": { 22 | "codemirror_mode": { 23 | "name": "ipython", 24 | "version": 3 25 | }, 26 | "file_extension": ".py", 27 | "mimetype": "text/x-python", 28 | "name": "python", 29 | "nbconvert_exporter": "python", 30 | "pygments_lexer": "ipython3", 31 | "version": "2.7.10" 32 | } 33 | }, 34 | "nbformat": 4, 35 | "nbformat_minor": 0 36 | } 37 | -------------------------------------------------------------------------------- /enterprise_gateway/tests/resources/failing_code3.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": { 7 | "collapsed": false 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import not-a-real-module" 12 | ] 13 | } 14 | ], 15 | "metadata": { 16 | "kernelspec": { 17 | "display_name": "Python 3", 18 | "language": "python", 19 | "name": "python3" 20 | }, 21 | "language_info": { 22 | "codemirror_mode": { 23 | "name": "ipython", 24 | "version": 3 25 | }, 26 | "file_extension": ".py", 27 | "mimetype": "text/x-python", 28 | "name": "python", 29 | "nbconvert_exporter": "python", 30 | "pygments_lexer": "ipython3", 31 | "version": "3.4.3" 32 | } 33 | }, 34 | "nbformat": 4, 35 | "nbformat_minor": 0 36 | } 37 | -------------------------------------------------------------------------------- /enterprise_gateway/tests/resources/kernels/kernel_defaults_test/kernel.json: -------------------------------------------------------------------------------- 1 | { 2 | "display_name": "Kernel Defaults Testing", 3 | "language": "python", 4 | "env": { 5 | "KERNEL_VAR1": "kernel_var1_default", 6 | "KERNEL_VAR2": "kernel_var2_default", 7 | "OTHER_VAR1": "other_var1_default", 8 | "OTHER_VAR2": "other_var2_default", 9 | "PROCESS_VAR1": "process_var1_default", 10 | "PROCESS_VAR2": "process_var2_default" 11 | }, 12 | "argv": ["python", "-m", "ipykernel_launcher", "-f", "{connection_file}"] 13 | } 14 | -------------------------------------------------------------------------------- /enterprise_gateway/tests/resources/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Hello world! 5 | 6 | 7 |

Hello world!

8 | 9 | 10 | -------------------------------------------------------------------------------- /enterprise_gateway/tests/resources/simple_api2.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import json" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": null, 17 | "metadata": { 18 | "collapsed": true 19 | }, 20 | "outputs": [], 21 | "source": [ 22 | "name = 'Test Name'" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": null, 28 | "metadata": { 29 | "collapsed": false 30 | }, 31 | "outputs": [], 32 | "source": [ 33 | "# GET /name\n", 34 | "print name " 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": null, 40 | "metadata": { 41 | "collapsed": true 42 | }, 43 | "outputs": [], 44 | "source": [ 45 | "# POST /name\n", 46 | "req = json.loads(REQUEST)\n", 47 | "name = req['body']\n", 48 | "print(name)" 49 | ] 50 | } 51 | ], 52 | "metadata": { 53 | "kernelspec": { 54 | "display_name": "Python 2", 55 | "language": "python", 56 | "name": "python2" 57 | }, 58 | "language_info": { 59 | "codemirror_mode": { 60 | "name": "ipython", 61 | "version": 3.0 62 | }, 63 | "file_extension": ".py", 64 | "mimetype": "text/x-python", 65 | "name": "python", 66 | "nbconvert_exporter": "python", 67 | "pygments_lexer": "ipython3", 68 | "version": "2.7.10" 69 | } 70 | }, 71 | "nbformat": 4, 72 | "nbformat_minor": 0 73 | } 74 | -------------------------------------------------------------------------------- /enterprise_gateway/tests/resources/simple_api3.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import json" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": null, 17 | "metadata": { 18 | "collapsed": true 19 | }, 20 | "outputs": [], 21 | "source": [ 22 | "name = 'Test Name'" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": null, 28 | "metadata": { 29 | "collapsed": false 30 | }, 31 | "outputs": [], 32 | "source": [ 33 | "# GET /name\n", 34 | "print(name)" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": null, 40 | "metadata": { 41 | "collapsed": true 42 | }, 43 | "outputs": [], 44 | "source": [ 45 | "# POST /name\n", 46 | "req = json.loads(REQUEST)\n", 47 | "name = req['body']\n", 48 | "print(name)" 49 | ] 50 | } 51 | ], 52 | "metadata": { 53 | "kernelspec": { 54 | "display_name": "Python 3", 55 | "language": "python", 56 | "name": "python3" 57 | }, 58 | "language_info": { 59 | "codemirror_mode": { 60 | "name": "ipython", 61 | "version": 3 62 | }, 63 | "file_extension": ".py", 64 | "mimetype": "text/x-python", 65 | "name": "python", 66 | "nbconvert_exporter": "python", 67 | "pygments_lexer": "ipython3", 68 | "version": "3.4.3" 69 | } 70 | }, 71 | "nbformat": 4, 72 | "nbformat_minor": 0 73 | } 74 | -------------------------------------------------------------------------------- /enterprise_gateway/tests/resources/unknown_kernel.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": { 7 | "collapsed": false 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "# GET /fake", 12 | "print 'I am not a real lang!'" 13 | ] 14 | } 15 | ], 16 | "metadata": { 17 | "kernelspec": { 18 | "display_name": "Fake Language 2000", 19 | "language": "fakelang", 20 | "name": "fakelang2000" 21 | }, 22 | "language_info": { 23 | "codemirror_mode": { 24 | "name": "fakelang", 25 | "version": 2000 26 | }, 27 | "file_extension": ".fl", 28 | "mimetype": "text/x-fake-lang", 29 | "name": "fakelang", 30 | "nbconvert_exporter": "fakelang", 31 | "pygments_lexer": "fakelang", 32 | "version": "2000" 33 | } 34 | }, 35 | "nbformat": 4, 36 | "nbformat_minor": 0 37 | } 38 | -------------------------------------------------------------------------------- /enterprise_gateway/tests/resources/zen2.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": { 7 | "collapsed": false 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import this" 12 | ] 13 | } 14 | ], 15 | "metadata": { 16 | "kernelspec": { 17 | "display_name": "Python 2", 18 | "language": "python", 19 | "name": "python2" 20 | }, 21 | "language_info": { 22 | "codemirror_mode": { 23 | "name": "ipython", 24 | "version": 3 25 | }, 26 | "file_extension": ".py", 27 | "mimetype": "text/x-python", 28 | "name": "python", 29 | "nbconvert_exporter": "python", 30 | "pygments_lexer": "ipython3", 31 | "version": "2.7.10" 32 | } 33 | }, 34 | "nbformat": 4, 35 | "nbformat_minor": 0 36 | } 37 | -------------------------------------------------------------------------------- /enterprise_gateway/tests/resources/zen3.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": { 7 | "collapsed": false 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import this" 12 | ] 13 | } 14 | ], 15 | "metadata": { 16 | "kernelspec": { 17 | "display_name": "Python 3", 18 | "language": "python", 19 | "name": "python3" 20 | }, 21 | "language_info": { 22 | "codemirror_mode": { 23 | "name": "ipython", 24 | "version": 3 25 | }, 26 | "file_extension": ".py", 27 | "mimetype": "text/x-python", 28 | "name": "python", 29 | "nbconvert_exporter": "python", 30 | "pygments_lexer": "ipython3", 31 | "version": "3.4.3" 32 | } 33 | }, 34 | "nbformat": 4, 35 | "nbformat_minor": 0 36 | } 37 | -------------------------------------------------------------------------------- /etc/docker/demo-base/README.md: -------------------------------------------------------------------------------- 1 | # What this image Gives You 2 | 3 | - Ubuntu base image : bionic 4 | - Hadoop 2.7.7 5 | - Apache Spark 2.4.6 6 | - Java 1.8 runtime 7 | - Mini-conda latest (python 3.8) with R packages 8 | - Toree 0.4.0-incubating 9 | - `jovyan` service user, with system users `elyra`, `bob`, and `alice`. The jovyan uid is `1000` to match other jupyter 10 | images. 11 | - Password-less ssh for service user 12 | - Users have HDFS folder setup at startup 13 | 14 | # Basic Use 15 | 16 | As of the 0.9.0 release of [Jupyter Enterprise Gateway](https://github.com/jupyter-server/enterprise_gateway/releases) 17 | this image can be started as a separate YARN cluster to better demonstrate remote kernel capabilities. See section 18 | [Dual Mode](https://hub.docker.com/r/elyra/enterprise-gateway/#dual_mode) on the enterprise-gateway page for command 19 | usage. 20 | -------------------------------------------------------------------------------- /etc/docker/demo-base/core-site.xml.template: -------------------------------------------------------------------------------- 1 | 2 | 3 | fs.defaultFS 4 | hdfs://HOSTNAME:9000 5 | 6 | 7 | -------------------------------------------------------------------------------- /etc/docker/demo-base/fix-permissions: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # set permissions on a directory 3 | # after any installation, if a directory needs to be (human) user-writable, 4 | # run this script on it. 5 | # It will make everything in the directory owned by the group $NB_GID 6 | # and writable by that group. 7 | # Deployments that want to set a specific user id can preserve permissions 8 | # by adding the `--group-add users` line to `docker run`. 9 | 10 | # uses find to avoid touching files that already have the right permissions, 11 | # which would cause massive image explosion 12 | 13 | # right permissions are: 14 | # group=$NB_GID 15 | # AND permissions include group rwX (directory-execute) 16 | # AND directories have setuid,setgid bits set 17 | 18 | set -e 19 | 20 | for d in "$@"; do 21 | find "$d" \ 22 | ! \( \ 23 | -group $NB_GID \ 24 | -a -perm -g+rwX \ 25 | \) \ 26 | -exec chgrp $NB_GID {} \; \ 27 | -exec chmod g+rwX {} \; 28 | # setuid,setgid *on directories only* 29 | find "$d" \ 30 | \( \ 31 | -type d \ 32 | -a ! -perm -6000 \ 33 | \) \ 34 | -exec chmod +6000 {} \; 35 | done 36 | -------------------------------------------------------------------------------- /etc/docker/demo-base/hdfs-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | dfs.replication 4 | 1 5 | 6 | 7 | -------------------------------------------------------------------------------- /etc/docker/demo-base/mapred-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | mapreduce.framework.name 4 | yarn 5 | 6 | 7 | -------------------------------------------------------------------------------- /etc/docker/demo-base/ssh_config: -------------------------------------------------------------------------------- 1 | Host * 2 | UserKnownHostsFile /dev/null 3 | StrictHostKeyChecking no 4 | LogLevel quiet 5 | Port 2122 6 | -------------------------------------------------------------------------------- /etc/docker/demo-base/yarn-site.xml.template: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | yarn.nodemanager.vmem-check-enabled 6 | false 7 | 8 | 9 | 10 | yarn.nodemanager.aux-services 11 | mapreduce_shuffle 12 | 13 | 14 | 15 | 16 | Number of seconds after an application finishes before the nodemanager's 17 | DeletionService will delete the application's localized file directory 18 | and log directory. 19 | 20 | To diagnose Yarn application problems, set this property's value large 21 | enough (for example, to 600 = 10 minutes) to permit examination of these 22 | directories. After changing the property's value, you must restart the 23 | nodemanager in order for it to have an effect. 24 | 25 | The roots of Yarn applications' work directories is configurable with 26 | the yarn.nodemanager.local-dirs property (see below), and the roots 27 | of the Yarn applications' log directories is configurable with the 28 | yarn.nodemanager.log-dirs property (see also below). 29 | 30 | yarn.nodemanager.delete.debug-delay-sec 31 | 600 32 | 33 | 34 | 35 | yarn.resourcemanager.scheduler.address 36 | HOSTNAME:8030 37 | 38 | 39 | yarn.resourcemanager.address 40 | HOSTNAME:8032 41 | 42 | 43 | yarn.resourcemanager.webapp.address 44 | HOSTNAME:8088 45 | 46 | 47 | yarn.resourcemanager.resource-tracker.address 48 | HOSTNAME:8031 49 | 50 | 51 | yarn.resourcemanager.admin.address 52 | HOSTNAME:8033 53 | 54 | 55 | yarn.application.classpath 56 | /usr/hdp/current/hadoop/etc/hadoop, /usr/hdp/current/hadoop/share/hadoop/common/*, /usr/hdp/current/hadoop/share/hadoop/common/lib/*, /usr/hdp/current/hadoop/share/hadoop/hdfs/*, /usr/hdp/current/hadoop/share/hadoop/hdfs/lib/*, /usr/hdp/current/hadoop/share/hadoop/mapreduce/*, /usr/hdp/current/hadoop/share/hadoop/mapreduce/lib/*, /usr/hdp/current/hadoop/share/hadoop/yarn/*, /usr/hdp/current/hadoop/share/hadoop/yarn/lib/* 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /etc/docker/enterprise-gateway-demo/bootstrap-enterprise-gateway.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This file is a copy of /etc/bootstrap.sh but invokes Jupyter Enterprise Gateway in its "deamon" case. 4 | # It also checks for --help or no options before starting anything... 5 | 6 | 7 | CMD=${1:-"--help"} 8 | if [[ "$CMD" == "--help" ]]; then 9 | echo "" 10 | echo "usage: docker run -it[d] --rm -h -p 8888:8888 [-p 8088:8088 -p 8042:8042] " 11 | echo "" 12 | echo "where is:" 13 | echo " --gateway ... Invokes Enterprise Gateway as user 'jovyan' directly. Useful for daemon behavior." 14 | echo " --yarn ... Runs container as standalone YARN master - no Enterprise Gateway is started." 15 | echo " --help ... Produces this message." 16 | echo " ... Invokes ''. Use ='/bin/bash' to explore within the container." 17 | echo "" 18 | echo "Tips:" 19 | echo "1) You can target a different YARN cluster by using '-e YARN_HOST='" 20 | echo "2) You can \"bring your own kernels\" by mounting to /tmp/byok/kernels (e.g., -v my-kernels-dir:/tmp/byok/kernels)" 21 | echo "3) It is advised that port '8888' be mapped to a host port, although the host port number is not" 22 | echo " required to be '8888'. Mapping of ports '8088' and '8042' is also strongly recommended" 23 | echo " for YARN application monitoring if running standalone." 24 | exit 0 25 | elif [[ "$CMD" != "--gateway" && "$CMD" != "--yarn" ]]; then # invoke w/o starting YARN 26 | "$*" 27 | exit 0 28 | fi 29 | 30 | : ${YARN_HOST:=$HOSTNAME} 31 | export FROM="EG" 32 | /usr/local/bin/bootstrap-yarn-spark.sh $* 33 | 34 | # Note that '--yarn' functionality is a subset of '--gateway' functionality 35 | 36 | if [[ "$CMD" == "--gateway" ]]; 37 | then 38 | sudo sed -i "s/HOSTNAME/$YARN_HOST/" /usr/local/bin/start-enterprise-gateway.sh 39 | /usr/local/bin/start-enterprise-gateway.sh 40 | fi 41 | 42 | exit 0 43 | -------------------------------------------------------------------------------- /etc/docker/enterprise-gateway-demo/start-enterprise-gateway.sh.template: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Allow for mounts of kernelspecs to /tmp/byok/kernels 4 | export JUPYTER_PATH=${JUPYTER_PATH:-/tmp/byok} 5 | 6 | # Enterprise Gateway variables 7 | export EG_REMOTE_HOSTS=${EG_REMOTE_HOSTS:-HOSTNAME} 8 | export EG_SSH_PORT=${EG_SSH_PORT:-2122} 9 | export KG_IP=${KG_IP:-0.0.0.0} 10 | export KG_PORT=${KG_PORT:-8888} 11 | export KG_PORT_RETRIES=${KG_PORT_RETRIES:-0} 12 | 13 | # To use tunneling set this variable to 'True' and run as root. 14 | export EG_ENABLE_TUNNELING=${EG_ENABLE_TUNNELING:-False} 15 | 16 | export EG_LOG_LEVEL=${EG_LOG_LEVEL:-DEBUG} 17 | export EG_CULL_IDLE_TIMEOUT=${EG_CULL_IDLE_TIMEOUT:-43200} # default to 12 hours 18 | export EG_CULL_CONNECTED=${EG_CULL_CONNECTED:-True} 19 | 20 | echo "Starting Jupyter Enterprise Gateway..." 21 | 22 | jupyter enterprisegateway \ 23 | --log-level=${EG_LOG_LEVEL} \ 24 | --EnterpriseGatewayApp.inherited_envs=PYSPARK_PYTHON \ 25 | --MappingKernelManager.cull_idle_timeout=${EG_CULL_IDLE_TIMEOUT} \ 26 | --MappingKernelManager.cull_interval=30 \ 27 | --MappingKernelManager.cull_connected=${EG_CULL_CONNECTED} 2>&1 | tee /usr/local/share/jupyter/enterprise-gateway.log 28 | -------------------------------------------------------------------------------- /etc/docker/enterprise-gateway/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG BASE_CONTAINER=jupyter/minimal-notebook:2023-03-13 2 | 3 | FROM $BASE_CONTAINER 4 | 5 | ARG SPARK_VERSION 6 | 7 | ENV SPARK_VER $SPARK_VERSION 8 | ENV SPARK_HOME /opt/spark 9 | 10 | 11 | RUN mamba install --quiet --yes \ 12 | cffi \ 13 | send2trash \ 14 | requests \ 15 | future \ 16 | pycryptodomex && \ 17 | conda clean --all && \ 18 | fix-permissions $CONDA_DIR && \ 19 | fix-permissions /home/$NB_USER 20 | 21 | USER root 22 | 23 | RUN apt update && apt install -yq curl openjdk-8-jdk 24 | 25 | ENV JAVA_HOME /usr/lib/jvm/java 26 | RUN ln -s $(readlink -f /usr/bin/javac | sed "s:/bin/javac::") ${JAVA_HOME} 27 | 28 | # Download and install Spark 29 | RUN curl -s https://archive.apache.org/dist/spark/spark-${SPARK_VER}/spark-${SPARK_VER}-bin-hadoop2.7.tgz | \ 30 | tar -xz -C /opt && \ 31 | ln -s ${SPARK_HOME}-${SPARK_VER}-bin-hadoop2.7 $SPARK_HOME && \ 32 | mkdir -p /usr/hdp/current && \ 33 | ln -s ${SPARK_HOME}-${SPARK_VER}-bin-hadoop2.7 /usr/hdp/current/spark2-client 34 | 35 | # Install Enterprise Gateway wheel and kernelspecs 36 | COPY jupyter_enterprise_gateway*.whl /tmp/ 37 | RUN pip install /tmp/jupyter_enterprise_gateway*.whl && \ 38 | rm -f /tmp/jupyter_enterprise_gateway*.whl 39 | 40 | ADD jupyter_enterprise_gateway_kernelspecs*.tar.gz /usr/local/share/jupyter/kernels/ 41 | ADD jupyter_enterprise_gateway_kernel_image_files*.tar.gz /usr/local/bin/ 42 | 43 | COPY start-enterprise-gateway.sh /usr/local/bin/ 44 | 45 | RUN chown jovyan:users /usr/local/bin/start-enterprise-gateway.sh && \ 46 | chmod 0755 /usr/local/bin/start-enterprise-gateway.sh && \ 47 | touch /usr/local/share/jupyter/enterprise-gateway.log && \ 48 | chown -R jovyan:users /usr/local/share/jupyter /usr/local/bin/kernel-launchers && \ 49 | chmod 0666 /usr/local/share/jupyter/enterprise-gateway.log && \ 50 | rm -f /usr/local/bin/bootstrap-kernel.sh 51 | 52 | USER jovyan 53 | 54 | CMD ["/usr/local/bin/start-enterprise-gateway.sh"] 55 | 56 | EXPOSE 8888 57 | 58 | WORKDIR /usr/local/bin 59 | -------------------------------------------------------------------------------- /etc/docker/enterprise-gateway/README.md: -------------------------------------------------------------------------------- 1 | This image adds support for [Jupyter Enterprise Gateway](https://jupyter-enterprise-gateway.readthedocs.io/en/latest/) within a Kubernetes or Docker Swarm cluster. It is currently built on jupyter/minimal-notebook as a base with Apache Spark 2.4.6 installed on top. 2 | 3 | **Note: If you're looking for the YARN-based image of this name, it has been moved to [elyra/enterprise-gateway-demo](https://hub.docker.com/r/elyra/enterprise-gateway-demo/).** 4 | 5 | # What it Gives You 6 | 7 | - [Jupyter Enterprise Gateway](https://github.com/jupyter-server/enterprise_gateway) 8 | - Python/R/Toree kernels that can be launched and distributed across a managed cluster. 9 | 10 | # Basic Use 11 | 12 | Pull this image, along with all of the elyra/kernel-\* images to each of your managed nodes. Although manual seeding of images across the cluster is not required, it is highly recommended since kernel startup times can timeout and image downloads can seriously undermine that window. 13 | 14 | ## Kubernetes 15 | 16 | Enterprise Gateway is deployed into Kubernetes using [Helm](https://helm.sh/). See the [Kubernetes section of our Operator's Guide](https://jupyter-enterprise-gateway.readthedocs.io/en/latest/operators/deploy-kubernetes.html) for further details. 17 | 18 | ## Docker Swarm 19 | 20 | Download the [`docker-compose.yml`](https://github.com/jupyter-server/enterprise_gateway/blob/main/etc/docker/docker-compose.yml) file and make any necessary changes for your configuration. The compose file consists of three pieces, the Enterprise Gateway container itself, a proxy layer container, and a Docker network. We recommend that a volume be used so that the kernelspec files can be accessed outside of the container since we've found those to require post-deployment modifications from time to time. 21 | 22 | ## Docker (Traditional) 23 | 24 | Same instructions as for Docker Swarm using [`docker-compose.yml`](https://github.com/jupyter-server/enterprise_gateway/blob/main/etc/docker/docker-compose.yml). Please note that you can still run Enterprise Gateway as a traditional docker container within a Docker Swarm cluster, yet have the kernel containers launched as Docker Swarm services since how the kernels are launched is a function of their configured process proxy class. 25 | 26 | For more information, check our [repo](https://github.com/jupyter-server/enterprise_gateway) and [docs](https://jupyter-enterprise-gateway.readthedocs.io/en/latest/). 27 | -------------------------------------------------------------------------------- /etc/docker/kernel-image-puller/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG BASE_CONTAINER=python:3.10 2 | FROM $BASE_CONTAINER 3 | 4 | WORKDIR /usr/src/app 5 | 6 | COPY requirements.txt ./ 7 | RUN pip install --no-cache-dir -r requirements.txt 8 | 9 | COPY kernel_image_puller.py ./ 10 | COPY image_fetcher.py ./ 11 | ARG OS=Debian_12 12 | 13 | # Install crictl for use by KIP when non-docker installations are encountered. 14 | RUN echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/${OS}/ /"|tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list 15 | RUN curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/Release.key | apt-key add - 16 | RUN apt-get update && apt-get install cri-tools 17 | 18 | RUN echo $PATH 19 | # The following environment variables are supported - defaults provided. Override as needed. 20 | ENV KIP_GATEWAY_HOST http://localhost:8888 21 | ENV KIP_INTERVAL 300 22 | ENV KIP_LOG_LEVEL INFO 23 | ENV KIP_NUM_PULLERS 2 24 | ENV KIP_NUM_RETRIES 3 25 | ENV KIP_PULL_POLICY 'IfNotPresent' 26 | 27 | CMD [ "python", "./kernel_image_puller.py" ] 28 | -------------------------------------------------------------------------------- /etc/docker/kernel-image-puller/README.md: -------------------------------------------------------------------------------- 1 | This image is responsible for contacting the configured [Jupyter Enterprise Gateway](https://jupyter-enterprise-gateway.readthedocs.io/en/latest/) instance within a Kubernetes or Docker Swarm cluster and pulling the set of kernel-based images to the node on which it is running. 2 | 3 | # What it Gives You 4 | 5 | - The ability to add new nodes and have kernel images on those nodes automatically populated. 6 | - The ability to configure new kernelspecs that use different images and have those images pulled to all cluster nodes. 7 | 8 | # Basic Use 9 | 10 | Deploy [enterprise-gateway](https://hub.docker.com/r/elyra/enterprise-gateway/) per its instructions and configured to the appropriate environment. 11 | 12 | As part of that deployment, Kernel Image Puller (KIP) will be launched on each node. On Kubernetes, this will be accomplished via a DaemonSet. On Docker Swarm, it will be via a global service. KIP will then contact the configured Enterprise Gateway instance, fetch the set of in-use kernelspecs, parse out the image names and pull those images. 13 | 14 | There are a few points of configuration listed below - all of which are environment variables (defaults in parenthesis). 15 | 16 | - `KIP_GATEWAY_HOST` (`http://localhost:8888`) 17 | - `KIP_INTERVAL` (`300`) 18 | - `KIP_LOG_LEVEL` (`INFO`) 19 | - `KIP_NUM_PULLERS` (`2`) 20 | - `KIP_NUM_RETRIES` (`3`) 21 | - `KIP_PULL_POLICY` (`IfNotPresent`) 22 | - `KIP_IMAGE_FETCHER` (`KernelSpecsFetcher`) 23 | 24 | For more information, check our [repo](https://github.com/jupyter-server/enterprise_gateway) and [docs](https://jupyter-enterprise-gateway.readthedocs.io/en/latest/). 25 | -------------------------------------------------------------------------------- /etc/docker/kernel-image-puller/requirements.txt: -------------------------------------------------------------------------------- 1 | docker>=3.7.2 2 | kubernetes>=17.17.0 3 | requests>=2.7,<3.0 4 | -------------------------------------------------------------------------------- /etc/docker/kernel-py/Dockerfile: -------------------------------------------------------------------------------- 1 | # Ubuntu 18.04.1 LTS Bionic 2 | ARG BASE_CONTAINER=jupyter/scipy-notebook:2023-03-13 3 | FROM $BASE_CONTAINER 4 | 5 | ENV PATH=$PATH:$CONDA_DIR/bin 6 | 7 | # Add debugger support 8 | RUN pip install --upgrade ipykernel 9 | 10 | RUN conda install --quiet --yes \ 11 | cffi \ 12 | future \ 13 | pycryptodomex && \ 14 | conda clean --all && \ 15 | fix-permissions $CONDA_DIR && \ 16 | fix-permissions /home/$NB_USER 17 | 18 | ADD jupyter_enterprise_gateway_kernel_image_files*.tar.gz /usr/local/bin/ 19 | 20 | USER root 21 | 22 | RUN apt-get update && apt-get install -yq --no-install-recommends \ 23 | libkrb5-dev \ 24 | && rm -rf /var/lib/apt/lists/* 25 | 26 | RUN chown jovyan:users /usr/local/bin/bootstrap-kernel.sh && \ 27 | chmod 0755 /usr/local/bin/bootstrap-kernel.sh && \ 28 | chown -R jovyan:users /usr/local/bin/kernel-launchers 29 | 30 | USER jovyan 31 | 32 | ENV KERNEL_LANGUAGE python 33 | 34 | # Disble healthcheck inherited from notebook image 35 | HEALTHCHECK NONE 36 | 37 | CMD /usr/local/bin/bootstrap-kernel.sh 38 | -------------------------------------------------------------------------------- /etc/docker/kernel-py/README.md: -------------------------------------------------------------------------------- 1 | This image enables the use of an IPython kernel launched from [Jupyter Enterprise Gateway](https://jupyter-enterprise-gateway.readthedocs.io/en/latest/) within a Kubernetes or Docker Swarm cluster. It is built on [jupyter/scipy-notebook](https://hub.docker.com/r/jupyter/scipy-notebook/). 2 | 3 | # What it Gives You 4 | 5 | - IPython kernel support (with debugger) 6 | - [Data science libraries](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#jupyter-scipy-notebook) 7 | 8 | # Basic Use 9 | 10 | Deploy [enterprise-gateway](https://hub.docker.com/r/elyra/enterprise-gateway/) per its instructions and configured to the appropriate environment. 11 | 12 | Launch a gateway-enabled Jupyter Notebook application against the Enterprise Gateway instance and pick the desired kernel to use in your notebook. 13 | 14 | For more information, check our [repo](https://github.com/jupyter-server/enterprise_gateway) and [docs](https://jupyter-enterprise-gateway.readthedocs.io/en/latest/). 15 | -------------------------------------------------------------------------------- /etc/docker/kernel-r/Dockerfile: -------------------------------------------------------------------------------- 1 | # Ubuntu 18.04.1 LTS Bionic 2 | ARG BASE_CONTAINER=jupyter/r-notebook:2023-03-13 3 | FROM $BASE_CONTAINER 4 | 5 | RUN conda install --quiet --yes \ 6 | 'r-argparse' \ 7 | pycryptodomex && \ 8 | conda clean --all && \ 9 | fix-permissions $CONDA_DIR 10 | 11 | ADD jupyter_enterprise_gateway_kernel_image_files*.tar.gz /usr/local/bin/ 12 | 13 | # Switch back to root to modify ownerships 14 | USER root 15 | 16 | RUN apt-get update && apt-get install -y \ 17 | less \ 18 | curl \ 19 | libkrb5-dev \ 20 | && rm -rf /var/lib/apt/lists/* 21 | 22 | RUN chown jovyan:users /usr/local/bin/bootstrap-kernel.sh && \ 23 | chmod 0755 /usr/local/bin/bootstrap-kernel.sh && \ 24 | chown -R jovyan:users /usr/local/bin/kernel-launchers 25 | 26 | USER jovyan 27 | 28 | ENV KERNEL_LANGUAGE R 29 | 30 | # Disble healthcheck inherited from notebook image 31 | HEALTHCHECK NONE 32 | 33 | CMD /usr/local/bin/bootstrap-kernel.sh 34 | -------------------------------------------------------------------------------- /etc/docker/kernel-r/README.md: -------------------------------------------------------------------------------- 1 | This image enables the use of an IRKernel kernel launched from [Jupyter Enterprise Gateway](https://jupyter-enterprise-gateway.readthedocs.io/en/latest/) within a Kubernetes or Docker Swarm cluster. It is currently built on [jupyter/r-notebook](https://hub.docker.com/r/jupyter/r-notebook/). 2 | 3 | # What it Gives You 4 | 5 | - IRKernel kernel support 6 | 7 | # Basic Use 8 | 9 | Deploy [enterprise-gateway](https://hub.docker.com/r/elyra/enterprise-gateway/) per its instructions and configured to the appropriate environment. 10 | 11 | Launch a gateway-enabled Jupyter Notebook application against the Enterprise Gateway instance and pick the desired kernel to use in your notebook. 12 | 13 | For more information, check our [repo](https://github.com/jupyter-server/enterprise_gateway) and [docs](https://jupyter-enterprise-gateway.readthedocs.io/en/latest/). 14 | -------------------------------------------------------------------------------- /etc/docker/kernel-scala/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG HUB_ORG 2 | ARG SPARK_VERSION 3 | 4 | # TODO: Restore usage of SPARK_VERSION ARG once https://github.com/jupyter/enterprise_gateway/pull/867 is merged 5 | ARG BASE_CONTAINER=$HUB_ORG/spark:v$SPARK_VERSION 6 | FROM $BASE_CONTAINER 7 | 8 | ADD jupyter_enterprise_gateway_kernel_image_files*.tar.gz /usr/local/bin/ 9 | 10 | USER root 11 | 12 | # Create/setup the jovyan system user 13 | RUN adduser --system -uid 1000 jovyan --ingroup users && \ 14 | chown jovyan:users /usr/local/bin/bootstrap-kernel.sh && \ 15 | chmod 0755 /usr/local/bin/bootstrap-kernel.sh && \ 16 | chmod 0777 /opt/spark/work-dir && \ 17 | chown -R jovyan:users /usr/local/bin/kernel-launchers 18 | 19 | USER jovyan 20 | ENV KERNEL_LANGUAGE scala 21 | CMD /usr/local/bin/bootstrap-kernel.sh 22 | -------------------------------------------------------------------------------- /etc/docker/kernel-scala/README.md: -------------------------------------------------------------------------------- 1 | This image enables the use of a Scala ([Apache Toree](https://toree.apache.org/)) kernel launched from [Jupyter Enterprise Gateway](http://jupyter-enterprise-gateway.readthedocs.io/en/latest/) within a Kubernetes or Docker Swarm cluster. It is built on [elyra/spark:v2.4.6](https://hub.docker.com/r/elyra/spark/) deriving from the [Apache Spark 2.4.6 release](https://spark.apache.org/docs/2.4.6/). Note: The ability to use the kernel within Spark within a Docker Swarm configuration probably won't yield the expected results. 2 | 3 | # What it Gives You 4 | 5 | - Scala (Toree) kernel support 6 | - Spark on kubernetes support from within a Jupyter Notebook 7 | 8 | # Basic Use 9 | 10 | Deploy [enterprise-gateway](https://hub.docker.com/r/elyra/enterprise-gateway/) per its instructions and configured to the appropriate environment. 11 | 12 | Launch a gateway-enabled Jupyter Notebook application against the Enterprise Gateway instance and pick the desired kernel to use in your notebook. 13 | 14 | For more information, check our [repo](https://github.com/jupyter-server/enterprise_gateway) and [docs](https://jupyter-enterprise-gateway.readthedocs.io/en/latest/). 15 | -------------------------------------------------------------------------------- /etc/docker/kernel-spark-py/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG HUB_ORG 2 | ARG TAG 3 | 4 | # Ubuntu 18.04.1 LTS Bionic 5 | ARG BASE_CONTAINER=$HUB_ORG/kernel-py:$TAG 6 | FROM $BASE_CONTAINER 7 | 8 | ARG SPARK_VERSION 9 | 10 | ENV SPARK_VER $SPARK_VERSION 11 | ENV SPARK_HOME /opt/spark 12 | ENV KERNEL_LANGUAGE python 13 | ENV R_LIBS_USER $R_LIBS_USER:${SPARK_HOME}/R/lib 14 | ENV PATH $PATH:$SPARK_HOME/bin 15 | 16 | USER root 17 | 18 | RUN dpkg --purge --force-depends ca-certificates-java \ 19 | && apt-get update \ 20 | && apt-get install -yq --no-install-recommends \ 21 | ca-certificates \ 22 | ca-certificates-java \ 23 | openjdk-8-jdk \ 24 | less \ 25 | curl \ 26 | libssl-dev \ 27 | && rm -rf /var/lib/apt/lists/* 28 | 29 | ENV JAVA_HOME /usr/lib/jvm/java 30 | RUN ln -s $(readlink -f /usr/bin/javac | sed "s:/bin/javac::") ${JAVA_HOME} 31 | 32 | # Download and install Spark 33 | RUN curl -s https://archive.apache.org/dist/spark/spark-${SPARK_VER}/spark-${SPARK_VER}-bin-hadoop2.7.tgz | \ 34 | tar -xz -C /opt && \ 35 | ln -s ${SPARK_HOME}-${SPARK_VER}-bin-hadoop2.7 $SPARK_HOME 36 | 37 | # Download entrypoint.sh from matching tag 38 | RUN cd /opt/ && \ 39 | wget https://raw.githubusercontent.com/apache/spark/v${SPARK_VER}/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh && \ 40 | chmod a+x /opt/entrypoint.sh && \ 41 | sed -i 's/tini -s/tini -g/g' /opt/entrypoint.sh 42 | 43 | WORKDIR $SPARK_HOME/work-dir 44 | # Ensure that work-dir is writable by everyone 45 | RUN chmod 0777 $SPARK_HOME/work-dir 46 | 47 | ENTRYPOINT [ "/opt/entrypoint.sh" ] 48 | 49 | USER jovyan 50 | -------------------------------------------------------------------------------- /etc/docker/kernel-spark-py/README.md: -------------------------------------------------------------------------------- 1 | This image enables the use of an IPython kernel launched from [Jupyter Enterprise Gateway](https://jupyter-enterprise-gateway.readthedocs.io/en/latest/) within a Kubernetes cluster. It is built on the base image [elyra/kernel-py](https://hub.docker.com/r/elyra/kernel-py/), and adds [Apache Spark 2.4.6](https://spark.apache.org/docs/2.4.6/). Note: The ability to use the kernel within Spark within a Docker Swarm configuration probably won't yield the expected results. 2 | 3 | # What it Gives You 4 | 5 | - IPython kernel support (with debugger) 6 | - [Data science libraries](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#jupyter-scipy-notebook) 7 | - Spark on kubernetes support from within a Jupyter Notebook 8 | 9 | # Basic Use 10 | 11 | Deploy [enterprise-gateway](https://hub.docker.com/r/elyra/enterprise-gateway/) per its instructions and configured to the appropriate environment. 12 | 13 | Launch a gateway-enabled Jupyter Notebook application against the Enterprise Gateway instance and pick the desired kernel to use in your notebook. 14 | 15 | For more information, check our [repo](https://github.com/jupyter-server/enterprise_gateway) and [docs](https://jupyter-enterprise-gateway.readthedocs.io/en/latest/). 16 | -------------------------------------------------------------------------------- /etc/docker/kernel-spark-r/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG HUB_ORG 2 | ARG TAG 3 | 4 | ARG BASE_CONTAINER=$HUB_ORG/kernel-r:$TAG 5 | FROM $BASE_CONTAINER 6 | 7 | ARG SPARK_VERSION 8 | 9 | USER root 10 | 11 | ENV SPARK_VER $SPARK_VERSION 12 | ENV SPARK_HOME /opt/spark 13 | ENV KERNEL_LANGUAGE=R 14 | ENV R_LIBS_USER $R_LIBS_USER:${R_HOME}/library:${SPARK_HOME}/R/lib 15 | ENV PATH $PATH:$SPARK_HOME/bin 16 | 17 | RUN dpkg --purge --force-depends ca-certificates-java \ 18 | && apt-get update \ 19 | && apt-get install -y \ 20 | ca-certificates \ 21 | ca-certificates-java \ 22 | openjdk-8-jdk \ 23 | libssl-dev \ 24 | && rm -rf /var/lib/apt/lists/* 25 | 26 | ENV JAVA_HOME /usr/lib/jvm/java 27 | RUN ln -s $(readlink -f /usr/bin/javac | sed "s:/bin/javac::") ${JAVA_HOME} 28 | 29 | # Download and install Spark 30 | RUN curl -s https://archive.apache.org/dist/spark/spark-${SPARK_VER}/spark-${SPARK_VER}-bin-hadoop2.7.tgz | \ 31 | tar -xz -C /opt && \ 32 | ln -s ${SPARK_HOME}-${SPARK_VER}-bin-hadoop2.7 $SPARK_HOME 33 | 34 | # Download entrypoint.sh from matching tag 35 | RUN cd /opt/ && \ 36 | wget https://raw.githubusercontent.com/apache/spark/v${SPARK_VER}/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh && \ 37 | chmod a+x /opt/entrypoint.sh && \ 38 | sed -i 's/tini -s/tini -g/g' /opt/entrypoint.sh 39 | 40 | WORKDIR $SPARK_HOME/work-dir 41 | # Ensure that work-dir is writable by everyone 42 | RUN chmod 0777 $SPARK_HOME/work-dir 43 | 44 | ENTRYPOINT [ "/opt/entrypoint.sh" ] 45 | 46 | USER jovyan 47 | -------------------------------------------------------------------------------- /etc/docker/kernel-spark-r/README.md: -------------------------------------------------------------------------------- 1 | This image enables the use of an IRKernel kernel launched from [Jupyter Enterprise Gateway](https://jupyter-enterprise-gateway.readthedocs.io/en/latest/) within a Kubernetes cluster. It is built on the base image [elyra/kernel-r](https://hub.docker.com/r/elyra/kernel-r/), and adds [Apache Spark 2.4.6](https://spark.apache.org/docs/2.4.6/). Note: The ability to use the kernel within Spark within a Docker Swarm configuration probably won't yield the expected results. 2 | 3 | # What it Gives You 4 | 5 | - IRkernel kernel support 6 | - Spark on kubernetes support from within a Jupyter Notebook 7 | 8 | # Basic Use 9 | 10 | Deploy [enterprise-gateway](https://hub.docker.com/r/elyra/enterprise-gateway/) per its instructions and configured to the appropriate environment. 11 | 12 | Launch a gateway-enabled Jupyter Notebook application against the Enterprise Gateway instance and pick the desired kernel to use in your notebook. 13 | 14 | For more information, check our [repo](https://github.com/jupyter-server/enterprise_gateway) and [docs](https://jupyter-enterprise-gateway.readthedocs.io/en/latest/). 15 | -------------------------------------------------------------------------------- /etc/docker/kernel-tf-gpu-py/Dockerfile: -------------------------------------------------------------------------------- 1 | # Ubuntu:xenial 2 | ARG BASE_CONTAINER=tensorflow/tensorflow:2.9.1-gpu 3 | FROM $BASE_CONTAINER 4 | 5 | ENV DEBIAN_FRONTEND=noninteractive 6 | 7 | RUN apt-get update && apt-get install -yq \ 8 | build-essential \ 9 | libsm6 \ 10 | libxext-dev \ 11 | libxrender1 \ 12 | netcat \ 13 | python3-dev \ 14 | tzdata \ 15 | unzip && \ 16 | rm -rf /var/lib/apt/lists/* && \ 17 | pip install --upgrade future pycryptodomex ipykernel 18 | 19 | ADD jupyter_enterprise_gateway_kernel_image_files*.tar.gz /usr/local/bin/ 20 | 21 | USER root 22 | 23 | RUN adduser --system --uid 1000 --gid 100 jovyan && \ 24 | chown jovyan:users /usr/local/bin/bootstrap-kernel.sh && \ 25 | chmod 0755 /usr/local/bin/bootstrap-kernel.sh && \ 26 | chown -R jovyan:users /usr/local/bin/kernel-launchers 27 | 28 | 29 | USER jovyan 30 | ENV KERNEL_LANGUAGE python 31 | CMD /usr/local/bin/bootstrap-kernel.sh 32 | -------------------------------------------------------------------------------- /etc/docker/kernel-tf-gpu-py/README.md: -------------------------------------------------------------------------------- 1 | This image enables the use of an IPython kernel launched from [Jupyter Enterprise Gateway](https://jupyter-enterprise-gateway.readthedocs.io/en/latest/) within a Kubernetes or Docker Swarm cluster that can perform Tensorflow operations. It is currently built on [tensorflow/tensorflow:2.7.0-gpu-jupyter](https://hub.docker.com/r/tensorflow/tensorflow/) deriving from the [tensorflow](https://github.com/tensorflow/tensorflow) project. 2 | 3 | # What it Gives You 4 | 5 | - IPython kernel support supplemented with Tensorflow functionality (and debugger) 6 | 7 | # Basic Use 8 | 9 | Deploy [enterprise-gateway](https://hub.docker.com/r/elyra/enterprise-gateway/) per its instructions and configured to the appropriate environment. 10 | 11 | Launch a gateway-enabled Jupyter Notebook application against the Enterprise Gateway instance and pick the desired kernel to use in your notebook. 12 | 13 | For more information, check our [repo](https://github.com/jupyter-server/enterprise_gateway) and [docs](https://jupyter-enterprise-gateway.readthedocs.io/en/latest/). 14 | -------------------------------------------------------------------------------- /etc/docker/kernel-tf-py/Dockerfile: -------------------------------------------------------------------------------- 1 | # Ubuntu:Bionic 2 | # TensorFlow 2.4.0 3 | ARG BASE_CONTAINER=jupyter/tensorflow-notebook:2023-10-20 4 | 5 | FROM $BASE_CONTAINER 6 | 7 | ENV KERNEL_LANGUAGE python 8 | 9 | ADD jupyter_enterprise_gateway_kernel_image_files*.tar.gz /usr/local/bin/ 10 | 11 | RUN conda install --quiet --yes \ 12 | pillow \ 13 | future \ 14 | pycryptodomex && \ 15 | fix-permissions $CONDA_DIR 16 | 17 | USER root 18 | 19 | RUN chown jovyan:users /usr/local/bin/bootstrap-kernel.sh && \ 20 | chmod 0755 /usr/local/bin/bootstrap-kernel.sh && \ 21 | chown -R jovyan:users /usr/local/bin/kernel-launchers 22 | 23 | USER jovyan 24 | 25 | # Disble healthcheck inherited from notebook image 26 | HEALTHCHECK NONE 27 | 28 | CMD [ "/usr/local/bin/bootstrap-kernel.sh" ] 29 | -------------------------------------------------------------------------------- /etc/docker/kernel-tf-py/README.md: -------------------------------------------------------------------------------- 1 | This image enables the use of an IPython kernel launched from [Jupyter Enterprise Gateway](https://jupyter-enterprise-gateway.readthedocs.io/en/latest/) within a Kubernetes or Docker Swarm cluster that can perform Tensorflow operations. It is currently built on the [jupyter/tensorflow-notebook](https://hub.docker.com/r/jupyter/tensorflow-notebook) image deriving from the [jupyter/tensorflow-notebook](https://github.com/jupyter/docker-stacks/tree/main/images/tensorflow-notebook) project. 2 | 3 | # What it Gives You 4 | 5 | - IPython kernel support supplemented with Tensorflow functionality (and debugger) 6 | 7 | # Basic Use 8 | 9 | Deploy [enterprise-gateway](https://hub.docker.com/r/elyra/enterprise-gateway/) per its instructions and configured to the appropriate environment. 10 | 11 | Launch a gateway-enabled Jupyter Notebook application against the Enterprise Gateway instance and pick the desired kernel to use in your notebook. 12 | 13 | For more information, check our [repo](https://github.com/jupyter-server/enterprise_gateway) and [docs](https://jupyter-enterprise-gateway.readthedocs.io/en/latest/). 14 | -------------------------------------------------------------------------------- /etc/kernel-launchers/operators/scripts/sparkoperator.k8s.io-v1beta2.yaml.j2: -------------------------------------------------------------------------------- 1 | apiVersion: "sparkoperator.k8s.io/v1beta2" 2 | kind: SparkApplication 3 | metadata: 4 | name: {{ kernel_resource_name }} 5 | spec: 6 | restartPolicy: 7 | type: Never 8 | type: Python 9 | pythonVersion: "3" 10 | sparkVersion: 2.4.5 11 | image: {{ kernel_image }} 12 | mainApplicationFile: "local:///usr/local/bin/kernel-launchers/python/scripts/launch_ipykernel.py" 13 | arguments: 14 | - "--kernel-id" 15 | - "{{ kernel_id }}" 16 | - "--spark-context-initialization-mode" 17 | - "{{ spark_context_initialization_mode }}" 18 | - "--response-address" 19 | - "{{ eg_response_address }}" 20 | - "--port-range" 21 | - "{{ eg_port_range }}" 22 | - "--public-key" 23 | - "{{ eg_public_key }}" 24 | driver: 25 | annotations: 26 | cluster-autoscaler.kubernetes.io/safe-to-evict: "false" 27 | env: 28 | # Add any custom envs here that aren't already configured for the kernel's environment 29 | # Note: For envs to flow to the pods, the webhook server must be enabled during deployment 30 | # e.g., helm install my-release spark-operator/spark-operator --namespace spark-operator --set webhook.enable=true 31 | # - name: MY_DRIVER_ENV 32 | # value: "my_driver_value" 33 | serviceAccount: "{{ kernel_service_account_name }}" 34 | labels: 35 | kernel_id: "{{ kernel_id }}" 36 | app: enterprise-gateway 37 | component: kernel 38 | cores: 1 39 | coreLimit: 1000m 40 | memory: 1g 41 | executor: 42 | env: 43 | # Add any custom envs here that aren't already configured for the kernel's environment 44 | # Note: For envs to flow to the pods, the webhook server must be enabled during deployment 45 | # e.g., helm install my-release spark-operator/spark-operator --namespace spark-operator --set webhook.enable=true 46 | # - name: MY_EXECUTOR_ENV 47 | # value: "my_executor_value" 48 | labels: 49 | kernel_id: "{{ kernel_id }}" 50 | app: enterprise-gateway 51 | component: worker 52 | image: {{ kernel_executor_image }} 53 | instances: 2 54 | cores: 1 55 | coreLimit: 1000m 56 | memory: 1g 57 | {% if kernel_sparkapp_config_map %} 58 | sparkConfigMap: {{ kernel_sparkapp_config_map }} 59 | {% endif %} 60 | -------------------------------------------------------------------------------- /etc/kernel-launchers/scala/toree-launcher/build.sbt: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Jupyter Development Team. 3 | * Distributed under the terms of the Modified BSD License. 4 | */ 5 | 6 | name := "toree-launcher" 7 | 8 | version := sys.props.getOrElse("version", default = "1.0").replaceAll("dev[0-9]", "SNAPSHOT") 9 | 10 | scalaVersion := "2.12.12" 11 | 12 | resolvers += "Typesafe Repo" at "https://repo.typesafe.com/typesafe/releases/" 13 | /* resolvers += "Sonatype Repository" at "https://oss.sonatype.org/content/repositories/releases/" */ 14 | resolvers += "Sonatype Maven Central Mirror" at "https://maven-central.storage-download.googleapis.com/maven2/" 15 | 16 | libraryDependencies += "com.typesafe.play" %% "play-json" % "2.7.4" // Apache v2 17 | libraryDependencies += "org.apache.toree" % "toree-assembly" % "0.5.0-incubating" from "https://repository.apache.org/content/repositories/orgapachetoree-1020/org/apache/toree/toree-assembly/0.5.0-incubating/toree-assembly-0.5.0-incubating.jar" 18 | -------------------------------------------------------------------------------- /etc/kernel-launchers/scala/toree-launcher/project/build.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Jupyter Development Team. 3 | # Distributed under the terms of the Modified BSD License. 4 | # 5 | sbt.version = 1.3.12 6 | -------------------------------------------------------------------------------- /etc/kernel-launchers/scala/toree-launcher/project/plugins.sbt: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Jupyter Development Team. 3 | * Distributed under the terms of the Modified BSD License. 4 | */ 5 | 6 | logLevel := Level.Warn 7 | 8 | /* 9 | * Following plugins have a dependency on sbt v0.13 10 | */ 11 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.5") 12 | addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "1.0.0") 13 | -------------------------------------------------------------------------------- /etc/kernel-launchers/scala/toree-launcher/src/main/scala/launcher/KernelProfile.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) Jupyter Development Team. 3 | * Distributed under the terms of the Modified BSD License. 4 | */ 5 | 6 | package launcher 7 | 8 | 9 | import java.util.UUID.randomUUID 10 | import play.api.libs.json._ 11 | import scala.util.Random 12 | import launcher.utils.SocketUtils 13 | 14 | 15 | case class KernelProfile(hb_port : Int, 16 | control_port : Int, 17 | iopub_port : Int, 18 | stdin_port : Int, 19 | shell_port : Int, 20 | key : String, 21 | kernel_name : String, 22 | signature_scheme : String, 23 | transport : String, 24 | ip : String) 25 | 26 | object KernelProfile { 27 | 28 | def newKey() : String = randomUUID.toString 29 | 30 | def createJsonProfile(portLowerBound: Int = -1, 31 | portUpperBound: Int = -1) : String = { 32 | 33 | implicit val writes = Json.writes[KernelProfile] 34 | 35 | val newKernelProfile = new KernelProfile( 36 | hb_port = SocketUtils.findPort(portLowerBound, portUpperBound), 37 | control_port = SocketUtils.findPort(portLowerBound, portUpperBound), 38 | iopub_port = SocketUtils.findPort(portLowerBound, portUpperBound), 39 | stdin_port = SocketUtils.findPort(portLowerBound, portUpperBound), 40 | shell_port = SocketUtils.findPort(portLowerBound, portUpperBound), 41 | key = newKey(), 42 | kernel_name = "Apache Toree Scala", transport = "tcp", ip = "0.0.0.0", 43 | signature_scheme = "hmac-sha256" 44 | ) 45 | 46 | Json.prettyPrint(Json.toJson(newKernelProfile)) 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /etc/kernel-launchers/scala/toree-launcher/src/main/scala/launcher/utils/SecurityUtils.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) Jupyter Development Team. 3 | * Distributed under the terms of the Modified BSD License. 4 | */ 5 | 6 | package launcher.utils 7 | import scala.util.Random 8 | import java.nio.charset.StandardCharsets 9 | import java.security.Key 10 | import java.security.KeyFactory 11 | import java.security.PublicKey 12 | import java.security.spec.X509EncodedKeySpec 13 | import java.util.Base64 14 | import javax.crypto.Cipher 15 | import javax.crypto.spec.SecretKeySpec 16 | import play.api.libs.json._ 17 | import org.apache.toree.utils.LogLike 18 | 19 | 20 | case class Payload(key : String, conn_info : String, version : Int = 1) 21 | 22 | object Payload { 23 | 24 | def createJson(key: String, conn_info: String) : String = { 25 | implicit val writes = Json.writes[Payload] 26 | val newPayload = new Payload(key = key, conn_info = conn_info) 27 | Json.prettyPrint(Json.toJson(newPayload)) 28 | } 29 | } 30 | 31 | 32 | object SecurityUtils extends LogLike { 33 | 34 | def encrypt(publicKey: String, jsonContent: String): String = { 35 | // Generate an AES key and encrypt the connection information... 36 | logger.info("publicKey: %s".format(publicKey)) 37 | val random: Random = new Random() 38 | val preKey: Array[Byte] = new Array[Byte](16) 39 | random.nextBytes(preKey) 40 | logger.info("aes_key: '%s'".format(preKey)) 41 | val aesKey: Key = new SecretKeySpec(preKey, "AES") 42 | val aesCipher: Cipher = Cipher.getInstance("AES") 43 | aesCipher.init(Cipher.ENCRYPT_MODE, aesKey) 44 | val connInfo = Base64.getEncoder.encodeToString(aesCipher.doFinal(jsonContent.getBytes(StandardCharsets.UTF_8))) 45 | 46 | // Encrypt the AES key using the public key... 47 | val encodedPK: Array[Byte] = publicKey.getBytes(StandardCharsets.UTF_8) 48 | val b64Key = Base64.getDecoder.decode(encodedPK) 49 | val keySpec: X509EncodedKeySpec = new X509EncodedKeySpec(b64Key) 50 | val keyFactory: KeyFactory = KeyFactory.getInstance("RSA") 51 | val rsaKey: PublicKey = keyFactory.generatePublic(keySpec) 52 | 53 | val rsaCipher: Cipher = Cipher.getInstance("RSA") 54 | rsaCipher.init(Cipher.ENCRYPT_MODE, rsaKey) 55 | val key = Base64.getEncoder.encodeToString(rsaCipher.doFinal(aesKey.getEncoded())) 56 | Base64.getEncoder.encodeToString(Payload.createJson(key, connInfo).getBytes(StandardCharsets.UTF_8)) 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /etc/kernel-resources/apache_toree/logo-64x64.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/etc/kernel-resources/apache_toree/logo-64x64.png -------------------------------------------------------------------------------- /etc/kernel-resources/ir/logo-64x64.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/etc/kernel-resources/ir/logo-64x64.png -------------------------------------------------------------------------------- /etc/kernel-resources/python/logo-64x64.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/etc/kernel-resources/python/logo-64x64.png -------------------------------------------------------------------------------- /etc/kernel-resources/tensorflow/logo-64x64.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/etc/kernel-resources/tensorflow/logo-64x64.png -------------------------------------------------------------------------------- /etc/kernelspecs/R_docker/kernel.json: -------------------------------------------------------------------------------- 1 | { 2 | "language": "R", 3 | "display_name": "R on Docker", 4 | "metadata": { 5 | "process_proxy": { 6 | "class_name": "enterprise_gateway.services.processproxies.docker_swarm.DockerSwarmProcessProxy", 7 | "config": { 8 | "image_name": "elyra/kernel-r:VERSION" 9 | } 10 | } 11 | }, 12 | "env": {}, 13 | "argv": [ 14 | "python", 15 | "/usr/local/share/jupyter/kernels/R_docker/scripts/launch_docker.py", 16 | "--RemoteProcessProxy.kernel-id", 17 | "{kernel_id}", 18 | "--RemoteProcessProxy.port-range", 19 | "{port_range}", 20 | "--RemoteProcessProxy.response-address", 21 | "{response_address}", 22 | "--RemoteProcessProxy.public-key", 23 | "{public_key}" 24 | ] 25 | } 26 | -------------------------------------------------------------------------------- /etc/kernelspecs/R_kubernetes/kernel.json: -------------------------------------------------------------------------------- 1 | { 2 | "language": "R", 3 | "display_name": "R on Kubernetes", 4 | "metadata": { 5 | "process_proxy": { 6 | "class_name": "enterprise_gateway.services.processproxies.k8s.KubernetesProcessProxy", 7 | "config": { 8 | "image_name": "elyra/kernel-r:VERSION" 9 | } 10 | } 11 | }, 12 | "env": {}, 13 | "argv": [ 14 | "python", 15 | "/usr/local/share/jupyter/kernels/R_kubernetes/scripts/launch_kubernetes.py", 16 | "--RemoteProcessProxy.kernel-id", 17 | "{kernel_id}", 18 | "--RemoteProcessProxy.port-range", 19 | "{port_range}", 20 | "--RemoteProcessProxy.response-address", 21 | "{response_address}", 22 | "--RemoteProcessProxy.public-key", 23 | "{public_key}" 24 | ] 25 | } 26 | -------------------------------------------------------------------------------- /etc/kernelspecs/dask_python_yarn_remote/bin/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ "${EG_IMPERSONATION_ENABLED}" = "True" ]; then 4 | IMPERSONATION_OPTS="--user ${KERNEL_USERNAME:-UNSPECIFIED}" 5 | USER_CLAUSE="as user ${KERNEL_USERNAME:-UNSPECIFIED}" 6 | else 7 | IMPERSONATION_OPTS="" 8 | USER_CLAUSE="on behalf of user ${KERNEL_USERNAME:-UNSPECIFIED}" 9 | fi 10 | 11 | echo 12 | echo "Starting IPython kernel for Dask ${USER_CLAUSE}" 13 | echo 14 | 15 | PROG_HOME="$(cd "`dirname "$0"`"/..; pwd)" 16 | 17 | set -x 18 | eval exec \ 19 | "${DASK_YARN_EXE}" submit \ 20 | "${DASK_OPTS}" \ 21 | "${IMPERSONATION_OPTS}" \ 22 | "${PROG_HOME}/scripts/launch_ipykernel.py" \ 23 | "${LAUNCH_OPTS}" \ 24 | "$@" 25 | set +x 26 | -------------------------------------------------------------------------------- /etc/kernelspecs/dask_python_yarn_remote/kernel.json: -------------------------------------------------------------------------------- 1 | { 2 | "language": "python", 3 | "display_name": "Dask - Python (YARN Remote Mode)", 4 | "metadata": { 5 | "process_proxy": { 6 | "class_name": "enterprise_gateway.services.processproxies.yarn.YarnClusterProcessProxy" 7 | }, 8 | "debugger": true 9 | }, 10 | "env": { 11 | "SPARK_HOME": "/usr/hdp/current/spark2-client", 12 | "DASK_YARN_EXE": "/opt/conda/bin/dask-yarn", 13 | "DASK_OPTS": "--name ${KERNEL_ID:-ERROR__NO__KERNEL_ID} --environment python:///opt/conda/bin/python --temporary-security-credentials --deploy-mode remote", 14 | "LAUNCH_OPTS": "" 15 | }, 16 | "argv": [ 17 | "/usr/local/share/jupyter/kernels/dask_python_yarn_remote/bin/run.sh", 18 | "--RemoteProcessProxy.kernel-id", 19 | "{kernel_id}", 20 | "--RemoteProcessProxy.response-address", 21 | "{response_address}", 22 | "--RemoteProcessProxy.public-key", 23 | "{public_key}", 24 | "--RemoteProcessProxy.port-range", 25 | "{port_range}", 26 | "--RemoteProcessProxy.cluster-type", 27 | "dask" 28 | ] 29 | } 30 | -------------------------------------------------------------------------------- /etc/kernelspecs/python_distributed/kernel.json: -------------------------------------------------------------------------------- 1 | { 2 | "display_name": "Python 3 (distributed)", 3 | "language": "python", 4 | "metadata": { 5 | "process_proxy": { 6 | "class_name": "enterprise_gateway.services.processproxies.distributed.DistributedProcessProxy" 7 | }, 8 | "debugger": true 9 | }, 10 | "argv": [ 11 | "python", 12 | "/usr/local/share/jupyter/kernels/python_distributed/scripts/launch_ipykernel.py", 13 | "--RemoteProcessProxy.kernel-id", 14 | "{kernel_id}", 15 | "--RemoteProcessProxy.response-address", 16 | "{response_address}", 17 | "--RemoteProcessProxy.public-key", 18 | "{public_key}", 19 | "--RemoteProcessProxy.port-range", 20 | "{port_range}", 21 | "--RemoteProcessProxy.spark-context-initialization-mode", 22 | "none" 23 | ] 24 | } 25 | -------------------------------------------------------------------------------- /etc/kernelspecs/python_docker/kernel.json: -------------------------------------------------------------------------------- 1 | { 2 | "language": "python", 3 | "display_name": "Python on Docker", 4 | "metadata": { 5 | "process_proxy": { 6 | "class_name": "enterprise_gateway.services.processproxies.docker_swarm.DockerSwarmProcessProxy", 7 | "config": { 8 | "image_name": "elyra/kernel-py:VERSION" 9 | } 10 | }, 11 | "debugger": true 12 | }, 13 | "env": {}, 14 | "argv": [ 15 | "python", 16 | "/usr/local/share/jupyter/kernels/python_docker/scripts/launch_docker.py", 17 | "--RemoteProcessProxy.kernel-id", 18 | "{kernel_id}", 19 | "--RemoteProcessProxy.port-range", 20 | "{port_range}", 21 | "--RemoteProcessProxy.response-address", 22 | "{response_address}", 23 | "--RemoteProcessProxy.public-key", 24 | "{public_key}" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /etc/kernelspecs/python_kubernetes/kernel.json: -------------------------------------------------------------------------------- 1 | { 2 | "language": "python", 3 | "display_name": "Python on Kubernetes", 4 | "metadata": { 5 | "process_proxy": { 6 | "class_name": "enterprise_gateway.services.processproxies.k8s.KubernetesProcessProxy", 7 | "config": { 8 | "image_name": "elyra/kernel-py:VERSION" 9 | } 10 | }, 11 | "debugger": true 12 | }, 13 | "env": {}, 14 | "argv": [ 15 | "python", 16 | "/usr/local/share/jupyter/kernels/python_kubernetes/scripts/launch_kubernetes.py", 17 | "--RemoteProcessProxy.kernel-id", 18 | "{kernel_id}", 19 | "--RemoteProcessProxy.port-range", 20 | "{port_range}", 21 | "--RemoteProcessProxy.response-address", 22 | "{response_address}", 23 | "--RemoteProcessProxy.public-key", 24 | "{public_key}" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /etc/kernelspecs/python_tf_docker/kernel.json: -------------------------------------------------------------------------------- 1 | { 2 | "language": "python", 3 | "display_name": "Python on Docker with Tensorflow", 4 | "metadata": { 5 | "process_proxy": { 6 | "class_name": "enterprise_gateway.services.processproxies.docker_swarm.DockerSwarmProcessProxy", 7 | "config": { 8 | "image_name": "elyra/kernel-tf-py:VERSION" 9 | } 10 | }, 11 | "debugger": true 12 | }, 13 | "env": {}, 14 | "argv": [ 15 | "python", 16 | "/usr/local/share/jupyter/kernels/python_tf_docker/scripts/launch_docker.py", 17 | "--RemoteProcessProxy.kernel-id", 18 | "{kernel_id}", 19 | "--RemoteProcessProxy.port-range", 20 | "{port_range}", 21 | "--RemoteProcessProxy.response-address", 22 | "{response_address}", 23 | "--RemoteProcessProxy.public-key", 24 | "{public_key}" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /etc/kernelspecs/python_tf_gpu_docker/kernel.json: -------------------------------------------------------------------------------- 1 | { 2 | "language": "python", 3 | "display_name": "Python on Docker with Tensorflow with GPUs", 4 | "metadata": { 5 | "process_proxy": { 6 | "class_name": "enterprise_gateway.services.processproxies.docker_swarm.DockerSwarmProcessProxy", 7 | "config": { 8 | "image_name": "elyra/kernel-tf-gpu-py:VERSION" 9 | } 10 | }, 11 | "debugger": true 12 | }, 13 | "env": {}, 14 | "argv": [ 15 | "python", 16 | "/usr/local/share/jupyter/kernels/python_tf_gpu_docker/scripts/launch_docker.py", 17 | "--RemoteProcessProxy.kernel-id", 18 | "{kernel_id}", 19 | "--RemoteProcessProxy.port-range", 20 | "{port_range}", 21 | "--RemoteProcessProxy.response-address", 22 | "{response_address}", 23 | "--RemoteProcessProxy.public-key", 24 | "{public_key}" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /etc/kernelspecs/python_tf_gpu_kubernetes/kernel.json: -------------------------------------------------------------------------------- 1 | { 2 | "language": "python", 3 | "display_name": "Python on Kubernetes with Tensorflow with GPUs", 4 | "metadata": { 5 | "process_proxy": { 6 | "class_name": "enterprise_gateway.services.processproxies.k8s.KubernetesProcessProxy", 7 | "config": { 8 | "image_name": "elyra/kernel-tf-gpu-py:VERSION" 9 | } 10 | }, 11 | "debugger": true 12 | }, 13 | "env": {}, 14 | "argv": [ 15 | "python", 16 | "/usr/local/share/jupyter/kernels/python_tf_gpu_kubernetes/scripts/launch_kubernetes.py", 17 | "--RemoteProcessProxy.kernel-id", 18 | "{kernel_id}", 19 | "--RemoteProcessProxy.port-range", 20 | "{port_range}", 21 | "--RemoteProcessProxy.response-address", 22 | "{response_address}", 23 | "--RemoteProcessProxy.public-key", 24 | "{public_key}" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /etc/kernelspecs/python_tf_kubernetes/kernel.json: -------------------------------------------------------------------------------- 1 | { 2 | "language": "python", 3 | "display_name": "Python on Kubernetes with Tensorflow", 4 | "metadata": { 5 | "process_proxy": { 6 | "class_name": "enterprise_gateway.services.processproxies.k8s.KubernetesProcessProxy", 7 | "config": { 8 | "image_name": "elyra/kernel-tf-py:VERSION" 9 | } 10 | }, 11 | "debugger": true 12 | }, 13 | "env": {}, 14 | "argv": [ 15 | "python", 16 | "/usr/local/share/jupyter/kernels/python_tf_kubernetes/scripts/launch_kubernetes.py", 17 | "--RemoteProcessProxy.kernel-id", 18 | "{kernel_id}", 19 | "--RemoteProcessProxy.port-range", 20 | "{port_range}", 21 | "--RemoteProcessProxy.response-address", 22 | "{response_address}", 23 | "--RemoteProcessProxy.public-key", 24 | "{public_key}" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /etc/kernelspecs/scala_docker/kernel.json: -------------------------------------------------------------------------------- 1 | { 2 | "language": "scala", 3 | "display_name": "Scala on Docker", 4 | "metadata": { 5 | "process_proxy": { 6 | "class_name": "enterprise_gateway.services.processproxies.docker_swarm.DockerSwarmProcessProxy", 7 | "config": { 8 | "image_name": "elyra/kernel-scala:VERSION" 9 | } 10 | } 11 | }, 12 | "env": {}, 13 | "argv": [ 14 | "python", 15 | "/usr/local/share/jupyter/kernels/scala_docker/scripts/launch_docker.py", 16 | "--RemoteProcessProxy.kernel-id", 17 | "{kernel_id}", 18 | "--RemoteProcessProxy.port-range", 19 | "{port_range}", 20 | "--RemoteProcessProxy.response-address", 21 | "{response_address}", 22 | "--RemoteProcessProxy.public-key", 23 | "{public_key}" 24 | ] 25 | } 26 | -------------------------------------------------------------------------------- /etc/kernelspecs/scala_kubernetes/kernel.json: -------------------------------------------------------------------------------- 1 | { 2 | "language": "scala", 3 | "display_name": "Scala on Kubernetes", 4 | "metadata": { 5 | "process_proxy": { 6 | "class_name": "enterprise_gateway.services.processproxies.k8s.KubernetesProcessProxy", 7 | "config": { 8 | "image_name": "elyra/kernel-scala:VERSION" 9 | } 10 | } 11 | }, 12 | "env": {}, 13 | "argv": [ 14 | "python", 15 | "/usr/local/share/jupyter/kernels/scala_kubernetes/scripts/launch_kubernetes.py", 16 | "--RemoteProcessProxy.kernel-id", 17 | "{kernel_id}", 18 | "--RemoteProcessProxy.port-range", 19 | "{port_range}", 20 | "--RemoteProcessProxy.response-address", 21 | "{response_address}", 22 | "--RemoteProcessProxy.public-key", 23 | "{public_key}" 24 | ] 25 | } 26 | -------------------------------------------------------------------------------- /etc/kernelspecs/spark_R_conductor_cluster/bin/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ "${EG_IMPERSONATION_ENABLED}" = "True" ]; then 4 | IMPERSONATION_OPTS="--proxy-user ${KERNEL_USERNAME:-UNSPECIFIED}" 5 | USER_CLAUSE="as user ${KERNEL_USERNAME:-UNSPECIFIED}" 6 | else 7 | IMPERSONATION_OPTS="" 8 | USER_CLAUSE="on behalf of user ${KERNEL_USERNAME:-UNSPECIFIED}" 9 | fi 10 | 11 | echo 12 | echo "Starting IRkernel for Spark Cluster mode ${USER_CLAUSE}" 13 | echo 14 | 15 | if [ -z "${SPARK_HOME}" ]; then 16 | echo "SPARK_HOME must be set to the location of a Spark distribution!" 17 | exit 1 18 | fi 19 | 20 | PROG_HOME="$(cd "`dirname "$0"`"/..; pwd)" 21 | 22 | # Add server_listener.py to files for spark-opts 23 | ADDITIONAL_OPTS="--files ${PROG_HOME}/scripts/server_listener.py" 24 | 25 | eval exec \ 26 | "${SPARK_HOME}/bin/spark-submit" \ 27 | "${SPARK_OPTS}" \ 28 | "${ADDITIONAL_OPTS}" \ 29 | "${IMPERSONATION_OPTS}" \ 30 | "${PROG_HOME}/scripts/launch_IRkernel.R" \ 31 | "${LAUNCH_OPTS}" \ 32 | "$@" 33 | -------------------------------------------------------------------------------- /etc/kernelspecs/spark_R_conductor_cluster/kernel.json: -------------------------------------------------------------------------------- 1 | { 2 | "language": "R", 3 | "display_name": "Spark R (Spark Cluster Mode)", 4 | "metadata": { 5 | "process_proxy": { 6 | "class_name": "enterprise_gateway.services.processproxies.conductor.ConductorClusterProcessProxy" 7 | } 8 | }, 9 | "env": { 10 | "SPARK_OPTS": "--name ${KERNEL_ID:-ERROR__NO__KERNEL_ID} --conf spark.yarn.maxAppAttempts=1 ${KERNEL_EXTRA_SPARK_OPTS}", 11 | "LAUNCH_OPTS": "--customAppName ${KERNEL_ID}" 12 | }, 13 | "argv": [ 14 | "--RemoteProcessProxy.kernel-id", 15 | "{kernel_id}", 16 | "--RemoteProcessProxy.response-address", 17 | "{response_address}", 18 | "--RemoteProcessProxy.public-key", 19 | "{public_key}", 20 | "--RemoteProcessProxy.port-range", 21 | "{port_range}", 22 | "--RemoteProcessProxy.spark-context-initialization-mode", 23 | "eager" 24 | ] 25 | } 26 | -------------------------------------------------------------------------------- /etc/kernelspecs/spark_R_kubernetes/bin/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ "${EG_IMPERSONATION_ENABLED}" = "True" ]; then 4 | # IMPERSONATION_OPTS="--proxy-user ${KERNEL_USERNAME:-UNSPECIFIED}" 5 | USER_CLAUSE="as user ${KERNEL_USERNAME:-UNSPECIFIED}" 6 | else 7 | # IMPERSONATION_OPTS="" 8 | USER_CLAUSE="on behalf of user ${KERNEL_USERNAME:-UNSPECIFIED}" 9 | fi 10 | 11 | echo 12 | echo "Starting IRkernel for Spark in Kubernetes mode ${USER_CLAUSE}" 13 | echo 14 | 15 | if [ -z "${SPARK_HOME}" ]; then 16 | echo "SPARK_HOME must be set to the location of a Spark distribution!" 17 | exit 1 18 | fi 19 | 20 | if [ -z "${KERNEL_ID}" ]; then 21 | echo "KERNEL_ID must be set for discovery and lifecycle management!" 22 | exit 1 23 | fi 24 | 25 | KERNEL_LAUNCHERS_DIR=${KERNEL_LAUNCHERS_DIR:-/usr/local/bin/kernel-launchers} 26 | PROG_HOME=${KERNEL_LAUNCHERS_DIR}/R 27 | 28 | EG_POD_TEMPLATE_DIR=${EG_POD_TEMPLATE_DIR:-/tmp} 29 | SCRIPTS_HOME="$(cd "`dirname "$0"`"/../scripts; pwd)" 30 | pod_template_file=${EG_POD_TEMPLATE_DIR}/kpt_${KERNEL_ID} 31 | spark_opts_out=${EG_POD_TEMPLATE_DIR}/spark_opts_${KERNEL_ID} 32 | python ${SCRIPTS_HOME}/launch_kubernetes.py $@ --pod-template=${pod_template_file} --spark-opts-out=${spark_opts_out} 33 | additional_spark_opts=`cat ${spark_opts_out}` 34 | SPARK_OPTS="${SPARK_OPTS} ${additional_spark_opts}" 35 | rm -f ${spark_opts_out} 36 | 37 | set -x 38 | eval exec \ 39 | "${SPARK_HOME}/bin/spark-submit" \ 40 | "${SPARK_OPTS}" \ 41 | "local://${PROG_HOME}/scripts/launch_IRkernel.R" \ 42 | "${LAUNCH_OPTS}" \ 43 | "$@" 44 | set +x 45 | -------------------------------------------------------------------------------- /etc/kernelspecs/spark_R_kubernetes/kernel.json: -------------------------------------------------------------------------------- 1 | { 2 | "language": "R", 3 | "display_name": "Spark - R (Kubernetes Mode)", 4 | "metadata": { 5 | "process_proxy": { 6 | "class_name": "enterprise_gateway.services.processproxies.k8s.KubernetesProcessProxy", 7 | "config": { 8 | "image_name": "elyra/kernel-spark-r:VERSION", 9 | "executor_image_name": "elyra/kernel-spark-r:VERSION" 10 | } 11 | } 12 | }, 13 | "env": { 14 | "SPARK_HOME": "/opt/spark", 15 | "SPARK_OPTS": "--master k8s://https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT} --deploy-mode cluster --name ${KERNEL_USERNAME}-${KERNEL_ID} --conf spark.kubernetes.namespace=${KERNEL_NAMESPACE} --conf spark.kubernetes.driver.label.app=enterprise-gateway --conf spark.kubernetes.driver.label.kernel_id=${KERNEL_ID} --conf spark.kubernetes.driver.label.component=kernel --conf spark.kubernetes.executor.label.app=enterprise-gateway --conf spark.kubernetes.executor.label.kernel_id=${KERNEL_ID} --conf spark.kubernetes.executor.label.component=worker --conf spark.kubernetes.driver.container.image=${KERNEL_IMAGE} --conf spark.kubernetes.executor.container.image=${KERNEL_EXECUTOR_IMAGE} --conf spark.kubernetes.authenticate.driver.serviceAccountName=${KERNEL_SERVICE_ACCOUNT_NAME} --conf spark.kubernetes.submission.waitAppCompletion=false --conf spark.kubernetes.driverEnv.HTTP2_DISABLE=true ${KERNEL_EXTRA_SPARK_OPTS}", 16 | "HTTP2_DISABLE": "true", 17 | "LAUNCH_OPTS": "" 18 | }, 19 | "argv": [ 20 | "/usr/local/share/jupyter/kernels/spark_R_kubernetes/bin/run.sh", 21 | "--RemoteProcessProxy.kernel-id", 22 | "{kernel_id}", 23 | "--RemoteProcessProxy.port-range", 24 | "{port_range}", 25 | "--RemoteProcessProxy.response-address", 26 | "{response_address}", 27 | "--RemoteProcessProxy.public-key", 28 | "{public_key}", 29 | "--RemoteProcessProxy.spark-context-initialization-mode", 30 | "lazy" 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /etc/kernelspecs/spark_R_yarn_client/bin/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ "${EG_IMPERSONATION_ENABLED}" = "True" ]; then 4 | IMPERSONATION_OPTS="sudo PATH=${PATH} -H -E -u ${KERNEL_USERNAME:-UNSPECIFIED}" 5 | USER_CLAUSE="as user ${KERNEL_USERNAME:-UNSPECIFIED}" 6 | else 7 | IMPERSONATION_OPTS="" 8 | USER_CLAUSE="on behalf of user ${KERNEL_USERNAME:-UNSPECIFIED}" 9 | fi 10 | 11 | echo 12 | echo "Starting IRkernel for Spark in Yarn Client mode ${USER_CLAUSE}" 13 | echo 14 | 15 | if [ -z "${SPARK_HOME}" ]; then 16 | echo "SPARK_HOME must be set to the location of a Spark distribution!" 17 | exit 1 18 | fi 19 | 20 | PROG_HOME="$(cd "`dirname "$0"`"/..; pwd)" 21 | 22 | set -x 23 | eval exec "${IMPERSONATION_OPTS}" \ 24 | "${SPARK_HOME}/bin/spark-submit" \ 25 | "${SPARK_OPTS}" \ 26 | "${PROG_HOME}/scripts/launch_IRkernel.R" \ 27 | "${LAUNCH_OPTS}" \ 28 | "$@" 29 | set +x 30 | -------------------------------------------------------------------------------- /etc/kernelspecs/spark_R_yarn_client/kernel.json: -------------------------------------------------------------------------------- 1 | { 2 | "language": "R", 3 | "display_name": "Spark - R (YARN Client Mode)", 4 | "metadata": { 5 | "process_proxy": { 6 | "class_name": "enterprise_gateway.services.processproxies.distributed.DistributedProcessProxy" 7 | } 8 | }, 9 | "env": { 10 | "SPARK_HOME": "/usr/hdp/current/spark2-client", 11 | "SPARK_OPTS": "--master yarn --deploy-mode client --name ${KERNEL_ID:-ERROR__NO__KERNEL_ID} --conf spark.sparkr.r.command=/opt/conda/lib/R/bin/Rscript ${KERNEL_EXTRA_SPARK_OPTS}", 12 | "LAUNCH_OPTS": "" 13 | }, 14 | "argv": [ 15 | "/usr/local/share/jupyter/kernels/spark_R_yarn_client/bin/run.sh", 16 | "--RemoteProcessProxy.kernel-id", 17 | "{kernel_id}", 18 | "--RemoteProcessProxy.response-address", 19 | "{response_address}", 20 | "--RemoteProcessProxy.public-key", 21 | "{public_key}", 22 | "--RemoteProcessProxy.port-range", 23 | "{port_range}", 24 | "--RemoteProcessProxy.spark-context-initialization-mode", 25 | "lazy" 26 | ] 27 | } 28 | -------------------------------------------------------------------------------- /etc/kernelspecs/spark_R_yarn_cluster/bin/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ "${EG_IMPERSONATION_ENABLED}" = "True" ]; then 4 | IMPERSONATION_OPTS="--proxy-user ${KERNEL_USERNAME:-UNSPECIFIED}" 5 | USER_CLAUSE="as user ${KERNEL_USERNAME:-UNSPECIFIED}" 6 | else 7 | IMPERSONATION_OPTS="" 8 | USER_CLAUSE="on behalf of user ${KERNEL_USERNAME:-UNSPECIFIED}" 9 | fi 10 | 11 | echo 12 | echo "Starting IRkernel for Spark in Yarn Cluster mode ${USER_CLAUSE}" 13 | echo 14 | 15 | if [ -z "${SPARK_HOME}" ]; then 16 | echo "SPARK_HOME must be set to the location of a Spark distribution!" 17 | exit 1 18 | fi 19 | 20 | PROG_HOME="$(cd "`dirname "$0"`"/..; pwd)" 21 | 22 | # Add server_listener.py to files for spark-opts 23 | ADDITIONAL_OPTS="--files ${PROG_HOME}/scripts/server_listener.py" 24 | 25 | set -x 26 | eval exec \ 27 | "${SPARK_HOME}/bin/spark-submit" \ 28 | "${SPARK_OPTS}" \ 29 | "${ADDITIONAL_OPTS}" \ 30 | "${IMPERSONATION_OPTS}" \ 31 | "${PROG_HOME}/scripts/launch_IRkernel.R" \ 32 | "${LAUNCH_OPTS}" \ 33 | "$@" 34 | set +x 35 | -------------------------------------------------------------------------------- /etc/kernelspecs/spark_R_yarn_cluster/kernel.json: -------------------------------------------------------------------------------- 1 | { 2 | "language": "R", 3 | "display_name": "Spark - R (YARN Cluster Mode)", 4 | "metadata": { 5 | "process_proxy": { 6 | "class_name": "enterprise_gateway.services.processproxies.yarn.YarnClusterProcessProxy" 7 | } 8 | }, 9 | "env": { 10 | "SPARK_HOME": "/usr/hdp/current/spark2-client", 11 | "SPARK_OPTS": "--master yarn --deploy-mode cluster --name ${KERNEL_ID:-ERROR__NO__KERNEL_ID} --conf spark.yarn.submit.waitAppCompletion=false --conf spark.yarn.am.waitTime=1d --conf spark.yarn.appMasterEnv.PATH=/opt/conda/bin:$PATH --conf spark.sparkr.r.command=/opt/conda/lib/R/bin/Rscript --conf spark.yarn.maxAppAttempts=1 ${KERNEL_EXTRA_SPARK_OPTS}", 12 | "LAUNCH_OPTS": "" 13 | }, 14 | "argv": [ 15 | "/usr/local/share/jupyter/kernels/spark_R_yarn_cluster/bin/run.sh", 16 | "--RemoteProcessProxy.kernel-id", 17 | "{kernel_id}", 18 | "--RemoteProcessProxy.response-address", 19 | "{response_address}", 20 | "--RemoteProcessProxy.public-key", 21 | "{public_key}", 22 | "--RemoteProcessProxy.port-range", 23 | "{port_range}", 24 | "--RemoteProcessProxy.spark-context-initialization-mode", 25 | "eager" 26 | ] 27 | } 28 | -------------------------------------------------------------------------------- /etc/kernelspecs/spark_python_conductor_cluster/bin/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ "${EG_IMPERSONATION_ENABLED}" = "True" ]; then 4 | IMPERSONATION_OPTS="sudo PATH=${PATH} -H -E -u ${KERNEL_USERNAME:-UNSPECIFIED}" 5 | USER_CLAUSE="as user ${KERNEL_USERNAME:-UNSPECIFIED}" 6 | else 7 | IMPERSONATION_OPTS="" 8 | USER_CLAUSE="on behalf of user ${KERNEL_USERNAME:-UNSPECIFIED}" 9 | fi 10 | 11 | echo 12 | echo "Starting IPython kernel for Spark Cluster mode ${USER_CLAUSE}" 13 | echo 14 | 15 | if [ -z "${SPARK_HOME}" ]; then 16 | echo "SPARK_HOME must be set to the location of a Spark distribution!" 17 | exit 1 18 | fi 19 | 20 | if [ -z "${KERNEL_IG_UUID}" ]; then 21 | PROG_HOME="$(cd "`dirname "$0"`"/..; pwd)" 22 | else 23 | PROG_HOME="${SPARK_HOME}" 24 | fi 25 | 26 | eval exec "${IMPERSONATION_OPTS}" \ 27 | "${SPARK_HOME}/bin/spark-submit" \ 28 | "${SPARK_OPTS}" \ 29 | "${PROG_HOME}/scripts/launch_ipykernel.py" \ 30 | "${LAUNCH_OPTS}" \ 31 | "$@" 32 | -------------------------------------------------------------------------------- /etc/kernelspecs/spark_python_conductor_cluster/kernel.json: -------------------------------------------------------------------------------- 1 | { 2 | "language": "python", 3 | "display_name": "Spark Python (Spark Cluster Mode)", 4 | "metadata": { 5 | "process_proxy": { 6 | "class_name": "enterprise_gateway.services.processproxies.conductor.ConductorClusterProcessProxy" 7 | }, 8 | "debugger": true 9 | }, 10 | "env": { 11 | "SPARK_OPTS": "--name ${KERNEL_ID:-ERROR__NO__KERNEL_ID} --conf spark.yarn.maxAppAttempts=1 ${KERNEL_EXTRA_SPARK_OPTS}", 12 | "LAUNCH_OPTS": "" 13 | }, 14 | "argv": [ 15 | "--RemoteProcessProxy.kernel-id", 16 | "{kernel_id}", 17 | "--RemoteProcessProxy.response-address", 18 | "{response_address}", 19 | "--RemoteProcessProxy.public-key", 20 | "{public_key}", 21 | "--RemoteProcessProxy.port-range", 22 | "{port_range}", 23 | "--RemoteProcessProxy.spark-context-initialization-mode", 24 | "eager" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /etc/kernelspecs/spark_python_kubernetes/bin/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ "${EG_IMPERSONATION_ENABLED}" = "True" ]; then 4 | # IMPERSONATION_OPTS="--proxy-user ${KERNEL_USERNAME:-UNSPECIFIED}" 5 | USER_CLAUSE="as user ${KERNEL_USERNAME:-UNSPECIFIED}" 6 | else 7 | # IMPERSONATION_OPTS="" 8 | USER_CLAUSE="on behalf of user ${KERNEL_USERNAME:-UNSPECIFIED}" 9 | fi 10 | 11 | echo 12 | echo "Starting IPython kernel for Spark in Kubernetes mode ${USER_CLAUSE}" 13 | echo 14 | 15 | if [ -z "${SPARK_HOME}" ]; then 16 | echo "SPARK_HOME must be set to the location of a Spark distribution!" 17 | exit 1 18 | fi 19 | 20 | if [ -z "${KERNEL_ID}" ]; then 21 | echo "KERNEL_ID must be set for discovery and lifecycle management!" 22 | exit 1 23 | fi 24 | 25 | KERNEL_LAUNCHERS_DIR=${KERNEL_LAUNCHERS_DIR:-/usr/local/bin/kernel-launchers} 26 | PROG_HOME=${KERNEL_LAUNCHERS_DIR}/python 27 | 28 | EG_POD_TEMPLATE_DIR=${EG_POD_TEMPLATE_DIR:-/tmp} 29 | SCRIPTS_HOME="$(cd "`dirname "$0"`"/../scripts; pwd)" 30 | pod_template_file=${EG_POD_TEMPLATE_DIR}/kpt_${KERNEL_ID} 31 | spark_opts_out=${EG_POD_TEMPLATE_DIR}/spark_opts_${KERNEL_ID} 32 | python ${SCRIPTS_HOME}/launch_kubernetes.py $@ --pod-template=${pod_template_file} --spark-opts-out=${spark_opts_out} 33 | additional_spark_opts=`cat ${spark_opts_out}` 34 | SPARK_OPTS="${SPARK_OPTS} ${additional_spark_opts}" 35 | rm -f ${spark_opts_out} 36 | 37 | set -x 38 | eval exec \ 39 | "${SPARK_HOME}/bin/spark-submit" \ 40 | "${SPARK_OPTS}" \ 41 | "local://${PROG_HOME}/scripts/launch_ipykernel.py" \ 42 | "${LAUNCH_OPTS}" \ 43 | "$@" 44 | set +x 45 | -------------------------------------------------------------------------------- /etc/kernelspecs/spark_python_kubernetes/kernel.json: -------------------------------------------------------------------------------- 1 | { 2 | "language": "python", 3 | "display_name": "Spark - Python (Kubernetes Mode)", 4 | "metadata": { 5 | "process_proxy": { 6 | "class_name": "enterprise_gateway.services.processproxies.k8s.KubernetesProcessProxy", 7 | "config": { 8 | "image_name": "elyra/kernel-spark-py:VERSION", 9 | "executor_image_name": "elyra/kernel-spark-py:VERSION" 10 | } 11 | }, 12 | "debugger": true 13 | }, 14 | "env": { 15 | "SPARK_HOME": "/opt/spark", 16 | "SPARK_OPTS": "--master k8s://https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT} --deploy-mode cluster --name ${KERNEL_USERNAME}-${KERNEL_ID} --conf spark.kubernetes.namespace=${KERNEL_NAMESPACE} --conf spark.kubernetes.driver.label.app=enterprise-gateway --conf spark.kubernetes.driver.label.kernel_id=${KERNEL_ID} --conf spark.kubernetes.driver.label.component=kernel --conf spark.kubernetes.executor.label.app=enterprise-gateway --conf spark.kubernetes.executor.label.kernel_id=${KERNEL_ID} --conf spark.kubernetes.executor.label.component=worker --conf spark.kubernetes.driver.container.image=${KERNEL_IMAGE} --conf spark.kubernetes.executor.container.image=${KERNEL_EXECUTOR_IMAGE} --conf spark.kubernetes.authenticate.driver.serviceAccountName=${KERNEL_SERVICE_ACCOUNT_NAME} --conf spark.kubernetes.submission.waitAppCompletion=false --conf spark.kubernetes.driverEnv.HTTP2_DISABLE=true ${KERNEL_EXTRA_SPARK_OPTS}", 17 | "HTTP2_DISABLE": "true", 18 | "LAUNCH_OPTS": "" 19 | }, 20 | "argv": [ 21 | "/usr/local/share/jupyter/kernels/spark_python_kubernetes/bin/run.sh", 22 | "--RemoteProcessProxy.kernel-id", 23 | "{kernel_id}", 24 | "--RemoteProcessProxy.port-range", 25 | "{port_range}", 26 | "--RemoteProcessProxy.response-address", 27 | "{response_address}", 28 | "--RemoteProcessProxy.public-key", 29 | "{public_key}", 30 | "--RemoteProcessProxy.spark-context-initialization-mode", 31 | "lazy" 32 | ] 33 | } 34 | -------------------------------------------------------------------------------- /etc/kernelspecs/spark_python_operator/kernel.json: -------------------------------------------------------------------------------- 1 | { 2 | "language": "python", 3 | "display_name": "Spark Operator (Python)", 4 | "metadata": { 5 | "process_proxy": { 6 | "class_name": "enterprise_gateway.services.processproxies.spark_operator.SparkOperatorProcessProxy", 7 | "config": { 8 | "image_name": "elyra/kernel-spark-py:VERSION", 9 | "executor_image_name": "elyra/kernel-spark-py:VERSION" 10 | } 11 | } 12 | }, 13 | "argv": [ 14 | "python", 15 | "/usr/local/share/jupyter/kernels/spark_python_operator/scripts/launch_custom_resource.py", 16 | "--RemoteProcessProxy.kernel-id", 17 | "{kernel_id}", 18 | "--RemoteProcessProxy.port-range", 19 | "{port_range}", 20 | "--RemoteProcessProxy.response-address", 21 | "{response_address}", 22 | "--RemoteProcessProxy.public-key", 23 | "{public_key}" 24 | ] 25 | } 26 | -------------------------------------------------------------------------------- /etc/kernelspecs/spark_python_yarn_client/bin/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ "${EG_IMPERSONATION_ENABLED}" = "True" ]; then 4 | IMPERSONATION_OPTS="sudo PATH=${PATH} -H -E -u ${KERNEL_USERNAME:-UNSPECIFIED}" 5 | USER_CLAUSE="as user ${KERNEL_USERNAME:-UNSPECIFIED}" 6 | else 7 | IMPERSONATION_OPTS="" 8 | USER_CLAUSE="on behalf of user ${KERNEL_USERNAME:-UNSPECIFIED}" 9 | fi 10 | 11 | echo 12 | echo "Starting IPython kernel for Spark in Yarn Client mode ${USER_CLAUSE}" 13 | echo 14 | 15 | if [ -z "${SPARK_HOME}" ]; then 16 | echo "SPARK_HOME must be set to the location of a Spark distribution!" 17 | exit 1 18 | fi 19 | 20 | PROG_HOME="$(cd "`dirname "$0"`"/..; pwd)" 21 | 22 | set -x 23 | eval exec "${IMPERSONATION_OPTS}" \ 24 | "${SPARK_HOME}/bin/spark-submit" \ 25 | "${SPARK_OPTS}" \ 26 | "${PROG_HOME}/scripts/launch_ipykernel.py" \ 27 | "${LAUNCH_OPTS}" \ 28 | "$@" 29 | set +x 30 | -------------------------------------------------------------------------------- /etc/kernelspecs/spark_python_yarn_client/kernel.json: -------------------------------------------------------------------------------- 1 | { 2 | "language": "python", 3 | "display_name": "Spark - Python (YARN Client Mode)", 4 | "metadata": { 5 | "process_proxy": { 6 | "class_name": "enterprise_gateway.services.processproxies.distributed.DistributedProcessProxy" 7 | }, 8 | "debugger": true 9 | }, 10 | "env": { 11 | "SPARK_HOME": "/usr/hdp/current/spark2-client", 12 | "PYSPARK_PYTHON": "/opt/conda/bin/python", 13 | "PYTHONPATH": "${HOME}/.local/lib/python3.8/site-packages:/usr/hdp/current/spark2-client/python:/usr/hdp/current/spark2-client/python/lib/py4j-0.10.6-src.zip", 14 | "SPARK_OPTS": "--master yarn --deploy-mode client --name ${KERNEL_ID:-ERROR__NO__KERNEL_ID} ${KERNEL_EXTRA_SPARK_OPTS}", 15 | "LAUNCH_OPTS": "" 16 | }, 17 | "argv": [ 18 | "/usr/local/share/jupyter/kernels/spark_python_yarn_client/bin/run.sh", 19 | "--RemoteProcessProxy.kernel-id", 20 | "{kernel_id}", 21 | "--RemoteProcessProxy.response-address", 22 | "{response_address}", 23 | "--RemoteProcessProxy.public-key", 24 | "{public_key}", 25 | "--RemoteProcessProxy.port-range", 26 | "{port_range}", 27 | "--RemoteProcessProxy.spark-context-initialization-mode", 28 | "lazy" 29 | ] 30 | } 31 | -------------------------------------------------------------------------------- /etc/kernelspecs/spark_python_yarn_cluster/bin/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ "${EG_IMPERSONATION_ENABLED}" = "True" ]; then 4 | IMPERSONATION_OPTS="--proxy-user ${KERNEL_USERNAME:-UNSPECIFIED}" 5 | USER_CLAUSE="as user ${KERNEL_USERNAME:-UNSPECIFIED}" 6 | else 7 | IMPERSONATION_OPTS="" 8 | USER_CLAUSE="on behalf of user ${KERNEL_USERNAME:-UNSPECIFIED}" 9 | fi 10 | 11 | echo 12 | echo "Starting IPython kernel for Spark in Yarn Cluster mode ${USER_CLAUSE}" 13 | echo 14 | 15 | if [ -z "${SPARK_HOME}" ]; then 16 | echo "SPARK_HOME must be set to the location of a Spark distribution!" 17 | exit 1 18 | fi 19 | 20 | PROG_HOME="$(cd "`dirname "$0"`"/..; pwd)" 21 | 22 | set -x 23 | eval exec \ 24 | "${SPARK_HOME}/bin/spark-submit" \ 25 | "${SPARK_OPTS}" \ 26 | "${IMPERSONATION_OPTS}" \ 27 | "${PROG_HOME}/scripts/launch_ipykernel.py" \ 28 | "${LAUNCH_OPTS}" \ 29 | "$@" 30 | set +x 31 | -------------------------------------------------------------------------------- /etc/kernelspecs/spark_python_yarn_cluster/kernel.json: -------------------------------------------------------------------------------- 1 | { 2 | "language": "python", 3 | "display_name": "Spark - Python (YARN Cluster Mode)", 4 | "metadata": { 5 | "process_proxy": { 6 | "class_name": "enterprise_gateway.services.processproxies.yarn.YarnClusterProcessProxy" 7 | }, 8 | "debugger": true 9 | }, 10 | "env": { 11 | "SPARK_HOME": "/usr/hdp/current/spark2-client", 12 | "PYSPARK_PYTHON": "/opt/conda/bin/python", 13 | "PYTHONPATH": "${HOME}/.local/lib/python3.8/site-packages:/usr/hdp/current/spark2-client/python:/usr/hdp/current/spark2-client/python/lib/py4j-0.10.6-src.zip", 14 | "SPARK_OPTS": "--master yarn --deploy-mode cluster --name ${KERNEL_ID:-ERROR__NO__KERNEL_ID} --conf spark.yarn.submit.waitAppCompletion=false --conf spark.yarn.appMasterEnv.PYTHONUSERBASE=/home/${KERNEL_USERNAME}/.local --conf spark.yarn.appMasterEnv.PYTHONPATH=${HOME}/.local/lib/python3.8/site-packages:/usr/hdp/current/spark2-client/python:/usr/hdp/current/spark2-client/python/lib/py4j-0.10.6-src.zip --conf spark.yarn.appMasterEnv.PATH=/opt/conda/bin:$PATH --conf spark.yarn.maxAppAttempts=1 ${KERNEL_EXTRA_SPARK_OPTS}", 15 | "LAUNCH_OPTS": "" 16 | }, 17 | "argv": [ 18 | "/usr/local/share/jupyter/kernels/spark_python_yarn_cluster/bin/run.sh", 19 | "--RemoteProcessProxy.kernel-id", 20 | "{kernel_id}", 21 | "--RemoteProcessProxy.response-address", 22 | "{response_address}", 23 | "--RemoteProcessProxy.public-key", 24 | "{public_key}", 25 | "--RemoteProcessProxy.port-range", 26 | "{port_range}", 27 | "--RemoteProcessProxy.spark-context-initialization-mode", 28 | "lazy" 29 | ] 30 | } 31 | -------------------------------------------------------------------------------- /etc/kernelspecs/spark_scala_conductor_cluster/bin/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ "${EG_IMPERSONATION_ENABLED}" = "True" ]; then 4 | IMPERSONATION_OPTS="--proxy-user ${KERNEL_USERNAME:-UNSPECIFIED}" 5 | USER_CLAUSE="as user ${KERNEL_USERNAME:-UNSPECIFIED}" 6 | else 7 | IMPERSONATION_OPTS="" 8 | USER_CLAUSE="on behalf of user ${KERNEL_USERNAME:-UNSPECIFIED}" 9 | fi 10 | 11 | echo 12 | echo "Starting Scala kernel for Spark Cluster mode ${USER_CLAUSE}" 13 | echo 14 | 15 | if [ -z "${SPARK_HOME}" ]; then 16 | echo "SPARK_HOME must be set to the location of a Spark distribution!" 17 | exit 1 18 | fi 19 | 20 | PROG_HOME="$(cd "`dirname "$0"`"/..; pwd)" 21 | KERNEL_ASSEMBLY=`(cd "${PROG_HOME}/lib"; ls -1 toree-assembly-*.jar;)` 22 | TOREE_ASSEMBLY="${PROG_HOME}/lib/${KERNEL_ASSEMBLY}" 23 | if [ ! -f ${TOREE_ASSEMBLY} ]; then 24 | echo "Toree assembly '${PROG_HOME}/lib/toree-assembly-*.jar' is missing. Exiting..." 25 | exit 1 26 | fi 27 | 28 | # The SPARK_OPTS values during installation are stored in __TOREE_SPARK_OPTS__. This allows values to be specified during 29 | # install, but also during runtime. The runtime options take precedence over the install options. 30 | if [ "${SPARK_OPTS}" = "" ]; then 31 | SPARK_OPTS=${__TOREE_SPARK_OPTS__} 32 | fi 33 | 34 | if [ "${TOREE_OPTS}" = "" ]; then 35 | TOREE_OPTS=${__TOREE_OPTS__} 36 | fi 37 | 38 | # Toree launcher jar path, plus required lib jars (toree-assembly) 39 | JARS="${TOREE_ASSEMBLY}" 40 | # Toree launcher app path 41 | LAUNCHER_JAR=`(cd "${PROG_HOME}/lib"; ls -1 toree-launcher*.jar;)` 42 | LAUNCHER_APP="${PROG_HOME}/lib/${LAUNCHER_JAR}" 43 | if [ ! -f ${LAUNCHER_APP} ]; then 44 | echo "Scala launcher jar '${PROG_HOME}/lib/toree-launcher*.jar' is missing. Exiting..." 45 | exit 1 46 | fi 47 | 48 | eval exec \ 49 | "${SPARK_HOME}/bin/spark-submit" \ 50 | "${SPARK_OPTS}" \ 51 | "${IMPERSONATION_OPTS}" \ 52 | --jars "${JARS}" \ 53 | --class launcher.ToreeLauncher \ 54 | "${LAUNCHER_APP}" \ 55 | "${TOREE_OPTS}" \ 56 | "${LAUNCH_OPTS}" \ 57 | "$@" 58 | -------------------------------------------------------------------------------- /etc/kernelspecs/spark_scala_conductor_cluster/kernel.json: -------------------------------------------------------------------------------- 1 | { 2 | "language": "scala", 3 | "display_name": "Spark Scala (Spark Cluster Mode)", 4 | "metadata": { 5 | "process_proxy": { 6 | "class_name": "enterprise_gateway.services.processproxies.conductor.ConductorClusterProcessProxy" 7 | } 8 | }, 9 | "env": { 10 | "SPARK_OPTS": "--name ${KERNEL_ID:-ERROR__NO__KERNEL_ID} --conf spark.yarn.maxAppAttempts=1 ${KERNEL_EXTRA_SPARK_OPTS}", 11 | "__TOREE_OPTS__": "--alternate-sigint USR2 --spark-context-initialization-mode eager", 12 | "LAUNCH_OPTS": "", 13 | "DEFAULT_INTERPRETER": "Scala" 14 | }, 15 | "argv": [ 16 | "--RemoteProcessProxy.kernel-id", 17 | "{kernel_id}", 18 | "--RemoteProcessProxy.response-address", 19 | "{response_address}", 20 | "--RemoteProcessProxy.public-key", 21 | "{public_key}", 22 | "--RemoteProcessProxy.port-range", 23 | "{port_range}" 24 | ] 25 | } 26 | -------------------------------------------------------------------------------- /etc/kernelspecs/spark_scala_kubernetes/kernel.json: -------------------------------------------------------------------------------- 1 | { 2 | "language": "scala", 3 | "display_name": "Spark - Scala (Kubernetes Mode)", 4 | "metadata": { 5 | "process_proxy": { 6 | "class_name": "enterprise_gateway.services.processproxies.k8s.KubernetesProcessProxy", 7 | "config": { 8 | "image_name": "elyra/kernel-scala:VERSION", 9 | "executor_image_name": "elyra/kernel-scala:VERSION" 10 | } 11 | } 12 | }, 13 | "env": { 14 | "SPARK_HOME": "/opt/spark", 15 | "__TOREE_SPARK_OPTS__": "--master k8s://https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT} --deploy-mode cluster --name ${KERNEL_USERNAME}-${KERNEL_ID} --conf spark.kubernetes.namespace=${KERNEL_NAMESPACE} --driver-memory 2G --conf spark.kubernetes.driver.label.app=enterprise-gateway --conf spark.kubernetes.driver.label.kernel_id=${KERNEL_ID} --conf spark.kubernetes.driver.label.component=kernel --conf spark.kubernetes.executor.label.app=enterprise-gateway --conf spark.kubernetes.executor.label.kernel_id=${KERNEL_ID} --conf spark.kubernetes.executor.label.component=worker --conf spark.kubernetes.driver.container.image=${KERNEL_IMAGE} --conf spark.kubernetes.executor.container.image=${KERNEL_EXECUTOR_IMAGE} --conf spark.kubernetes.authenticate.driver.serviceAccountName=${KERNEL_SERVICE_ACCOUNT_NAME} --conf spark.kubernetes.submission.waitAppCompletion=false --conf spark.kubernetes.driverEnv.HTTP2_DISABLE=true ${KERNEL_EXTRA_SPARK_OPTS}", 16 | "__TOREE_OPTS__": "--alternate-sigint USR2", 17 | "HTTP2_DISABLE": "true", 18 | "LAUNCH_OPTS": "", 19 | "DEFAULT_INTERPRETER": "Scala" 20 | }, 21 | "argv": [ 22 | "/usr/local/share/jupyter/kernels/spark_scala_kubernetes/bin/run.sh", 23 | "--RemoteProcessProxy.kernel-id", 24 | "{kernel_id}", 25 | "--RemoteProcessProxy.port-range", 26 | "{port_range}", 27 | "--RemoteProcessProxy.response-address", 28 | "{response_address}", 29 | "--RemoteProcessProxy.public-key", 30 | "{public_key}", 31 | "--RemoteProcessProxy.spark-context-initialization-mode", 32 | "eager" 33 | ] 34 | } 35 | -------------------------------------------------------------------------------- /etc/kernelspecs/spark_scala_yarn_client/bin/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ "${EG_IMPERSONATION_ENABLED}" = "True" ]; then 4 | IMPERSONATION_OPTS="sudo PATH=${PATH} -H -E -u ${KERNEL_USERNAME:-UNSPECIFIED}" 5 | USER_CLAUSE="as user ${KERNEL_USERNAME:-UNSPECIFIED}" 6 | else 7 | IMPERSONATION_OPTS="" 8 | USER_CLAUSE="on behalf of user ${KERNEL_USERNAME:-UNSPECIFIED}" 9 | fi 10 | 11 | echo 12 | echo "Starting Scala kernel for Spark in Yarn Client mode ${USER_CLAUSE}" 13 | echo 14 | 15 | if [ -z "${SPARK_HOME}" ]; then 16 | echo "SPARK_HOME must be set to the location of a Spark distribution!" 17 | exit 1 18 | fi 19 | 20 | PROG_HOME="$(cd "`dirname "$0"`"/..; pwd)" 21 | KERNEL_ASSEMBLY=`(cd "${PROG_HOME}/lib"; ls -1 toree-assembly-*.jar;)` 22 | TOREE_ASSEMBLY="${PROG_HOME}/lib/${KERNEL_ASSEMBLY}" 23 | if [ ! -f ${TOREE_ASSEMBLY} ]; then 24 | echo "Toree assembly '${PROG_HOME}/lib/toree-assembly-*.jar' is missing. Exiting..." 25 | exit 1 26 | fi 27 | 28 | # The SPARK_OPTS values during installation are stored in __TOREE_SPARK_OPTS__. This allows values to be specified during 29 | # install, but also during runtime. The runtime options take precedence over the install options. 30 | if [ "${SPARK_OPTS}" = "" ]; then 31 | SPARK_OPTS=${__TOREE_SPARK_OPTS__} 32 | fi 33 | 34 | if [ "${TOREE_OPTS}" = "" ]; then 35 | TOREE_OPTS=${__TOREE_OPTS__} 36 | fi 37 | 38 | # Toree launcher jar path, plus required lib jars (toree-assembly) 39 | JARS="${TOREE_ASSEMBLY}" 40 | # Toree launcher app path 41 | LAUNCHER_JAR=`(cd "${PROG_HOME}/lib"; ls -1 toree-launcher*.jar;)` 42 | LAUNCHER_APP="${PROG_HOME}/lib/${LAUNCHER_JAR}" 43 | if [ ! -f ${LAUNCHER_APP} ]; then 44 | echo "Scala launcher jar '${PROG_HOME}/lib/toree-launcher*.jar' is missing. Exiting..." 45 | exit 1 46 | fi 47 | 48 | set -x 49 | eval exec "${IMPERSONATION_OPTS}" \ 50 | "${SPARK_HOME}/bin/spark-submit" \ 51 | "${SPARK_OPTS}" \ 52 | --jars "${JARS}" \ 53 | --class launcher.ToreeLauncher \ 54 | "${LAUNCHER_APP}" \ 55 | "${TOREE_OPTS}" \ 56 | "${LAUNCH_OPTS}" \ 57 | "$@" 58 | set +x 59 | -------------------------------------------------------------------------------- /etc/kernelspecs/spark_scala_yarn_client/kernel.json: -------------------------------------------------------------------------------- 1 | { 2 | "language": "scala", 3 | "display_name": "Spark - Scala (YARN Client Mode)", 4 | "metadata": { 5 | "process_proxy": { 6 | "class_name": "enterprise_gateway.services.processproxies.distributed.DistributedProcessProxy" 7 | } 8 | }, 9 | "env": { 10 | "SPARK_HOME": "/usr/hdp/current/spark2-client", 11 | "__TOREE_SPARK_OPTS__": "--master yarn --deploy-mode client --name ${KERNEL_ID:-ERROR__NO__KERNEL_ID} ${KERNEL_EXTRA_SPARK_OPTS}", 12 | "__TOREE_OPTS__": "--alternate-sigint USR2", 13 | "LAUNCH_OPTS": "", 14 | "DEFAULT_INTERPRETER": "Scala" 15 | }, 16 | "argv": [ 17 | "/usr/local/share/jupyter/kernels/spark_scala_yarn_client/bin/run.sh", 18 | "--RemoteProcessProxy.kernel-id", 19 | "{kernel_id}", 20 | "--RemoteProcessProxy.response-address", 21 | "{response_address}", 22 | "--RemoteProcessProxy.public-key", 23 | "{public_key}", 24 | "--RemoteProcessProxy.port-range", 25 | "{port_range}", 26 | "--RemoteProcessProxy.spark-context-initialization-mode", 27 | "lazy" 28 | ] 29 | } 30 | -------------------------------------------------------------------------------- /etc/kernelspecs/spark_scala_yarn_cluster/bin/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ "${EG_IMPERSONATION_ENABLED}" = "True" ]; then 4 | IMPERSONATION_OPTS="--proxy-user ${KERNEL_USERNAME:-UNSPECIFIED}" 5 | USER_CLAUSE="as user ${KERNEL_USERNAME:-UNSPECIFIED}" 6 | else 7 | IMPERSONATION_OPTS="" 8 | USER_CLAUSE="on behalf of user ${KERNEL_USERNAME:-UNSPECIFIED}" 9 | fi 10 | 11 | echo 12 | echo "Starting Scala kernel for Spark in Yarn Cluster mode ${USER_CLAUSE}" 13 | echo 14 | 15 | if [ -z "${SPARK_HOME}" ]; then 16 | echo "SPARK_HOME must be set to the location of a Spark distribution!" 17 | exit 1 18 | fi 19 | 20 | PROG_HOME="$(cd "`dirname "$0"`"/..; pwd)" 21 | KERNEL_ASSEMBLY=`(cd "${PROG_HOME}/lib"; ls -1 toree-assembly-*.jar;)` 22 | TOREE_ASSEMBLY="${PROG_HOME}/lib/${KERNEL_ASSEMBLY}" 23 | if [ ! -f ${TOREE_ASSEMBLY} ]; then 24 | echo "Toree assembly '${PROG_HOME}/lib/toree-assembly-*.jar' is missing. Exiting..." 25 | exit 1 26 | fi 27 | 28 | # The SPARK_OPTS values during installation are stored in __TOREE_SPARK_OPTS__. This allows values to be specified during 29 | # install, but also during runtime. The runtime options take precedence over the install options. 30 | if [ "${SPARK_OPTS}" = "" ]; then 31 | SPARK_OPTS=${__TOREE_SPARK_OPTS__} 32 | fi 33 | 34 | if [ "${TOREE_OPTS}" = "" ]; then 35 | TOREE_OPTS=${__TOREE_OPTS__} 36 | fi 37 | 38 | # Toree launcher jar path, plus required lib jars (toree-assembly) 39 | JARS="${TOREE_ASSEMBLY}" 40 | # Toree launcher app path 41 | LAUNCHER_JAR=`(cd "${PROG_HOME}/lib"; ls -1 toree-launcher*.jar;)` 42 | LAUNCHER_APP="${PROG_HOME}/lib/${LAUNCHER_JAR}" 43 | if [ ! -f ${LAUNCHER_APP} ]; then 44 | echo "Scala launcher jar '${PROG_HOME}/lib/toree-launcher*.jar' is missing. Exiting..." 45 | exit 1 46 | fi 47 | 48 | set -x 49 | eval exec \ 50 | "${SPARK_HOME}/bin/spark-submit" \ 51 | "${SPARK_OPTS}" \ 52 | "${IMPERSONATION_OPTS}" \ 53 | --jars "${JARS}" \ 54 | --class launcher.ToreeLauncher \ 55 | "${LAUNCHER_APP}" \ 56 | "${TOREE_OPTS}" \ 57 | "${LAUNCH_OPTS}" \ 58 | "$@" 59 | set +x 60 | -------------------------------------------------------------------------------- /etc/kernelspecs/spark_scala_yarn_cluster/kernel.json: -------------------------------------------------------------------------------- 1 | { 2 | "language": "scala", 3 | "display_name": "Spark - Scala (YARN Cluster Mode)", 4 | "metadata": { 5 | "process_proxy": { 6 | "class_name": "enterprise_gateway.services.processproxies.yarn.YarnClusterProcessProxy" 7 | } 8 | }, 9 | "env": { 10 | "SPARK_HOME": "/usr/hdp/current/spark2-client", 11 | "__TOREE_SPARK_OPTS__": "--master yarn --deploy-mode cluster --name ${KERNEL_ID:-ERROR__NO__KERNEL_ID} --conf spark.yarn.submit.waitAppCompletion=false --conf spark.yarn.am.waitTime=1d --conf spark.yarn.maxAppAttempts=1 ${KERNEL_EXTRA_SPARK_OPTS}", 12 | "__TOREE_OPTS__": "--alternate-sigint USR2", 13 | "LAUNCH_OPTS": "", 14 | "DEFAULT_INTERPRETER": "Scala" 15 | }, 16 | "argv": [ 17 | "/usr/local/share/jupyter/kernels/spark_scala_yarn_cluster/bin/run.sh", 18 | "--RemoteProcessProxy.kernel-id", 19 | "{kernel_id}", 20 | "--RemoteProcessProxy.response-address", 21 | "{response_address}", 22 | "--RemoteProcessProxy.public-key", 23 | "{public_key}", 24 | "--RemoteProcessProxy.port-range", 25 | "{port_range}", 26 | "--RemoteProcessProxy.spark-context-initialization-mode", 27 | "lazy" 28 | ] 29 | } 30 | -------------------------------------------------------------------------------- /etc/kubernetes/helm/enterprise-gateway/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: enterprise-gateway 3 | description: A helm chart to deploy Jupyter Enterprise Gateway 4 | # This is the chart version. This version number should be incremented each time you make changes 5 | # to the chart and its templates, including the app version. 6 | # Versions are expected to follow Semantic Versioning (https://semver.org/) 7 | version: 3.3.0-dev0 8 | 9 | # This is the version number of the application being deployed. This version number should be 10 | # incremented each time you make changes to the application. Versions are not expected to 11 | # follow Semantic Versioning. They should reflect the version the application is using. 12 | appVersion: 3.3.0.dev0 13 | 14 | icon: https://avatars1.githubusercontent.com/u/7388996?s=200&v=4 15 | home: https://jupyter.org 16 | 17 | # A chart can be either an 'application' or a 'library' chart. 18 | # 19 | # Application charts are a collection of templates that can be packaged into versioned archives 20 | # to be deployed. 21 | # 22 | # Library charts provide useful utilities or functions for the chart developer. They're included as 23 | # a dependency of application charts to inject those utilities and functions into the rendering 24 | # pipeline. Library charts do not define any templates and therefore cannot be deployed. 25 | type: application 26 | 27 | sources: 28 | - https://github.com/jupyter-server/enterprise_gateway 29 | kubeVersion: '>=1.18.0-0' 30 | -------------------------------------------------------------------------------- /etc/kubernetes/helm/enterprise-gateway/templates/eg-clusterrole.yaml: -------------------------------------------------------------------------------- 1 | {{- if and (.Values.deployment.serviceAccountName) (.Values.global.rbac) }} 2 | --- 3 | apiVersion: rbac.authorization.k8s.io/v1 4 | kind: ClusterRole 5 | metadata: 6 | name: enterprise-gateway-controller 7 | labels: 8 | app: enterprise-gateway 9 | component: enterprise-gateway 10 | chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} 11 | release: {{ .Release.Name }} 12 | heritage: {{ .Release.Service }} 13 | {{- range $key, $val := .Values.global.commonLabels }} 14 | {{ $key }}: "{{ $val }}" 15 | {{- end }} 16 | rules: 17 | - apiGroups: [""] 18 | resources: ["pods", "namespaces", "services", "configmaps", "secrets", "persistentvolumes", "persistentvolumeclaims"] 19 | verbs: ["get", "watch", "list", "create", "delete"] 20 | - apiGroups: ["rbac.authorization.k8s.io"] 21 | resources: ["rolebindings"] 22 | verbs: ["get", "list", "create", "delete"] 23 | - apiGroups: ["sparkoperator.k8s.io"] 24 | resources: ["sparkapplications", "sparkapplications/status", "scheduledsparkapplications", "scheduledsparkapplications/status"] 25 | verbs: ["get", "watch", "list", "create", "delete"] 26 | --- 27 | apiVersion: rbac.authorization.k8s.io/v1 28 | kind: ClusterRole 29 | metadata: 30 | # Referenced by EG_KERNEL_CLUSTER_ROLE in the Deployment 31 | name: kernel-controller 32 | labels: 33 | app: enterprise-gateway 34 | component: kernel 35 | chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} 36 | release: {{ .Release.Name }} 37 | heritage: {{ .Release.Service }} 38 | rules: 39 | - apiGroups: [""] 40 | resources: ["pods"] 41 | verbs: ["get", "watch", "list", "create", "delete"] 42 | - apiGroups: [""] 43 | resources: ["configmaps"] 44 | verbs: ["list", "create"] 45 | - apiGroups: [""] 46 | resources: ["services", "persistentvolumeclaims"] 47 | verbs: ["list"] 48 | {{- end }} 49 | -------------------------------------------------------------------------------- /etc/kubernetes/helm/enterprise-gateway/templates/eg-clusterrolebinding.yaml: -------------------------------------------------------------------------------- 1 | {{- if and (.Values.deployment.serviceAccountName) (.Values.global.rbac) }} 2 | --- 3 | apiVersion: rbac.authorization.k8s.io/v1 4 | kind: ClusterRoleBinding 5 | metadata: 6 | name: enterprise-gateway-controller 7 | labels: 8 | app: enterprise-gateway 9 | component: enterprise-gateway 10 | chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} 11 | release: {{ .Release.Name }} 12 | heritage: {{ .Release.Service }} 13 | {{- range $key, $val := .Values.global.commonLabels }} 14 | {{ $key }}: "{{ $val }}" 15 | {{- end }} 16 | subjects: 17 | - kind: ServiceAccount 18 | name: {{ .Values.deployment.serviceAccountName }} 19 | namespace: {{ .Release.Namespace }} 20 | roleRef: 21 | kind: ClusterRole 22 | name: enterprise-gateway-controller 23 | apiGroup: rbac.authorization.k8s.io 24 | {{- end }} 25 | -------------------------------------------------------------------------------- /etc/kubernetes/helm/enterprise-gateway/templates/eg-serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if and (.Values.deployment.serviceAccountName) (.Values.global.rbac) }} 2 | --- 3 | apiVersion: v1 4 | kind: ServiceAccount 5 | {{- if .Values.global.imagePullSecrets }} 6 | imagePullSecrets: 7 | {{- $parent := . -}} 8 | {{- range .Values.global.imagePullSecrets }} 9 | - name: {{ . }} 10 | {{- end }} 11 | {{- end }} 12 | metadata: 13 | name: {{ .Values.deployment.serviceAccountName }} 14 | namespace: {{ .Release.Namespace }} 15 | labels: 16 | app: enterprise-gateway 17 | component: enterprise-gateway 18 | chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} 19 | release: {{ .Release.Name }} 20 | heritage: {{ .Release.Service }} 21 | {{- range $key, $val := .Values.global.commonLabels }} 22 | {{ $key }}: "{{ $val }}" 23 | {{- end }} 24 | {{- if .Values.deployment.annotations }} 25 | annotations: 26 | {{- range $key, $val := .Values.deployment.annotations }} 27 | {{ $key }}: "{{ $val }}" 28 | {{- end }} 29 | {{- end }} 30 | {{- end }} 31 | -------------------------------------------------------------------------------- /etc/kubernetes/helm/enterprise-gateway/templates/imagepullSecret.yaml: -------------------------------------------------------------------------------- 1 | {{- if and (.Values.imagePullSecretsCreate.enabled) (.Values.imagePullSecretsCreate.secrets) -}} 2 | --- 3 | {{- $root := .Values }} 4 | {{- range .Values.imagePullSecretsCreate.secrets }} 5 | apiVersion: v1 6 | data: 7 | .dockerconfigjson: {{ .data }} 8 | kind: Secret 9 | metadata: 10 | name: {{ .name }} 11 | {{- if ($root.imagePullSecretsCreate.annotations) -}} 12 | {{- with $root.imagePullSecretsCreate.annotations }} 13 | annotations: 14 | {{- toYaml . | nindent 4 }} 15 | {{- end }} 16 | {{- end }} 17 | 18 | type: kubernetes.io/dockerconfigjson 19 | {{- end }} 20 | --- 21 | {{- end }} 22 | -------------------------------------------------------------------------------- /etc/kubernetes/helm/enterprise-gateway/templates/ingress.yaml: -------------------------------------------------------------------------------- 1 | {{ if .Values.ingress.enabled }} 2 | {{- $parent := . -}} 3 | {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} 4 | apiVersion: networking.k8s.io/v1 5 | {{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} 6 | apiVersion: networking.k8s.io/v1beta1 7 | {{- else -}} 8 | apiVersion: extensions/v1beta1 9 | {{- end }} 10 | kind: Ingress 11 | metadata: 12 | namespace: {{ .Release.Namespace }} 13 | name: enterprise-gateway-ingress 14 | {{- if .Values.ingress.annotations }} 15 | annotations: 16 | {{ toYaml .Values.ingress.annotations | indent 4}} 17 | {{- end }} 18 | spec: 19 | {{ if .Values.ingress.ingressClassName }} 20 | ingressClassName: {{ .Values.ingress.ingressClassName }} 21 | {{ end }} 22 | rules: 23 | - host: {{ .Values.ingress.hostName }} 24 | http: 25 | paths: 26 | - path: {{ .Values.ingress.path }} 27 | {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} 28 | pathType: {{ .Values.ingress.pathType }} 29 | {{- end }} 30 | backend: 31 | {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} 32 | service: 33 | name: "enterprise-gateway" 34 | port: 35 | {{ with index .Values.service.ports 0 }} 36 | number: {{ .port }} 37 | {{ end }} 38 | {{- else }} 39 | serviceName: "enterprise-gateway" 40 | {{ with index .Values.service.ports 0 }} 41 | servicePort: {{ .port }} 42 | {{- end }} 43 | {{- end }} 44 | {{- if .Values.ingress.tls }} 45 | tls: 46 | {{- range .Values.ingress.tls }} 47 | - hosts: 48 | {{- range .hosts }} 49 | - {{ . }} 50 | {{- end }} 51 | secretName: {{ .secretName }} 52 | {{- end }} 53 | {{- end }} 54 | {{- end }} 55 | -------------------------------------------------------------------------------- /etc/kubernetes/helm/enterprise-gateway/templates/kip-clusterrole.yaml: -------------------------------------------------------------------------------- 1 | {{- if and (.Values.kip.serviceAccountName) (.Values.global.rbac) }} 2 | --- 3 | apiVersion: rbac.authorization.k8s.io/v1 4 | kind: ClusterRole 5 | metadata: 6 | name: kip-controller 7 | labels: 8 | app: enterprise-gateway 9 | component: kernel-image-puller 10 | chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} 11 | release: {{ .Release.Name }} 12 | heritage: {{ .Release.Service }} 13 | {{- range $key, $val := .Values.global.commonLabels }} 14 | {{ $key }}: "{{ $val }}" 15 | {{- end }} 16 | rules: 17 | - apiGroups: [""] 18 | resources: ["pods"] 19 | verbs: ["get", "watch", "list", "create", "delete"] 20 | {{- if .Values.kip.podSecurityPolicy.create }} 21 | - apiGroups: 22 | - policy 23 | resources: 24 | - podsecuritypolicies 25 | resourceNames: 26 | - "kip-psp" 27 | verbs: 28 | - use 29 | {{- end }} 30 | {{- end }} 31 | -------------------------------------------------------------------------------- /etc/kubernetes/helm/enterprise-gateway/templates/kip-clusterrolebinding.yaml: -------------------------------------------------------------------------------- 1 | {{- if and (.Values.kip.serviceAccountName) (.Values.global.rbac) }} 2 | --- 3 | apiVersion: rbac.authorization.k8s.io/v1 4 | kind: ClusterRoleBinding 5 | metadata: 6 | name: enterprise-gateway-kip 7 | labels: 8 | app: enterprise-gateway 9 | component: kernel-image-puller 10 | chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} 11 | release: {{ .Release.Name }} 12 | heritage: {{ .Release.Service }} 13 | {{- range $key, $val := .Values.global.commonLabels }} 14 | {{ $key }}: "{{ $val }}" 15 | {{- end }} 16 | 17 | subjects: 18 | - kind: ServiceAccount 19 | name: {{ .Values.kip.serviceAccountName }} 20 | namespace: {{ .Release.Namespace }} 21 | roleRef: 22 | kind: ClusterRole 23 | name: kip-controller 24 | apiGroup: rbac.authorization.k8s.io 25 | {{- end }} 26 | -------------------------------------------------------------------------------- /etc/kubernetes/helm/enterprise-gateway/templates/kip-serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if and (.Values.kip.serviceAccountName) (.Values.global.rbac) }} 2 | 3 | apiVersion: v1 4 | kind: ServiceAccount 5 | {{- if .Values.global.imagePullSecrets }} 6 | imagePullSecrets: 7 | {{- $parent := . -}} 8 | {{- range .Values.global.imagePullSecrets }} 9 | - name: {{ . }} 10 | {{- end }} 11 | {{- end }} 12 | metadata: 13 | name: {{ .Values.kip.serviceAccountName }} 14 | namespace: {{ .Release.Namespace }} 15 | labels: 16 | app: enterprise-gateway 17 | component: enterprise-gateway 18 | chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} 19 | release: {{ .Release.Name }} 20 | heritage: {{ .Release.Service }} 21 | {{- range $key, $val := .Values.global.commonLabels }} 22 | {{ $key }}: "{{ $val }}" 23 | {{- end }} 24 | {{- if .Values.kip.annotations }} 25 | annotations: 26 | {{- range $key, $val := .Values.kip.annotations }} 27 | {{ $key }}: "{{ $val }}" 28 | {{- end }} 29 | {{- end }} 30 | {{- end }} 31 | -------------------------------------------------------------------------------- /etc/kubernetes/helm/enterprise-gateway/templates/psp.yaml: -------------------------------------------------------------------------------- 1 | {{- if and (.Values.kip.podSecurityPolicy.create) (.Values.global.rbac) }} 2 | apiVersion: policy/v1beta1 3 | kind: PodSecurityPolicy 4 | metadata: 5 | name: "kip-psp" 6 | {{- if .Values.kip.podSecurityPolicy.annotations }} 7 | annotations: 8 | {{- toYaml .Values.kip.podSecurityPolicy.annotations | nindent 4 }} 9 | {{- end }} 10 | spec: 11 | privileged: false 12 | # Required to prevent escalations to root. 13 | allowPrivilegeEscalation: false 14 | # This is redundant with non-root + disallow privilege escalation, 15 | # but we can provide it for defense in depth. 16 | requiredDropCapabilities: 17 | - ALL 18 | hostNetwork: false 19 | hostIPC: false 20 | hostPID: false 21 | runAsUser: 22 | # TODO: Require the container to run without root privileges. 23 | rule: 'RunAsAny' 24 | seLinux: 25 | # This policy assumes the nodes are using AppArmor rather than SELinux. 26 | rule: 'RunAsAny' 27 | supplementalGroups: 28 | rule: 'MustRunAs' 29 | ranges: 30 | # Forbid adding the root group. 31 | - min: 1 32 | max: 65535 33 | fsGroup: 34 | rule: 'MustRunAs' 35 | ranges: 36 | # Forbid adding the root group. 37 | - min: 1 38 | max: 65535 39 | readOnlyRootFilesystem: false 40 | allowedHostPaths: 41 | - pathPrefix: /var/run 42 | readOnly: true # only allow read-only mounts 43 | volumes: 44 | - '*' 45 | # - 'secret' 46 | # - 'hostPath' 47 | {{- end }} 48 | -------------------------------------------------------------------------------- /etc/kubernetes/helm/enterprise-gateway/templates/service.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.deployment.enabled }} 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | labels: 6 | app: enterprise-gateway 7 | component: enterprise-gateway 8 | chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} 9 | release: {{ .Release.Name }} 10 | heritage: {{ .Release.Service }} 11 | {{- range $key, $val := .Values.global.commonLabels }} 12 | {{ $key }}: "{{ $val }}" 13 | {{- end }} 14 | 15 | name: enterprise-gateway 16 | namespace: {{ .Release.Namespace }} 17 | spec: 18 | ports: 19 | {{- range $key, $val := .Values.service.ports }} 20 | - 21 | {{- range $pkey, $pval := $val }} 22 | {{ $pkey}}: {{ $pval }} 23 | {{- end }} 24 | {{- end }} 25 | selector: 26 | gateway-selector: enterprise-gateway 27 | sessionAffinity: ClientIP 28 | type: {{ .Values.service.type }} 29 | {{- if .Values.service.externalIPs.k8sMasterPublicIP }} 30 | externalIPs: 31 | - {{ .Values.service.externalIPs.k8sMasterPublicIP }} 32 | {{- end }} 33 | {{- end }} 34 | -------------------------------------------------------------------------------- /requirements.yml: -------------------------------------------------------------------------------- 1 | channels: 2 | - conda-forge 3 | - defaults 4 | dependencies: 5 | - docker-py>=3.5.0 6 | - future 7 | - jinja2>=3.1 8 | - jupyter_client>=6.1 9 | - jupyter_core>=4.6.0 10 | - jupyter_server>=1.7 11 | - paramiko>=2.1.2 12 | - pexpect>=4.2.0 13 | - pip 14 | - pre-commit 15 | - pycryptodomex>=3.9.7 16 | - python-kubernetes>=18.20.0 17 | - pyzmq>=20.0.0 18 | - requests>=2.7,<3.0 19 | - tornado>=6.1 20 | - traitlets>=4.2.0 21 | - watchdog 22 | - yarn-api-client>=1.0 23 | 24 | # Test Requirements 25 | - ipykernel 26 | - mock 27 | - pytest 28 | - pytest-tornasync 29 | - websocket-client 30 | 31 | # Code Style 32 | - flake8 33 | 34 | - pip: 35 | - . 36 | - -r docs/doc-requirements.txt 37 | -------------------------------------------------------------------------------- /website/.gitignore: -------------------------------------------------------------------------------- 1 | _site/ 2 | .sass-cache/ 3 | -------------------------------------------------------------------------------- /website/README.md: -------------------------------------------------------------------------------- 1 | # Jupyter Enterprise Gateway website 2 | 3 | A Jekyll based website describing a general overview of the Jupyter Enterprise Gateway project 4 | 5 | ## Building the project 6 | 7 | jekyll serve --watch 8 | -------------------------------------------------------------------------------- /website/_config.yml: -------------------------------------------------------------------------------- 1 | # Site settings 2 | title: Jupyter Enterprise Gateway 3 | email: 4 | description: > # this means to ignore newlines until "baseurl:" 5 | baseurl: "/enterprise_gateway" # the subpath of your site, e.g. /blog/ 6 | url: "" # the base hostname & protocol for your site 7 | 8 | twitter_username: 9 | github_username: 10 | 11 | # Build settings 12 | markdown: kramdown 13 | -------------------------------------------------------------------------------- /website/_data/navigation.yml: -------------------------------------------------------------------------------- 1 | topnav: 2 | 3 | - title: Enterprise Gateway 4 | subcategories: 5 | - title: What is it? 6 | url: /enterprise_gateway/#about 7 | - title: Features 8 | url: /enterprise_gateway/#features 9 | - title: Supported Platforms 10 | url: /enterprise_gateway/#platforms 11 | - title: Contact 12 | url: /enterprise_gateway/#contact 13 | 14 | 15 | - title: Documentation 16 | url: https://jupyter-enterprise-gateway.readthedocs.io/en/latest/ 17 | 18 | - title: GitHub 19 | url: https://github.com/jupyter/enterprise_gateway 20 | 21 | - title: Privacy 22 | url: /enterprise_gateway/privacy-policy 23 | -------------------------------------------------------------------------------- /website/_includes/call-to-action.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 |
5 |

Jupyter Enterprise Gateway

6 |
7 |

A lightweight, multi-tenant, scalable and secure gateway that enables Jupyter Notebooks to share resources across distributed clusters such as Apache Spark, Kubernetes and others.

8 | Get Started! 9 |
10 |
11 |
12 |
13 | -------------------------------------------------------------------------------- /website/_includes/contact.html: -------------------------------------------------------------------------------- 1 |
2 | 3 |
4 |
5 |
6 |
7 |

Let's Get In Touch!

8 |

Join the Jupyter Gateway community by interacting with us on the Jupyter mailing list or via the project Github.

9 |
10 |
11 | 12 | 13 |

{{ site.email }}

14 | 15 |
16 |
17 | 18 | 19 |

{{ site.github_username }}

20 | 21 |
22 |
23 |
24 |
25 | -------------------------------------------------------------------------------- /website/_includes/head.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Jupyter Enterprise Gateway 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /website/_includes/header.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 |

Jupyter
Enterprise Gateway

5 |
6 |

The Jupyter Enterprise Gateway project is dedicated to making Jupyter Notebook stack multi-tenant, scalable, secure and ready for Enterprise scenarios such as Big Data Analytics, Machine Learning and Deep Learning model development.

7 | Find Out More 8 |
9 |
10 |
11 | -------------------------------------------------------------------------------- /website/_includes/nav.html: -------------------------------------------------------------------------------- 1 | 43 | -------------------------------------------------------------------------------- /website/_includes/scripts.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 24 | -------------------------------------------------------------------------------- /website/_layouts/home.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include head.html %} 5 | 6 | 7 | {% include nav.html %} 8 | {% include header.html %} 9 | {% include call-to-action.html %} 10 | {% include features.html %} 11 | {% include platforms.html %} 12 | {% include contact.html %} 13 | {% include scripts.html %} 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /website/_layouts/page.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include head.html %} 5 | 6 | 7 | {% include nav.html %} 8 | 9 |
10 |
11 |
12 |
13 | {{ content }} 14 |
15 |
16 |
17 | 18 | {% include scripts.html %} 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /website/_sass/_mixins.scss: -------------------------------------------------------------------------------- 1 | @mixin transition-all() 2 | { 3 | -webkit-transition: all 0.35s; 4 | -moz-transition: all 0.35s; 5 | transition: all 0.35s; 6 | } 7 | 8 | @mixin background-cover() 9 | { 10 | -webkit-background-size: cover; 11 | -moz-background-size: cover; 12 | background-size: cover; 13 | -o-background-size: cover; 14 | } 15 | 16 | @mixin button-variant($color, $background, $border) 17 | { 18 | color: $color; 19 | background-color: $background; 20 | border-color: $border; 21 | @include transition-all; 22 | 23 | &:hover, 24 | &:focus, 25 | &.focus, 26 | &:active, 27 | &.active, 28 | .open > .dropdown-toggle & { 29 | color: $color; 30 | background-color: darken($background, 5%); 31 | border-color: darken($border, 7%); 32 | } 33 | &:active, 34 | &.active, 35 | .open > .dropdown-toggle & { 36 | background-image: none; 37 | } 38 | &.disabled, 39 | &[disabled], 40 | fieldset[disabled] & { 41 | &, 42 | &:hover, 43 | &:focus, 44 | &.focus, 45 | &:active, 46 | &.active { 47 | background-color: $background; 48 | border-color: $border; 49 | } 50 | } 51 | 52 | .badge { 53 | color: $background; 54 | background-color: $color; 55 | } 56 | } 57 | 58 | @mixin sans-serif-font() 59 | { 60 | font-family: 'Open Sans', 'Helvetica Neue', Arial, sans-serif; 61 | } 62 | 63 | @mixin serif-font() 64 | { 65 | font-family: 'Merriweather', 'Helvetica Neue', Arial, sans-serif; 66 | } 67 | -------------------------------------------------------------------------------- /website/css/main.scss: -------------------------------------------------------------------------------- 1 | --- 2 | --- 3 | 4 | @charset "utf-8"; 5 | 6 | // Example Defaults 7 | // $base-font-family: Helvetica, Arial, sans-serif; 8 | // $base-font-size: 16px; 9 | // $small-font-size: $base-font-size * 0.875; 10 | // $base-line-height: 1.5; 11 | 12 | // $spacing-unit: 30px; 13 | 14 | // $text-color: #111; 15 | // $background-color: #fdfdfd; 16 | // $brand-color: #2a7ae2; 17 | 18 | // $grey-color: #828282; 19 | // $grey-color-light: lighten($grey-color, 40%); 20 | // $grey-color-dark: darken($grey-color, 25%); 21 | 22 | // // Width of the content area 23 | // $content-width: 800px; 24 | 25 | // $on-palm: 600px; 26 | // $on-laptop: 800px; 27 | 28 | // Using media queries with like this: 29 | // @include media-query($on-palm) { 30 | // .wrapper { 31 | // padding-right: $spacing-unit / 2; 32 | // padding-left: $spacing-unit / 2; 33 | // } 34 | // } 35 | 36 | // variables.less 37 | $theme-primary: #F05F40; 38 | $theme-dark: #222; 39 | 40 | // Import partials from `sass_dir` (defaults to `_sass`) 41 | @import 42 | "mixins", 43 | "base" 44 | ; 45 | -------------------------------------------------------------------------------- /website/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/website/favicon.ico -------------------------------------------------------------------------------- /website/font-awesome/fonts/FontAwesome.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/website/font-awesome/fonts/FontAwesome.otf -------------------------------------------------------------------------------- /website/font-awesome/fonts/fontawesome-webfont.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/website/font-awesome/fonts/fontawesome-webfont.eot -------------------------------------------------------------------------------- /website/font-awesome/fonts/fontawesome-webfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/website/font-awesome/fonts/fontawesome-webfont.ttf -------------------------------------------------------------------------------- /website/font-awesome/fonts/fontawesome-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/website/font-awesome/fonts/fontawesome-webfont.woff -------------------------------------------------------------------------------- /website/font-awesome/fonts/fontawesome-webfont.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/website/font-awesome/fonts/fontawesome-webfont.woff2 -------------------------------------------------------------------------------- /website/font-awesome/less/animated.less: -------------------------------------------------------------------------------- 1 | // Animated Icons 2 | // -------------------------- 3 | 4 | .@{fa-css-prefix}-spin { 5 | -webkit-animation: fa-spin 2s infinite linear; 6 | animation: fa-spin 2s infinite linear; 7 | } 8 | 9 | .@{fa-css-prefix}-pulse { 10 | -webkit-animation: fa-spin 1s infinite steps(8); 11 | animation: fa-spin 1s infinite steps(8); 12 | } 13 | 14 | @-webkit-keyframes fa-spin { 15 | 0% { 16 | -webkit-transform: rotate(0deg); 17 | transform: rotate(0deg); 18 | } 19 | 100% { 20 | -webkit-transform: rotate(359deg); 21 | transform: rotate(359deg); 22 | } 23 | } 24 | 25 | @keyframes fa-spin { 26 | 0% { 27 | -webkit-transform: rotate(0deg); 28 | transform: rotate(0deg); 29 | } 30 | 100% { 31 | -webkit-transform: rotate(359deg); 32 | transform: rotate(359deg); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /website/font-awesome/less/bordered-pulled.less: -------------------------------------------------------------------------------- 1 | // Bordered & Pulled 2 | // ------------------------- 3 | 4 | .@{fa-css-prefix}-border { 5 | padding: .2em .25em .15em; 6 | border: solid .08em @fa-border-color; 7 | border-radius: .1em; 8 | } 9 | 10 | .pull-right { float: right; } 11 | .pull-left { float: left; } 12 | 13 | .@{fa-css-prefix} { 14 | &.pull-left { margin-right: .3em; } 15 | &.pull-right { margin-left: .3em; } 16 | } 17 | -------------------------------------------------------------------------------- /website/font-awesome/less/core.less: -------------------------------------------------------------------------------- 1 | // Base Class Definition 2 | // ------------------------- 3 | 4 | .@{fa-css-prefix} { 5 | display: inline-block; 6 | font: normal normal normal @fa-font-size-base/1 FontAwesome; // shortening font declaration 7 | font-size: inherit; // can't have font-size inherit on line above, so need to override 8 | text-rendering: auto; // optimizelegibility throws things off #1094 9 | -webkit-font-smoothing: antialiased; 10 | -moz-osx-font-smoothing: grayscale; 11 | transform: translate(0, 0); // ensures no half-pixel rendering in firefox 12 | 13 | } 14 | -------------------------------------------------------------------------------- /website/font-awesome/less/fixed-width.less: -------------------------------------------------------------------------------- 1 | // Fixed Width Icons 2 | // ------------------------- 3 | .@{fa-css-prefix}-fw { 4 | width: (18em / 14); 5 | text-align: center; 6 | } 7 | -------------------------------------------------------------------------------- /website/font-awesome/less/font-awesome.less: -------------------------------------------------------------------------------- 1 | /*! 2 | * Font Awesome 4.3.0 by @davegandy - http://fontawesome.io - @fontawesome 3 | * License - http://fontawesome.io/license (Font: SIL OFL 1.1, CSS: MIT License) 4 | */ 5 | 6 | @import "variables.less"; 7 | @import "mixins.less"; 8 | @import "path.less"; 9 | @import "core.less"; 10 | @import "larger.less"; 11 | @import "fixed-width.less"; 12 | @import "list.less"; 13 | @import "bordered-pulled.less"; 14 | @import "animated.less"; 15 | @import "rotated-flipped.less"; 16 | @import "stacked.less"; 17 | @import "icons.less"; 18 | -------------------------------------------------------------------------------- /website/font-awesome/less/larger.less: -------------------------------------------------------------------------------- 1 | // Icon Sizes 2 | // ------------------------- 3 | 4 | /* makes the font 33% larger relative to the icon container */ 5 | .@{fa-css-prefix}-lg { 6 | font-size: (4em / 3); 7 | line-height: (3em / 4); 8 | vertical-align: -15%; 9 | } 10 | .@{fa-css-prefix}-2x { font-size: 2em; } 11 | .@{fa-css-prefix}-3x { font-size: 3em; } 12 | .@{fa-css-prefix}-4x { font-size: 4em; } 13 | .@{fa-css-prefix}-5x { font-size: 5em; } 14 | -------------------------------------------------------------------------------- /website/font-awesome/less/list.less: -------------------------------------------------------------------------------- 1 | // List Icons 2 | // ------------------------- 3 | 4 | .@{fa-css-prefix}-ul { 5 | padding-left: 0; 6 | margin-left: @fa-li-width; 7 | list-style-type: none; 8 | > li { position: relative; } 9 | } 10 | .@{fa-css-prefix}-li { 11 | position: absolute; 12 | left: -@fa-li-width; 13 | width: @fa-li-width; 14 | top: (2em / 14); 15 | text-align: center; 16 | &.@{fa-css-prefix}-lg { 17 | left: (-@fa-li-width + (4em / 14)); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /website/font-awesome/less/mixins.less: -------------------------------------------------------------------------------- 1 | // Mixins 2 | // -------------------------- 3 | 4 | .fa-icon() { 5 | display: inline-block; 6 | font: normal normal normal @fa-font-size-base/1 FontAwesome; // shortening font declaration 7 | font-size: inherit; // can't have font-size inherit on line above, so need to override 8 | text-rendering: auto; // optimizelegibility throws things off #1094 9 | -webkit-font-smoothing: antialiased; 10 | -moz-osx-font-smoothing: grayscale; 11 | transform: translate(0, 0); // ensures no half-pixel rendering in firefox 12 | 13 | } 14 | 15 | .fa-icon-rotate(@degrees, @rotation) { 16 | filter: progid:DXImageTransform.Microsoft.BasicImage(rotation=@rotation); 17 | -webkit-transform: rotate(@degrees); 18 | -ms-transform: rotate(@degrees); 19 | transform: rotate(@degrees); 20 | } 21 | 22 | .fa-icon-flip(@horiz, @vert, @rotation) { 23 | filter: progid:DXImageTransform.Microsoft.BasicImage(rotation=@rotation, mirror=1); 24 | -webkit-transform: scale(@horiz, @vert); 25 | -ms-transform: scale(@horiz, @vert); 26 | transform: scale(@horiz, @vert); 27 | } 28 | -------------------------------------------------------------------------------- /website/font-awesome/less/path.less: -------------------------------------------------------------------------------- 1 | /* FONT PATH 2 | * -------------------------- */ 3 | 4 | @font-face { 5 | font-family: 'FontAwesome'; 6 | src: url('@{fa-font-path}/fontawesome-webfont.eot?v=@{fa-version}'); 7 | src: url('@{fa-font-path}/fontawesome-webfont.eot?#iefix&v=@{fa-version}') format('embedded-opentype'), 8 | url('@{fa-font-path}/fontawesome-webfont.woff2?v=@{fa-version}') format('woff2'), 9 | url('@{fa-font-path}/fontawesome-webfont.woff?v=@{fa-version}') format('woff'), 10 | url('@{fa-font-path}/fontawesome-webfont.ttf?v=@{fa-version}') format('truetype'), 11 | url('@{fa-font-path}/fontawesome-webfont.svg?v=@{fa-version}#fontawesomeregular') format('svg'); 12 | // src: url('@{fa-font-path}/FontAwesome.otf') format('opentype'); // used when developing fonts 13 | font-weight: normal; 14 | font-style: normal; 15 | } 16 | -------------------------------------------------------------------------------- /website/font-awesome/less/rotated-flipped.less: -------------------------------------------------------------------------------- 1 | // Rotated & Flipped Icons 2 | // ------------------------- 3 | 4 | .@{fa-css-prefix}-rotate-90 { .fa-icon-rotate(90deg, 1); } 5 | .@{fa-css-prefix}-rotate-180 { .fa-icon-rotate(180deg, 2); } 6 | .@{fa-css-prefix}-rotate-270 { .fa-icon-rotate(270deg, 3); } 7 | 8 | .@{fa-css-prefix}-flip-horizontal { .fa-icon-flip(-1, 1, 0); } 9 | .@{fa-css-prefix}-flip-vertical { .fa-icon-flip(1, -1, 2); } 10 | 11 | // Hook for IE8-9 12 | // ------------------------- 13 | 14 | :root .@{fa-css-prefix}-rotate-90, 15 | :root .@{fa-css-prefix}-rotate-180, 16 | :root .@{fa-css-prefix}-rotate-270, 17 | :root .@{fa-css-prefix}-flip-horizontal, 18 | :root .@{fa-css-prefix}-flip-vertical { 19 | filter: none; 20 | } 21 | -------------------------------------------------------------------------------- /website/font-awesome/less/stacked.less: -------------------------------------------------------------------------------- 1 | // Stacked Icons 2 | // ------------------------- 3 | 4 | .@{fa-css-prefix}-stack { 5 | position: relative; 6 | display: inline-block; 7 | width: 2em; 8 | height: 2em; 9 | line-height: 2em; 10 | vertical-align: middle; 11 | } 12 | .@{fa-css-prefix}-stack-1x, .@{fa-css-prefix}-stack-2x { 13 | position: absolute; 14 | left: 0; 15 | width: 100%; 16 | text-align: center; 17 | } 18 | .@{fa-css-prefix}-stack-1x { line-height: inherit; } 19 | .@{fa-css-prefix}-stack-2x { font-size: 2em; } 20 | .@{fa-css-prefix}-inverse { color: @fa-inverse; } 21 | -------------------------------------------------------------------------------- /website/font-awesome/scss/_animated.scss: -------------------------------------------------------------------------------- 1 | // Spinning Icons 2 | // -------------------------- 3 | 4 | .#{$fa-css-prefix}-spin { 5 | -webkit-animation: fa-spin 2s infinite linear; 6 | animation: fa-spin 2s infinite linear; 7 | } 8 | 9 | .#{$fa-css-prefix}-pulse { 10 | -webkit-animation: fa-spin 1s infinite steps(8); 11 | animation: fa-spin 1s infinite steps(8); 12 | } 13 | 14 | @-webkit-keyframes fa-spin { 15 | 0% { 16 | -webkit-transform: rotate(0deg); 17 | transform: rotate(0deg); 18 | } 19 | 100% { 20 | -webkit-transform: rotate(359deg); 21 | transform: rotate(359deg); 22 | } 23 | } 24 | 25 | @keyframes fa-spin { 26 | 0% { 27 | -webkit-transform: rotate(0deg); 28 | transform: rotate(0deg); 29 | } 30 | 100% { 31 | -webkit-transform: rotate(359deg); 32 | transform: rotate(359deg); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /website/font-awesome/scss/_bordered-pulled.scss: -------------------------------------------------------------------------------- 1 | // Bordered & Pulled 2 | // ------------------------- 3 | 4 | .#{$fa-css-prefix}-border { 5 | padding: .2em .25em .15em; 6 | border: solid .08em $fa-border-color; 7 | border-radius: .1em; 8 | } 9 | 10 | .pull-right { float: right; } 11 | .pull-left { float: left; } 12 | 13 | .#{$fa-css-prefix} { 14 | &.pull-left { margin-right: .3em; } 15 | &.pull-right { margin-left: .3em; } 16 | } 17 | -------------------------------------------------------------------------------- /website/font-awesome/scss/_core.scss: -------------------------------------------------------------------------------- 1 | // Base Class Definition 2 | // ------------------------- 3 | 4 | .#{$fa-css-prefix} { 5 | display: inline-block; 6 | font: normal normal normal #{$fa-font-size-base}/1 FontAwesome; // shortening font declaration 7 | font-size: inherit; // can't have font-size inherit on line above, so need to override 8 | text-rendering: auto; // optimizelegibility throws things off #1094 9 | -webkit-font-smoothing: antialiased; 10 | -moz-osx-font-smoothing: grayscale; 11 | transform: translate(0, 0); // ensures no half-pixel rendering in firefox 12 | 13 | } 14 | -------------------------------------------------------------------------------- /website/font-awesome/scss/_fixed-width.scss: -------------------------------------------------------------------------------- 1 | // Fixed Width Icons 2 | // ------------------------- 3 | .#{$fa-css-prefix}-fw { 4 | width: (18em / 14); 5 | text-align: center; 6 | } 7 | -------------------------------------------------------------------------------- /website/font-awesome/scss/_larger.scss: -------------------------------------------------------------------------------- 1 | // Icon Sizes 2 | // ------------------------- 3 | 4 | /* makes the font 33% larger relative to the icon container */ 5 | .#{$fa-css-prefix}-lg { 6 | font-size: (4em / 3); 7 | line-height: (3em / 4); 8 | vertical-align: -15%; 9 | } 10 | .#{$fa-css-prefix}-2x { font-size: 2em; } 11 | .#{$fa-css-prefix}-3x { font-size: 3em; } 12 | .#{$fa-css-prefix}-4x { font-size: 4em; } 13 | .#{$fa-css-prefix}-5x { font-size: 5em; } 14 | -------------------------------------------------------------------------------- /website/font-awesome/scss/_list.scss: -------------------------------------------------------------------------------- 1 | // List Icons 2 | // ------------------------- 3 | 4 | .#{$fa-css-prefix}-ul { 5 | padding-left: 0; 6 | margin-left: $fa-li-width; 7 | list-style-type: none; 8 | > li { position: relative; } 9 | } 10 | .#{$fa-css-prefix}-li { 11 | position: absolute; 12 | left: -$fa-li-width; 13 | width: $fa-li-width; 14 | top: (2em / 14); 15 | text-align: center; 16 | &.#{$fa-css-prefix}-lg { 17 | left: -$fa-li-width + (4em / 14); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /website/font-awesome/scss/_mixins.scss: -------------------------------------------------------------------------------- 1 | // Mixins 2 | // -------------------------- 3 | 4 | @mixin fa-icon() { 5 | display: inline-block; 6 | font: normal normal normal #{$fa-font-size-base}/1 FontAwesome; // shortening font declaration 7 | font-size: inherit; // can't have font-size inherit on line above, so need to override 8 | text-rendering: auto; // optimizelegibility throws things off #1094 9 | -webkit-font-smoothing: antialiased; 10 | -moz-osx-font-smoothing: grayscale; 11 | transform: translate(0, 0); // ensures no half-pixel rendering in firefox 12 | 13 | } 14 | 15 | @mixin fa-icon-rotate($degrees, $rotation) { 16 | filter: progid:DXImageTransform.Microsoft.BasicImage(rotation=#{$rotation}); 17 | -webkit-transform: rotate($degrees); 18 | -ms-transform: rotate($degrees); 19 | transform: rotate($degrees); 20 | } 21 | 22 | @mixin fa-icon-flip($horiz, $vert, $rotation) { 23 | filter: progid:DXImageTransform.Microsoft.BasicImage(rotation=#{$rotation}); 24 | -webkit-transform: scale($horiz, $vert); 25 | -ms-transform: scale($horiz, $vert); 26 | transform: scale($horiz, $vert); 27 | } 28 | -------------------------------------------------------------------------------- /website/font-awesome/scss/_path.scss: -------------------------------------------------------------------------------- 1 | /* FONT PATH 2 | * -------------------------- */ 3 | 4 | @font-face { 5 | font-family: 'FontAwesome'; 6 | src: url('#{$fa-font-path}/fontawesome-webfont.eot?v=#{$fa-version}'); 7 | src: url('#{$fa-font-path}/fontawesome-webfont.eot?#iefix&v=#{$fa-version}') format('embedded-opentype'), 8 | url('#{$fa-font-path}/fontawesome-webfont.woff2?v=#{$fa-version}') format('woff2'), 9 | url('#{$fa-font-path}/fontawesome-webfont.woff?v=#{$fa-version}') format('woff'), 10 | url('#{$fa-font-path}/fontawesome-webfont.ttf?v=#{$fa-version}') format('truetype'), 11 | url('#{$fa-font-path}/fontawesome-webfont.svg?v=#{$fa-version}#fontawesomeregular') format('svg'); 12 | // src: url('#{$fa-font-path}/FontAwesome.otf') format('opentype'); // used when developing fonts 13 | font-weight: normal; 14 | font-style: normal; 15 | } 16 | -------------------------------------------------------------------------------- /website/font-awesome/scss/_rotated-flipped.scss: -------------------------------------------------------------------------------- 1 | // Rotated & Flipped Icons 2 | // ------------------------- 3 | 4 | .#{$fa-css-prefix}-rotate-90 { @include fa-icon-rotate(90deg, 1); } 5 | .#{$fa-css-prefix}-rotate-180 { @include fa-icon-rotate(180deg, 2); } 6 | .#{$fa-css-prefix}-rotate-270 { @include fa-icon-rotate(270deg, 3); } 7 | 8 | .#{$fa-css-prefix}-flip-horizontal { @include fa-icon-flip(-1, 1, 0); } 9 | .#{$fa-css-prefix}-flip-vertical { @include fa-icon-flip(1, -1, 2); } 10 | 11 | // Hook for IE8-9 12 | // ------------------------- 13 | 14 | :root .#{$fa-css-prefix}-rotate-90, 15 | :root .#{$fa-css-prefix}-rotate-180, 16 | :root .#{$fa-css-prefix}-rotate-270, 17 | :root .#{$fa-css-prefix}-flip-horizontal, 18 | :root .#{$fa-css-prefix}-flip-vertical { 19 | filter: none; 20 | } 21 | -------------------------------------------------------------------------------- /website/font-awesome/scss/_stacked.scss: -------------------------------------------------------------------------------- 1 | // Stacked Icons 2 | // ------------------------- 3 | 4 | .#{$fa-css-prefix}-stack { 5 | position: relative; 6 | display: inline-block; 7 | width: 2em; 8 | height: 2em; 9 | line-height: 2em; 10 | vertical-align: middle; 11 | } 12 | .#{$fa-css-prefix}-stack-1x, .#{$fa-css-prefix}-stack-2x { 13 | position: absolute; 14 | left: 0; 15 | width: 100%; 16 | text-align: center; 17 | } 18 | .#{$fa-css-prefix}-stack-1x { line-height: inherit; } 19 | .#{$fa-css-prefix}-stack-2x { font-size: 2em; } 20 | .#{$fa-css-prefix}-inverse { color: $fa-inverse; } 21 | -------------------------------------------------------------------------------- /website/font-awesome/scss/font-awesome.scss: -------------------------------------------------------------------------------- 1 | /*! 2 | * Font Awesome 4.3.0 by @davegandy - http://fontawesome.io - @fontawesome 3 | * License - http://fontawesome.io/license (Font: SIL OFL 1.1, CSS: MIT License) 4 | */ 5 | 6 | @import "variables"; 7 | @import "mixins"; 8 | @import "path"; 9 | @import "core"; 10 | @import "larger"; 11 | @import "fixed-width"; 12 | @import "list"; 13 | @import "bordered-pulled"; 14 | @import "animated"; 15 | @import "rotated-flipped"; 16 | @import "stacked"; 17 | @import "icons"; 18 | -------------------------------------------------------------------------------- /website/fonts/glyphicons-halflings-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/website/fonts/glyphicons-halflings-regular.eot -------------------------------------------------------------------------------- /website/fonts/glyphicons-halflings-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/website/fonts/glyphicons-halflings-regular.ttf -------------------------------------------------------------------------------- /website/fonts/glyphicons-halflings-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/website/fonts/glyphicons-halflings-regular.woff -------------------------------------------------------------------------------- /website/fonts/glyphicons-halflings-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/website/fonts/glyphicons-halflings-regular.woff2 -------------------------------------------------------------------------------- /website/img/dask-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/website/img/dask-logo.png -------------------------------------------------------------------------------- /website/img/docker-swarm-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/website/img/docker-swarm-logo.png -------------------------------------------------------------------------------- /website/img/header.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/website/img/header.jpg -------------------------------------------------------------------------------- /website/img/kubernetes-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/website/img/kubernetes-logo.png -------------------------------------------------------------------------------- /website/img/platform-kubernetes-jupyterhub.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/website/img/platform-kubernetes-jupyterhub.png -------------------------------------------------------------------------------- /website/img/platform-kubernetes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/website/img/platform-kubernetes.png -------------------------------------------------------------------------------- /website/img/platform-spark-hdp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/website/img/platform-spark-hdp.png -------------------------------------------------------------------------------- /website/img/platform-spark-yarn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/website/img/platform-spark-yarn.png -------------------------------------------------------------------------------- /website/img/spark-logo-trademark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/website/img/spark-logo-trademark.png -------------------------------------------------------------------------------- /website/img/spectrum-conductor-logo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/website/img/spectrum-conductor-logo.jpg -------------------------------------------------------------------------------- /website/img/spectrum-conductor-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jupyter-server/enterprise_gateway/ffebbaff9184cc48e0c48c59682eac0a2fbf1c45/website/img/spectrum-conductor-logo.png -------------------------------------------------------------------------------- /website/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: home 3 | --- 4 | -------------------------------------------------------------------------------- /website/js/cbpAnimatedHeader.js: -------------------------------------------------------------------------------- 1 | /** 2 | * cbpAnimatedHeader.js v1.0.0 3 | * http://www.codrops.com 4 | * 5 | * Licensed under the MIT license. 6 | * http://www.opensource.org/licenses/mit-license.php 7 | * 8 | * Copyright 2013, Codrops 9 | * http://www.codrops.com 10 | */ 11 | var cbpAnimatedHeader = (function() { 12 | 13 | var docElem = document.documentElement, 14 | header = document.querySelector( '.navbar-default' ), 15 | didScroll = false, 16 | changeHeaderOn = 300; 17 | 18 | function init() { 19 | window.addEventListener( 'scroll', function( event ) { 20 | if( !didScroll ) { 21 | didScroll = true; 22 | setTimeout( scrollPage, 250 ); 23 | } 24 | }, false ); 25 | } 26 | 27 | function scrollPage() { 28 | var sy = scrollY(); 29 | if ( sy >= changeHeaderOn ) { 30 | classie.add( header, 'navbar-shrink' ); 31 | } 32 | else { 33 | classie.remove( header, 'navbar-shrink' ); 34 | } 35 | didScroll = false; 36 | } 37 | 38 | function scrollY() { 39 | return window.pageYOffset || docElem.scrollTop; 40 | } 41 | 42 | init(); 43 | 44 | })(); 45 | -------------------------------------------------------------------------------- /website/js/classie.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * classie - class helper functions 3 | * from bonzo https://github.com/ded/bonzo 4 | * 5 | * classie.has( elem, 'my-class' ) -> true/false 6 | * classie.add( elem, 'my-new-class' ) 7 | * classie.remove( elem, 'my-unwanted-class' ) 8 | * classie.toggle( elem, 'my-class' ) 9 | */ 10 | 11 | /*jshint browser: true, strict: true, undef: true */ 12 | /*global define: false */ 13 | 14 | ( function( window ) { 15 | 16 | 'use strict'; 17 | 18 | // class helper functions from bonzo https://github.com/ded/bonzo 19 | 20 | function classReg( className ) { 21 | return new RegExp("(^|\\s+)" + className + "(\\s+|$)"); 22 | } 23 | 24 | // classList support for class management 25 | // altho to be fair, the api sucks because it won't accept multiple classes at once 26 | var hasClass, addClass, removeClass; 27 | 28 | if ( 'classList' in document.documentElement ) { 29 | hasClass = function( elem, c ) { 30 | return elem.classList.contains( c ); 31 | }; 32 | addClass = function( elem, c ) { 33 | elem.classList.add( c ); 34 | }; 35 | removeClass = function( elem, c ) { 36 | elem.classList.remove( c ); 37 | }; 38 | } 39 | else { 40 | hasClass = function( elem, c ) { 41 | return classReg( c ).test( elem.className ); 42 | }; 43 | addClass = function( elem, c ) { 44 | if ( !hasClass( elem, c ) ) { 45 | elem.className = elem.className + ' ' + c; 46 | } 47 | }; 48 | removeClass = function( elem, c ) { 49 | elem.className = elem.className.replace( classReg( c ), ' ' ); 50 | }; 51 | } 52 | 53 | function toggleClass( elem, c ) { 54 | var fn = hasClass( elem, c ) ? removeClass : addClass; 55 | fn( elem, c ); 56 | } 57 | 58 | var classie = { 59 | // full names 60 | hasClass: hasClass, 61 | addClass: addClass, 62 | removeClass: removeClass, 63 | toggleClass: toggleClass, 64 | // short names 65 | has: hasClass, 66 | add: addClass, 67 | remove: removeClass, 68 | toggle: toggleClass 69 | }; 70 | 71 | // transport 72 | if ( typeof define === 'function' && define.amd ) { 73 | // AMD 74 | define( classie ); 75 | } else { 76 | // browser global 77 | window.classie = classie; 78 | } 79 | 80 | })( window ); 81 | -------------------------------------------------------------------------------- /website/js/creative.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * Start Bootstrap - Creative Bootstrap Theme (http://startbootstrap.com) 3 | * Code licensed under the Apache License v2.0. 4 | * For details, see http://www.apache.org/licenses/LICENSE-2.0. 5 | */ 6 | 7 | (function($) { 8 | "use strict"; // Start of use strict 9 | 10 | // jQuery for page scrolling feature - requires jQuery Easing plugin 11 | $('a.page-scroll').bind('click', function(event) { 12 | var $anchor = $(this); 13 | $('html, body').stop().animate({ 14 | scrollTop: ($($anchor.attr('href')).offset().top - 50) 15 | }, 1250, 'easeInOutExpo'); 16 | event.preventDefault(); 17 | }); 18 | 19 | // Highlight the top nav as scrolling occurs 20 | $('body').scrollspy({ 21 | target: '.navbar-fixed-top', 22 | offset: 51 23 | }) 24 | 25 | // Closes the Responsive Menu on Menu Item Click 26 | $('.navbar-collapse ul li a').click(function() { 27 | $('.navbar-toggle:visible').click(); 28 | }); 29 | 30 | // Fit Text Plugin for Main Header 31 | $("h1").fitText( 32 | 1.2, { 33 | minFontSize: '35px', 34 | maxFontSize: '65px' 35 | } 36 | ); 37 | 38 | // Offset for Main Navigation 39 | $('#mainNav').affix({ 40 | offset: { 41 | top: 100 42 | } 43 | }) 44 | 45 | // Initialize WOW.js Scrolling Animations 46 | new WOW().init(); 47 | 48 | })(jQuery); // End of use strict 49 | -------------------------------------------------------------------------------- /website/js/jquery.fittext.js: -------------------------------------------------------------------------------- 1 | /*global jQuery */ 2 | /*! 3 | * FitText.js 1.2 4 | * 5 | * Copyright 2011, Dave Rupert http://daverupert.com 6 | * Released under the WTFPL license 7 | * http://sam.zoy.org/wtfpl/ 8 | * 9 | * Date: Thu May 05 14:23:00 2011 -0600 10 | */ 11 | 12 | (function( $ ){ 13 | 14 | $.fn.fitText = function( kompressor, options ) { 15 | 16 | // Setup options 17 | var compressor = kompressor || 1, 18 | settings = $.extend({ 19 | 'minFontSize' : Number.NEGATIVE_INFINITY, 20 | 'maxFontSize' : Number.POSITIVE_INFINITY 21 | }, options); 22 | 23 | return this.each(function(){ 24 | 25 | // Store the object 26 | var $this = $(this); 27 | 28 | // Resizer() resizes items based on the object width divided by the compressor * 10 29 | var resizer = function () { 30 | $this.css('font-size', Math.max(Math.min($this.width() / (compressor*10), parseFloat(settings.maxFontSize)), parseFloat(settings.minFontSize))); 31 | }; 32 | 33 | // Call once to set. 34 | resizer(); 35 | 36 | // Call on resize. Opera debounces their resize by default. 37 | $(window).on('resize.fittext orientationchange.fittext', resizer); 38 | 39 | }); 40 | 41 | }; 42 | 43 | })( jQuery ); 44 | -------------------------------------------------------------------------------- /website/platform-kubernetes.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: Jupyter Enterprise Gateway and Kubernetes 4 | --- 5 | 6 | Recently, we have experienced various advances in AI, in particular around Deep Learning. This have 7 | increased the popularity of Deep Learning use cases and also the proliferation of several development 8 | frameworks that have different runtime and deployment requirements. Containers provides a very flexible 9 | way to build such heterogenous environments and Kubernetes provides an easy way to deploy and manage such 10 | deployments with the benefit of elasticity and other quality of services. 11 | 12 | Jupyter Enterprise Gateway extends the Jupyter Notebook platform and enables Jupyter Notebook 13 | kernels to run as independent pods in a Kubernetes cluster, providing the necessary environment 14 | isolation to support the development and training of Deep Learning models. 15 | 16 | Using the Kubernetes support in Jupyter Enterprise Gateway, the container image where the kernel will be 17 | launched becomes a choice, where you can easily start a Python kernel on a TensorFlow community image 18 | to enable working on TensorFlow models, or you can have a Python kernel on a PyTorch community image. 19 | 20 | Kubernetes also gives the ability to associate/share specialized hardwares such as GPUs and TPUs 21 | to the kernel pod, providing necessary power for training Deep Learning models. 22 | 23 |
24 | 25 |
26 | 27 |
28 | 29 |
30 | 31 | ### Deployment 32 | 33 |
34 | 35 | Jupyter Enterprise Gateway can easily be deployed into your Kubernetes cluster: 36 | 37 |
38 | 
kubectl apply -f https://raw.githubusercontent.com/jupyter/enterprise_gateway/main/etc/kubernetes/enterprise-gateway.yaml
39 |
40 | 41 | #### Deployment Scripts 42 | 43 |
44 | 45 | The Jupyter Enterprise Gateway development team uses some Ansible scripts for provisioning 46 | test environments, these scripts might be useful for users trying to get started with the gateway 47 | on a Kubernetes environment. 48 | 49 | - Ansible Deployment scripts : ansible-kubernetes-cluster 50 | -------------------------------------------------------------------------------- /website/platform-spark.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: Jupyter Enterprise Gateway and Apache Spark 4 | --- 5 | 6 | The Big Data Analytics use cases require processing large data sets which are not containable by 7 | the resources available on a single machine. 8 | 9 | Jupyter Enterprise Gateway extends the Jupyter Notebook platform and enables Jupyter Notebook 10 | kernels to run as Apache Spark applications in YARN cluster mode. 11 | 12 | By leveraging the functionality 13 | of the underlying resource management applications like Hadoop YARN, etc., Jupyter Enterprise Gateway 14 | distributes kernels across the compute cluster, dramatically increasing the number of simultaneously 15 | active notebooks/kernels. 16 | 17 |
18 | 19 | ### Deployment 20 | 21 |
22 | 23 | Jupyter Enterprise Gateway can easily be incorporated into your Analytics Platform. 24 | 25 | If you are using a distribution like HDP, the gateway can be installed in an edge node 26 | (optionally secured by Knox). Jupyter Notebooks can then connect via the gateway and 27 | have access to run the Notebook kernels in the Spark/YARN nodes. If you have Kerberos 28 | security enable on the cluster, then each notebook kernel will be running as the userid 29 | from the users that requested the notebook kernel, thus leveraging all configured ACLs 30 | when accessing HDFS, and other secured resources. 31 | 32 |
33 | 34 |
35 | 36 | Note that the use of a distribution is not a requirement, and we also support running 37 | Jupyter Enterprise Gateway in a vanilla deployment of Spark and YARN. 38 | 39 |
40 | 41 |
42 | 43 | #### Deployment Scripts 44 | 45 |
46 | 47 | The Jupyter Enterprise Gateway development team uses some Ansible scripts for provisioning 48 | test environments, these scripts might be useful for users trying to get started with the gateway. 49 | 50 | - Ansible Deployment scripts : ansible-spark-cluster 51 | -------------------------------------------------------------------------------- /website/privacy-policy.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: Jupyter Enterprise Gateway Privacy Policy 4 | --- 5 | 6 | ## Jupyter Enterprise Gateway Privacy Policy 7 | 8 | Information about your use of this website is collected using server access logs and a tracking cookie. 9 | The collected information consists of the following: 10 | 11 | - The IP address from which you access the website; 12 | - The type of browser and operating system you use to access our site; 13 | - The date and time you access our site; 14 | - The pages you visit; and 15 | - The addresses of pages from where you followed a link to our site. 16 | 17 | Part of this information is gathered using a tracking cookie set by the [Google Analytics](https://www.google.com/analytics/) 18 | service and handled by Google as described in their [privacy policy](https://www.google.com/privacy.html). 19 | See your browser documentation for instructions on how to disable the cookie if you prefer not to share this data with Google. 20 | 21 | We use the gathered information to help us make our site more useful to visitors and to better understand how and when our site is used. We do not track or collect personally identifiable information or associate gathered data with any personally identifying information from other sources. 22 | 23 | By using this website, you consent to the collection of this data in the manner and for the purpose described above. 24 | -------------------------------------------------------------------------------- /website/publish.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright (c) Jupyter Development Team. 4 | # Distributed under the terms of the Modified BSD License. 5 | 6 | 7 | set -e 8 | 9 | BASE_DIR=$(pwd) 10 | WORK_DIR=$(pwd)/build 11 | SOURCE_DIR=$(pwd)/build/enterprise_gateway/website 12 | HTML_DIR=$(pwd)/build/website 13 | 14 | 15 | echo " " 16 | echo "-------------------------------------------------------------" 17 | echo "------- Build and publish project website -------" 18 | echo "-------------------------------------------------------------" 19 | echo " " 20 | echo "Base directory: $BASE_DIR" 21 | echo "Work directory: $WORK_DIR" 22 | echo "Source directory: $SOURCE_DIR" 23 | echo "HTML directory: $HTML_DIR" 24 | echo " " 25 | 26 | set -o xtrace 27 | 28 | 29 | function checkout_code { 30 | rm -rf $WORK_DIR 31 | mkdir -p $WORK_DIR 32 | cd $WORK_DIR 33 | # Checkout code 34 | git clone git@github.com:jupyter/enterprise_gateway.git 35 | cd enterprise_gateway 36 | git_hash=`git rev-parse --short HEAD` 37 | echo "Checked out Jupyter Enterprise Gateway git hash $git_hash" 38 | } 39 | 40 | function build_website { 41 | rm -rf $HTML_DIR 42 | mkdir -p $HTML_DIR 43 | cd $SOURCE_DIR 44 | jekyll clean 45 | jekyll build -d $HTML_DIR 46 | } 47 | 48 | function publish_website { 49 | cd $WORK_DIR/enterprise_gateway 50 | git checkout gh-pages 51 | git branch --set-upstream-to=origin/gh-pages gh-pages 52 | git pull --rebase 53 | rm -rf * 54 | git checkout .gitignore 55 | git checkout README.md 56 | cp -r $HTML_DIR/ $WORK_DIR/enterprise_gateway 57 | git add * 58 | git commit -a -m"Publishing website using commit $git_hash" 59 | echo "Publishing website using commit $git_hash" 60 | git push origin gh-pages 61 | } 62 | 63 | echo "Preparing to publish website..." 64 | 65 | checkout_code 66 | 67 | build_website 68 | 69 | publish_website 70 | 71 | 72 | echo "Website published..." 73 | echo " https://jupyter.org/enterprise_gateway/" 74 | echo " https://jupyter.github.io/enterprise_gateway/" 75 | 76 | 77 | cd "$BASE_DIR" #return to base dir 78 | --------------------------------------------------------------------------------