├── .github └── workflows │ └── python-lint.yaml ├── .gitignore ├── LICENSE ├── README.md ├── dev-requirements.txt ├── docs ├── Makefile ├── conf.py ├── doc-requirements.txt ├── howto │ ├── hubploy-build-jupyterhub-image.rst │ ├── hubploy-deploy-jupyterhub-repo-setup.rst │ ├── hubploy-setup-dev-environment.rst │ └── index.rst ├── index.rst ├── reference │ ├── contribution-guide.rst │ ├── index.rst │ └── reference-hubploy-configuration-values.rst └── topics │ ├── index.rst │ ├── topic-directory-structure.rst │ ├── topic-helm-versions.rst │ └── topic-values-yaml-overriding.rst ├── hubploy ├── __init__.py ├── __main__.py ├── auth.py ├── config.py └── helm.py └── setup.py /.github/workflows/python-lint.yaml: -------------------------------------------------------------------------------- 1 | name: "python lint" 2 | on: 3 | - pull_request 4 | 5 | jobs: 6 | lint: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v4 10 | 11 | - name: Install ruff 12 | run: pip install ruff==0.6.9 13 | 14 | - name: Lint python files 15 | run: ruff check . 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | # Terraform modules 107 | .terraform 108 | *.tfstate 109 | *.tfstate.backup 110 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2018, Yuvi Panda 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # hubploy 2 | 3 | Toolkit to deploy many z2jh based JupyterHubs 4 | 5 | Usage: 6 | 7 | ``` 8 | hubploy deploy 9 | ``` 10 | 11 | Help text: 12 | 13 | ``` 14 | $ hubploy --help 15 | usage: hubploy [-h] [-d] [-D] [-v] {deploy} ... 16 | 17 | positional arguments: 18 | {deploy} 19 | deploy Deploy a chart to the given environment. 20 | 21 | options: 22 | -h, --help show this help message and exit 23 | -d, --debug Enable tool debug output (not including helm debug). 24 | -D, --helm-debug Enable Helm debug output. This is not allowed to be used in a CI environment due to secrets being displayed in plain text, and the script will exit. To enable this option, set a local environment varible HUBPLOY_LOCAL_DEBUG=true 25 | -v, --verbose Enable verbose output. 26 | ``` 27 | 28 | Deploy help: 29 | 30 | ``` 31 | hubploy deploy --help 32 | usage: hubploy deploy [-h] [--namespace NAMESPACE] [--set SET] [--set-string SET_STRING] [--version VERSION] [--timeout TIMEOUT] [--force] [--atomic] 33 | [--cleanup-on-fail] [--dry-run] [--image-overrides IMAGE_OVERRIDES [IMAGE_OVERRIDES ...]] 34 | deployment chart {develop,staging,prod} 35 | 36 | positional arguments: 37 | deployment The name of the hub to deploy. 38 | chart The path to the main hub chart. 39 | {develop,staging,prod} 40 | The environment to deploy to. 41 | 42 | options: 43 | -h, --help show this help message and exit 44 | --namespace NAMESPACE 45 | Helm option: the namespace to deploy to. If not specified, the namespace will be derived from the environment argument. 46 | --set SET Helm option: set values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2) 47 | --set-string SET_STRING 48 | Helm option: set STRING values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2) 49 | --version VERSION Helm option: specify a version constraint for the chart version to use. This constraint can be a specific tag (e.g. 1.1.1) or it may reference a 50 | valid range (e.g. ^2.0.0). If this is not specified, the latest version is used. 51 | --timeout TIMEOUT Helm option: time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks, etc). Defaults to 300 seconds. 52 | --force Helm option: force resource updates through a replacement strategy. 53 | --atomic Helm option: if set, upgrade process rolls back changes made in case of failed upgrade. The --wait flag will be set automatically if --atomic is 54 | used. 55 | --cleanup-on-fail Helm option: allow deletion of new resources created in this upgrade when upgrade fails. 56 | --dry-run Dry run the helm upgrade command. This also renders the chart to STDOUT. This is not allowed to be used in a CI environment due to secrets being 57 | displayed in plain text, and the script will exit. To enable this option, set a local environment varible HUBPLOY_LOCAL_DEBUG=true 58 | --image-overrides IMAGE_OVERRIDES [IMAGE_OVERRIDES ...] 59 | Override one or more images and tags to deploy. Format is: : : ... IMPORTANT: 60 | The order of images passed in must match the order in which they appear in hubploy.yaml and separated by spaces without quotes. You must always 61 | specify a tag when overriding images. 62 | ``` -------------------------------------------------------------------------------- /dev-requirements.txt: -------------------------------------------------------------------------------- 1 | pytest 2 | pytest-cov 3 | codecov 4 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = hubploy 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Configuration file for the Sphinx documentation builder. 4 | # 5 | # This file does only contain a selection of the most common options. For a 6 | # full list see the documentation: 7 | # http://www.sphinx-doc.org/en/master/config 8 | 9 | # -- Path setup -------------------------------------------------------------- 10 | 11 | # If extensions (or modules to document with autodoc) are in another directory, 12 | # add these directories to sys.path here. If the directory is relative to the 13 | # documentation root, use os.path.abspath to make it absolute, like shown here. 14 | # 15 | # import os 16 | # import sys 17 | # sys.path.insert(0, os.path.abspath('.')) 18 | 19 | 20 | # -- Project information ----------------------------------------------------- 21 | 22 | project = "hubploy" 23 | copyright = "2018, Yuvi Panda" 24 | author = "Yuvi Panda" 25 | 26 | # The short X.Y version 27 | version = "" 28 | # The full version, including alpha/beta/rc tags 29 | release = "0.1" 30 | 31 | 32 | # -- General configuration --------------------------------------------------- 33 | 34 | # If your documentation needs a minimal Sphinx version, state it here. 35 | # 36 | # needs_sphinx = '1.0' 37 | 38 | # Add any Sphinx extension module names here, as strings. They can be 39 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 40 | # ones. 41 | extensions = ["sphinx.ext.intersphinx", "sphinxcontrib.mermaid"] 42 | 43 | # Add any paths that contain templates here, relative to this directory. 44 | templates_path = ["_templates"] 45 | 46 | # The suffix(es) of source filenames. 47 | # You can specify multiple suffix as a list of string: 48 | # 49 | # source_suffix = ['.rst', '.md'] 50 | source_suffix = ".rst" 51 | 52 | # The master toctree document. 53 | master_doc = "index" 54 | 55 | # The language for content autogenerated by Sphinx. Refer to documentation 56 | # for a list of supported languages. 57 | # 58 | # This is also used if you do content translation via gettext catalogs. 59 | # Usually you set "language" from the command line for these cases. 60 | language = None 61 | 62 | # List of patterns, relative to source directory, that match files and 63 | # directories to ignore when looking for source files. 64 | # This pattern also affects html_static_path and html_extra_path . 65 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] 66 | 67 | # The name of the Pygments (syntax highlighting) style to use. 68 | pygments_style = "sphinx" 69 | 70 | 71 | # -- Options for HTML output ------------------------------------------------- 72 | 73 | # The theme to use for HTML and HTML Help pages. See the documentation for 74 | # a list of builtin themes. 75 | # 76 | html_theme = "alabaster" 77 | 78 | # Theme options are theme-specific and customize the look and feel of a theme 79 | # further. For a list of options available for each theme, see the 80 | # documentation. 81 | # 82 | # html_theme_options = {} 83 | 84 | # Add any paths that contain custom static files (such as style sheets) here, 85 | # relative to this directory. They are copied after the builtin static files, 86 | # so a file named "default.css" will overwrite the builtin "default.css". 87 | html_static_path = ["_static"] 88 | 89 | # Custom sidebar templates, must be a dictionary that maps document names 90 | # to template names. 91 | # 92 | # The default sidebars (for documents that don't match any pattern) are 93 | # defined by theme itself. Builtin themes are using these templates by 94 | # default: ``['localtoc.html', 'relations.html', 'sourcelink.html', 95 | # 'searchbox.html']``. 96 | # 97 | # html_sidebars = {} 98 | 99 | 100 | # -- Options for HTMLHelp output --------------------------------------------- 101 | 102 | # Output file base name for HTML help builder. 103 | htmlhelp_basename = "hubploydoc" 104 | 105 | 106 | # -- Options for LaTeX output ------------------------------------------------ 107 | 108 | latex_elements = { 109 | # The paper size ('letterpaper' or 'a4paper'). 110 | # 111 | # 'papersize': 'letterpaper', 112 | # The font size ('10pt', '11pt' or '12pt'). 113 | # 114 | # 'pointsize': '10pt', 115 | # Additional stuff for the LaTeX preamble. 116 | # 117 | # 'preamble': '', 118 | # Latex figure (float) alignment 119 | # 120 | # 'figure_align': 'htbp', 121 | } 122 | 123 | # Grouping the document tree into LaTeX files. List of tuples 124 | # (source start file, target name, title, 125 | # author, documentclass [howto, manual, or own class]). 126 | latex_documents = [ 127 | (master_doc, "hubploy.tex", "hubploy Documentation", "Yuvi Panda", "manual"), 128 | ] 129 | 130 | 131 | # -- Options for manual page output ------------------------------------------ 132 | 133 | # One entry per manual page. List of tuples 134 | # (source start file, name, description, authors, manual section). 135 | man_pages = [(master_doc, "hubploy", "hubploy Documentation", [author], 1)] 136 | 137 | 138 | # -- Options for Texinfo output ---------------------------------------------- 139 | 140 | # Grouping the document tree into Texinfo files. List of tuples 141 | # (source start file, target name, title, author, 142 | # dir menu entry, description, category) 143 | texinfo_documents = [ 144 | ( 145 | master_doc, 146 | "hubploy", 147 | "hubploy Documentation", 148 | author, 149 | "hubploy", 150 | "One line description of project.", 151 | "Miscellaneous", 152 | ), 153 | ] 154 | 155 | 156 | # -- Extension configuration ------------------------------------------------- 157 | 158 | # -- Options for intersphinx extension --------------------------------------- 159 | 160 | # Example configuration for intersphinx: refer to the Python standard library. 161 | intersphinx_mapping = {"https://docs.python.org/": None} 162 | 163 | 164 | mermaid_params = ["--theme", "neutral"] 165 | -------------------------------------------------------------------------------- /docs/doc-requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx 2 | sphinxcontrib-mermaid 3 | -------------------------------------------------------------------------------- /docs/howto/hubploy-build-jupyterhub-image.rst: -------------------------------------------------------------------------------- 1 | =============================== 2 | How to Build a JupyterHub Image 3 | =============================== 4 | -------------------------------------------------------------------------------- /docs/howto/hubploy-deploy-jupyterhub-repo-setup.rst: -------------------------------------------------------------------------------- 1 | ============================================================= 2 | How to Setup a Repository to Deploy a JupyterHub with Hubploy 3 | ============================================================= 4 | 5 | This is a guide on how to deploy a JupyterHub with Hubploy. 6 | 7 | General Procedure: 8 | 9 | * `Step 0: Setup Prerequisites`_ 10 | * `Step 1: Get the hubploy-template Repository`_ 11 | * `Step 2: Install Hubploy`_ 12 | * `Step 3: Configure the Hub`_ 13 | * `Step 4: Build and Push the Image`_ 14 | * `Step 5: Deploy the Staging Hub`_ 15 | * `Step 6: Deploy the Production Hub`_ 16 | * `Step 7: Setup git-crypt for Secrets`_ 17 | * `Step 8: GitHub Workflows`_ 18 | 19 | 20 | Step 0: Setup Prerequisites 21 | =========================== 22 | 23 | Hubploy does not manage your cloud resources - only your *Kubernetes* resources. You should use 24 | some other means to create your cloud resources. Example infrastructure deployments can be found 25 | at the `terraform-deploy repository `_. At a 26 | minimum, Hubloy expects a Kubernetes cluster. Many installations want to use a shared file system 27 | for home directories, so in those cases you want to hvae that managed outside Hubploy as well. 28 | 29 | You also need the following tools installed: 30 | 31 | #. Your cloud vendor's commandline tool. 32 | 33 | * `Google Cloud SDK `_ for Google Cloud 34 | * `AWS CLI `_ for AWS 35 | * `Azure CLI `_ for Azure 36 | 37 | #. A local install of `helm 3 `_. Helm 2 is also supported, 38 | but requires the same version of Helm to be present locally and on the cluster. If you are sing 39 | Helm 2, you can find both versions with ``helm version``. 40 | 41 | #. A `docker environment `_ that you can use. This is only 42 | needed when building images. 43 | 44 | 45 | Step 1: Get the ``hubploy-template`` Repository 46 | ================================================= 47 | 48 | There are a couple different options for acquiring the content in `this repository`_. 49 | 50 | * Use the repository as a template. Click the "Use this template" button on the GitHub 51 | repository's page, then input your own repo name. You can then use ``git clone`` as normal to 52 | get your repository onto your local machine. 53 | 54 | * Fork the repository. 55 | 56 | * Clone it directly with ``git clone https://github.com/yuvipanda/hubploy-template.git``. The 57 | disadvantage here is that you probably won't have permissions to push changes and will have to 58 | only develop locally. Not recommended. 59 | 60 | 61 | Step 2: Install Hubploy 62 | ======================= 63 | 64 | .. code:: bash 65 | 66 | python3 -m venv . 67 | source bin/activate 68 | python3 -m pip install -r requirements.txt 69 | 70 | This installs hubploy and its dependencies. 71 | 72 | 73 | Step 3: Configure the Hub 74 | ========================= 75 | 76 | Rename the Hub 77 | -------------- 78 | 79 | Each directory inside ``deployments/`` represents an installation of JupyterHub. The default is 80 | called ``myhub``, but *please* rename it to something more descriptive. ``git commit`` the result 81 | as well. 82 | 83 | .. code:: bash 84 | 85 | git mv deployments/myhub deployments/ 86 | git commit 87 | 88 | 89 | Fill in the Minimum Config Details 90 | ---------------------------------- 91 | 92 | You need to find all things marked TODO and fill them in. In particular, 93 | 94 | #. ``hubploy.yaml`` needs information about where your docker registry & kubernetes cluster is, 95 | and paths to access keys as well. These access key files should be in the deployment's 96 | ``secret/`` folder. 97 | #. ``secrets/prod.yaml`` and ``secrets/staging.yaml`` require secure random keys you can generate 98 | and fill in. 99 | 100 | If you are deploying onto AWS infrastructure, your access key file should look like the aws 101 | credentials file (usually found at ``~/.aws/credentials``). However, the profile you use *must* 102 | be named ``default``. 103 | 104 | If you want to try deploying to staging now, that is fine! Hub Customization can come later as you 105 | try things out. 106 | 107 | 108 | Hub Customizations 109 | ------------------ 110 | 111 | You can customize your hub in two major ways: 112 | 113 | #. Customize the hub image. `repo2docker`_ is used to build the image, so you can put any of the 114 | `supported configuration files`_ under ``deployments//image``. You *must* make a git 115 | commit after modifying this for ``hubploy build --push --check-registry`` to work, 116 | since it uses the commit hash as the image tag. 117 | 118 | #. Customize hub configuration with various YAML files. 119 | 120 | * ``hub/values.yaml`` is common to *all* hubs that exist in this repo (multiple hubs can live 121 | under ``deployments/``). 122 | 123 | * ``deployments//config/common.yaml`` is where most of the config specific to each 124 | hub should go. Examples include memory / cpu limits, home directory definitions, etc 125 | 126 | * ``deployments//config/staging.yaml`` and 127 | ``deployments//config/prod.yaml`` 128 | are files specific to the staging & prod versions of the hub. These should be *as minimal as 129 | possible*. Ideally, only DNS entries, IP addresses, should be here. 130 | 131 | * ``deployments//secrets/staging.yaml`` and 132 | ``deployments//secrets/prod.yaml`` 133 | should contain information that mustn't be public. This would be proxy / hub secret 134 | tokens, any authentication tokens you have, etc. These files *must* be protected by something 135 | like `git-crypt `_ or 136 | `sops `_. 137 | 138 | 139 | You can customize the staging hub, deploy it with ``hubploy deploy hub staging``, and 140 | iterate until you like how it behaves. 141 | 142 | 143 | Step 4: Build and Push the Image 144 | ================================ 145 | 146 | #. Make sure tha appropriate docker credential helper is installed, so hubploy can push to the 147 | registry you need. 148 | 149 | For AWS, you need `docker-ecr-credential-helper `_ 151 | For Google Cloud, you need the `gcloud commandline tool `_ 152 | 153 | #. Make sure you are in your repo's root directory, so hubploy can find the directory structure it 154 | expects. 155 | 156 | #. Build and push the image to the registry 157 | 158 | .. code:: bash 159 | 160 | hubploy build --push --check-registry 161 | 162 | This should check if the user image for your hub needs to be rebuilt, and if so, it’ll build 163 | and push it. 164 | 165 | 166 | Step 5: Deploy the Staging Hub 167 | ============================== 168 | 169 | Each hub will always have two versions - a *staging* hub that isn’t used by actual users, and a * 170 | production* hub that is. These two should be kept as similar as possible, so you can fearlessly 171 | test stuff on the staging hub without feaer that it is going to crash & burn when deployed to 172 | production. 173 | 174 | To deploy to the staging hub, 175 | 176 | .. code:: bash 177 | 178 | hubploy deploy hub staging 179 | 180 | This should take a while, but eventually return successfully. You can then find the public IP of 181 | your hub with: 182 | 183 | .. code:: bash 184 | 185 | kubectl -n -staging get svc public-proxy 186 | 187 | If you access that, you should be able to get in with any username & password. 188 | 189 | The defaults provision each user their own EBS / Persistent Disk, so this can get expensive 190 | quickly :) Watch out! 191 | 192 | If you didn't do more `Hub Customizations`_, you can do so now! 193 | 194 | 195 | Step 6: Deploy the Production Hub 196 | ================================= 197 | 198 | You can then do a production deployment with: ``hubploy deploy hub prod``, and test it 199 | out! 200 | 201 | 202 | Step 7: Setup git-crypt for Secrets 203 | =================================== 204 | 205 | `git crypt `_ is used to keep encrypted secrets in the git 206 | repository. We would eventually like to use something like 207 | `sops `_ 208 | but for now... 209 | 210 | #. Install git-crypt. You can get it from brew or your package manager. 211 | 212 | #. In your repo, initialize it. 213 | 214 | .. code:: bash 215 | 216 | git crypt init 217 | 218 | #. In ``.gitattributes`` have the following contents: 219 | 220 | .. code:: 221 | 222 | deployments/*/secrets/** filter=git-crypt diff=git-crypt 223 | deployments/**/secrets/** filter=git-crypt diff=git-crypt 224 | support/secrets.yaml filter=git-crypt diff=git-crypt 225 | 226 | #. Make a copy of your encryption key. This will be used to decrypt the secrets. You will need to 227 | share it with your CD provider, and anyone else. 228 | 229 | .. code:: 230 | 231 | git crypt export-key key 232 | 233 | This puts the key in a file called 'key' 234 | 235 | 236 | Step 8: GitHub Workflows 237 | ======================== 238 | 239 | #. Get a base64 copy of your key 240 | 241 | .. code:: bash 242 | 243 | cat key | base64 244 | 245 | #. Put it as a secret named GIT_CRYPT_KEY in github secrets. 246 | 247 | #. Make sure you change the `myhub` to your deployment name in the 248 | workflows under `.github/workflows`. 249 | 250 | #. Push to the staging branch, and check out GitHub actions, to 251 | see if your action goes to completion. 252 | 253 | #. If the staging action succeeds, make a PR from staging to prod, 254 | and merge this PR. This should also trigger an action - see if 255 | this works out. 256 | 257 | **Note**: *Always* make a PR from staging to prod, never push directly to prod. We want to keep 258 | the staging and prod branches as close to each other as possible, and this is the only long term 259 | guaranteed way to do that. 260 | 261 | .. _this repository: https://github.com/yuvipanda/hubploy-template 262 | .. _repo2docker: https://repo2docker.readthedocs.io/ 263 | .. _supported configuration files: https://repo2docker.readthedocs.io/en/latest/config_files.html 264 | -------------------------------------------------------------------------------- /docs/howto/hubploy-setup-dev-environment.rst: -------------------------------------------------------------------------------- 1 | ============================================== 2 | How to Setup a Hubploy Development Environment 3 | ============================================== 4 | 5 | This is a guide on how to setup a development environment for Hubploy. Use cases would be for 6 | making a custom Hubploy image for your own use or contributing to the Hubploy repository. 7 | 8 | * `Prerequisites`_ 9 | * `Modifying Hubploy Files`_ 10 | * `Using a Custom Hubploy Locally`_ 11 | * `Building a Custom Hubploy on DockerHub`_ 12 | * `Contributing to Hubploy`_ 13 | 14 | Prerequisites 15 | =========================== 16 | 17 | To start, fork the `main Hubploy repository `_ 18 | and then clone your fork. This will enable easier setup for pull requests and 19 | independent development. Methodology for testing Hubploy is limited right now but it is 20 | recommendation that you have a working JupyterHub configuration so you can try to 21 | build and deploy. 22 | 23 | If you don't have such a configuration set up, we recommend setting one up using the 24 | `hubploy template repository `_ and following the 25 | How-To guide 26 | `Deploying a JupyterHub with Hubploy `_. 27 | 28 | 29 | Modifying Hubploy Files 30 | ======================= 31 | 32 | The code for Hubploy is contained in the ``hubploy/hubploy`` folder. All of it is in Python, so 33 | there is no compiling necessary to use it locally. As long as the files are saved, their changes 34 | should be reflected the next time you run a ``hubploy`` command. 35 | 36 | 37 | Using a Custom Hubploy Locally 38 | ============================== 39 | 40 | Hubploy can be installed via ``pip install hubploy``, but this version is very out-of-date. 41 | Using a custom version of Hubploy will require different installation methods. 42 | 43 | If you are just using your custom Hubploy locally, you can link it with ``pip``. Go to the top 44 | folder of your ``hubploy-template`` or JupyterHub deployment repo and run:: 45 | 46 | pip install -e ~//hubploy 47 | 48 | You can then make changes to your local Hubploy files and rerun Hubploy commands in the other 49 | folder for quick development. 50 | 51 | `hubploy` can also be installed at any specific commit with the following line in a 52 | `requirements.txt` file: 53 | :: 54 | 55 | git+https://github.com/yuvipanda/hubploy@ 56 | 57 | 58 | Building a Custom Hubploy on DockerHub 59 | ====================================== 60 | 61 | Another way to use Hubploy is by building a Docker image and pushing it to DockerHub. For this, 62 | you will need to have forked the Hubploy repository to your personal GitHub account. You will also 63 | need a personal DockerHub account. 64 | 65 | Modify the file ``hubploy/.github/workflows/docker-push.yaml``. Change ``name: yuvipanda/hubploy`` 66 | to ``name: /hubploy``. You will need to input your DockerHub credentials as 67 | secrets in your personal Hubploy GitHub repository as ``DOCKER_USERNAME`` and ``DOCKER_PASSWORD``. 68 | Also in the GitHub repository, go to the Actions tab and allow the repo to run workflows by 69 | clicking "I understand my workflows, go ahead and run them." 70 | 71 | Once you have made the changes you want for your custom Hubploy, you can ``git push`` your local 72 | changes. The file mentioned above will automatically attempt to push your Hubploy to DockerHub! If 73 | it fails, there will be output in the Actions tab that should have some insights. 74 | 75 | Now that you have a publicly-hosted image for your custom Hubploy, you can reference it anywhere 76 | you want! In ``hubploy-template``, these references are in the ``hubploy/.github/workflows/`` files 77 | :: 78 | 79 | jobs: 80 | build: 81 | name: 82 | # This job runs on Linux 83 | runs-on: ubuntu-latest 84 | steps: 85 | - uses: actions/checkout@v1 86 | - uses: docker://yuvipanda/hubploy:20191210215236cfab2d 87 | 88 | You will need to change the docker link everywhere you see it in these files to the link of your 89 | image on DockerHub. 90 | 91 | 92 | Contributing to Hubploy 93 | ======================= 94 | 95 | If you have your own fork of Hubploy, and have a feature that would be generally useful, feel free 96 | to join the dicussions in the Issues section or contribute a PR! 97 | 98 | For more details, see the full 99 | `contribution guide `_. 100 | -------------------------------------------------------------------------------- /docs/howto/index.rst: -------------------------------------------------------------------------------- 1 | ================== 2 | How-To Guides Home 3 | ================== 4 | 5 | .. toctree:: 6 | 7 | hubploy-deploy-jupyterhub-repo-setup 8 | hubploy-setup-dev-environment 9 | hubploy-build-jupyterhub-image 10 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | HubPloy 3 | ======= 4 | 5 | ``hubploy`` is a suite of commandline tools and an opinionated 6 | repository structure for continuously deploying JupyterHubs on Kubernetes (with 7 | `Zero to JupyterHub `_). Find the ``hubploy`` 8 | `repository `_ on GitHub. 9 | 10 | 11 | Hubploy workflow 12 | ================ 13 | 14 | **Every change to your hub configuration must be made via a pull request 15 | to your git repository**. Guided by principles of `continuous delivery `_, 16 | this informs hubploy's design. 17 | 18 | Components 19 | ---------- 20 | 21 | The following components make up a hubploy based deployment workflow: 22 | 23 | #. A deployment *git repository*, containing *all* the configuration for your 24 | JupyterHubs. This includes image configuration, zero-to-jupyterhub configuration, 25 | and any secrets if necessary. hubploy is designed to support many different 26 | hubs deploying to different cloud providers from the same repository. 27 | #. A *staging hub* for each JupyterHub in the git repo. End users rarely use 28 | this hub, and it is primarily used for testing by devs. The ``staging`` branch 29 | in the git repo contains the config for these hubs. 30 | #. A *prod(uction) hub* for each JupyterHub in the git repo. End users actively 31 | use this hub, and we try to have minimal downtime here. The ``prod`` branch 32 | in the git repo contains the config for these hubs. However, since we want 33 | prod and staging to be as close as possible, the *prod branch match the 34 | staging branch completely* under normal circumstances. The only commits that 35 | can be in prod but not in staging are merge commits. 36 | 37 | Deploying a change 38 | ------------------ 39 | 40 | .. mermaid:: 41 | 42 | graph TD 43 | Change-Configuration[Change configuration] --> Create-Staging-PR[Create PR to 'staging' branch] 44 | subgraph iterate on config change 45 | Create-Staging-PR --> Automated-Tests[CI runs automated tests] 46 | Automated-Tests --> Code-Review[Code Review] 47 | Code-Review --> Automated-Tests 48 | end 49 | Code-Review --> Merge-Staging-PR[Merge PR to 'staging' branch] 50 | subgraph test in staging 51 | Merge-Staging-PR --> Deploy-To-Staging[CI deploys staging hub] 52 | Deploy-To-Staging --> Test-Staging[Manually test staging hub] 53 | Test-Staging --> |Success| Create-Prod-PR[Create PR from 'staging' to 'prod'] 54 | Test-Staging --> |Fail| Try-Again[Debug & Try Again] 55 | end 56 | Create-Prod-PR --> Merge-Prod-PR[Merge PR to prod branch] 57 | subgraph promote to prod 58 | Merge-Prod-PR --> Deploy-To-Prod[CI deploys prod hub] 59 | Deploy-To-Prod --> Happy-Users[Users are happy!] 60 | end 61 | 62 | 63 | How-To Guides 64 | ============= 65 | 66 | These how-to guides are intended to walk you through the basics of particular tasks that you might do with Hubploy. 67 | 68 | .. toctree:: 69 | :maxdepth: 2 70 | 71 | howto/index 72 | 73 | 74 | Topic Guides 75 | ============ 76 | 77 | These topic guides are meant as informative reference documents about various pieces of Hubploy. 78 | 79 | .. toctree:: 80 | :maxdepth: 2 81 | 82 | topics/index 83 | 84 | 85 | Reference Documentation 86 | ======================= 87 | 88 | These reference documents are here to describe the configuration values of various files in Hubploy 89 | . 90 | 91 | .. toctree:: 92 | :maxdepth: 2 93 | 94 | reference/index 95 | 96 | 97 | Known Limitations 98 | ================= 99 | 100 | #. hubploy requires you already have infrastructure set up - Kubernetes 101 | cluster, persistent home directories, image repositories, etc. There 102 | are `ongoing efforts `_ to fix 103 | this, however. 104 | #. More documentation and tests, as always! -------------------------------------------------------------------------------- /docs/reference/contribution-guide.rst: -------------------------------------------------------------------------------- 1 | ================== 2 | Contribution Guide 3 | ================== 4 | 5 | * `Setting up for Documentation Development`_ 6 | * `Setting up for Hubploy Development`_ 7 | 8 | 9 | ``hubploy`` is open-source and anyone can contribute to it. We welcome 10 | the help! Yuvi Panda is the original author and can give GitHub contributor 11 | access to those who are committed to making ``hubploy`` better. You do not 12 | have to be a contributor on GitHub to suggest changes in 13 | `the Issues section `_ or make 14 | pull requests. A contributor will have to accept your changes before they become 15 | a part of ``hubploy``. 16 | 17 | If you don't have `git `_ 18 | already, install it and clone this repository. 19 | 20 | .. code:: bash 21 | 22 | git clone https://github.com/yuvipanda/hubploy 23 | 24 | Using a 25 | `forking workflow `_ 26 | is also useful and will make seting up pull requests easier. 27 | 28 | Once you have made changes that you are ready to offer to ``hubploy``, 29 | make a 30 | `pull request `_ 31 | to the main `hubploy repository `_. 32 | Someone will get back to you soon on your changes. 33 | 34 | If you want to dicuss 35 | changes before they get onto GitHub or contact a contributor, try the 36 | `JupyterHub Gitter channel `_. 37 | 38 | 39 | Setting up for Documentation Development 40 | ======================================== 41 | 42 | The ``hubploy`` documentation is automatically built on each commit 43 | `as configured on ReadTheDocs `_. 44 | Source files are in the ``docs/`` folder of the main repository. 45 | 46 | To set up your local machine for documentation development, install the 47 | required packages with: 48 | 49 | .. code:: bash 50 | 51 | # From the docs/ folder 52 | pip install -r doc-requirements.txt 53 | 54 | To test your updated documentation, run: 55 | 56 | .. code:: bash 57 | 58 | # From the docs/ folder 59 | make html 60 | 61 | Make sure there are no warnings or errors. From there, you can check 62 | the ``_build/html/`` folder and launch the ``.html`` files locally to 63 | check that formatting is as you expect. 64 | 65 | 66 | Setting up for Hubploy Development 67 | ================================== 68 | 69 | See the How-To guide on 70 | `setting up a development environment `_ 71 | for ``hubploy``. 72 | 73 | In short, you can install ``hubploy`` and its dependencies easily 74 | with the above guide but you will need a 75 | `kubernetes `_ cluster to do local deployment 76 | tests. Some good resources for deploying a kubernetes cluster are: 77 | 78 | #. `Zero to JupyterHub K8s `_ 79 | #. `AWS Terraform K8s Examples `_ 80 | 81 | You will also need to reference the section 82 | `Using a Custom Hubploy Locally `_, 83 | rather than doing a default ``hubploy`` installation. 84 | 85 | -------------------------------------------------------------------------------- /docs/reference/index.rst: -------------------------------------------------------------------------------- 1 | =================== 2 | Reference Docs Home 3 | =================== 4 | 5 | .. toctree:: 6 | 7 | reference-hubploy-configuration-values 8 | contribution-guide 9 | -------------------------------------------------------------------------------- /docs/reference/reference-hubploy-configuration-values.rst: -------------------------------------------------------------------------------- 1 | ====================================== 2 | Hubploy Configuration Values Reference 3 | ====================================== 4 | 5 | This reference doc will detail the various configuration values present in ``hubploy.yaml``. 6 | Here is the ``hubploy.yaml`` file that comes with cloning hubploy-template:: 7 | 8 | images: 9 | image_name: # TODO: Full path to your docker image, based on the following pattern 10 | # On AWS: .dkr.ecr..amazonaws.com/-user-image 11 | # On Google Cloud: gcr.io//-user-image 12 | registry: 13 | provider: # TODO: Pick 'gcloud' or 'aws', and fill up other config accordingly 14 | gcloud: 15 | # Pushes to Google Container Registry. 16 | project: # TODO: GCloud Project Name 17 | # Make a service account with GCR push permissions, put it in secrets/gcr-key.json 18 | service_key: gcr-key.json 19 | aws: 20 | # Pushes to Amazon ECR 21 | account_id: # TODO: AWS account id 22 | region: # TODO: Zone in which your container image should live. Match your cluster's zone 23 | # TODO: Get AWS credentials that can push to ECR, in same format as ~/.aws/credentials 24 | # then put them in secrets/aws-ecr-config.cfg 25 | service_key: aws-ecr-config.cfg 26 | 27 | cluster: 28 | provider: # TODO: gcloud or aws 29 | gcloud: 30 | project: # TODO: Name of your Google Cloud project with the cluster in it 31 | cluster: # TODO: Name of your Kubernetes cluster 32 | zone: # TODO: Zone or region your cluster is in 33 | # Make a service key with permissions to talk to your cluster, put it in secrets/gkee-key.json 34 | service_key: gke-key.json 35 | aws: 36 | account_id: # TODO: AWS account id 37 | region: # TODO: Zone or region in which your cluster is set up 38 | cluster: # TODO: The name of your EKS cluster 39 | # TODO: Get AWS credentials that can access your EKS cluster, in same format as ~/.aws credentials 40 | # then put them in secrets/aws-eks-config.cfg 41 | service_key: aws-eks-config.cfg 42 | 43 | The various values are described below. 44 | 45 | 46 | images 47 | ====== 48 | 49 | The entire ``images`` block is optional. If you don't need it, comment it out or delete it. 50 | 51 | image_name 52 | ---------- 53 | 54 | Full path to your docker image, based on the following pattern: 55 | * On AWS: .dkr.ecr..amazonaws.com/-user-image 56 | * On Google Cloud: gcr.io//-user-image 57 | 58 | registry 59 | -------- 60 | 61 | provider 62 | ^^^^^^^^ 63 | 64 | Either 'aws' or 'gcloud'. More options will be present in the future. 65 | Both the ``aws`` and ``gcloud`` blocks are uncommented. The one that you do not pick should be 66 | commented out. 67 | 68 | gcloud 69 | ^^^^^^ 70 | 71 | project 72 | """"""" 73 | 74 | GCloud Project Name 75 | 76 | service_key 77 | """"""""""" 78 | 79 | ``gcr-key.json`` by default. 80 | 81 | Make a service account with GCR push permissions and put it in ``secrets/gcr-key.json``. You can 82 | rename this file, but you will also need put the new filename here. 83 | 84 | aws 85 | ^^^ 86 | 87 | account_id 88 | """""""""" 89 | 90 | AWS account ID 91 | 92 | region 93 | """""" 94 | 95 | The zone in which your ECR image will live. This should match the zone where your cluster will 96 | live. 97 | 98 | service_key 99 | """"""""""" 100 | 101 | ``aws-ecr-config.cfg`` by default. 102 | 103 | Get AWS Credentials that can push images to ECR. These credentials should be in the same format as 104 | found in ``~/.aws/credentials`` and put in to the file ``secrets/aws-ecr-config.cfg``. You can 105 | rename this file, but you will also need put the new filename here. 106 | 107 | 108 | cluster 109 | ======= 110 | 111 | provider 112 | -------- 113 | 114 | Either 'aws' or 'gcloud'. More options will be present in the future. 115 | Both the ``aws`` and ``gcloud`` blocks are uncommented. The one that you do not pick should be 116 | commented out. 117 | 118 | gcloud 119 | ------ 120 | 121 | project 122 | ^^^^^^^ 123 | 124 | Name of your Google Cloud project with the cluster you will create. 125 | 126 | cluster 127 | ^^^^^^^ 128 | 129 | Name of the Kubernetes cluster you will create. 130 | 131 | zone 132 | ^^^^ 133 | 134 | Zone or region this cluster will sit in. 135 | 136 | service_key 137 | ^^^^^^^^^^^ 138 | 139 | ``gke-key.json`` by default. 140 | 141 | Make a service key with permissions to talk to your cluster and put it in ``secrets/gke-key.json``. 142 | You can rename this file, but you will also need put the new filename here. 143 | 144 | aws 145 | --- 146 | 147 | account_id 148 | ^^^^^^^^^^ 149 | 150 | AWS account ID 151 | 152 | cluster 153 | ^^^^^^^ 154 | 155 | The name of the EKS cluster you will create. 156 | 157 | region 158 | ^^^^^^ 159 | 160 | Zone or region this cluster will sit in. 161 | 162 | service_key 163 | ^^^^^^^^^^^ 164 | 165 | ``aws-eks-config.cfg`` by default. 166 | 167 | Get AWS credentials that can access your EKS cluster. These credentials should be in the same 168 | format as found in ``~/.aws/credentials`` and put in to the file ``secrets/aws-eks-config.cfg``. 169 | You can rename this file, but you will also need put the new filename here. 170 | 171 | -------------------------------------------------------------------------------- /docs/topics/index.rst: -------------------------------------------------------------------------------- 1 | =========== 2 | Topics Home 3 | =========== 4 | 5 | .. toctree:: 6 | 7 | topic-directory-structure 8 | topic-values-yaml-overriding 9 | topic-helm-versions 10 | -------------------------------------------------------------------------------- /docs/topics/topic-directory-structure.rst: -------------------------------------------------------------------------------- 1 | ====================================== 2 | Hubploy's Expected Directory Structure 3 | ====================================== 4 | 5 | Hubploy expects the directory structure shown in the 6 | `hubploy template repository `_. The folders must be 7 | set up in this fashion:: 8 | 9 | hubploy-template/ 10 | ├── .github 11 | │   └── workflows 12 | │   ├── deploy.yaml 13 | │   └── image-build.yaml 14 | ├── deployments 15 | │   └── hub 16 | │   ├── config 17 | │   │   ├── common.yaml 18 | │   │   ├── prod.yaml 19 | │   │   └── staging.yaml 20 | │   ├── hubploy.yaml 21 | │   ├── image 22 | │   │   ├── ipython_config.py 23 | │   │   ├── postBuild 24 | │   │   └── requirements.txt 25 | │   └── secrets 26 | │   ├── aws-ecr-config.cfg 27 | │   ├── aws-eks-config.cfg 28 | │   ├── prod.yaml 29 | │   └── staging.yaml 30 | ├── hub 31 | │   ├── Chart.yaml 32 | │   ├── requirements.lock 33 | │   ├── requirements.yaml 34 | │   ├── templates 35 | │   │   ├── jupyter-notebook-config.yaml 36 | │   │   └── nfs-pvc.yaml 37 | │   └── values.yaml 38 | ├── LICENSE 39 | ├── README.rst 40 | └── requirements.txt 41 | 42 | 43 | .github Folder 44 | -------------- 45 | 46 | This folder houses the GitHub Workflow files that you can use for Continuous Integration with 47 | Hubploy. ``deploy.yaml`` will attempt to build the staging or production JupyterHub upon updates 48 | to the respective GitHub branch. ``image-build.yaml`` will attempt to build the JupyterHub image 49 | upon updates to only the production branch. 50 | 51 | These files have references to a Docker image that uses Hubploy. You can change this image. Some 52 | options are listed in :doc:`../howto/hubploy-setup-dev-environment`. 53 | 54 | 55 | Deployments Folder 56 | ------------------ 57 | 58 | The deployments folder can hold multiple subfolders, but each must have the same structure as the 59 | hub folder. Renaming the hub folder is part of the recommended workflow for deploying a JupyterHub. 60 | Each subfolder directly under deployments needs a different name so that Hubploy can distinguish 61 | between them in Hubploy commands. 62 | 63 | Each JupyterHub is deployed with YAML files. The YAML files listed under deployments must have 64 | these names. 65 | 66 | Hubploy takes in secrets for credentialing via the ``.cfg`` files. You can rename these freely, 67 | just be sure to put the proper names into ``hubploy.yaml``. 68 | 69 | The image folder can have additional files depending on how you are building the image. See more 70 | in the image building how-to. If you are not specifying ``images`` in your ``hubploy.yaml`` file, 71 | the ``images/`` folder can be deleted. 72 | 73 | 74 | Hub Folder 75 | ---------- 76 | 77 | The hub folder houses a local `Helm Chart`_. This chart and folder can be renamed, but the name 78 | needs to be present in Hubploy commands, the files in the ``.github`` folder, and in ``Chart.yaml`` 79 | . Modification of the files in here should be done as you would change a Helm Chart. 80 | 81 | 82 | .. _Helm Chart: https://helm.sh/docs/intro/using_helm/ 83 | -------------------------------------------------------------------------------- /docs/topics/topic-helm-versions.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | Helm Versions in Hubploy 3 | ======================== 4 | 5 | * `Helm Versions Present by Default`_ 6 | * `Using a Custom Version of Helm`_ 7 | * `Local Usage`_ 8 | * `GitHub Action Usage`_ 9 | 10 | Helm Versions Present by Default 11 | ================================ 12 | 13 | The ``hubploy`` Docker image has `Helm `_ v2.16.9 14 | and v3.2.4 installed by default. This may depend on the specific version 15 | of ``hubploy`` that is installed. Versions can be found in the 16 | `Dockerfile `_ 17 | present in the base folder of the 18 | `hubploy `_ repository. There isn't 19 | a version matrix to help find which versions of ``helm`` ship with certain 20 | versions of ``hubploy``. You can look at the ``Dockerfile``'s commit history 21 | or just use the most recent version of ``hubploy``, which has the versions 22 | listed above. 23 | 24 | 25 | Using a Custom Version of Helm 26 | ============================== 27 | 28 | To use your own installed version of ``helm``, set the environment variable 29 | ``HELM_EXECUTABLE``. ``hubploy`` will pick up the value from this environment 30 | variable to use when running ``helm`` commands. It will default to ``helm``, 31 | ie. v2.16.9, if nothing else is installed. You can find the line of code that 32 | does this 33 | `here `_. 34 | 35 | 36 | Local Usage 37 | =========== 38 | 39 | You can use several versions of ``helm`` in local usage of ``hubploy`` 40 | This does require that you have installed ``helm`` or are using the 41 | ``hubploy`` Docker image on your local machine. 42 | 43 | To use this environment variable on a local installation of ``hubploy``, 44 | use the following command from your terminal: 45 | 46 | .. code:: bash 47 | 48 | export HELM_EXECUTABLE=~/absolute/path/to/helm/binary 49 | 50 | For example, if you wanted to use ``helm`` v3 locally and had installed 51 | and moved it to ``/usr/local/bin/helm3``, you would run the following from 52 | your terminal: 53 | 54 | .. code:: bash 55 | 56 | export HELM_EXECUTABLE=/usr/local/bin/helm3 57 | 58 | If you already have ``helm`` v2 installed, no extra steps are necessary. 59 | 60 | 61 | GitHub Action Usage 62 | =================== 63 | 64 | To use this environment variable in a GitHub Action, use the following lines 65 | in your workflow file: 66 | 67 | .. code:: yaml 68 | 69 | env: 70 | HELM_EXECUTABLE: /absolute/path/to/helm/binary 71 | 72 | More information on this second option can be found on the 73 | `Environment variables page `_ 74 | on GitHub Docs. 75 | 76 | -------------------------------------------------------------------------------- /docs/topics/topic-values-yaml-overriding.rst: -------------------------------------------------------------------------------- 1 | ===================================== 2 | YAML File Value Overriding in Hubploy 3 | ===================================== 4 | 5 | There are several ``.yaml`` files present in the hubploy-template repository. It can be unclear 6 | which settings go in which files. This topic hopes to clear that up a bit. As a reminder, here is 7 | the directory structure that Hubploy expects (minimized for focus on the yaml files):: 8 | 9 | hubploy-template/ 10 | ├── .github 11 | │   └── workflows 12 | │   ├── deploy.yaml 13 | │   └── image-build.yaml 14 | ├── deployments 15 | │   └── hub 16 | │   ├── config 17 | │   │   ├── common.yaml 18 | │   │   ├── prod.yaml 19 | │   │   └── staging.yaml 20 | │   ├── hubploy.yaml 21 | │   └── secrets 22 | │   ├── aws-ecr-config.cfg 23 | │   ├── aws-eks-config.cfg 24 | │   ├── prod.yaml 25 | │   └── staging.yaml 26 | └── hub 27 |    ├── Chart.yaml 28 |    ├── requirements.lock 29 |    ├── requirements.yaml 30 |    ├── templates 31 |    │   ├── jupyter-notebook-config.yaml 32 |    │   └── nfs-pvc.yaml 33 |    └── values.yaml 34 | 35 | 36 | GitHub Action Files 37 | ------------------- 38 | 39 | The two files under ``.github/workflows/`` manage individual GitHub Actions for Hubploy. They are 40 | independent of most of the rest of Hubploy. 41 | 42 | 43 | JupyterHub Deployment Files 44 | --------------------------- 45 | 46 | The main value files are related to the JupyterHub Helm release. The lowest level of these are 47 | specified in ``hub/values.yaml`` via:: 48 | 49 | jupyterhub: {} 50 | 51 | The braces can be removed once there are yaml values in the file, but they are needed if this block 52 | is empty. Appropriate values to put in this file are those that will span both versions of all 53 | JupyterHubs that you will deploy with Hubploy, as this file will be used for all of them. 54 | 55 | The next file in the heirarchy is ``deployments/hub/config/common.yaml``. This file covers 56 | deployment values that are common to both the staging and production hubs that Hubploy named 57 | "hub," or what you had changed that folder name to. If there are multiple JupyterHubs being managed 58 | , each one will have a ``common.yaml``. Values in this file will overwrite ``hub/values.yaml``. 59 | 60 | The next two files in the heirarchy are also in the config folder: ``staging.yaml`` and 61 | ``prod.yaml``. These contain values for the staging and production hubs, respectively. Values in 62 | these files will override the previous two. These two files do not override each other ever since they are for two different hubs. 63 | 64 | The last files in the heirarchy are under the ``secrets`` directory. These are set in a folder that 65 | we tell git-crypt to encrypt when pushing code to GitHub. In general, there shouldn't be anything 66 | in these files that overwrites the other ``staging.yaml`` and ``prod.yaml``. It is more expected 67 | that values in these files will overwrite default credentials or paths present in the first two 68 | files. 69 | 70 | A quick summary of the heirarchy follows in descending priority (lower overwrites higher) 71 | but ascending generality (higher applies to more hubs):: 72 | 73 | hub/values.yaml 74 | deployments/hub/config/common.yaml 75 | deployments/hub/config/staging.yaml 76 | deployments/hub/config/prod.yaml 77 | deployments/hub/secrets/staging.yaml 78 | deployments/hub/secrets/prod.yaml 79 | 80 | 81 | Local Hub Helm Chart Files 82 | -------------------------- 83 | 84 | Everything under the hub folder is related to the Helm Chart. In ``Chart.yaml``, the main 85 | specification is what the Chart is named and what version you are on. In ``requirements.yaml``, 86 | the JupyterHub Helm chart is listed as the only dependency and you can pick a specific version. 87 | ``values.yaml`` is used to provide the lowest level of values for JupyterHub configuration and 88 | other deployment pieces that are present in the ``templates/`` folder or other dependencies 89 | you choose to add to the Helm chart. 90 | 91 | -------------------------------------------------------------------------------- /hubploy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/berkeley-dsep-infra/hubploy/b6ee8e6f0bb1bfbfb3db3f08cb928dd19cb97fff/hubploy/__init__.py -------------------------------------------------------------------------------- /hubploy/__main__.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import hubploy 3 | import logging 4 | import os 5 | import sys 6 | import textwrap 7 | 8 | from hubploy import helm 9 | from argparse import RawTextHelpFormatter 10 | 11 | logging.basicConfig(stream=sys.stdout, level=logging.WARNING) 12 | logger = logging.getLogger(__name__) 13 | 14 | 15 | def main(): 16 | argparser = argparse.ArgumentParser(formatter_class=RawTextHelpFormatter) 17 | subparsers = argparser.add_subparsers(dest="command") 18 | 19 | argparser.add_argument( 20 | "-d", 21 | "--debug", 22 | action="store_true", 23 | help="Enable tool debug output (not including helm debug).", 24 | ) 25 | argparser.add_argument( 26 | "-D", 27 | "--helm-debug", 28 | action="store_true", 29 | help="Enable Helm debug output. This is not allowed to be used in a " 30 | + "CI environment due to secrets being displayed in plain text, and " 31 | + "the script will exit. To enable this option, set a local environment " 32 | + "varible HUBPLOY_LOCAL_DEBUG=true", 33 | ) 34 | argparser.add_argument( 35 | "-v", "--verbose", action="store_true", help="Enable verbose output." 36 | ) 37 | 38 | deploy_parser = subparsers.add_parser( 39 | "deploy", help="Deploy a chart to the given environment." 40 | ) 41 | 42 | deploy_parser.add_argument("deployment", help="The name of the hub to deploy.") 43 | deploy_parser.add_argument("chart", help="The path to the main hub chart.") 44 | deploy_parser.add_argument( 45 | "environment", 46 | choices=["develop", "staging", "prod"], 47 | help="The environment to deploy to.", 48 | ) 49 | deploy_parser.add_argument( 50 | "--namespace", 51 | default=None, 52 | help="Helm option: the namespace to deploy to. If not specified, " 53 | + "the namespace will be derived from the environment argument.", 54 | ) 55 | deploy_parser.add_argument( 56 | "--set", 57 | action="append", 58 | help="Helm option: set values on the command line (can specify " 59 | + "multiple or separate values with commas: key1=val1,key2=val2)", 60 | ) 61 | deploy_parser.add_argument( 62 | "--set-string", 63 | action="append", 64 | help="Helm option: set STRING values on the command line (can " 65 | + "specify multiple or separate values with commas: key1=val1,key2=val2)", 66 | ) 67 | deploy_parser.add_argument( 68 | "--version", 69 | help="Helm option: specify a version constraint for the chart " 70 | + "version to use. This constraint can be a specific tag (e.g. 1.1.1) " 71 | + "or it may reference a valid range (e.g. ^2.0.0). If this is not " 72 | + "specified, the latest version is used.", 73 | ) 74 | deploy_parser.add_argument( 75 | "--timeout", 76 | help="Helm option: time in seconds to wait for any individual " 77 | + "Kubernetes operation (like Jobs for hooks, etc). Defaults to 300 " 78 | + "seconds.", 79 | ) 80 | deploy_parser.add_argument( 81 | "--force", 82 | action="store_true", 83 | help="Helm option: force resource updates through a replacement strategy.", 84 | ) 85 | deploy_parser.add_argument( 86 | "--atomic", 87 | action="store_true", 88 | help="Helm option: if set, upgrade process rolls back changes made " 89 | + "in case of failed upgrade. The --wait flag will be set automatically " 90 | + "if --atomic is used.", 91 | ) 92 | deploy_parser.add_argument( 93 | "--cleanup-on-fail", 94 | action="store_true", 95 | help="Helm option: allow deletion of new resources created in this " 96 | + "upgrade when upgrade fails.", 97 | ) 98 | deploy_parser.add_argument( 99 | "--dry-run", 100 | default=False, 101 | action="store_true", 102 | help="Dry run the helm upgrade command. This also renders the " 103 | + "chart to STDOUT. This is not allowed to be used in a " 104 | + "CI environment due to secrets being displayed in plain text, and " 105 | + "the script will exit. To enable this option, set a local environment " 106 | + "varible HUBPLOY_LOCAL_DEBUG=true", 107 | ) 108 | deploy_parser.add_argument( 109 | "--image-overrides", 110 | nargs="+", 111 | help=textwrap.dedent( 112 | """\ 113 | Override one or more images and tags to deploy. Format is:\n 114 | : : ...\n \n 115 | IMPORTANT: The order of images passed in must match the order in which 116 | they appear in hubploy.yaml and separated by spaces without quotes. You 117 | must always specify a tag when overriding images. 118 | """ 119 | ), 120 | ) 121 | 122 | args = argparser.parse_args() 123 | 124 | if args.command is None: 125 | argparser.print_help() 126 | sys.exit(1) 127 | 128 | if args.verbose: 129 | logger.setLevel(logging.INFO) 130 | elif args.debug: 131 | logger.setLevel(logging.DEBUG) 132 | logger.info(args) 133 | 134 | is_on_ci = os.environ.get("CI", False) 135 | if is_on_ci: 136 | if args.helm_debug or args.dry_run: 137 | print( 138 | "--helm-debug and --dry-run are not allowed to be used in a CI environment." 139 | ) 140 | print("Exiting...") 141 | sys.exit(1) 142 | else: 143 | if args.helm_debug or args.dry_run: 144 | if os.environ.get("HUBPLOY_LOCAL_DEBUG", False): 145 | print( 146 | "Local debug mode enabled. Proceeding with --helm-debug and --dry-run." 147 | ) 148 | else: 149 | print( 150 | "To enable local debug mode, set a local environment variable HUBPLOY_LOCAL_DEBUG=true" 151 | ) 152 | print("Exiting...") 153 | sys.exit(1) 154 | 155 | # Attempt to load the config early, fail if it doesn't exist or is invalid 156 | try: 157 | config = hubploy.config.get_config(args.deployment, debug=False, verbose=False) 158 | if not config: 159 | raise hubploy.config.DeploymentNotFoundError( 160 | "Deployment '{}' not found in hubploy.yaml".format(args.deployment) 161 | ) 162 | except hubploy.config.DeploymentNotFoundError as e: 163 | print(e, file=sys.stderr) 164 | sys.exit(1) 165 | 166 | helm.deploy( 167 | args.deployment, 168 | args.chart, 169 | args.environment, 170 | args.namespace, 171 | args.set, 172 | args.set_string, 173 | args.version, 174 | args.timeout, 175 | args.force, 176 | args.atomic, 177 | args.cleanup_on_fail, 178 | args.debug, 179 | args.verbose, 180 | args.helm_debug, 181 | args.dry_run, 182 | args.image_overrides, 183 | ) 184 | 185 | 186 | if __name__ == "__main__": 187 | main() 188 | -------------------------------------------------------------------------------- /hubploy/auth.py: -------------------------------------------------------------------------------- 1 | """ 2 | Utils to authenticate with a set of cloud providers' container registries 3 | (registry_auth) and Kubernetes clusters (cluster_auth) for use in 4 | with-statements. 5 | 6 | Current cloud providers supported: gcloud, aws, and azure. 7 | """ 8 | 9 | import boto3 10 | import json 11 | import logging 12 | import os 13 | import subprocess 14 | import tempfile 15 | 16 | from contextlib import contextmanager 17 | from hubploy.config import get_config 18 | from ruamel.yaml import YAML 19 | from ruamel.yaml.scanner import ScannerError 20 | 21 | logger = logging.getLogger(__name__) 22 | yaml = YAML(typ="rt") 23 | 24 | 25 | @contextmanager 26 | def cluster_auth(deployment, debug=False, verbose=False): 27 | """ 28 | Do appropriate cluster authentication for given deployment 29 | """ 30 | if verbose: 31 | logger.setLevel(logging.INFO) 32 | elif debug: 33 | logger.setLevel(logging.DEBUG) 34 | 35 | logger.info(f"Getting auth config for {deployment}") 36 | config = get_config(deployment, debug, verbose) 37 | 38 | if "cluster" in config: 39 | cluster = config["cluster"] 40 | provider = cluster.get("provider") 41 | orig_kubeconfig = os.environ.get("KUBECONFIG", None) 42 | 43 | try: 44 | if provider == "kubeconfig": 45 | logger.info( 46 | f"Attempting to authenticate to {cluster} with " 47 | + "existing kubeconfig." 48 | ) 49 | logger.debug( 50 | "Using kubeconfig file " 51 | + f"deploylemts/{deployment}/secrets/{cluster['kubeconfig']['filename']}" 52 | ) 53 | encrypted_kubeconfig_path = os.path.join( 54 | "deployments", 55 | deployment, 56 | "secrets", 57 | cluster["kubeconfig"]["filename"], 58 | ) 59 | with decrypt_file(encrypted_kubeconfig_path) as kubeconfig_path: 60 | os.environ["KUBECONFIG"] = kubeconfig_path 61 | yield 62 | else: 63 | # Temporarily kubeconfig file 64 | with tempfile.NamedTemporaryFile() as temp_kubeconfig: 65 | os.environ["KUBECONFIG"] = temp_kubeconfig.name 66 | logger.info(f"Attempting to authenticate with {provider}...") 67 | 68 | if provider == "gcloud": 69 | yield from cluster_auth_gcloud(deployment, **cluster["gcloud"]) 70 | elif provider == "aws": 71 | yield from cluster_auth_aws(deployment, **cluster["aws"]) 72 | elif provider == "azure": 73 | yield from cluster_auth_azure(deployment, **cluster["azure"]) 74 | else: 75 | raise ValueError( 76 | f"Unknown provider {provider} found in " + "hubploy.yaml" 77 | ) 78 | finally: 79 | unset_env_var("KUBECONFIG", orig_kubeconfig) 80 | 81 | 82 | def cluster_auth_gcloud(deployment, project, cluster, zone, service_key): 83 | """ 84 | Setup GKE authentication with service_key 85 | 86 | This changes *global machine state* on what current kubernetes cluster is! 87 | """ 88 | current_login_command = [ 89 | "gcloud", 90 | "config", 91 | "get-value", 92 | "account", 93 | ] 94 | logger.info("Saving current gcloud login") 95 | logger.debug( 96 | "Running gcloud command: " + " ".join(x for x in current_login_command) 97 | ) 98 | current_login = ( 99 | subprocess.check_output(current_login_command).decode("utf-8").strip() 100 | ) 101 | logger.info(f"Current gcloud login: {current_login}") 102 | 103 | encrypted_service_key_path = os.path.join( 104 | "deployments", deployment, "secrets", service_key 105 | ) 106 | with decrypt_file(encrypted_service_key_path) as decrypted_service_key_path: 107 | gcloud_auth_command = [ 108 | "gcloud", 109 | "auth", 110 | "activate-service-account", 111 | "--key-file", 112 | os.path.abspath(decrypted_service_key_path), 113 | ] 114 | logger.info(f"Activating service account for {project}") 115 | logger.debug( 116 | "Running gcloud command: " + " ".join(x for x in gcloud_auth_command) 117 | ) 118 | subprocess.check_call(gcloud_auth_command) 119 | 120 | gcloud_cluster_credential_command = [ 121 | "gcloud", 122 | "container", 123 | "clusters", 124 | f"--zone={zone}", 125 | f"--project={project}", 126 | "get-credentials", 127 | cluster, 128 | ] 129 | logger.info(f"Getting credentials for {cluster} in {zone}") 130 | logger.debug( 131 | "Running gcloud command: " 132 | + " ".join(x for x in gcloud_cluster_credential_command) 133 | ) 134 | subprocess.check_call(gcloud_cluster_credential_command) 135 | 136 | yield current_login 137 | 138 | 139 | @contextmanager 140 | def revert_gcloud_auth(current_login): 141 | """ 142 | Revert gcloud authentication to previous state 143 | """ 144 | if current_login: 145 | logger.info(f"Reverting gcloud login to {current_login}") 146 | subprocess.check_call(["gcloud", "config", "set", "account", current_login]) 147 | else: 148 | logger.info("Reverting gcloud login to default") 149 | subprocess.check_call(["gcloud", "config", "unset", "account"]) 150 | yield 151 | 152 | 153 | @contextmanager 154 | def _auth_aws(deployment, service_key=None, role_arn=None, role_session_name=None): 155 | """ 156 | This helper contextmanager will update AWS_SHARED_CREDENTIALS_FILE if 157 | service_key is provided and AWS_SESSION_TOKEN if role_arn is provided. 158 | """ 159 | # validate arguments 160 | if bool(service_key) == bool(role_arn): 161 | raise Exception( 162 | "AWS authentication require either service_key or role_arn, but not both." 163 | ) 164 | if role_arn: 165 | assert role_session_name, "always pass role_session_name along with role_arn" 166 | 167 | try: 168 | original_access_key_id = os.environ.get("AWS_ACCESS_KEY_ID", None) 169 | original_secret_access_key = os.environ.get("AWS_SECRET_ACCESS_KEY", None) 170 | original_session_token = os.environ.get("AWS_SESSION_TOKEN", None) 171 | if service_key: 172 | original_credential_file_loc = os.environ.get( 173 | "AWS_SHARED_CREDENTIALS_FILE", None 174 | ) 175 | 176 | # Get path to service_key and validate its around 177 | encrypted_service_key_path = os.path.join( 178 | "deployments", deployment, "secrets", service_key 179 | ) 180 | if not os.path.isfile(encrypted_service_key_path): 181 | raise FileNotFoundError( 182 | f"The service_key file {encrypted_service_key_path} does not exist" 183 | ) 184 | 185 | logger.info(f"Decrypting service key {encrypted_service_key_path}") 186 | with decrypt_file(encrypted_service_key_path) as decrypted_service_key_path: 187 | auth = yaml.load(open(decrypted_service_key_path)) 188 | os.environ["AWS_ACCESS_KEY_ID"] = auth["creds"]["aws_access_key_id"] 189 | os.environ["AWS_SECRET_ACCESS_KEY"] = auth["creds"][ 190 | "aws_secret_access_key" 191 | ] 192 | logger.info("Set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY") 193 | 194 | elif role_arn: 195 | original_access_key_id = os.environ.get("AWS_ACCESS_KEY_ID", None) 196 | original_secret_access_key = os.environ.get("AWS_SECRET_ACCESS_KEY", None) 197 | original_session_token = os.environ.get("AWS_SESSION_TOKEN", None) 198 | 199 | sts_client = boto3.client("sts") 200 | assumed_role_object = sts_client.assume_role( 201 | RoleArn=role_arn, RoleSessionName=role_session_name 202 | ) 203 | 204 | creds = assumed_role_object["Credentials"] 205 | os.environ["AWS_ACCESS_KEY_ID"] = creds["AccessKeyId"] 206 | os.environ["AWS_SECRET_ACCESS_KEY"] = creds["SecretAccessKey"] 207 | os.environ["AWS_SESSION_TOKEN"] = creds["SessionToken"] 208 | 209 | # return until context exits 210 | yield 211 | 212 | finally: 213 | if service_key: 214 | unset_env_var("AWS_SHARED_CREDENTIALS_FILE", original_credential_file_loc) 215 | unset_env_var("AWS_ACCESS_KEY_ID", original_access_key_id) 216 | unset_env_var("AWS_SECRET_ACCESS_KEY", original_secret_access_key) 217 | unset_env_var("AWS_SESSION_TOKEN", original_session_token) 218 | elif role_arn: 219 | unset_env_var("AWS_ACCESS_KEY_ID", original_access_key_id) 220 | unset_env_var("AWS_SECRET_ACCESS_KEY", original_secret_access_key) 221 | unset_env_var("AWS_SESSION_TOKEN", original_session_token) 222 | 223 | 224 | def cluster_auth_aws(deployment, cluster, region, service_key=None, role_arn=None): 225 | """ 226 | Setup AWS authentication with service_key or with a role 227 | 228 | This changes *global machine state* on what current kubernetes cluster is! 229 | """ 230 | with _auth_aws( 231 | deployment, 232 | service_key=service_key, 233 | role_arn=role_arn, 234 | role_session_name="hubploy-cluster-auth", 235 | ): 236 | subprocess.check_call( 237 | ["aws", "eks", "update-kubeconfig", "--name", cluster, "--region", region], 238 | stdout=subprocess.DEVNULL, 239 | stderr=subprocess.STDOUT, 240 | ) 241 | yield 242 | 243 | 244 | def cluster_auth_azure(deployment, resource_group, cluster, auth_file): 245 | """ 246 | 247 | Azure authentication for AKS 248 | 249 | In hubploy.yaml include: 250 | 251 | cluster: 252 | provider: azure 253 | azure: 254 | resource_group: resource_group_name 255 | cluster: cluster_name 256 | auth_file: azure_auth_file.yaml 257 | 258 | The azure_service_principal.json file should have the following keys: 259 | appId, tenant, password. 260 | 261 | This is the format produced by the az command when creating a service 262 | principal. 263 | """ 264 | 265 | # parse Azure auth file 266 | auth_file_path = os.path.join("deployments", deployment, "secrets", auth_file) 267 | with open(auth_file_path) as f: 268 | auth = yaml.load(f) 269 | 270 | # log in 271 | subprocess.check_call( 272 | [ 273 | "az", 274 | "login", 275 | "--service-principal", 276 | "--user", 277 | auth["appId"], 278 | "--tenant", 279 | auth["tenant"], 280 | "--password", 281 | auth["password"], 282 | ] 283 | ) 284 | 285 | # get cluster credentials 286 | subprocess.check_call( 287 | [ 288 | "az", 289 | "aks", 290 | "get-credentials", 291 | "--name", 292 | cluster, 293 | "--resource-group", 294 | resource_group, 295 | ] 296 | ) 297 | 298 | yield 299 | 300 | 301 | def unset_env_var(env_var, old_env_var_value): 302 | """ 303 | If the old environment variable's value exists, replace the current one 304 | with the old one. 305 | 306 | If the old environment variable's value does not exist, delete the current 307 | one. 308 | """ 309 | 310 | if env_var in os.environ: 311 | del os.environ[env_var] 312 | if old_env_var_value is not None: 313 | os.environ[env_var] = old_env_var_value 314 | 315 | 316 | @contextmanager 317 | def decrypt_file(encrypted_path): 318 | """ 319 | Provide secure temporary decrypted contents of a given file 320 | 321 | If file isn't a sops encrypted file, we assume no encryption is used 322 | and return the current path. 323 | """ 324 | # We must first determine if the file is using sops 325 | # sops files are JSON/YAML with a `sops` key. So we first check 326 | # if the file is valid JSON/YAML, and then if it has a `sops` key 327 | logger.info(f"Decrypting {encrypted_path}") 328 | with open(encrypted_path) as f: 329 | _, ext = os.path.splitext(encrypted_path) 330 | # Support the (clearly wrong) people who use .yml instead of .yaml 331 | if ext == ".yaml" or ext == ".yml": 332 | try: 333 | encrypted_data = yaml.load(f) 334 | except ScannerError: 335 | yield encrypted_path 336 | return 337 | elif ext == ".json": 338 | try: 339 | encrypted_data = json.load(f) 340 | except json.JSONDecodeError: 341 | yield encrypted_path 342 | return 343 | elif ext == ".cfg": 344 | try: 345 | with open(encrypted_path) as f: 346 | encrypted_data = f.read() 347 | except Exception: 348 | yield encrypted_path 349 | return 350 | 351 | if "sops" not in encrypted_data: 352 | logger.info("File is not sops encrypted, returning path") 353 | yield encrypted_path 354 | return 355 | 356 | else: 357 | # If file has a `sops` key, we assume it's sops encrypted 358 | sops_command = ["sops", "--decrypt", encrypted_path] 359 | 360 | logger.info("File is sops encrypted, decrypting...") 361 | logger.debug( 362 | "Executing: " 363 | + " ".join(sops_command) 364 | + " (with output to a temporary file)" 365 | ) 366 | with tempfile.NamedTemporaryFile() as f: 367 | subprocess.check_call( 368 | ["sops", "--output", f.name, "--decrypt", encrypted_path] 369 | ) 370 | yield f.name 371 | -------------------------------------------------------------------------------- /hubploy/config.py: -------------------------------------------------------------------------------- 1 | """ 2 | A util (get_config) that process hubploy.yaml deployment configuration and 3 | returns it embedded with a set of LocalImage objects with filesystem paths made 4 | absolute. 5 | """ 6 | 7 | import logging 8 | import os 9 | from ruamel.yaml import YAML 10 | 11 | logger = logging.getLogger(__name__) 12 | yaml = YAML(typ="safe") 13 | 14 | 15 | class DeploymentNotFoundError(Exception): 16 | def __init__(self, deployment, path, *args, **kwargs): 17 | super().__init__(*args, **kwargs) 18 | self.deployment = deployment 19 | self.path = path 20 | 21 | def __str__(self): 22 | return f"deployment {self.deployment} not found at {self.path}" 23 | 24 | 25 | class RemoteImage: 26 | """ 27 | A simple class to represent a remote image 28 | """ 29 | 30 | def __init__( 31 | self, name, tag=None, helm_substitution_path="jupyterhub.singleuser.image" 32 | ): 33 | """ 34 | Define an Image from the hubploy config 35 | 36 | name: Fully qualified name of image 37 | tag: Tag of image (github hash) 38 | helm_substitution_path: Dot separated path in a helm file that should 39 | be populated with this image spec 40 | """ 41 | # name must not be empty 42 | # FIXME: Validate name to conform to docker image name guidelines 43 | if not name or name.strip() == "": 44 | raise ValueError( 45 | "Name of image to be built is not specified. Check " 46 | + "hubploy.yaml of your deployment" 47 | ) 48 | self.name = name 49 | self.tag = tag 50 | self.helm_substitution_path = helm_substitution_path 51 | 52 | if self.tag is None: 53 | self.image_spec = f"{self.name}" 54 | else: 55 | self.image_spec = f"{self.name}:{self.tag}" 56 | 57 | 58 | def get_config(deployment, debug=False, verbose=False): 59 | """ 60 | Returns hubploy.yaml configuration as a Python dictionary if it exists for 61 | a given deployment, and also augments it with a set of RemoteImage objects 62 | in ["images"]["images"]. 63 | """ 64 | if verbose: 65 | logger.setLevel(logging.INFO) 66 | elif debug: 67 | logger.setLevel(logging.DEBUG) 68 | 69 | deployment_path = os.path.abspath(os.path.join("deployments", deployment)) 70 | if not os.path.exists(deployment_path): 71 | raise DeploymentNotFoundError(deployment, deployment_path) 72 | 73 | config_path = os.path.join(deployment_path, "hubploy.yaml") 74 | logger.info(f"Loading hubploy config from {config_path}") 75 | with open(config_path) as f: 76 | # If config_path isn't found, this will raise a FileNotFoundError with 77 | # useful info 78 | config = yaml.load(f) 79 | 80 | if "images" in config: 81 | images_config = config["images"] 82 | 83 | # A single image is being deployed 84 | if "image_name" in images_config: 85 | if ":" in images_config["image_name"]: 86 | image_name, tag = images_config["image_name"].split(":") 87 | images = [{"name": image_name, "tag": tag}] 88 | else: 89 | images = [{"name": images_config["image_name"]}] 90 | 91 | else: 92 | # Multiple images are being deployed 93 | image_list = images_config["images"] 94 | images = [] 95 | for i in image_list: 96 | if ":" in i["name"]: 97 | image_name, tag = i["name"].split(":") 98 | logger.info(f"Tag for {image_name}: {tag}") 99 | images.append( 100 | { 101 | "name": image_name, 102 | "tag": tag, 103 | } 104 | ) 105 | else: 106 | images.append({"name": i["name"]}) 107 | 108 | config["images"]["images"] = [RemoteImage(**i) for i in images] 109 | 110 | logger.debug(f"Config loaded and parsed: {config}") 111 | return config 112 | -------------------------------------------------------------------------------- /hubploy/helm.py: -------------------------------------------------------------------------------- 1 | """ 2 | Convention based helm deploys 3 | 4 | Expects the following configuration layout from cwd: 5 | 6 | chart-name/ (Helm deployment chart) 7 | deployments/ 8 | - deployment-name 9 | - secrets/ 10 | - prod.yaml 11 | - staging.yaml 12 | - config/ 13 | - common.yaml 14 | - staging.yaml 15 | - prod.yaml 16 | 17 | Util to deploy a Helm chart (deploy) given hubploy configuration and Helm chart 18 | configuration located in accordance to hubploy conventions. 19 | """ 20 | 21 | import itertools 22 | import kubernetes.config 23 | import logging 24 | import os 25 | import subprocess 26 | 27 | from contextlib import ExitStack 28 | from kubernetes.client import CoreV1Api, rest 29 | from kubernetes.client.models import V1Namespace, V1ObjectMeta 30 | 31 | from hubploy.config import get_config 32 | from hubploy.auth import decrypt_file, cluster_auth, revert_gcloud_auth 33 | 34 | logger = logging.getLogger(__name__) 35 | HELM_EXECUTABLE = os.environ.get("HELM_EXECUTABLE", "helm") 36 | 37 | 38 | def helm_upgrade( 39 | name, 40 | namespace, 41 | context, 42 | chart, 43 | config_files, 44 | config_overrides_implicit, 45 | config_overrides_string, 46 | version, 47 | timeout, 48 | force, 49 | atomic, 50 | cleanup_on_fail, 51 | debug, 52 | verbose, 53 | helm_debug, 54 | dry_run, 55 | ): 56 | if verbose: 57 | logger.setLevel(logging.INFO) 58 | elif debug: 59 | logger.setLevel(logging.DEBUG) 60 | 61 | logger.info(f"Deploying {name} in namespace {namespace}") 62 | logger.debug(f"Running helm dep up in subdirectory '{chart}'") 63 | subprocess.check_call([HELM_EXECUTABLE, "dep", "up"], cwd=chart) 64 | 65 | # Create namespace explicitly, since helm3 removes support for it 66 | # See https://github.com/helm/helm/issues/6794 67 | # helm2 only creates the namespace if it doesn't exist, so we should be fine 68 | kubeconfig = os.environ.get("KUBECONFIG", None) 69 | logger.debug("Loading kubeconfig for k8s access") 70 | try: 71 | kubernetes.config.load_kube_config(config_file=kubeconfig, context=context) 72 | logger.info(f"Loaded kubeconfig {kubeconfig} for context {context}") 73 | except Exception as e: 74 | logger.info( 75 | f"Failed to load kubeconfig {kubeconfig} context {context} with " 76 | + f"exception:\n{e}\nTrying in-cluster config..." 77 | ) 78 | kubernetes.config.load_incluster_config() 79 | logger.info("Loaded in-cluster kubeconfig") 80 | logger.debug(f"Checking for namespace {namespace} and creating if it doesn't exist") 81 | api = CoreV1Api() 82 | try: 83 | api.read_namespace(namespace) 84 | except rest.ApiException as e: 85 | if e.status == 404: 86 | # Create namespace 87 | print(f"Namespace {namespace} does not exist, creating it...") 88 | api.create_namespace(V1Namespace(metadata=V1ObjectMeta(name=namespace))) 89 | else: 90 | raise 91 | 92 | cmd = [ 93 | HELM_EXECUTABLE, 94 | "upgrade", 95 | "--wait", 96 | "--install", 97 | "--namespace", 98 | namespace, 99 | name, 100 | chart, 101 | ] 102 | if context: 103 | cmd += ["--kube-context", context] 104 | if version: 105 | cmd += ["--version", version] 106 | if timeout: 107 | cmd += ["--timeout", timeout] 108 | if force: 109 | cmd += ["--force"] 110 | if atomic: 111 | cmd += ["--atomic"] 112 | if cleanup_on_fail: 113 | cmd += ["--cleanup-on-fail"] 114 | if helm_debug: 115 | cmd += ["--debug"] 116 | if dry_run: 117 | cmd += ["--dry-run"] 118 | cmd += itertools.chain(*[["-f", cf] for cf in config_files]) 119 | cmd += itertools.chain(*[["--set", v] for v in config_overrides_implicit]) 120 | cmd += itertools.chain(*[["--set-string", v] for v in config_overrides_string]) 121 | 122 | logger.info(f"Running helm upgrade on {name}.") 123 | logger.debug("Helm upgrade command: " + " ".join(x for x in cmd)) 124 | subprocess.check_call(cmd) 125 | 126 | 127 | def deploy( 128 | deployment, 129 | chart, 130 | environment, 131 | namespace=None, 132 | helm_config_overrides_implicit=None, 133 | helm_config_overrides_string=None, 134 | version=None, 135 | timeout=None, 136 | force=False, 137 | atomic=False, 138 | cleanup_on_fail=False, 139 | debug=False, 140 | verbose=False, 141 | helm_debug=False, 142 | dry_run=False, 143 | image_overrides=None, 144 | ): 145 | """ 146 | Deploy a JupyterHub. 147 | 148 | Expects the following files to exist in current directory 149 | 150 | {chart}/ (Helm deployment chart) 151 | deployments/ 152 | - {deployment} 153 | - secrets/ 154 | - {environment}.yaml 155 | - config/ 156 | - common.yaml 157 | - {environment}.yaml 158 | 159 | A docker image is expected to have already been built and tagged with 160 | "name" containing the full path to the repo, image name and tag. 161 | 162 | `jupyterhub.singleuser.image.tag` will be automatically set to this image 163 | tag. 164 | """ 165 | if verbose: 166 | logger.setLevel(logging.INFO) 167 | elif debug: 168 | logger.setLevel(logging.DEBUG) 169 | 170 | logger.info(f"Deploying {deployment} to {environment}") 171 | 172 | if helm_config_overrides_implicit is None: 173 | helm_config_overrides_implicit = [] 174 | if helm_config_overrides_string is None: 175 | helm_config_overrides_string = [] 176 | 177 | logger.info(f"Getting image and deployment config for {deployment}") 178 | config = get_config(deployment, debug, verbose) 179 | name = f"{deployment}-{environment}" 180 | 181 | if namespace is None: 182 | namespace = name 183 | helm_config_files = [ 184 | f 185 | for f in [ 186 | os.path.join("deployments", deployment, "config", "common.yaml"), 187 | os.path.join("deployments", deployment, "config", f"{environment}.yaml"), 188 | ] 189 | if os.path.exists(f) 190 | ] 191 | logger.debug(f"Using helm config files: {helm_config_files}") 192 | 193 | helm_secret_files = [ 194 | f 195 | for f in [ 196 | # Support for secrets in same repo 197 | os.path.join("deployments", deployment, "secrets", f"{environment}.yaml"), 198 | # Support for secrets in a submodule repo 199 | os.path.join( 200 | "secrets", "deployments", deployment, "secrets", f"{environment}.yaml" 201 | ), 202 | ] 203 | if os.path.exists(f) 204 | ] 205 | logger.debug(f"Using helm secret files: {helm_secret_files}") 206 | 207 | if config.get("images"): 208 | if image_overrides is not None: 209 | print(f"Image overrides found: {image_overrides}") 210 | num_images = len(config["images"]["images"]) 211 | num_overrides = len(image_overrides) 212 | 213 | if num_images != num_overrides: 214 | raise ValueError( 215 | f"Number of image overrides ({num_overrides}) must match " 216 | + "number of images found in " 217 | + f"deployments/{deployment}/hubploy.yaml ({num_images})" 218 | ) 219 | for override in image_overrides: 220 | if ":" not in override: 221 | raise ValueError( 222 | "Image override must be in the format " 223 | + f":. Got: {override}" 224 | ) 225 | 226 | count = 0 227 | for image in config["images"]["images"]: 228 | # We can support other charts that wrap z2jh by allowing various 229 | # config paths where we set image tags and names. 230 | # We default to one sublevel, but we can do multiple levels. 231 | if image_overrides is not None: 232 | override = image_overrides[count] 233 | override_image, override_tag = override.split(":") 234 | print( 235 | f"Overriding image {image.name}:{image.tag} to " 236 | + f"{override_image}:{override_tag}" 237 | ) 238 | image.name = override_image 239 | image.tag = override_tag 240 | 241 | if image.tag is not None: 242 | logger.info( 243 | f"Using image {image.name}:{image.tag} for " 244 | + f"{image.helm_substitution_path}" 245 | ) 246 | helm_config_overrides_string.append( 247 | f"{image.helm_substitution_path}.tag={image.tag}" 248 | ) 249 | helm_config_overrides_string.append( 250 | f"{image.helm_substitution_path}.name={image.name}" 251 | ) 252 | else: 253 | logger.info( 254 | f"Using image {image.name} for " + f"{image.helm_substitution_path}" 255 | ) 256 | helm_config_overrides_string.append( 257 | f"{image.helm_substitution_path}.name={image.name}" 258 | ) 259 | 260 | count += 1 261 | 262 | with ExitStack() as stack: 263 | # Use any specified kubeconfig context. A value of {namespace} will be 264 | # templated. A value of None will be interpreted as the current context. 265 | template_vars = dict(namespace=namespace) 266 | context = config.get("cluster", {}).get("kubeconfig", {}).get("context") 267 | if context: 268 | context = context.format(**template_vars) 269 | 270 | decrypted_secret_files = [ 271 | stack.enter_context(decrypt_file(f)) for f in helm_secret_files 272 | ] 273 | 274 | # Just in time for k8s access, activate the cluster credentials 275 | logger.debug( 276 | "Activating cluster credentials for deployment " 277 | + f"{deployment} and performing deployment upgrade." 278 | ) 279 | provider = config.get("cluster", {}).get("provider") 280 | if provider == "gcloud": 281 | current_login = stack.enter_context( 282 | cluster_auth(deployment, debug, verbose) 283 | ) 284 | else: 285 | stack.enter_context(cluster_auth(deployment, debug, verbose)) 286 | helm_upgrade( 287 | name, 288 | namespace, 289 | context, 290 | chart, 291 | helm_config_files + decrypted_secret_files, 292 | helm_config_overrides_implicit, 293 | helm_config_overrides_string, 294 | version, 295 | timeout, 296 | force, 297 | atomic, 298 | cleanup_on_fail, 299 | debug, 300 | verbose, 301 | helm_debug, 302 | dry_run, 303 | ) 304 | # Revert the gcloud auth 305 | if provider == "gcloud": 306 | stack.enter_context(revert_gcloud_auth(current_login)) 307 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | setuptools.setup( 4 | name="hubploy", 5 | version="0.4.2", 6 | url="https://github.com/berkeley-dsep-infra/hubploy", 7 | author="Yuvi Panda and Shane Knapp", 8 | packages=setuptools.find_packages(), 9 | install_requires=["kubernetes", "boto3"], 10 | python_requires=">=3.6", 11 | entry_points={ 12 | "console_scripts": [ 13 | "hubploy = hubploy.__main__:main", 14 | ], 15 | }, 16 | ) 17 | --------------------------------------------------------------------------------