├── .coveragerc ├── .dockerignore ├── .github └── stale.yml ├── .gitignore ├── .travis.yml ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── docker_interface ├── __init__.py ├── cli.py ├── docker_interface.py ├── plugins │ ├── __init__.py │ ├── base.py │ ├── build.py │ ├── google.py │ ├── python.py │ ├── run.py │ └── user.py └── util.py ├── docs ├── autodoc.py ├── conf.py ├── guide.rst ├── index.rst ├── plugins.rst └── schema.rst ├── examples ├── cython │ ├── Dockerfile │ ├── README.rst │ ├── cython_example │ │ ├── __init__.py │ │ └── speedy_code.pyx │ ├── di.yml │ ├── requirements.txt │ └── setup.py ├── env │ ├── README.rst │ ├── check_env_var.py │ └── di.yml ├── notebook │ ├── Dockerfile │ ├── README.rst │ └── di.yml └── ports │ ├── README.rst │ ├── bind_to_port.py │ └── di.yml ├── pytest.ini ├── requirements.in ├── requirements.txt ├── setup.py └── tests ├── configurations ├── comprehensive.yml ├── jupyter.yml └── plugins_list.yml ├── test_dry_run.py ├── test_examples.py ├── test_schema.py └── test_util.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [report] 2 | # Regexes for lines to exclude from consideration 3 | exclude_lines = 4 | # Have to re-enable the standard pragma 5 | pragma: no cover 6 | 7 | # Don't complain if tests don't raise exceptions 8 | raise 9 | 10 | # Don't complain about representations not being covered 11 | __repr__ 12 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | **/__pycache__ 2 | -------------------------------------------------------------------------------- /.github/stale.yml: -------------------------------------------------------------------------------- 1 | # Configuration for probot-stale - https://github.com/probot/stale 2 | 3 | # Number of days of inactivity before an Issue or Pull Request becomes stale 4 | daysUntilStale: 60 5 | # Number of days of inactivity before a stale Issue or Pull Request is closed 6 | daysUntilClose: 7 7 | # Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable 8 | exemptLabels: 9 | - pinned 10 | - security 11 | - "[Status] Maybe Later" 12 | # Label to use when marking as stale 13 | staleLabel: wontfix 14 | # Comment to post when marking as stale. Set to `false` to disable 15 | markComment: > 16 | This issue has been automatically marked as stale because it has not had 17 | recent activity. It will be closed if no further activity occurs. Thank you 18 | for your contributions. 19 | # Comment to post when removing the stale label. Set to `false` to disable 20 | unmarkComment: false 21 | # Comment to post when closing a stale Issue or Pull Request. Set to `false` to disable 22 | closeComment: false 23 | # Limit to only `issues` or `pulls` 24 | # only: issues 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | .static_storage/ 56 | .media/ 57 | local_settings.py 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | playground/ 107 | .vscode/ 108 | .DS_Store 109 | 110 | docs/plugin_reference.rst 111 | docs/schema.json 112 | docs/examples.rst 113 | /di.yml 114 | 115 | .pytest_cache 116 | 117 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | services: 3 | - docker 4 | language: python 5 | python: 6 | - '3.6' 7 | cache: pip 8 | install: 9 | - pip install -r requirements.txt 10 | script: 11 | - make tests 12 | - make html 13 | deploy: 14 | provider: pypi 15 | skip_existing: true 16 | user: tillahoffmann 17 | password: 18 | secure: yO2h21yzPnHFyLETmvlLuBrRWPmR0HWAPZ6gJyOkf72Q6G2/ZKDd2GaF28kCE8I8hg64o4w96WcNgdbdaleUJkAT2aIRiNG3hepJ/oE8rQe7D8XK99OdOcE3PdaOZFQh0KenjjylJw6/ITXufZ6xofjLKPmVx0mvb/QUeqO6hBWUtCo826xVqAE0ls/fhT8VPq+tRKuk4DsPuINJhTrvz582+AdjZW5OJg3Bh+MkfwVdAQpzy9QM9sX9txyV4sWGy64TUvPyOcKQGtQC2MxIj8MsbYDY484sThgIvd3nMRyluspcIN8vimwUT0IDFaMEn8b/yRoTdc43nIFjP0gDbwpctwETL5E6TIvLE2gkVNLibXZBqZrXN3m1oV+sg9YZScSQJoj1esh74l4BAv+1r6Pf82iiX0F8vYAk1Jfs0uCI3Y4fng0D9iJtciYIuNzOZ+MmMoQ8sy5swN9BVvzYY+YsjwE2mx/vpoNTF0G99NcJYXLnIdnRTWHkiYjJPaUzG5Rh/oVl3ySskl9EJAVnEY05i/lemk7uEW9wkeCvES2dEiiNwqNn+6pW2hK0/6lPFHIW7Ji8bL58i+YK6S5yUPU1OVSFuAAtX4QPo6U5GyZN746UdUfcPRtN15RYWaKF/YRVz/93/X1KkVNLigUCJPCBvsvC03HMfZxEOmtt99Q= 19 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3 2 | 3 | RUN apt-get update \ 4 | && apt-get install zip -y --no-install-recommends \ 5 | && rm -rf /var/lib/apt/lists/* 6 | 7 | WORKDIR /workspace 8 | COPY requirements.txt . 9 | RUN pip install -r requirements.txt --no-cache-dir 10 | COPY . . 11 | RUN pip install -e . 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # You can set these variables from the command line. 2 | SPHINXOPTS = 3 | SPHINXBUILD = sphinx-build 4 | SPHINXPROJ = docker_interface 5 | SOURCEDIR = docs 6 | BUILDDIR = docs/_build 7 | 8 | all : tests html 9 | 10 | tests : code_tests 11 | 12 | code_tests : 13 | pytest --cov docker_interface --cov-report=html --cov-report=term-missing -v --durations=10 -s 14 | 15 | help: 16 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 17 | 18 | # Catch-all target: route all unknown targets to Sphinx using the new 19 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 20 | html : Makefile 21 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 22 | 23 | clean : 24 | rm -rf docs/_build 25 | 26 | .PHONY: help Makefile docs/plugin_reference.rst clean tests code_tests 27 | 28 | sdist : 29 | python setup.py sdist 30 | 31 | testpypi : sdist 32 | twine upload --repository-url https://test.pypi.org/legacy/dist/docker-interface-* 33 | 34 | pypi : sdist 35 | twine upload dist/docker_interface-* 36 | 37 | requirements.txt : requirements.in setup.py 38 | pip-compile --upgrade $< 39 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Docker Interface [![Build Status](https://travis-ci.org/spotify/docker_interface.svg?branch=master)](https://travis-ci.org/spotify/docker_interface) ![Development Status](https://img.shields.io/badge/status-beta-orange.svg) 2 | 3 | Docker Interface (DI) is a declarative interface for building images and running commands in containers using Docker. At Spotify, we use Docker Interface to minimise environment drift by running all of our code in containers–during development, production, or to train machine learning models. 4 | 5 | ## Installing Docker Interface 6 | 7 | You can install Docker Interface using the following `pip` command (you need a python3 interpreter). 8 | 9 | ``` 10 | pip install docker-interface 11 | ``` 12 | 13 | To check that Docker Interface was installed successfully, run 14 | ``` 15 | di --help 16 | ``` 17 | 18 | ## Using Docker Interface 19 | 20 | Docker Interface can be invoked from the command line. By default, it reads the configuration from the file `di.yml` in the current working directory, a basic version of which is shown below. 21 | 22 | ```yaml 23 | build: 24 | tag: name-of-your-image 25 | ``` 26 | 27 | Docker interface supports two commands: 28 | 29 | * `build` builds and tags Docker image using the current working directory as the build context. 30 | * `run` runs a Docker command in a container and mounts the current working directory with appropriate permissions at `/workspace` so you can access your local files without having to rebuild the image. 31 | 32 | You can find more extensive examples in the [`examples` folder](https://github.com/spotify/docker_interface/tree/master/examples) in this repository. You can find more detailed information [here](http://docker-interface.readthedocs.io/en/latest/). Check the [schema](http://docker-interface.readthedocs.io/en/latest/schema.html) to get a comprehensive overview of the declarative syntax supported by Docker Interface. 33 | 34 | ## Contributing to Docker Interface 35 | 36 | To contribute to the development of Docker Interface, please create a [fork](https://help.github.com/articles/fork-a-repo/) of the repository and send any changes as a pull request. 37 | 38 | You can test your local installation of Docker Interface as follows. 39 | 40 | ``` 41 | # 0. Set up a virtual environment (optional but recommended) 42 | # 1. Install development requirements 43 | pip install -r requirements.txt 44 | # 2. Run the tests 45 | make tests 46 | ``` 47 | 48 | See [`virtualenv`](https://virtualenv.pypa.io/en/stable/) or [`conda`](https://conda.io/docs/) for details on how to set up a virtual environment in step 0. 49 | 50 | ## Code of conduct 51 | 52 | This project adheres to the [Open Code of Conduct](https://github.com/spotify/code-of-conduct/blob/master/code-of-conduct.md). By participating, you are expected to honour this code. 53 | -------------------------------------------------------------------------------- /docker_interface/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Spotify AB 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /docker_interface/cli.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Spotify AB 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import argparse 16 | import json 17 | import logging 18 | 19 | from .plugins import Plugin, BasePlugin 20 | from . import util 21 | 22 | 23 | def entry_point(args=None, configuration=None): 24 | """ 25 | Standard entry point for the docker interface CLI. 26 | 27 | Parameters 28 | ---------- 29 | args : list or None 30 | list of command line arguments or `None` to use `sys.argv` 31 | configuration : dict 32 | parsed configuration or `None` to load and build a configuration given the command line 33 | arguments 34 | 35 | Raises 36 | ------ 37 | SystemExit 38 | if the configuration is malformed or the docker subprocesses returns a non-zero status code 39 | """ 40 | # Parse basic information 41 | parser = argparse.ArgumentParser('di') 42 | base = BasePlugin() 43 | base.add_arguments(parser) 44 | args, remainder = parser.parse_known_args(args) 45 | command = args.command 46 | configuration = base.apply(configuration, None, args) 47 | 48 | logger = logging.getLogger('di') 49 | 50 | # Load all plugins and en/disable as desired 51 | plugin_cls = Plugin.load_plugins() 52 | plugins = configuration.get('plugins') 53 | if isinstance(plugins, list): 54 | plugins = [plugin_cls[name.lower()] for name in plugins] 55 | else: 56 | # Disable and enable specific plugins 57 | if isinstance(plugins, dict): 58 | try: 59 | for name in plugins.get('enable', []): 60 | plugin_cls[name.lower()].ENABLED = True 61 | for name in plugins.get('disable', []): 62 | plugin_cls[name.lower()].ENABLED = False 63 | except KeyError as ex: # pragma: no cover 64 | logger.fatal("could not resolve plugin %s. Available plugins: %s", 65 | ex, ", ".join(plugin_cls)) 66 | raise SystemExit(2) 67 | elif plugins is not None: # pragma: no cover 68 | logger.fatal("'plugins' must be a `list`, `dict`, or `None` but got `%s`", 69 | type(plugins)) 70 | raise SystemExit(2) 71 | 72 | # Restrict plugins to enabled ones 73 | plugins = list(sorted([cls() for cls in plugin_cls.values() if cls.ENABLED], 74 | key=lambda x: x.ORDER)) 75 | 76 | # Construct the schema 77 | schema = base.SCHEMA 78 | for cls in plugin_cls.values(): 79 | schema = util.merge(schema, cls.SCHEMA) 80 | 81 | # Ensure that the plugins are relevant to the command 82 | plugins = [plugin for plugin in plugins 83 | if plugin.COMMANDS == 'all' or command in plugin.COMMANDS] 84 | parser = argparse.ArgumentParser('di %s' % command) 85 | for plugin in plugins: 86 | plugin.add_arguments(parser) 87 | args = parser.parse_args(remainder) 88 | 89 | # Apply defaults 90 | util.set_default_from_schema(configuration, schema) 91 | 92 | # Apply all the plugins in order 93 | status_code = 0 94 | logger.debug("configuration:\n%s", json.dumps(configuration, indent=4)) 95 | for plugin in plugins: 96 | logger.debug("applying plugin '%s'", plugin) 97 | try: 98 | configuration = plugin.apply(configuration, schema, args) 99 | assert configuration is not None, "plugin '%s' returned `None`" % plugin 100 | except Exception as ex: # pragma: no cover 101 | logger.exception("failed to apply plugin '%s': %s", plugin, ex) 102 | message = "please rerun the command using `di --log-level debug` and file a new " \ 103 | "issue containing the output of the command here: https://github.com/" \ 104 | "spotify/docker_interface/issues/new" 105 | logger.fatal("\033[%dm%s\033[0m", 31, message) 106 | status_code = 3 107 | break 108 | logger.debug("configuration:\n%s", json.dumps(configuration, indent=4)) 109 | 110 | for plugin in reversed(plugins): 111 | logger.debug("tearing down plugin '%s'", plugin) 112 | plugin.cleanup() 113 | 114 | status_code = configuration.get('status-code', status_code) 115 | if status_code: 116 | raise SystemExit(status_code) 117 | -------------------------------------------------------------------------------- /docker_interface/docker_interface.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Spotify AB 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | 17 | 18 | def build_parameter_parts(configuration, *parameters): 19 | """ 20 | Construct command parts for one or more parameters. 21 | 22 | Parameters 23 | ---------- 24 | configuration : dict 25 | configuration 26 | parameters : list 27 | list of parameters to create command line arguments for 28 | 29 | Yields 30 | ------ 31 | argument : str 32 | command line argument 33 | """ 34 | for parameter in parameters: 35 | values = configuration.pop(parameter, []) 36 | if values: 37 | if not isinstance(values, list): 38 | values = [values] 39 | for value in values: 40 | yield '--%s=%s' % (parameter, value) 41 | 42 | 43 | def build_dict_parameter_parts(configuration, *parameters, **defaults): 44 | """ 45 | Construct command parts for one or more parameters, each of which constitutes an assignment of 46 | the form `key=value`. 47 | 48 | Parameters 49 | ---------- 50 | configuration : dict 51 | configuration 52 | parameters : list 53 | list of parameters to create command line arguments for 54 | defaults : dict 55 | default values to use if a parameter is missing 56 | 57 | Yields 58 | ------ 59 | argument : str 60 | command line argument 61 | """ 62 | for parameter in parameters: 63 | for key, value in configuration.pop(parameter, {}).items(): 64 | yield '--%s=%s=%s' % (parameter, key, value) 65 | 66 | 67 | def build_docker_run_command(configuration): 68 | """ 69 | Translate a declarative docker `configuration` to a `docker run` command. 70 | 71 | Parameters 72 | ---------- 73 | configuration : dict 74 | configuration 75 | 76 | Returns 77 | ------- 78 | args : list 79 | sequence of command line arguments to run a command in a container 80 | """ 81 | parts = configuration.pop('docker').split() 82 | parts.append('run') 83 | 84 | run = configuration.pop('run') 85 | 86 | # Ensure all env-files have proper paths 87 | if 'env-file' in run: 88 | run['env-file'] = [os.path.join(configuration['workspace'], env_file) 89 | for env_file in run['env-file']] 90 | 91 | parts.extend(build_parameter_parts( 92 | run, 'user', 'workdir', 'rm', 'interactive', 'tty', 'env-file', 'cpu-shares', 'name', 93 | 'network', 'label', 'memory', 'entrypoint', 'runtime', 'privileged', 'group-add', 'gpus' 94 | )) 95 | 96 | # Add the mounts 97 | # The following code requires docker >= 17.06 98 | '''for mount in run.pop('mount', []): 99 | if mount['type'] == 'bind': 100 | mount['source'] = os.path.join( 101 | configuration['workspace'], mount['source']) 102 | parts.extend(['--mount', ",".join(["%s=%s" % item for item in mount.items()])])''' 103 | 104 | # Add the mounts 105 | for mount in run.pop('mount', []): 106 | if mount['type'] == 'tmpfs': 107 | raise RuntimeError('tmpfs-mounts are currently not supported via the mount ' + 108 | 'directive in docker_interface. Consider using the tmpfs ' + 109 | 'directive instead.') 110 | if mount['type'] == 'bind': 111 | mount['source'] = os.path.abspath( 112 | os.path.join(configuration['workspace'], mount['source'])) 113 | vol_config = '--volume=%s:%s' % (mount['source'], mount['destination']) 114 | if 'readonly' in mount and mount['readonly']: 115 | vol_config += ':ro' 116 | parts.append(vol_config) 117 | 118 | # Set or forward environment variables 119 | for key, value in run.pop('env', {}).items(): 120 | if value is None: 121 | parts.append('--env=%s' % key) 122 | else: 123 | parts.append('--env=%s=%s' % (key, value)) 124 | parts.append('--env=DOCKER_INTERFACE=true') 125 | 126 | # Forward ports 127 | for publish in run.pop('publish', []): 128 | parts.append('--publish=%s:%s:%s' % tuple([ 129 | publish.get(key, '') for key in "ip host container".split()])) 130 | 131 | # Add temporary file systems 132 | for tmpfs in run.pop('tmpfs', []): 133 | destination = tmpfs['destination'] 134 | options = tmpfs.pop('options', []) 135 | for key in ['mode', 'size']: 136 | if key in tmpfs: 137 | options.append('%s=%s' % (key, tmpfs[key])) 138 | if options: 139 | destination = "%s:%s" % (destination, ",".join(options)) 140 | parts.extend(['--tmpfs', destination]) 141 | 142 | parts.append(run.pop('image')) 143 | parts.extend(run.pop('cmd', [])) 144 | 145 | return parts 146 | 147 | 148 | def build_docker_build_command(configuration): 149 | """ 150 | Translate a declarative docker `configuration` to a `docker build` command. 151 | 152 | Parameters 153 | ---------- 154 | configuration : dict 155 | configuration 156 | 157 | Returns 158 | ------- 159 | args : list 160 | sequence of command line arguments to build an image 161 | """ 162 | parts = configuration.pop('docker', 'docker').split() 163 | parts.append('build') 164 | 165 | build = configuration.pop('build') 166 | 167 | build['path'] = os.path.join(configuration['workspace'], build['path']) 168 | build['file'] = os.path.join(build['path'], build['file']) 169 | 170 | parts.extend(build_parameter_parts( 171 | build, 'tag', 'file', 'no-cache', 'quiet', 'cpu-shares', 'memory')) 172 | 173 | parts.extend(build_dict_parameter_parts(build, 'build-arg')) 174 | parts.append(build.pop('path')) 175 | 176 | return parts 177 | -------------------------------------------------------------------------------- /docker_interface/plugins/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Spotify AB 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .base import Plugin, BasePlugin, HomeDirPlugin, SubstitutionPlugin, WorkspaceMountPlugin, \ 16 | ValidationPlugin, ExecutePlugin 17 | from .user import UserPlugin 18 | from .run import RunPlugin, RunConfigurationPlugin 19 | from .build import BuildPlugin, BuildConfigurationPlugin 20 | from .python import JupyterPlugin 21 | from .google import GoogleCloudCredentialsPlugin, GoogleContainerRegistryPlugin 22 | -------------------------------------------------------------------------------- /docker_interface/plugins/base.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Spotify AB 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import argparse 16 | import functools as ft 17 | import itertools as it 18 | import logging 19 | import os 20 | import re 21 | 22 | import jsonschema 23 | import pkg_resources 24 | import yaml 25 | 26 | from .. import util 27 | 28 | 29 | class Plugin: 30 | """ 31 | Abstract base class for plugins. 32 | """ 33 | ENABLED = True 34 | SCHEMA = {} 35 | ORDER = None 36 | COMMANDS = None 37 | 38 | def __init__(self): 39 | self.logger = logging.getLogger(self.__class__.__name__) 40 | self.arguments = {} 41 | 42 | def add_argument(self, parser, path, name=None, schema=None, **kwargs): 43 | """ 44 | Add an argument to the `parser` based on a schema definition. 45 | 46 | Parameters 47 | ---------- 48 | parser : argparse.ArgumentParser 49 | parser to add an argument to 50 | path : str 51 | path in the configuration document to add an argument for 52 | name : str or None 53 | name of the command line parameter (defaults to the name in the schema) 54 | schema : dict 55 | JSON schema definition (defaults to the schema of the plugin) 56 | 57 | Returns 58 | ------- 59 | arg : 60 | command line argument definition 61 | """ 62 | schema = schema or self.SCHEMA 63 | name = name or ('--%s' % os.path.basename(path)) 64 | self.arguments[name.strip('-')] = path 65 | # Build a path to the help in the schema 66 | path = util.split_path(path) 67 | path = os.path.sep.join( 68 | it.chain([os.path.sep], *zip(it.repeat("properties"), path))) 69 | property_ = util.get_value(schema, path) 70 | kwargs.setdefault('choices', property_.get('enum')) 71 | kwargs.setdefault('help', property_.get('description')) 72 | type_ = property_.get('type') 73 | if type_: 74 | kwargs.setdefault('type', util.TYPES[type_]) 75 | return parser.add_argument(name, **kwargs) 76 | 77 | def add_arguments(self, parser): 78 | """ 79 | Add arguments to the parser. 80 | 81 | Inheriting plugins should implement this method to add parameters to the command line 82 | parser. 83 | 84 | Parameters 85 | ---------- 86 | parser : argparse.ArgumentParser 87 | parser to add arguments to 88 | """ 89 | pass 90 | 91 | def apply(self, configuration, schema, args): 92 | """ 93 | Apply the plugin to the configuration. 94 | 95 | Inheriting plugins should implement this method to add additional functionality. 96 | 97 | Parameters 98 | ---------- 99 | configuration : dict 100 | configuration 101 | schema : dict 102 | JSON schema 103 | args : argparse.NameSpace 104 | parsed command line arguments 105 | 106 | Returns 107 | ------- 108 | configuration : dict 109 | updated configuration after applying the plugin 110 | """ 111 | # Set values from the command line 112 | for name, path in self.arguments.items(): 113 | value = getattr(args, name.replace('-', '_')) 114 | if value is not None: 115 | util.set_value(configuration, path, value) 116 | 117 | return configuration 118 | 119 | @staticmethod 120 | def load_plugins(): 121 | """ 122 | Load all availabe plugins. 123 | 124 | Returns 125 | ------- 126 | plugin_cls : dict 127 | mapping from plugin names to plugin classes 128 | """ 129 | plugin_cls = {} 130 | for entry_point in pkg_resources.iter_entry_points('docker_interface.plugins'): 131 | cls = entry_point.load() 132 | assert cls.COMMANDS is not None, \ 133 | "plugin '%s' does not define its commands" % entry_point.name 134 | assert cls.ORDER is not None, \ 135 | "plugin '%s' does not define its priority" % entry_point.name 136 | plugin_cls[entry_point.name] = cls 137 | return plugin_cls 138 | 139 | def cleanup(self): 140 | """ 141 | Tear down the plugin and clean up any resources used. 142 | 143 | Inheriting plugins should implement this method to add additional functionality. 144 | """ 145 | pass 146 | 147 | class ValidationPlugin(Plugin): 148 | """ 149 | Validate the configuration document. 150 | """ 151 | COMMANDS = 'all' 152 | ORDER = 990 153 | 154 | def apply(self, configuration, schema, args): 155 | super(ValidationPlugin, self).apply(configuration, schema, args) 156 | validator = jsonschema.validators.validator_for(schema)(schema) 157 | errors = list(validator.iter_errors(configuration)) 158 | if errors: # pragma: no cover 159 | for error in errors: 160 | self.logger.fatal(error.message) 161 | raise ValueError("failed to validate configuration") 162 | return configuration 163 | 164 | 165 | class ExecutePlugin(Plugin): 166 | """ 167 | Base class for plugins that execute shell commands. 168 | 169 | Inheriting classes should define the method :code:`build_command` which takes a configuration 170 | document as its only argument. 171 | """ 172 | def build_command(self, configuration): 173 | """ 174 | Construct a command and return its parts. 175 | 176 | Parameters 177 | ---------- 178 | configuration : dict 179 | configuration 180 | 181 | Returns 182 | ------- 183 | args : list 184 | sequence of command line arguments 185 | """ 186 | raise NotImplementedError 187 | 188 | def apply(self, configuration, schema, args): 189 | super(ExecutePlugin, self).apply(configuration, schema, args) 190 | parts = self.build_command(configuration) 191 | if parts: 192 | configuration['status-code'] = self.execute_command(parts, configuration['dry-run']) 193 | else: 194 | configuration['status-code'] = 0 195 | return configuration 196 | 197 | def execute_command(self, parts, dry_run): 198 | """ 199 | Execute a command. 200 | 201 | Parameters 202 | ---------- 203 | parts : list 204 | Sequence of strings constituting a command. 205 | dry_run : bool 206 | Whether to just log the command instead of executing it. 207 | 208 | Returns 209 | ------- 210 | status : int 211 | Status code of the executed command or 0 if `dry_run` is `True`. 212 | """ 213 | if dry_run: 214 | self.logger.info("dry-run command '%s'", " ".join(map(str, parts))) 215 | return 0 216 | else: # pragma: no cover 217 | self.logger.debug("executing command '%s'", " ".join(map(str, parts))) 218 | status_code = os.spawnvpe(os.P_WAIT, parts[0], parts, os.environ) 219 | if status_code: 220 | self.logger.warning("command '%s' returned status code %d", 221 | " ".join(map(str, parts)), status_code) 222 | return status_code 223 | 224 | 225 | class BasePlugin(Plugin): 226 | """ 227 | Load or create a default configuration and set up logging. 228 | """ 229 | SCHEMA = { 230 | "title": "Declarative Docker Interface (DI) definition.", 231 | "$schema": "http://json-schema.org/draft-04/schema", 232 | "additionalProperties": False, 233 | "required": ["workspace", "docker"], 234 | "properties": { 235 | "workspace": { 236 | "type": "string", 237 | "description": "Path defining the DI workspace (absolute or relative to the URI of this document). All subsequent path definitions must be absolute or relative to the `workspace`." 238 | }, 239 | "docker": { 240 | "type": "string", 241 | "description": "Name of the docker CLI.", 242 | "default": "docker" 243 | }, 244 | "log-level": { 245 | "type": "string", 246 | "enum": ["debug", "info", "warning", "error", "critical", "fatal"], 247 | "default": "info" 248 | }, 249 | "dry-run": { 250 | "type": "boolean", 251 | "description": "Whether to just construct the docker command.", 252 | "default": False 253 | }, 254 | "status-code": { 255 | "type": "integer", 256 | "description": "status code returned by docker" 257 | }, 258 | "plugins": { 259 | "oneOf": [ 260 | { 261 | "type": "array", 262 | "description": "Enable the listed plugins and disable all plugins not listed.", 263 | "items": { 264 | "type": "string" 265 | } 266 | }, 267 | { 268 | "type": "object", 269 | "properties": { 270 | "enable": { 271 | "type": "array", 272 | "description": "Enable the listed plugins.", 273 | "items": { 274 | "type": "string" 275 | } 276 | }, 277 | "disable": { 278 | "type": "array", 279 | "description": "Disable the listed plugins.", 280 | "items": { 281 | "type": "string" 282 | } 283 | } 284 | }, 285 | "additionalProperties": False 286 | } 287 | ] 288 | } 289 | } 290 | } 291 | 292 | def add_arguments(self, parser): 293 | parser.add_argument('--file', '-f', help='Configuration file.', default='di.yml') 294 | self.add_argument(parser, '/workspace') 295 | self.add_argument(parser, '/docker') 296 | self.add_argument(parser, '/log-level') 297 | self.add_argument(parser, '/dry-run') 298 | parser.add_argument('command', help='Docker interface command to execute.', 299 | choices=['run', 'build']) 300 | 301 | def apply(self, configuration, schema, args): 302 | # Load the configuration 303 | if configuration is None and os.path.isfile(args.file): 304 | filename = os.path.abspath(args.file) 305 | with open(filename) as fp: # pylint: disable=invalid-name 306 | configuration = yaml.safe_load(fp) 307 | self.logger.debug("loaded configuration from '%s'", filename) 308 | dirname = os.path.dirname(filename) 309 | configuration['workspace'] = os.path.join(dirname, configuration.get('workspace', '.')) 310 | elif configuration is None: 311 | raise FileNotFoundError( 312 | "missing configuration; could not find configuration file '%s'" % args.file) 313 | 314 | configuration = super(BasePlugin, self).apply(configuration, schema, args) 315 | 316 | logging.basicConfig(level=configuration.get('log-level', 'info').upper()) 317 | return configuration 318 | 319 | 320 | class SubstitutionPlugin(Plugin): 321 | """ 322 | Substitute variables in strings. 323 | 324 | String values in the configuration document may 325 | 326 | * reference other parts of the configuration document using :code:`#{path}`, where :code:`path` 327 | may be an absolute or relative path in the document. 328 | * reference a variable using :code:`${path}`, where :code:`path` is assumed to be an absolute 329 | path in the :code:`VARIABLES` class attribute of the plugin. 330 | 331 | By default, the plugin provides environment variables using the :code:`env` prefix. For example, 332 | a value could reference the user name on the host using :code:`${env/USER}`. Other plugins can 333 | provide variables for substitution by extending the :code:`VARIABLES` class attribute and should 334 | do so using a unique prefix. 335 | """ 336 | REF_PATTERN = re.compile(r'#\{(?P.*?)\}') 337 | VAR_PATTERN = re.compile(r'\$\{(?P.*?)\}') 338 | COMMANDS = 'all' 339 | ORDER = 980 340 | VARIABLES = { 341 | 'env': dict(os.environ) 342 | } 343 | 344 | @classmethod 345 | def substitute_variables(cls, configuration, value, ref): 346 | """ 347 | Substitute variables in `value` from `configuration` where any path reference is relative to 348 | `ref`. 349 | 350 | Parameters 351 | ---------- 352 | configuration : dict 353 | configuration (required to resolve intra-document references) 354 | value : 355 | value to resolve substitutions for 356 | ref : str 357 | path to `value` in the `configuration` 358 | 359 | Returns 360 | ------- 361 | value : 362 | value after substitution 363 | """ 364 | if isinstance(value, str): 365 | # Substitute all intra-document references 366 | while True: 367 | match = cls.REF_PATTERN.search(value) 368 | if match is None: 369 | break 370 | path = os.path.join(os.path.dirname(ref), match.group('path')) 371 | try: 372 | value = value.replace( 373 | match.group(0), str(util.get_value(configuration, path))) 374 | except KeyError: 375 | raise KeyError(path) 376 | 377 | # Substitute all variable references 378 | while True: 379 | match = cls.VAR_PATTERN.search(value) 380 | if match is None: 381 | break 382 | value = value.replace( 383 | match.group(0), 384 | str(util.get_value(cls.VARIABLES, match.group('path'), '/'))) 385 | return value 386 | 387 | def apply(self, configuration, schema, args): 388 | super(SubstitutionPlugin, self).apply(configuration, schema, args) 389 | return util.apply(configuration, ft.partial(self.substitute_variables, configuration)) 390 | 391 | 392 | class WorkspaceMountPlugin(Plugin): 393 | """ 394 | Mount the workspace inside the container. 395 | """ 396 | SCHEMA = { 397 | "properties": { 398 | "run": { 399 | "properties": { 400 | "workspace-dir": { 401 | "type": "string", 402 | "description": "Path at which to mount the workspace in the container.", 403 | "default": "/workspace" 404 | }, 405 | "workdir": { 406 | "type": "string", 407 | "default": "#{workspace-dir}" 408 | } 409 | }, 410 | "additionalProperties": False 411 | } 412 | }, 413 | "additionalProperties": False 414 | } 415 | COMMANDS = ['run'] 416 | ORDER = 500 417 | 418 | def add_arguments(self, parser): 419 | self.add_argument(parser, '/run/workspace-dir') 420 | self.add_argument(parser, '/run/workdir') 421 | 422 | def apply(self, configuration, schema, args): 423 | super(WorkspaceMountPlugin, self).apply(configuration, schema, args) 424 | configuration['run'].setdefault('mount', []).append({ 425 | 'type': 'bind', 426 | 'source': '#{/workspace}', 427 | 'destination': util.get_value(configuration, '/run/workspace-dir') 428 | }) 429 | return configuration 430 | 431 | 432 | class HomeDirPlugin(Plugin): 433 | """ 434 | Mount a home directory placed in the current directory. 435 | """ 436 | ORDER = 520 437 | COMMANDS = ['run'] 438 | 439 | def apply(self, configuration, schema, args): 440 | super(HomeDirPlugin, self).apply(configuration, schema, args) 441 | configuration['run'].setdefault('mount', []).append({ 442 | 'destination': '#{/run/env/HOME}', 443 | 'source': '#{/workspace}/.di/home', 444 | 'type': 'bind', 445 | }) 446 | configuration['run'].setdefault('env', {}).setdefault('HOME', '/${user/name}') 447 | return configuration 448 | -------------------------------------------------------------------------------- /docker_interface/plugins/build.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Spotify AB 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .base import Plugin, ExecutePlugin 16 | from ..docker_interface import build_docker_build_command 17 | 18 | 19 | class BuildPlugin(ExecutePlugin): 20 | """ 21 | Build a docker image. 22 | """ 23 | COMMANDS = ['build'] 24 | ORDER = 1000 25 | build_command = staticmethod(build_docker_build_command) 26 | 27 | 28 | class BuildConfigurationPlugin(Plugin): 29 | """ 30 | Configure how to build a docker image. 31 | """ 32 | COMMANDS = ['build'] 33 | ORDER = 950 34 | SCHEMA = { 35 | "properties": { 36 | "build": { 37 | "properties": { 38 | "path": { 39 | "type": "string", 40 | "description": "Path of the build context.", 41 | "default": "#{/workspace}" 42 | }, 43 | "tag": { 44 | "type": "string", 45 | "description": "Name and optionally a tag in the 'name:tag' format.", 46 | "default": "docker-interface-image" 47 | }, 48 | "file": { 49 | "type": "string", 50 | "description": "Name of the Dockerfile.", 51 | "default": "#{path}/Dockerfile" 52 | }, 53 | "build-arg": { 54 | "type": "object", 55 | "description": "Set build-time variables.", 56 | "additionalProperties": { 57 | "type": "string" 58 | } 59 | }, 60 | "no-cache": { 61 | "type": "boolean", 62 | "description": "Do not use cache when building the image" 63 | }, 64 | "quiet": { 65 | "type": "boolean", 66 | "description": "Suppress the build output and print image ID on success" 67 | }, 68 | "cpu-shares": { 69 | "type": "integer", 70 | "description": "CPU shares (relative weight)", 71 | "minimum": 0, 72 | "maximum": 1024 73 | }, 74 | "memory": { 75 | "type": "string", 76 | "description": "Memory limit" 77 | } 78 | }, 79 | "required": [ 80 | "tag", 81 | "path", 82 | "file" 83 | ], 84 | "additionalProperties": False 85 | }, 86 | }, 87 | "additionalProperties": False 88 | } 89 | -------------------------------------------------------------------------------- /docker_interface/plugins/google.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Spotify AB 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import contextlib 16 | import datetime 17 | import os 18 | import sqlite3 19 | from .base import Plugin, ExecutePlugin 20 | 21 | 22 | class GoogleCloudCredentialsPlugin(Plugin): 23 | """ 24 | Mount Google Cloud credentials in the Docker container. 25 | """ 26 | ORDER = 560 27 | COMMANDS = ['run'] 28 | 29 | def apply(self, configuration, schema, args): 30 | configuration['run'].setdefault('mount', []).append({ 31 | 'type': 'bind', 32 | 'source': '${env/HOME}/.config/gcloud', 33 | 'destination': '#{/run/env/HOME}/.config/gcloud' 34 | }) 35 | return configuration 36 | 37 | 38 | class GoogleContainerRegistryPlugin(ExecutePlugin): 39 | """ 40 | Configure docker authorization for Google services such as Google Container Registry. 41 | """ 42 | # We want to authorize before any other plugins that may depend on access to Google's services. 43 | ORDER = 10 44 | COMMANDS = 'all' 45 | ENABLED = False 46 | 47 | def build_command(self, configuration): 48 | filename = os.path.expanduser('~/.config/gcloud/access_tokens.db') 49 | if os.path.isfile(filename): 50 | with contextlib.closing(sqlite3.connect(filename, detect_types=sqlite3.PARSE_DECLTYPES)) as conn, \ 51 | contextlib.closing(conn.cursor()) as cursor: 52 | cursor.execute("SELECT token_expiry FROM access_tokens") 53 | token_expiry, = cursor.fetchone() 54 | if token_expiry > datetime.datetime.now() + datetime.timedelta(seconds=30): 55 | self.logger.debug('skipping gcr.io authentication; token is valid until %s', 56 | token_expiry) 57 | return None 58 | return ['gcloud', 'docker', '--authorize-only', '--quiet'] 59 | -------------------------------------------------------------------------------- /docker_interface/plugins/python.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Spotify AB 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | import socket 17 | import uuid 18 | 19 | from .base import Plugin 20 | from ..util import get_free_port 21 | 22 | 23 | class JupyterPlugin(Plugin): 24 | """ 25 | Forward the port required by Jupyter Notebook to the host machine and print a URL for easily 26 | accessing the notebook server. 27 | """ 28 | ORDER = 960 29 | COMMANDS = ['run'] 30 | 31 | def apply(self, configuration, schema, args): 32 | cmd = configuration.setdefault('run', {}).get('cmd', []) 33 | # Check whether the user is starting a notebook 34 | if cmd and cmd[0] == 'jupyter' and cmd[1] in ('notebook', 'lab'): 35 | # Don't try to start a browser 36 | if '--no-browser' not in cmd: 37 | cmd.append('--no-browser') 38 | # Open the standard port for the notebook 39 | free_port = get_free_port(range(8888, 9999)) 40 | configuration['run'].setdefault('publish', []).append({ 41 | 'container': 8888, 42 | 'host': free_port, 43 | }) 44 | 45 | if not any([x.startswith('--NotebookApp.token=') for x in cmd]): 46 | token = uuid.uuid4().hex 47 | cmd.append("--NotebookApp.token='%s'" % token) 48 | 49 | self.logger.info( 50 | "containerized notebook server will be available at http://%s:%d?token=%s", 51 | socket.gethostname(), free_port, token) 52 | 53 | if not any(x.startswith('--ip') for x in cmd): 54 | cmd.append('--ip=0.0.0.0') 55 | 56 | return configuration 57 | -------------------------------------------------------------------------------- /docker_interface/plugins/run.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Spotify AB 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import argparse 16 | import sys 17 | from ..docker_interface import build_docker_run_command 18 | from .. import util 19 | from .base import Plugin, ExecutePlugin 20 | 21 | 22 | class RunPlugin(ExecutePlugin): 23 | """ 24 | Run a command inside a docker container. 25 | """ 26 | COMMANDS = ['run'] 27 | ORDER = 1000 28 | build_command = staticmethod(build_docker_run_command) 29 | 30 | 31 | class RunConfigurationPlugin(Plugin): 32 | """ 33 | Configure how to run a command inside a docker container. 34 | """ 35 | COMMANDS = ['run'] 36 | ORDER = 950 37 | SCHEMA = { 38 | "properties": { 39 | "run": { 40 | "properties": { 41 | "image": { 42 | "type": "string", 43 | "description": "Image to derive the container from.", 44 | "default": "#{/build/tag}" 45 | }, 46 | "env": { 47 | "type": "object", 48 | "description": "Set environment variables (use `null` to forward environment variables).", 49 | "additionalProperties": { 50 | "type": [ 51 | "string", 52 | "null" 53 | ] 54 | } 55 | }, 56 | "env-file": { 57 | "type": "array", 58 | "description": "Read in a file of environment variables.", 59 | "items": { 60 | "type": "string" 61 | } 62 | }, 63 | "mount": { 64 | "type": "array", 65 | "description": "Attach a filesystem mount to the container.", 66 | "items": { 67 | "type": "object", 68 | "properties": { 69 | "type": { 70 | "type": "string", 71 | "enum": [ 72 | "bind", 73 | "tmpfs", 74 | "volume" 75 | ] 76 | }, 77 | "source": { 78 | "type": "string", 79 | "description": "Volume name or path on the host." 80 | }, 81 | "destination": { 82 | "type": "string", 83 | "description": "Absolute mount path in the container." 84 | }, 85 | "readonly": { 86 | "type": "boolean", 87 | "description": "Whether to mount the volume read-only." 88 | } 89 | }, 90 | "required": [ 91 | "type", 92 | "destination" 93 | ], 94 | "additionalProperties": False 95 | } 96 | }, 97 | "publish": { 98 | "type": "array", 99 | "description": "Publish a container's port(s), or range(s) of ports, to the host.", 100 | "items": { 101 | "type": "object", 102 | "properties": { 103 | "ip": { 104 | "type": "string", 105 | "description": "" 106 | }, 107 | "host": { 108 | "anyOf": [ 109 | { 110 | "type": "number" 111 | }, 112 | { 113 | "type": "string", 114 | "pattern": "\\d+-\\d+" 115 | } 116 | ], 117 | "description": "Port (e.g. `8000`) or range of ports (e.g. `8000-8100`) on the host." 118 | }, 119 | "container": { 120 | "anyOf": [ 121 | { 122 | "type": "number" 123 | }, 124 | { 125 | "type": "string", 126 | "pattern": "\\d+-\\d+" 127 | } 128 | ], 129 | "description": "Port (e.g. `8000`) or range of ports (e.g. `8000-8100`) on the container." 130 | } 131 | }, 132 | "required": [ 133 | "container" 134 | ], 135 | "additionalProperties": False 136 | } 137 | }, 138 | "runtime": { 139 | "type": "string", 140 | "description": "Runtime to use for this container." 141 | }, 142 | "tmpfs": { 143 | "type": "array", 144 | "description": "Mount a tmpfs directory", 145 | "items": { 146 | "type": "object", 147 | "properties": { 148 | "destination": { 149 | "type": "string", 150 | "description": "Absolute mount path in the container." 151 | }, 152 | "options": { 153 | "type": "array", 154 | "description": "Mount options for the temporary file system.", 155 | "items": { 156 | "type": "string" 157 | } 158 | }, 159 | "size": { 160 | "type": "integer", 161 | "description": "Size of the tmpfs mount in bytes." 162 | }, 163 | "mode": { 164 | "type": "integer", 165 | "description": "File mode of the tmpfs in octal." 166 | } 167 | }, 168 | "required": [ 169 | "destination" 170 | ], 171 | "additionalProperties": False 172 | } 173 | }, 174 | "cmd": { 175 | "type": "array", 176 | "description": "Command to execute inside the container.", 177 | "items": { 178 | "type": "string" 179 | } 180 | }, 181 | "tty": { 182 | "type": "boolean", 183 | "description": "Allocate a pseudo-TTY" 184 | }, 185 | "cpu-shares": { 186 | "type": "integer", 187 | "description": "CPU shares (relative weight)", 188 | "minimum": 0, 189 | "maximum": 1024 190 | }, 191 | "name": { 192 | "type": "string", 193 | "description": "Assign a name to the container" 194 | }, 195 | "network": { 196 | "type": "string", 197 | "description": "Connect a container to a network (default \"default\")" 198 | }, 199 | "label": { 200 | "type": "array", 201 | "description": "Set meta data on a container" 202 | }, 203 | "rm": { 204 | "type": "boolean", 205 | "description": "Automatically remove the container when it exits", 206 | "default": True 207 | }, 208 | "privileged": { 209 | "type": "boolean", 210 | "description": "Give extended privileges to this container", 211 | "default": False 212 | }, 213 | "memory": { 214 | "type": "string", 215 | "description": "Memory limit" 216 | }, 217 | "interactive": { 218 | "type": "boolean", 219 | "description": "Keep STDIN open even if not attached" 220 | }, 221 | "entrypoint": { 222 | "type": "string", 223 | "description": "Overwrite the default ENTRYPOINT of the image" 224 | }, 225 | "workdir": { 226 | "type": "string", 227 | "description": "Working directory inside the container" 228 | }, 229 | "user": { 230 | "type": "string", 231 | "description": "Username or UID (format: [:])" 232 | }, 233 | "group-add": { 234 | "type": "array", 235 | "description": "Additional groups to run as.", 236 | "items": { 237 | "type": "string" 238 | } 239 | }, 240 | "gpus": { 241 | "type": "string", 242 | "description": "GPU devices to add to the container (‘all’ to pass all GPUs)", 243 | } 244 | }, 245 | "additionalProperties": False 246 | } 247 | }, 248 | "additionalProperties": False 249 | } 250 | 251 | def add_arguments(self, parser): 252 | super(RunConfigurationPlugin, self).add_arguments(parser) 253 | self.add_argument(parser, '/run/cmd', name='cmd', nargs=argparse.REMAINDER, type=None) 254 | 255 | def apply(self, configuration, schema, args): 256 | super(RunConfigurationPlugin, self).apply(configuration, schema, args) 257 | # Set some sensible defaults (could also be published as variables) 258 | util.set_default(configuration, '/run/tty', sys.stdout.isatty()) 259 | util.set_default(configuration, '/run/interactive', sys.stdout.isatty()) 260 | return configuration 261 | -------------------------------------------------------------------------------- /docker_interface/plugins/user.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Spotify AB 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import pwd 16 | import grp 17 | import os 18 | import subprocess 19 | import tempfile 20 | import uuid 21 | 22 | from .base import Plugin, SubstitutionPlugin 23 | from .run import RunConfigurationPlugin 24 | from .. import util 25 | 26 | 27 | class UserPlugin(Plugin): 28 | """ 29 | Share the host user id and group id with the container. 30 | 31 | The plugin provides the following additional variables for substitution: 32 | 33 | * :code:`user/name`: Name of the user on the host. 34 | * :code:`user/uid`: User id of the user on the host. 35 | * :code:`group/name`: Name of the user group on the host. 36 | * :code:`group/gid`: Group id of the user group on the host. 37 | """ 38 | COMMANDS = ['run'] 39 | ORDER = 510 40 | SCHEMA = { 41 | "properties": { 42 | "run": { 43 | "properties": { 44 | "user": util.get_value( 45 | RunConfigurationPlugin.SCHEMA, '/properties/run/properties/user') 46 | }, 47 | "additionalProperties": False 48 | } 49 | }, 50 | "additionalProperties": False 51 | } 52 | 53 | def __init__(self): 54 | super(UserPlugin, self).__init__() 55 | self.tempdir = None 56 | 57 | def add_arguments(self, parser): 58 | self.add_argument(parser, '/run/user') 59 | 60 | def get_user_group(self, user=None, group=None): 61 | """ 62 | Get the user and group information. 63 | 64 | Parameters 65 | ---------- 66 | user : str 67 | User name or user id (default is the `os.getuid()`). 68 | group : str 69 | Group name or group id (default is the group of `user`). 70 | 71 | Returns 72 | ------- 73 | user : pwd.struct_passwd 74 | User object. 75 | group : grp.struct_group 76 | Group object. 77 | """ 78 | user = user or os.getuid() 79 | # Convert the information we have obtained to a user object 80 | try: 81 | try: 82 | user = pwd.getpwuid(int(user)) 83 | except ValueError: 84 | user = pwd.getpwnam(user) 85 | except KeyError as ex: # pragma: no cover 86 | self.logger.fatal("could not resolve user: %s", ex) 87 | raise 88 | 89 | # Get the group 90 | group = group or user.pw_gid 91 | try: 92 | try: 93 | group = grp.getgrgid(int(group)) 94 | except ValueError: 95 | group = grp.getgrnam(group) 96 | except KeyError as ex: # pragma: no cover 97 | self.logger.fatal("could not resolve group:%s", ex) 98 | raise 99 | 100 | return user, group 101 | 102 | def apply(self, configuration, schema, args): 103 | # Do not call the super class because we want to do something more sophisticated with the 104 | # arguments 105 | user, group = self.get_user_group(*(args.user or '').split(':')) 106 | SubstitutionPlugin.VARIABLES['user'] = { 107 | 'uid': user.pw_uid, 108 | 'name': user.pw_name, 109 | } 110 | SubstitutionPlugin.VARIABLES['group'] = { 111 | 'gid': group.gr_gid, 112 | 'name': group.gr_name, 113 | } 114 | util.set_value(configuration, '/run/user', "${user/uid}:${group/gid}") 115 | 116 | # Create a temporary directory and copy the group and passwd files 117 | if configuration['dry-run']: 118 | self.logger.warning("cannot mount /etc/passwd and /etc/groups during dry-run") 119 | else: 120 | self.tempdir = tempfile.TemporaryDirectory(dir='/tmp') 121 | name = uuid.uuid4().hex 122 | # Create a docker image 123 | image = util.get_value(configuration, '/run/image') 124 | image = SubstitutionPlugin.substitute_variables(configuration, image, '/run') 125 | status = subprocess.call([configuration['docker'], 'create', '--name', name, image, 'sh']) 126 | if status: 127 | raise RuntimeError( 128 | "Could not create container from image '%s'. Did you run `di build`?" % image) 129 | # Copy out the passwd and group files, mount them, and append the necessary information 130 | for filename in ['passwd', 'group']: 131 | path = os.path.join(self.tempdir.name, filename) 132 | subprocess.check_call([ 133 | configuration['docker'], 'cp', '%s:/etc/%s' % (name, filename), path]) 134 | util.set_default(configuration, '/run/mount', []).append({ 135 | 'type': 'bind', 136 | 'source': path, 137 | 'destination': '/etc/%s' % filename 138 | }) 139 | with open(path, 'a') as fp: 140 | variables = { 141 | 'user': user.pw_name, 142 | 'uid': user.pw_uid, 143 | 'group': group.gr_name, 144 | 'gid': group.gr_gid 145 | } 146 | if filename == 'passwd': 147 | line = "%(user)s:x:%(uid)d:%(gid)d:%(user)s:/%(user)s:/bin/sh\n" % variables 148 | else: 149 | line = "%(group)s:x:%(gid)d:%(user)s\n" % variables 150 | fp.write(line) 151 | assert os.path.isfile(path) 152 | 153 | # Destroy the container 154 | subprocess.check_call(['docker', 'rm', name]) 155 | 156 | return configuration 157 | 158 | def cleanup(self): 159 | if self.tempdir: 160 | self.tempdir.cleanup() 161 | -------------------------------------------------------------------------------- /docker_interface/util.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Spotify AB 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import contextlib 16 | import os 17 | import socket 18 | 19 | 20 | TYPES = { 21 | 'integer': int, 22 | 'string': str, 23 | 'number': float, 24 | 'boolean': bool, 25 | 'array': list, 26 | } 27 | 28 | 29 | def abspath(path, ref=None): 30 | """ 31 | Create an absolute path. 32 | 33 | Parameters 34 | ---------- 35 | path : str 36 | absolute or relative path with respect to `ref` 37 | ref : str or None 38 | reference path if `path` is relative 39 | 40 | Returns 41 | ------- 42 | path : str 43 | absolute path 44 | 45 | Raises 46 | ------ 47 | ValueError 48 | if an absolute path cannot be constructed 49 | """ 50 | if ref: 51 | path = os.path.join(ref, path) 52 | if not os.path.isabs(path): 53 | raise ValueError("expected an absolute path but got '%s'" % path) 54 | return path 55 | 56 | 57 | def split_path(path, ref=None): 58 | """ 59 | Split a path into its components. 60 | 61 | Parameters 62 | ---------- 63 | path : str 64 | absolute or relative path with respect to `ref` 65 | ref : str or None 66 | reference path if `path` is relative 67 | 68 | Returns 69 | ------- 70 | list : str 71 | components of the path 72 | """ 73 | path = abspath(path, ref) 74 | return path.strip(os.path.sep).split(os.path.sep) 75 | 76 | 77 | def get_value(instance, path, ref=None): 78 | """ 79 | Get the value from `instance` at the given `path`. 80 | 81 | Parameters 82 | ---------- 83 | instance : dict or list 84 | instance from which to retrieve a value 85 | path : str 86 | path to retrieve a value from 87 | ref : str or None 88 | reference path if `path` is relative 89 | 90 | Returns 91 | ------- 92 | value : 93 | value at `path` in `instance` 94 | 95 | Raises 96 | ------ 97 | KeyError 98 | if `path` is not valid 99 | TypeError 100 | if a value along the `path` is not a list or dictionary 101 | """ 102 | for part in split_path(path, ref): 103 | if isinstance(instance, list): 104 | part = int(part) 105 | elif not isinstance(instance, dict): 106 | raise TypeError("expected `list` or `dict` but got `%s`" % instance) 107 | try: 108 | instance = instance[part] 109 | except KeyError: 110 | raise KeyError(abspath(path, ref)) 111 | return instance 112 | 113 | 114 | def pop_value(instance, path, ref=None): 115 | """ 116 | Pop the value from `instance` at the given `path`. 117 | 118 | Parameters 119 | ---------- 120 | instance : dict or list 121 | instance from which to retrieve a value 122 | path : str 123 | path to retrieve a value from 124 | ref : str or None 125 | reference path if `path` is relative 126 | 127 | Returns 128 | ------- 129 | value : 130 | value at `path` in `instance` 131 | """ 132 | head, tail = os.path.split(abspath(path, ref)) 133 | instance = get_value(instance, head) 134 | if isinstance(instance, list): 135 | tail = int(tail) 136 | return instance.pop(tail) 137 | 138 | 139 | def set_value(instance, path, value, ref=None): 140 | """ 141 | Set `value` on `instance` at the given `path` and create missing intermediate objects. 142 | 143 | Parameters 144 | ---------- 145 | instance : dict or list 146 | instance from which to retrieve a value 147 | path : str 148 | path to retrieve a value from 149 | value : 150 | value to set 151 | ref : str or None 152 | reference path if `path` is relative 153 | """ 154 | *head, tail = split_path(path, ref) 155 | for part in head: 156 | instance = instance.setdefault(part, {}) 157 | instance[tail] = value 158 | 159 | 160 | def set_default(instance, path, value, ref=None): 161 | """ 162 | Set `value` on `instance` at the given `path` and create missing intermediate objects. 163 | 164 | Parameters 165 | ---------- 166 | instance : dict or list 167 | instance from which to retrieve a value 168 | path : str 169 | path to retrieve a value from 170 | value : 171 | value to set 172 | ref : str or None 173 | reference path if `path` is relative 174 | """ 175 | *head, tail = split_path(path, ref) 176 | for part in head: 177 | instance = instance.setdefault(part, {}) 178 | return instance.setdefault(tail, value) 179 | 180 | 181 | def merge(x, y): 182 | """ 183 | Merge two dictionaries and raise an error for inconsistencies. 184 | 185 | Parameters 186 | ---------- 187 | x : dict 188 | dictionary x 189 | y : dict 190 | dictionary y 191 | 192 | Returns 193 | ------- 194 | x : dict 195 | merged dictionary 196 | 197 | Raises 198 | ------ 199 | ValueError 200 | if `x` and `y` are inconsistent 201 | """ 202 | keys_x = set(x) 203 | keys_y = set(y) 204 | 205 | for key in keys_y - keys_x: 206 | x[key] = y[key] 207 | 208 | for key in keys_x & keys_y: 209 | value_x = x[key] 210 | value_y = y[key] 211 | 212 | if isinstance(value_x, dict) and isinstance(value_y, dict): 213 | x[key] = merge(value_x, value_y) 214 | else: 215 | if value_x != value_y: 216 | raise ValueError 217 | 218 | return x 219 | 220 | 221 | def set_default_from_schema(instance, schema): 222 | """ 223 | Populate default values on an `instance` given a `schema`. 224 | 225 | Parameters 226 | ---------- 227 | instance : dict 228 | instance to populate default values for 229 | schema : dict 230 | JSON schema with default values 231 | 232 | Returns 233 | ------- 234 | instance : dict 235 | instance with populated default values 236 | """ 237 | for name, property_ in schema.get('properties', {}).items(): 238 | # Set the defaults at this level of the schema 239 | if 'default' in property_: 240 | instance.setdefault(name, property_['default']) 241 | # Descend one level if the property is an object 242 | if 'properties' in property_: 243 | set_default_from_schema(instance.setdefault(name, {}), property_) 244 | return instance 245 | 246 | 247 | def apply(instance, func, path=None): 248 | """ 249 | Apply `func` to all fundamental types of `instance`. 250 | 251 | Parameters 252 | ---------- 253 | instance : dict 254 | instance to apply functions to 255 | func : callable 256 | function with two arguments (instance, path) to apply to all fundamental types recursively 257 | path : str 258 | path in the document (defaults to '/') 259 | 260 | Returns 261 | ------- 262 | instance : dict 263 | instance after applying `func` to fundamental types 264 | """ 265 | path = path or os.path.sep 266 | if isinstance(instance, list): 267 | return [apply(item, func, os.path.join(path, str(i))) for i, item in enumerate(instance)] 268 | elif isinstance(instance, dict): 269 | return {key: apply(value, func, os.path.join(path, key)) for key, value in instance.items()} 270 | return func(instance, path) 271 | 272 | 273 | def get_free_port(ports=None): 274 | """ 275 | Get a free port. 276 | 277 | Parameters 278 | ---------- 279 | ports : iterable 280 | ports to check (obtain a random port by default) 281 | 282 | Returns 283 | ------- 284 | port : int 285 | a free port 286 | """ 287 | if ports is None: 288 | with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as _socket: 289 | _socket.bind(('', 0)) 290 | _, port = _socket.getsockname() 291 | return port 292 | 293 | # Get ports from the specified list 294 | for port in ports: 295 | with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as _socket: 296 | try: 297 | _socket.bind(('', port)) 298 | return port 299 | except socket.error as ex: 300 | if ex.errno not in (48, 98): 301 | raise 302 | 303 | raise RuntimeError("could not find a free port") 304 | -------------------------------------------------------------------------------- /docs/autodoc.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Spotify AB 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | import json 17 | import textwrap 18 | from docker_interface import plugins, util 19 | 20 | 21 | def build_property_tree(schema, depth=0): 22 | for name, property_ in schema.get('properties', {}).items(): 23 | parts = [' ' * depth, '* %s' % name] 24 | type_ = property_.get('type') 25 | if type_: 26 | parts.append(' (:code:`%s`)' % type_) 27 | parts.append(': %s' % property_.get('description', '').strip('.')) 28 | default = property_.get('default') 29 | if default: 30 | parts.append(' (default: :code:`%s`)' % default) 31 | parts.append('.') 32 | yield ''.join(parts) 33 | if 'properties' in property_: 34 | for line in build_property_tree(property_, depth + 4): 35 | yield line 36 | 37 | # Generate the plugin reference 38 | lines = """ 39 | Plugin reference 40 | ================ 41 | 42 | This document lists all plugins in order of execution. 43 | 44 | """.splitlines()[1:] 45 | 46 | classes = plugins.Plugin.load_plugins() 47 | classes['base'] = plugins.base.BasePlugin 48 | schema = {} 49 | for name, plugin_cls in sorted(classes.items(), key=lambda x: x[1].ORDER or 0): 50 | title = plugin_cls.__name__ 51 | lines.extend([title, '-' * len(title), '']) 52 | lines.extend([textwrap.dedent(plugin_cls.__doc__), '']) 53 | 54 | tree = list(build_property_tree(plugin_cls.SCHEMA)) 55 | if tree: 56 | lines.extend(['Properties', '~~~~~~~~~~', '']) 57 | lines.extend(tree) 58 | lines.append('') 59 | schema = util.merge(schema, plugin_cls.SCHEMA) 60 | 61 | dirname = os.path.dirname(os.path.abspath(__file__)) 62 | with open(os.path.join(dirname, 'plugin_reference.rst'), 'w') as fp: 63 | fp.write("\n".join(lines)) 64 | 65 | # Generate the schema 66 | with open(os.path.join(dirname, 'schema.json'), 'w') as fp: 67 | json.dump(schema, fp, indent=4) 68 | 69 | # Generate the list of examples 70 | 71 | lines = [""" 72 | Examples 73 | ======== 74 | 75 | This document lists example use cases for Docker Interface that are available on `GitHub `_. Additional, comprehensive examples can be found in the `tests `_. 76 | 77 | """] 78 | 79 | for path in os.listdir('../examples'): 80 | if os.path.isdir('../examples/' + path): 81 | header = '`%s `_' % (path, path) 82 | lines.append(header) 83 | lines.append('~' * len(header)) 84 | path = 'examples/%s/README.rst' % path 85 | lines.append('.. include:: ../%s' % path) 86 | 87 | with open(os.path.join(dirname, 'examples.rst'), 'w') as fp: 88 | fp.write("\n".join(lines)) 89 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Spotify AB 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # docker_interface documentation build configuration file, created by 16 | # sphinx-quickstart on Thu Dec 7 18:55:35 2017. 17 | # 18 | # This file is execfile()d with the current directory set to its 19 | # containing dir. 20 | # 21 | # Note that not all possible configuration values are present in this 22 | # autogenerated file. 23 | # 24 | # All configuration values have a default; values that are commented out 25 | # serve to show the default. 26 | 27 | # If extensions (or modules to document with autodoc) are in another directory, 28 | # add these directories to sys.path here. If the directory is relative to the 29 | # documentation root, use os.path.abspath to make it absolute, like shown here. 30 | # 31 | # import os 32 | # import sys 33 | # sys.path.insert(0, os.path.abspath('.')) 34 | 35 | # -- General configuration ------------------------------------------------ 36 | 37 | # If your documentation needs a minimal Sphinx version, state it here. 38 | # 39 | # needs_sphinx = '1.0' 40 | 41 | # Add any Sphinx extension module names here, as strings. They can be 42 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 43 | # ones. 44 | extensions = ['sphinx.ext.autodoc', 45 | 'sphinx.ext.doctest', 46 | 'sphinx.ext.intersphinx', 47 | 'sphinx.ext.coverage', 48 | 'sphinx.ext.viewcode', 49 | 'sphinx.ext.githubpages'] 50 | 51 | # Add any paths that contain templates here, relative to this directory. 52 | templates_path = ['_templates'] 53 | 54 | # The suffix(es) of source filenames. 55 | # You can specify multiple suffix as a list of string: 56 | # 57 | # source_suffix = ['.rst', '.md'] 58 | source_suffix = '.rst' 59 | 60 | # The master toctree document. 61 | master_doc = 'index' 62 | 63 | # General information about the project. 64 | project = 'docker_interface' 65 | copyright = '2017, Till Hoffmann' 66 | author = 'Till Hoffmann' 67 | 68 | # The version info for the project you're documenting, acts as replacement for 69 | # |version| and |release|, also used in various other places throughout the 70 | # built documents. 71 | # 72 | # The short X.Y version. 73 | version = '' 74 | # The full version, including alpha/beta/rc tags. 75 | release = '' 76 | 77 | # The language for content autogenerated by Sphinx. Refer to documentation 78 | # for a list of supported languages. 79 | # 80 | # This is also used if you do content translation via gettext catalogs. 81 | # Usually you set "language" from the command line for these cases. 82 | language = None 83 | 84 | # List of patterns, relative to source directory, that match files and 85 | # directories to ignore when looking for source files. 86 | # This patterns also effect to html_static_path and html_extra_path 87 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 88 | 89 | # The name of the Pygments (syntax highlighting) style to use. 90 | pygments_style = 'sphinx' 91 | 92 | # If true, `todo` and `todoList` produce output, else they produce nothing. 93 | todo_include_todos = False 94 | 95 | # -- Options for HTML output ---------------------------------------------- 96 | 97 | # The theme to use for HTML and HTML Help pages. See the documentation for 98 | # a list of builtin themes. 99 | # 100 | html_theme = 'sphinx_rtd_theme' 101 | 102 | # Theme options are theme-specific and customize the look and feel of a theme 103 | # further. For a list of options available for each theme, see the 104 | # documentation. 105 | # 106 | # html_theme_options = {} 107 | 108 | # Add any paths that contain custom static files (such as style sheets) here, 109 | # relative to this directory. They are copied after the builtin static files, 110 | # so a file named "default.css" will overwrite the builtin "default.css". 111 | html_static_path = [] 112 | 113 | # Custom sidebar templates, must be a dictionary that maps document names 114 | # to template names. 115 | # 116 | # This is required for the alabaster theme 117 | # refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars 118 | html_sidebars = { 119 | '**': [ 120 | 'relations.html', # needs 'show_related': True theme option to display 121 | 'searchbox.html', 122 | ] 123 | } 124 | 125 | # -- Options for HTMLHelp output ------------------------------------------ 126 | 127 | # Output file base name for HTML help builder. 128 | htmlhelp_basename = 'docker_interfacedoc' 129 | 130 | # -- Options for LaTeX output --------------------------------------------- 131 | 132 | latex_elements = { 133 | # The paper size ('letterpaper' or 'a4paper'). 134 | # 135 | # 'papersize': 'letterpaper', 136 | 137 | # The font size ('10pt', '11pt' or '12pt'). 138 | # 139 | # 'pointsize': '10pt', 140 | 141 | # Additional stuff for the LaTeX preamble. 142 | # 143 | # 'preamble': '', 144 | 145 | # Latex figure (float) alignment 146 | # 147 | # 'figure_align': 'htbp', 148 | } 149 | 150 | # Grouping the document tree into LaTeX files. List of tuples 151 | # (source start file, target name, title, 152 | # author, documentclass [howto, manual, or own class]). 153 | latex_documents = [ 154 | (master_doc, 'docker_interface.tex', 'docker\\_interface Documentation', 155 | 'Till Hoffmann', 'manual'), 156 | ] 157 | 158 | # -- Options for manual page output --------------------------------------- 159 | 160 | # One entry per manual page. List of tuples 161 | # (source start file, name, description, authors, manual section). 162 | man_pages = [ 163 | (master_doc, 'docker_interface', 'docker_interface Documentation', 164 | [author], 1) 165 | ] 166 | 167 | # -- Options for Texinfo output ------------------------------------------- 168 | 169 | # Grouping the document tree into Texinfo files. List of tuples 170 | # (source start file, target name, title, author, 171 | # dir menu entry, description, category) 172 | texinfo_documents = [ 173 | (master_doc, 'docker_interface', 'docker_interface Documentation', 174 | author, 'docker_interface', 'One line description of project.', 175 | 'Miscellaneous'), 176 | ] 177 | 178 | # Example configuration for intersphinx: refer to the Python standard library. 179 | intersphinx_mapping = {'https://docs.python.org/': None} 180 | 181 | import subprocess 182 | subprocess.check_call(['python', 'autodoc.py']) 183 | -------------------------------------------------------------------------------- /docs/guide.rst: -------------------------------------------------------------------------------- 1 | Introduction 2 | ============ 3 | 4 | `Docker `_ provides a containerised runtime that enables easy and reproducible deployment of applications to production. Unfortunately, these applications are often developed using the local environment of the developer such that it can be difficult to reproduce the results on another machine. Using Docker as a development environment is possible in principle but plagued by problems in practice. For example, 5 | 6 | * `mounting a folder from the host in the container can cause permission problems `_, 7 | * `ports need to be manually forwarded to run Jupyter notebook servers `_, 8 | * or `credentials are not available inside the container `_. 9 | 10 | These issues can be addressed directly by modifying the arguments passed to the Docker `command line interface `_ (CLI), but the resulting commands can be formidable. Docker Interface allows users to define a Docker command declaratively in a configuration file rather than having to remember to type out all required arguments on the command line. In short, Docker Interface is a translator from a command declaration to a Docker command. 11 | 12 | Installing Docker interface 13 | --------------------------- 14 | 15 | You can install Docker Interface using the following :code:`pip` command (you need a python3 interpreter). 16 | 17 | .. code-block:: bash 18 | 19 | pip install docker-interface 20 | 21 | 22 | To check that Docker Interface was installed successfully, run 23 | 24 | .. code-block:: bash 25 | 26 | di --help 27 | 28 | 29 | Using Docker Interface 30 | ---------------------- 31 | 32 | Docker Interface will attempt to locate a configuration file :code:`di.yml` in the current working directory. A basic configuration (as a YAML or JSON file) might look like so. 33 | 34 | .. code-block:: yaml 35 | 36 | docker: docker # The docker command to use, e.g. nvidia-docker 37 | workspace: . # The workspace path (relative to the directory containing the configuration) 38 | 39 | All paths in the configuration are relative to the :code:`workspace`. The values shown above are default values and you can omit them unless you want to change them. 40 | 41 | Docker Interface supports two commands: 42 | 43 | * `build `_ to build a Docker image, 44 | * and `run `_ to execute a command inside a Docker container. 45 | 46 | Information that is relevant to a particular command is stored in a corresponding section of the configuration file. For example, you can run the :code:`bash` shell in the latest :code:`ubuntu` like so: First, create the following configuration file. 47 | 48 | .. code-block:: yaml 49 | 50 | run: 51 | image: ubuntu 52 | 53 | Second, run :code:`di run bash` from the command line. In contrast to :code:`docker run ubuntu bash`, the :code:`di` command will open an interactive shell because it starts the container interactively if it detects that Docker Interface was launched interactively. By default, it will also create an ephemeral container which is deleted as soon as you log out of the shell. 54 | 55 | Before delving into the plugin architecture that powers Docker Interface, let us consider a simple example for building your own Docker image. Create a :code:`Dockerfile` with the following content 56 | 57 | .. code-block:: Dockerfile 58 | 59 | FROM python 60 | RUN pip install ipython 61 | 62 | and modify your :code:`di.yml` configuration to read: 63 | 64 | .. code-block:: yaml 65 | 66 | build: 67 | tag: my-ipython 68 | 69 | Running :code:`di build` from the command line will build your image, and :code:`di run ipython` will run the :code:`ipython` command inside the container. Unless otherwise specified, Docker Interface uses the image built in the :code:`build` step to start a new container when you use the :code:`run` command. Note: the :code:`run` command also sets the environment variable :code:`DOCKER_INTERFACE=true` which allows you to dynamically detect when running under the control of :code:`di`. 70 | 71 | A comprehensive list of variables that can be set in the :code:`di.yml` configuration can be found in the :doc:`plugin_reference`. 72 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Docker Interface 2 | ================ 3 | 4 | Docker Interface (DI) is a declarative interface for building images and running commands in containers using Docker. 5 | 6 | .. toctree:: 7 | :maxdepth: 2 8 | :caption: Contents: 9 | 10 | guide 11 | plugins 12 | plugin_reference 13 | examples 14 | schema 15 | 16 | 17 | Indices and tables 18 | ================== 19 | 20 | * :ref:`genindex` 21 | * :ref:`modindex` 22 | * :ref:`search` 23 | -------------------------------------------------------------------------------- /docs/plugins.rst: -------------------------------------------------------------------------------- 1 | Plugins 2 | ======= 3 | 4 | Docker Interface is a simple framework leveraging a suite of plugins to do most of its work. Each plugin is a :code:`python` class that defines at least two static attributes: 5 | 6 | * :code:`COMANDS` is a sequence of commands such as :code:`run` or :code:`build` and defines for which command the plugin should be active. Setting :code:`COMMANDS` to :code:`all` will enable the plugin for all commands. 7 | * :code:`ORDER` is an integer that determines the order of execution with lower numbers being executed earlier. 8 | 9 | Furthermore, each plugin can additionally define: 10 | 11 | * :code:`ENABLED` (defaults to :code:`True`) which indicates whether the plugin is enabled. Set :code:`ENABLED` to :code:`False` if you want a plugin to be disabled by default. 12 | * :code:`SCHEMA` (defaults to :code:`{}`) is a JSON schema definition that is specific to the plugin. The Docker Interface configuration is validated against the union of schemas defined by all enabled plugins. 13 | 14 | How plugins work 15 | ---------------- 16 | 17 | Each plugin has two methods used by Docker Interface: 18 | 19 | * :code:`add_arguments(parser)` is called for each enabled plugin before Docker Interface attempts to parse the command line arguments. Each plugin may add arbitrary arguments to the :code:`parser` of the command line interface as long as they do not interfere with one another. 20 | * :code:`apply(configuration, schema, args)` is called for each plugin after :code:`args` have been parsed. The :code:`schema` passed to the plugins is the union of all plugins' schemas. Finally, :code:`configuration` is the configuration returned by the :code:`apply` method of a plugin with lower :code:`ORDER`. The plugin may modify the configuration (as :code:`UserPlugin` does), execute a Docker command (as :code:`BuildExecutePlugin` does), or run any other python code. 21 | 22 | Enable and disabling plugins 23 | ---------------------------- 24 | 25 | Unless otherwise specified, a plugin is enabled if and only if its class-level attribute :code:`ENABLED` is :code:`TRUE`. But you can specify which plugins to enable or disable in the configuration like so. 26 | 27 | .. code-block:: yaml 28 | 29 | plugins: 30 | - user 31 | - homedir 32 | 33 | The above configuration will enable only the :code:`user` and the :code:`homedir` plugins. Alternatively, you can specify which plugins to enable or disable explicitly. 34 | 35 | .. code-block:: yaml 36 | 37 | plugins: 38 | enable: 39 | - user 40 | disable: 41 | - homedir 42 | 43 | Which will enable the :code:`user` plugin, disable the :code:`homedir` plugin, and leave all other plugins unchanged. 44 | 45 | Schema validation 46 | ----------------- 47 | 48 | Docker Interface validates the configuration against the union of schemas defined by all enabled plugins. Different plugins may define the same schema as long as the definitions are consistent with one another. Schema definitions are also the preferred way to provide default values for the configuration. For example, the schema for the :code:`BasePlugin` responsible for loading the configuration, handling the workspace, and other global variables looks like so. 49 | 50 | .. code-block:: json 51 | 52 | { 53 | "properties": { 54 | "workspace": { 55 | "type": "string", 56 | "description": "Path defining the DI workspace (absolute or relative to the URI of the configuration document). All subsequent path definitions must be absolute or relative to the `workspace`." 57 | }, 58 | "docker": { 59 | "type": "string", 60 | "description": "Name of the docker CLI.", 61 | "default": "docker" 62 | } 63 | }, 64 | "required": [ 65 | "docker", 66 | "workspace" 67 | ] 68 | } 69 | 70 | The configuration can define the parameters :code:`docker` and :code:`workspace`, and we provide a default value for :code:`docker`. We have omitted some properties for easier readability. If your plugin adds new configuration values, it should define a :code:`SCHEMA`. 71 | -------------------------------------------------------------------------------- /docs/schema.rst: -------------------------------------------------------------------------------- 1 | Schema 2 | ====== 3 | 4 | The document below provides a comprehensive schema definition for Docker Interface. 5 | 6 | .. literalinclude:: schema.json 7 | :language: json 8 | -------------------------------------------------------------------------------- /examples/cython/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3 2 | 3 | WORKDIR /workspace 4 | COPY requirements.txt . 5 | RUN pip install -r requirements.txt 6 | COPY . . 7 | RUN pip install -e . 8 | -------------------------------------------------------------------------------- /examples/cython/README.rst: -------------------------------------------------------------------------------- 1 | This simple example addresses some of the difficulties associated with using cython and :code:`di` together: the cython code is compiled when the Docker image is built. But when the workspace is mounted in the container, the binaries are hidden and the code can no longer be executed. We thus use `pyximport `_ to compile the cython code on the fly. See :code:`cython_example/__init__.py` for details. 2 | -------------------------------------------------------------------------------- /examples/cython/cython_example/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Spotify AB 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import pyximport 15 | pyximport.install() 16 | 17 | from .speedy_code import add_two_numbers 18 | -------------------------------------------------------------------------------- /examples/cython/cython_example/speedy_code.pyx: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Spotify AB 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | cpdef add_two_numbers(int a, int b): 16 | return a + b 17 | -------------------------------------------------------------------------------- /examples/cython/di.yml: -------------------------------------------------------------------------------- 1 | build: 2 | tag: cython-example 3 | -------------------------------------------------------------------------------- /examples/cython/requirements.txt: -------------------------------------------------------------------------------- 1 | cython 2 | ipython 3 | -------------------------------------------------------------------------------- /examples/cython/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Spotify AB 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from setuptools import setup, find_packages 16 | from Cython.Build import cythonize 17 | 18 | ext_modules = cythonize('cython_example/*.pyx') 19 | 20 | setup( 21 | name='cython_example', 22 | version='0.1', 23 | ext_modules=ext_modules, 24 | ) 25 | -------------------------------------------------------------------------------- /examples/env/README.rst: -------------------------------------------------------------------------------- 1 | This example demonstrates that a default environment variable is set whe using :code:`di run`. 2 | -------------------------------------------------------------------------------- /examples/env/check_env_var.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019Spotify AB 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | 17 | if 'DOCKER_INTERFACE' not in os.environ: 18 | raise RuntimeError('DOCKER_INTERFACE variable not set') 19 | 20 | -------------------------------------------------------------------------------- /examples/env/di.yml: -------------------------------------------------------------------------------- 1 | run: 2 | image: python:3 3 | -------------------------------------------------------------------------------- /examples/notebook/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3 2 | 3 | RUN pip install jupyter matplotlib numpy scipy 4 | -------------------------------------------------------------------------------- /examples/notebook/README.rst: -------------------------------------------------------------------------------- 1 | This example demonstrates automatic port forwarding for the Jupyter notebook using :code:`di`. Port forwarding is implemented using the :code:`JupyterPlugin` in :code:`docker_interface/plugins/python.py`. 2 | -------------------------------------------------------------------------------- /examples/notebook/di.yml: -------------------------------------------------------------------------------- 1 | build: 2 | tag: jupyter-example 3 | run: 4 | cmd: 5 | - jupyter 6 | - notebook 7 | -------------------------------------------------------------------------------- /examples/ports/README.rst: -------------------------------------------------------------------------------- 1 | This example demonstrates how to forward ports using :code:`di`'s declarative syntax. 2 | -------------------------------------------------------------------------------- /examples/ports/bind_to_port.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Spotify AB 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import contextlib 16 | import socket 17 | 18 | 19 | with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as _socket: 20 | _socket.bind(('127.0.0.1', 8888)) 21 | print("Successfully bound to port 8888.") 22 | -------------------------------------------------------------------------------- /examples/ports/di.yml: -------------------------------------------------------------------------------- 1 | run: 2 | image: python:3 3 | publish: 4 | - container: 8888 5 | host: 9999 6 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | testpaths = tests 3 | -------------------------------------------------------------------------------- /requirements.in: -------------------------------------------------------------------------------- 1 | -e .[tests] 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile 3 | # To update, run: 4 | # 5 | # pip-compile requirements.in 6 | # 7 | -e . 8 | alabaster==0.7.12 # via sphinx 9 | atomicwrites==1.3.0 # via pytest 10 | attrs==19.1.0 # via jsonschema, packaging, pytest 11 | babel==2.7.0 # via sphinx 12 | certifi==2019.6.16 # via requests 13 | chardet==3.0.4 # via requests 14 | coverage==4.5.4 # via pytest-cov 15 | docutils==0.15.2 # via sphinx 16 | idna==2.8 # via requests 17 | imagesize==1.1.0 # via sphinx 18 | importlib-metadata==0.20 # via pluggy, pytest 19 | jinja2==2.10.1 # via sphinx 20 | jsonschema==3.0.2 21 | markupsafe==1.1.1 # via jinja2 22 | more-itertools==7.2.0 # via pytest, zipp 23 | packaging==19.1 # via pytest, sphinx 24 | pluggy==0.12.0 # via pytest 25 | py==1.8.0 # via pytest 26 | pygments==2.4.2 # via sphinx 27 | pyparsing==2.4.2 # via packaging 28 | pyrsistent==0.15.4 # via jsonschema 29 | pytest-cov==2.7.1 30 | pytest==5.1.2 # via pytest-cov 31 | pytz==2019.2 # via babel 32 | pyyaml==5.1.2 33 | requests==2.22.0 # via sphinx 34 | six==1.12.0 # via jsonschema, packaging, pyrsistent 35 | snowballstemmer==1.9.0 # via sphinx 36 | sphinx-rtd-theme==0.4.3 37 | sphinx==2.2.0 # via sphinx-rtd-theme 38 | sphinxcontrib-applehelp==1.0.1 # via sphinx 39 | sphinxcontrib-devhelp==1.0.1 # via sphinx 40 | sphinxcontrib-htmlhelp==1.0.2 # via sphinx 41 | sphinxcontrib-jsmath==1.0.1 # via sphinx 42 | sphinxcontrib-qthelp==1.0.2 # via sphinx 43 | sphinxcontrib-serializinghtml==1.1.3 # via sphinx 44 | urllib3==1.25.3 # via requests 45 | wcwidth==0.1.7 # via pytest 46 | zipp==0.6.0 # via importlib-metadata 47 | 48 | # The following packages are considered to be unsafe in a requirements file: 49 | # setuptools==41.2.0 # via jsonschema, sphinx 50 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Spotify AB 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from setuptools import setup, find_packages 16 | 17 | PLUGINS = [ 18 | 'Run', 'Build', 'WorkspaceMount', 'Substitution', 'User', 'HomeDir', 'RunConfiguration', 19 | 'BuildConfiguration', 'Validation', 'GoogleCloudCredentials', 'GoogleContainerRegistry', 20 | 'Jupyter' 21 | ] 22 | 23 | 24 | with open('README.md') as fp: 25 | long_description = fp.read() 26 | 27 | 28 | setup( 29 | name="docker_interface", 30 | version="0.4.3", 31 | packages=find_packages(), 32 | install_requires=[ 33 | 'jsonschema>=2.6.0', 34 | 'PyYAML>=3.12', 35 | ], 36 | extras_require={ 37 | 'tests': [ 38 | "pytest>=3.3.1", 39 | "pytest-cov>=2.5.1", 40 | "Sphinx>=1.6.5", 41 | "sphinx_rtd_theme>=0.2.4", 42 | ] 43 | }, 44 | entry_points={ 45 | 'console_scripts': [ 46 | 'di = docker_interface.cli:entry_point' 47 | ], 48 | 'docker_interface.plugins': [ 49 | '%s = docker_interface.plugins:%sPlugin' % (name.lower(), name) for name in PLUGINS 50 | ], 51 | }, 52 | author="Till Hoffmann", 53 | author_email="till@spotify.com", 54 | url="https://github.com/spotify/docker_interface", 55 | license="License :: OSI Approved :: Apache Software License", 56 | description="Declarative interface for building images and running commands in containers " 57 | "using Docker.", 58 | long_description=long_description, 59 | long_description_content_type="text/markdown", 60 | ) 61 | -------------------------------------------------------------------------------- /tests/configurations/comprehensive.yml: -------------------------------------------------------------------------------- 1 | build: 2 | build-arg: 3 | arg: value 4 | file: "#{path}/Dockerfile.test" 5 | no-cache: true 6 | tag: di-test 7 | quiet: false 8 | cpu-shares: 3 9 | memory: 1G 10 | run: 11 | env: 12 | my_variable: Hello world! 13 | TMPDIR: 14 | env-file: 15 | - ".env.test" 16 | mount: 17 | - type: bind 18 | destination: "/bind" 19 | source: "." 20 | publish: 21 | - host: 52729 22 | container: 8888 23 | tmpfs: 24 | - destination: "/tmpfs" 25 | options: 26 | - exec 27 | tty: true 28 | interactive: true 29 | memory: 2G 30 | plugins: 31 | enable: 32 | - user 33 | disable: 34 | - GoogleContainerRegistry 35 | -------------------------------------------------------------------------------- /tests/configurations/jupyter.yml: -------------------------------------------------------------------------------- 1 | run: 2 | cmd: 3 | - jupyter 4 | - notebook 5 | -------------------------------------------------------------------------------- /tests/configurations/plugins_list.yml: -------------------------------------------------------------------------------- 1 | plugins: 2 | - user 3 | - workspacemount 4 | -------------------------------------------------------------------------------- /tests/test_dry_run.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Spotify AB 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import glob 16 | import yaml 17 | from docker_interface import cli 18 | import pytest 19 | 20 | 21 | @pytest.fixture(params=glob.glob('tests/configurations/*.yml')) 22 | def configuration(request): 23 | with open(request.param) as fp: 24 | return yaml.safe_load(fp) 25 | 26 | 27 | @pytest.mark.parametrize('command', ['build', 'run']) 28 | def test_cli(configuration, command): 29 | assert configuration is not None 30 | configuration['dry-run'] = True 31 | configuration['workspace'] = '.' 32 | # Run the command 33 | cli.entry_point([command], configuration) 34 | -------------------------------------------------------------------------------- /tests/test_examples.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Spotify AB 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import pytest 16 | import docker_interface as di 17 | import docker_interface.cli 18 | 19 | 20 | @pytest.fixture(params=[ 21 | ('cython', ['python', '-c', '"add_two_numbers(1, 2)"'], True), 22 | ('ports', ['python', 'bind_to_port.py'], False), 23 | ('env', ['python', 'check_env_var.py'], False), 24 | ]) 25 | def example_definition(request): 26 | return request.param 27 | 28 | 29 | @pytest.fixture 30 | def build_image(example_definition): 31 | # Only build the image if desired 32 | if example_definition[2]: 33 | di.cli.entry_point(['-f', 'examples/%s/di.yml' % example_definition[0], 'build']) 34 | return example_definition 35 | 36 | 37 | def test_command(build_image): 38 | example, command, _ = build_image 39 | di.cli.entry_point(['-f', 'examples/%s/di.yml' % example, 'run'] + command) 40 | -------------------------------------------------------------------------------- /tests/test_schema.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Spotify AB 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | import pytest 17 | from docker_interface import plugins 18 | 19 | 20 | @pytest.mark.parametrize('plugin, cls', plugins.Plugin.load_plugins().items()) 21 | def test_properties(plugin, cls): 22 | if not cls.SCHEMA: 23 | pytest.skip() 24 | required = ["additionalProperties", "anyOf", "oneOf", "allOf", "not"] 25 | queue = [('/', cls.SCHEMA)] 26 | while queue: 27 | path, property_ = queue.pop() 28 | assert any([r in property_ for r in required]), \ 29 | "additionalProperties missing for plugin %s: %s" % (plugin, path) 30 | 31 | for name, child in property_.get('properties', {}).items(): 32 | if child.get("type", "object") == "object": 33 | queue.append((os.path.join(path, name), child)) 34 | child = child.get('items') 35 | if child and child["type"] == "object": 36 | queue.append((os.path.join(path, name, "items"), child)) 37 | 38 | child = property_.get('additionalProperties') 39 | if child and child["type"] == "object": 40 | queue.append((os.path.join(path, "additionalProperties"), child)) 41 | -------------------------------------------------------------------------------- /tests/test_util.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Spotify AB 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import pytest 16 | from docker_interface import util 17 | 18 | 19 | def test_set_value(): 20 | obj = {} 21 | util.set_value(obj, '/some/value', 1) 22 | assert obj == {'some': {'value': 1}} 23 | 24 | 25 | def test_get_value(): 26 | obj = {'some': [{'path': 3}]} 27 | assert util.get_value(obj, '/some/0/path') == 3 28 | 29 | 30 | def test_pop_value(): 31 | obj = {'some': [{'path': 3}]} 32 | assert util.pop_value(obj, '/some/0/path') == 3 33 | assert obj['some'][0] == {} 34 | assert util.pop_value(obj, '/some/0') == {} 35 | assert obj['some'] == [] 36 | 37 | 38 | def test_set_default(): 39 | obj = {'a': 1} 40 | assert util.set_default(obj, '/a', 5) == 1 41 | assert obj['a'] == 1 42 | assert util.set_default(obj, '/b', 3) == 3 43 | assert obj['b'] == 3 44 | assert util.set_default(obj, 'd', 'hello', ref='/c') == 'hello' 45 | assert obj['c']['d'] == 'hello' 46 | 47 | 48 | def test_merge(): 49 | merged = util.merge({'a': 1, 'b': {'c': 3}, 'd': 8}, {'a': 1, 'b': {'d': 7}, 'e': 9}) 50 | assert merged == { 51 | 'a': 1, 52 | 'b': { 53 | 'c': 3, 54 | 'd': 7 55 | }, 56 | 'd': 8, 57 | 'e': 9 58 | } 59 | 60 | 61 | def test_merge_conflict(): 62 | with pytest.raises(ValueError): 63 | util.merge({'a': 1}, {'a': 2}) 64 | 65 | 66 | def test_set_default_from_schema(): 67 | schema = { 68 | "properties": { 69 | "a": { 70 | "default": 1 71 | }, 72 | "b": { 73 | "properties": { 74 | "c": { 75 | "default": [] 76 | } 77 | } 78 | } 79 | } 80 | } 81 | assert util.set_default_from_schema({}, schema) == {'a': 1, 'b': {'c': []}} 82 | 83 | 84 | def test_apply(): 85 | assert util.apply({'a': 3, 'b': [4, 5]}, lambda x, _: x ** 2) == {'a': 9, 'b': [16, 25]} 86 | 87 | 88 | def test_get_free_port_random(): 89 | assert util.get_free_port() > 0 90 | 91 | 92 | def test_get_free_port_bounded(): 93 | assert 8888 <= util.get_free_port((8888, 9999)) <= 9999 94 | --------------------------------------------------------------------------------