├── .gitignore ├── CONTRIBUTIONS.md ├── LICENSE ├── README.md ├── dockerfabric ├── __init__.py ├── actions.py ├── api.py ├── apiclient.py ├── base.py ├── cli.py ├── socat.py ├── tasks.py ├── tunnel.py ├── utils │ ├── __init__.py │ ├── base.py │ ├── containers.py │ ├── files.py │ ├── net.py │ ├── output.py │ └── users.py └── yaml.py ├── docs ├── Makefile ├── api │ ├── dockerfabric.rst │ ├── dockerfabric.utils.rst │ └── modules.rst ├── changes.rst ├── conf.py ├── guide │ ├── apiclient.rst │ ├── cli.rst │ ├── containers.rst │ ├── tasks.rst │ └── utils.rst ├── index.rst ├── installation.rst └── start.rst └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Sphinx documentation 49 | docs/_build/ 50 | 51 | # pyenv 52 | .python-version 53 | 54 | # dotenv 55 | .env 56 | 57 | # virtualenv 58 | venv/ 59 | ENV/ 60 | -------------------------------------------------------------------------------- /CONTRIBUTIONS.md: -------------------------------------------------------------------------------- 1 | Thanks to the following contributors 2 | ==================================== 3 | * [Laurent Fasnacht](https://github.com/lfasnacht) for publishing an implementation for a local tunnel to a Fabric 4 | client in the [pull request #939 of Fabric](https://github.com/fabric/fabric/pull/939). 5 | * [zalan-axis](https://github.com/zalan-axis): Make it possible to set raise_on_error with pull. 6 | [PR #3](https://github.com/merll/docker-fabric/pull/3) 7 | * [bountin](https://github.com/bountin): Fixed docstring typo. [PR #5](https://github.com/merll/docker-fabric/pull/5) 8 | * [Technology Admin](https://github.com/ambsw-technology): Many detailed issue reports and suggestions for improvement. 9 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014-2016 Matthias Erll 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Docker-Fabric 2 | ============= 3 | 4 | Build Docker images, and run Docker containers in Fabric. 5 | --------------------------------------------------------- 6 | 7 | Project: https://github.com/merll/docker-fabric 8 | 9 | Docs: https://docker-fabric.readthedocs.io/en/latest/ 10 | 11 | 12 | Overview 13 | ======== 14 | With a few preparations, Docker images can easily be generated and tested on development 15 | machines, and transferred on to a production environment. This package supports managing 16 | container configurations along with their dependencies within Fabric-based deployments. 17 | DockerFiles can also be easily implemented in Fabric tasks. 18 | 19 | Local Docker clients can be controlled directly through ``docker-py``. Remote Docker 20 | API services make use of Fabric's SSH connection. 21 | 22 | API access 23 | ========== 24 | This project is based on [Docker-Map](https://github.com/merll/docker-map), and adapts 25 | its container configuration methods. 26 | 27 | As with Docker-Map, container configurations can be generated as objects, updated from 28 | Python dictionaries, or imported from YAML files in order to control remote clients 29 | via the API. Docker-Fabric includes the following enhancements: 30 | 31 | Docker client 32 | ------------- 33 | `DockerFabricClient` adds Fabric-like logging in the context of container instances on 34 | top of Fabric hosts, and enables automatic creation of tunnel connections for access to a 35 | remote Docker host using Fabric's SSH connection. By using the tool `socat`, the Docker 36 | client can access a remote service without re-configuration. 37 | 38 | Client configuration 39 | -------------------- 40 | `DockerClientConfiguration` adds the capability of running containers to Fabric hosts 41 | with specific Docker settings for each, e.g. the version number. 42 | 43 | Running container configurations 44 | -------------------------------- 45 | `ContainerFabric` is a simple wrapper that combines Docker-Map's `DockerFabricClient`, 46 | `DockerClientConfiguration` objects, and container maps. 47 | 48 | Command-line based access 49 | ------------------------- 50 | The following features are provided by running the appropriate commands on a remote Docker 51 | command line: 52 | 53 | * Copy resources from a container to a Fabric host. 54 | * Copy resources from a container and download them in a compressed tarball. The Docker 55 | Remote API currently does not support creating compressed tarballs. 56 | * Copy resources from a container and store them in a new blank image. 57 | * Generate a compressed image tarball. The Docker Remote API currently does not support 58 | creating compressed tarballs, but is capable of importing them. 59 | 60 | Tasks 61 | ===== 62 | All essential container actions (`create`, `start`, `stop`, `remove`) and some advanced 63 | (e.g. `update`) can be triggered from the command line as Fabric tasks and executed on 64 | the remote service, e.g. via SSH. 65 | 66 | Additionally the following tasks are included in this package, that can be run by Fabric 67 | directly: 68 | 69 | * `check_version`: Returns version information of the remote Docker service and provides 70 | useful insight if permissions are set up properly. 71 | * `cleanup_containers`: Removes all containers that have stopped. 72 | * `cleanup_images`: Removes all untagged images, that do not have a dependent container 73 | or other dependent images. 74 | * `remove_all_containers`: Stops and removes all containers on the remote Docker service. 75 | 76 | 77 | Contributions 78 | ============= 79 | Thanks to [lfasnacht](https://github.com/lfasnacht) for publishing an implementation for 80 | a local tunnel to a Fabric client in the [pull request 939 of Fabric](https://github.com/fabric/fabric/pull/939). 81 | 82 | Further contributions are maintained in [CONTRIBUTIONS.md](https://github.com/merll/docker-fabric/blob/master/CONTRIBUTIONS.md) of the project. 83 | -------------------------------------------------------------------------------- /dockerfabric/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | 5 | __version__ = '0.5.0' 6 | -------------------------------------------------------------------------------- /dockerfabric/actions.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | import os 5 | import posixpath 6 | 7 | from fabric.api import get, put, puts, task 8 | from fabric.utils import error 9 | 10 | from dockermap.map.action import ContainerUtilAction 11 | from .apiclient import container_fabric 12 | from .utils.files import temp_dir 13 | 14 | 15 | @task 16 | def perform(action_name, container, **kwargs): 17 | """ 18 | Performs an action on the given container map and configuration. 19 | 20 | :param action_name: Name of the action (e.g. ``update``). 21 | :param container: Container configuration name. 22 | :param kwargs: Keyword arguments for the action implementation. 23 | """ 24 | cf = container_fabric() 25 | cf.call(action_name, container, **kwargs) 26 | 27 | 28 | @task 29 | def create(container, **kwargs): 30 | """ 31 | Creates a container and its dependencies. 32 | 33 | :param container: Container configuration name. 34 | :param kwargs: Keyword arguments to the action implementation. 35 | """ 36 | container_fabric().create(container, **kwargs) 37 | 38 | 39 | @task 40 | def start(container, **kwargs): 41 | """ 42 | Starts a container and its dependencies. 43 | 44 | :param container: Container configuration name. 45 | :param kwargs: Keyword arguments to the action implementation. 46 | """ 47 | container_fabric().start(container, **kwargs) 48 | 49 | 50 | @task 51 | def stop(container, **kwargs): 52 | """ 53 | Stops a container and its dependents. 54 | 55 | :param container: Container configuration name. 56 | :param kwargs: Keyword arguments to the action implementation. 57 | """ 58 | container_fabric().stop(container, **kwargs) 59 | 60 | 61 | @task 62 | def remove(container, **kwargs): 63 | """ 64 | Removes a container and its dependents. 65 | 66 | :param container: Container configuration name. 67 | :param kwargs: Keyword arguments to the action implementation. 68 | """ 69 | container_fabric().remove(container, **kwargs) 70 | 71 | 72 | @task 73 | def restart(container, **kwargs): 74 | """ 75 | Restarts a container and starts its dependencies if necessary. 76 | 77 | :param container: Container configuration name. 78 | :param kwargs: Keyword arguments to the action implementation. 79 | """ 80 | container_fabric().restart(container, **kwargs) 81 | 82 | 83 | @task 84 | def startup(container, **kwargs): 85 | """ 86 | Creates and starts a container and its dependencies. 87 | 88 | :param container: Container configuration name. 89 | :param kwargs: Keyword arguments to the action implementation. 90 | """ 91 | container_fabric().startup(container, **kwargs) 92 | 93 | 94 | @task 95 | def shutdown(container, **kwargs): 96 | """ 97 | Stops and removes a container and its dependents. 98 | 99 | :param container: Container configuration name. 100 | :param kwargs: Keyword arguments to the action implementation. 101 | """ 102 | container_fabric().shutdown(container, **kwargs) 103 | 104 | 105 | @task 106 | def update(container, **kwargs): 107 | """ 108 | Updates a container and its dependencies. Creates and starts containers as necessary. 109 | 110 | :param container: Container configuration name. 111 | :param kwargs: Keyword arguments to the action implementation. 112 | """ 113 | container_fabric().update(container, **kwargs) 114 | 115 | 116 | @task 117 | def kill(container, **kwargs): 118 | """ 119 | Sends a signal to a container, by default ``SIGKILL``. You can also pass a different signal such as ``SIGHUP``. 120 | 121 | :param container: Container configuration name. 122 | :param kwargs: Keyword arguments to the action implementation. 123 | """ 124 | container_fabric().signal(container, **kwargs) 125 | 126 | 127 | @task 128 | def pull_images(container, **kwargs): 129 | """ 130 | Pulls missing images, including dependencies. 131 | 132 | :param container: Container configuration name. 133 | :param kwargs: Keyword arguments to the action implementation. 134 | """ 135 | container_fabric().pull_images(container, **kwargs) 136 | 137 | 138 | @task 139 | def script(container, script_path, fail_nonzero=False, upload_dir=False, **kwargs): 140 | """ 141 | Runs a script inside a container, which is created with all its dependencies. The container is removed after it 142 | has been run, whereas the dependencies are not destroyed. The output is printed to the console. 143 | 144 | :param container: Container configuration name. 145 | :param script_path: Local path to the script file. 146 | :param fail_nonzero: Fail if the script returns with a nonzero exit code. 147 | :param upload_dir: Upload the entire parent directory of the script file to the remote. 148 | :param kwargs: Additional keyword arguments to the run_script action. 149 | """ 150 | full_script_path = os.path.abspath(script_path) 151 | prefix, name = os.path.split(full_script_path) 152 | with temp_dir() as remote_tmp: 153 | if upload_dir: 154 | prefix_path, prefix_name = os.path.split(prefix) 155 | remote_script = posixpath.join(remote_tmp, prefix_name, name) 156 | put(prefix, remote_tmp, mirror_local_mode=True) 157 | else: 158 | remote_script = posixpath.join(remote_tmp, name) 159 | put(script_path, remote_script, mirror_local_mode=True) 160 | results = [output.result 161 | for output in container_fabric().run_script(container, script_path=remote_script, **kwargs) 162 | if o.action_type == ContainerUtilAction.SCRIPT] 163 | for res in results: 164 | puts("Exit code: {0}".format(res['exit_code'])) 165 | if res['exit_code'] == 0 or not fail_nonzero: 166 | puts(res['log']) 167 | else: 168 | error(res['log']) 169 | 170 | 171 | @task 172 | def single_cmd(container, command, fail_nonzero=False, download_result=None, **kwargs): 173 | """ 174 | Runs a script inside a container, which is created with all its dependencies. The container is removed after it 175 | has been run, whereas the dependencies are not destroyed. The output is printed to the console. 176 | 177 | :param container: Container configuration name. 178 | :param command: Command line to run. 179 | :param fail_nonzero: Fail if the script returns with a nonzero exit code. 180 | :param download_result: Download any results that the command has written back to a temporary directory. 181 | :param kwargs: Additional keyword arguments to the run_script action. 182 | """ 183 | with temp_dir() as remote_tmp: 184 | kwargs.setdefault('command_format', ['-c', command]) 185 | results = [output.result 186 | for output in container_fabric().run_script(container, script_path=remote_tmp, **kwargs) 187 | if o.action_type == ContainerUtilAction.SCRIPT] 188 | if download_result: 189 | get(posixpath.join(remote_tmp, '*'), local_path=download_result) 190 | for res in results: 191 | puts("Exit code: {0}".format(res['exit_code'])) 192 | if res['exit_code'] == 0 or not fail_nonzero: 193 | puts(res['log']) 194 | else: 195 | error(res['log']) 196 | -------------------------------------------------------------------------------- /dockerfabric/api.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from fabric.api import env 5 | 6 | from .apiclient import DockerFabricApiConnections, ContainerApiFabricClient, DockerClientConfiguration 7 | from .cli import DockerCliConnections, ContainerCliFabricClient, DockerCliConfig 8 | 9 | CLIENT_API = 'API' 10 | CLIENT_CLI = 'CLI' 11 | 12 | docker_api = DockerFabricApiConnections().get_connection 13 | docker_cli = DockerCliConnections().get_connection 14 | 15 | 16 | def docker_fabric(*args, **kwargs): 17 | """ 18 | :param args: Positional arguments to Docker client. 19 | :param kwargs: Keyword arguments to Docker client. 20 | :return: Docker client. 21 | :rtype: dockerfabric.apiclient.DockerFabricClient | dockerfabric.cli.DockerCliClient 22 | """ 23 | ci = kwargs.get('client_implementation') or env.get('docker_fabric_implementation') or CLIENT_API 24 | if ci == CLIENT_API: 25 | return docker_api(*args, **kwargs) 26 | elif ci == CLIENT_CLI: 27 | return docker_cli(*args, **kwargs) 28 | raise ValueError("Invalid client implementation.", ci) 29 | 30 | 31 | def container_fabric(container_maps=None, docker_client=None, clients=None, client_implementation=None): 32 | """ 33 | :param container_maps: Container map or a tuple / list thereof. 34 | :type container_maps: list[dockermap.map.config.main.ContainerMap] | dockermap.map.config.main.ContainerMap 35 | :param docker_client: Default Docker client instance. 36 | :type docker_client: dockerfabric.base.FabricClientConfiguration or docker.docker.Client 37 | :param clients: Optional dictionary of Docker client configuration objects. 38 | :type clients: dict[unicode | str, dockerfabric.base.FabricClientConfiguration] 39 | :param client_implementation: Client implementation to use (API or CLI). 40 | :type client_implementation: unicode | str 41 | :return: Container mapping client. 42 | :rtype: dockerfabric.base.FabricContainerClient 43 | """ 44 | ci = client_implementation or env.get('docker_fabric_implementation') or CLIENT_API 45 | if ci == CLIENT_API: 46 | return ContainerApiFabricClient(container_maps, docker_client, clients) 47 | elif ci == CLIENT_CLI: 48 | return ContainerCliFabricClient(container_maps, docker_client, clients) 49 | raise ValueError("Invalid client implementation.", ci) 50 | -------------------------------------------------------------------------------- /dockerfabric/apiclient.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from fabric.api import env, sudo 5 | from fabric.utils import puts, fastprint, error 6 | 7 | from dockermap.client.base import LOG_PROGRESS_FORMAT, DockerStatusError 8 | from dockermap.api import DockerClientWrapper 9 | from .base import (get_local_port, set_raise_on_error, DockerConnectionDict, FabricClientConfiguration, 10 | FabricContainerClient) 11 | from .socat import socat_tunnels 12 | from .tunnel import local_tunnels 13 | 14 | 15 | DEFAULT_TCP_HOST = 'tcp://127.0.0.1' 16 | DEFAULT_SOCKET = '/var/run/docker.sock' 17 | progress_fmt = LOG_PROGRESS_FORMAT.format 18 | 19 | 20 | def _get_port_number(expr, port_loc): 21 | try: 22 | return int(expr) 23 | except TypeError: 24 | raise ValueError("Missing or invalid {0} port ({1}).".format(port_loc, expr)) 25 | 26 | 27 | def _get_socat_tunnel(address, local_port): 28 | init_local_port = _get_port_number(local_port, 'local') 29 | tunnel_local_port = get_local_port(init_local_port) 30 | socat_tunnel = socat_tunnels[(address, tunnel_local_port)] 31 | return '{0}:{1}'.format(DEFAULT_TCP_HOST, socat_tunnel.bind_port), socat_tunnel 32 | 33 | 34 | def _get_local_tunnel(address, remote_port, local_port): 35 | host_port = address.partition('/')[0] 36 | host, __, port = host_port.partition(':') 37 | service_remote_port = _get_port_number(port or remote_port, 'remote') 38 | init_local_port = _get_port_number(local_port or port or remote_port, 'local') 39 | local_tunnel = local_tunnels[(host, service_remote_port, 'localhost', init_local_port)] 40 | return '{0}:{1}'.format(DEFAULT_TCP_HOST, local_tunnel.bind_port), local_tunnel 41 | 42 | 43 | def _get_connection_args(base_url, remote_port, local_port): 44 | if env.host_string: 45 | if base_url: 46 | proto_idx = base_url.find(':/') 47 | if proto_idx >= 0: 48 | proto = base_url[:proto_idx] 49 | address = base_url[proto_idx + 2:] 50 | if proto in ('http+unix', 'unix'): 51 | if address[:3] == '//': 52 | address = address[1:] 53 | elif address[0] != '/': 54 | address = ''.join(('/', address)) 55 | return _get_socat_tunnel(address, local_port) 56 | return _get_local_tunnel(address.lstrip('/'), remote_port, local_port) 57 | elif base_url[0] == '/': 58 | return _get_socat_tunnel(base_url, local_port) 59 | return _get_local_tunnel(base_url, remote_port, local_port) 60 | return _get_socat_tunnel(DEFAULT_SOCKET, local_port) 61 | return base_url, None 62 | 63 | 64 | class DockerFabricClient(DockerClientWrapper): 65 | """ 66 | Docker client for Fabric. 67 | 68 | For functional enhancements to :class:`docker.client.Client`, see 69 | :class:`~dockermap.client.base.DockerClientWrapper`. This implementation only adds the possibility to build a 70 | tunnel through the current SSH connection and adds Fabric-usual logging. 71 | 72 | If a unix socket is used, `socat` will be started on the remote side to redirect it to a TCP port. 73 | 74 | :param base_url: URL to connect to; if not set, will refer to ``env.docker_base_url`` or use ``None``, which by 75 | default attempts a connection on a Unix socket at ``/var/run/docker.sock``. 76 | :type base_url: unicode 77 | :param tls: Whether to use TLS on the connection to the Docker service. 78 | :type tls: bool 79 | :param version: API version; if not set, will try to use ``env.docker_api_version``; otherwise defaults to 80 | :const:`~docker.constants.DEFAULT_DOCKER_API_VERSION`. 81 | :type version: unicode 82 | :param timeout: Client timeout for Docker; if not set, will try to use ``env.docker_timeout``; otherwise defaults to 83 | :const:`~docker.constants.DEFAULT_TIMEOUT_SECONDS`. 84 | :type timeout: int 85 | :param tunnel_remote_port: Optional, port of the remote service; if port is included in ``base_url``, the latter 86 | is preferred. If not set, will try to use ``env.docker_tunnel_remote_port``; otherwise defaults to ``None``. 87 | :type tunnel_remote_port: int 88 | :param tunnel_local_port: Optional, for SSH tunneling: Port to open towards the local end for the tunnel; if not 89 | provided, will try to use ``env.docker_tunnel_local_port``; otherwise defaults to the value of 90 | ``tunnel_remote_port`` or ``None`` for direct connections without an SSH tunnel. 91 | :type tunnel_local_port: int 92 | :param kwargs: Additional kwargs for :class:`docker.client.Client` 93 | """ 94 | def __init__(self, base_url=None, tls=None, version=None, timeout=None, tunnel_remote_port=None, 95 | tunnel_local_port=None, **kwargs): 96 | url = base_url or env.get('docker_base_url') 97 | use_tls = tls or (tls is None and env.get('docker_tls', False)) 98 | api_version = version or env.get('docker_api_version') 99 | client_timeout = timeout or env.get('docker_timeout') 100 | remote_port = tunnel_remote_port or env.get('docker_tunnel_remote_port') 101 | local_port = tunnel_local_port or env.get('docker_tunnel_local_port', remote_port) 102 | conn_url, self._tunnel = _get_connection_args(url, remote_port, local_port) 103 | super(DockerFabricClient, self).__init__(base_url=conn_url, version=api_version, timeout=client_timeout, 104 | tls=use_tls, **kwargs) 105 | 106 | def push_log(self, info, level=None, *args, **kwargs): 107 | """ 108 | Prints the log as usual for fabric output, enhanced with the prefix "docker". 109 | 110 | :param info: Log output. 111 | :type info: unicode 112 | :param level: Logging level. Has no effect here. 113 | :type level: int 114 | """ 115 | if args: 116 | msg = info % args 117 | else: 118 | msg = info 119 | try: 120 | puts('docker: {0}'.format(msg)) 121 | except UnicodeDecodeError: 122 | puts('docker: -- non-printable output --') 123 | 124 | def push_progress(self, status, object_id, progress): 125 | """ 126 | Prints progress information. 127 | 128 | :param status: Status text. 129 | :type status: unicode 130 | :param object_id: Object that the progress is reported on. 131 | :type object_id: unicode 132 | :param progress: Progress bar. 133 | :type progress: unicode 134 | """ 135 | fastprint(progress_fmt(status, object_id, progress), end='\n') 136 | 137 | def close(self): 138 | """ 139 | Closes the connection and any tunnels created for it. 140 | """ 141 | try: 142 | super(DockerFabricClient, self).close() 143 | finally: 144 | if self._tunnel is not None: 145 | self._tunnel.close() 146 | 147 | def build(self, tag, **kwargs): 148 | """ 149 | Identical to :meth:`dockermap.client.base.DockerClientWrapper.build` with additional logging. 150 | """ 151 | self.push_log("Building image '{0}'.".format(tag)) 152 | set_raise_on_error(kwargs) 153 | try: 154 | return super(DockerFabricClient, self).build(tag, **kwargs) 155 | except DockerStatusError as e: 156 | error(e.message) 157 | 158 | def create_container(self, image, name=None, **kwargs): 159 | """ 160 | Identical to :meth:`docker.api.container.ContainerApiMixin.create_container` with additional logging. 161 | """ 162 | name_str = " '{0}'".format(name) if name else "" 163 | self.push_log("Creating container{0} from image '{1}'.".format(name_str, image)) 164 | return super(DockerFabricClient, self).create_container(image, name=name, **kwargs) 165 | 166 | def copy_resource(self, container, resource, local_filename): 167 | """ 168 | Identical to :meth:`dockermap.client.base.DockerClientWrapper.copy_resource` with additional logging. 169 | """ 170 | self.push_log("Receiving tarball for resource '{0}:{1}' and storing as {2}".format(container, resource, local_filename)) 171 | super(DockerFabricClient, self).copy_resource(container, resource, local_filename) 172 | 173 | def cleanup_containers(self, include_initial=False, exclude=None, **kwargs): 174 | """ 175 | Identical to :meth:`dockermap.client.docker_util.DockerUtilityMixin.cleanup_containers` with additional logging. 176 | """ 177 | self.push_log("Generating list of stopped containers.") 178 | set_raise_on_error(kwargs, False) 179 | return super(DockerFabricClient, self).cleanup_containers(include_initial=include_initial, exclude=exclude, 180 | **kwargs) 181 | 182 | def cleanup_images(self, remove_old=False, keep_tags=None, **kwargs): 183 | """ 184 | Identical to :meth:`dockermap.client.docker_util.DockerUtilityMixin.cleanup_images` with additional logging. 185 | """ 186 | self.push_log("Checking images for dependent images and containers.") 187 | set_raise_on_error(kwargs, False) 188 | return super(DockerFabricClient, self).cleanup_images(remove_old=remove_old, keep_tags=keep_tags, **kwargs) 189 | 190 | def import_image(self, image=None, tag='latest', **kwargs): 191 | """ 192 | Identical to :meth:`docker.api.image.ImageApiMixin.import_image` with additional logging. 193 | """ 194 | self.push_log("Fetching image '{0}' from registry.".format(image)) 195 | return super(DockerFabricClient, self).import_image(image=image, tag=tag, **kwargs) 196 | 197 | def login(self, **kwargs): 198 | """ 199 | Identical to :meth:`dockermap.client.base.DockerClientWrapper.login` with two enhancements: 200 | 201 | * additional logging; 202 | * login parameters can be passed through ``kwargs``, or set as default using the following ``env`` 203 | variables: 204 | 205 | * ``env.docker_registry_user`` (kwarg: ``username``), 206 | * ``env.docker_registry_password`` (kwarg: ``password``), 207 | * ``env.docker_registry_mail`` (kwarg: ``email``), 208 | * ``env.docker_registry_repository`` (kwarg: ``registry``), 209 | * ``env.docker_registry_insecure`` (kwarg: ``insecure_registry``). 210 | """ 211 | c_user = kwargs.pop('username', env.get('docker_registry_user')) 212 | c_pass = kwargs.pop('password', env.get('docker_registry_password')) 213 | c_mail = kwargs.pop('email', env.get('docker_registry_mail')) 214 | c_registry = kwargs.pop('registry', env.get('docker_registry_repository')) 215 | c_insecure = kwargs.pop('insecure_registry', env.get('docker_registry_insecure')) 216 | if super(DockerFabricClient, self).login(c_user, password=c_pass, email=c_mail, registry=c_registry, 217 | insecure_registry=c_insecure, **kwargs): 218 | self.push_log("Login at registry '{0}' succeeded.".format(c_registry)) 219 | return True 220 | self.push_log("Login at registry '{0}' failed.".format(c_registry)) 221 | return False 222 | 223 | def pull(self, repository, tag=None, stream=True, **kwargs): 224 | """ 225 | Identical to :meth:`dockermap.client.base.DockerClientWrapper.pull` with two enhancements: 226 | 227 | * additional logging; 228 | * the ``insecure_registry`` flag can be passed through ``kwargs``, or set as default using 229 | ``env.docker_registry_insecure``. 230 | """ 231 | c_insecure = kwargs.pop('insecure_registry', env.get('docker_registry_insecure')) 232 | set_raise_on_error(kwargs) 233 | try: 234 | return super(DockerFabricClient, self).pull(repository, tag=tag, stream=stream, 235 | insecure_registry=c_insecure, **kwargs) 236 | except DockerStatusError as e: 237 | error(e.message) 238 | 239 | def push(self, repository, stream=True, **kwargs): 240 | """ 241 | Identical to :meth:`dockermap.client.base.DockerClientWrapper.push` with two enhancements: 242 | 243 | * additional logging; 244 | * the ``insecure_registry`` flag can be passed through ``kwargs``, or set as default using 245 | ``env.docker_registry_insecure``. 246 | """ 247 | c_insecure = kwargs.pop('insecure_registry', env.get('docker_registry_insecure')) 248 | set_raise_on_error(kwargs) 249 | try: 250 | return super(DockerFabricClient, self).push(repository, stream=stream, insecure_registry=c_insecure, 251 | **kwargs) 252 | except DockerStatusError as e: 253 | error(e.message) 254 | 255 | def restart(self, container, **kwargs): 256 | """ 257 | Identical to :meth:`docker.api.container.ContainerApiMixin.restart` with additional logging. 258 | """ 259 | self.push_log("Restarting container '{0}'.".format(container)) 260 | super(DockerFabricClient, self).restart(container, **kwargs) 261 | 262 | def remove_all_containers(self, **kwargs): 263 | """ 264 | Identical to :meth:`dockermap.client.docker_util.DockerUtilityMixin.remove_all_containers` with additional 265 | logging. 266 | """ 267 | self.push_log("Fetching container list.") 268 | set_raise_on_error(kwargs) 269 | super(DockerFabricClient, self).remove_all_containers(**kwargs) 270 | 271 | def remove_container(self, container, **kwargs): 272 | """ 273 | Identical to :meth:`dockermap.client.base.DockerClientWrapper.remove_container` with additional logging. 274 | """ 275 | self.push_log("Removing container '{0}'.".format(container)) 276 | set_raise_on_error(kwargs) 277 | super(DockerFabricClient, self).remove_container(container, **kwargs) 278 | 279 | def remove_image(self, image, **kwargs): 280 | """ 281 | Identical to :meth:`dockermap.client.base.DockerClientWrapper.remove_image` with additional logging. 282 | """ 283 | self.push_log("Removing image '{0}'.".format(image)) 284 | set_raise_on_error(kwargs) 285 | super(DockerFabricClient, self).remove_image(image, **kwargs) 286 | 287 | def save_image(self, image, local_filename): 288 | """ 289 | Identical to :meth:`dockermap.client.base.DockerClientWrapper.save_image` with additional logging. 290 | """ 291 | self.push_log("Receiving tarball for image '{0}' and storing as '{1}'".format(image, local_filename)) 292 | super(DockerFabricClient, self).save_image(image, local_filename) 293 | 294 | def start(self, container, **kwargs): 295 | """ 296 | Identical to :meth:`docker.api.container.ContainerApiMixin.start` with additional logging. 297 | """ 298 | self.push_log("Starting container '{0}'.".format(container)) 299 | super(DockerFabricClient, self).start(container, **kwargs) 300 | 301 | def stop(self, container, **kwargs): 302 | """ 303 | Identical to :meth:`dockermap.client.base.DockerClientWrapper.stop` with additional logging. 304 | """ 305 | self.push_log("Stopping container '{0}'.".format(container)) 306 | super(DockerFabricClient, self).stop(container, **kwargs) 307 | 308 | def wait(self, container, **kwargs): 309 | """ 310 | Identical to :meth:`docker.api.container.ContainerApiMixin.wait` with additional logging. 311 | """ 312 | self.push_log("Waiting for container '{0}'.".format(container)) 313 | super(DockerFabricClient, self).wait(container, **kwargs) 314 | 315 | def create_network(self, name, **kwargs): 316 | self.push_log("Creating network '{0}'.".format(name)) 317 | return super(DockerFabricClient, self).create_network(name, **kwargs) 318 | 319 | def remove_network(self, net_id, **kwargs): 320 | self.push_log("Removing network '{0}'.".format(net_id)) 321 | super(DockerFabricClient, self).remove_network(net_id, **kwargs) 322 | 323 | def connect_container_to_network(self, container, net_id, **kwargs): 324 | self.push_log("Connecting container '{0}' to network '{1}'.".format(container, net_id)) 325 | super(DockerFabricClient, self).connect_container_to_network(container, net_id, **kwargs) 326 | 327 | def disconnect_container_from_network(self, container, net_id, **kwargs): 328 | self.push_log("Disconnecting container '{0}' from network '{1}'.".format(container, net_id)) 329 | super(DockerFabricClient, self).disconnect_container_from_network(container, net_id, **kwargs) 330 | 331 | def create_volume(self, name, **kwargs): 332 | self.push_log("Creating volume '{0}'.".format(name)) 333 | super(DockerFabricClient, self).create_volume(name, **kwargs) 334 | 335 | def remove_volume(self, name, **kwargs): 336 | self.push_log("Removing volume '{0}'.".format(name)) 337 | super(DockerFabricClient, self).remove_volume(name, **kwargs) 338 | 339 | def run_cmd(self, command): 340 | sudo(command) 341 | 342 | 343 | class DockerClientConfiguration(FabricClientConfiguration): 344 | init_kwargs = FabricClientConfiguration.init_kwargs + ('tunnel_remote_port', 'tunnel_local_port') 345 | client_constructor = DockerFabricClient 346 | 347 | 348 | class DockerFabricApiConnections(DockerConnectionDict): 349 | configuration_class = DockerClientConfiguration 350 | 351 | 352 | class ContainerApiFabricClient(FabricContainerClient): 353 | configuration_class = DockerClientConfiguration 354 | 355 | 356 | # Still defined here for backwards compatibility. 357 | docker_fabric = DockerFabricApiConnections().get_connection 358 | container_fabric = ContainerApiFabricClient 359 | -------------------------------------------------------------------------------- /dockerfabric/base.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | import ctypes 5 | import logging 6 | import multiprocessing 7 | 8 | from dockermap.api import MappingDockerClient, ClientConfiguration 9 | from fabric.api import env, settings 10 | 11 | log = logging.getLogger(__name__) 12 | port_offset = multiprocessing.Value(ctypes.c_ulong) 13 | 14 | 15 | def _get_default_config(client_configs): 16 | clients = client_configs or env.get('docker_clients') 17 | host_string = env.get('host_string') 18 | if not host_string or not clients: 19 | return None 20 | for c in clients.values(): 21 | host = c.get('fabric_host') 22 | if host == host_string: 23 | return c 24 | return None 25 | 26 | 27 | class ConnectionDict(dict): 28 | def get_or_create_connection(self, key, d, *args, **kwargs): 29 | e = self.get(key) 30 | if e is None: 31 | log.debug("Creating new %s connection for key %s with args: %s, kwargs: %s", 32 | self.__class__.__name__, key, args, kwargs) 33 | self[key] = e = d(*args, **kwargs) 34 | return e 35 | 36 | 37 | class DockerConnectionDict(ConnectionDict): 38 | """ 39 | Cache for connections to Docker clients. 40 | """ 41 | configuration_class = None 42 | 43 | def get_connection(self, *args, **kwargs): 44 | """ 45 | Create a new connection, or return an existing one from the cache. Uses Fabric's current ``env.host_string`` 46 | and the URL to the Docker service. 47 | 48 | :param args: Additional arguments for the client constructor, if a new client has to be instantiated. 49 | :param kwargs: Additional keyword args for the client constructor, if a new client has to be instantiated. 50 | """ 51 | key = env.get('host_string'), kwargs.get('base_url', env.get('docker_base_url')) 52 | default_config = _get_default_config(None) 53 | if default_config: 54 | if key not in self: 55 | self[key] = default_config 56 | return default_config.get_client() 57 | config = self.get_or_create_connection(key, self.configuration_class, *args, **kwargs) 58 | return config.get_client() 59 | 60 | 61 | class FabricClientConfiguration(ClientConfiguration): 62 | def get_client(self): 63 | if 'fabric_host' in self: 64 | with settings(host_string=self.fabric_host): 65 | return super(FabricClientConfiguration, self).get_client() 66 | return super(FabricClientConfiguration, self).get_client() 67 | 68 | 69 | class FabricContainerClient(MappingDockerClient): 70 | """ 71 | Convenience class for using a :class:`~dockermap.map.config.main.ContainerMap` on a :class:`DockerFabricClient`. 72 | 73 | :param container_maps: Container map or a tuple / list thereof. 74 | :type container_maps: list[dockermap.map.config.main.ContainerMap] | dockermap.map.config.main.ContainerMap 75 | :param docker_client: Default Docker client instance. 76 | :type docker_client: FabricClientConfiguration 77 | :param clients: Optional dictionary of Docker client configuration objects. 78 | :type clients: dict[unicode | str, FabricClientConfiguration] 79 | """ 80 | def __init__(self, container_maps=None, docker_client=None, clients=None): 81 | all_maps = container_maps or env.get('docker_maps', ()) 82 | if not isinstance(all_maps, (list, tuple)): 83 | env_maps = all_maps, 84 | else: 85 | env_maps = all_maps 86 | all_configs = clients or env.get('docker_clients', dict()) 87 | current_clients = dict() 88 | 89 | default_client = docker_client or _get_default_config(all_configs) 90 | for c_map in env_maps: 91 | map_clients = set(c_map.clients or ()) 92 | for config_name, c_config in c_map: 93 | if c_config.clients: 94 | map_clients.update(c_config.clients) 95 | for map_client in map_clients: 96 | if map_client not in current_clients: 97 | client_config = all_configs.get(map_client) 98 | if not client_config: 99 | raise ValueError("Client '{0}' used in map '{1}' not configured.".format(map_client, c_map.name)) 100 | client_host = client_config.get('fabric_host') 101 | if not client_host: 102 | raise ValueError("Client '{0}' is configured, but has no 'fabric_host' definition.".format(map_client)) 103 | current_clients[map_client] = client_config 104 | 105 | if not (default_client or clients): 106 | default_client = self.configuration_class() 107 | super(FabricContainerClient, self).__init__(container_maps=all_maps, docker_client=default_client, 108 | clients=current_clients) 109 | 110 | def __enter__(self): 111 | return self 112 | 113 | def __exit__(self, exc_type, exc_val, exc_tb): 114 | pass 115 | 116 | 117 | def get_local_port(init_port): 118 | with port_offset.get_lock(): 119 | current_offset = port_offset.value 120 | port_offset.value += 1 121 | return int(init_port) + current_offset 122 | 123 | 124 | def set_raise_on_error(kwargs, default=True): 125 | r = kwargs.get('raise_on_error') 126 | if r is None: 127 | return env.get('docker_default_raise_on_error', default) 128 | return r 129 | -------------------------------------------------------------------------------- /dockerfabric/cli.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | import os 5 | import posixpath 6 | 7 | from fabric.api import cd, env, fastprint, get, put, run, settings, sudo 8 | from fabric.network import needs_host 9 | 10 | from dockermap.api import USE_HC_MERGE 11 | from dockermap.client.cli import (DockerCommandLineOutput, parse_containers_output, parse_inspect_output, 12 | parse_images_output, parse_version_output, parse_top_output, parse_networks_output, 13 | parse_volumes_output) 14 | from dockermap.client.docker_util import DockerUtilityMixin 15 | from dockermap.shortcuts import chmod, chown, targz, mkdir 16 | 17 | from .base import DockerConnectionDict, FabricContainerClient, FabricClientConfiguration 18 | from .utils.containers import temp_container 19 | from .utils.files import temp_dir, is_directory 20 | 21 | 22 | def _find_image_id(output): 23 | for line in reversed(output.splitlines()): 24 | if line and line.startswith('Successfully built '): 25 | return line[19:] # Remove prefix 26 | return None 27 | 28 | 29 | class DockerCliClient(DockerUtilityMixin): 30 | """ 31 | Docker client for Fabric using the command line interface on a remote host. 32 | 33 | :param cmd_prefix: Custom prefix to prepend to the Docker command line. 34 | :type cmd_prefix: unicode 35 | :param default_bin: Docker binary to use. If not set, uses ``docker``. 36 | :type default_bin: unicode 37 | :param base_url: URL to connect to; if not set, will refer to ``env.docker_base_url`` or use ``None``, which by 38 | default attempts a connection on a Unix socket at ``/var/run/docker.sock``. 39 | :type base_url: unicode 40 | :param tls: Whether to use TLS on the connection to the Docker service. 41 | :type tls: bool 42 | :param use_sudo: Whether to use ``sudo`` when performing Docker commands. 43 | :type use_sudo: bool 44 | :param debug: If set to ``True``, echoes each command and its console output. Some commands are echoed either way 45 | for some feedback. 46 | :type debug: bool 47 | """ 48 | def __init__(self, cmd_prefix=None, default_bin=None, base_url=None, tls=None, use_sudo=None, debug=None): 49 | super(DockerCliClient, self).__init__() 50 | base_url = base_url or env.get('docker_base_url') 51 | if base_url: 52 | cmd_args = ['-H {0}'.format(base_url)] 53 | else: 54 | cmd_args = [] 55 | if tls or (tls is None and env.get('docker_tls')): 56 | cmd_args.append('--tls') 57 | self._out = DockerCommandLineOutput(cmd_prefix or env.get('docker_cli_prefix'), 58 | default_bin or env.get('docker_cli_bin', 'docker'), cmd_args or None) 59 | if use_sudo or (use_sudo is None and env.get('docker_cli_sudo')): 60 | self._call_method = sudo 61 | else: 62 | self._call_method = run 63 | self._quiet = not (debug or (debug is None and env.get('docker_cli_debug'))) 64 | self.api_version = None 65 | self._update_api_version() 66 | 67 | def _call(self, cmd, quiet=False): 68 | if cmd: 69 | return self._call_method(cmd, shell=False, quiet=quiet and self._quiet) 70 | return None 71 | 72 | def create_container(self, *args, **kwargs): 73 | cmd_str = self._out.get_cmd('create_container', *args, **kwargs) 74 | return {'Id': self._call(cmd_str)} 75 | 76 | def start(self, *args, **kwargs): 77 | cmd_str = self._out.get_cmd('start', *args, **kwargs) 78 | self._call(cmd_str) 79 | 80 | def restart(self, *args, **kwargs): 81 | cmd_str = self._out.get_cmd('restart', *args, **kwargs) 82 | self._call(cmd_str) 83 | 84 | def stop(self, *args, **kwargs): 85 | cmd_str = self._out.get_cmd('stop', *args, **kwargs) 86 | self._call(cmd_str) 87 | 88 | def remove_container(self, *args, **kwargs): 89 | cmd_str = self._out.get_cmd('remove_container', *args, **kwargs) 90 | self._call(cmd_str) 91 | 92 | def remove_image(self, *args, **kwargs): 93 | cmd_str = self._out.get_cmd('remove_image', *args, **kwargs) 94 | self._call(cmd_str) 95 | 96 | def kill(self, *args, **kwargs): 97 | cmd_str = self._out.get_cmd('kill', *args, **kwargs) 98 | self._call(cmd_str) 99 | 100 | def wait(self, *args, **kwargs): 101 | cmd_str = self._out.get_cmd('wait', *args, **kwargs) 102 | self._call(cmd_str) 103 | 104 | def containers(self, *args, **kwargs): 105 | cmd_str = self._out.get_cmd('containers', *args, **kwargs) 106 | res = self._call(cmd_str, quiet=True) 107 | return parse_containers_output(res) 108 | 109 | def inspect_container(self, *args, **kwargs): 110 | cmd_str = self._out.get_cmd('inspect_container', *args, **kwargs) 111 | res = self._call(cmd_str, quiet=True) 112 | return parse_inspect_output(res, 'container') 113 | 114 | def images(self, *args, **kwargs): 115 | cmd_str = self._out.get_cmd('images', *args, **kwargs) 116 | res = self._call(cmd_str, quiet=True) 117 | return parse_images_output(res) 118 | 119 | def pull(self, repository, tag=None, **kwargs): 120 | repo_tag = '{0}:{1}'.format(repository, tag) if tag else repository 121 | cmd_str = self._out.get_cmd('pull', repo_tag, **kwargs) 122 | self._call(cmd_str) 123 | 124 | def push(self, repository, tag=None, **kwargs): 125 | repo_tag = '{0}:{1}'.format(repository, tag) if tag else repository 126 | cmd_str = self._out.get_cmd('push', repo_tag, **kwargs) 127 | self._call(cmd_str) 128 | 129 | def create_network(self, *args, **kwargs): 130 | cmd_str = self._out.get_cmd('create_network', *args, **kwargs) 131 | return {'Id': self._call(cmd_str)} 132 | 133 | def remove_network(self, *args, **kwargs): 134 | cmd_str = self._out.get_cmd('remove_network', *args, **kwargs) 135 | self._call(cmd_str) 136 | 137 | def connect_container_to_network(self, *args, **kwargs): 138 | cmd_str = self._out.get_cmd('connect_container_to_network', *args, **kwargs) 139 | self._call(cmd_str) 140 | 141 | def disconnect_container_from_network(self, *args, **kwargs): 142 | cmd_str = self._out.get_cmd('disconnect_container_from_network', *args, **kwargs) 143 | self._call(cmd_str) 144 | 145 | def networks(self, *args, **kwargs): 146 | cmd_str = self._out.get_cmd('networks', *args, **kwargs) 147 | res = self._call(cmd_str, quiet=True) 148 | return parse_networks_output(res) 149 | 150 | def inspect_network(self, *args, **kwargs): 151 | cmd_str = self._out.get_cmd('inspect_network', *args, **kwargs) 152 | res = self._call(cmd_str, quiet=True) 153 | return parse_inspect_output(res, 'network') 154 | 155 | def create_volume(self, *args, **kwargs): 156 | cmd_str = self._out.get_cmd('create_volume', *args, **kwargs) 157 | return {'Name': self._call(cmd_str)} 158 | 159 | def remove_volume(self, *args, **kwargs): 160 | cmd_str = self._out.get_cmd('remove_volume', *args, **kwargs) 161 | self._call(cmd_str) 162 | 163 | def volumes(self, *args, **kwargs): 164 | cmd_str = self._out.get_cmd('volumes', *args, **kwargs) 165 | res = self._call(cmd_str, quiet=True) 166 | return {'Volumes': parse_volumes_output(res), 'Warnings': None} 167 | 168 | def inspect_volume(self, *args, **kwargs): 169 | cmd_str = self._out.get_cmd('inspect_volume', *args, **kwargs) 170 | res = self._call(cmd_str, quiet=True) 171 | return parse_inspect_output(res, 'volume') 172 | 173 | def exec_create(self, *args, **kwargs): 174 | cmd_str = self._out.get_cmd('exec_create', *args, **kwargs) 175 | self._call(cmd_str) 176 | 177 | def exec_start(self, *args, **kwargs): 178 | cmd_str = self._out.get_cmd('exec_start', *args, **kwargs) 179 | self._call(cmd_str) 180 | 181 | def top(self, container, ps_args): 182 | if ps_args: 183 | cmd_str = self._out.get_cmd('top', container, ps_args) 184 | else: 185 | cmd_str = self._out.get_cmd('top', container) 186 | res = self._call(cmd_str, quiet=True) 187 | return parse_top_output(res) 188 | 189 | def tag(self, image, repository, tag=None, **kwargs): 190 | if tag: 191 | repo_tag = '{0}:{1}'.format(repository, tag) 192 | else: 193 | repo_tag = repository 194 | cmd_str = self._out.get_cmd('tag', image, repo_tag, **kwargs) 195 | return self._call(cmd_str) 196 | 197 | def logs(self, *args, **kwargs): 198 | kwargs.pop('stream', None) 199 | cmd_str = self._out.get_cmd('logs', *args, **kwargs) 200 | return self._call(cmd_str, quiet=True) 201 | 202 | def login(self, **kwargs): 203 | for key, variable in [ 204 | ('username', 'user'), 205 | ('password', 'password'), 206 | ('email', 'mail'), 207 | ('registry', 'repository'), 208 | ('insecure_registry', 'insecure') 209 | ]: 210 | if key not in kwargs: 211 | env_value = env.get('docker_registry_{0}'.format(variable)) 212 | if env_value: 213 | kwargs[key] = env_value 214 | registry = kwargs.pop('registry', env.get('docker_registry_repository')) 215 | if registry: 216 | cmd_str = self._out.get_cmd('login', registry, **kwargs) 217 | else: 218 | cmd_str = self._out.get_cmd('login', **kwargs) 219 | res = self._call(cmd_str, quiet=True) 220 | lines = res.splitlines() 221 | fastprint(lines) 222 | return 'Login Succeeded' in lines 223 | 224 | def build(self, tag, add_latest_tag=False, add_tags=None, raise_on_error=True, **kwargs): 225 | try: 226 | context = kwargs.pop('fileobj') 227 | except KeyError: 228 | raise ValueError("'fileobj' needs to be provided. Using 'path' is currently not implemented.") 229 | for a in ['custom_context', 'encoding']: 230 | kwargs.pop(a, None) 231 | 232 | with temp_dir() as remote_tmp: 233 | remote_fn = posixpath.join(remote_tmp, 'context') 234 | put(context, remote_fn) 235 | cmd_str = self._out.get_cmd('build', '- <', remote_fn, tag=tag, **kwargs) 236 | with settings(warn_only=not raise_on_error): 237 | res = self._call(cmd_str) 238 | if res: 239 | image_id = _find_image_id(res) 240 | if image_id: 241 | self.add_extra_tags(image_id, tag, add_tags, add_latest_tag) 242 | return image_id 243 | return None 244 | 245 | def version(self, **kwargs): 246 | kwargs.pop('api_version', None) 247 | cmd_str = self._out.get_cmd('version') 248 | res = self._call(cmd_str, quiet=True) 249 | version_dict = parse_version_output(res) 250 | return version_dict 251 | 252 | def push_log(self, info, level, *args, **kwargs): 253 | pass 254 | 255 | def _update_api_version(self): 256 | if self.api_version and self.api_version != 'auto': 257 | return 258 | version_dict = self.version() 259 | if 'APIVersion' in version_dict: 260 | self.api_version = version_dict['APIVersion'] 261 | elif 'ApiVersion' in version_dict: 262 | self.api_version = version_dict['ApiVersion'] 263 | 264 | def run_cmd(self, command): 265 | sudo(command) 266 | 267 | 268 | class DockerCliConfig(FabricClientConfiguration): 269 | init_kwargs = 'base_url', 'tls', 'cmd_prefix', 'default_bin', 'use_sudo', 'debug' 270 | client_constructor = DockerCliClient 271 | 272 | def update_settings(self, **kwargs): 273 | super(DockerCliConfig, self).update_settings(**kwargs) 274 | self.use_host_config = USE_HC_MERGE 275 | 276 | 277 | class DockerCliConnections(DockerConnectionDict): 278 | configuration_class = DockerCliConfig 279 | 280 | 281 | class ContainerCliFabricClient(FabricContainerClient): 282 | configuration_class = DockerCliConfig 283 | 284 | 285 | docker_cli = DockerCliConnections().get_connection 286 | container_cli = ContainerCliFabricClient 287 | 288 | 289 | @needs_host 290 | def copy_resource(container, resource, local_filename, contents_only=True): 291 | """ 292 | Copies a resource from a container to a compressed tarball and downloads it. 293 | 294 | :param container: Container name or id. 295 | :type container: unicode 296 | :param resource: Name of resource to copy. 297 | :type resource: unicode 298 | :param local_filename: Path to store the tarball locally. 299 | :type local_filename: unicode 300 | :param contents_only: In case ``resource`` is a directory, put all contents at the root of the tar file. If this is 301 | set to ``False``, the directory itself will be at the root instead. 302 | :type contents_only: bool 303 | """ 304 | with temp_dir() as remote_tmp: 305 | base_name = os.path.basename(resource) 306 | copy_path = posixpath.join(remote_tmp, 'copy_tmp') 307 | run(mkdir(copy_path, check_if_exists=True)) 308 | remote_name = posixpath.join(copy_path, base_name) 309 | archive_name = 'container_{0}.tar.gz'.format(container) 310 | archive_path = posixpath.join(remote_tmp, archive_name) 311 | run('docker cp {0}:{1} {2}'.format(container, resource, copy_path), shell=False) 312 | if contents_only and is_directory(remote_name): 313 | src_dir = remote_name 314 | src_files = '*' 315 | else: 316 | src_dir = copy_path 317 | src_files = base_name 318 | with cd(src_dir): 319 | run(targz(archive_path, src_files)) 320 | get(archive_path, local_filename) 321 | 322 | 323 | @needs_host 324 | def copy_resources(src_container, src_resources, storage_dir, dst_directories=None, apply_chown=None, apply_chmod=None): 325 | """ 326 | Copies files and directories from a Docker container. Multiple resources can be copied and additional options are 327 | available than in :func:`copy_resource`. Unlike in :func:`copy_resource`, Resources are copied as they are and not 328 | compressed to a tarball, and they are left on the remote machine. 329 | 330 | :param src_container: Container name or id. 331 | :type src_container: unicode 332 | :param src_resources: Resources, as (file or directory) names to copy. 333 | :type src_resources: iterable 334 | :param storage_dir: Remote directory to store the copied objects in. 335 | :type storage_dir: unicode 336 | :param dst_directories: Optional dictionary of destination directories, in the format ``resource: destination``. If 337 | not set, resources will be in the same relative structure to one another as inside the container. For setting a 338 | common default, use ``*`` as the resource key. 339 | :type dst_directories: dict 340 | :param apply_chown: Owner to set for the copied resources. Can be a user name or id, group name or id, both in the 341 | notation ``user:group``, or as a tuple ``(user, group)``. 342 | :type apply_chown: unicode or tuple 343 | :param apply_chmod: File system permissions to set for the copied resources. Can be any notation as accepted by 344 | `chmod`. 345 | :type apply_chmod: unicode 346 | """ 347 | def _copy_resource(resource): 348 | default_dest_path = generic_path if generic_path is not None else resource 349 | dest_path = directories.get(resource, default_dest_path).strip(posixpath.sep) 350 | head, tail = posixpath.split(dest_path) 351 | rel_path = posixpath.join(storage_dir, head) 352 | run(mkdir(rel_path, check_if_exists=True)) 353 | run('docker cp {0}:{1} {2}'.format(src_container, resource, rel_path), shell=False) 354 | 355 | directories = dst_directories or {} 356 | generic_path = directories.get('*') 357 | for res in src_resources: 358 | _copy_resource(res) 359 | if apply_chmod: 360 | run(chmod(apply_chmod, storage_dir)) 361 | if apply_chown: 362 | sudo(chown(apply_chown, storage_dir)) 363 | 364 | 365 | @needs_host 366 | def isolate_and_get(src_container, src_resources, local_dst_dir, **kwargs): 367 | """ 368 | Uses :func:`copy_resources` to copy resources from a container, but afterwards generates a compressed tarball 369 | and downloads it. 370 | 371 | :param src_container: Container name or id. 372 | :type src_container: unicode 373 | :param src_resources: Resources, as (file or directory) names to copy. 374 | :type src_resources: iterable 375 | :param local_dst_dir: Local directory to store the compressed tarball in. Can also be a file name; the default file 376 | name is ``container_.tar.gz``. 377 | :type local_dst_dir: unicode 378 | :param kwargs: Additional kwargs for :func:`copy_resources`. 379 | """ 380 | with temp_dir() as remote_tmp: 381 | copy_path = posixpath.join(remote_tmp, 'copy_tmp') 382 | archive_path = posixpath.join(remote_tmp, 'container_{0}.tar.gz'.format(src_container)) 383 | copy_resources(src_container, src_resources, copy_path, **kwargs) 384 | with cd(copy_path): 385 | sudo(targz(archive_path, '*')) 386 | get(archive_path, local_dst_dir) 387 | 388 | 389 | @needs_host 390 | def isolate_to_image(src_container, src_resources, dst_image, **kwargs): 391 | """ 392 | Uses :func:`copy_resources` to copy resources from a container, but afterwards imports the contents into a new 393 | (otherwise empty) Docker image. 394 | 395 | :param src_container: Container name or id. 396 | :type src_container: unicode 397 | :param src_resources: Resources, as (file or directory) names to copy. 398 | :type src_resources: iterable 399 | :param dst_image: Tag for the new image. 400 | :type dst_image: unicode 401 | :param kwargs: Additional kwargs for :func:`copy_resources`. 402 | """ 403 | with temp_dir() as remote_tmp: 404 | copy_resources(src_container, src_resources, remote_tmp, **kwargs) 405 | with cd(remote_tmp): 406 | sudo('tar -cz * | docker import - {0}'.format(dst_image)) 407 | 408 | 409 | @needs_host 410 | def save_image(image, local_filename): 411 | """ 412 | Saves a Docker image as a compressed tarball. This command line client method is a suitable alternative, if the 413 | Remove API method is too slow. 414 | 415 | :param image: Image id or tag. 416 | :type image: unicode 417 | :param local_filename: Local file name to store the image into. If this is a directory, the image will be stored 418 | there as a file named ``image_.tar.gz``. 419 | """ 420 | r_name, __, i_name = image.rpartition('/') 421 | i_name, __, __ = i_name.partition(':') 422 | with temp_dir() as remote_tmp: 423 | archive = posixpath.join(remote_tmp, 'image_{0}.tar.gz'.format(i_name)) 424 | run('docker save {0} | gzip --stdout > {1}'.format(image, archive), shell=False) 425 | get(archive, local_filename) 426 | 427 | 428 | @needs_host 429 | def flatten_image(image, dest_image=None, no_op_cmd='/bin/true', create_kwargs={}, start_kwargs={}): 430 | """ 431 | Exports a Docker image's file system and re-imports it into a new (otherwise new) image. Note that this does not 432 | transfer the image configuration. In order to gain access to the container contents, the image is started with a 433 | non-operational command, such as ``/bin/true``. The container is removed once the new image has been created. 434 | 435 | :param image: Image id or tag. 436 | :type image: unicode 437 | :param dest_image: Tag for the new image. 438 | :type dest_image: unicode 439 | :param no_op_cmd: Dummy command for starting temporary container. 440 | :type no_op_cmd: unicode 441 | :param create_kwargs: Optional additional kwargs for creating the temporary container. 442 | :type create_kwargs: dict 443 | :param start_kwargs: Optional additional kwargs for starting the temporary container. 444 | :type start_kwargs: dict 445 | """ 446 | dest_image = dest_image or image 447 | with temp_container(image, no_op_cmd=no_op_cmd, create_kwargs=create_kwargs, start_kwargs=start_kwargs) as c: 448 | run('docker export {0} | docker import - {1}'.format(c, dest_image), shell=False) 449 | -------------------------------------------------------------------------------- /dockerfabric/socat.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from fabric.state import env 5 | from fabric.utils import puts 6 | 7 | from .base import ConnectionDict, get_local_port 8 | from .tunnel import LocalTunnel 9 | 10 | 11 | class SocketTunnels(ConnectionDict): 12 | """ 13 | Cache for **socat** tunnels to the remote machine. 14 | 15 | Instantiation of :class:`SocketTunnel` can be configured with ``env.socat_quiet``, setting 16 | the ``quiet`` keyword argument. 17 | """ 18 | def __getitem__(self, item): 19 | """ 20 | :param item: Tuple of remote socket name, remote port, and local port number. 21 | :type item: tuple 22 | :return: Socket tunnel 23 | :rtype: SocketTunnel 24 | """ 25 | def _connect_socket_tunnel(): 26 | local_port = get_local_port(init_local_port) 27 | svc = SocketTunnel(remote_socket, local_port, env.get('socat_quiet', True)) 28 | svc.connect() 29 | return svc 30 | 31 | remote_socket, init_local_port = item 32 | key = env.host_string, remote_socket 33 | return self.get_or_create_connection(key, _connect_socket_tunnel) 34 | 35 | 36 | socat_tunnels = SocketTunnels() 37 | 38 | 39 | class SocketTunnel(LocalTunnel): 40 | """ 41 | Establish a tunnel from the local machine to the SSH host and from there start a **socat** process for forwarding 42 | traffic between the remote-end `stdout` and a Unix socket. 43 | 44 | :param remote_socket: Unix socket to connect to on the remote machine. 45 | :type remote_socket: unicode 46 | :param local_port: Local TCP port to use for the tunnel. 47 | :type local_port: int 48 | :param quiet: If set to ``False``, the **socat** command line on the SSH channel will be written to `stdout`. 49 | :type quiet: bool 50 | """ 51 | def __init__(self, remote_socket, local_port, quiet=True): 52 | dest = 'STDIO' 53 | src = 'UNIX-CONNECT:{0}'.format(remote_socket) 54 | self.quiet = quiet 55 | self._socat_cmd = ' '.join(('socat', dest, src)) 56 | super(SocketTunnel, self).__init__(local_port) 57 | 58 | def get_channel(self, transport, remote_addr, local_peer): 59 | channel = transport.open_channel('session') 60 | if channel is None: 61 | raise Exception("Failed to open channel on the SSH server.") 62 | if not self.quiet: 63 | puts(self._socat_cmd) 64 | channel.exec_command(self._socat_cmd) 65 | return channel 66 | -------------------------------------------------------------------------------- /dockerfabric/tasks.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from datetime import datetime 5 | import itertools 6 | from fabric.api import env, run, runs_once, sudo, task 7 | from fabric.utils import puts, fastprint 8 | import six 9 | 10 | from dockermap.utils import expand_path 11 | from . import cli 12 | from .api import docker_fabric 13 | from .utils.net import get_ip4_address, get_ip6_address 14 | from .utils.output import stdout_result 15 | 16 | 17 | IMAGE_COLUMNS = ('Id', 'RepoTags', 'ParentId', 'Created', 'VirtualSize', 'Size') 18 | CONTAINER_COLUMNS = ('Id', 'Names', 'Image', 'Command', 'Ports', 'Status', 'Created') 19 | NETWORK_COLUMNS = ('Id', 'Name', 'Driver', 'Scope') 20 | VOLUME_COLUMNS = ('Name', 'Driver') 21 | 22 | 23 | def _format_output_table(data_dict, columns, full_ids=False, full_cmd=False, short_image=False): 24 | def _format_port(port_dict): 25 | if 'PublicPort' in port_dict and 'IP' in port_dict: 26 | return '{IP}:{PublicPort}->{PrivatePort}/{Type}'.format(**port_dict) 27 | return '{PrivatePort}/{Type}'.format(**port_dict) 28 | 29 | def _get_column(item, column): 30 | data = item.get(column, '') 31 | if isinstance(data, list): 32 | if column == 'Ports': 33 | return map(_format_port, data) 34 | return data 35 | if column in ('Id', 'ParentId') and not full_ids: 36 | return data[:12], 37 | if column == 'Created': 38 | return datetime.utcfromtimestamp(data).isoformat(), 39 | if column == 'Command' and not full_cmd: 40 | return data[:25], 41 | if column == 'Image' and short_image: 42 | __, __, i_name = data.rpartition('/') 43 | return i_name, 44 | return unicode(data), 45 | 46 | def _max_len(col_data): 47 | if col_data: 48 | return max(map(len, col_data)) 49 | return 0 50 | 51 | puts('') 52 | rows = [[[c] for c in columns]] 53 | rows.extend([_get_column(i, col) for col in columns] for i in data_dict) 54 | col_lens = map(max, (map(_max_len, c) for c in zip(*rows))) 55 | row_format = ' '.join('{{{0}:{1}}}'.format(i, l) for i, l in enumerate(col_lens)) 56 | for row in rows: 57 | for c in itertools.izip_longest(*row, fillvalue=''): 58 | fastprint(row_format.format(*c), end='\n', flush=False) 59 | fastprint('', flush=True) 60 | 61 | 62 | @task 63 | def reset_socat(use_sudo=False): 64 | """ 65 | Finds and closes all processes of `socat`. 66 | 67 | :param use_sudo: Use `sudo` command. As Docker-Fabric does not run `socat` with `sudo`, this is by default set to 68 | ``False``. Setting it to ``True`` could unintentionally remove instances from other users. 69 | :type use_sudo: bool 70 | """ 71 | output = stdout_result('ps -o pid -C socat', quiet=True) 72 | pids = output.split('\n')[1:] 73 | puts("Removing process(es) with id(s) {0}.".format(', '.join(pids))) 74 | which = sudo if use_sudo else run 75 | which('kill {0}'.format(' '.join(pids)), quiet=True) 76 | 77 | 78 | @task 79 | def version(): 80 | """ 81 | Shows version information of the remote Docker service, similar to ``docker version``. 82 | """ 83 | output = docker_fabric().version() 84 | col_len = max(map(len, output.keys())) + 1 85 | puts('') 86 | for k, v in six.iteritems(output): 87 | fastprint('{0:{1}} {2}'.format(''.join((k, ':')), col_len, v), end='\n', flush=False) 88 | fastprint('', flush=True) 89 | 90 | 91 | @task 92 | def get_ip(interface_name='docker0'): 93 | """ 94 | Shows the IP4 address of a network interface. 95 | 96 | :param interface_name: Name of the network interface. Default is ``docker0``. 97 | :type interface_name: unicode 98 | """ 99 | puts(get_ip4_address(interface_name)) 100 | 101 | 102 | @task 103 | def get_ipv6(interface_name='docker0', expand=False): 104 | """ 105 | Shows the IP6 address of a network interface. 106 | 107 | :param interface_name: Name of the network interface. Default is ``docker0``. 108 | :type interface_name: unicode 109 | :param expand: Expand the abbreviated IP6 address. Default is ``False``. 110 | :type expand: bool 111 | """ 112 | puts(get_ip6_address(interface_name, expand=expand)) 113 | 114 | 115 | @task 116 | def list_images(list_all=False, full_ids=False): 117 | """ 118 | Lists images on the Docker remote host, similar to ``docker images``. 119 | 120 | :param list_all: Lists all images (e.g. dependencies). Default is ``False``, only shows named images. 121 | :type list_all: bool 122 | :param full_ids: Shows the full ids. When ``False`` (default) only shows the first 12 characters. 123 | :type full_ids: bool 124 | """ 125 | images = docker_fabric().images(all=list_all) 126 | _format_output_table(images, IMAGE_COLUMNS, full_ids) 127 | 128 | 129 | @task 130 | def list_containers(list_all=True, short_image=True, full_ids=False, full_cmd=False): 131 | """ 132 | Lists containers on the Docker remote host, similar to ``docker ps``. 133 | 134 | :param list_all: Shows all containers. Default is ``False``, which omits exited containers. 135 | :type list_all: bool 136 | :param short_image: Hides the repository prefix for preserving space. Default is ``True``. 137 | :type short_image: bool 138 | :param full_ids: Shows the full image ids. When ``False`` (default) only shows the first 12 characters. 139 | :type full_ids: bool 140 | :param full_cmd: Shows the full container command. When ``False`` (default) only shows the first 25 characters. 141 | :type full_cmd: bool 142 | """ 143 | containers = docker_fabric().containers(all=list_all) 144 | _format_output_table(containers, CONTAINER_COLUMNS, full_ids, full_cmd, short_image) 145 | 146 | 147 | @task 148 | def list_networks(full_ids=False): 149 | """ 150 | Lists networks on the Docker remote host, similar to ``docker network ls``. 151 | 152 | :param full_ids: Shows the full network ids. When ``False`` (default) only shows the first 12 characters. 153 | :type full_ids: bool 154 | """ 155 | networks = docker_fabric().networks() 156 | _format_output_table(networks, NETWORK_COLUMNS, full_ids) 157 | 158 | 159 | @task 160 | def list_volumes(): 161 | """ 162 | Lists volumes on the Docker remote host, similar to ``docker volume ls``. 163 | """ 164 | volumes = docker_fabric().volumes()['Volumes'] or () 165 | _format_output_table(volumes, VOLUME_COLUMNS) 166 | 167 | 168 | @task 169 | def cleanup_containers(**kwargs): 170 | """ 171 | Removes all containers that have finished running. Similar to the ``prune`` functionality in newer Docker versions. 172 | """ 173 | containers = docker_fabric().cleanup_containers(**kwargs) 174 | if kwargs.get('list_only'): 175 | puts('Existing containers:') 176 | for c_id, c_name in containers: 177 | fastprint('{0} {1}'.format(c_id, c_name), end='\n') 178 | 179 | 180 | @task 181 | def cleanup_images(remove_old=False, **kwargs): 182 | """ 183 | Removes all images that have no name, and that are not references as dependency by any other named image. Similar 184 | to the ``prune`` functionality in newer Docker versions, but supports more filters. 185 | 186 | :param remove_old: Also remove images that do have a name, but no `latest` tag. 187 | :type remove_old: bool 188 | """ 189 | keep_tags = env.get('docker_keep_tags') 190 | if keep_tags is not None: 191 | kwargs.setdefault('keep_tags', keep_tags) 192 | removed_images = docker_fabric().cleanup_images(remove_old=remove_old, **kwargs) 193 | if kwargs.get('list_only'): 194 | puts('Unused images:') 195 | for image_name in removed_images: 196 | fastprint(image_name, end='\n') 197 | 198 | 199 | @task 200 | def remove_all_containers(**kwargs): 201 | """ 202 | Stops and removes all containers from the remote. Use with caution outside of a development environment! 203 | :return: 204 | """ 205 | containers = docker_fabric().remove_all_containers(**kwargs) 206 | if kwargs.get('list_only'): 207 | puts('Existing containers:') 208 | for c_id in containers[1]: 209 | fastprint(c_id, end='\n') 210 | 211 | 212 | @task 213 | @runs_once 214 | def save_image(image, filename=None): 215 | """ 216 | Saves a Docker image from the remote to a local files. For performance reasons, uses the Docker command line client 217 | on the host, generates a gzip-tarball and downloads that. 218 | 219 | :param image: Image name or id. 220 | :type image: unicode 221 | :param filename: File name to store the local file. If not provided, will use ``.tar.gz`` in the current 222 | working directory. 223 | :type filename: unicode 224 | """ 225 | local_name = filename or '{0}.tar.gz'.format(image) 226 | cli.save_image(image, local_name) 227 | 228 | 229 | @task 230 | def load_image(filename, timeout=120): 231 | """ 232 | Uploads an image from a local file to a Docker remote. Note that this temporarily has to extend the service timeout 233 | period. 234 | 235 | :param filename: Local file name. 236 | :type filename: unicode 237 | :param timeout: Timeout in seconds to set temporarily for the upload. 238 | :type timeout: int 239 | """ 240 | c = docker_fabric() 241 | with open(expand_path(filename), 'r') as f: 242 | _timeout = c._timeout 243 | c._timeout = timeout 244 | try: 245 | c.load_image(f) 246 | finally: 247 | c._timeout = _timeout 248 | -------------------------------------------------------------------------------- /dockerfabric/tunnel.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | import select 5 | import socket 6 | import errno 7 | 8 | from fabric.network import needs_host 9 | from fabric.state import connections, env 10 | from fabric.thread_handling import ThreadHandler 11 | 12 | from .base import ConnectionDict, get_local_port 13 | 14 | 15 | class LocalTunnels(ConnectionDict): 16 | """ 17 | Cache for local tunnels to the remote machine. 18 | """ 19 | def __getitem__(self, item): 20 | """ 21 | :param item: Tuple of remote host, remote port, local port number, and local bind address. 22 | :type item: tuple 23 | :return: Local tunnel 24 | :rtype: LocalTunnel 25 | """ 26 | def _connect_local_tunnel(): 27 | bind_port = get_local_port(init_bind_port) 28 | tun = LocalTunnel(remote_port, remote_host, bind_port, bind_host) 29 | tun.connect() 30 | return tun 31 | 32 | remote_host, remote_port, bind_host, init_bind_port = item 33 | key = remote_host, remote_port 34 | return self.get_or_create_connection(key, _connect_local_tunnel) 35 | 36 | 37 | local_tunnels = LocalTunnels() 38 | 39 | 40 | def _forwarder(chan, sock): 41 | # Bidirectionally forward data between a socket and a Paramiko channel. 42 | try: 43 | while True: 44 | r, w, x = select.select([sock, chan], [], [], 1) 45 | if sock in r: 46 | data = sock.recv(1024) 47 | if len(data) == 0: 48 | break 49 | chan.sendall(data) 50 | if chan in r: 51 | data = chan.recv(1024) 52 | if len(data) == 0: 53 | break 54 | sock.sendall(data) 55 | except socket.error as e: 56 | #Sockets return bad file descriptor if closed. 57 | #Maybe there is a cleaner way of doing this? 58 | if e.errno not in (socket.EBADF, errno.ECONNRESET): 59 | raise 60 | except select.error as e: 61 | if e[0] != socket.EBADF: 62 | raise 63 | 64 | try: 65 | chan.close() 66 | except socket.error: 67 | pass 68 | 69 | try: 70 | sock.close() 71 | except socket.error: 72 | pass 73 | 74 | 75 | class LocalTunnel(object): 76 | """ 77 | Adapted from PR #939 of Fabric: https://github.com/fabric/fabric/pull/939 78 | 79 | Forward a local port to a given host and port on the remote side. 80 | 81 | :param remote_port: Remote port to forward connections to. 82 | :type remote_port: int 83 | :param remote_host: Host to connect to. Optional, default is ``localhost``. 84 | :type remote_host: unicode 85 | :param bind_port: Local port to bind to. Optional, default is same as ``remote_port``. 86 | :type bind_port: int 87 | :param bind_host: Local address to bind to. Optional, default is ``localhost``. 88 | """ 89 | def __init__(self, remote_port, remote_host=None, bind_port=None, bind_host=None, remote_cmd=None): 90 | self.remote_port = remote_port 91 | self.remote_host = remote_host or 'localhost' 92 | self.bind_port = bind_port or remote_port 93 | self.bind_host = bind_host or 'localhost' 94 | self.remote_cmd = remote_cmd 95 | self.sockets = [] 96 | self.channels = [] 97 | self.threads = [] 98 | self.listening_socket = None 99 | self.listening_thread = None 100 | 101 | def get_channel(self, transport, remote_addr, local_peer): 102 | channel = transport.open_channel('direct-tcpip', remote_addr, local_peer) 103 | if channel is None: 104 | raise Exception('Incoming request to %s:%d was rejected by the SSH server.' % remote_addr) 105 | return channel 106 | 107 | @needs_host 108 | def connect(self): 109 | def listener_thread_main(thead_sock, callback, *a, **kw): 110 | try: 111 | while True: 112 | selsockets = select.select([thead_sock], [], [], 1) 113 | if thead_sock in selsockets[0]: 114 | callback(thead_sock, *a, **kw) 115 | except socket.error as e: 116 | #Sockets return bad file descriptor if closed. 117 | #Maybe there is a cleaner way of doing this? 118 | if e.errno != socket.EBADF: 119 | raise 120 | except select.error as e: 121 | if e[0] != socket.EBADF: 122 | raise 123 | 124 | listening_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 125 | listening_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 126 | listening_socket.bind((self.bind_host, self.bind_port)) 127 | listening_socket.listen(1) 128 | 129 | def accept(listen_sock, transport, remote_addr): 130 | accept_sock, local_peer = listen_sock.accept() 131 | channel = self.get_channel(transport, remote_addr, local_peer) 132 | 133 | handler = ThreadHandler('fwd', _forwarder, channel, accept_sock) 134 | 135 | self.sockets.append(accept_sock) 136 | self.channels.append(channel) 137 | self.threads.append(handler) 138 | 139 | self.sockets = [] 140 | self.channels = [] 141 | self.threads = [] 142 | self.listening_socket = listening_socket 143 | self.listening_thread = ThreadHandler('local_bind', listener_thread_main, 144 | listening_socket, accept, 145 | connections[env.host_string].get_transport(), 146 | (self.remote_host, self.remote_port)) 147 | 148 | def close(self): 149 | for sock, chan, th in zip(self.sockets, self.channels, self.threads): 150 | sock.close() 151 | if not chan.closed: 152 | chan.close() 153 | th.thread.join() 154 | th.raise_if_needed() 155 | 156 | self.listening_socket.close() 157 | self.listening_thread.thread.join() 158 | self.listening_thread.raise_if_needed() 159 | -------------------------------------------------------------------------------- /dockerfabric/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | -------------------------------------------------------------------------------- /dockerfabric/utils/base.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | import six 5 | 6 | from fabric.api import env 7 | from fabric.network import needs_host 8 | 9 | 10 | @needs_host 11 | def get_current_roles(): 12 | """ 13 | Determines the list of roles, that the current host is assigned to. If ``env.roledefs`` is not set, an empty list 14 | is returned. 15 | 16 | :return: List of roles of the current host. 17 | :rtype: list 18 | """ 19 | current_host = env.host_string 20 | roledefs = env.get('roledefs') 21 | if roledefs: 22 | return [role for role, hosts in six.iteritems(roledefs) if current_host in hosts] 23 | return [] 24 | 25 | 26 | def get_role_addresses(role_name, interface_name): 27 | roledefs = env.get('roledefs') 28 | clients = env.get('docker_clients') 29 | if roledefs and clients: 30 | role_hosts = roledefs.get(role_name) 31 | if role_hosts: 32 | return set(client_config.interfaces[interface_name] 33 | for client_name, client_config in six.iteritems(clients) 34 | if client_config.get('fabric_host') in role_hosts) 35 | return set() 36 | -------------------------------------------------------------------------------- /dockerfabric/utils/containers.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from fabric.context_managers import documented_contextmanager 5 | from dockerfabric.apiclient import docker_fabric 6 | 7 | 8 | @documented_contextmanager 9 | def temp_container(image, no_op_cmd='/bin/true', create_kwargs=None, start_kwargs=None): 10 | """ 11 | Creates a temporary container, which can be used e.g. for copying resources. The container is removed once it 12 | is no longer needed. Note that ``no_op_cmd`` needs to be set appropriately, since the method will wait for the 13 | container to finish before copying resources. 14 | 15 | :param image: Image name or id to create the container from. 16 | :type image: unicode 17 | :param no_op_cmd: Dummy-command to run, only for being able to access the container. 18 | :type no_op_cmd: unicode 19 | :param create_kwargs: Additional kwargs for creating the container. The ``entrypoint`` will be set to ``no_op_cmd``. 20 | :type create_kwargs: dict 21 | :param start_kwargs: Additional kwargs for starting the container. ``restart_policy`` will be set to ``None``. 22 | :type start_kwargs: dict 23 | :return: Id of the temporary container. 24 | :rtype: unicode 25 | """ 26 | df = docker_fabric() 27 | create_kwargs = create_kwargs.copy() if create_kwargs else dict() 28 | start_kwargs = start_kwargs.copy() if start_kwargs else dict() 29 | create_kwargs.update(entrypoint=no_op_cmd) 30 | start_kwargs.update(restart_policy=None) 31 | container = df.create_container(image, **create_kwargs)['Id'] 32 | df.start(container, **start_kwargs) 33 | df.wait(container) 34 | yield container 35 | df.remove_container(container) 36 | -------------------------------------------------------------------------------- /dockerfabric/utils/files.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | import shutil 5 | import tarfile 6 | import tempfile 7 | 8 | from fabric.api import run, sudo 9 | from fabric.context_managers import documented_contextmanager 10 | 11 | from dockermap.shortcuts import rm, chmod, chown 12 | from .output import single_line_stdout 13 | 14 | 15 | def _safe_name(tarinfo): 16 | return tarinfo.name[0] != '/' and '..' not in tarinfo.name 17 | 18 | 19 | def get_remote_temp(): 20 | """ 21 | Creates a temporary directory on the remote end. Uses the command ``mktemp`` to do so. 22 | 23 | :return: Path to the temporary directory. 24 | :rtype: unicode 25 | """ 26 | return single_line_stdout('mktemp -d') 27 | 28 | 29 | def remove_ignore(path, use_sudo=False, force=False): 30 | """ 31 | Recursively removes a file or directory, ignoring any errors that may occur. Should only be used for temporary 32 | files that can be assumed to be cleaned up at a later point. 33 | 34 | :param path: Path to file or directory to remove. 35 | :type path: unicode 36 | :param use_sudo: Use the `sudo` command. 37 | :type use_sudo: bool 38 | :param force: Force the removal. 39 | :type force: bool 40 | """ 41 | which = sudo if use_sudo else run 42 | which(rm(path, recursive=True, force=force), warn_only=True) 43 | 44 | 45 | def is_directory(path, use_sudo=False): 46 | """ 47 | Check if the remote path exists and is a directory. 48 | 49 | :param path: Remote path to check. 50 | :type path: unicode 51 | :param use_sudo: Use the `sudo` command. 52 | :type use_sudo: bool 53 | :return: `True` if the path exists and is a directory; `False` if it exists, but is a file; `None` if it does not 54 | exist. 55 | :rtype: bool or ``None`` 56 | """ 57 | result = single_line_stdout('if [[ -f {0} ]]; then echo 0; elif [[ -d {0} ]]; then echo 1; else echo -1; fi'.format(path), sudo=use_sudo, quiet=True) 58 | if result == '0': 59 | return False 60 | elif result == '1': 61 | return True 62 | else: 63 | return None 64 | 65 | 66 | @documented_contextmanager 67 | def temp_dir(apply_chown=None, apply_chmod=None, remove_using_sudo=None, remove_force=False): 68 | """ 69 | Creates a temporary directory on the remote machine. The directory is removed when no longer needed. Failure to do 70 | so will be ignored. 71 | 72 | :param apply_chown: Optional; change the owner of the directory. 73 | :type apply_chown: unicode 74 | :param apply_chmod: Optional; change the permissions of the directory. 75 | :type apply_chmod: unicode 76 | :param remove_using_sudo: Use sudo for removing the directory. ``None`` (default) means it is used depending on 77 | whether ``apply_chown`` has been set. 78 | :type remove_using_sudo: bool | NoneType 79 | :param remove_force: Force the removal. 80 | :type remove_force: bool 81 | :return: Path to the temporary directory. 82 | :rtype: unicode 83 | """ 84 | path = get_remote_temp() 85 | try: 86 | if apply_chmod: 87 | run(chmod(apply_chmod, path)) 88 | if apply_chown: 89 | if remove_using_sudo is None: 90 | remove_using_sudo = True 91 | sudo(chown(apply_chown, path)) 92 | yield path 93 | finally: 94 | remove_ignore(path, use_sudo=remove_using_sudo, force=remove_force) 95 | 96 | 97 | @documented_contextmanager 98 | def local_temp_dir(): 99 | """ 100 | Creates a local temporary directory. The directory is removed when no longer needed. Failure to do 101 | so will be ignored. 102 | 103 | :return: Path to the temporary directory. 104 | :rtype: unicode 105 | """ 106 | path = tempfile.mkdtemp() 107 | yield path 108 | shutil.rmtree(path, ignore_errors=True) 109 | 110 | 111 | def extract_tar(filename, dest_path, **kwargs): 112 | """ 113 | Extracts a TAR archive. All element names starting with ``/`` (indicating an absolute path) or that contain ``..`` 114 | as references to a parent directory are not extracted. 115 | 116 | :param filename: Path to the tar file. 117 | :type filename: unicode 118 | :param dest_path: Destination path to extract the contents to. 119 | :type dest_path: unicode 120 | :param kwargs: Additional kwargs for opening the TAR file (:func:`tarfile.open`). 121 | """ 122 | with tarfile.open(filename, 'r', **kwargs) as tf: 123 | safe_members = [name for name in tf.getmembers() if _safe_name(name)] 124 | if safe_members: 125 | tf.extractall(dest_path, safe_members) 126 | -------------------------------------------------------------------------------- /dockerfabric/utils/net.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from itertools import repeat 5 | import re 6 | 7 | from fabric.utils import error 8 | from .output import stdout_result 9 | 10 | 11 | IP4_PATTERN = re.compile(r'\s+inet addr:\s*((?:\d{1,3}\.){3}\d{1,3})') 12 | IP6_PATTERN = re.compile(r'\s+inet6 addr:\s*((?:[a-f0-9]{1,4}::?){1,7}[a-f0-9]{1,4})') 13 | 14 | 15 | def _get_address(interface_name, pattern): 16 | out = stdout_result('ifconfig {0}'.format(interface_name), (1,), shell=False, quiet=True) 17 | if not out: 18 | error("Network interface {0} not found.".format(interface_name)) 19 | match = pattern.search(out) 20 | if match: 21 | return match.group(1) 22 | return None 23 | 24 | 25 | def _expand_groups(address): 26 | for group in address.split(':'): 27 | if group: 28 | yield group.zfill(4) 29 | else: 30 | zero_groups = 8 - address.count(':') if '::' in address else 0 31 | for z in repeat('0000', zero_groups): 32 | yield z 33 | 34 | 35 | def get_ip4_address(interface_name): 36 | """ 37 | Extracts the IPv4 address for a particular interface from `ifconfig`. 38 | 39 | :param interface_name: Name of the network interface (e.g. ``eth0``). 40 | :type interface_name: unicode 41 | :return: IPv4 address; ``None`` if the interface is present but no address could be extracted. 42 | :rtype: unicode 43 | """ 44 | return _get_address(interface_name, IP4_PATTERN) 45 | 46 | 47 | def get_ip6_address(interface_name, expand=False): 48 | """ 49 | Extracts the IPv6 address for a particular interface from `ifconfig`. 50 | 51 | :param interface_name: Name of the network interface (e.g. ``eth0``). 52 | :type interface_name: unicode 53 | :param expand: If set to ``True``, an abbreviated address is expanded to the full address. 54 | :type expand: bool 55 | :return: IPv6 address; ``None`` if the interface is present but no address could be extracted. 56 | :rtype: unicode 57 | """ 58 | address = _get_address(interface_name, IP6_PATTERN) 59 | if address and expand: 60 | return ':'.join(_expand_groups(address)) 61 | return address 62 | -------------------------------------------------------------------------------- /dockerfabric/utils/output.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from fabric import operations 5 | from fabric.context_managers import hide 6 | from fabric.utils import error 7 | 8 | 9 | def stdout_result(cmd, expected_errors=(), shell=True, sudo=False, quiet=False): 10 | """ 11 | Runs a command and returns the result, that would be written to `stdout`, as a string. The output itself can 12 | be suppressed. 13 | 14 | :param cmd: Command to run. 15 | :type cmd: unicode 16 | :param expected_errors: If the return code is non-zero, but found in this tuple, it will be ignored. ``None`` is 17 | returned in this case. 18 | :type expected_errors: tuple 19 | :param shell: Use a shell. 20 | :type shell: bool 21 | :param sudo: Use `sudo`. 22 | :type sudo: bool 23 | :param quiet: If set to ``True``, does not show any output. 24 | :type quiet: bool 25 | :return: The result of the command as would be written to `stdout`. 26 | :rtype: unicode 27 | """ 28 | which = operations.sudo if sudo else operations.run 29 | with hide('warnings'): 30 | result = which(cmd, shell=shell, quiet=quiet, warn_only=True) 31 | if result.return_code == 0: 32 | return result 33 | 34 | if result.return_code not in expected_errors: 35 | error("Received unexpected error code {0} while executing!".format(result.return_code)) 36 | return None 37 | 38 | 39 | def check_int(value): 40 | """ 41 | Tests a given string for a possible conversion to integer. Uses Fabric's :func:`fabric.utils.error` instead of 42 | raising a :class:`TypeError`. ``None`` is not converted but returns ``None`` instead. 43 | 44 | :param value: Value to test for conversion. 45 | :type value: unicode 46 | :return: Integer value. 47 | :rtype: int 48 | """ 49 | if value is not None: 50 | try: 51 | return int(value) 52 | except TypeError: 53 | error("Unexpected non-integer value '{0}'.".format(value)) 54 | return None 55 | 56 | 57 | single_line = lambda val: val.split('\n')[0] if val is not None else None 58 | 59 | 60 | def single_line_stdout(cmd, expected_errors=(), shell=True, sudo=False, quiet=False): 61 | """ 62 | Runs a command and returns the first line of the result, that would be written to `stdout`, as a string. 63 | The output itself can be suppressed. 64 | 65 | :param cmd: Command to run. 66 | :type cmd: unicode 67 | :param expected_errors: If the return code is non-zero, but found in this tuple, it will be ignored. ``None`` is 68 | returned in this case. 69 | :type expected_errors: tuple 70 | :param shell: Use a shell. 71 | :type shell: bool 72 | :param sudo: Use `sudo`. 73 | :type sudo: bool 74 | :param quiet: If set to ``True``, does not show any output. 75 | :type quiet: bool 76 | :return: The result of the command as would be written to `stdout`. 77 | :rtype: unicode 78 | """ 79 | return single_line(stdout_result(cmd, expected_errors, shell, sudo, quiet)) 80 | -------------------------------------------------------------------------------- /dockerfabric/utils/users.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from fabric.api import sudo 5 | from fabric.utils import error 6 | 7 | from dockermap.shortcuts import addgroup, adduser, assignuser 8 | from .output import single_line_stdout, check_int 9 | 10 | 11 | def get_group_id(groupname): 12 | """ 13 | Returns the group id to a given group name. Returns ``None`` if the group does not exist. 14 | 15 | :param groupname: Group name. 16 | :type groupname: unicode 17 | :return: Group id. 18 | :rtype: int 19 | """ 20 | gid = single_line_stdout('id -g {0}'.format(groupname), expected_errors=(1,), shell=False) 21 | return check_int(gid) 22 | 23 | 24 | def get_user_id(username): 25 | """ 26 | Returns the user id to a given user name. Returns ``None`` if the user does not exist. 27 | 28 | :param username: User name. 29 | :type username: unicode 30 | :return: User id. 31 | :rtype: int 32 | """ 33 | uid = single_line_stdout('id -u {0}'.format(username), expected_errors=(1,), shell=False) 34 | return check_int(uid) 35 | 36 | 37 | def get_user_groups(username): 38 | """ 39 | Returns the list if group names for a given user name, omitting the default group. 40 | Returns ``None`` if the user does not exist. 41 | 42 | :param username: User name. 43 | :type username: unicode 44 | :return: Group names. 45 | :rtype: list 46 | """ 47 | out = single_line_stdout('groups {0}'.format(username)) 48 | if out: 49 | return out.split()[2:] 50 | return None 51 | 52 | 53 | def create_group(groupname, gid, system=True): 54 | """ 55 | Creates a new user group with a specific id. 56 | 57 | :param groupname: Group name. 58 | :type groupname: unicode 59 | :param gid: Group id. 60 | :type gid: int or unicode 61 | :param system: Creates a system group. 62 | """ 63 | sudo(addgroup(groupname, gid, system)) 64 | 65 | 66 | def create_user(username, uid, system=False, no_login=True, no_password=False, group=False, gecos=None): 67 | """ 68 | Creates a new user with a specific id. 69 | 70 | :param username: User name. 71 | :type username: unicode 72 | :param uid: User id. 73 | :type uid: int or unicode 74 | :param system: Creates a system user. 75 | :type system: bool 76 | :param no_login: Disallow login of this user and group, and skip creating the home directory. Default is ``True``. 77 | :type no_login: bool 78 | :param no_password: Do not set a password for the new user. 79 | :type: no_password: bool 80 | :param group: Create a group with the same id. 81 | :type group: bool 82 | :param gecos: Provide GECOS info and suppress prompt. 83 | :type gecos: unicode 84 | """ 85 | sudo(adduser(username, uid, system, no_login, no_password, group, gecos)) 86 | 87 | 88 | def assign_user_groups(username, groupnames): 89 | """ 90 | Assigns a user to a set of groups. User and group need to exists. The new groups are appended to existing group 91 | assignments. 92 | 93 | :param username: User name. 94 | :type username: unicode 95 | :param groupnames: Group names. 96 | :type groupnames: iterable 97 | """ 98 | sudo(assignuser(username, groupnames)) 99 | 100 | 101 | def get_or_create_group(groupname, gid_preset, system=False, id_dependent=True): 102 | """ 103 | Returns the id for the given group, and creates it first in case it does not exist. 104 | 105 | :param groupname: Group name. 106 | :type groupname: unicode 107 | :param gid_preset: Group id to set if a new group is created. 108 | :type gid_preset: int or unicode 109 | :param system: Create a system group. 110 | :type system: bool 111 | :param id_dependent: If the group exists, but its id does not match `gid_preset`, an error is thrown. 112 | :type id_dependent: bool 113 | :return: Group id of the existing or new group. 114 | :rtype: int 115 | """ 116 | gid = get_group_id(groupname) 117 | if gid is None: 118 | create_group(groupname, gid_preset, system) 119 | return gid_preset 120 | elif id_dependent and gid != gid_preset: 121 | error("Present group id '{0}' does not match the required id of the environment '{1}'.".format(gid, gid_preset)) 122 | return gid 123 | 124 | 125 | def get_or_create_user(username, uid_preset, groupnames=[], system=False, no_password=False, no_login=True, 126 | gecos=None, id_dependent=True): 127 | """ 128 | Returns the id of the given user name, and creates it first in case it does not exist. A default group is created 129 | as well. 130 | 131 | :param username: User name. 132 | :type username: unicode 133 | :param uid_preset: User id to set in case a new user is created. 134 | :type uid_preset: int or unicode 135 | :param groupnames: Additional names of groups to assign the user to. If the user exists, these will be appended to 136 | existing group assignments. 137 | :type groupnames: iterable 138 | :param system: Create a system user. 139 | :type system: bool 140 | :param no_login: Disallow login of this user and group, and skip creating the home directory. Default is ``True``. 141 | :type no_login: bool 142 | :param no_password: Do not set a password for the new user. 143 | :type: no_password: bool 144 | :param gecos: Provide GECOS info and suppress prompt. 145 | :type gecos: unicode 146 | :param id_dependent: If the user exists, but its id does not match `uid_preset`, an error is thrown. 147 | :type id_dependent: bool 148 | :return: 149 | """ 150 | uid = get_user_id(username) 151 | gid = get_group_id(username) 152 | if id_dependent and gid is not None and gid != uid_preset: 153 | error("Present group id '{0}' does not match the required id of the environment '{1}'.".format(gid, uid_preset)) 154 | if gid is None: 155 | create_group(username, uid_preset, system) 156 | gid = uid_preset 157 | if uid is None: 158 | create_user(username, gid, system, no_login, no_password, False, gecos) 159 | if groupnames: 160 | assign_user_groups(username, groupnames) 161 | return uid 162 | elif id_dependent and uid != uid_preset: 163 | error("Present user id '{0}' does not match the required id of the environment '{1}'.".format(uid, uid_preset)) 164 | current_groups = get_user_groups(username) 165 | new_groups = set(groupnames).discard(tuple(current_groups)) 166 | if new_groups: 167 | assign_user_groups(username, new_groups) 168 | return uid 169 | -------------------------------------------------------------------------------- /dockerfabric/yaml.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals, absolute_import 3 | 4 | from fabric.api import env 5 | from dockermap.functional import lazy_once 6 | # noinspection PyUnresolvedReferences 7 | from dockermap.map.yaml import (yaml, load_file, load_map, load_map_file, 8 | load_clients as _load_clients, 9 | load_clients_file as _load_clients_file) 10 | from .apiclient import DockerClientConfiguration 11 | 12 | 13 | env_get = lambda v: env[v] 14 | 15 | 16 | def expand_env_lazy(loader, node): 17 | """ 18 | Substitutes a variable read from a YAML node with the value stored in Fabric's ``env`` dictionary. Creates an 19 | object for late resolution. 20 | 21 | :param loader: YAML loader. 22 | :type loader: yaml.loader.SafeLoader 23 | :param node: Document node. 24 | :type node: ScalarNode 25 | :return: Corresponding value stored in the ``env`` dictionary. 26 | :rtype: any 27 | """ 28 | val = loader.construct_scalar(node) 29 | return lazy_once(env_get, val) 30 | 31 | 32 | def expand_env(loader, node): 33 | """ 34 | Substitutes a variable read from a YAML node with the value stored in Fabric's ``env`` dictionary. 35 | 36 | :param loader: YAML loader. 37 | :type loader: yaml.loader.SafeLoader 38 | :param node: Document node. 39 | :type node: ScalarNode 40 | :return: Corresponding value stored in the ``env`` dictionary. 41 | :rtype: any 42 | """ 43 | val = loader.construct_scalar(node) 44 | return env[val] 45 | 46 | 47 | def load_clients(stream): 48 | """ 49 | Loads client configurations from a YAML document stream. 50 | 51 | :param stream: YAML stream. 52 | :type stream: file 53 | :return: A dictionary of client configuration objects. 54 | :rtype: dict[unicode, dockerfabric.apiclient.DockerClientConfiguration] 55 | """ 56 | return _load_clients(stream, configuration_class=DockerClientConfiguration) 57 | 58 | 59 | def load_clients_file(filename): 60 | """ 61 | Loads client configurations from a YAML file. 62 | 63 | :param filename: YAML file name. 64 | :type filename: unicode 65 | :return: A dictionary of client configuration objects. 66 | :rtype: dict[unicode, dockerfabric.apiclient.DockerClientConfiguration] 67 | """ 68 | return _load_clients_file(filename, configuration_class=DockerClientConfiguration) 69 | 70 | 71 | yaml.add_constructor('!env_lazy', expand_env_lazy, yaml.SafeLoader) 72 | yaml.add_constructor('!env', expand_env, yaml.SafeLoader) 73 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | 49 | clean: 50 | rm -rf $(BUILDDIR)/* 51 | 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | dirhtml: 58 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 59 | @echo 60 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 61 | 62 | singlehtml: 63 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 64 | @echo 65 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 66 | 67 | pickle: 68 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 69 | @echo 70 | @echo "Build finished; now you can process the pickle files." 71 | 72 | json: 73 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 74 | @echo 75 | @echo "Build finished; now you can process the JSON files." 76 | 77 | htmlhelp: 78 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 79 | @echo 80 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 81 | ".hhp project file in $(BUILDDIR)/htmlhelp." 82 | 83 | qthelp: 84 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 85 | @echo 86 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 87 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 88 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Docker-Fabric.qhcp" 89 | @echo "To view the help file:" 90 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Docker-Fabric.qhc" 91 | 92 | devhelp: 93 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 94 | @echo 95 | @echo "Build finished." 96 | @echo "To view the help file:" 97 | @echo "# mkdir -p $$HOME/.local/share/devhelp/Docker-Fabric" 98 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Docker-Fabric" 99 | @echo "# devhelp" 100 | 101 | epub: 102 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 103 | @echo 104 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 105 | 106 | latex: 107 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 108 | @echo 109 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 110 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 111 | "(use \`make latexpdf' here to do that automatically)." 112 | 113 | latexpdf: 114 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 115 | @echo "Running LaTeX files through pdflatex..." 116 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 117 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 118 | 119 | latexpdfja: 120 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 121 | @echo "Running LaTeX files through platex and dvipdfmx..." 122 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 123 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 124 | 125 | text: 126 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 127 | @echo 128 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 129 | 130 | man: 131 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 132 | @echo 133 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 134 | 135 | texinfo: 136 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 137 | @echo 138 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 139 | @echo "Run \`make' in that directory to run these through makeinfo" \ 140 | "(use \`make info' here to do that automatically)." 141 | 142 | info: 143 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 144 | @echo "Running Texinfo files through makeinfo..." 145 | make -C $(BUILDDIR)/texinfo info 146 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 147 | 148 | gettext: 149 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 150 | @echo 151 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 152 | 153 | changes: 154 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 155 | @echo 156 | @echo "The overview file is in $(BUILDDIR)/changes." 157 | 158 | linkcheck: 159 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 160 | @echo 161 | @echo "Link check complete; look for any errors in the above output " \ 162 | "or in $(BUILDDIR)/linkcheck/output.txt." 163 | 164 | doctest: 165 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 166 | @echo "Testing of doctests in the sources finished, look at the " \ 167 | "results in $(BUILDDIR)/doctest/output.txt." 168 | 169 | xml: 170 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 171 | @echo 172 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 173 | 174 | pseudoxml: 175 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 176 | @echo 177 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 178 | -------------------------------------------------------------------------------- /docs/api/dockerfabric.rst: -------------------------------------------------------------------------------- 1 | dockerfabric package 2 | ==================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | dockerfabric.utils 10 | 11 | Submodules 12 | ---------- 13 | 14 | dockerfabric.actions module 15 | ----------------------------- 16 | 17 | .. automodule:: dockerfabric.actions 18 | :members: 19 | :undoc-members: 20 | :show-inheritance: 21 | 22 | dockerfabric.apiclient module 23 | ----------------------------- 24 | 25 | .. automodule:: dockerfabric.apiclient 26 | :members: 27 | :undoc-members: 28 | :show-inheritance: 29 | 30 | dockerfabric.cli module 31 | ----------------------- 32 | 33 | .. automodule:: dockerfabric.cli 34 | :members: 35 | :undoc-members: 36 | :show-inheritance: 37 | 38 | dockerfabric.socat module 39 | ------------------------- 40 | 41 | .. automodule:: dockerfabric.socat 42 | :members: 43 | :undoc-members: 44 | :show-inheritance: 45 | 46 | dockerfabric.tasks module 47 | ------------------------- 48 | 49 | .. automodule:: dockerfabric.tasks 50 | :members: 51 | :undoc-members: 52 | :show-inheritance: 53 | 54 | dockerfabric.tunnel module 55 | -------------------------- 56 | 57 | .. automodule:: dockerfabric.tunnel 58 | :members: 59 | :undoc-members: 60 | :show-inheritance: 61 | 62 | dockerfabric.yaml module 63 | ------------------------ 64 | 65 | .. automodule:: dockerfabric.yaml 66 | :members: 67 | :undoc-members: 68 | :show-inheritance: 69 | 70 | 71 | Module contents 72 | --------------- 73 | 74 | .. automodule:: dockerfabric 75 | :members: 76 | :undoc-members: 77 | :show-inheritance: 78 | -------------------------------------------------------------------------------- /docs/api/dockerfabric.utils.rst: -------------------------------------------------------------------------------- 1 | dockerfabric.utils package 2 | ========================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | dockerfabric.utils.containers module 8 | ------------------------------------ 9 | 10 | .. automodule:: dockerfabric.utils.containers 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | dockerfabric.utils.files module 16 | ------------------------------- 17 | 18 | .. automodule:: dockerfabric.utils.files 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | dockerfabric.utils.net module 24 | ----------------------------- 25 | 26 | .. automodule:: dockerfabric.utils.net 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | dockerfabric.utils.output module 32 | -------------------------------- 33 | 34 | .. automodule:: dockerfabric.utils.output 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | dockerfabric.utils.users module 40 | ------------------------------- 41 | 42 | .. automodule:: dockerfabric.utils.users 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | 48 | Module contents 49 | --------------- 50 | 51 | .. automodule:: dockerfabric.utils 52 | :members: 53 | :undoc-members: 54 | :show-inheritance: 55 | -------------------------------------------------------------------------------- /docs/api/modules.rst: -------------------------------------------------------------------------------- 1 | API 2 | === 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | dockerfabric 8 | -------------------------------------------------------------------------------- /docs/changes.rst: -------------------------------------------------------------------------------- 1 | .. _change_history: 2 | 3 | Change History 4 | ============== 5 | 0.5.0 6 | ----- 7 | * Minor addition to CLI client. 8 | 9 | 0.5.0b4 10 | ------- 11 | * Fixed image id parsing after successful build. 12 | 13 | 0.5.0b3 14 | ------- 15 | * Added new CLI functions and utility task for volumes, as now implemented in Docker-Map 0.8.0b3. 16 | 17 | 0.5.0b2 18 | ------- 19 | * Fixed setup (requirements). 20 | 21 | 0.5.0b1 22 | ------- 23 | * Adapted to recent Docker-Map developments, which includes networking support and improved error handling. 24 | * Dropped setup tasks for Docker and Socat. Socat is included in most Linux distributions, so that the task of compiling 25 | it from source was likely not used. Installation instructions for Docker have changed too frequently, and a correct 26 | (supported) setup depends strongly on the environment it is installed in, with more aspects than these simple tasks 27 | could consider. 28 | 29 | 0.4.2 30 | ----- 31 | * Added ``top`` method to CLI client. 32 | 33 | 0.4.1 34 | ----- 35 | * Fixed side-effects of modifying the ``base_url`` for SSH tunnels, causing problems when re-using a client returned 36 | by the ``docker_fabric()`` function (`Issue #12 `_). 37 | * Added ``version`` method to CLI client. 38 | * Added ``env.docker_cli_debug`` for echoing commands. 39 | * API clients' ``remove_all_containers`` now forwards keyword arguments. 40 | 41 | 0.4.0 42 | ----- 43 | * Added Docker-Map's new features (keeping certain tags during cleanup and adding extra tags during build). 44 | * Added experimental :ref:`cli_client`. This has changed the module structure a bit, but previous imports should still work. 45 | From now on however, ``docker_fabric`` and ``container_fabric`` should be imported from ``dockerfabric.api`` instead 46 | of ``dockerfabric.apiclient``. 47 | * Fixed installation task for CentOS. 48 | 49 | 0.3.10 50 | ------ 51 | * Updated Docker service installation to follow reference instructions. 52 | * Added separate utility tasks for CentOS. 53 | * Fixed build failures in case of unicode errors. 54 | 55 | 0.3.9 56 | ----- 57 | * Client configuration is not required, if defaults are used. 58 | 59 | 0.3.8 60 | ----- 61 | * Implemented local (faster) method for adjusting permissions on containers. 62 | * Fixed issues with non-existing directories when downloading resources from containers. 63 | 64 | 0.3.7 65 | ----- 66 | * Minor logging changes. 67 | * Make it possible to set raise_on_error with pull (`PR #3 `_) 68 | 69 | 0.3.6 70 | ----- 71 | * Added script and single-command actions. 72 | 73 | 0.3.5 74 | ----- 75 | * docker-py update compatibility. 76 | 77 | 0.3.4 78 | ----- 79 | * Added Fabric error handling when build fails. 80 | * Fixed re-use of existing local tunnels when connections are opened through different methods. 81 | 82 | 0.3.3 83 | ----- 84 | * More consistent arguments to connection behavior. 85 | 86 | 0.3.2 87 | ----- 88 | * Added ``!env_lazy`` YAML tag. 89 | * Fixed bug on local connections. 90 | 91 | 0.3.1 92 | ----- 93 | * Added restart utility task. 94 | 95 | 0.3.0 96 | ----- 97 | * Added Docker-Map's new features (host-client-configuration and multiple maps). 98 | 99 | 0.2.0 100 | ----- 101 | * Revised SSH tunnelling of Docker service connections; not exposing a port on the host any longer. 102 | 103 | 0.1.4 104 | ----- 105 | * Intermediate step to 0.2.0, not published on PyPI. 106 | * Better tolerance on missing parameters. 107 | * Improved multiprocessing behavior (parallel tasks in Fabric). 108 | 109 | 0.1.3 110 | ----- 111 | * Only setup fix, no functional changes. 112 | 113 | 0.1.2 114 | ----- 115 | * Added more utility tasks, functions, and context managers. 116 | * Improved output format of builtin tasks. 117 | * Cleanups and fixes in utility functions. 118 | 119 | 0.1.1 120 | ----- 121 | * Added YAML import. 122 | * Added default host root path and repository prefix. 123 | * Added Docker registry actions. 124 | * Added import/export utility functions. 125 | * Attempts to fix reconnect and multiple connection issues. 126 | 127 | 0.1.0 128 | ----- 129 | Initial release. 130 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Docker-Fabric documentation build configuration file, created by 4 | # sphinx-quickstart on Tue Sep 2 07:43:06 2014. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | 15 | import sys 16 | import os 17 | 18 | # on_rtd is whether we are on readthedocs.org, this line of code grabbed from docs.readthedocs.org 19 | on_rtd = os.environ.get('READTHEDOCS', None) == 'True' 20 | 21 | # If extensions (or modules to document with autodoc) are in another directory, 22 | # add these directories to sys.path here. If the directory is relative to the 23 | # documentation root, use os.path.abspath to make it absolute, like shown here. 24 | sys.path.insert(0, os.path.abspath('..')) 25 | import dockerfabric 26 | 27 | # -- General configuration ------------------------------------------------ 28 | 29 | # If your documentation needs a minimal Sphinx version, state it here. 30 | #needs_sphinx = '1.0' 31 | 32 | # Add any Sphinx extension module names here, as strings. They can be 33 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 34 | # ones. 35 | extensions = [ 36 | 'sphinx.ext.autodoc', 37 | 'sphinx.ext.intersphinx', 38 | ] 39 | 40 | # Add any paths that contain templates here, relative to this directory. 41 | templates_path = ['_templates'] 42 | 43 | # The suffix of source filenames. 44 | source_suffix = '.rst' 45 | 46 | # The encoding of source files. 47 | #source_encoding = 'utf-8-sig' 48 | 49 | # The master toctree document. 50 | master_doc = 'index' 51 | 52 | # General information about the project. 53 | project = u'Docker-Fabric' 54 | copyright = u'2016, Matthias Erll' 55 | 56 | # The version info for the project you're documenting, acts as replacement for 57 | # |version| and |release|, also used in various other places throughout the 58 | # built documents. 59 | # 60 | # The short X.Y version. 61 | version = dockerfabric.__version__ 62 | # The full version, including alpha/beta/rc tags. 63 | release = dockerfabric.__version__ 64 | 65 | # The language for content autogenerated by Sphinx. Refer to documentation 66 | # for a list of supported languages. 67 | #language = None 68 | 69 | # There are two options for replacing |today|: either, you set today to some 70 | # non-false value, then it is used: 71 | #today = '' 72 | # Else, today_fmt is used as the format for a strftime call. 73 | #today_fmt = '%B %d, %Y' 74 | 75 | # List of patterns, relative to source directory, that match files and 76 | # directories to ignore when looking for source files. 77 | exclude_patterns = ['_build'] 78 | 79 | # The reST default role (used for this markup: `text`) to use for all 80 | # documents. 81 | #default_role = None 82 | 83 | # If true, '()' will be appended to :func: etc. cross-reference text. 84 | #add_function_parentheses = True 85 | 86 | # If true, the current module name will be prepended to all description 87 | # unit titles (such as .. function::). 88 | #add_module_names = True 89 | 90 | # If true, sectionauthor and moduleauthor directives will be shown in the 91 | # output. They are ignored by default. 92 | #show_authors = False 93 | 94 | # The name of the Pygments (syntax highlighting) style to use. 95 | pygments_style = 'sphinx' 96 | 97 | # A list of ignored prefixes for module index sorting. 98 | #modindex_common_prefix = [] 99 | 100 | # If true, keep warnings as "system message" paragraphs in the built documents. 101 | #keep_warnings = False 102 | 103 | 104 | # -- Options for HTML output ---------------------------------------------- 105 | 106 | if not on_rtd: # only import and set the theme if we're building docs locally 107 | try: 108 | import sphinx_rtd_theme 109 | except ImportError: 110 | pass # Ignore, go with default options 111 | else: 112 | html_theme = 'sphinx_rtd_theme' 113 | html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] 114 | 115 | 116 | # Theme options are theme-specific and customize the look and feel of a theme 117 | # further. For a list of options available for each theme, see the 118 | # documentation. 119 | #html_theme_options = {} 120 | 121 | # The name for this set of Sphinx documents. If None, it defaults to 122 | # " v documentation". 123 | #html_title = None 124 | 125 | # A shorter title for the navigation bar. Default is the same as html_title. 126 | #html_short_title = None 127 | 128 | # The name of an image file (relative to this directory) to place at the top 129 | # of the sidebar. 130 | #html_logo = None 131 | 132 | # The name of an image file (within the static path) to use as favicon of the 133 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 134 | # pixels large. 135 | #html_favicon = None 136 | 137 | # Add any paths that contain custom static files (such as style sheets) here, 138 | # relative to this directory. They are copied after the builtin static files, 139 | # so a file named "default.css" will overwrite the builtin "default.css". 140 | html_static_path = ['_static'] 141 | 142 | # Add any extra paths that contain custom files (such as robots.txt or 143 | # .htaccess) here, relative to this directory. These files are copied 144 | # directly to the root of the documentation. 145 | #html_extra_path = [] 146 | 147 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 148 | # using the given strftime format. 149 | #html_last_updated_fmt = '%b %d, %Y' 150 | 151 | # If true, SmartyPants will be used to convert quotes and dashes to 152 | # typographically correct entities. 153 | #html_use_smartypants = True 154 | 155 | # Custom sidebar templates, maps document names to template names. 156 | #html_sidebars = {} 157 | 158 | # Additional templates that should be rendered to pages, maps page names to 159 | # template names. 160 | #html_additional_pages = {} 161 | 162 | # If false, no module index is generated. 163 | #html_domain_indices = True 164 | 165 | # If false, no index is generated. 166 | #html_use_index = True 167 | 168 | # If true, the index is split into individual pages for each letter. 169 | #html_split_index = False 170 | 171 | # If true, links to the reST sources are added to the pages. 172 | #html_show_sourcelink = True 173 | 174 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 175 | #html_show_sphinx = True 176 | 177 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 178 | #html_show_copyright = True 179 | 180 | # If true, an OpenSearch description file will be output, and all pages will 181 | # contain a tag referring to it. The value of this option must be the 182 | # base URL from which the finished HTML is served. 183 | #html_use_opensearch = '' 184 | 185 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 186 | #html_file_suffix = None 187 | 188 | # Output file base name for HTML help builder. 189 | htmlhelp_basename = 'Docker-Fabricdoc' 190 | 191 | 192 | # -- Options for LaTeX output --------------------------------------------- 193 | 194 | latex_elements = { 195 | # The paper size ('letterpaper' or 'a4paper'). 196 | #'papersize': 'letterpaper', 197 | 198 | # The font size ('10pt', '11pt' or '12pt'). 199 | #'pointsize': '10pt', 200 | 201 | # Additional stuff for the LaTeX preamble. 202 | #'preamble': '', 203 | } 204 | 205 | # Grouping the document tree into LaTeX files. List of tuples 206 | # (source start file, target name, title, 207 | # author, documentclass [howto, manual, or own class]). 208 | latex_documents = [ 209 | ('index', 'Docker-Fabric.tex', u'Docker-Fabric Documentation', 210 | u'Matthias Erll', 'manual'), 211 | ] 212 | 213 | # The name of an image file (relative to this directory) to place at the top of 214 | # the title page. 215 | #latex_logo = None 216 | 217 | # For "manual" documents, if this is true, then toplevel headings are parts, 218 | # not chapters. 219 | #latex_use_parts = False 220 | 221 | # If true, show page references after internal links. 222 | #latex_show_pagerefs = False 223 | 224 | # If true, show URL addresses after external links. 225 | #latex_show_urls = False 226 | 227 | # Documents to append as an appendix to all manuals. 228 | #latex_appendices = [] 229 | 230 | # If false, no module index is generated. 231 | #latex_domain_indices = True 232 | 233 | 234 | # -- Options for manual page output --------------------------------------- 235 | 236 | # One entry per manual page. List of tuples 237 | # (source start file, name, description, authors, manual section). 238 | man_pages = [ 239 | ('index', 'docker-fabric', u'Docker-Fabric Documentation', 240 | [u'Matthias Erll'], 1) 241 | ] 242 | 243 | # If true, show URL addresses after external links. 244 | #man_show_urls = False 245 | 246 | 247 | # -- Options for Texinfo output ------------------------------------------- 248 | 249 | # Grouping the document tree into Texinfo files. List of tuples 250 | # (source start file, target name, title, author, 251 | # dir menu entry, description, category) 252 | texinfo_documents = [ 253 | ('index', 'Docker-Fabric', u'Docker-Fabric Documentation', 254 | u'Matthias Erll', 'Docker-Fabric', 'One line description of project.', 255 | 'Miscellaneous'), 256 | ] 257 | 258 | # Documents to append as an appendix to all manuals. 259 | #texinfo_appendices = [] 260 | 261 | # If false, no module index is generated. 262 | #texinfo_domain_indices = True 263 | 264 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 265 | #texinfo_show_urls = 'footnote' 266 | 267 | # If true, do not generate a @detailmenu in the "Top" node's menu. 268 | #texinfo_no_detailmenu = False 269 | 270 | intersphinx_mapping = { 271 | 'python': ('https://docs.python.org/', None), 272 | 'dockermap': ('https://docker-map.readthedocs.org/en/latest/', None), 273 | 'fabric': ('http://docs.fabfile.org/en/latest/', None), 274 | } 275 | -------------------------------------------------------------------------------- /docs/guide/apiclient.rst: -------------------------------------------------------------------------------- 1 | .. _api_client: 2 | 3 | Docker Remote API client for Fabric 4 | =================================== 5 | :class:`~dockerfabric.apiclient.DockerFabricClient` is a client for access to a remote Docker host, which has been 6 | enhanced with some additional functionality. It is a Fabric adaption to 7 | :class:`~dockermap.map.base.DockerClientWrapper` from the Docker-Map_ package, which again is based on docker-py_, the 8 | reference Python client for the `Docker Remote API`_. 9 | 10 | :class:`~dockermap.map.base.DockerClientWrapper` wraps some functions of `docker-py`, but most methods of its original 11 | implementation can also be used directly. This is described in more depth in the 12 | :ref:`Docker-Map documentation `. The following sections focus on the details specific for 13 | Docker-Fabric. 14 | 15 | 16 | Basic usage 17 | ----------- 18 | The constructor of :class:`~dockerfabric.apiclient.DockerFabricClient` accepts the same arguments as the `docker-py` 19 | implementation (``base_url``, ``version``, and ``timeout``), which are passed through. Moreover, ``tunnel_remote_port`` 20 | and ``tunnel_local_port`` are available. The following arguments of :class:`~dockerfabric.apiclient.DockerFabricClient` 21 | fall back to Fabric ``env`` variables, if not specified explicitly: 22 | 23 | * ``base_url``: ``env.docker_base_url`` 24 | * ``version``: ``env.docker_api_version`` 25 | * ``timeout``: ``env.docker_timeout`` 26 | * ``tunnel_remote_port``: ``env.docker_tunnel_remote_port`` 27 | * ``tunnel_local_port``: ``env.docker_tunnel_local_port`` 28 | 29 | Although instances of :class:`~dockerfabric.apiclient.DockerFabricClient` can 30 | be created directly, it is more practical to do so implicitly by calling :func:`~dockerfabric.apiclient.docker_fabric` 31 | instead: 32 | 33 | * If parameters are set up in the Fabric environment, as listed in the :ref:`fabric_env` section, no further 34 | configuration is necessary. 35 | * More importantly, existing client connections (and possibly tunnels) are cached and reused, similar to Fabric's 36 | connection caching. Therefore, you do not need to keep global references to the client around. 37 | 38 | For example, consider the following task:: 39 | 40 | from dockerfabric.api import docker_fabric 41 | 42 | @task 43 | def sample_task(): 44 | images = docker_fabric().images() 45 | ... 46 | containers = docker_fabric().containers(all=True) 47 | ... 48 | 49 | 50 | The fist call to ``docker_fabric()`` opens the connection, and although you may choose to reference the client object 51 | with an extra variable, it will not use significantly more time to run ``docker_fabric()`` a second time. This becomes 52 | important especially on tunnelled connections. 53 | 54 | New connections are opened for each combination of Fabric's host string and the Docker base URL. Therefore, you can run 55 | the task on multiple machines at once, just as any other Fabric task. 56 | 57 | 58 | Working with multiple clients 59 | ----------------------------- 60 | Whereas ``docker_fabric()`` always opens the connection on the current host (determined by ``env.host_string``), it may 61 | be beneficial to run Docker commands without a ``host_string`` or ``roles`` assignment if 62 | 63 | * the set of clients, that are supposed to run container configurations, does not match the role definitions in 64 | Fabric; 65 | * you do not feel like creating a separate task with host or role lists for each container configuration to be 66 | launched; 67 | * or the client in some way require different instantiation parameters (e.g. different service URL, tunnel ports, or 68 | individual timeout settings). 69 | 70 | Docker-Fabric enhances the client configuration from Docker-Map_ in 71 | :class:`~dockerfabric.apiclient.DockerClientConfiguration`. Setting any of ``base_url``, ``version``, ``timeout``, 72 | ``tunnel_remote_port`` or ``tunnel_local_port`` overrides the global settings from the ``env`` variables mentioned in 73 | the last section. The object is mapped to Fabric's host configurations by the ``fabric_host`` variable. 74 | 75 | If stored as a dictionary in ``env.docker_clients``, configurations are used automatically by ``container_fabric()``. 76 | 77 | 78 | SSH Tunnelling 79 | -------------- 80 | Docker is by default configured to only accept connections on a Unix socket. This is good practice for security reasons, 81 | as the socket can be protected with file system permissions, whereas the attack surface with TCP-IP would be larger. 82 | However, it also makes outside access for administrative purposes more difficult. 83 | 84 | Fabric's SSH connection can tunnel connections from the local client to the remote host. If the service is 85 | only exposed over a Unix domain socket, the client additionally launches a **socat** process on the remote end for 86 | forwarding traffic between the remote tunnel endpoint and that Unix socket. That way, no permanent reconfiguration of 87 | Docker is necessary. 88 | 89 | 90 | Tunnel functionality 91 | ^^^^^^^^^^^^^^^^^^^^ 92 | Without a host connection in Fabric, the client attempts to make all connection locally (i.e. acts just like the 93 | `docker-py` client). With a ``host_string`` set, the :class:`~dockerfabric.apiclient.DockerFabricClient` opens a tunnel 94 | to forward traffic between the local machine and the Docker service on the remote host. Practically, a modified URL 95 | ``tcp://127.0.0.1:`` is passed to `docker-py`, where ```` is the specified via 96 | ``tunnel_local_port``. There are two tunnel methods, depending on the connection type to Docker: 97 | 98 | #. If ``base_url`` indicates a Unix domain socket, i.e. it is prefixed with any ``http+unix:``, ``unix:``, ``/``, or 99 | if it is left empty, **socat** is started on the remote end and forwards traffic between the remote tunnel endpoint 100 | and the socket. 101 | #. In other cases of ``base_url``, the client attempts to connect directly through the established tunnel to the 102 | Docker service on the remote end. The service has to be exposed to the port included in the ``base_url`` or set in 103 | ``tunnel_remote_port`` or. 104 | 105 | As there needs to be a separate local port for every connection, the exact port ``tunnel_local_port`` is only used once 106 | between multiple clients. :class:`~dockerfabric.apiclient.DockerFabricClient` increases this by one for each additional 107 | host. From version 0.1.4, this also works with :ref:`parallel tasks in Fabric `. 108 | 109 | Socat options 110 | ^^^^^^^^^^^^^ 111 | From version 0.2.0, **socat** does not expose a port on the remote end and therefore does not require further 112 | configuration. For information purposes, the client can however be set to echo the command to `stdout` by setting 113 | ``env.socat_quiet`` to ``False``. 114 | 115 | The utility task ``reset_socat`` removes **socat** processes, in case of occasional re-connection issues. Since 116 | **socat** no longer forks on accepting a connection, this should no longer occur. 117 | 118 | 119 | Configuration example 120 | --------------------- 121 | 122 | Single-client configuration 123 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ 124 | Consider the following lines in your project's ``fabfile.py``:: 125 | 126 | env.docker_tunnel_local_port = 2224 127 | env.docker_timeout = 20 128 | 129 | 130 | With this configuration, ``docker_fabric()`` in a task running on each host 131 | 132 | #. opens a channel on the existing SSH connection and launches **socat** on the remote, forwarding traffic between 133 | the remote `stdout` and ``/var/run/docker.sock`` (the default base URL); 134 | #. opens a tunnel through the existing SSH connection on port 2224 (increased by 1 for every additional host); 135 | #. cancels operations that take longer than 20 seconds. 136 | 137 | Multi-client configuration 138 | ^^^^^^^^^^^^^^^^^^^^^^^^^^ 139 | In addition to the previous example, adding the following additional lines in your project's ``fabfile.py``:: 140 | 141 | env.docker_clients = { 142 | 'client1': DockerClientConfiguration( 143 | fabric_host='host1', 144 | timeout=40, # Host needs longer timeouts than usual. 145 | ), 146 | 'client2': DockerClientConfiguration( 147 | fabric_host='host2', 148 | interfaces={ 149 | 'private': '10.x.x.11', # Host will be publishing some ports. 150 | 'public': '178.x.x.11', 151 | }, 152 | ), 153 | } 154 | 155 | A single client can be instantiated using:: 156 | 157 | env.docker_clients['client1'].get_client() 158 | 159 | Similar to ``docker_fabric()`` each client per host and service URL is only instantiated once. 160 | 161 | 162 | Registry connections 163 | -------------------- 164 | Docker-Fabric offers the following additional options for configuring registry access from the Docker host to a 165 | registry, as described in the :ref:`fabric_env` section. Those can be either set with keyword arguments at run-time, 166 | or with the environment variables: 167 | 168 | * ``username``: ``env.docker_registry_user`` 169 | * ``password``: ``env.docker_registry_password`` 170 | * ``email``: ``env.docker_registry_mail`` 171 | * ``registry``: ``env.docker_registry_repository`` 172 | * ``insecure_registry``: ``env.docker_registry_insecure`` 173 | 174 | Whereas ``env.docker_registry_insecure`` applies to :meth:`~dockerfabric.apiclient.DockerFabricClient.login`, 175 | :meth:`~dockerfabric.apiclient.DockerFabricClient.pull`, and :meth:`~dockerfabric.apiclient.DockerFabricClient.push`, 176 | the others are only evaluated during :meth:`~dockerfabric.apiclient.DockerFabricClient.login`. 177 | 178 | .. note:: Before a registry action, the local Docker client uses the `ping` endpoint of the registry to check on the 179 | connection. This has implications for using HTTPS connections between your Docker host(s) and the registry: 180 | Although everything is working fine on the Docker command line of the host, your client may reject the 181 | certificate because it does not trust it. This is very common with self-signed certificates, but can happen 182 | even with purchased ones. This behavior is defined by `docker-py`. 183 | 184 | There are two methods to circumvent this issue: Either set ``insecure_registry`` (or 185 | ``env.docker_registry_insecure``) to ``True``; or add the certificate authority that signed the registry's 186 | certificate to your local trust store. 187 | 188 | 189 | Docker-Map utilities 190 | -------------------- 191 | As it is based on Docker-Map_, Docker-Fabric has also inherited all of its functionality. Regarding container maps, 192 | a few adaptions are described in the section :ref:`containers`. The process of generating a `Dockerfile` and building an 193 | image from that is however very similar to the description in the 194 | :ref:`Docker-Map documentation `:: 195 | 196 | from dockermap.api import DockerFile 197 | 198 | dockerfile = DockerFile('ubuntu', maintainer='ME, me@example.com') 199 | ... 200 | docker_fabric().build_from_file(dockerfile, 'new_image') 201 | 202 | 203 | .. _Docker-Map: https://pypi.python.org/pypi/docker-map 204 | .. _Docker Remote API: https://docs.docker.com/reference/api/docker_remote_api/ 205 | .. _docker-py: https://github.com/docker/docker-py 206 | .. _running Docker with HTTPS: https://docs.docker.com/articles/https/ 207 | -------------------------------------------------------------------------------- /docs/guide/cli.rst: -------------------------------------------------------------------------------- 1 | .. _cli_client: 2 | 3 | Remote CLI client 4 | ================= 5 | Following a `feature request on GitHub `_, an alternative client 6 | implementation has been added to Docker-Fabric and Docker-Map, for using a command-line-based interface to Docker. It 7 | supports the same options, methods, and arguments. However, it directly runs commands through Fabric instead of opening 8 | its an additional SSH channel. Due to different requirements in parsing output and handling errors this should be 9 | considered experimental. 10 | 11 | Usage is very similar to the API client. There are two ways of changing between the two implementations: 12 | 13 | #. By setting a Fabric environment variable:: 14 | 15 | from fabric.api import env 16 | from dockerfabric.api import docker_fabric, container_fabric, CLIENT_CLI 17 | 18 | env.docker_fabric_implementation = CLIENT_CLI # Default is CLIENT_API. 19 | 20 | 21 | This is the preferred method, as this also applies to utility tasks defined in Docker-Fabric. 22 | 23 | #. Directly by importing a different method:: 24 | 25 | from dockerfabric.cli import docker_cli, container_cli 26 | 27 | docker_cli().images() # Instead of docker_fabric().images() 28 | container_cli().update(config_name) # Instead of container_fabric().update(config_name) 29 | 30 | -------------------------------------------------------------------------------- /docs/guide/containers.rst: -------------------------------------------------------------------------------- 1 | .. _containers: 2 | 3 | Fabric with Container maps 4 | ========================== 5 | Using :func:`~dockerfabric.apiclient.docker_fabric`, the standard API to Docker is accessible in a similar way to 6 | other Fabric commands. With Docker-Map_, the API has been enhanced with a set of utilities to configure containers 7 | and their dependencies. 8 | 9 | The configuration is in full discussed in the :ref:`Docker-Map configuration `, along with 10 | examples. This section explains how to apply this to Docker-Fabric. 11 | 12 | Managing containers 13 | ------------------- 14 | In order to have the map available in your Fabric project, it is practical to store a reference in the global 15 | ``env`` object. The :ref:`example from Docker-Map ` could be initialized with 16 | reference to other configuration variables:: 17 | 18 | env.host_root_path = '/var/lib/site' 19 | env.registry_prefix = 'registry.example.com' 20 | env.nginx_config_path = 'config/nginx' 21 | env.app1_config_path = 'config/app1' 22 | env.app2_config_path = 'config/app2' 23 | env.app1_data_path = 'data/app1' 24 | env.app2_data_path = 'data/app2' 25 | 26 | env.docker_maps = ContainerMap('example_map', { 27 | 'repository': env.registry_prefix, 28 | 'host_root': env.host_root_path, 29 | 'web_server': { # Configure container creation and startup 30 | 'image': 'nginx', 31 | 'binds': {'/etc/nginx': ('env.nginx_config_path', 'ro')}, 32 | 'uses': 'app_server_socket', 33 | 'attaches': 'web_log', 34 | 'exposes': { 35 | 80: 80, 36 | 443: 443, 37 | }, 38 | }, 39 | 'app_server': { 40 | 'image': 'app', 41 | 'instances': ('instance1', 'instance2'), 42 | 'binds': ( 43 | {'app_config': 'ro'}, 44 | 'app_data', 45 | ), 46 | 'attaches': ('app_log', 'app_server_socket'), 47 | 'user': 2000, 48 | 'permissions': 'u=rwX,g=rX,o=', 49 | }, 50 | 'volumes': { # Configure volume paths inside containers 51 | 'web_log': '/var/log/nginx', 52 | 'app_server_socket': '/var/lib/app/socket', 53 | 'app_config': '/var/lib/app/config', 54 | 'app_log': '/var/lib/app/log', 55 | 'app_data': '/var/lib/app/data', 56 | }, 57 | 'host': { # Configure volume paths on the Docker host 58 | 'app_config': { 59 | 'instance1': env.app1_config_path, 60 | 'instance2': env.app2_config_path, 61 | }, 62 | 'app_data': { 63 | 'instance1': env.app1_data_path, 64 | 'instance2': env.app2_data_path, 65 | }, 66 | }, 67 | }) 68 | 69 | In order to use this configuration set, create a :class:`~dockerfabric.apiclient.ContainerFabric` instance from this 70 | map. For example, in order to launch the web server and all dependencies, run:: 71 | 72 | from dockerfabric.api import container_fabric 73 | 74 | container_fabric().startup('web_server') 75 | 76 | :class:`~dockerfabric.apiclient.ContainerFabric` (aliased with ``container_fabric()``) calls 77 | :func:`~dockerfabric.apiclient.docker_fabric` with the host strings on demand, and therefore runs the selected map on 78 | each host where required. 79 | 80 | ``env.docker_maps`` can store one container map, or a list / tuple of multiple container maps. You can also store host 81 | definitions in any variable you like and pass them to ``container_fabric``:: 82 | 83 | container_fabric(env.container_maps) 84 | 85 | Multi-client configurations are automatically considered when stored in ``env.docker_clients``, but can also be passed 86 | through a variable:: 87 | 88 | container_fabric(maps=custom_maps, clients=custom_clients) 89 | 90 | .. _yaml-import: 91 | 92 | YAML import 93 | ----------- 94 | Import of YAML files works identically to :ref:`Docker-Map's implementation `, but with one 95 | more added tag: ``!env``. Where applied, the following string is substituted with the current value of a 96 | corresponding ``env`` variable. 97 | 98 | When using the ``!env`` tag, the order of setting variables is relevant, since values are substituted at the time the 99 | YAML file is read. For cases where this is impractical some configuration elements support a 'lazy' behavior, i.e. they 100 | are not resolved to their actual values until the first attempt to access them. In order to use that, just apply 101 | ``!env_lazy`` in place of ``!env``. For example volume paths and host ports can be assigned with this tag instead. A 102 | full list of variables supporting the late value resolution is maintained in the 103 | :ref:`Docker-Map documentation `. 104 | 105 | .. note:: If the variable is still missing at the time it is needed, a ``KeyError`` exception is raised. 106 | 107 | In order to make use of the ``!env`` and ``!env_lazy`` tag, import the module from Docker-Fabric instead of Docker-Map:: 108 | 109 | from dockerfabric import yaml 110 | 111 | env.docker_maps = yaml.load_map_file('/path/to/example_map.yaml', 'example_map') 112 | env.docker_clients = yaml.load_clients_file('/path/to/example_clients.yaml') 113 | 114 | One more difference to the Docker-Map ``yaml`` module is that :func:`load_clients_file` creates object instances of 115 | :func:`~dockerfabric.apiclient.DockerClientConfiguration`. The latter consider specific settings as the tunnel ports, 116 | which are not part of Docker-Map. 117 | 118 | Container map 119 | ^^^^^^^^^^^^^ 120 | In the file ``example_map.yaml``, the above-quoted map could be represented like this: 121 | 122 | .. code-block:: yaml 123 | 124 | repository: !env registry_prefix 125 | host_root: /var/lib/site 126 | web_server: 127 | image: nginx 128 | binds: 129 | /etc/nginx: 130 | - !env nginx_config_path 131 | - ro 132 | uses: app_server_socket 133 | attaches: web_log 134 | exposes: 135 | 80: 80 136 | 443: 443 137 | app_server: 138 | image: app 139 | instances: 140 | - instance1 141 | - instance2 142 | binds: 143 | - app_config: ro 144 | - app_data: 145 | attaches: 146 | - app_log 147 | - app_server_socket 148 | user: 2000 149 | permissions: u=rwX,g=rX,o= 150 | volumes: 151 | web_log: /var/log/nginx 152 | app_server_socket: /var/lib/app/socket 153 | app_config: /var/lib/app/config 154 | app_log: /var/lib/app/log 155 | app_data: /var/lib/app/data 156 | host: 157 | app_config: 158 | instance1: !env app1_config_path 159 | instance2: !env app2_config_path 160 | app_data: 161 | instance1: !env app1_data_path 162 | instance2: !env app2_data_path 163 | 164 | 165 | Client configurations 166 | ^^^^^^^^^^^^^^^^^^^^^ 167 | With some modifications, this map could also run a setup on multiple hosts, for example one web server running as 168 | reverse proxy for multiple identical app servers:: 169 | 170 | env.docker_maps.update( 171 | web_server={ 172 | 'clients': 'web', 173 | 'uses': [], # No longer look for a socket 174 | }, 175 | app_server={ 176 | 'clients': ('apps1', 'apps2', 'apps3'), 177 | 'attaches': 'app_log', # No longer create a socket 178 | 'exposes': [(8443, 8443, 'private')], # Expose a TCP port on 8443 of the private network interface 179 | } 180 | ) 181 | 182 | The modifications could of course have been included in the aforementioned map right away. Moreover, all of this has to 183 | be set up in the web server's and app servers' configuration accordingly. 184 | 185 | A client configuration in ``example_clients.yaml`` could look like this: 186 | 187 | .. code-block:: yaml 188 | 189 | web: 190 | fabric_host: web_host # Set the Fabric host here. 191 | apps1: 192 | fabric_host: app_host1 193 | interfaces: 194 | private: 10.x.x.21 # Provide the individual IP address for each host. 195 | apps2: 196 | fabric_host: app_host2 197 | interfaces: 198 | private: 10.x.x.22 199 | apps3: 200 | fabric_host: app_host3 201 | interfaces: 202 | private: 10.x.x.23 203 | 204 | 205 | Since there is no dependency indicated by the configuration between the web and app servers, two startup commands are 206 | required; still they will connect to each host as necessary:: 207 | 208 | with container_fabric() as cf: 209 | cf.startup('web_server') 210 | cf.startup('app_server') 211 | 212 | In addition to creating and starting the containers, ports will be bound to each private network adapter individually. 213 | 214 | .. _Docker-Map: https://docker-map.readthedocs.org/ 215 | -------------------------------------------------------------------------------- /docs/guide/tasks.rst: -------------------------------------------------------------------------------- 1 | .. _tasks: 2 | 3 | Utility tasks for Fabric 4 | ======================== 5 | Included utility tasks perform some basic actions within Docker. When importing them into your ``fabfile.py``, you 6 | might want to assign an alias to the module, for having a clear task namespace:: 7 | 8 | import dockerfabric.tasks as docker 9 | 10 | Then the following commands work directly from the command line, e.g. ``fab ``. A basic description of 11 | each task is displayed when running ``fab --list`` -- the following sections describe a few further details. 12 | 13 | 14 | General purpose tasks 15 | --------------------- 16 | **Socat** does not terminate after all connections to the host have been closed. Although this can be changed by setting 17 | ``env.socat_fork`` to ``False``, there may be instances where it may be necessary to close the process manually, e.g. 18 | when the ``fork`` setting has just recently been set. The task :func:`~dockerfabric.tasks.reset_socat` finds **socat**'s 19 | process id(s) and sends a `kill` signal. 20 | 21 | For configuration between containers and firewalls, the host's IP address can be obtained using the tasks 22 | :func:`~dockerfabric.tasks.get_ip` and :func:`~dockerfabric.tasks.get_ipv6`. Without further arguments it returns 23 | the address of the `docker0` interface. Specifying a different interface is possible via the first argument: 24 | 25 | .. code-block:: bash 26 | 27 | fab get_ip:eth0 28 | 29 | returns the IPv4 address of the first network adapter. IPv6 addresses can additionally be expanded, e.g. 30 | 31 | .. code-block:: bash 32 | 33 | fab get_ipv6:eth0:True 34 | 35 | returns the full address instead of the abbreviated version provided by ``ifconfig``. 36 | 37 | .. tip:: If you would like to handle this information directly in code, use the utility functions 38 | :func:`~dockerfabric.utils.net.get_ip4_address` and :func:`~dockerfabric.utils.net.get_ip6_address` instead. 39 | 40 | 41 | Docker tasks 42 | ------------ 43 | The following tasks are directly related to Docker and processed by the service on the remote host. 44 | 45 | Information tasks 46 | ^^^^^^^^^^^^^^^^^ 47 | As mentioned in the :ref:`installation_and_configuration` section, :func:`~dockerfabric.tasks.version` provides a 48 | similar output to running ``docker version`` on the command line. 49 | 50 | Similarly, :func:`~dockerfabric.tasks.list_images` and :func:`~dockerfabric.tasks.list_containers` print a list of 51 | available images and running containers. The output is slightly different from the corresponding command line's. For 52 | ``list_containers`` 53 | 54 | * Ports and multiple container names (e.g. linking aliases) are broken into multiple lines, 55 | * images are by default shown without their registry prefix (can be changed by passing ``short_image=False``), 56 | * the absolute creation timestamp is printed, 57 | * and by default all containers are shown (can be changed by passing an empty string as the first argument). 58 | 59 | In the output of ``list_images`` 60 | 61 | * parent image ids are shown, 62 | * and also here the absolute creation timestamp is printed. 63 | 64 | Container tasks 65 | ^^^^^^^^^^^^^^^ 66 | As of version 0.3.0, container maps are recommended to be set in ``env.docker_maps`` (as list or single entry) and 67 | multiple clients to be configured in ``env.docker_clients``. In that setup, the lifecycle of containers, including their 68 | dependencies, can be entirely managed from the command line without creating individual tasks for them. 69 | The module :mod:`~dockerfabric.actions` contains the following actions: 70 | 71 | * :func:`~dockerfabric.actions.create` - Creates a container and its dependencies. 72 | * :func:`~dockerfabric.actions.start` - Starts a container and its dependencies. 73 | * :func:`~dockerfabric.actions.stop` - Stops a container and its dependents. 74 | * :func:`~dockerfabric.actions.remove` - Removes a container and its dependents. 75 | * :func:`~dockerfabric.actions.startup` - Creates and starts a container and its dependencies. 76 | * :func:`~dockerfabric.actions.shutdown` - Stops and removes a container and its dependents. 77 | * :func:`~dockerfabric.actions.update` - Updates a container and its dependencies. Creates and starts containers as 78 | necessary. 79 | * :func:`~dockerfabric.actions.kill` - Sends a ``SIGKILL`` signal to a single container. A different signal can be 80 | sent by specifying in the keyword argument ``signal``, e.g. ``signal=SIGHUP``. 81 | * :func:`~dockerfabric.actions.pull_images` - Pulls the image of the specified container if it is absent, and all of 82 | its dependency container images. 83 | * :func:`~dockerfabric.actions.script` - Uploads and runs a script inside a container, which is created specifically 84 | for that purpose, along with its dependencies. The container is removed after the script has completed. 85 | * :func:`~dockerfabric.actions.single_cmd` - Similar to :func:`~dockerfabric.actions.script`, but not uploading 86 | contents beforehand, for running a self-contained command (e.g. Django `migrate`, Redis `flusdhdb` etc.). If this 87 | produces files, the results can be downloaded however. 88 | 89 | .. note:: 90 | 91 | There is also a generic action :func:`~dockerfabric.actions.perform`. Performs an action on the given container map 92 | and configuration. There needs to be a matching implementation in the policy class. 93 | 94 | Given the lines in ``fabfile.py``:: 95 | 96 | from dockerfabric import yaml, actions 97 | 98 | env.docker_maps = yaml.load_map_file('/path/to/example_map.yaml', 'example_map') 99 | env.docker_clients = yaml.load_clients_file('/path/to/example_clients.yaml') 100 | 101 | 102 | The web server from the :ref:`yaml-import` example may be started with 103 | 104 | .. code-block:: bash 105 | 106 | fab actions.startup:example_map,web_server 107 | 108 | runs the web server and its dependencies. The command 109 | 110 | .. code-block:: bash 111 | 112 | fab actions.update:example_map,web_server 113 | 114 | stops, removes, re-creates, and starts the container if the image as specified in the container configuration (e.g. 115 | ``nginx:latest``) has been updated, or mapped volumes virtual filesystems are found to mismatch the dependency 116 | containers' shared volumes. 117 | 118 | Maintencance tasks 119 | ^^^^^^^^^^^^^^^^^^ 120 | The maintenance tasks :func:`~dockerfabric.tasks.cleanup_containers`, :func:`~dockerfabric.tasks.cleanup_images`, and 121 | :func:`~dockerfabric.tasks.remove_all_containers` simply call the corresponding methods of 122 | :class:`~dockerfabric.apiclient.DockerFabricClient`: 123 | 124 | * :func:`~dockerfabric.tasks.cleanup_containers` removes all containers that have the `Exited` status; 125 | * :func:`~dockerfabric.tasks.cleanup_images` removes all untagged images, optionally with the argument ``True`` also 126 | images without a ``latest`` tag. Additional tags can be specified by setting the environment variable 127 | ``docker_keep_tags``. 128 | * :func:`~dockerfabric.tasks.remove_all_containers` stops and removes all containers from the host. 129 | 130 | Image transfer 131 | ^^^^^^^^^^^^^^ 132 | Especially during the initial deployment you may run into a situation where manual image transfer is necessary. For 133 | example, when you plan to use your own registry, but would like to use your own web server image for a reverse proxy, 134 | the following tasks help to download the image from your build system to the client, and upload it to the production 135 | server: 136 | 137 | Use :func:`~dockerfabric.tasks.save_image` with two arguments: Image name or id, and file name. If the file name is 138 | omitted, the image is stored in the current working directory, as ``.tar.gz``. For performance reasons, 139 | :func:`~dockerfabric.tasks.save_image` currently relies on the command line client. The compressed tarball is generated 140 | on the host. 141 | 142 | .. code-block:: bash 143 | 144 | fab docker.save_image:new_image.tar.gz 145 | 146 | In reverse, :func:`~dockerfabric.tasks.load_image` uploads a local image to the Docker host. In this case the Docker 147 | Remote API is used. It accepts plain and gzip-compressed tarballs. The local image file name is the first argument. 148 | Since the API often times out for larger images (default is 60 seconds), the period is extended temporarily to 149 | 120 seconds. This can optionally be adjusted with a second argument, e.g. 150 | 151 | .. code-block:: bash 152 | 153 | fab docker.load_image:new_image.tar.gz:600 154 | 155 | for an image that might take longer to upload due to a slow connection. 156 | -------------------------------------------------------------------------------- /docs/guide/utils.rst: -------------------------------------------------------------------------------- 1 | .. _cli_utils: 2 | 3 | Miscellaneous utilities 4 | ======================= 5 | Docker-Fabric supports deployments with a set of additional tools. Some components rely on the command line interface 6 | (CLI) of Docker, that can be run through Fabric. 7 | 8 | .. _cli: 9 | 10 | Docker command line interface 11 | ----------------------------- 12 | Some functionality has been implemented using the Docker CLI mainly for performance reasons. It has turned out in 13 | earlier tests that the download speed of container and image data through the SSH tunnel was extremely slow. This may 14 | be due to the tunnelling. The effect is further increased by the fact that the remote API currently does not compress 15 | any downstream data (e.g. container and image transfers to a client), although it accepts gzip-compressed upstream. 16 | 17 | Containers 18 | ^^^^^^^^^^ 19 | The two functions :func:`~dockerfabric.cli.copy_resource` and :func:`~dockerfabric.cli.copy_resources`, as the name may 20 | suggest, extract files and directories from a container. They behave slightly different from one another however: 21 | Whereas the former is more simple, the latter aims for flexibility. 22 | 23 | For downloading some files and packaging them into a tarball (similar to what the ``copy`` function of the API would do) 24 | :func:`~dockerfabric.cli.copy_resource` is more appropriate. Example:: 25 | 26 | from dockerfabric import cli 27 | cli.copy_resource('app_container', '/var/log/app', 'app_logs.tar.gz') 28 | 29 | This downloads all files from ``/var/log/app`` in the container ``app_container`` onto the host, packages them into 30 | a compressed tarball, and downloads that to your client. Finally, it removes the downloaded source files from the host. 31 | 32 | If the copied resource is a directory, contents of this directory are packaged into the top level of the archive. This 33 | behavior can be changed (i.e. having the directory on the root level) by setting the optional keyword argument 34 | ``contents_only=False``. 35 | 36 | The more advanced :func:`~dockerfabric.cli.copy_resources` is suitable for complex tasks. It does not create 37 | a tarball and does not download to your client, but can copy multiple resources, and modify file ownership (`chown`) as 38 | well as file permissions (`chmod`) after the download:: 39 | 40 | ... 41 | resources = ['/var/lib/app/data1', '/var/lib/app/data2'] 42 | temp_mapping = {'/var/lib/app/data1': 'd1', '/var/lib/app/data1': 'd2'} 43 | cli.copy_resources('app_container', resources, '/home/data', dst_directories=temp_mapping, apply_chmod='0750') 44 | 45 | This example downloads two directories from ``app_container``, stores them in a folder ``/home/data``, and 46 | changes the file system permissions to read-write for the owner, read-only for the group, and no access for anyone else. 47 | 48 | The directories would by default be stored within their original structure, but in this example are renamed to ``d1`` 49 | and ``d2``. This is also possible with files. In order to override the generic fallback mapping (e.g. to something else 50 | than the resource name), add a key ``*`` to the dictionary. That way contents of multiple directories can be merged into 51 | one. 52 | 53 | In case you would rather compress and download these files and directories, instead use 54 | :func:`~dockerfabric.cli.isolate_and_get`. Similar to ``copy_resource`` contents are only stored temporarily and 55 | downloaded as a single gzip-compressed tarball, but with the same options as :func:`~dockerfabric.cli.copy_resources`:: 56 | 57 | cli.isolate_and_get('app_container', resources, 'container.tar.gz', dst_directories=temp_mapping) 58 | 59 | It results in tar archive with ``d1`` and ``d2`` as top-level elements. 60 | 61 | Since Docker also supports creating images from tar files, :func:`~dockerfabric.cli.isolate_to_image` can generate an 62 | image that contains only the selected resources. Instead of a target file or directory, specify an image name instead:: 63 | 64 | cli.isolate_to_image('app_container', resources, 'new_image', dst_directories=temp_mapping) 65 | 66 | Note that the image at that point still has no configuration. In order for being able to run it as a container, some 67 | executable file needs to be included. 68 | 69 | Images 70 | ^^^^^^ 71 | As an alternative to the remote API ``save_image``, :func:`~dockerfabric.cli.save_image` stores the contents of an 72 | entire image into a compressed tarball and downloads that. It takes two arguments, the image and the tarball:: 73 | 74 | cli.save_image('app_image', 'app_image.tar.gz') 75 | 76 | The function :func:`~dockerfabric.cli.flatten_image` works different from ``save_image``: It downloads the contents of 77 | an image and stores them in a new one. This can reduce the size, but comes with a couple of limitations. 78 | 79 | A template container has to be created from the image first, and started with a command that makes no further 80 | modifications. For Linux images including the core utilities, such a command is typically ``/bin/true``; where 81 | applicable it should be changed using the keyword argument ``no_op_cmd``:: 82 | 83 | cli.flatten_image('app_image', 'new_image', no_op_cmd='/true') 84 | 85 | If the second argument is not provided, the original image is overwritten. Like ``isolate_to_image``, the original 86 | configuration is not transferred to the new image. 87 | 88 | Fabric context managers 89 | ----------------------- 90 | The following context managers complement :mod:`fabric.context_managers`. They are referenced in 91 | other areas of Docker-Fabric, and can also be used directly for deployments. 92 | 93 | .. note:: Docker-Fabric includes more utility functions. Not all are described here, but are documented with the 94 | package :mod:`dockerfabric.utils`. 95 | 96 | For some purposes it may be useful to create a temporary container from an image, copy some data from it, and destroy it 97 | afterwards. This is provided by :func:`~dockerfabric.utils.containers.temp_container`:: 98 | 99 | from dockerfabric.utils.containers import temp_containers 100 | 101 | with temp_container('app_image', no_op_cmd='/true'): 102 | ... 103 | 104 | In fact it is not a requirement that the command provided in the keyword argument ``no_op_cmd`` actually performs no 105 | changes. The command should finish without any interaction however, as the function waits before 106 | processing further commands inside that block. Further supported arguments are ``create_kwargs`` and ``start_kwargs``, 107 | for cases where it is necessary to modify the create and start options of a temporary container. 108 | 109 | Management of local files, e.g. for copying around container contents, is supported with two more temporary contexts: 110 | :func:`~dockerfabric.utils.files.temp_dir` creates a temporary directory on the remote host, that is removed after 111 | leaving the context block. An alias should be assigned for use inside the context block:: 112 | 113 | from dockerfabric.utils.files import temp_dir 114 | 115 | with temp_dir() as remote_tmp: 116 | cli.copy_resources('app_container', resources, remote_tmp, dst_directories=temp_mapping, apply_chmod='0750') 117 | ... 118 | ... 119 | # Directory is removed at this point 120 | 121 | The local counterpart is :func:`~dockerfabric.utils.files.local_temp_dir`: It creates a temporary folder on the client 122 | side:: 123 | 124 | from dockerfabric.utils.files import local_temp_dir 125 | 126 | with local_temp_dir() as local_tmp: 127 | cli.copy_resource('app_container', '/var/log/app', os.path.join(local_tmp, 'app_logs.tar.gz')) 128 | ... 129 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. Docker-Fabric documentation master file, created by 2 | sphinx-quickstart on Tue Sep 2 07:43:06 2014. 3 | 4 | Welcome to Docker-Fabric's documentation! 5 | ========================================= 6 | 7 | Docker-Fabric provides a set of utilities for controlling Docker on a local test machine or a remote production 8 | environment. It combines Fabric_ with extensions to docker-py_ in Docker-Map_. 9 | 10 | The project is hosted on GitHub_. 11 | 12 | 13 | Features 14 | ======== 15 | * Integration of :ref:`Docker-Map's container structure ` into Fabric deployments. 16 | * Complements Docker API commands with command line shortcuts. 17 | * Uses Fabric's SSH tunnel for connecting to the Docker Remote API. 18 | * Fabric-like console feedback for Docker image and container management. 19 | 20 | 21 | Contents 22 | ======== 23 | 24 | .. toctree:: 25 | :maxdepth: 2 26 | 27 | installation 28 | start 29 | guide/apiclient 30 | guide/cli 31 | guide/utils 32 | guide/tasks 33 | guide/containers 34 | changes 35 | 36 | 37 | Status 38 | ====== 39 | Docker-Fabric is being used for small-scale deployment scenarios in test and production. It should currently considered 40 | beta, due to pending new features, generalizations, and unit tests. 41 | 42 | 43 | Indices and tables 44 | ================== 45 | 46 | * :ref:`genindex` 47 | * :ref:`modindex` 48 | * :ref:`search` 49 | 50 | 51 | .. _Fabric: http://www.fabfile.org 52 | .. _docker-py: https://github.com/docker/docker-py 53 | .. _Docker-Map: https://github.com/merll/docker-map 54 | .. _GitHub: https://github.com/merll/docker-fabric 55 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | .. _installation_and_configuration: 2 | 3 | Installation and configuration 4 | ============================== 5 | 6 | Installation 7 | ------------ 8 | The current stable release, published on PyPI_, can be installed using the following command: 9 | 10 | .. code-block:: bash 11 | 12 | pip install docker-fabric 13 | 14 | 15 | For importing YAML configurations for Docker-Map_, you can install Docker-Fabric using 16 | 17 | .. code-block:: bash 18 | 19 | pip install docker-fabric[yaml] 20 | 21 | 22 | Dependencies 23 | ^^^^^^^^^^^^ 24 | The following libraries will be automatically installed from PyPI: 25 | 26 | * Fabric (tested with >=1.8.0) 27 | * docker-py (>=1.9.0) 28 | * docker-map (>=0.8.0) 29 | * Optional: PyYAML (tested with 3.11) for YAML configuration import 30 | 31 | 32 | Docker service 33 | ^^^^^^^^^^^^^^ 34 | Docker needs to be installed on the target machine. There used to be a utility task for this, but the required steps for 35 | installation tended to change too much too quickly for maintaining them properly. Please follow the 36 | `Docker installation instructions`_ according to your operating system. 37 | 38 | 39 | Socat 40 | ^^^^^ 41 | The tool Socat_ is needed in order to tunnel local TCP-IP connections to a unix socket on the target machine. The 42 | ``socat`` binary needs to be in the search path. It is included in most common Linux distributions, e.g. for CentOS 43 | you can install it using ``yum install socat``; or you can download the source code and compile it yourself. 44 | 45 | 46 | Configuration 47 | ------------- 48 | 49 | Docker service 50 | ^^^^^^^^^^^^^^ 51 | On every target machine, Docker-Fabric needs access to the Docker Remote API and (optionally) to the command line 52 | client. With the default Docker configuration, this requires for the connecting SSH user to be in the `docker` 53 | user group. The group assignment provides access to the unix socket. 54 | 55 | For assigning an existing user to that group, run 56 | 57 | .. code-block:: bash 58 | 59 | usermod -aG docker 60 | 61 | 62 | Note that if you run this command with the same user (using `sudo`), you need to re-connect. Use 63 | :func:`~fabric.network.disconnect_all` if necessary. 64 | 65 | 66 | Tasks 67 | ^^^^^ 68 | If you plan to use the built-in tasks, include the module in your fabfile module (e.g. `fabfile.py`). Most likely 69 | you might want to assign an alias for the task namespace:: 70 | 71 | from dockerfabric import tasks as docker 72 | 73 | 74 | .. _fabric_env: 75 | 76 | Environment 77 | ^^^^^^^^^^^ 78 | In order to customize the general behavior of the client, the following variables can be set in `Fabric's env`_. All 79 | of them are generally optional, but some are needed when tunnelling connections over SSH: 80 | 81 | * ``docker_base_url``: The URL of the Docker service. If not set, defaults to a socket connection in 82 | ``/var/run/docker.sock``, which is also the default behavior of the `docker-py` client. 83 | If ``docker_tunnel_remote_port`` and/or ``docker_tunnel_local_port`` is set, the connection will be tunnelled through 84 | SSH, otherwise the value is simply passed to `docker-py`. For socket connections (i.e. this is blank, starts with 85 | a forward slash, or is prefixed with ``http+unix:``, ``unix:``), **socat** will be used to forward the TCP-IP tunnel 86 | to the socket. 87 | * ``docker_tunnel_local_port``: Set this, if you need a tunneled socket connection. Alternatively, the value 88 | ``docker_tunnel_remote_port`` is used (unless empty as well). This is the first local port for tunnelling 89 | connections to a Docker service on the remote. Since during simultaneous connections, a separate local port has to be 90 | available for each, the port number is increased by one on every new connection. This means for example, that when 91 | setting this to 2224 and connecting to 10 servers, ports from 2224 through 2233 will be temporarily occupied. 92 | * ``docker_tunnel_remote_port``: Port of the Docker service. 93 | 94 | - On TCP connections, this is the remote endpoint of the tunnel. If a different port is included in 95 | ``docker_base_url``, this setting is ignored. 96 | - For socket connections, this is the initial local tunnel port. If specified by ``docker_tunnel_local_port``, this 97 | setting has no effect. 98 | 99 | * ``docker_timeout``: Request timeout of the Docker service; by default uses 100 | :const:`~docker-py.docker.client.DEFAULT_TIMEOUT_SECONDS`. 101 | * ``docker_api_version``: API version used to communicate with the Docker service, as a string, such as ``1.16``. 102 | Must be lower or equal to the accepted version. By default uses 103 | :const:`~docker-py.docker.client.DEFAULT_DOCKER_API_VERSION`. 104 | 105 | 106 | Additionally, the following variables are specific for Docker registry access. They can be overridden in the relevant 107 | commands (:meth:`~dockerfabric.apiclient.DockerFabricClient.login`, 108 | :meth:`~dockerfabric.apiclient.DockerFabricClient.push`, and 109 | :meth:`~dockerfabric.apiclient.DockerFabricClient.pull`). 110 | 111 | * ``docker_registry_user``: User name to use when authenticating against a Docker registry. 112 | * ``docker_registry_password``: Password to use when authenticating against a Docker registry. 113 | * ``docker_registry_mail``: E-Mail to use when authenticating against a Docker registry. 114 | * ``docker_registry_repository``: Optional; the registry to connect to. This will be expanded to a URL automatically. 115 | If not set, registry operations will run on the public Docker index. 116 | * ``docker_registry_insecure``: Whether to set the `insecure` flag on Docker registry operations, e.g. when accessing your 117 | self-hosted registry over plain HTTP. Default is ``False``. 118 | 119 | 120 | Examples 121 | ^^^^^^^^ 122 | For connecting to a remote Docker instance over a socket, install **socat** on the remote, and put the following in 123 | your ``fabfile``:: 124 | 125 | from fabric.api import env 126 | from dockerfabric import tasks as docker 127 | 128 | env.docker_tunnel_local_port = 22024 # or any other available port above 1024 of your choice 129 | 130 | 131 | If the remote Docker instance accepts connections on port 8000 from localhost (not recommended), use the following:: 132 | 133 | from fabric.api import env 134 | from dockerfabric import tasks as docker 135 | 136 | env.docker_base_url = 'tcp://127.0.0.1:8000' 137 | env.docker_tunnel_local_port = 22024 # or any other available port above 1024 of your choice 138 | 139 | 140 | Checking the setup 141 | ------------------ 142 | For checking if everything is set up properly, you can run the included task `version`. For example, running 143 | 144 | .. code-block:: bash 145 | 146 | fab docker.version 147 | 148 | 149 | against a local Vagrant machine (using the default setup, only allowing socket connections) and tunnelling through 150 | port 2224 should show a similar result:: 151 | 152 | [127.0.0.1] Executing task 'docker.version' 153 | [127.0.0.1] 154 | KernelVersion: 3.13.0-34-generic 155 | Arch: amd64 156 | ApiVersion: 1.14 157 | Version: 1.2.0 158 | GitCommit: fa7b24f 159 | Os: linux 160 | GoVersion: go1.3.1 161 | 162 | Done. 163 | Disconnecting from 127.0.0.1:2222... done. 164 | 165 | 166 | .. _PyPI: https://pypi.python.org/pypi/docker-fabric 167 | .. _Docker-Map: https://pypi.python.org/pypi/docker-map 168 | .. _Socat: http://www.dest-unreach.org/socat/ 169 | .. _Fabric's env: http://docs.fabfile.org/en/latest/usage/env.html 170 | .. _Docker installation instructions: https://docs.docker.com/engine/installation/ 171 | -------------------------------------------------------------------------------- /docs/start.rst: -------------------------------------------------------------------------------- 1 | .. _getting_started: 2 | 3 | Getting started 4 | =============== 5 | 6 | In order to connect with the Docker service, make sure that 7 | 8 | 1. Docker is installed on the remote machine; 9 | 2. **socat** is installed and in the remote's search path, if you are using the SSH tunnel; 10 | 3. and the SSH user has access to the service. 11 | 12 | (For details, refer to :ref:`installation_and_configuration`). 13 | 14 | 15 | Calls to the Remote API can be made by using :func:`~dockerfabric.apiclient.docker_fabric`. This function uses Fabric's 16 | usual SSH connection (creates a new one if necessary) and opens a separate channel for forwarding requests to the 17 | Docker Remote API. 18 | 19 | Since this is merely a wrapper, all commands to `docker-py` are supported. Some additional functionality is provided 20 | by Docker-Map. However, instead of repeatedly passing in similar parameters (e.g. the service URL), settings can be 21 | preset globally for the project. Additionally, it provides a caching functionality for open tunnels and connections, 22 | which speeds up access to Docker significantly. 23 | 24 | 25 | Short examples:: 26 | 27 | from dockerfabric.apiclient import docker_fabric 28 | docker_fabric().version() 29 | 30 | returns version information from the installed Docker service. This function is directly passed through to 31 | ``docker-py`` and formatted. The utility function:: 32 | 33 | docker_fabric().cleanup_containers() 34 | 35 | removes all containers on the target Docker service that have exited. 36 | 37 | For building images, use Docker-Map's :class:`~dockermap.build.dockerfile.DockerFile` for generating an environment, 38 | and run:: 39 | 40 | docker_fabric().build_from_file(dockerfile, 'new_image_tag:1.0', rm=True) 41 | 42 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from distutils.spawn import find_executable 2 | import os 3 | from setuptools import setup, find_packages 4 | 5 | from dockerfabric import __version__ 6 | 7 | 8 | def include_readme(): 9 | try: 10 | import pandoc 11 | except ImportError: 12 | return '' 13 | pandoc.core.PANDOC_PATH = find_executable('pandoc') 14 | readme_file = os.path.join(os.path.dirname(__file__), 'README.md') 15 | doc = pandoc.Document() 16 | with open(readme_file, 'r') as rf: 17 | doc.markdown = rf.read() 18 | return doc.rst 19 | 20 | 21 | setup( 22 | name='docker-fabric', 23 | version=__version__, 24 | packages=find_packages(), 25 | install_requires=['six', 'Fabric>=1.8.0', 'docker-py>=1.9.0', 'docker-map>=0.8.0b2'], 26 | extras_require={ 27 | 'yaml': ['PyYAML'], 28 | }, 29 | license='MIT', 30 | author='Matthias Erll', 31 | author_email='matthias@erll.de', 32 | url='https://github.com/merll/docker-fabric', 33 | description='Build Docker images, and run Docker containers in Fabric.', 34 | long_description=include_readme(), 35 | platforms=['OS Independent'], 36 | keywords=['docker', 'fabric'], 37 | classifiers=[ 38 | 'Environment :: Web Environment', 39 | 'Intended Audience :: Developers', 40 | 'License :: OSI Approved :: MIT License', 41 | 'Operating System :: OS Independent', 42 | 'Programming Language :: Python', 43 | 'Topic :: Software Development :: Libraries :: Python Modules', 44 | 'Topic :: Software Development :: Build Tools', 45 | 'Topic :: System :: Software Distribution', 46 | 'Development Status :: 4 - Beta', 47 | 'Programming Language :: Python :: 2.7', 48 | ], 49 | include_package_data=True, 50 | ) 51 | --------------------------------------------------------------------------------