├── dirg ├── __init__.py ├── _version.py ├── container_utils.py ├── service_utils.py └── dirg.py ├── tests └── __init__.py ├── dirg.cfg.sample ├── .gitignore ├── tox.ini ├── dirg-services.yml.sample ├── LICENSE ├── setup.py ├── README.md └── README.rst /dirg/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dirg/_version.py: -------------------------------------------------------------------------------- 1 | __version__ = '1.0.0.' 2 | -------------------------------------------------------------------------------- /dirg.cfg.sample: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | dirg_services=dirg-services.yml.sample -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | venv 3 | dirg.egg-info 4 | dist 5 | .tox 6 | .idea 7 | __pycache__ 8 | *.pyc 9 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist=py27,py33,py34 3 | [testenv] 4 | deps=nose 5 | argparse 6 | commands=nosetests -------------------------------------------------------------------------------- /dirg-services.yml.sample: -------------------------------------------------------------------------------- 1 | --- 2 | service1: 3 | - container1 4 | - container2 5 | service2: 6 | - container3 7 | - container4 8 | all: 9 | - container1 10 | - container2 11 | - container3 12 | - container4 13 | - container5 14 | --- 15 | 16 | container1: 17 | image: imagename 18 | volumes: volumes 19 | volume_bindings: volume bindings 20 | 21 | container2: 22 | image: imagename 23 | volumes: volumes 24 | volume_bindings: volume bindings 25 | 26 | container3: 27 | image: imagename 28 | volumes: volumes 29 | volume_bindings: volume bindings 30 | 31 | container4: 32 | image: imagename 33 | volumes: volumes 34 | volume_bindings: volume bindings 35 | 36 | container5: 37 | image: imagename 38 | volumes: volumes 39 | volume_bindings: volume bindings 40 | 41 | container6: 42 | image: imagename 43 | volumes: volumes 44 | volume_bindings: volume bindings 45 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015 Smaato Inc. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import os 3 | 4 | try: 5 | from setuptools import setup, find_packages, Command, convert_path 6 | except ImportError: 7 | from distutils.core import setup, Command 8 | from distutils.util import convert_path 9 | 10 | 11 | def _find_packages(where='.', exclude=()): 12 | """Return a list all Python packages found within directory 'where' 13 | 14 | 'where' should be supplied as a "cross-platform" (i.e. URL-style) path; it 15 | will be converted to the appropriate local path syntax. 'exclude' is a 16 | sequence of package names to exclude; '*' can be used as a wildcard in the 17 | names, such that 'foo.*' will exclude all subpackages of 'foo' (but not 18 | 'foo' itself). 19 | """ 20 | out = [] 21 | stack = [(convert_path(where), '')] 22 | while stack: 23 | where, prefix = stack.pop(0) 24 | for name in os.listdir(where): 25 | fn = os.path.join(where, name) 26 | if ('.' not in name and os.path.isdir(fn) and 27 | os.path.isfile(os.path.join(fn, '__init__.py'))): 28 | out.append(prefix + name) 29 | stack.append((fn, prefix + name + '.')) 30 | for pat in list(exclude) + ['ez_setup', 'distribute_setup']: 31 | from fnmatch import fnmatchcase 32 | out = [item for item in out if not fnmatchcase(item, pat)] 33 | return out 34 | 35 | find_packages = _find_packages 36 | 37 | setup( 38 | name='dirg', 39 | version='1.0.0', 40 | packages=find_packages('.', exclude=('tests',)), 41 | description='A docker orchestration tool.', 42 | author='Stephan Brosinski', 43 | author_email='stephan.brosinski@smaato.com', 44 | url='https://github.com/smaato/dirg', 45 | download_url='https://github.com/smaato/dirg', 46 | license='MIT License', 47 | keywords=['docker', 'orchestration', 'docker-py'], 48 | classifiers=[ 49 | 'Programming Language :: Python :: 2.7', 50 | 'Development Status :: 3 - Alpha', 51 | 'License :: OSI Approved :: MIT License', 52 | 'Natural Language :: English', 53 | 'Environment :: Other Environment', 54 | 'Intended Audience :: Developers', 55 | 'Topic :: Software Development :: Build Tools', 56 | ], 57 | package_data={'': ['LICENSE']}, 58 | include_package_data=True, 59 | entry_points={ 60 | 'console_scripts': [ 61 | 'dirg = dirg.dirg:main' 62 | ] 63 | }, 64 | install_requires=['jinja2', 'docker-py>=1.1.0', 'pyyaml'] 65 | ) 66 | -------------------------------------------------------------------------------- /dirg/container_utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from docker.utils import kwargs_from_env 3 | import sys 4 | import docker 5 | import json 6 | 7 | 8 | def find_docker_host(cli, name, conf): 9 | new_cli = cli 10 | if conf.get('docker_host'): 11 | args = kwargs_from_env(assert_hostname=False) 12 | args['base_url'] = conf.get('docker_host') 13 | args['version'] = '1.17' 14 | return docker.Client(**args) 15 | 16 | if not new_cli: 17 | print('DOCKER_HOST not set globally or for container %s' % name) 18 | sys.exit(-1) 19 | 20 | return new_cli 21 | 22 | 23 | def create_container(cli, name, conf): 24 | host = find_docker_host(cli, name, conf) 25 | sys.stdout.flush() 26 | try: 27 | host.create_container(name=name, image=conf.get('image'), 28 | hostname=name, ports=conf.get('ports'), 29 | environment=conf.get('env'), 30 | volumes=conf.get('volumes'), 31 | command=conf.get('command', '')) 32 | except docker.errors.APIError as e: 33 | print(e) 34 | sys.exit(-1) 35 | 36 | 37 | def pull_container(cli, name, conf): 38 | print('Pulling image %s ... ' % conf.get('image')) 39 | cli = find_docker_host(cli, name, conf) 40 | sys.stdout.flush() 41 | try: 42 | for line in cli.pull(conf.get('image'), stream=True): 43 | print(json.dumps(json.loads(line), indent=4)) 44 | except docker.errors.APIError as e: 45 | print(e) 46 | sys.exit(-1) 47 | 48 | 49 | def start_container(cli, name, conf): 50 | print('Starting container %s ... ' % name, end='') 51 | host = find_docker_host(cli, name, conf) 52 | sys.stdout.flush() 53 | try: 54 | host.start( 55 | container=name, port_bindings=conf.get('port_bindings'), 56 | binds=conf.get('volume_bindings'), links=conf.get('links'), 57 | network_mode=conf.get('net', 'bridge')) 58 | except docker.errors.APIError as e: 59 | print(e) 60 | sys.exit(-1) 61 | print('ok') 62 | 63 | 64 | def stop_container(cli, name, conf): 65 | print('Stopping container %s ... ' % name, end='') 66 | host = find_docker_host(cli, name, conf) 67 | sys.stdout.flush() 68 | try: 69 | host.stop(container=name) 70 | except docker.errors.APIError as e: 71 | print(e) 72 | sys.exit(-1) 73 | print('ok') 74 | 75 | 76 | def remove_container(cli, name, conf): 77 | print('Removing container %s ... ' % name, end='') 78 | cli = find_docker_host(cli, name, conf) 79 | sys.stdout.flush() 80 | try: 81 | cli.stop(container=name) 82 | cli.remove_container(container=name, v=True) 83 | except docker.errors.APIError: 84 | print('not found') 85 | return 86 | print('ok') 87 | 88 | 89 | def build_container(cli, name, conf): 90 | print('Building container %s ... ' % name) 91 | cli = find_docker_host(cli, name, conf) 92 | sys.stdout.flush() 93 | try: 94 | if 'path' in conf: 95 | [print(json.loads(line).get('stream', ''), end='') 96 | for line in cli.build(path=conf.get('path'), rm=True)] 97 | except docker.errors.APIError as e: 98 | print(e) 99 | sys.exit(-1) 100 | 101 | 102 | def show_container_logs(cli, name, conf): 103 | print('Showing logs for container %s ... ' % name) 104 | cli = find_docker_host(cli, name, conf) 105 | try: 106 | [print(line, end='') for line in cli.logs(container=name, stderr=True, stdout=True, stream=True)] 107 | except docker.errors.APIError as e: 108 | print(e) 109 | sys.exit(-1) -------------------------------------------------------------------------------- /dirg/service_utils.py: -------------------------------------------------------------------------------- 1 | import container_utils 2 | import yaml 3 | from sys import stdin 4 | import docker 5 | import sys 6 | import json 7 | 8 | 9 | def run_service(cli, service): 10 | print('Running service ' + service['name']) 11 | for container in service['container']: 12 | container_utils.create_container( 13 | cli, container['name'], container['conf']) 14 | container_utils.start_container( 15 | cli, container['name'], container['conf']) 16 | 17 | 18 | def start_service(cli, service): 19 | print('Starting service ' + service['name']) 20 | for container in service['container']: 21 | container_utils.start_container( 22 | cli, container['name'], container['conf']) 23 | 24 | 25 | def stop_service(cli, service): 26 | print('Stopping service ' + service['name']) 27 | for container in service['container']: 28 | container_utils.stop_container( 29 | cli, container['name'], container['conf']) 30 | 31 | 32 | def update_service(cli, service): 33 | print('Updating service ' + service['name']) 34 | for container in service['container']: 35 | container_utils.pull_container( 36 | cli, container['name'], container['conf']) 37 | container_utils.remove_container( 38 | cli, container['name'], container['conf']) 39 | container_utils.create_container( 40 | cli, container['name'], container['conf']) 41 | container_utils.start_container( 42 | cli, container['name'], container['conf']) 43 | 44 | 45 | def build_service(cli, service): 46 | print('Building service ' + service['name']) 47 | for container in service['container']: 48 | container_utils.build_container( 49 | cli, container['name'], container['conf']) 50 | 51 | 52 | def show_service(cli, service): 53 | for container in service['container']: 54 | print('Container %s:' % container['name']) 55 | print('\n' + yaml.dump(container['conf'])) 56 | 57 | 58 | def pull_service(cli, service): 59 | for container in service['container']: 60 | container_utils.pull_container( 61 | cli, container['name'], container['conf']) 62 | 63 | 64 | def remove_service(cli, service): 65 | print('Removing service ' + service['name']) 66 | for container in service['container']: 67 | container_utils.remove_container( 68 | cli, container['name'], container['conf']) 69 | 70 | 71 | def show_service_logs(cli, service): 72 | choice = '1' 73 | if len(service['container']) > 1: 74 | print('Choose container:') 75 | num = 1 76 | for container in service['container']: 77 | print('%s) %s' % (num, container['name'])) 78 | num += 1 79 | choice = stdin.readline() 80 | container = service['container'][int(choice) - 1] 81 | container_utils.show_container_logs( 82 | cli, container['name'], container['conf']) 83 | 84 | 85 | def show_service_stats(cli, service): 86 | for container in service['container']: 87 | host = container_utils.find_docker_host(cli, container['name'], container['conf']) 88 | try: 89 | for stats_json in host.stats(container['name']): 90 | stats = json.loads(stats_json) 91 | #print(json.dumps(json.loads(line), indent=4)) 92 | print(stats['cpu_stats']) 93 | except docker.errors.APIError as e: 94 | print(e) 95 | sys.exit(-1) 96 | 97 | 98 | 99 | def list_services(cli, service): 100 | print('\n{:<25} {:<25} {:<25} {:<25}'.format('Service', 'Container', 'Status', 'Host')) 101 | print('-' * 120) 102 | 103 | for container in service['container']: 104 | host = container_utils.find_docker_host(cli, container['name'], container['conf']) 105 | container_status = host.containers() 106 | status = next( 107 | (c for c in container_status 108 | if '/' + container['name'] in c['Names']), None) 109 | 110 | if status: 111 | print('{:<25} {:<25} {:<25} {:<25}'.format( 112 | service['name'], container['name'], status['Status'], container['conf'].get('docker_host', ''))) 113 | else: 114 | print('{:<25} {:<25} not available'.format( 115 | service['name'], container['name'])) 116 | 117 | print(' ') 118 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Dirg - a docker container configuration and orchestration tool 2 | 3 | Maintained by [Stephan Brosinski](https://github.com/sbrosinski) for [Smaato Inc.](https://github.com/smaato/). 4 | 5 | Dirg is an orchestration tool for docker. It reads a yaml file describing services made of docker container definitions and allows to apply a number of commands to these groups of containers. 6 | 7 | # Why another orchestration tool? 8 | 9 | * Support for multi-host docker setups 10 | * Support for templating in service description 11 | 12 | # Installation 13 | 14 | Make sure you have 15 | 16 | * Python 2.7, 3.x is not supported yet 17 | * Python setuptools are installed 18 | 19 | You can install Dirg from the Python Package Index with 20 | 21 | $ pip install dirg 22 | 23 | Or you can clone the repository, then 24 | 25 | $ python setup.py install 26 | 27 | To check if the installation was successful, execute 28 | 29 | $ dirg info 30 | 31 | # Setting the docker host 32 | 33 | You can either set the `DOCKER_HOST` environment variable or set a specific docker host per container in the service description. 34 | 35 | Using a local docker host: 36 | 37 | $ export DOCKER_HOST=unix:///var/run/docker.sock 38 | 39 | Using a remote docker host via HTTP: 40 | 41 | $ export DOCKER_HOST=tcp://remote.host:2375 42 | 43 | Using a remote docker host via HTTPS: 44 | 45 | $ export DOCKER_HOST=https://remote.host:2375 46 | $ export DOCKER_CERT_PATH=/path/to/client/cert.pem 47 | $ export DOCKER_TLS_VERIFY=1 48 | 49 | # Dirg Commands 50 | 51 | Most commands have the form: 52 | 53 | $ dirg COMMAND SERVICE_NAME 54 | 55 | If `SERVICE_NAME` is missing, `all` is the default service name. 56 | 57 | `COMMAND` can be 58 | 59 | run Create all container and start them. 60 | info Prints out environment info. 61 | start Start all service container. 62 | stop Stop all service container 63 | rm Remove all service service. 64 | build Build all service container images. 65 | ps List all services and their container status. 66 | show Show service container config. 67 | pull Pull service container images. 68 | logs Show service logs. 69 | update Update service. 70 | stats Show service stats. 71 | 72 | Adding `-d` will print out additional debug information. This is valuable when you want to make sure Dirg is finding the right service configuration. `info` shows all environment variables needed for a SSL connection. 73 | 74 | # Service Configuration 75 | 76 | To configure Dirg you need a configuration file called `dirg.cfg` and a yaml description of your services. When you execute Dirg, it looks for file named `dirg.cfg` in the current directory. You can set an environment variable `DIRG_CFG` to point to your `dirg.cfg` file. 77 | 78 | A minimal `dirg.cfg` looks like this: 79 | 80 | [DEFAULT] 81 | dirg_services = /path/to/dirg-services.yml 82 | 83 | It holds a reference to the file describing your docker based services. In addition, you may define your own properties and values which you can then use in your service description. E.g. you could add you docker image registry URL to `dirg.cfg` and then reference it in your container definitions. 84 | 85 | A `dirg-services.yml` looks like this: 86 | 87 | --- 88 | service1: 89 | - container1 90 | - container2 91 | service2: 92 | - container3 93 | - container4 94 | all: 95 | - container1 96 | - container2 97 | - container3 98 | - container4 99 | - container5 100 | --- 101 | 102 | container1: 103 | image: imagename 104 | volumes: volumes 105 | volume_bindings: volume bindings 106 | 107 | container2: 108 | image: imagename 109 | volumes: volumes 110 | volume_bindings: volume bindings 111 | 112 | ... 113 | 114 | This yaml file contains 2 sub-documents (separated by ---). The first document describes all existing services. The second one describes the containers used by the services above. 115 | 116 | If you name a service `all` it will be the default service used by Dirg when you don't name a service upon calling Dirg commands. 117 | 118 | # Container Configuration 119 | 120 | Dirg supports the following container properties (more will be added as needed): 121 | 122 | | Property | Description | 123 | |-------------------|-------------------------------------------| 124 | | image | Image to use | 125 | | docker_host | Docker host to run this container on | 126 | | net | Network config | 127 | | env | Environment variables | 128 | | volumes | Volumes for the container | 129 | | volume_bindings | Mapping of container volumes | 130 | | ports | Ports opened by the container | 131 | | port_bindings | Mapping to host ports | 132 | | links | Docker links to other container | 133 | | command | Command to execute when container starts | 134 | 135 | This is a commented sample container definition using every configuration possible: 136 | 137 | # You can use comments in dirg-services.yml, block comments start with {# and end with #} 138 | # my_container will be set as container name on the docker host. 139 | my_container: 140 | 141 | # Stay DRY by using properties defined in dirg.cfg 142 | # Variables are enclosed in {{property_name}} 143 | image: {{registry}}/my_image_name 144 | 145 | # Run each command concerning this container on the following docker host 146 | docker_host: https://my.docker.host:2376 147 | 148 | # Use host network instead of bridge, which is default 149 | net: host 150 | 151 | # Define environment variables 152 | env: 153 | ENV1: value1 154 | ENV2: value2 155 | 156 | # Anywhere in dirg-services.yml you can also reference properties defined 157 | # as environment variables in the shell Dirg is running in. 158 | # This fills the docker environment variable with the contents of an 159 | # environment variable defined in the shell. If the shell environment 160 | # variable is not available, 'secret' is used as a default 161 | env: 162 | MY_PASSWORD: {{env['PASSWORD'] or 'secret'}} 163 | 164 | # Define volumes for the container 165 | volumes: [/logs, /data] 166 | 167 | # Then map them to host directories, specified in a property read from dirg.cfg 168 | volume_bindings: 169 | {{data_dir}}: {bind: /data} 170 | {{logs_dir}}: {bind: /logs} 171 | 172 | # Define ports exposed by the container 173 | ports: [80, 90] 174 | 175 | # Then map them to host ports 176 | port_bindings: {80: 8080, 90: 9090} 177 | 178 | # Ugly workaround to define a UDP port. This will be improved in a later version: 179 | ports: 180 | - !!python/tuple [8125, udp] 181 | port_bindings: {8125: 8125} 182 | 183 | # Link containers 184 | links: {db: db} 185 | 186 | # Execute command in container when it starts 187 | command: '/app/run_benchmark -p 80 -c 90' 188 | 189 | ## Advanced Templating 190 | 191 | Since the service description is a Jinja2 template you may do everything you can do in Jinja2. Take a look at the Jinja2 template designer documentation at http://jinja.pocoo.org/docs/dev/templates/ . 192 | 193 | Some ideas of what you could do: 194 | 195 | --- 196 | # Define a service my_service with 3 containers 197 | my_service: 198 | {% for idx in [1, 2, 3] %} 199 | - container{{idx}} 200 | {% endfor %} 201 | --- 202 | 203 | # Define 3 container to run on 3 different docker hosts 204 | {% for idx in [1, 2, 3] %} 205 | container{{idx}}: 206 | image: {{registry}}/my-image 207 | docker_host: https://docker-host0{{idx}} 208 | {% endfor %} 209 | 210 | To check the result of your templating you can call `dirg show my_service` which would result in the following output: 211 | 212 | container1: 213 | image: my-registry:5000/my-image 214 | docker_host: https://docker-host01 215 | 216 | container2: 217 | image: my-registry:5000/my-image 218 | docker_host: https://docker-host02 219 | 220 | container3: 221 | image: my-registry:5000/my-image 222 | docker_host: https://docker-host03 223 | 224 | Or you could define certain container or services only when run in a certain environment: 225 | 226 | # Only define this container if there is an environment variable 'dev' 227 | {% if env['dev'] %} 228 | container: 229 | image: my-registry:5000/my-image 230 | {% endif %} 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | 243 | 244 | 245 | 246 | -------------------------------------------------------------------------------- /dirg/dirg.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import print_function 3 | import docker 4 | from docker.utils import kwargs_from_env 5 | import argparse 6 | import yaml 7 | import ConfigParser 8 | import os 9 | import jinja2 10 | import service_utils 11 | import sys 12 | import _version 13 | import requests 14 | 15 | 16 | HELP = ''' 17 | Dirg reads a yaml file describing services made of docker container definitions 18 | and allows those to apply a number of command to these groups of containers. 19 | 20 | The current dir needs to have a file called dirg.cfg or you need an environment 21 | variable DIRG_CFG pointing to one. This config file contains a reference to 22 | the service description to be used. 23 | 24 | The DOCKER_HOST environment variable is used to determin to docker server. 25 | ''' 26 | 27 | DIRG_CFG_FILE = 'dirg.cfg' 28 | DIRG_CFG_ENV = 'DIRG_CFG' 29 | 30 | # removes urllib3's InsecurePlatformWarning SSL warning when in 31 | # use with older python version 32 | requests.packages.urllib3.disable_warnings() 33 | 34 | config = ConfigParser.ConfigParser() 35 | 36 | # when using SSL: 37 | # $ export DOCKER_HOST=https://docker.local:2376 38 | # $ export DOCKER_CERT_PATH=/home/user/.docker/ 39 | # $ export DOCKER_TLS_VERIFY=1 40 | 41 | if 'DOCKER_HOST' in os.environ: 42 | try: 43 | args = kwargs_from_env(assert_hostname=False) 44 | args['base_url'] = os.environ.get('DOCKER_HOST') 45 | args['version'] = '1.17' 46 | cli = docker.Client(**args) 47 | except Exception as e: 48 | print('Error connecting to docker host.') 49 | print(e) 50 | sys.exit(-1) 51 | else: 52 | cli = None 53 | 54 | container = {} 55 | services = {} 56 | 57 | 58 | def service_by_name(name): 59 | if name in services: 60 | return services[name] 61 | print('Invalid service %s' % name) 62 | sys.exit(-1) 63 | 64 | 65 | def foreach_service(args, command): 66 | # check if all services exist before calling the 67 | # first one 68 | for name in args.name: 69 | service_by_name(name) 70 | 71 | for name in args.name: 72 | command(cli, service_by_name(name)) 73 | 74 | 75 | def run_service_cmd(args): 76 | foreach_service(args, service_utils.run_service) 77 | 78 | 79 | def start_service_cmd(args): 80 | foreach_service(args, service_utils.start_service) 81 | 82 | 83 | def stop_service_cmd(args): 84 | foreach_service(args, service_utils.stop_service) 85 | 86 | 87 | def build_service_cmd(args): 88 | foreach_service(args, service_utils.build_service) 89 | 90 | 91 | def show_service_cmd(args): 92 | foreach_service(args, service_utils.show_service) 93 | 94 | 95 | def pull_service_cmd(args): 96 | foreach_service(args, service_utils.pull_service) 97 | 98 | 99 | def remove_service_cmd(args): 100 | foreach_service(args, service_utils.remove_service) 101 | 102 | 103 | def update_service_cmd(args): 104 | foreach_service(args, service_utils.update_service) 105 | 106 | 107 | def list_services_cmd(args): 108 | foreach_service(args, service_utils.list_services) 109 | 110 | 111 | def show_service_logs_cmd(args): 112 | foreach_service(args, service_utils.show_service_logs) 113 | 114 | 115 | def show_service_stats_cmd(args): 116 | foreach_service(args, service_utils.show_service_stats) 117 | 118 | 119 | def info_cmd(args): 120 | check_environment() 121 | 122 | 123 | def print_debug(args, message): 124 | if args.debug: 125 | print(message) 126 | 127 | 128 | def load_service_config(args): 129 | """loads service definition specified in config file""" 130 | try: 131 | service_file = config.get('DEFAULT', 'dirg_services', None) 132 | except ConfigParser.NoOptionError: 133 | print('Dirg config needs a key called "dirg_services".') 134 | sys.exit(-1) 135 | 136 | if not os.path.isfile(service_file): 137 | print('Can not read service config file: %s' % service_file) 138 | sys.exit(-1) 139 | 140 | print_debug(args, 'Reading services from %s' % service_file) 141 | 142 | with open(config.get('DEFAULT', 'dirg_services'), 'r') as f: 143 | service_content = f.read() 144 | service_template = jinja2.Template(service_content) 145 | service_yml = service_template.render(dict(config.items('DEFAULT'), env=os.environ)) 146 | try: 147 | services_def, container = yaml.load_all(service_yml) 148 | except ValueError as e: 149 | print('Error reading service description %s: %s' % (service_file, e)) 150 | sys.exit(-1) 151 | 152 | if services_def is not None and container is not None: 153 | global services 154 | for s in services_def: 155 | service_container = [] 156 | for container_name in services_def[s]: 157 | if container_name not in container: 158 | print('"%s" not a valid container name for service "%s"' % (container_name, s)) 159 | sys.exit(-1) 160 | service_container.append( 161 | {'name': container_name, 162 | 'conf': container[container_name]}) 163 | services[s] = {'name': s, 'container': service_container} 164 | 165 | 166 | def load_config(args): 167 | """loads config file from local dir or from DIRG_CFG_ENV""" 168 | if DIRG_CFG_ENV in os.environ and os.path.isfile(DIRG_CFG_FILE): 169 | print_debug(args, 'Reading cfg from %s then from %s' 170 | % (DIRG_CFG_FILE, os.environ[DIRG_CFG_ENV])) 171 | config.readfp(open(DIRG_CFG_FILE)) 172 | config.read([os.environ[DIRG_CFG_ENV]]) 173 | elif DIRG_CFG_ENV in os.environ and not os.path.isfile(DIRG_CFG_FILE): 174 | print_debug(args, 'Reading cfg from %s' % os.environ[DIRG_CFG_ENV]) 175 | config.read([os.environ[DIRG_CFG_ENV]]) 176 | elif os.path.isfile(DIRG_CFG_FILE): 177 | print_debug(args, 'Reading cfg from %s' % DIRG_CFG_FILE) 178 | config.readfp(open(DIRG_CFG_FILE)) 179 | else: 180 | print('dirg.cfg not found and env variable %s not set.' % DIRG_CFG_ENV) 181 | sys.exit(-1) 182 | 183 | 184 | def check_environment(): 185 | print('Version %s' % _version.__version__) 186 | print('DIRG_CFG = %s' % os.environ.get('DIRG_CFG')) 187 | print('DOCKER_HOST = %s' % os.environ.get('DOCKER_HOST')) 188 | print('DOCKER_CERT_PATH = %s' 189 | % os.environ.get('DOCKER_CERT_PATH')) 190 | print('DOCKER_TLS_VERIFY = %s' 191 | % os.environ.get('DOCKER_TLS_VERIFY')) 192 | 193 | 194 | def main(): 195 | parent_parser = argparse.ArgumentParser(add_help=False) 196 | parent_parser.add_argument( 197 | 'name', 198 | default=['all'], 199 | nargs='*', 200 | help='service name', 201 | action='store') 202 | parent_parser.add_argument( 203 | '-d', '--debug', 204 | default=False, 205 | dest='debug', 206 | help='Print debug info.', 207 | action='store_true') 208 | 209 | parser = argparse.ArgumentParser( 210 | formatter_class=argparse.RawDescriptionHelpFormatter, 211 | description=HELP, 212 | epilog='') 213 | parser.add_argument( 214 | '-d', '--debug', 215 | default=False, 216 | dest='debug', 217 | help='Print debug info.', 218 | action='store_true') 219 | 220 | subparsers = parser.add_subparsers() 221 | 222 | parser_run = subparsers.add_parser( 223 | 'run', 224 | help='Create all container and start them.', 225 | parents=[parent_parser]) 226 | parser_run.set_defaults(func=run_service_cmd) 227 | 228 | parser_info = subparsers.add_parser( 229 | 'info', help='Prints out environment info.') 230 | parser_info.set_defaults(func=info_cmd) 231 | 232 | parser_start = subparsers.add_parser( 233 | 'start', help='Start all service container.', parents=[parent_parser]) 234 | parser_start.set_defaults(func=start_service_cmd) 235 | 236 | parser_stop = subparsers.add_parser( 237 | 'stop', help='Stop all service container', parents=[parent_parser]) 238 | parser_stop.set_defaults(func=stop_service_cmd) 239 | 240 | parser_remove = subparsers.add_parser( 241 | 'rm', help='Remove all service service.', parents=[parent_parser]) 242 | parser_remove.set_defaults(func=remove_service_cmd) 243 | 244 | parser_build = subparsers.add_parser( 245 | 'build', 246 | help='Build all service container images.', 247 | parents=[parent_parser]) 248 | parser_build.set_defaults(func=build_service_cmd) 249 | 250 | parser_list = subparsers.add_parser( 251 | 'ps', 252 | help='List all services and their container status.', 253 | parents=[parent_parser]) 254 | parser_list.set_defaults(func=list_services_cmd) 255 | 256 | parser_show = subparsers.add_parser( 257 | 'show', help='Show service container config.', parents=[parent_parser]) 258 | parser_show.set_defaults(func=show_service_cmd) 259 | 260 | parser_pull = subparsers.add_parser( 261 | 'pull', help='Pull service container images.', parents=[parent_parser]) 262 | parser_pull.set_defaults(func=pull_service_cmd) 263 | 264 | parser_logs = subparsers.add_parser( 265 | 'logs', help='Show service logs.', parents=[parent_parser]) 266 | parser_logs.set_defaults(func=show_service_logs_cmd) 267 | 268 | parser_update = subparsers.add_parser( 269 | 'update', help='Update service.', parents=[parent_parser]) 270 | parser_update.set_defaults(func=update_service_cmd) 271 | 272 | parser_stats = subparsers.add_parser( 273 | 'stats', help='Show service stats.', parents=[parent_parser]) 274 | parser_stats.set_defaults(func=show_service_stats_cmd) 275 | 276 | args = parser.parse_args() 277 | 278 | if args.debug: 279 | check_environment() 280 | 281 | load_config(args) 282 | load_service_config(args) 283 | 284 | args.func(args) 285 | 286 | if __name__ == '__main__': 287 | try: 288 | main() 289 | except KeyboardInterrupt: 290 | print('Interrupted') 291 | try: 292 | sys.exit(0) 293 | except SystemExit: 294 | os._exit(0) 295 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Dirg is an orchestration tool for docker. It reads a yaml file 2 | describing services made of docker container definitions and allows to 3 | apply a number of commands to these groups of containers. 4 | 5 | Why another orchestration tool? 6 | =============================== 7 | 8 | - Support for multi-host docker setups 9 | - Support for templating in service description 10 | 11 | Installation 12 | ============ 13 | 14 | Make sure you have 15 | 16 | - Python 2.7, 3.x is not supported yet 17 | - Python setuptools are installed 18 | 19 | You can install Dirg from the Python Package Index with 20 | 21 | :: 22 | 23 | $ pip install dirg 24 | 25 | Or you can clone the repository, then 26 | 27 | :: 28 | 29 | $ python setup.py install 30 | 31 | To check if the installation was successful, execute 32 | 33 | :: 34 | 35 | $ dirg info 36 | 37 | Setting the docker host 38 | ======================= 39 | 40 | You can either set the ``DOCKER_HOST`` environment variable or set a 41 | specific docker host per container in the service description. 42 | 43 | Using a local docker host: 44 | 45 | :: 46 | 47 | $ export DOCKER_HOST=unix:///var/run/docker.sock 48 | 49 | Using a remote docker host via HTTP: 50 | 51 | :: 52 | 53 | $ export DOCKER_HOST=tcp://remote.host:2375 54 | 55 | Using a remote docker host via HTTPS: 56 | 57 | :: 58 | 59 | $ export DOCKER_HOST=https://remote.host:2375 60 | $ export DOCKER_CERT_PATH=/path/to/client/cert.pem 61 | $ export DOCKER_TLS_VERIFY=1 62 | 63 | Dirg Commands 64 | ============= 65 | 66 | Most commands have the form: 67 | 68 | :: 69 | 70 | $ dirg COMMAND SERVICE_NAME 71 | 72 | If ``SERVICE_NAME`` is missing, ``all`` is the default service name. 73 | 74 | ``COMMAND`` can be 75 | 76 | :: 77 | 78 | run Create all container and start them. 79 | info Prints out environment info. 80 | start Start all service container. 81 | stop Stop all service container 82 | rm Remove all service service. 83 | build Build all service container images. 84 | ps List all services and their container status. 85 | show Show service container config. 86 | pull Pull service container images. 87 | logs Show service logs. 88 | update Update service. 89 | stats Show service stats. 90 | 91 | Adding ``-d`` will print out additional debug information. This is 92 | valuable when you want to make sure Dirg is finding the right service 93 | configuration. ``info`` shows all environment variables needed for a SSL 94 | connection. 95 | 96 | Service Configuration 97 | ===================== 98 | 99 | To configure Dirg you need a configuration file called ``dirg.cfg`` and 100 | a yaml description of your services. When you execute Dirg, it looks for 101 | file named ``dirg.cfg`` in the current directory. You can set an 102 | environment variable ``DIRG_CFG`` to point to your ``dirg.cfg`` file. 103 | 104 | A minimal ``dirg.cfg`` looks like this: 105 | 106 | :: 107 | 108 | [DEFAULT] 109 | dirg_services = /path/to/dirg-services.yml 110 | 111 | It holds a reference to the file describing your docker based services. 112 | In addition, you may define your own properties and values which you can 113 | then use in your service description. E.g. you could add you docker 114 | image registry URL to ``dirg.cfg`` and then reference it in your 115 | container definitions. 116 | 117 | A ``dirg-services.yml`` looks like this: 118 | 119 | :: 120 | 121 | --- 122 | service1: 123 | - container1 124 | - container2 125 | service2: 126 | - container3 127 | - container4 128 | all: 129 | - container1 130 | - container2 131 | - container3 132 | - container4 133 | - container5 134 | --- 135 | 136 | container1: 137 | image: imagename 138 | volumes: volumes 139 | volume_bindings: volume bindings 140 | 141 | container2: 142 | image: imagename 143 | volumes: volumes 144 | volume_bindings: volume bindings 145 | 146 | ... 147 | 148 | This yaml file contains 2 sub-documents (separated by ---). The first 149 | document describes all existing services. The second one describes the 150 | containers used by the services above. 151 | 152 | If you name a service ``all`` it will be the default service used by 153 | Dirg when you don't name a service upon calling Dirg commands. 154 | 155 | Container Configuration 156 | ======================= 157 | 158 | Dirg supports the following container properties (more will be added as 159 | needed): 160 | 161 | +--------------------+--------------------------------------------+ 162 | | Property | Description | 163 | +====================+============================================+ 164 | | image | Image to use | 165 | +--------------------+--------------------------------------------+ 166 | | docker\_host | Docker host to run this container on | 167 | +--------------------+--------------------------------------------+ 168 | | net | Network config | 169 | +--------------------+--------------------------------------------+ 170 | | env | Environment variables | 171 | +--------------------+--------------------------------------------+ 172 | | volumes | Volumes for the container | 173 | +--------------------+--------------------------------------------+ 174 | | volume\_bindings | Mapping of container volumes | 175 | +--------------------+--------------------------------------------+ 176 | | ports | Ports opened by the container | 177 | +--------------------+--------------------------------------------+ 178 | | port\_bindings | Mapping to host ports | 179 | +--------------------+--------------------------------------------+ 180 | | links | Docker links to other container | 181 | +--------------------+--------------------------------------------+ 182 | | command | Command to execute when container starts | 183 | +--------------------+--------------------------------------------+ 184 | 185 | This is a commented sample container definition using every 186 | configuration possible: 187 | 188 | :: 189 | 190 | # You can use comments in dirg-services.yml, block comments start with {# and end with #} 191 | # my_container will be set as container name on the docker host. 192 | my_container: 193 | 194 | # Stay DRY by using properties defined in dirg.cfg 195 | # Variables are enclosed in {{property_name}} 196 | image: {{registry}}/my_image_name 197 | 198 | # Run each command concerning this container on the following docker host 199 | docker_host: https://my.docker.host:2376 200 | 201 | # Use host network instead of bridge, which is default 202 | net: host 203 | 204 | # Define environment variables 205 | env: 206 | ENV1: value1 207 | ENV2: value2 208 | 209 | # Anywhere in dirg-services.yml you can also reference properties defined 210 | # as environment variables in the shell Dirg is running in. 211 | # This fills the docker environment variable with the contents of an 212 | # environment variable defined in the shell. If the shell environment 213 | # variable is not available, 'secret' is used as a default 214 | env: 215 | MY_PASSWORD: {{env['PASSWORD'] or 'secret'}} 216 | 217 | # Define volumes for the container 218 | volumes: [/logs, /data] 219 | 220 | # Then map them to host directories, specified in a property read from dirg.cfg 221 | volume_bindings: 222 | {{data_dir}}: {bind: /data} 223 | {{logs_dir}}: {bind: /logs} 224 | 225 | # Define ports exposed by the container 226 | ports: [80, 90] 227 | 228 | # Then map them to host ports 229 | port_bindings: {80: 8080, 90: 9090} 230 | 231 | # Ugly workaround to define a UDP port. This will be improved in a later version: 232 | ports: 233 | - !!python/tuple [8125, udp] 234 | port_bindings: {8125: 8125} 235 | 236 | # Link containers 237 | links: {db: db} 238 | 239 | # Execute command in container when it starts 240 | command: '/app/run_benchmark -p 80 -c 90' 241 | 242 | Advanced Templating 243 | ------------------- 244 | 245 | Since the service description is a Jinja2 template you may do everything 246 | you can do in Jinja2. Take a look at the Jinja2 template designer 247 | documentation at http://jinja.pocoo.org/docs/dev/templates/ . 248 | 249 | Some ideas of what you could do: 250 | 251 | :: 252 | 253 | --- 254 | # Define a service my_service with 3 containers 255 | my_service: 256 | {% for idx in [1, 2, 3] %} 257 | - container{{idx}} 258 | {% endfor %} 259 | --- 260 | 261 | # Define 3 container to run on 3 different docker hosts 262 | {% for idx in [1, 2, 3] %} 263 | container{{idx}}: 264 | image: {{registry}}/my-image 265 | docker_host: https://docker-host0{{idx}} 266 | {% endfor %} 267 | 268 | To check the result of your templating you can call 269 | ``dirg show my_service`` which would result in the following output: 270 | 271 | :: 272 | 273 | container1: 274 | image: my-registry:5000/my-image 275 | docker_host: https://docker-host01 276 | 277 | container2: 278 | image: my-registry:5000/my-image 279 | docker_host: https://docker-host02 280 | 281 | container3: 282 | image: my-registry:5000/my-image 283 | docker_host: https://docker-host03 284 | 285 | Or you could define certain container or services only when run in a 286 | certain environment: 287 | 288 | :: 289 | 290 | # Only define this container if there is an environment variable 'dev' 291 | {% if env['dev'] %} 292 | container: 293 | image: my-registry:5000/my-image 294 | {% endif %} 295 | --------------------------------------------------------------------------------