├── .gitignore ├── LICENSE ├── README.md ├── bin └── maestro ├── examples ├── meteor-dev │ ├── app │ │ ├── .meteor │ │ │ ├── .gitignore │ │ │ ├── packages │ │ │ └── release │ │ ├── app.css │ │ ├── app.html │ │ └── app.js │ └── maestro.yml ├── mongo-replicaset.yml ├── new-format.yml ├── nodejs-mongodb │ └── maestro.yml └── salt-stack │ └── maestro.yml ├── maestro ├── __init__.py ├── cli.py ├── container.py ├── environment.py ├── exceptions.py ├── py_backend.py ├── service.py ├── template.py └── utils.py ├── requirements.txt ├── setup.py └── tests ├── fixtures ├── count.yml ├── default.yml ├── dockerfile.yml ├── maestro.yml ├── require-cycle.yml ├── require.yml ├── startstop.yml └── template │ ├── invalid_base.yml │ ├── invalid_buildspec.yml │ ├── invalid_dockerfile.yml │ ├── mount.yml │ ├── no_base.yml │ ├── valid_base.yml │ ├── valid_base_tag.yml │ ├── valid_build_url.yml │ └── valid_dockerfile.yml ├── test_container.py ├── test_maestro.py ├── test_py_backend.py ├── test_service.py └── test_template.py /.gitignore: -------------------------------------------------------------------------------- 1 | environment.yml 2 | dockermix.log 3 | maestro.log 4 | src/ 5 | *.py[cod] 6 | 7 | # C extensions 8 | *.so 9 | 10 | # Packages 11 | *.egg 12 | *.egg-info 13 | dist 14 | build 15 | eggs 16 | parts 17 | var 18 | sdist 19 | develop-eggs 20 | .installed.cfg 21 | lib 22 | lib64 23 | 24 | # Installer logs 25 | pip-log.txt 26 | 27 | # Unit test / coverage reports 28 | .coverage 29 | .tox 30 | nosetests.xml 31 | 32 | # Translations 33 | *.mo 34 | 35 | # Mr Developer 36 | .mr.developer.cfg 37 | .project 38 | .pydevproject 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2013 Kimbro Staken 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Maestro 2 | ============ 3 | 4 | Maestro provides the ability to easily launch, orchestrate and manage mulitple Docker containers as single unit. Container sets are defined in a simple YAML format that allows you to define how the containers should be created and to specify relationships between containers. The intention is to make easy to create and use complex multi-node container envionments for testing and development. 5 | 6 | This is what it currently looks like to use maestro and deploy a multi-tier node.js/mongodb application. All that's required is a maestro.yml in the root of the repository. 7 | 8 | ``` 9 | $ git clone https://github.com/kstaken/express-todo-example.git 10 | $ cd express-todo-example && maestro build 11 | Building template mongodb 12 | Building template nodejs 13 | Launching instance of template mongodb named mongodb 14 | Starting container mongodb - 144af5ca089d 15 | Starting nodejs: waiting for service mongodb on ip 172.16.0.61 and port 27017 16 | Found service mongodb on ip 172.16.0.61 and port 27017 17 | Launching instance of template nodejs named nodejs 18 | Starting container nodejs - 52be61a3242c 19 | Launched. 20 | 21 | $ maestro ps 22 | ID NODE COMMAND STATUS PORTS 23 | 144af5ca089d mongodb /usr/bin/mongod --config /etc/mongodb.conf Running 24 | 52be61a3242c nodejs /usr/bin/node /var/www/app.js Running 49184->80 25 | 26 | $ maestro stop 27 | Stopping container mongodb - 144af5ca089d 28 | Stopping container nodejs - 52be61a3242c 29 | Stopped. 30 | 31 | $ maestro start 32 | Starting container mongodb - 144af5ca089d 33 | Starting nodejs: waiting for service mongodb on ip 172.16.0.63 and port 27017 34 | Found service mongodb on ip 172.16.0.63 and port 27017 35 | Starting container nodejs - 52be61a3242c 36 | Started. 37 | ``` 38 | 39 | In this example the app would be accessible on http://localhost:49184/. 40 | 41 | Status 42 | ====== 43 | 44 | Early development. It can be useful for testing and development but the feature set and configuration format are changing rapidly. 45 | 46 | Note: this project used to be called DockerMix. 47 | 48 | Features 49 | ======== 50 | 51 | - Build/start/stop/destroy multi-container docker environments via simple commands 52 | - Specify dependencies between containers so they start in order and wait for services to become available 53 | - Automatically configure dependent containers to know where to locate services from other containers in the same environment 54 | - Easily launch and manage multiple copies of the same environment 55 | - Declarative YAML format to specify container configurations for the environment 56 | - Easily launch multiple instances of the same container for testing cluster operations 57 | - Share data between the host machine and containers running in the environment 58 | - ... Much more to come 59 | 60 | Dependencies 61 | ============= 62 | 63 | - Docker: https://github.com/dotcloud/docker 64 | - docker-py: https://github.com/dotcloud/docker-py 65 | - Python pip package manager 66 | 67 | Installation 68 | ============ 69 | 70 | Install Docker as described here: http://www.docker.io/gettingstarted/ 71 | 72 | Note: Docker 0.5.2 changed from listening on a network socket to listening on a unix socket due to a security issue. At this time, to use Maestro with Docker 0.5.2 you must re-enable the TCP socket in Docker. `/usr/bin/docker -d -H=tcp://127.0.0.1:4243` This is safe if you're running Docker inside a VM dedicated to that purpose but not if you're running Docker directly on your physical computer. This will be fixed in the future. 73 | 74 | Then: 75 | ``` 76 | sudo apt-get install -y python-pip 77 | git clone https://github.com/toscanini/maestro.git 78 | cd maestro 79 | sudo pip install -r requirements.txt 80 | sudo python setup.py install 81 | docker pull ubuntu 82 | ``` 83 | 84 | Configuration File Format 85 | ========================= 86 | 87 | **Note:** This format is changing heavily. 88 | 89 | The configuration file defines an environment that is made up of multiple templates that can be used to generate containers. The templates can have relationships defined between them to specify start order and Maestro will handle starting instances of the templates in containers in the correct order and then providing environment configuration to the containers. 90 | 91 | This example will setup two templates, one for nodejs and one for mongodb. The nodejs template depends on mongodb so that when you start the environment no nodejs containers will be started until a mongdb container is fully up and running. 92 | 93 | In this instance the templates are built from repositories stored on Github but there are various ways to set these up. 94 | 95 | ``` 96 | templates: 97 | nodejs: 98 | config: 99 | command: /usr/bin/node /var/www/app.js 100 | ports: 101 | - '80' 102 | environment: 103 | - PORT=80 104 | buildspec: 105 | url: github.com/toscanini/docker-nodejs 106 | require: 107 | mongodb: 108 | port: '27017' 109 | mongodb: 110 | config: 111 | command: /usr/bin/mongod --config /etc/mongodb.conf 112 | buildspec: 113 | url: github.com/toscanini/docker-mongodb 114 | ``` 115 | 116 | To build and launch an environment you just place this config in a file named `maestro.yml` then run `maestro build`. It will take a few seconds to start as it waits for MongoDB to initialize. Currently the environment state lives in the current directory but that will have more flexibility in the future. 117 | 118 | Templates also define a basic docker configuration so that you can pre-define the parameters used on docker run. 119 | 120 | `base_image` is the Docker container to use to run the command. It must already exist on the system and won't be pulled automatically. 121 | 122 | `base_image` and `command` are the only required options if no `buildspec` is provided. If `buildspec` is provided `base_image` can be omitted 123 | 124 | `require` is used to specify dependencies between services. The start order will be adjusted and any container that requires a port on a another container will wait for that port to become available before starting. 125 | 126 | `mount` allows you to define bind mounts between a directory on the host and a directory in a container. This allows you to share files between the host and the container. 127 | Note: if you define a bind mount on a template then every instance of that template will mount the same host directory. 128 | 129 | This example yaml file shows how some of the docker parameters look: 130 | 131 | ``` 132 | templates: 133 | test_server_1: 134 | base_image: ubuntu 135 | mount: 136 | /host/path: /container/path 137 | config: 138 | ports: 139 | - '8080' 140 | command: '/bin/bash -c "apt-get install netcat ; nc -l 8080 -k"' 141 | hostname: test_server_1 142 | user: root 143 | detach: true 144 | stdin_open: true 145 | tty: true 146 | mem_limit: 2560000 147 | environment: 148 | - ENV_VAR=testing 149 | dns: 150 | - 8.8.8.8 151 | - 8.8.4.4 152 | volumes: 153 | /var/testing: {} 154 | ``` 155 | 156 | **Note:** *Command is required by the Docker Python api and having to specify it here can cause problems with images that pre-define entrypoints and commands.* 157 | 158 | Command Line Tools 159 | === 160 | 161 | The command line tool is called `maestro` and initial enironments are defined in `maestro.yml`. If there is a `maestro.yml` in the current directory it will be automatically used otherwise the `-f` option can be used to specify the location of the file. 162 | 163 | The environment state will be saved to a file named `environment.yml` and commands that manipulate existing environments will look for an `environment.yml` in the current directory or it can be specified by the `-e` option. 164 | 165 | If you want to create a named environment you can use `-n` to set the name and it will be made a global environment that lives either under ~/.maestro or /var/lib/maestro depending on your setup. 166 | 167 | `maestro build` 168 | 169 | Setup a new environment using a `maestro.yml` specification. 170 | 171 | `maestro start [node_name]` 172 | 173 | Start an existing environment that had been previously stopped and saved in `environment.yml`. If `node_name` is provided just that node will be stopped. 174 | 175 | `maestro stop [node_name]` 176 | 177 | Stop all containers in an environment and save the state to `environment.yml` If `node_name` is provided just that node will be stopped. 178 | 179 | `maestro run template [commandline]` 180 | 181 | Run a new instance of the template in the environment. *Limited functionality on this currently* 182 | 183 | `maestro destroy` 184 | 185 | Destroy all containers defined in an environment. Once destroyed the containers can not be recoved. 186 | 187 | `maestro ps` 188 | 189 | Show the status of the containers in an environment. 190 | 191 | Roadmap 192 | ==== 193 | 194 | - Bootstrap installer 195 | - Add the ability to share configuration data between containers. Limited capabilities exist for this currently. 196 | - ~~Explicitly specify startup order and dependencies~~ 197 | - ~~More powerful Docker Builder support~~ ~~(currently docker-py reimplements Docker Builder and it out of sync with the server implementation)~~ 198 | - ~~Add automatic pulling of base images~~ 199 | - Make it easier to run the full test suite 200 | - Add the ability to depend on external services 201 | - ~~Add the ability to have named global environments as well as environments stored in the local directory~~ 202 | - More robust support for running and adding containers to an existing environment 203 | - Direct build and instantiation of an environment from a git repo 204 | - ... 205 | -------------------------------------------------------------------------------- /bin/maestro: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | from maestro import cli 5 | 6 | if __name__ == '__main__': 7 | cmd = cli.MaestroCli() 8 | sys.exit(cmd.main()) 9 | -------------------------------------------------------------------------------- /examples/meteor-dev/app/.meteor/.gitignore: -------------------------------------------------------------------------------- 1 | local 2 | -------------------------------------------------------------------------------- /examples/meteor-dev/app/.meteor/packages: -------------------------------------------------------------------------------- 1 | # Meteor packages used by this project, one per line. 2 | # 3 | # 'meteor add' and 'meteor remove' will edit this file for you, 4 | # but you can also edit it by hand. 5 | 6 | autopublish 7 | insecure 8 | preserve-inputs 9 | -------------------------------------------------------------------------------- /examples/meteor-dev/app/.meteor/release: -------------------------------------------------------------------------------- 1 | 0.6.4.1 2 | -------------------------------------------------------------------------------- /examples/meteor-dev/app/app.css: -------------------------------------------------------------------------------- 1 | /* CSS declarations go here */ 2 | -------------------------------------------------------------------------------- /examples/meteor-dev/app/app.html: -------------------------------------------------------------------------------- 1 | 2 | app 3 | 4 | 5 | 6 | {{> hello}} 7 | 8 | 9 | 14 | -------------------------------------------------------------------------------- /examples/meteor-dev/app/app.js: -------------------------------------------------------------------------------- 1 | if (Meteor.isClient) { 2 | Template.hello.greeting = function () { 3 | return "Welcome to app. Does this work"; 4 | }; 5 | 6 | Template.hello.events({ 7 | 'click input' : function () { 8 | // template data, if any, is available in 'this' 9 | if (typeof console !== 'undefined') 10 | console.log("You pressed the button"); 11 | } 12 | }); 13 | } 14 | 15 | if (Meteor.isServer) { 16 | Meteor.startup(function () { 17 | // code to run on server at startup 18 | }); 19 | } 20 | -------------------------------------------------------------------------------- /examples/meteor-dev/maestro.yml: -------------------------------------------------------------------------------- 1 | templates: 2 | meteor: 3 | config: 4 | command: '/bin/bash -c "cd /var/www && MONGO_URL=mongodb://$MONGODB:27017/your_db meteor"' 5 | ports: 6 | - '3000' 7 | mounts: 8 | /vagrant/maestro/examples/meteor-dev/app: /var/www 9 | buildspec: 10 | dockerfile: | 11 | FROM ubuntu:12.10 12 | RUN apt-get update && apt-get install -y curl 13 | RUN curl http://install.meteor.com | /bin/sh 14 | RUN mkdir /var/www 15 | require: 16 | mongodb: 17 | port: '27017' 18 | mongodb: 19 | config: 20 | command: /usr/bin/mongod --config /etc/mongodb.conf 21 | buildspec: 22 | url: github.com/toscanini/docker-mongodb 23 | 24 | 25 | -------------------------------------------------------------------------------- /examples/mongo-replicaset.yml: -------------------------------------------------------------------------------- 1 | templates: 2 | mongodb: 3 | base_image: kstaken/mongodb 4 | count: 3 5 | config: 6 | command: /usr/bin/mongod --config /etc/mongodb.conf 7 | detach: true 8 | 9 | setup: 10 | base_image: kstaken/mongodb 11 | config: 12 | command: /bin/bash 13 | detach: false 14 | require: 15 | mongodb: 16 | count: 3 17 | port: 27017 18 | -------------------------------------------------------------------------------- /examples/new-format.yml: -------------------------------------------------------------------------------- 1 | --environment: 2 | -- name: Node.js / Mongo Deployment Platform 3 | -- version: 0.1 4 | -- What needs to be configured here at instantiation: 5 | -- External port 6 | -- Source for the application state 7 | -- Possibly commands to run? 8 | -- Name for the environment comes when the environment is created 9 | environment: 10 | description: Node.js / MongoDB deployment platform 11 | 12 | services: 13 | mongohq: 14 | description: Encapsulation of the mongohq external service 15 | host: db.mongohq.com 16 | port: 27017 17 | config: 18 | user: {{ mongohq_user }} -- Set as MONGOHQ_USER in env of any dependenct containers 19 | password: {{ mongohq_password }} -- Set as MONGOHQ_PASSWORD 20 | 21 | mongodb: -- only useful to services 22 | description: MongoDB Database service 23 | version: 0.1 24 | exposes: 27017 25 | templates: 26 | mongodb: -- only useful to templates 27 | base_image: kstaken/mongodb 28 | config: -- only useful to containers 29 | command: /usr/bin/mongod --config /etc/mongodb.conf 30 | detach: true 31 | 32 | 33 | nodejs: 34 | description: Deployment service for Node.js 35 | templates: 36 | nodejs: 37 | base_image: kstaken/nodejs 38 | config: 39 | command: /usr/bin/node /var/www/app.js 40 | detach: true 41 | ports: 42 | - '{{ public_port }}:80' 43 | environment: 44 | - PORT=80 45 | require: 46 | tosca_file: -- Platform provided service 47 | mount: /var/www 48 | 49 | require: 50 | mongodb: 51 | port: '27017' 52 | 53 | defaults: 54 | public_port: '8100' 55 | tosca_file.mount: /tmp/ 56 | mongohq_user: default 57 | mongohq_password: default 58 | 59 | -- --nodejs:mount=/vagrant/nodejs -------------------------------------------------------------------------------- /examples/nodejs-mongodb/maestro.yml: -------------------------------------------------------------------------------- 1 | --environment: 2 | -- name: Node.js / Mongo Deployment Platform 3 | -- version: 0.1 4 | templates: 5 | nodejs: 6 | config: 7 | command: /usr/bin/node /var/www/app.js 8 | ports: 9 | - '80' 10 | environment: 11 | - PORT=80 12 | buildspec: 13 | url: github.com/toscanini/docker-nodejs 14 | require: 15 | mongodb: 16 | port: '27017' 17 | mongodb: 18 | config: 19 | command: /usr/bin/mongod --config /etc/mongodb.conf 20 | buildspec: 21 | url: github.com/toscanini/docker-mongodb 22 | 23 | 24 | -------------------------------------------------------------------------------- /examples/salt-stack/maestro.yml: -------------------------------------------------------------------------------- 1 | templates: 2 | salt_master: 3 | config: 4 | command: /usr/bin/supervisord -n -c /etc/supervisor/supervisord.conf 5 | buildspec: 6 | url: github.com/toscanini/docker-salt-master 7 | salt_minion: 8 | count: 10 9 | config: 10 | command: /usr/bin/supervisord -n -c /etc/supervisor/supervisord.conf 11 | buildspec: 12 | url: github.com/toscanini/docker-salt-minion 13 | require: 14 | salt_master: 15 | port: '4505' 16 | -------------------------------------------------------------------------------- /maestro/__init__.py: -------------------------------------------------------------------------------- 1 | __all__ = ["container", "service", "utils", "cli"] 2 | 3 | 4 | LOCAL_ENV=".maestro" 5 | GLOBAL_ENV="/var/lib/maestro" 6 | 7 | # Maintain a list of environments on disk 8 | # By default an environemnt is created in .maestro unless -g is specified to make it global. 9 | # Global enviroments are stored in /var/lib/maestro. Permission setting will come into play for this. 10 | # The environment directory contains: 11 | # environment.yml capturing the state of the running system 12 | # settings.yml capturing the user configuration settings 13 | # maestro.yml ?? The original environment description used to create the environment 14 | 15 | # Initialize a new environment 16 | def init_environment(name, description="maestro.yml", system=False): 17 | # Verify the environment doesn't already exist 18 | # Check for both local and system environments that may conflict 19 | 20 | if (system): 21 | # Create a system wide environment 22 | pass 23 | else: 24 | # We're just creating an environment that lives relative to the local directory 25 | pass 26 | 27 | # retrieve environment 28 | def get_environment(name): 29 | pass 30 | 31 | # list environments 32 | def list_environments(): 33 | # Include the local environment if there is one 34 | # Include a list of the system environments 35 | pass 36 | 37 | def destroy_environment(name): 38 | pass 39 | 40 | -------------------------------------------------------------------------------- /maestro/cli.py: -------------------------------------------------------------------------------- 1 | import sys, os 2 | import cmdln 3 | from . import service 4 | 5 | class MaestroCli(cmdln.Cmdln): 6 | """Usage: 7 | maestro SUBCOMMAND [ARGS...] 8 | maestro help SUBCOMMAND 9 | 10 | Maestro provides a command to manage multiple Docker containers 11 | from a single configuration. 12 | 13 | ${command_list} 14 | ${help_list} 15 | """ 16 | name = "maestro" 17 | 18 | def __init__(self, *args, **kwargs): 19 | cmdln.Cmdln.__init__(self, *args, **kwargs) 20 | cmdln.Cmdln.do_help.aliases.append("h") 21 | 22 | @cmdln.option("-f", "--maestro_file", 23 | help='path to the maestro file to use') 24 | @cmdln.option("-e", "--environment_file", 25 | help='path to the environment file to use to save the state of running containers') 26 | @cmdln.option("-n", "--name", 27 | help='Create a global named environment using the provided name') 28 | def do_build(self, subcmd, opts, *args): 29 | """Setup and start a set of Docker containers. 30 | 31 | usage: 32 | build 33 | 34 | ${cmd_option_list} 35 | """ 36 | config = opts.maestro_file 37 | if not config: 38 | config = os.path.join(os.getcwd(), 'maestro.yml') 39 | 40 | if not config.startswith('/'): 41 | config = os.path.join(os.getcwd(), config) 42 | 43 | if not os.path.exists(config): 44 | sys.stderr.write("No maestro configuration found {0}\n".format(config)) 45 | exit(1) 46 | 47 | containers = service.Service(config) 48 | containers.build() 49 | 50 | environment = opts.environment_file 51 | name = opts.name 52 | if name: 53 | environment = self._create_global_environment(name) 54 | else: 55 | environment = self._create_local_environment(opts) 56 | 57 | containers.save(environment) 58 | 59 | print "Launched." 60 | 61 | 62 | @cmdln.option("-e", "--environment_file", 63 | help='path to the environment file to use to save the state of running containers') 64 | @cmdln.option("-n", "--name", 65 | help='Create a global named environment using the provided name') 66 | def do_start(self, subcmd, opts, *args): 67 | """Start a set of Docker containers that had been previously stopped. Container state is defined in an environment file. 68 | 69 | usage: 70 | start [container_name] 71 | 72 | ${cmd_option_list} 73 | """ 74 | container = None 75 | if (len(args) > 0): 76 | container = args[0] 77 | 78 | environment = self._verify_environment(opts) 79 | 80 | containers = service.Service(environment=environment) 81 | if containers.start(container): 82 | containers.save(environment) 83 | print "Started." 84 | 85 | @cmdln.option("-e", "--environment_file", 86 | help='path to the environment file to use to save the state of running containers') 87 | @cmdln.option("-n", "--name", 88 | help='Create a global named environment using the provided name') 89 | def do_stop(self, subcmd, opts, *args): 90 | """Stop a set of Docker containers as defined in an environment file. 91 | 92 | usage: 93 | stop [container_name] 94 | 95 | ${cmd_option_list} 96 | """ 97 | container = None 98 | if (len(args) > 0): 99 | container = args[0] 100 | 101 | environment = self._verify_environment(opts) 102 | 103 | containers = service.Service(environment=environment) 104 | if containers.stop(container): 105 | containers.save(environment) 106 | print "Stopped." 107 | 108 | @cmdln.option("-e", "--environment_file", 109 | help='path to the environment file to use to save the state of running containers') 110 | @cmdln.option("-n", "--name", 111 | help='Create a global named environment using the provided name') 112 | def do_restart(self, subcmd, opts, *args): 113 | """Restart a set of containers as defined in an environment file. 114 | 115 | usage: 116 | restart [container_name] 117 | 118 | ${cmd_option_list} 119 | """ 120 | self.do_stop('stop', opts, args) 121 | self.do_start('start', opts, args) 122 | 123 | @cmdln.option("-e", "--environment_file", 124 | help='path to the environment file to use to save the state of running containers') 125 | @cmdln.option("-n", "--name", 126 | help='Create a global named environment using the provided name') 127 | def do_destroy(self, subcmd, opts, *args): 128 | """Stop and destroy a set of Docker containers as defined in an environment file. 129 | 130 | usage: 131 | destroy 132 | 133 | ${cmd_option_list} 134 | """ 135 | environment = self._verify_environment(opts) 136 | 137 | containers = service.Service(environment=environment) 138 | if containers.destroy(): 139 | containers.save(environment) 140 | print "Destroyed." 141 | 142 | @cmdln.option("-e", "--environment_file", 143 | help='path to the environment file to use to save the state of running containers') 144 | @cmdln.option("-n", "--name", 145 | help='Create a global named environment using the provided name') 146 | @cmdln.option("-a", "--attach", action="store_true", 147 | help='Attach to the running container to view output') 148 | @cmdln.option("-d", "--dont_add", action="store_true", 149 | help='Just run the command and exit. Don\'t add the container to the environment') 150 | def do_run(self, subcmd, opts, *args): 151 | """Start a set of Docker containers that had been previously stopped. Container state is defined in an environment file. 152 | 153 | usage: 154 | run template_name [commandline] 155 | 156 | ${cmd_option_list} 157 | """ 158 | container = None 159 | if (len(args) == 0): 160 | sys.stderr.write("Error: Container name must be provided\n") 161 | exit(1) 162 | 163 | environment = self._verify_environment(opts) 164 | 165 | template = args[0] 166 | commandline = args[1:] 167 | print " ".join(commandline) 168 | containers = service.Service(environment=environment) 169 | containers.run(template, commandline, attach=opts.attach, dont_add=opts.dont_add) 170 | containers.save(environment) 171 | 172 | if opts.dont_add: 173 | print "Execution of " + template + " complete." 174 | else: 175 | print "Adding a new instance of " + template + "." 176 | 177 | @cmdln.option("-e", "--environment_file", 178 | help='path to the environment file to use to save the state of running containers') 179 | @cmdln.option("-n", "--name", 180 | help='Create a global named environment using the provided name') 181 | def do_ps(self, subcmd, opts, *args): 182 | """Show the status of a set of containers as defined in an environment file. 183 | 184 | usage: 185 | ps 186 | 187 | ${cmd_option_list} 188 | """ 189 | environment = self._verify_environment(opts) 190 | 191 | containers = service.Service(environment=environment) 192 | print containers.ps() 193 | 194 | def _verify_global_environment(self, name): 195 | """ 196 | Setup the global environment. 197 | """ 198 | # Default to /var/lib/maestro and check there first 199 | path = '/var/lib/maestro' 200 | if not os.path.exists(path) or not os.access(path, os.W_OK): 201 | env_path = os.path.join(path, name) 202 | # See if the environment exists in /var/lib maestro 203 | if not os.path.exists(env_path): 204 | # If the environment doesn't exist or is not accessible then we check ~/.maestro instead 205 | path = os.path.expanduser(os.path.join('~', '.maestro')) 206 | if not os.path.exists(path): 207 | sys.stderr.write("Global named environments directory does not exist {0}\n".format(path)) 208 | exit(1) 209 | 210 | env_path = os.path.join(path, name) 211 | if not os.path.exists(env_path): 212 | sys.stderr.write("Environment named {0} does not exist\n".format(env_path)) 213 | exit(1) 214 | 215 | if not os.access(env_path, os.W_OK): 216 | sys.stderr.write("Environment named {0} is not writable\n".format(env_path)) 217 | exit(1) 218 | 219 | return os.path.join(env_path, 'environment.yml') 220 | 221 | def _create_global_environment(self, name): 222 | """ 223 | Setup the global environment. 224 | """ 225 | # Default to /var/lib/maestro 226 | # It has to exist and be writable, otherwise we'll just use a directory relative to ~ 227 | path = '/var/lib/maestro' 228 | if not os.path.exists(path) or not os.access(path, os.W_OK): 229 | # If /var/lib/maestro doesn't exist or is not accessible then we use ~/.maestro instead 230 | path = os.path.expanduser(os.path.join('~', '.maestro')) 231 | if not os.path.exists(path): 232 | print "Creating ~/.maestro to hold named environments" 233 | os.makedirs(path) 234 | 235 | # The environment will live in a directory under path 236 | env_path = os.path.join(path, name) 237 | if not os.path.exists(env_path): 238 | print "Initializing ~/.maestro/" + name 239 | os.makedirs(env_path) 240 | return os.path.join(env_path, 'environment.yml') 241 | 242 | def _verify_environment(self, opts): 243 | """ 244 | Verify that the provided environment file exists. 245 | """ 246 | if opts.name: 247 | environment = self._verify_global_environment(opts.name) 248 | else: 249 | environment = self._create_local_environment(opts) 250 | 251 | if not os.path.exists(environment): 252 | sys.stderr.write("Could not locate the environments file {0}\n".format(environment)) 253 | exit(1) 254 | 255 | if not os.access(environment, os.W_OK): 256 | sys.stderr.write("Environment file {0} is not writable\n".format(environment)) 257 | exit(1) 258 | 259 | return environment 260 | 261 | def _create_local_environment(self, opts): 262 | environment = opts.environment_file 263 | if not environment: 264 | base = os.path.join(os.getcwd(), '.maestro') 265 | environment = os.path.join(base, 'environment.yml') 266 | if not os.path.exists(base): 267 | print "Initializing " + base 268 | os.makedirs(base) 269 | 270 | return environment -------------------------------------------------------------------------------- /maestro/container.py: -------------------------------------------------------------------------------- 1 | import os, sys 2 | from exceptions import ContainerError 3 | import utils, StringIO, logging 4 | import py_backend 5 | 6 | class Container: 7 | def __init__(self, name, state, config, mounts=None): 8 | self.log = logging.getLogger('maestro') 9 | 10 | self.state = state 11 | self.config = config 12 | self.name = name 13 | self.mounts = mounts 14 | 15 | if 'hostname' not in self.config: 16 | self.config['hostname'] = name 17 | 18 | #if 'command' not in self.config: 19 | # self.log.error("Error: No command specified for container " + name + "\n") 20 | # raise ContainerError('No command specified in configuration') 21 | 22 | self.backend = py_backend.PyBackend() 23 | 24 | def create(self): 25 | self._start_container(False) 26 | 27 | def run(self): 28 | self._start_container() 29 | 30 | def rerun(self): 31 | # Commit the current container and then use that image_id to restart. 32 | self.state['image_id'] = self.backend.commit_container(self.state['container_id'])['Id'] 33 | self._start_container() 34 | 35 | def start(self): 36 | utils.status("Starting container %s - %s" % (self.name, self.state['container_id'])) 37 | self.backend.start_container(self.state['container_id'], self.mounts) 38 | 39 | def stop(self, timeout=10): 40 | utils.status("Stopping container %s - %s" % (self.name, self.state['container_id'])) 41 | self.backend.stop_container(self.state['container_id'], timeout=timeout) 42 | 43 | def destroy(self, timeout=None): 44 | self.stop(timeout) 45 | utils.status("Destroying container %s - %s" % (self.name, self.state['container_id'])) 46 | self.backend.remove_container(self.state['container_id']) 47 | 48 | def get_ip_address(self): 49 | return self.backend.get_ip_address(self.state['container_id']) 50 | 51 | def inspect(self): 52 | return self.backend.inspect_container(self.state['container_id']) 53 | 54 | def attach(self): 55 | # should probably catch ctrl-c here so that the process doesn't abort 56 | for line in self.backend.attach_container(self.state['container_id']): 57 | sys.stdout.write(line) 58 | 59 | def _start_container(self, start=True): 60 | # Start the container 61 | self.state['container_id'] = self.backend.create_container(self.state['image_id'], self.config) 62 | 63 | if (start): 64 | self.start() 65 | 66 | self.log.info('Container started: %s %s', self.name, self.state['container_id']) 67 | -------------------------------------------------------------------------------- /maestro/environment.py: -------------------------------------------------------------------------------- 1 | class Environment: 2 | def __init__(self): 3 | # Maintains a list of services ordered in start order 4 | self.services = [] 5 | 6 | def start(self): 7 | pass 8 | 9 | def stop(self): 10 | pass 11 | 12 | def destroy(self): 13 | pass 14 | 15 | def load(self): 16 | pass 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /maestro/exceptions.py: -------------------------------------------------------------------------------- 1 | 2 | class MaestroError(Exception): 3 | pass 4 | 5 | class TemplateError(MaestroError): 6 | pass 7 | 8 | class ContainerError(MaestroError): 9 | pass -------------------------------------------------------------------------------- /maestro/py_backend.py: -------------------------------------------------------------------------------- 1 | import docker 2 | 3 | class PyBackend: 4 | def __init__(self): 5 | self.docker_client = docker.Client() 6 | 7 | ## Container management 8 | 9 | def create_container(self, image_id, config): 10 | return self._start_container(image_id, config, False) 11 | 12 | def run_container(self, image_id, config): 13 | return self._start_container(image_id, config) 14 | 15 | def start_container(self, container_id, mounts=None): 16 | self.docker_client.start(container_id, binds=mounts) 17 | 18 | def stop_container(self, container_id, timeout=10): 19 | self.docker_client.stop(container_id, timeout=timeout) 20 | 21 | def remove_container(self, container_id, timeout=None): 22 | self.stop_container(timeout) 23 | self.docker_client.remove_container(container_id) 24 | 25 | def inspect_container(self, container_id): 26 | return self.docker_client.inspect_container(container_id) 27 | 28 | def commit_container(self, container_id): 29 | return self.docker_client.commit(container_id) 30 | 31 | def attach_container(self, container_id): 32 | return self.docker_client.attach(container_id) 33 | 34 | ## Image management 35 | 36 | def build_image(self, fileobj=None, path=None): 37 | return self.docker_client.build(path=path, fileobj=fileobj) 38 | 39 | def remove_image(self, image_id): 40 | self.docker_client.remove_image(image_id) 41 | 42 | def inspect_image(self, image_id): 43 | return self.docker_client.inspect_image(image_id) 44 | 45 | def images(self, name): 46 | return self.docker_client.images(name=name) 47 | 48 | def tag_image(self, image_id, name, tag): 49 | self.docker_client.tag(image_id, name, tag=tag) 50 | 51 | def pull_image(self, name): 52 | return self.docker_client.pull(name) 53 | 54 | ## Helpers 55 | 56 | def get_ip_address(self, container_id): 57 | state = self.docker_client.inspect_container(container_id) 58 | return state['NetworkSettings']['IPAddress'] 59 | 60 | def _start_container(self, image_id, config, start=True): 61 | # Start the container 62 | container_id = self.docker_client.create_container(image_id, **config)['Id'] 63 | 64 | if (start): 65 | self.start_container(container_id) 66 | 67 | return container_id -------------------------------------------------------------------------------- /maestro/service.py: -------------------------------------------------------------------------------- 1 | import docker 2 | import os, sys, yaml, copy, string, StringIO 3 | import maestro, template, utils 4 | from requests.exceptions import HTTPError 5 | from .container import Container 6 | 7 | class ContainerError(Exception): 8 | pass 9 | 10 | class Service: 11 | def __init__(self, conf_file=None, environment=None): 12 | self.log = utils.setupLogging() 13 | self.containers = {} 14 | self.templates = {} 15 | self.state = 'live' 16 | 17 | if environment: 18 | self.load(environment) 19 | else: 20 | # If we didn't get an absolute path to a file, look for it in the current directory. 21 | if not conf_file.startswith('/'): 22 | conf_file = os.path.join(os.path.dirname(sys.argv[0]), conf_file) 23 | 24 | data = open(conf_file, 'r') 25 | self.config = yaml.load(data) 26 | 27 | # On load, order templates into the proper startup sequence 28 | self.start_order = utils.order(self.config['templates']) 29 | 30 | def get(self, container): 31 | return self.containers[container] 32 | 33 | def build(self, wait_time=60): 34 | # Setup and build all the templates 35 | for tmpl in self.start_order: 36 | if not self.config['templates'][tmpl]: 37 | sys.stderr.write('Error: no configuration found for template: ' + tmpl + '\n') 38 | exit(1) 39 | 40 | config = self.config['templates'][tmpl] 41 | 42 | # Create the template. The service name and version will be dynamic once the new config format is implemented 43 | utils.status('Building template %s' % (tmpl)) 44 | tmpl_instance = template.Template(tmpl, config, 'service', '0.1') 45 | tmpl_instance.build() 46 | 47 | 48 | self.templates[tmpl] = tmpl_instance 49 | 50 | # We'll store the running instances as a dict under the template 51 | self.containers[tmpl] = {} 52 | 53 | # Start the envrionment 54 | for tmpl in self.start_order: 55 | self._handleRequire(tmpl, wait_time) 56 | 57 | tmpl_instance = self.templates[tmpl] 58 | config = self.config['templates'][tmpl] 59 | 60 | # If count is defined in the config then we're launching multiple instances of the same thing 61 | # and they'll need to be tagged accordingly. Count only applies on build. 62 | count = tag_name = 1 63 | if 'count' in config: 64 | count = tag_name = config['count'] 65 | 66 | while count > 0: 67 | name = tmpl 68 | if tag_name > 1: 69 | name = name + '__' + str(count) 70 | 71 | utils.status('Launching instance of template %s named %s' % (tmpl, name)) 72 | instance = tmpl_instance.instantiate(name) 73 | instance.run() 74 | 75 | self.containers[tmpl][name] = instance 76 | 77 | count = count - 1 78 | 79 | def destroy(self, timeout=None): 80 | for tmpl in reversed(self.start_order): 81 | for container in self.containers[tmpl]: 82 | self.log.info('Destroying container: %s', container) 83 | self.containers[tmpl][container].destroy(timeout) 84 | 85 | self.state = 'destroyed' 86 | return True 87 | 88 | def start(self, container=None, wait_time=60): 89 | if not self._live(): 90 | utils.status('Environment has been destroyed and can\'t be started') 91 | return False 92 | 93 | # If a container is provided we just start that container 94 | # TODO: may need an abstraction here to handle names of multi-container groups 95 | if container: 96 | tmpl = self._getTemplate(container) 97 | 98 | rerun = self._handleRequire(tmpl, wait_time) 99 | 100 | # We need to see if env has changed and then commit and run a new container. 101 | # This rerun functionality should only be a temporary solution as each time the 102 | # container is restarted this way it will consume a layer. 103 | # This is only necessary because docker start won't take a new set of env vars 104 | if rerun: 105 | self.containers[tmpl][container].rerun() 106 | else: 107 | self.containers[tmpl][container].start() 108 | else: 109 | for tmpl in self.start_order: 110 | rerun = self._handleRequire(tmpl, wait_time) 111 | 112 | for container in self.containers[tmpl]: 113 | if rerun: 114 | self.containers[tmpl][container].rerun() 115 | else: 116 | self.containers[tmpl][container].start() 117 | 118 | return True 119 | 120 | def stop(self, container=None, timeout=None): 121 | if not self._live(): 122 | utils.status('Environment has been destroyed and can\'t be stopped.') 123 | return False 124 | 125 | if container: 126 | self.containers[self._getTemplate(container)][container].stop(timeout) 127 | else: 128 | for tmpl in reversed(self.start_order): 129 | for container in self.containers[tmpl]: 130 | self.containers[tmpl][container].stop(timeout) 131 | 132 | return True 133 | 134 | def load(self, filename='envrionment.yml'): 135 | self.log.info('Loading environment from: %s', filename) 136 | 137 | with open(filename, 'r') as input_file: 138 | self.config = yaml.load(input_file) 139 | 140 | self.state = self.config['state'] 141 | 142 | for tmpl in self.config['templates']: 143 | # TODO fix hardcoded service name and version 144 | self.templates[tmpl] = template.Template(tmpl, self.config['templates'][tmpl], 'service', '0.1') 145 | self.containers[tmpl] = {} 146 | 147 | self.start_order = utils.order(self.config['templates']) 148 | for container in self.config['containers']: 149 | tmpl = self.config['containers'][container]['template'] 150 | 151 | self.containers[tmpl][container] = Container(container, self.config['containers'][container], 152 | self.config['templates'][tmpl]['config']) 153 | 154 | def save(self, filename='environment.yml'): 155 | self.log.info('Saving environment state to: %s', filename) 156 | 157 | with open(filename, 'w') as output_file: 158 | output_file.write(self.dump()) 159 | 160 | def run(self, template, commandline=None, wait_time=60, attach=False, dont_add=False): 161 | if template in self.templates: 162 | self._handleRequire(template, wait_time) 163 | 164 | name = template + "-" + str(os.getpid()) 165 | # TODO: name need to be dynamic here. Need to handle static and temporary cases. 166 | container = self.templates[template].instantiate(name, commandline) 167 | container.run() 168 | 169 | # For temporary containers we may not want to save it in the environment 170 | if not dont_add: 171 | self.containers[template][name] = container 172 | 173 | # for dynamic runs there needs to be a way to display the output of the command. 174 | if attach: 175 | container.attach() 176 | return container 177 | else: 178 | # Should handle arbitrary containers 179 | raise ContainerError('Unknown template') 180 | 181 | def ps(self): 182 | columns = '{0:<14}{1:<19}{2:<44}{3:<11}{4:<15}\n' 183 | result = columns.format('ID', 'NODE', 'COMMAND', 'STATUS', 'PORTS') 184 | 185 | for tmpl in self.templates: 186 | for container in self.containers[tmpl]: 187 | container_id = self.containers[tmpl][container].state['container_id'] 188 | 189 | node_name = (container[:15] + '..') if len(container) > 17 else container 190 | 191 | command = '' 192 | status = 'Stopped' 193 | ports = '' 194 | try: 195 | state = docker.Client().inspect_container(container_id) 196 | command = string.join([state['Path']] + state['Args']) 197 | command = (command[:40] + '..') if len(command) > 42 else command 198 | p = [] 199 | if state['NetworkSettings']['PortMapping']: 200 | p = state['NetworkSettings']['PortMapping']['Tcp'] 201 | 202 | for port in p: 203 | if ports: 204 | ports += ', ' 205 | ports += p[port] + '->' + port 206 | if state['State']['Running']: 207 | status = 'Running' 208 | except HTTPError: 209 | status = 'Destroyed' 210 | 211 | result += columns.format(container_id, node_name, command, status, ports) 212 | 213 | return result.rstrip('\n') 214 | 215 | def dump(self): 216 | result = {} 217 | result['state'] = self.state 218 | result['templates'] = {} 219 | result['containers'] = {} 220 | 221 | for template in self.templates: 222 | result['templates'][template] = self.templates[template].config 223 | 224 | for container in self.containers[template]: 225 | result['containers'][container] = self.containers[template][container].state 226 | 227 | return yaml.dump(result, Dumper=yaml.SafeDumper) 228 | 229 | def _getTemplate(self, container): 230 | # Find the template for this container 231 | for tmpl in self.containers: 232 | if container in self.containers[tmpl]: 233 | return tmpl 234 | 235 | def _live(self): 236 | return self.state == 'live' 237 | 238 | def _pollService(self, container, service, name, port, wait_time): 239 | # Based on start_order the service should already be running 240 | service_ip = self.containers[service][name].get_ip_address() 241 | utils.status('Starting %s: waiting for service %s on ip %s and port %s' % (container, service, service_ip, port)) 242 | 243 | result = utils.waitForService(service_ip, int(port), wait_time) 244 | if result < 0: 245 | utils.status('Never found service %s on port %s' % (service, port)) 246 | raise ContainerError('Couldn\d find required services, aborting') 247 | 248 | utils.status('Found service %s on ip %s and port %s' % (service, service_ip, port)) 249 | 250 | #return service_ip + ":" + str(port) 251 | return service_ip 252 | 253 | def _handleRequire(self, tmpl, wait_time): 254 | env = [] 255 | # Wait for any required services to finish registering 256 | config = self.config['templates'][tmpl] 257 | if 'require' in config: 258 | try: 259 | # Containers can depend on mulitple services 260 | for service in config['require']: 261 | service_env = [] 262 | port = config['require'][service]['port'] 263 | if port: 264 | # If count is defined then we need to wait for all instances to start 265 | count = config['require'][service].get('count', 1) 266 | if count > 1: 267 | while count > 0: 268 | name = service + '__' + str(count) 269 | service_env.append(self._pollService(tmpl, service, name, port, wait_time)) 270 | count = count - 1 271 | else: 272 | service_env.append(self._pollService(tmpl, service, service, port, wait_time)) 273 | 274 | env.append(service.upper() + '=' + ' '.join(service_env)) 275 | except: 276 | utils.status('Failure on require. Shutting down the environment') 277 | self.destroy() 278 | raise 279 | 280 | # If the environment changes then dependent containers will need to be re-run not just restarted 281 | rerun = False 282 | # Setup the env for dependent services 283 | if 'environment' in config['config']: 284 | for entry in env: 285 | name, value = entry.split('=') 286 | result = [] 287 | replaced = False 288 | # See if an existing variable exists and needs to be updated 289 | for var in config['config']['environment']: 290 | var_name, var_value = var.split('=') 291 | if var_name == name and var_value != value: 292 | replaced = True 293 | rerun = True 294 | result.append(entry) 295 | elif var_name == name and var_value == value: 296 | # Just drop any full matches. We'll add it back later 297 | pass 298 | else: 299 | result.append(var) 300 | 301 | if not replaced: 302 | result.append(entry) 303 | 304 | config['config']['environment'] = result 305 | else: 306 | config['config']['environment'] = env 307 | 308 | # Determines whether or not a container can simply be restarted 309 | return rerun 310 | -------------------------------------------------------------------------------- /maestro/template.py: -------------------------------------------------------------------------------- 1 | import exceptions, utils, container, py_backend 2 | import StringIO, copy, logging, sys 3 | from requests.exceptions import HTTPError 4 | 5 | class Template: 6 | def __init__(self, name, config, service, version): 7 | self.name = name 8 | self.config = config 9 | self.service = service 10 | self.version = version 11 | self.log = logging.getLogger('maestro') 12 | 13 | self.backend = py_backend.PyBackend() 14 | 15 | def build(self): 16 | # If there is a docker file or url hand off to Docker builder 17 | if 'buildspec' in self.config: 18 | if self.config['buildspec']: 19 | if 'dockerfile' in self.config['buildspec']: 20 | self._build(dockerfile=self.config['buildspec']['dockerfile']) 21 | elif 'url' in self.config['buildspec']: 22 | self._build(url=self.config['buildspec']['url']) 23 | else: 24 | raise exceptions.TemplateError("Template: " + self.name + " Buildspec specified but no dockerfile or url found.") 25 | else: 26 | # verify the base image and pull it if necessary 27 | try: 28 | base = self.config['base_image'] 29 | self.backend.inspect_image(base) 30 | except HTTPError: 31 | # Attempt to pull the image. 32 | self.log.info('Attempting to pull base: %s', base) 33 | result = self.backend.pull_image(base) 34 | if 'error' in result: 35 | self.log.error('No base image could be pulled under the name: %s', base) 36 | raise exceptions.TemplateError("No base image could be pulled under the name: " + base) 37 | except KeyError: 38 | raise exceptions.TemplateError("Template: " + self.name + "No base image specified.") 39 | 40 | # There doesn't seem to be a way to currently remove tags so we'll generate a new image. 41 | # More consistent for all cases this way too but it does feel kinda wrong. 42 | dockerfile = """ 43 | FROM %s 44 | MAINTAINER %s 45 | """ % (base, self._mid()) 46 | self._build(dockerfile=dockerfile) 47 | 48 | return True 49 | 50 | # Launches an instance of the template in a new container 51 | def instantiate(self, name, command=None): 52 | config = copy.deepcopy(self.config['config']) 53 | 54 | # Setup bind mounts to the host system 55 | bind_mounts = {} 56 | if 'mounts' in self.config: 57 | bind_mounts = self.config['mounts'] 58 | for src, dest in self.config['mounts'].items(): 59 | if 'volumes' not in config: 60 | config['volumes'] = {} 61 | 62 | config['volumes'][dest] = {} 63 | 64 | if command: 65 | if isinstance(command, basestring): 66 | config['command'] = command 67 | else: 68 | config['command'] = " ".join(command) 69 | 70 | return container.Container(name, {'template': self.name, 'image_id': self.config['image_id']}, config, mounts=bind_mounts) 71 | 72 | def destroy(self): 73 | # If there is an image_id then we need to destroy the image. 74 | if 'image_id' in self.config: 75 | self.backend.remove_image(self.config['image_id']) 76 | 77 | def full_name(self): 78 | return self.service + "." + self.name 79 | 80 | def _base_id(self, base): 81 | tag = 'latest' 82 | if ':' in base: 83 | base, tag = base.split(':') 84 | 85 | result = self.backend.images(name=base) 86 | for image in result: 87 | if image['Tag'] == tag: 88 | return image['Id'] 89 | 90 | return None 91 | 92 | # Generate the meastro specific ID for this template. 93 | def _mid(self): 94 | return self.service + "." + self.name + ":" + self.version 95 | 96 | def _build(self, dockerfile=None, url=None): 97 | self.log.info('Building container: %s', self._mid()) 98 | 99 | if (dockerfile): 100 | result = self.backend.build_image(fileobj=StringIO.StringIO(dockerfile)) 101 | elif (url): 102 | result = self.backend.build_image(path=url) 103 | else: 104 | raise exceptions.TemplateError("Can't build if no buildspec is provided: " + self.name) 105 | 106 | if result[0] == None: 107 | # TODO: figure out what to do with the result of this execution 108 | print result 109 | raise exceptions.TemplateError("Build failed for template: " + self.name) 110 | 111 | self.config['image_id'] = result[0] 112 | 113 | self._tag(self.config['image_id']) 114 | 115 | self.log.info('Container registered with tag: %s', self._mid()) 116 | 117 | def _tag(self, image_id): 118 | # Tag the container with the name and process id 119 | self.backend.tag_image(image_id, self.service + "." + self.name, tag=self.version) 120 | 121 | # TODO: make sure this is always appropriate to do as there may be cases where tagging a build as latest isn't desired. 122 | self.backend.tag_image(image_id, self.service + "." + self.name, tag='latest') 123 | 124 | -------------------------------------------------------------------------------- /maestro/utils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os, sys, time, socket 3 | import docker 4 | 5 | def setupLogging(): 6 | log = logging.getLogger('maestro') 7 | if not len(log.handlers): 8 | log.setLevel(logging.DEBUG) 9 | 10 | formatter = logging.Formatter("%(asctime)s %(levelname)-10s %(message)s") 11 | filehandler = logging.FileHandler('maestro.log') 12 | filehandler.setLevel(logging.DEBUG) 13 | filehandler.setFormatter(formatter) 14 | log.addHandler(filehandler) 15 | return log 16 | 17 | quiet=False 18 | def setQuiet(state=True): 19 | global quiet 20 | quiet = state 21 | 22 | # Display the status 23 | def status(string): 24 | global quiet 25 | log = logging.getLogger('maestro') 26 | log.info(string) 27 | 28 | if not quiet: 29 | print string 30 | 31 | def order(raw_list): 32 | def _process(wait_list): 33 | new_wait = [] 34 | for item in wait_list: 35 | match = False 36 | for dependency in raw_list[item]['require']: 37 | if dependency in ordered_list: 38 | match = True 39 | else: 40 | match = False 41 | break 42 | 43 | if match: 44 | ordered_list.append(item) 45 | else: 46 | new_wait.append(item) 47 | 48 | if len(new_wait) > 0: 49 | # Guard against circular dependencies 50 | if len(new_wait) == len(wait_list): 51 | raise Exception("Unable to satisfy the require for: " + item) 52 | 53 | # Do it again for any remaining items 54 | _process(new_wait) 55 | 56 | ordered_list = [] 57 | wait_list = [] 58 | # Start by building up the list of items that do not have any dependencies 59 | for item in raw_list: 60 | if 'require' not in raw_list[item]: 61 | ordered_list.append(item) 62 | else: 63 | wait_list.append(item) 64 | 65 | # Then recursively order the items that do define dependencies 66 | _process(wait_list) 67 | 68 | return ordered_list 69 | 70 | def waitForService(ip, port, retries=60): 71 | while retries >= 0: 72 | try: 73 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 74 | s.settimeout(1) 75 | s.connect((ip, port)) 76 | s.close() 77 | break 78 | except: 79 | time.sleep(0.5) 80 | retries = retries - 1 81 | continue 82 | 83 | return retries 84 | 85 | def findImage(name, tag="latest"): 86 | result = docker.Client().images(name=name) 87 | 88 | for image in result: 89 | if image['Tag'] == tag: 90 | return image['Id'] 91 | return None -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | cmdln>=1.0 2 | -e git+http://github.com/dotcloud/docker-py.git#egg=docker-py -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | setup(name='maestro', 3 | version='0.1', 4 | description='Orchestration tools for multi-container docker environments', 5 | author='Kimbro Staken', 6 | author_email='kstaken@kstaken.com', 7 | url='https://github.com/toscanini/maestro', 8 | packages=['maestro'], 9 | scripts=['bin/maestro'] 10 | ) 11 | -------------------------------------------------------------------------------- /tests/fixtures/count.yml: -------------------------------------------------------------------------------- 1 | templates: 2 | service1: 3 | base_image: ubuntu 4 | count: 3 5 | config: 6 | command: '/bin/bash -c "apt-get install netcat ; nc -l 8080 -k"' 7 | detach: true 8 | service_post: 9 | base_image: ubuntu 10 | config: 11 | command: '/bin/bash -c "while true; do echo hello world; sleep 60; done;"' 12 | detach: true 13 | require: 14 | service1: 15 | count: 3 16 | port: 8080 -------------------------------------------------------------------------------- /tests/fixtures/default.yml: -------------------------------------------------------------------------------- 1 | templates: 2 | test_server_1: 3 | base_image: ubuntu 4 | config: 5 | ports: 6 | - '8080' 7 | command: 'ps aux' 8 | hostname: test_server_1 9 | user: root 10 | detach: true 11 | stdin_open: true 12 | tty: true 13 | mem_limit: 2560000 14 | environment: 15 | - ENV_VAR=testing 16 | dns: 17 | - 8.8.8.8 18 | - 8.8.4.4 19 | volumes: 20 | /var/testing: {} 21 | 22 | #volumes_from: container_id 23 | test_server_2: 24 | base_image: ubuntu 25 | config: 26 | command: 'ls -l' 27 | hostname: test_server_2 -------------------------------------------------------------------------------- /tests/fixtures/dockerfile.yml: -------------------------------------------------------------------------------- 1 | templates: 2 | test_server_1: 3 | base_image: ubuntu 4 | config: 5 | command: ns -l 8080 -k 6 | buildspec: 7 | dockerfile: | 8 | FROM ubuntu 9 | 10 | RUN apt-get update 11 | RUN apt-get -y install netcat 12 | 13 | test_server_2: 14 | base_image: ubuntu 15 | config: 16 | command: 'ls -l' 17 | buildspec: 18 | url: github.com/toscanini/maestro-dockerfile-test 19 | tags: 20 | - kstaken/test_server_2 -------------------------------------------------------------------------------- /tests/fixtures/maestro.yml: -------------------------------------------------------------------------------- 1 | environment: 2 | description: Test Environment 3 | 4 | services: 5 | service_1: 6 | description: Test service 1 7 | version: 0.1 8 | templates: 9 | template_test: 10 | base_image: ubuntu 11 | config: 12 | command: /bin/bash 13 | detach: true 14 | -------------------------------------------------------------------------------- /tests/fixtures/require-cycle.yml: -------------------------------------------------------------------------------- 1 | containers: 2 | test_server_1: 3 | base_image: ubuntu 4 | config: 5 | ports: 6 | - '8080' 7 | command: '/bin/bash -c "while true; do echo hello world; sleep 60; done;"' 8 | hostname: test_server_1 9 | detach: true 10 | require: 11 | service: 12 | name: test_server_2 13 | port: '8080' 14 | test_server_2: 15 | base_image: ubuntu 16 | config: 17 | ports: 18 | - '8080' 19 | command: '/bin/bash -c "while true; do echo hello world; sleep 60; done;"' 20 | hostname: test_server_2 21 | detach: true 22 | require: 23 | service: 24 | name: test_server_1 25 | port: '8080' 26 | test_server_3: 27 | base_image: ubuntu 28 | config: 29 | ports: 30 | - '8080' 31 | command: '/bin/bash -c "while true; do echo hello world; sleep 60; done;"' 32 | hostname: test_server_2 33 | detach: true 34 | require: 35 | service: 36 | name: test_server_1 37 | port: '8080' -------------------------------------------------------------------------------- /tests/fixtures/require.yml: -------------------------------------------------------------------------------- 1 | templates: 2 | test_server_1: 3 | base_image: ubuntu 4 | config: 5 | ports: 6 | - '8080' 7 | command: '/bin/bash -c "apt-get install netcat ; nc -l 8080 -k"' 8 | hostname: test_server_1 9 | detach: true 10 | require: 11 | test_server_2: 12 | port: '8080' 13 | test_server_2: 14 | base_image: ubuntu 15 | config: 16 | ports: 17 | - '8080' 18 | command: '/bin/bash -c "apt-get install netcat ; nc -l 8080 -k"' 19 | hostname: test_server_2 20 | detach: true 21 | test_server_3: 22 | base_image: ubuntu 23 | config: 24 | ports: 25 | - '8080' 26 | command: '/bin/bash -c "apt-get install netcat ; nc -l 8080 -k"' 27 | hostname: test_server_2 28 | detach: true 29 | require: 30 | test_server_1: 31 | port: '8080' -------------------------------------------------------------------------------- /tests/fixtures/startstop.yml: -------------------------------------------------------------------------------- 1 | templates: 2 | test_server_1: 3 | base_image: ubuntu 4 | config: 5 | ports: 6 | - '8080' 7 | command: '/bin/bash -c "while true; do echo hello world; sleep 60; done;"' 8 | hostname: test_server_1 9 | detach: true 10 | test_server_2: 11 | base_image: ubuntu 12 | config: 13 | ports: 14 | - '8080' 15 | command: '/bin/bash -c "while true; do echo hello world; sleep 60; done;"' 16 | hostname: test_server_2 17 | detach: true 18 | -------------------------------------------------------------------------------- /tests/fixtures/template/invalid_base.yml: -------------------------------------------------------------------------------- 1 | base_image: ubuntu2 2 | config: 3 | command: /bin/bash 4 | detach: true -------------------------------------------------------------------------------- /tests/fixtures/template/invalid_buildspec.yml: -------------------------------------------------------------------------------- 1 | base_image: ubuntu 2 | config: 3 | command: /bin/bash 4 | detach: true 5 | buildspec: -------------------------------------------------------------------------------- /tests/fixtures/template/invalid_dockerfile.yml: -------------------------------------------------------------------------------- 1 | base_image: ubuntu 2 | config: 3 | command: /bin/bash 4 | detach: true 5 | buildspec: 6 | dockerfile: | 7 | FRO ubuntu -------------------------------------------------------------------------------- /tests/fixtures/template/mount.yml: -------------------------------------------------------------------------------- 1 | base_image: ubuntu 2 | config: 3 | command: /bin/bash 4 | detach: true 5 | mounts: 6 | /tmp: /var/www -------------------------------------------------------------------------------- /tests/fixtures/template/no_base.yml: -------------------------------------------------------------------------------- 1 | config: 2 | command: /bin/bash 3 | detach: true -------------------------------------------------------------------------------- /tests/fixtures/template/valid_base.yml: -------------------------------------------------------------------------------- 1 | base_image: ubuntu 2 | config: 3 | command: /bin/bash 4 | detach: true -------------------------------------------------------------------------------- /tests/fixtures/template/valid_base_tag.yml: -------------------------------------------------------------------------------- 1 | base_image: ubuntu:precise 2 | config: 3 | command: /bin/bash 4 | detach: true -------------------------------------------------------------------------------- /tests/fixtures/template/valid_build_url.yml: -------------------------------------------------------------------------------- 1 | base_image: ubuntu 2 | config: 3 | command: /bin/bash 4 | detach: true 5 | buildspec: 6 | url: github.com/toscanini/maestro-dockerfile-test -------------------------------------------------------------------------------- /tests/fixtures/template/valid_dockerfile.yml: -------------------------------------------------------------------------------- 1 | base_image: ubuntu 2 | config: 3 | command: /bin/bash 4 | detach: true 5 | buildspec: 6 | dockerfile: | 7 | FROM ubuntu -------------------------------------------------------------------------------- /tests/test_container.py: -------------------------------------------------------------------------------- 1 | import unittest, sys 2 | sys.path.append('.') 3 | from maestro import container, exceptions, utils 4 | from requests.exceptions import HTTPError 5 | 6 | utils.setQuiet(True) 7 | 8 | class TestContainer(unittest.TestCase): 9 | def testInit(self): 10 | # with self.assertRaises(exceptions.ContainerError) as e: 11 | # container.Container('test_container', { 'image_id': utils.findImage('ubuntu') }, {}) 12 | pass 13 | def testGetIpAddress(self): 14 | # TODO: image_id will change 15 | c = container.Container('test_container', { 'image_id': utils.findImage('ubuntu') }, {'command': 'ps aux'}) 16 | 17 | c.run() 18 | 19 | self.assertIsNotNone(c.state['container_id']) 20 | self.assertIsNotNone(c.get_ip_address()) 21 | 22 | def testDestroy(self): 23 | c = container.Container('test_container', { 'image_id': utils.findImage('ubuntu') }, {'command': 'ps aux'}) 24 | 25 | c.run() 26 | 27 | self.assertIsNotNone(c.state['container_id']) 28 | 29 | c.destroy() 30 | 31 | with self.assertRaises(HTTPError) as e: 32 | c.backend.inspect_container(c.state['container_id']) 33 | 34 | self.assertEqual(str(e.exception), '404 Client Error: Not Found') 35 | 36 | if __name__ == '__main__': 37 | unittest.main() -------------------------------------------------------------------------------- /tests/test_maestro.py: -------------------------------------------------------------------------------- 1 | import unittest, sys 2 | sys.path.append('.') 3 | import maestro 4 | 5 | class TestMaestro(unittest.TestCase): 6 | def testCreateGlobalEnvironment(self): 7 | maestro.init_environment() 8 | 9 | def testCreateLocalEnvironment(self): 10 | env = maestro.init_environment("testEnvironment") 11 | 12 | self.assertIsNotNone(env) 13 | 14 | def testCreateExistingEnvironment(self): 15 | maestro.init_environment() 16 | 17 | def testGetEnvironment(self): 18 | pass 19 | 20 | def testListEnvironment(self): 21 | pass 22 | 23 | def testDestroyLocalEnvironment(self): 24 | pass 25 | 26 | def testDestroyGlobalEnvironment(self): 27 | pass 28 | 29 | if __name__ == '__main__': 30 | unittest.main() -------------------------------------------------------------------------------- /tests/test_py_backend.py: -------------------------------------------------------------------------------- 1 | import unittest, sys, StringIO 2 | sys.path.append('.') 3 | from maestro import py_backend, exceptions, utils 4 | from requests.exceptions import HTTPError 5 | 6 | utils.setQuiet(True) 7 | 8 | class TestContainer(unittest.TestCase): 9 | #@unittest.skip("skipping") 10 | def testStartStopRm(self): 11 | p = py_backend.PyBackend() 12 | 13 | c = p.create_container(utils.findImage('ubuntu'), {'command': '/bin/bash -c "while true; do echo hello world; sleep 60; done;"'}) 14 | state = p.docker_client.inspect_container(c) 15 | self.assertFalse(state['State']['Running']) 16 | 17 | p.start_container(c) 18 | state = p.docker_client.inspect_container(c) 19 | self.assertTrue(state['State']['Running']) 20 | 21 | p.stop_container(c, 1) 22 | state = p.docker_client.inspect_container(c) 23 | self.assertFalse(state['State']['Running']) 24 | 25 | p.remove_container(c, 1) 26 | with self.assertRaises(HTTPError) as e: 27 | p.docker_client.inspect_container(c) 28 | 29 | self.assertEqual(str(e.exception), '404 Client Error: Not Found') 30 | 31 | def testBuildImage(self): 32 | dockerfile = """ 33 | FROM ubuntu 34 | MAINTAINER test 35 | """ 36 | p = py_backend.PyBackend() 37 | image_id = p.build_image(fileobj=StringIO.StringIO(dockerfile))[0] 38 | self.assertEqual(p.inspect_image(image_id)['author'], 'test') 39 | 40 | p.remove_image(image_id) 41 | with self.assertRaises(HTTPError) as e: 42 | p.inspect_image(image_id) 43 | 44 | #@unittest.skip("skipping") 45 | def testGetIpAddress(self): 46 | # TODO: image_id will change 47 | p = py_backend.PyBackend() 48 | 49 | c = p.run_container(utils.findImage('ubuntu'), {'command': 'ps aux'}) 50 | 51 | self.assertIsNotNone(c) 52 | 53 | self.assertIsNotNone(p.get_ip_address(c)) 54 | 55 | if __name__ == '__main__': 56 | unittest.main() -------------------------------------------------------------------------------- /tests/test_service.py: -------------------------------------------------------------------------------- 1 | import unittest, sys, yaml 2 | import docker 3 | sys.path.append('.') 4 | from maestro import service, utils 5 | from requests.exceptions import HTTPError 6 | 7 | utils.setQuiet(True) 8 | 9 | class TestContainer(unittest.TestCase): 10 | def setUp(self): 11 | self.mix = service.Service('fixtures/default.yml') 12 | self.mix.build() 13 | 14 | def tearDown(self): 15 | self.mix.destroy(timeout=1) 16 | 17 | #@unittest.skip("skipping") 18 | def testBuild(self): 19 | env = yaml.load(self.mix.dump()) 20 | self._configCheck(env) 21 | 22 | #@unittest.skip("Skipping") 23 | def testBuildDockerfile(self): 24 | mix = service.Service('fixtures/dockerfile.yml') 25 | mix.build() 26 | env = yaml.load(mix.dump()) 27 | 28 | for container in env['containers']: 29 | state = docker.Client().inspect_container(env['containers'][container]['container_id']) 30 | 31 | self.assertIsNotNone(state) 32 | self.assertIn(container, ['test_server_1', 'test_server_2']) 33 | 34 | self.assertEqual(state['State']['ExitCode'], 0) 35 | 36 | if container == 'test_server_1': 37 | self.assertNotEqual(state['Config']['Image'], 'ubuntu') 38 | self.assertEqual(state['Path'], 'ns') 39 | self.assertEqual(state['Args'][0], '-l') 40 | 41 | #elif container == 'test_server_2': 42 | # self.assertNotEqual(state['Config']['Image'], 'ubuntu') 43 | # self.assertEqual(state['Path'], 'ls') 44 | # self.assertEqual(state['Args'][0], '-l') 45 | 46 | mix.destroy(timeout=1) 47 | 48 | 49 | #@unittest.skip("skipping") 50 | def testPorts(self): 51 | env = yaml.load(self.mix.dump()) 52 | self.mix.save() 53 | for container in env['containers']: 54 | state = docker.Client().inspect_container(env['containers'][container]['container_id']) 55 | 56 | self.assertIsNotNone(state) 57 | if container == 'test_server_1': 58 | self.assertIn('8080', state['NetworkSettings']['PortMapping']['Tcp']) 59 | elif container == 'test_server_2': 60 | self.assertEqual(state['NetworkSettings']['PortMapping']['Tcp'], {}) 61 | else: 62 | # Shouldn't get here 63 | self.assertFalse(True) 64 | 65 | #@unittest.skip("skipping") 66 | def testDestroy(self): 67 | mix = service.Service('fixtures/default.yml') 68 | mix.build() 69 | 70 | env = yaml.load(mix.dump()) 71 | mix.destroy(timeout=1) 72 | 73 | for container in env['containers']: 74 | with self.assertRaises(HTTPError) as e: 75 | docker.Client().inspect_container(env['containers'][container]['container_id']) 76 | 77 | self.assertEqual(str(e.exception), '404 Client Error: Not Found') 78 | 79 | #@unittest.skip("skipping") 80 | def testSave(self): 81 | self.mix.save() 82 | with open('environment.yml', 'r') as input_file: 83 | env = yaml.load(input_file) 84 | 85 | self._configCheck(env) 86 | 87 | #@unittest.skip("skipping") 88 | def testDependencyEnv(self): 89 | mix = service.Service('fixtures/count.yml') 90 | 91 | mix.build() 92 | 93 | # Verify that all three services are running 94 | env = yaml.load(mix.dump()) 95 | 96 | self.assertEqual(len(env['containers']), 4) 97 | 98 | state = docker.Client().inspect_container(env['containers']['service_post']['container_id']) 99 | #self.assertIn("SERVICE1", state['Config']['Env']) 100 | 101 | mix.destroy(timeout=1) 102 | 103 | #@unittest.skip("skipping") 104 | def testCount(self): 105 | mix = service.Service('fixtures/count.yml') 106 | 107 | mix.build(180) 108 | 109 | # Verify that all three services are running 110 | env = yaml.load(mix.dump()) 111 | 112 | self.assertEqual(len(env['containers']), 4) 113 | 114 | for container in env['containers']: 115 | state = docker.Client().inspect_container(env['containers'][container]['container_id']) 116 | 117 | #Verify the containers are running 118 | self.assertTrue(state['State']['Running']) 119 | self.assertEqual(state['State']['ExitCode'], 0) 120 | 121 | mix.destroy(timeout=1) 122 | 123 | #@unittest.skip("skipping") 124 | def testRequire(self): 125 | mix = service.Service('fixtures/require.yml') 126 | 127 | # Verify that it determined the correct start order 128 | start_order = mix.start_order 129 | self.assertEqual(start_order[0], 'test_server_2') 130 | self.assertEqual(start_order[1], 'test_server_1') 131 | self.assertEqual(start_order[2], 'test_server_3') 132 | 133 | mix.build() 134 | 135 | # Verify that all three services are running 136 | env = yaml.load(mix.dump()) 137 | for container in env['containers']: 138 | state = docker.Client().inspect_container(env['containers'][container]['container_id']) 139 | 140 | #Verify the containers are running 141 | self.assertTrue(state['State']['Running']) 142 | self.assertEqual(state['State']['ExitCode'], 0) 143 | 144 | mix.destroy(timeout=1) 145 | 146 | #@unittest.skip("skipping") 147 | def testStop(self): 148 | mix = service.Service('fixtures/startstop.yml') 149 | mix.build() 150 | 151 | env = yaml.load(mix.dump()) 152 | for container in env['containers']: 153 | state = docker.Client().inspect_container(env['containers'][container]['container_id']) 154 | 155 | #Verify the containers are running 156 | self.assertTrue(state['State']['Running']) 157 | self.assertEqual(state['State']['ExitCode'], 0) 158 | 159 | mix.stop(timeout=1) 160 | env = yaml.load(mix.dump()) 161 | 162 | for container in env['containers']: 163 | state = docker.Client().inspect_container(env['containers'][container]['container_id']) 164 | 165 | #Verify the containers are stopped 166 | self.assertFalse(state['State']['Running']) 167 | self.assertNotEqual(state['State']['ExitCode'], 0) 168 | 169 | # restart the environment and then stop one of the containers 170 | mix.start() 171 | mix.stop('test_server_2', timeout=1) 172 | 173 | #Verify that test_server_2 is stopped 174 | state = docker.Client().inspect_container(env['containers']['test_server_2']['container_id']) 175 | self.assertFalse(state['State']['Running']) 176 | self.assertNotEqual(state['State']['ExitCode'], 0) 177 | 178 | #But test_server_1 should still be running 179 | state = docker.Client().inspect_container(env['containers']['test_server_1']['container_id']) 180 | self.assertTrue(state['State']['Running']) 181 | self.assertEqual(state['State']['ExitCode'], 0) 182 | 183 | mix.destroy(timeout=1) 184 | 185 | #@unittest.skip("skipping") 186 | def testStart(self): 187 | mix = service.Service('fixtures/startstop.yml') 188 | mix.build() 189 | 190 | mix.stop(timeout=1) 191 | env = yaml.load(mix.dump()) 192 | 193 | for container in env['containers']: 194 | state = docker.Client().inspect_container(env['containers'][container]['container_id']) 195 | 196 | # Verify the containers are stopped 197 | self.assertFalse(state['State']['Running']) 198 | self.assertNotEqual(state['State']['ExitCode'], 0) 199 | 200 | mix.start() 201 | env = yaml.load(mix.dump()) 202 | 203 | for container in env['containers']: 204 | state = docker.Client().inspect_container(env['containers'][container]['container_id']) 205 | 206 | # Verify the containers are running again 207 | self.assertTrue(state['State']['Running']) 208 | self.assertEqual(state['State']['ExitCode'], 0) 209 | 210 | mix.stop(timeout=1) 211 | mix.start('test_server_1') 212 | 213 | #Verify that test_server_2 is still stopped 214 | state = docker.Client().inspect_container(env['containers']['test_server_2']['container_id']) 215 | self.assertFalse(state['State']['Running']) 216 | self.assertNotEqual(state['State']['ExitCode'], 0) 217 | 218 | #But test_server_1 should now be running 219 | state = docker.Client().inspect_container(env['containers']['test_server_1']['container_id']) 220 | self.assertTrue(state['State']['Running']) 221 | self.assertEqual(state['State']['ExitCode'], 0) 222 | 223 | mix.destroy(timeout=1) 224 | 225 | #@unittest.skip("skipping") 226 | def testStatus(self): 227 | mix = service.Service('fixtures/startstop.yml') 228 | mix.build() 229 | 230 | status = mix.ps() 231 | 232 | lines = status.split("\n") 233 | # Skip over the headers 234 | del(lines[0]) 235 | for line in lines: 236 | if len(line) > 0: 237 | self.assertIn(line[14:29].rstrip(), ['test_server_1', 'test_server_2']) 238 | self.assertEqual(line[77:87].rstrip(), "Running") 239 | 240 | mix.destroy(timeout=1) 241 | 242 | status = mix.ps() 243 | 244 | lines = status.split("\n") 245 | # Skip over the headers 246 | del(lines[0]) 247 | for line in lines: 248 | if len(line) > 0: 249 | self.assertIn(line[14:29].rstrip(), ['test_server_1', 'test_server_2']) 250 | self.assertEqual(line[77:87].rstrip(), "Destroyed") 251 | 252 | #@unittest.skip("skipping") 253 | def testLoad(self): 254 | self.mix.save() 255 | mix = service.Service(environment = 'environment.yml') 256 | 257 | env = yaml.load(mix.dump()) 258 | 259 | self._configCheck(env) 260 | 261 | #@unittest.skip("skipping") 262 | def testRun(self): 263 | # Test the default command run 264 | container = self.mix.run("test_server_1") 265 | state = docker.Client().inspect_container(container.state['container_id']) 266 | self.assertEqual(state['State']['ExitCode'], 0) 267 | self.assertNotEqual(state['Config']['Image'], 'ubuntu') 268 | self.assertEqual(state['Path'], 'ps') 269 | 270 | # Test run of an overridden command 271 | container = self.mix.run("test_server_1", "uptime") 272 | state = docker.Client().inspect_container(container.state['container_id']) 273 | self.assertEqual(state['State']['ExitCode'], 0) 274 | self.assertEqual(state['Path'], 'uptime') 275 | 276 | 277 | def _configCheck(self, env): 278 | self.assertIsNotNone(env) 279 | 280 | for container in env['containers']: 281 | self.assertIn(container, ['test_server_1', 'test_server_2']) 282 | 283 | state = docker.Client().inspect_container(env['containers'][container]['container_id']) 284 | 285 | self.assertEqual(state['State']['ExitCode'], 0) 286 | 287 | if container == 'test_server_1': 288 | self.assertEqual(state['Path'], 'ps') 289 | self.assertEqual(state['Args'][0], 'aux') 290 | self.assertEqual(state['Config']['Hostname'], 'test_server_1') 291 | self.assertEqual(state['Config']['User'], 'root') 292 | self.assertTrue(state['Config']['OpenStdin']) 293 | self.assertTrue(state['Config']['Tty']) 294 | self.assertEqual(state['Config']['Memory'], 2560000) 295 | self.assertIn("ENV_VAR=testing", state['Config']['Env']) 296 | self.assertIn("8.8.8.8", state['Config']['Dns']) 297 | 298 | elif container == 'test_server_2': 299 | self.assertEqual(state['Path'], 'ls') 300 | self.assertEqual(state['Args'][0], '-l') 301 | self.assertEqual(state['Config']['Hostname'], 'test_server_2') 302 | 303 | if __name__ == '__main__': 304 | unittest.main() -------------------------------------------------------------------------------- /tests/test_template.py: -------------------------------------------------------------------------------- 1 | import unittest, sys, yaml, os 2 | 3 | sys.path.append('.') 4 | from maestro import template, utils, exceptions 5 | 6 | utils.setQuiet(True) 7 | 8 | class TestTemplate(unittest.TestCase): 9 | 10 | def testBuild(self): 11 | # Test correct build 12 | config = self._loadFixture("valid_base.yml") 13 | # This will create a template named test.service.template_test:0.1 14 | t = template.Template('template_test', config, 'test.service', '0.1') 15 | self.assertTrue(t.build()) 16 | 17 | # Verify the image really exists with docker. 18 | self.assertIsNotNone(utils.findImage(t.full_name(), t.version)) 19 | t.destroy() 20 | 21 | # Test correct build with a tag 22 | config = self._loadFixture("valid_base_tag.yml") 23 | t = template.Template('template_test', config, 'test.service', '0.1') 24 | self.assertTrue(t.build()) 25 | self.assertIsNotNone(utils.findImage(t.full_name(), t.version)) 26 | t.destroy() 27 | 28 | # Test invalid base image 29 | config = self._loadFixture("invalid_base.yml") 30 | t = template.Template('template_test', config, 'test.service', '0.1') 31 | with self.assertRaises(exceptions.TemplateError) as e: 32 | t.build() 33 | t.destroy() 34 | 35 | # Test no base image specified 36 | config = self._loadFixture("no_base.yml") 37 | t = template.Template('template_test', config, 'test.service', '0.1') 38 | with self.assertRaises(exceptions.TemplateError) as e: 39 | t.build() 40 | t.destroy() 41 | 42 | def testBuildDockerfile(self): 43 | # Test correct build using a minimal Dockerfile 44 | config = self._loadFixture("valid_dockerfile.yml") 45 | t = template.Template('template_test', config, 'test.service', '0.1') 46 | self.assertTrue(t.build()) 47 | t.destroy() 48 | 49 | # Test error on incorrectly formatted Dockerfile 50 | config = self._loadFixture("invalid_dockerfile.yml") 51 | t = template.Template('template_test', config, 'test.service', '0.1') 52 | with self.assertRaises(exceptions.TemplateError) as e: 53 | t.build() 54 | t.destroy() 55 | 56 | # Test error on incorrect format for buildspec 57 | config = self._loadFixture("invalid_buildspec.yml") 58 | t = template.Template('template_test', config, 'test.service', '0.1') 59 | with self.assertRaises(exceptions.TemplateError) as e: 60 | t.build() 61 | t.destroy() 62 | 63 | def testBuildUrl(self): 64 | # Test correct build using a minimal Dockerfile 65 | config = self._loadFixture("valid_build_url.yml") 66 | t = template.Template('template_test', config, 'test.service', '0.1') 67 | self.assertTrue(t.build()) 68 | t.destroy() 69 | 70 | def testMount(self): 71 | config = self._loadFixture("mount.yml") 72 | t = template.Template('template_test', config, 'test.service', '0.1') 73 | self.assertTrue(t.build()) 74 | 75 | container = t.instantiate('template_test') 76 | container.run() 77 | self.assertIsNotNone(container.inspect()['Volumes']['/var/www']) 78 | container.destroy() 79 | t.destroy() 80 | 81 | def testInstantiate(self): 82 | config = self._loadFixture("valid_base.yml") 83 | t = template.Template('template_test', config, 'test.service', '0.1') 84 | self.assertTrue(t.build()) 85 | 86 | container = t.instantiate('template_test') 87 | container.run() 88 | self.assertIsNotNone(container.get_ip_address()) 89 | container.destroy() 90 | t.destroy() 91 | 92 | def testDestroy(self): 93 | config = self._loadFixture("valid_base.yml") 94 | t = template.Template('template_test', config, 'test.service', '0.1') 95 | self.assertTrue(t.build()) 96 | 97 | # Make sure the image is there 98 | self.assertIsNotNone(self._findImage(t, t.full_name(), t.version)) 99 | t.destroy() 100 | # Now make sure it's gone 101 | self.assertIsNone(self._findImage(t, t.full_name(), t.version)) 102 | 103 | 104 | def _loadFixture(self, name): 105 | return yaml.load(file(os.path.join(os.path.dirname(__file__), "fixtures/template", name), "r")) 106 | 107 | def _findImage(self, t, name, tag="latest"): 108 | result = t.backend.images(name=name) 109 | 110 | for image in result: 111 | if image['Tag'] == tag: 112 | return image['Id'] 113 | return None 114 | 115 | if __name__ == '__main__': 116 | unittest.main() --------------------------------------------------------------------------------