├── .gitignore ├── MANIFEST.in ├── README.md ├── docker_sync ├── Manager.py ├── TestManager.py ├── __init__.py ├── __main__.py ├── cli.py └── lib │ ├── Container.py │ ├── ContainerDefinition.py │ ├── DockerSyncWrapper.py │ ├── Image.py │ ├── ImageTag.py │ ├── TestContainerDefinition.py │ ├── TestDockerSyncWrapper.py │ ├── TestImage.py │ ├── TestImageTag.py │ └── __init__.py ├── example ├── 00-hosts.yaml ├── 00-private-registry.yaml ├── 10-skydns.yaml ├── 20-skydock.yaml └── 30-elasticsearch.yaml ├── packaging └── rpm │ └── redhat │ ├── README.md │ └── python-docker-sync.spec ├── requirements.txt ├── scripts └── docker-sync └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | setup.cfg 3 | *.egg-info/ 4 | temp/ 5 | dist/ 6 | *.egg 7 | _ve/ 8 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # docker-sync 2 | 3 | Helper utility for configuration management tools like Chef and Puppet. 4 | 5 | Given a directory of config files (see the `example` directory), docker-sync 6 | will ensure that the running containers are kept in sync with the config files, 7 | and will remove containers that no longer have a related config file. 8 | 9 | It's a little bit opinionated: container links aren't supported (tho they could 10 | be in the future); all containers are detached. The image a container is 11 | instantiated from is compared to its tag in its appropriate registry, and the 12 | pull is only done if the registry tag is different than the local tag (a `docker 13 | pull` is slow even when there are no changes). 14 | 15 | In the future I may support [dogestry][dogestry] as an alternative (or companion 16 | to) a traditional Docker registry. 17 | 18 | ## installation 19 | 20 | pip install docker-sync 21 | 22 | Or from a clone: 23 | 24 | pip install -r requirements.txt 25 | pip install . 26 | 27 | ## example usage 28 | 29 | docker-sync ./example 30 | 31 | or 32 | 33 | ./docker_sync/cli.py ./example 34 | 35 | ## options 36 | 37 | You can add `--no-pull` to skip pulling images; very useful when you're 38 | iterating on your configs. 39 | 40 | ## running tests 41 | 42 | nosetests 43 | 44 | Requires nosetests and httpretty 45 | 46 | [dogestry]: https://github.com/blake-education/dogestry 47 | -------------------------------------------------------------------------------- /docker_sync/Manager.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- encoding: utf-8 -*- 3 | 4 | ## I'm beginning to think container links are just plain evil. the dependency 5 | ## tracking is a pain in the ass. So instead of trying to trace dependencies, 6 | ## all containers will be "linked" via explicit port mappings, via the host. 7 | 8 | import logging 9 | logging.basicConfig(level=logging.WARN) 10 | 11 | import os 12 | import glob 13 | 14 | ## cached objects: http://stackoverflow.com/questions/13054250/python-object-cache 15 | 16 | from lib import DockerSyncWrapper as Docker 17 | from lib import ContainerDefinition 18 | 19 | DOCKER = Docker() 20 | 21 | LOGGER = logging.getLogger("main") 22 | 23 | def containerIsOutOfSync(container_def, container_info, image_info): 24 | out_of_sync = False 25 | 26 | ## the effective command being executed is different from the configured 27 | ## command because the effective command includes the entrypoint. 28 | effective_command = image_info.entrypoint or [] 29 | 30 | if container_def.command: 31 | effective_command += container_def.command 32 | else: 33 | effective_command += image_info.command or [] 34 | 35 | ## include image's env vars when comparing container def's vars 36 | effective_env = dict(image_info.env.items() + container_def.env.items()) 37 | 38 | if container_info.image.id != image_info.id: 39 | LOGGER.info("image id does not match") 40 | LOGGER.debug("container_info.image.id %s != image_info.id %s", container_info.image.id, image_info.id) 41 | 42 | out_of_sync = True 43 | 44 | if container_def.hostname is not None and container_info.hostname != container_def.hostname: 45 | LOGGER.info("hostname is different") 46 | LOGGER.debug("container_info.hostname %s != container_def.hostname %s", container_info.hostname, container_def.hostname) 47 | 48 | out_of_sync = True 49 | 50 | if container_info.command != effective_command: 51 | LOGGER.info("command is different") 52 | LOGGER.debug("container_info.command %s != effective_command %s", container_info.command, effective_command) 53 | 54 | out_of_sync = True 55 | 56 | if container_info.env != effective_env: 57 | LOGGER.info("env is different") 58 | LOGGER.debug("container_info.env %s != effective_env %s", container_info.env, effective_env) 59 | 60 | out_of_sync = True 61 | 62 | if container_info.ports != container_def.ports: 63 | LOGGER.info("ports are different") 64 | LOGGER.debug("container_info.ports %s != container_def.ports %s", container_info.ports, container_def.ports) 65 | 66 | out_of_sync = True 67 | 68 | if container_info.volumes != container_def.volumes: 69 | LOGGER.info("volumes are different") 70 | LOGGER.debug("container_info.volumes %s != container_def.volumes %s", container_info.volumes, container_def.volumes) 71 | 72 | out_of_sync = True 73 | 74 | if not container_info.running: 75 | LOGGER.info("container not running") 76 | out_of_sync = True 77 | 78 | return out_of_sync 79 | 80 | 81 | def main(config_dir, pull=True, insecure_registry=False, remove_delay=None): 82 | LOGGER.setLevel(logging.DEBUG) 83 | 84 | container_defs = [] 85 | 86 | for conf in sorted(glob.glob(os.path.join(config_dir, "*.yaml"))): 87 | container_defs.append(ContainerDefinition.parseFile(conf)) 88 | 89 | ## map of container name -> Container 90 | containers = DOCKER.getContainers() 91 | 92 | ## work through in sorted order so we can cheat and make private registries 93 | ## come first 94 | ## delete out-of-sync and unmanaged containers 95 | for container_def in container_defs: 96 | LOGGER.info(container_def.name) 97 | 98 | if pull: 99 | image_info = DOCKER.pullImage(container_def.image_tag, insecure_registry) 100 | else: 101 | image_info = DOCKER.getImage(container_def.image_tag) 102 | 103 | container_info = containers.get(container_def.name, None) 104 | 105 | out_of_sync = True 106 | 107 | if container_info is not None: 108 | out_of_sync = containerIsOutOfSync(container_def, container_info, image_info) 109 | 110 | if out_of_sync: 111 | if container_info: 112 | LOGGER.info("removing %s", container_def.name) 113 | DOCKER.removeContainer(container_def.name, remove_delay=remove_delay) 114 | 115 | LOGGER.info("creating %s", container_def.name) 116 | DOCKER.startContainer(container_def) 117 | 118 | ## delete unmanaged containers 119 | defined_container_names = [ c.name for c in container_defs ] 120 | for cont_name in DOCKER.getContainers(): 121 | if cont_name not in defined_container_names: 122 | LOGGER.warn("unmanaged container; removing %s", cont_name) 123 | 124 | DOCKER.removeContainer(cont_name) 125 | -------------------------------------------------------------------------------- /docker_sync/TestManager.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | 3 | import Manager 4 | from lib import ContainerDefinition, Container, Image 5 | import os 6 | 7 | from nose.tools import eq_ 8 | 9 | class TestManager: 10 | EXAMPLE_DIR = os.path.abspath(os.path.join(__file__, "../../example")) 11 | 12 | def test_containerIsOutOfSync(self): 13 | container_def = ContainerDefinition.parseFile(os.path.join(self.EXAMPLE_DIR, "00-private-registry.yaml")) 14 | 15 | img_info = Image.fromJson({ 16 | "Id": "2e2d7133e4a578bd861e85e7195412201f765d050af78c7906841ea62eb6f7dd", 17 | "Parent": "c79dab5561020bda9ce1b1cbe76283fc95f824cfb26a8a21a384993ed7f392bd", 18 | "Created": "2014-10-21T08:50:44.448455269Z", 19 | "Container": "b756100785c797b9f43d36f249b0d5688d88a1ca68df56d915cb436c4bfc7286", 20 | "Config": { 21 | "OnBuild": [], 22 | "NetworkDisabled": False, 23 | "Entrypoint": None, 24 | "WorkingDir": "", 25 | "Volumes": None, 26 | "Image": "c79dab5561020bda9ce1b1cbe76283fc95f824cfb26a8a21a384993ed7f392bd", 27 | "Cmd": [ 28 | "/bin/sh", 29 | "-c", 30 | "exec docker-registry" 31 | ], 32 | "AttachStdin": False, 33 | "Cpuset": "", 34 | "CpuShares": 0, 35 | "MemorySwap": 0, 36 | "Memory": 0, 37 | "User": "", 38 | "Domainname": "", 39 | "Hostname": "965c252e48c3", 40 | "AttachStdout": False, 41 | "AttachStderr": False, 42 | "PortSpecs": None, 43 | "ExposedPorts": { 44 | "5000/tcp": {} 45 | }, 46 | "Tty": False, 47 | "OpenStdin": False, 48 | "StdinOnce": False, 49 | "Env": [ 50 | "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", 51 | "DOCKER_REGISTRY_CONFIG=/docker-registry/config/config_sample.yml", 52 | "SETTINGS_FLAVOR=dev" 53 | ] 54 | }, 55 | }) 56 | 57 | container_info = Container.fromJson("private-registry", { 58 | "HostConfig": { 59 | "NetworkMode": "", 60 | "VolumesFrom": None, 61 | "DnsSearch": None, 62 | "Binds": [ 63 | "/tmp:/var/lib/docker/registry", 64 | ], 65 | "ContainerIDFile": "", 66 | "LxcConf": None, 67 | "Privileged": False, 68 | "PortBindings": { 69 | "5000/tcp": [ 70 | { 71 | "HostPort": "11003", 72 | "HostIp": "0.0.0.0" 73 | } 74 | ] 75 | }, 76 | "Links": None, 77 | "PublishAllPorts": False, 78 | "Dns": None 79 | }, 80 | "VolumesRW": { 81 | "/var/lib/docker/registry": True, 82 | }, 83 | "Volumes": { 84 | "/var/lib/docker/registry": "/tmp", 85 | }, 86 | "NetworkSettings": { 87 | "Ports": { 88 | "5000/tcp": [ 89 | { 90 | "HostPort": "11003", 91 | "HostIp": "0.0.0.0" 92 | } 93 | ] 94 | }, 95 | "PortMapping": None, 96 | "Bridge": "docker0", 97 | "Gateway": "172.17.42.1", 98 | "IPPrefixLen": 16, 99 | "IPAddress": "172.17.0.31" 100 | }, 101 | "Image": "2e2d7133e4a578bd861e85e7195412201f765d050af78c7906841ea62eb6f7dd", 102 | "State": { 103 | "FinishedAt": "0001-01-01T00:00:00Z", 104 | "StartedAt": "2014-10-28T16:38:31.491949274Z", 105 | "ExitCode": 0, 106 | "Pid": 18785, 107 | "Paused": False, 108 | "Running": True 109 | }, 110 | "Config": { 111 | "OnBuild": None, 112 | "NetworkDisabled": False, 113 | "Entrypoint": None, 114 | "WorkingDir": "", 115 | "Volumes": { 116 | "/var/lib/docker/registry": {}, 117 | }, 118 | "Image": "registry:0.8.1", 119 | "Cmd": [ 120 | "/bin/sh", 121 | "-c", 122 | "exec docker-registry" 123 | ], 124 | "AttachStdin": False, 125 | "Cpuset": "", 126 | "CpuShares": 0, 127 | "MemorySwap": 0, 128 | "Memory": 0, 129 | "User": "", 130 | "Domainname": "", 131 | "Hostname": "private-registry", 132 | "AttachStdout": False, 133 | "AttachStderr": False, 134 | "PortSpecs": None, 135 | "ExposedPorts": { 136 | "5000/tcp": {} 137 | }, 138 | "Tty": False, 139 | "OpenStdin": False, 140 | "StdinOnce": False, 141 | "Env": [ 142 | "SETTINGS_FLAVOR=local", 143 | "STORAGE_PATH=/var/lib/docker/registry", 144 | "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", 145 | "DOCKER_REGISTRY_CONFIG=/docker-registry/config/config_sample.yml" 146 | ] 147 | }, 148 | "Args": [ 149 | "-c", 150 | "exec docker-registry" 151 | ], 152 | "Path": "/bin/sh", 153 | "Created": "2014-10-28T16:38:31.20825271Z", 154 | "Id": "758a155a0374fa7e163e4fc71e96cd1bd7de37674dd5f552b9183789366e91f7", 155 | "ResolvConfPath": "/etc/resolv.conf", 156 | "HostnamePath": "/var/lib/docker/containers/758a155a0374fa7e163e4fc71e96cd1bd7de37674dd5f552b9183789366e91f7/hostname", 157 | "HostsPath": "/var/lib/docker/containers/758a155a0374fa7e163e4fc71e96cd1bd7de37674dd5f552b9183789366e91f7/hosts", 158 | "Name": "/private-registry", 159 | "Driver": "devicemapper", 160 | "ExecDriver": "native-0.2", 161 | "MountLabel": "", 162 | "ProcessLabel": "" 163 | }) 164 | 165 | eq_(Manager.containerIsOutOfSync(container_def, container_info, img_info), False) 166 | 167 | def test_containerIsOutOfSync_hosts(self): 168 | container_def = ContainerDefinition.parseFile(os.path.join(self.EXAMPLE_DIR, "00-hosts.yaml")) 169 | 170 | ## blalor/docker-hosts:latest 171 | img_info = Image.fromJson({ 172 | "Size": 0, 173 | "Os": "linux", 174 | "Architecture": "amd64", 175 | "Id": "98e7ca605530c6ee637e175f08e692149a4d019b384e421e661bd35601b25975", 176 | "Parent": "15e3a43eb69d67df5a6ae1f3b3e87407f3b82157bf54fe8a5dc997cf2ce6528a", 177 | "Created": "2014-07-30T01:02:04.516066768Z", 178 | "Container": "5d7384258a7ac29d8eabe30b6b1d83dfe4a8925440f33982b439731906a087f2", 179 | "Docker_version": "1.1.1", 180 | "Author": "Brian Lalor ", 181 | "Config": { 182 | "OnBuild": [], 183 | "NetworkDisabled": False, 184 | "Entrypoint": [ 185 | "/usr/local/bin/docker-hosts" 186 | ], 187 | "WorkingDir": "", 188 | "Volumes": None, 189 | "Image": "15e3a43eb69d67df5a6ae1f3b3e87407f3b82157bf54fe8a5dc997cf2ce6528a", 190 | "Cmd": None, 191 | "AttachStdin": False, 192 | "Cpuset": "", 193 | "CpuShares": 0, 194 | "MemorySwap": 0, 195 | "Memory": 0, 196 | "User": "", 197 | "Domainname": "", 198 | "Hostname": "5ca9d941ba62", 199 | "AttachStdout": False, 200 | "AttachStderr": False, 201 | "PortSpecs": None, 202 | "ExposedPorts": None, 203 | "Tty": False, 204 | "OpenStdin": False, 205 | "StdinOnce": False, 206 | "Env": [ 207 | "HOME=/", 208 | "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" 209 | ] 210 | } 211 | }) 212 | 213 | container_info = Container.fromJson("hosts", { 214 | "HostConfig": { 215 | "NetworkMode": "", 216 | "VolumesFrom": None, 217 | "DnsSearch": None, 218 | "Binds": [ 219 | "/var/run/docker.sock:/var/run/docker.sock:rw", 220 | "/var/lib/docker/hosts:/srv/hosts:rw" 221 | ], 222 | "ContainerIDFile": "", 223 | "LxcConf": None, 224 | "Privileged": False, 225 | "PortBindings": None, 226 | "Links": None, 227 | "PublishAllPorts": False, 228 | "Dns": None 229 | }, 230 | "VolumesRW": { 231 | "/var/run/docker.sock": True, 232 | "/srv/hosts": True 233 | }, 234 | "Volumes": { 235 | "/var/run/docker.sock": "/var/run/docker.sock", 236 | "/srv/hosts": "/var/lib/docker/hosts" 237 | }, 238 | "NetworkSettings": { 239 | "Ports": {}, 240 | "PortMapping": None, 241 | "Bridge": "docker0", 242 | "Gateway": "172.17.42.1", 243 | "IPPrefixLen": 16, 244 | "IPAddress": "172.17.0.17" 245 | }, 246 | "Image": "98e7ca605530c6ee637e175f08e692149a4d019b384e421e661bd35601b25975", 247 | "State": { 248 | "FinishedAt": "0001-01-01T00:00:00Z", 249 | "StartedAt": "2014-10-28T18:22:51.492441086Z", 250 | "ExitCode": 0, 251 | "Pid": 27669, 252 | "Paused": False, 253 | "Running": True 254 | }, 255 | "Config": { 256 | "OnBuild": None, 257 | "NetworkDisabled": False, 258 | "Entrypoint": [ 259 | "/usr/local/bin/docker-hosts" 260 | ], 261 | "WorkingDir": "", 262 | "Volumes": { 263 | "/var/run/docker.sock": {}, 264 | "/srv/hosts": {} 265 | }, 266 | "Image": "blalor/docker-hosts:latest", 267 | "Cmd": [ 268 | "--domain-name=dev.docker", 269 | "/srv/hosts" 270 | ], 271 | "AttachStdin": False, 272 | "Cpuset": "", 273 | "CpuShares": 0, 274 | "MemorySwap": 0, 275 | "Memory": 0, 276 | "User": "", 277 | "Domainname": "", 278 | "Hostname": "04bf6ca07d2c", 279 | "AttachStdout": False, 280 | "AttachStderr": False, 281 | "PortSpecs": None, 282 | "ExposedPorts": None, 283 | "Tty": False, 284 | "OpenStdin": False, 285 | "StdinOnce": False, 286 | "Env": [ 287 | "HOME=/", 288 | "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" 289 | ] 290 | }, 291 | "Args": [ 292 | "--domain-name=dev.docker", 293 | "/srv/hosts" 294 | ], 295 | "Path": "/usr/local/bin/docker-hosts", 296 | "Created": "2014-10-28T18:22:51.142918682Z", 297 | "Id": "04bf6ca07d2c610235f57b041e224c19b6fab51d400a599ee0f1b1c53e12201f", 298 | "ResolvConfPath": "/etc/resolv.conf", 299 | "HostnamePath": "/var/lib/docker/containers/04bf6ca07d2c610235f57b041e224c19b6fab51d400a599ee0f1b1c53e12201f/hostname", 300 | "HostsPath": "/var/lib/docker/containers/04bf6ca07d2c610235f57b041e224c19b6fab51d400a599ee0f1b1c53e12201f/hosts", 301 | "Name": "/hosts", 302 | "Driver": "devicemapper", 303 | "ExecDriver": "native-0.2", 304 | "MountLabel": "", 305 | "ProcessLabel": "" 306 | }) 307 | 308 | eq_(Manager.containerIsOutOfSync(container_def, container_info, img_info), False) 309 | -------------------------------------------------------------------------------- /docker_sync/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /docker_sync/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- encoding: utf-8 -*- 3 | 4 | ## the sole purpose of this file is so that I can run "python docker_sync" in 5 | ## the root of the project. I don't know how it works, but it does. :-) 6 | 7 | from cli import sync 8 | sync() 9 | -------------------------------------------------------------------------------- /docker_sync/cli.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- encoding: utf-8 -*- 3 | 4 | import argparse 5 | import Manager 6 | 7 | def sync(): 8 | parser = argparse.ArgumentParser(description="synchronize Docker containers") 9 | 10 | parser.add_argument( 11 | "-n", "--no-pull", 12 | dest="pull", 13 | action="store_false", 14 | help="don't pull images", 15 | ) 16 | 17 | parser.add_argument( 18 | "-i", "--allow-insecure-url", 19 | dest="insecure_registry", 20 | action="store_true", 21 | help="allow pulling from non-ssl repositories", 22 | ) 23 | 24 | parser.add_argument( 25 | "-s", "--remove-delay", 26 | dest="remove_delay", 27 | default=None, 28 | type=int, 29 | help="delay between stopping container and removing", 30 | ) 31 | 32 | parser.add_argument( 33 | "config_dir", 34 | help="directory containing yaml config files", 35 | ) 36 | 37 | parsed_args = parser.parse_args() 38 | 39 | Manager.main( 40 | parsed_args.config_dir, 41 | pull=parsed_args.pull, 42 | insecure_registry=parsed_args.insecure_registry, 43 | remove_delay=parsed_args.remove_delay, 44 | ) 45 | 46 | def gen(): 47 | pass 48 | 49 | if __name__ == "__main__": 50 | sync() 51 | -------------------------------------------------------------------------------- /docker_sync/lib/Container.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | 3 | import yaml 4 | 5 | from Image import Image 6 | 7 | class Container(object): 8 | @classmethod 9 | def fromJson(cls, canonical_name, cont_detail): 10 | container = Container(canonical_name, cont_detail["Id"], Image(cont_detail["Image"])) 11 | 12 | container.hostname = cont_detail["Config"]["Hostname"] 13 | container.running = cont_detail["State"]["Running"] 14 | 15 | env_strs = cont_detail["Config"]["Env"] 16 | if env_strs: 17 | container.env = dict([ e.split("=", 1) for e in env_strs]) 18 | 19 | ## remove stuff generated by Docker 20 | for key in ("PATH", "HOME"): 21 | if key in container.env: 22 | del container.env[key] 23 | 24 | if cont_detail["HostConfig"]["PortBindings"]: 25 | for port_def in cont_detail["HostConfig"]["PortBindings"]: 26 | binding = cont_detail["HostConfig"]["PortBindings"][port_def] 27 | 28 | ## binding is none if port not mapped to host 29 | if binding: 30 | if container.ports is None: 31 | container.ports = {} 32 | 33 | ## @todo uh, why is this a list? 34 | binding = binding[0] 35 | 36 | ## HostPort is a number in yaml but string in binding 37 | container.ports[port_def] = { 38 | "HostIp": binding["HostIp"], 39 | "HostPort": int(binding["HostPort"]), 40 | } 41 | 42 | ## { "/container/mount": { "HostPath": "/some/path", "ReadWrite": True}} 43 | if cont_detail["HostConfig"]["Binds"]: 44 | container.volumes = {} 45 | 46 | ## [ "/host/path:/container/path:ro" ] :ro is optional 47 | for bind_info in cont_detail["HostConfig"]["Binds"]: 48 | read_write = True 49 | 50 | if bind_info.endswith(":ro"): 51 | read_write = False 52 | bind_info = bind_info[:-3] 53 | elif bind_info.endswith(":rw"): 54 | read_write = True 55 | bind_info = bind_info[:-3] 56 | 57 | host_path, container_path = bind_info.split(":", 1) 58 | 59 | container.volumes[container_path] = { 60 | "HostPath": host_path, 61 | "ReadWrite": read_write, 62 | } 63 | 64 | container.command.append(cont_detail["Path"]) 65 | container.command.extend(cont_detail["Args"]) 66 | 67 | return container 68 | 69 | def __init__(self, name, id, image): 70 | assert isinstance(image, Image) 71 | 72 | super(Container, self).__init__() 73 | 74 | self._name = name 75 | self._id = id 76 | self.image = image 77 | 78 | self.hostname = None 79 | self.command = [] 80 | self.env = {} 81 | self.ports = None 82 | self.volumes = None 83 | self.running = None 84 | 85 | @property 86 | def name(self): 87 | return self._name 88 | 89 | @property 90 | def id(self): 91 | return self._id 92 | 93 | def toYaml(self): 94 | return yaml.safe_dump({ 95 | "name": self.name, 96 | "image": self.image.tags, 97 | "command": self.command, 98 | "hostname": self.hostname, 99 | "env": self.env, 100 | "ports": self.ports, 101 | "volumes": self.volumes, 102 | }, default_flow_style=False, indent=" ") 103 | -------------------------------------------------------------------------------- /docker_sync/lib/ContainerDefinition.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | 3 | import yaml 4 | 5 | from ImageTag import ImageTag 6 | 7 | class ContainerDefinition(object): 8 | def __init__(self, name, image_tag): 9 | ## containers always have at least one name. the name with only a 10 | ## leading slash shall be the canonical name and all others shall be 11 | ## aliases. 12 | 13 | assert isinstance(image_tag, ImageTag) 14 | 15 | super(ContainerDefinition, self).__init__() 16 | 17 | self._name = name 18 | self.image_tag = image_tag 19 | 20 | self.id = None 21 | 22 | self.hostname = None 23 | self.command = [] 24 | self.env = {} 25 | self.ports = None 26 | self.volumes = None 27 | 28 | @property 29 | def name(self): 30 | return self._name 31 | 32 | @classmethod 33 | def parseFile(cls, ymlFile): 34 | with open(ymlFile, "r") as ifp: 35 | yml = yaml.safe_load(ifp) 36 | 37 | img_tag = ImageTag.parse(yml["image"]) 38 | 39 | container = cls(yml["name"], img_tag) 40 | container.hostname = yml.get("hostname", None) 41 | 42 | container.command = yml.get("command", None) 43 | if container.command: 44 | ## convert all arguments to strings 45 | container.command = [ str(a) for a in container.command ] 46 | 47 | container.env = yml.get("env", {}) 48 | for key in container.env: 49 | ## coerce all values to strings 50 | container.env[key] = str(container.env[key]) 51 | 52 | container.volumes = yml.get("volumes", None) 53 | if container.volumes is not None: 54 | for v in container.volumes: 55 | if "ReadWrite" not in container.volumes[v]: 56 | container.volumes[v]["ReadWrite"] = True 57 | 58 | container.ports = yml.get("ports", None) 59 | if container.ports is not None: 60 | for p in container.ports: 61 | if "HostIp" not in container.ports[p]: 62 | container.ports[p]["HostIp"] = "0.0.0.0" 63 | 64 | return container 65 | -------------------------------------------------------------------------------- /docker_sync/lib/DockerSyncWrapper.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | 3 | import logging 4 | import os 5 | import time 6 | import types 7 | 8 | # http://docs.python-requests.org/en/v1.2.3/ 9 | import requests 10 | 11 | import semantic_version as semver 12 | 13 | import docker as DockerPy 14 | 15 | from ImageTag import ImageTag 16 | from Image import Image 17 | from Container import Container 18 | 19 | class DockerSyncWrapper(object): 20 | """interfaces with the local Docker daemon""" 21 | 22 | API_VERSION = "1.20" 23 | 24 | def __init__(self, docker_host=None): 25 | super(DockerSyncWrapper, self).__init__() 26 | 27 | self.logger = logging.getLogger("Docker") 28 | self.logger.setLevel(logging.DEBUG) 29 | 30 | self.client = DockerPy.Client( 31 | base_url=docker_host if docker_host else os.environ.get("DOCKER_HOST", None), 32 | version=DockerSyncWrapper.API_VERSION, 33 | ) 34 | 35 | self.images = {} 36 | self.containers = {} 37 | 38 | def getImages(self): 39 | """ returns a dict of image_tag => Image instance """ 40 | 41 | self.images = {} 42 | 43 | for img in self.client.images(all=True): 44 | tags = [ ImageTag.parse(t) for t in img["RepoTags"] ] 45 | 46 | image = Image(img["Id"]) 47 | image.tags = tags 48 | 49 | for t in image.tags: 50 | if t: 51 | self.images[t] = image 52 | 53 | return self.images 54 | 55 | def getImage(self, name): 56 | """ returns an Image for the given name, or None if not found """ 57 | retVal = None 58 | 59 | img_data = self.client.inspect_image(name) 60 | 61 | if img_data: 62 | retVal = Image.fromJson(img_data) 63 | retVal.tags = [ name if isinstance(name, ImageTag) else ImageTag.parse(name) ] 64 | 65 | return retVal 66 | 67 | def getContainers(self): 68 | """ returns a dict of container name => Container """ 69 | 70 | ## find all containers 71 | self.containers = {} 72 | 73 | for cont in self.client.containers(all=True): 74 | ## get names without leading "/" 75 | names = [ n[1:] for n in cont["Names"] ] 76 | 77 | ## canonical name has no slashes 78 | canonical_name = [ n for n in names if "/" not in n ][0] 79 | 80 | container = Container.fromJson(canonical_name, self.client.inspect_container(canonical_name)) 81 | self.containers[container.name] = container 82 | 83 | return self.containers 84 | 85 | def removeContainer(self, name, remove_delay=None): 86 | self.logger.info("stopping %s", name) 87 | 88 | self.client.stop(name) 89 | 90 | ## http://blog.hashbangbash.com/2014/11/docker-devicemapper-fix-for-device-or-resource-busy-ebusy/ 91 | ## https://github.com/docker/docker/issues/8176 92 | ## https://github.com/docker/docker/issues/5684 93 | ## attempted workaround is to sleep between stop and remove 94 | if remove_delay is not None: 95 | self.logger.info("delaying %ds before removing" % remove_delay) 96 | time.sleep(remove_delay) 97 | 98 | ## rm -v to remove volumes; we should always explicitly map a volume to 99 | ## the host, so this should be a non-issue. 100 | self.logger.info("removing %s", name) 101 | 102 | self.client.remove_container(name, v=True) 103 | 104 | def startContainer(self, container): 105 | create_container_params = { 106 | "name": container.name, 107 | "detach": True, 108 | "command": container.command, ## can be None 109 | "hostname": container.hostname, ## can be None 110 | "environment": container.env, ## can be None 111 | # "ports": None, 112 | # "volumes": None, 113 | # "environment": None, 114 | } 115 | 116 | start_container_params = { 117 | # "port_bindings": {}, 118 | # "binds": {}, 119 | } 120 | 121 | if container.volumes is not None: 122 | create_container_params["volumes"] = [] 123 | start_container_params["binds"] = {} 124 | 125 | for vol_name in container.volumes: 126 | vol_def = container.volumes[vol_name] 127 | 128 | create_container_params["volumes"].append(vol_name) 129 | 130 | ## https://github.com/dotcloud/docker-py/issues/175 131 | if semver.Version(DockerPy.__version__) < semver.Version("0.3.2"): 132 | start_container_params["binds"][vol_def["HostPath"]] = "%s%s" % ( 133 | vol_name, 134 | ":rw" if vol_def["ReadWrite"] else ":ro", 135 | ) 136 | else: 137 | start_container_params["binds"][vol_def["HostPath"]] = { 138 | "bind": vol_name, 139 | "ro": not vol_def["ReadWrite"], 140 | } 141 | 142 | 143 | if container.ports is not None: 144 | create_container_params["ports"] = [] 145 | start_container_params["port_bindings"] = {} 146 | 147 | for port_spec in container.ports: 148 | port_def = container.ports[port_spec] 149 | 150 | create_container_params["ports"].append(tuple(port_spec.split("/"))) 151 | start_container_params["port_bindings"][port_spec] = ( 152 | port_def["HostIp"], 153 | port_def["HostPort"], 154 | ) 155 | 156 | resp = self.client.create_container(str(container.image_tag), **create_container_params) 157 | container_id = resp["Id"] 158 | 159 | self.logger.info("created container for %s with id %s", container.name, container_id) 160 | 161 | if resp.get("Warnings", None): 162 | for warning in resp["Warnings"]: 163 | self.logger.warn(warning) 164 | 165 | self.client.start(container_id, **start_container_params) 166 | self.logger.info("started container %s", container.name) 167 | 168 | def getImageIdFromRegistry(self, image_tag): 169 | ## http://localhost:5000/v1/repositories/apps/mongodb/tags/latest 170 | ## actually returns a list of every layer with that tag. assuming you'd 171 | ## grab the first, but that seems weird. 172 | 173 | regUrl = "http://%s/v1/repositories/%s/tags" % ( 174 | image_tag.registry if image_tag.registry else "index.docker.io", 175 | image_tag.repository, 176 | ) 177 | 178 | attempts = 0 179 | success = False 180 | while not success and attempts < 3: 181 | attempts += 1 182 | 183 | self.logger.debug("querying registry: " + regUrl) 184 | 185 | try: 186 | start = time.time() 187 | resp = requests.get(regUrl) 188 | duration = time.time() - start 189 | 190 | self.logger.debug("%s %d %.2f", regUrl, resp.status_code, duration) 191 | 192 | success = True 193 | except requests.exceptions.ConnectionError, e: 194 | self.logger.warn("connection error; sleeping 5", exc_info=True) 195 | 196 | time.sleep(5) 197 | 198 | if not success: 199 | self.logger.error("giving up") 200 | raise e 201 | 202 | ## if public registry (index.docker.io): 203 | ## [ {"name":"tag", "layer": ""}, …] 204 | ## else 205 | ## { "tag": "", … } 206 | ## I'm probably looking in the wrong place, but I don't think I can 207 | ## query the registry without authenticating to the index, and that's 208 | ## just a pain in the ass. 209 | layers = resp.json() 210 | 211 | if type(layers) == types.ListType: 212 | layers = dict(map(lambda x: (x["name"], x["layer"]), layers)) 213 | 214 | return layers[image_tag.tag] 215 | 216 | def pullImage(self, image_tag, insecure_registry=False): 217 | # @todo don't need to pull all the images; use self.getImage() 218 | local_images = self.getImages() 219 | 220 | registry_img_id = self.getImageIdFromRegistry(image_tag) 221 | must_pull = True 222 | 223 | if image_tag in local_images and local_images[image_tag].id == registry_img_id: 224 | self.logger.info("%s is up to date", image_tag) 225 | must_pull = False 226 | 227 | if must_pull: 228 | self.logger.info("pulling %s", image_tag) 229 | repoUrl = image_tag.repository 230 | 231 | if image_tag.registry is not None: 232 | repoUrl = "/".join((image_tag.registry, repoUrl)) 233 | 234 | self.client.pull(repoUrl, tag=image_tag.tag, insecure_registry=insecure_registry) 235 | 236 | return self.getImage(str(image_tag)) 237 | -------------------------------------------------------------------------------- /docker_sync/lib/Image.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | 3 | class Image(object): 4 | @classmethod 5 | def fromJson(cls, json): 6 | """creates Image instance from a Docker image JSON dict""" 7 | 8 | img = Image(json["Id"]) 9 | config = json.get("Config", {}) 10 | img.entrypoint = config.get("Entrypoint", None) 11 | img.command = config.get("Cmd", None) 12 | 13 | for env in config.get("Env", []): 14 | k, v = env.split("=", 1) 15 | img.env[k] = v 16 | 17 | for k in ("PATH", "HOME"): 18 | if k in img.env: 19 | del img.env[k] 20 | 21 | return img 22 | 23 | def __init__(self, _id): 24 | super(Image, self).__init__() 25 | 26 | self._id = _id 27 | self._tags = set() 28 | self.entrypoint = None 29 | self.command = None 30 | self.env = {} 31 | 32 | @property 33 | def id(self): 34 | return self._id 35 | 36 | @property 37 | def tags(self): 38 | return self._tags 39 | 40 | @tags.setter 41 | def tags(self, tags): 42 | self._tags = set(tags) 43 | 44 | def __str__(self): 45 | return "" % (self.id, self.tags) 46 | 47 | __repr__ = __str__ 48 | -------------------------------------------------------------------------------- /docker_sync/lib/ImageTag.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | 3 | import re 4 | 5 | class ImageTag(object): 6 | """image tag representation""" 7 | 8 | @classmethod 9 | def parse(cls, tag_str): 10 | """ parses image tags """ 11 | 12 | ## : 13 | ## ubuntu => http://index.docker.io/v1/repositories/ubuntu/tags/latest 14 | ## ubuntu:12.04 15 | ## ubuntu:latest 16 | ## some/repo 17 | ## some/repo:aTag 18 | ## remote.registry:port/some/repo 19 | ## remote.registry:port/some/repo:aTag 20 | 21 | if tag_str == ":": 22 | return None 23 | 24 | 25 | registry = None 26 | repo_path = None 27 | repo_tag = None 28 | 29 | if tag_str.count("/") in (0, 1): 30 | repo_path = tag_str.split(":") 31 | 32 | if len(repo_path) == 2: 33 | repo_path, repo_tag = repo_path 34 | else: 35 | repo_path = repo_path[0] 36 | else: 37 | match = re.match( 38 | r"""(?P([a-z][a-z0-9.-]+|(\d+\.){3}\d+)(:\d+)?)/(?P[^:]+)(:(?P.+))?""", 39 | tag_str 40 | ) 41 | 42 | registry = match.group("registry") 43 | repo_path = match.group("repo_path") 44 | repo_tag = match.group("repo_tag") 45 | 46 | if repo_path: 47 | return ImageTag(repo_path, repo_tag, registry) 48 | 49 | def __init__(self, repository, tag=None, registry=None): 50 | super(ImageTag, self).__init__() 51 | 52 | assert repository 53 | self._repository = str(repository) 54 | 55 | self._tag = str(tag) if tag else "latest" 56 | self._registry = str(registry) if registry else None 57 | 58 | @property 59 | def repository(self): 60 | return self._repository 61 | 62 | @property 63 | def tag(self): 64 | return self._tag 65 | 66 | @property 67 | def registry(self): 68 | return self._registry 69 | 70 | def __hash__(self): 71 | return hash((self.registry, self.repository, self.tag)) 72 | 73 | def __str__(self): 74 | retVal = self.repository 75 | 76 | if self.tag is not None: 77 | retVal = ":".join((retVal, self.tag)) 78 | 79 | if self.registry is not None: 80 | retVal = "/".join((self.registry, retVal)) 81 | 82 | return retVal 83 | 84 | __repr__ = __str__ 85 | 86 | def __cmp__(self, other): 87 | return cmp(repr(self), repr(other)) 88 | -------------------------------------------------------------------------------- /docker_sync/lib/TestContainerDefinition.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | 3 | from ContainerDefinition import ContainerDefinition 4 | from ImageTag import ImageTag 5 | 6 | from nose.tools import eq_ 7 | import os 8 | 9 | class TestContainerDefinition: 10 | EXAMPLE_DIR = os.path.abspath(os.path.join(__file__, "../../../example")) 11 | 12 | def test_parseFile(self): 13 | cdef = ContainerDefinition.parseFile(os.path.join(self.EXAMPLE_DIR, "00-private-registry.yaml")) 14 | 15 | eq_(cdef.name, "private-registry") 16 | eq_(cdef.image_tag, ImageTag("registry", tag="0.8.1")) 17 | eq_(cdef.env["SETTINGS_FLAVOR"], "local") 18 | eq_(cdef.ports["5000/tcp"], { "HostIp": "0.0.0.0", "HostPort": 11003 }) 19 | eq_(cdef.volumes["/var/lib/docker/registry"], { "HostPath": "/tmp", "ReadWrite": True }) 20 | -------------------------------------------------------------------------------- /docker_sync/lib/TestDockerSyncWrapper.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | 3 | import httpretty 4 | import json 5 | import os 6 | from nose.tools import eq_ 7 | 8 | from DockerSyncWrapper import DockerSyncWrapper 9 | from ImageTag import ImageTag 10 | from ContainerDefinition import ContainerDefinition 11 | 12 | class TestDockerSyncWrapper(): 13 | DOCKER_HOST_URL = "http://docker.local:5454" 14 | EXAMPLE_DIR = os.path.abspath(os.path.join(__file__, "../../../example")) 15 | VERSION = "1.20" 16 | @httpretty.activate 17 | def test_getImages(self): 18 | docker = DockerSyncWrapper(docker_host=self.DOCKER_HOST_URL) 19 | 20 | httpretty.register_uri( 21 | httpretty.GET, 22 | self.DOCKER_HOST_URL + "/v%s/images/json" % (self.VERSION), 23 | body=json.dumps([ 24 | { 25 | "RepoTags": [ 26 | "ubuntu:12.04", 27 | "ubuntu:precise", 28 | "ubuntu:latest" 29 | ], 30 | "Id": "8dbd9e392a964056420e5d58ca5cc376ef18e2de93b5cc90e868a1bbc8318c1c", 31 | "Created": 1365714795, 32 | "Size": 131506275, 33 | "VirtualSize": 131506275 34 | }, 35 | { 36 | "RepoTags": [ 37 | "ubuntu:12.10", 38 | "ubuntu:quantal" 39 | ], 40 | "ParentId": "27cf784147099545", 41 | "Id": "b750fe79269d2ec9a3c593ef05b4332b1d1a02a62b4accb2c21d589ff2f5f2dc", 42 | "Created": 1364102658, 43 | "Size": 24653, 44 | "VirtualSize": 180116135 45 | } 46 | ]), 47 | content_type="application/json", 48 | ) 49 | 50 | images = docker.getImages() 51 | 52 | eq_(httpretty.last_request().querystring["all"], ["1"]) 53 | 54 | assert ImageTag.parse("ubuntu:12.10") in images, "tag not found" 55 | 56 | @httpretty.activate 57 | def test_getImageIdFromRegistry(self): 58 | docker = DockerSyncWrapper(docker_host=self.DOCKER_HOST_URL) 59 | 60 | httpretty.register_uri( 61 | httpretty.GET, 62 | "http://index.docker.io/v1/repositories/some/repo/tags", 63 | body=json.dumps([ 64 | { "layer": "84422536", "name": "latest" } 65 | ]), 66 | content_type="application/json", 67 | ) 68 | 69 | img_id = docker.getImageIdFromRegistry(ImageTag("some/repo", tag="latest")) 70 | 71 | eq_("84422536", img_id) 72 | 73 | @httpretty.activate 74 | def test_getContainers(self): 75 | docker = DockerSyncWrapper(docker_host=self.DOCKER_HOST_URL) 76 | 77 | httpretty.register_uri( 78 | httpretty.GET, 79 | self.DOCKER_HOST_URL + "/v%s/containers/json" % (self.VERSION), 80 | body=json.dumps([ 81 | { 82 | "Command":"/bin/sh -c 'exec docker-registry'", 83 | "Created":1413905954, 84 | "Id":"68afa73fe4d5a4012566a24b5f0487fd25b154d66a593b4e67425199487099a5", 85 | "Image":"registry:0.8.1", 86 | "Names":["/pensive_euclid"], 87 | "Ports":[ 88 | {"IP":"0.0.0.0","PrivatePort":5000,"PublicPort":49153,"Type":"tcp"} 89 | ], 90 | "Status":"", 91 | }, 92 | ]), 93 | content_type="application/json", 94 | ) 95 | 96 | httpretty.register_uri( 97 | httpretty.GET, 98 | self.DOCKER_HOST_URL + "/v%s/containers/pensive_euclid/json" % (self.VERSION), 99 | body=json.dumps( 100 | { 101 | "Id": "68afa73fe4d5a4012566a24b5f0487fd25b154d66a593b4e67425199487099a5", 102 | "Created": "2014-10-21T15:39:14.468411999Z", 103 | "Path": "/bin/sh", 104 | "Args": [ "-c", "exec docker-registry" ], 105 | "Config": { 106 | "Hostname": "68afa73fe4d5", 107 | "Domainname": "", 108 | "User": "", 109 | "Memory": 0, 110 | "MemorySwap": 0, 111 | "CpuShares": 0, 112 | "Cpuset": "", 113 | "AttachStdin": False, 114 | "AttachStdout": False, 115 | "AttachStderr": False, 116 | "PortSpecs": None, 117 | "ExposedPorts": { 118 | "5000/tcp": {} 119 | }, 120 | "Tty": False, 121 | "OpenStdin": False, 122 | "StdinOnce": False, 123 | "Env": [ 124 | "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", 125 | "DOCKER_REGISTRY_CONFIG=/docker-registry/config/config_sample.yml", 126 | "SETTINGS_FLAVOR=dev" 127 | ], 128 | "Cmd": [ 129 | "/bin/sh", 130 | "-c", 131 | "exec docker-registry" 132 | ], 133 | "Image": "registry:0.8.1", 134 | "Volumes": None, 135 | "WorkingDir": "", 136 | "Entrypoint": None, 137 | "NetworkDisabled": False, 138 | "OnBuild": None 139 | }, 140 | "State": { 141 | "Running": False, 142 | "Paused": False, 143 | "Pid": 0, 144 | "ExitCode": 0, 145 | "StartedAt": "0001-01-01T00:00:00Z", 146 | "FinishedAt": "0001-01-01T00:00:00Z" 147 | }, 148 | "Image": "326f0493022a886302ca28625533f006517836fb9a0645a7b7da7a220b3e3cf4", 149 | "NetworkSettings": { 150 | "IPAddress": "172.17.0.3", 151 | "IPPrefixLen": 16, 152 | "Gateway": "172.17.42.1", 153 | "Bridge": "docker0", 154 | "PortMapping": None, 155 | "Ports": { 156 | "5000/tcp": [ 157 | { 158 | "HostIp": "0.0.0.0", 159 | "HostPort": "49153" 160 | } 161 | ] 162 | } 163 | }, 164 | "ResolvConfPath": "/etc/resolv.conf", 165 | "HostnamePath": "/var/lib/docker/containers/68afa73fe4d5a4012566a24b5f0487fd25b154d66a593b4e67425199487099a5/hostname", 166 | "HostsPath": "/var/lib/docker/containers/68afa73fe4d5a4012566a24b5f0487fd25b154d66a593b4e67425199487099a5/hosts", 167 | "Name": "/pensive_euclid", 168 | "Driver": "devicemapper", 169 | "ExecDriver": "native-0.2", 170 | "MountLabel": "", 171 | "ProcessLabel": "", 172 | "Volumes": { 173 | "/tmp/registry": "/tmp/registry" 174 | }, 175 | "VolumesRW": { 176 | "/tmp/registry": True 177 | }, 178 | "HostConfig": { 179 | "Binds": [ 180 | "/tmp/registry:/tmp/registry" 181 | ], 182 | "ContainerIDFile": "", 183 | "LxcConf": [], 184 | "Privileged": False, 185 | "PortBindings": { 186 | "5000/tcp": [ 187 | { 188 | "HostIp": "0.0.0.0", 189 | "HostPort": "49153" 190 | } 191 | ] 192 | }, 193 | "Links": None, 194 | "PublishAllPorts": True, 195 | "Dns": None, 196 | "DnsSearch": None, 197 | "VolumesFrom": None, 198 | "NetworkMode": "bridge" 199 | } 200 | } 201 | ), 202 | content_type="application/json", 203 | ) 204 | 205 | containers = docker.getContainers() 206 | 207 | eq_(httpretty.HTTPretty.latest_requests[0].querystring["all"], ["1"]) 208 | 209 | assert "pensive_euclid" in containers 210 | 211 | @httpretty.activate 212 | def test_startContainer(self): 213 | docker = DockerSyncWrapper(docker_host=self.DOCKER_HOST_URL) 214 | cdef = ContainerDefinition.parseFile(os.path.join(self.EXAMPLE_DIR, "30-elasticsearch.yaml")) 215 | 216 | container_id = "e90e34656806" 217 | 218 | httpretty.register_uri( 219 | httpretty.POST, 220 | self.DOCKER_HOST_URL + "/v%s/containers/create" % (self.VERSION), 221 | body=json.dumps( 222 | { 223 | "Id": container_id, 224 | "Warnings": [], 225 | } 226 | ), 227 | content_type="application/json", 228 | status=201, 229 | ) 230 | 231 | httpretty.register_uri( 232 | httpretty.POST, 233 | "%s/v%s/containers/%s/start" % (self.DOCKER_HOST_URL, self.VERSION, container_id), 234 | content_type="text/plain", 235 | status=204, 236 | ) 237 | 238 | docker.startContainer(cdef) 239 | 240 | # create_req = json.loads(httpretty.HTTPretty.latest_requests[0].body) 241 | start_req = json.loads(httpretty.HTTPretty.latest_requests[1].body) 242 | 243 | eq_( 244 | set([ 245 | "/var/lib/docker_container_data/elasticsearch:/var/lib/elasticsearch:rw", 246 | "/var/lib/docker/hosts:/etc/hosts:ro", 247 | ]), 248 | set(start_req["Binds"]), 249 | ) 250 | -------------------------------------------------------------------------------- /docker_sync/lib/TestImage.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | 3 | from Image import Image 4 | 5 | from nose.tools import eq_ 6 | 7 | class TestImage: 8 | def test_fromJson(self): 9 | img = Image.fromJson({ 10 | "Id": "2e2d7133e4a578bd861e85e7195412201f765d050af78c7906841ea62eb6f7dd", 11 | "Parent": "c79dab5561020bda9ce1b1cbe76283fc95f824cfb26a8a21a384993ed7f392bd", 12 | "Created": "2014-10-21T08:50:44.448455269Z", 13 | "Container": "b756100785c797b9f43d36f249b0d5688d88a1ca68df56d915cb436c4bfc7286", 14 | "Config": { 15 | "Hostname": "965c252e48c3", 16 | "User": "", 17 | "Memory": 0, 18 | "MemorySwap": 0, 19 | "AttachStdin": False, 20 | "AttachStdout": False, 21 | "AttachStderr": False, 22 | "PortSpecs": None, 23 | "Tty": False, 24 | "OpenStdin": False, 25 | "StdinOnce": False, 26 | "Env": [ 27 | "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", 28 | "DOCKER_REGISTRY_CONFIG=/docker-registry/config/config_sample.yml", 29 | "SETTINGS_FLAVOR=dev", 30 | "HOME=/" 31 | ], 32 | "Entrypoint": None, 33 | "Cmd": [ "/bin/sh", "-c", "exec", "docker-registry" ], 34 | 35 | "Image": "c79dab5561020bda9ce1b1cbe76283fc95f824cfb26a8a21a384993ed7f392bd", 36 | "Volumes": None, 37 | 38 | "WorkingDir": "", 39 | }, 40 | }) 41 | 42 | eq_(img.id, "2e2d7133e4a578bd861e85e7195412201f765d050af78c7906841ea62eb6f7dd") 43 | eq_(img.entrypoint, None) 44 | eq_(img.command, [ "/bin/sh", "-c", "exec", "docker-registry" ]) 45 | eq_(img.env, { 46 | "DOCKER_REGISTRY_CONFIG": "/docker-registry/config/config_sample.yml", 47 | "SETTINGS_FLAVOR": "dev", 48 | }) 49 | -------------------------------------------------------------------------------- /docker_sync/lib/TestImageTag.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | 3 | from ImageTag import ImageTag 4 | 5 | from nose.tools import eq_ 6 | 7 | class TestImageTag: 8 | def test_untagged(self): 9 | eq_(None, ImageTag.parse(":")) 10 | 11 | def test_official_repo_only(self): 12 | # http://index.docker.io/v1/repositories/ubuntu/tags/latest 13 | eq_(ImageTag(repository="ubuntu", tag="latest"), ImageTag.parse("ubuntu")) 14 | 15 | def test_official_repo_with_simple_tag(self): 16 | eq_(ImageTag(repository="ubuntu", tag="12.04"), ImageTag.parse("ubuntu:12.04")) 17 | 18 | def test_official_repo_with_latest(self): 19 | eq_(ImageTag(repository="ubuntu", tag="latest"), ImageTag.parse("ubuntu:latest")) 20 | 21 | def test_repo_with_path(self): 22 | eq_(ImageTag(repository="some/repo", tag="latest"), ImageTag.parse("some/repo")) 23 | 24 | def test_repo_with_path_and_tag(self): 25 | eq_(ImageTag(repository="some/repo", tag="aTag"), ImageTag.parse("some/repo:aTag")) 26 | 27 | def test_remote_registry(self): 28 | eq_(ImageTag(repository="some/repo", tag="latest", registry="remote.registry:5000"), ImageTag.parse("remote.registry:5000/some/repo")) 29 | 30 | def test_remote_registry_with_tag(self): 31 | eq_(ImageTag(repository="some/repo", tag="aTag", registry="remote.registry:5000"), ImageTag.parse("remote.registry:5000/some/repo:aTag")) 32 | -------------------------------------------------------------------------------- /docker_sync/lib/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | 3 | from Container import Container 4 | from ContainerDefinition import ContainerDefinition 5 | from DockerSyncWrapper import DockerSyncWrapper 6 | from Image import Image 7 | from ImageTag import ImageTag 8 | -------------------------------------------------------------------------------- /example/00-hosts.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: hosts 3 | image: blalor/docker-hosts:latest 4 | 5 | volumes: 6 | /var/run/docker.sock: 7 | HostPath: /var/run/docker.sock 8 | /srv/hosts: 9 | HostPath: /var/lib/docker/hosts 10 | 11 | command: 12 | - --domain-name=dev.docker 13 | - /srv/hosts 14 | -------------------------------------------------------------------------------- /example/00-private-registry.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: private-registry 3 | image: registry:0.8.1 4 | 5 | env: 6 | SETTINGS_FLAVOR: local 7 | STORAGE_PATH: /var/lib/docker/registry 8 | 9 | ports: 10 | 5000/tcp: 11 | HostPort: 11003 12 | 13 | volumes: 14 | /var/lib/docker/registry: 15 | HostPath: /tmp 16 | -------------------------------------------------------------------------------- /example/10-skydns.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: skydns 3 | image: crosbymichael/skydns:latest 4 | command: [ "-nameserver", "8.8.8.8:53", "-domain", "docker" ] 5 | 6 | ports: 7 | 53/udp: 8 | HostIp: 172.17.42.1 9 | HostPort: 53 10 | -------------------------------------------------------------------------------- /example/20-skydock.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: skydock 3 | image: crosbymichael/skydock:latest 4 | 5 | command: 6 | ## the name of the skydns container 7 | - -name 8 | - skydns 9 | 10 | - -ttl 11 | - 30 12 | 13 | - -environment 14 | - dev 15 | 16 | - -s 17 | - /docker.sock 18 | 19 | - -domain 20 | - docker 21 | 22 | env: 23 | GOPATH: /go 24 | GOROOT: /usr/local/go 25 | 26 | volumes: 27 | /docker.sock: 28 | HostPath: /var/run/docker.sock 29 | -------------------------------------------------------------------------------- /example/30-elasticsearch.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: elasticsearch 3 | image: autosportlabs/elasticsearch:latest 4 | 5 | volumes: 6 | /etc/hosts: 7 | HostPath: /var/lib/docker/hosts 8 | ReadWrite: false 9 | /var/lib/elasticsearch: 10 | HostPath: /var/lib/docker_container_data/elasticsearch 11 | 12 | env: 13 | ES_HEAP_SIZE: 2048m 14 | SERVICE_9200_NAME: elasticsearch 15 | SERVICE_9200_TAGS: "rest,http" 16 | -------------------------------------------------------------------------------- /packaging/rpm/redhat/README.md: -------------------------------------------------------------------------------- 1 | # Building the RPM 2 | 3 | This spec file stages and builds the `python-docker-sync` rpm that is used by the ASL system. The steps below show you how to build the RPM file. 4 | 5 | ## Steps 6 | 1. Boot up the vagrant image found in the puppet repo. 7 | 2. Install the `rpmdevtools` and `rpm-build` packages if not already installed. 8 | 3. As the non root user, run `rpmdev-setuptree`. This builds the file structure you will need under the `~/rpmbuild` directory. 9 | 4. Copy the spec file into `~/rpmuild/SPECS`. 10 | 5. Use curl or wget to get the version of the source you are trying to build. The URL for this source is in the `Source:` tag in the spec file. Ensure this zip file ends up in the `SOURCES` directory. 11 | 6. Run `rpmbuild -ba ~/rpmbuild/SPECS/python-docker-sync.spec`. This should build both the SRPM and RPM files. Look in SRPM and RPM directories to find them. 12 | -------------------------------------------------------------------------------- /packaging/rpm/redhat/python-docker-sync.spec: -------------------------------------------------------------------------------- 1 | # Spec file for python-docker-sync. 2 | # 3 | # Copyright (c) 2015 Andrew Stiegmann (andrew.stiegmann@gmail.com) 4 | # 5 | 6 | Name: python-docker-sync 7 | Version: 1.2.5 8 | Release: 1%{?dist} 9 | Summary: Docker container sync and management utility 10 | License: UNKNOWN 11 | Group: Applications/Internet 12 | Url: https://github.com/autosportlabs/docker-sync 13 | Source: https://github.com/autosportlabs/docker-sync/archive/v%{version}.zip 14 | 15 | BuildRequires: python2-devel 16 | Requires: python2 17 | Requires: python-docker-py >= 0.6.0 18 | Requires: PyYAML >= 3.10 19 | Requires: python-semantic_version >= 2.3.1 20 | Requires: docker >= 1.8.0 21 | 22 | %{?python_provide: %python_provide python-%{srcname}} 23 | 24 | BuildRoot: %{_tmppath}/%{name}-%{version}-build 25 | BuildArch: noarch 26 | 27 | 28 | %description 29 | A utility written to sync and spin up the various docker containers 30 | referenced by a directory provided on the command line. 31 | 32 | 33 | %prep 34 | %setup -qn docker-sync-%{version} 35 | 36 | 37 | %build 38 | # Nothing to do 39 | 40 | 41 | %install 42 | mkdir -p %{buildroot}%{python2_sitelib}/docker_sync/lib 43 | install -m0644 docker_sync/*.py %{buildroot}%{python2_sitelib}/docker_sync/ 44 | install -m0644 docker_sync/lib/*.py %{buildroot}%{python2_sitelib}/docker_sync/lib 45 | 46 | mkdir -p %{buildroot}%{_bindir} 47 | install -m0755 scripts/docker-sync %{buildroot}%{_bindir}/docker-sync 48 | 49 | 50 | %files 51 | %{_bindir}/docker-sync 52 | %{python2_sitelib}/docker_sync 53 | 54 | 55 | %changelog 56 | * Fri Jan 22 2016 Stieg - 1.2.5-1.el7.centos 57 | - Update package to support Docker API version 1.20 from 1.8 58 | - Add new Docker dependency >= 1.8.0 since that is when API v 1.20 was 59 | introduced. 60 | 61 | * Thu Oct 29 2015 Stieg - 1.2.4-1.el7.centos 62 | - Initial creation of the RHEL 7 spec file 63 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | PyYAML >= 3.10, < 4.0 2 | argparse >= 1.1 3 | docker-py >= 0.6.0, < 0.7.0 4 | semantic_version >= 2.3.1, < 2.4.0 5 | -------------------------------------------------------------------------------- /scripts/docker-sync: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- encoding: utf-8 -*- 3 | 4 | from docker_sync.cli import sync 5 | 6 | sync() 7 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- encoding: utf-8 -*- 3 | 4 | from distutils.core import setup 5 | 6 | setup( 7 | name="docker-sync", 8 | version="1.2.4", 9 | description="Configuration management for Docker containers", 10 | long_description=open("README.md").read(), 11 | author="Brian Lalor", 12 | author_email="brian@autosportlabs.com", 13 | url="http://github.com/autosportlabs/docker-sync", 14 | packages=[ "docker_sync", "docker_sync.lib" ], 15 | scripts=[ "scripts/docker-sync" ], 16 | ) 17 | --------------------------------------------------------------------------------