├── tests ├── __init__.py ├── test_legacy_file_arg.py ├── test_merge.py ├── test_main.py └── test_updater.py ├── diu ├── __init__.py ├── merge.py ├── updater.py └── main.py ├── requirements ├── base └── development ├── MANIFEST.in ├── tox.ini ├── setup.cfg ├── Makefile ├── .travis.yml ├── .gitignore ├── LICENSE ├── setup.py └── README.rst /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /diu/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "1.0.0" 2 | -------------------------------------------------------------------------------- /requirements/base: -------------------------------------------------------------------------------- 1 | attrs>=15.0.0a1 2 | colorlog 3 | docker-py 4 | pyyaml 5 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include requirements * 2 | recursive-include tests * 3 | include *.rst 4 | -------------------------------------------------------------------------------- /requirements/development: -------------------------------------------------------------------------------- 1 | --requirement base 2 | bumpversion 3 | flake8 4 | mock 5 | pytest 6 | pytest-capturelog 7 | pytest-cov 8 | tox 9 | wheel 10 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27,py33,py34 3 | skip_missing_interpreters = True 4 | 5 | [testenv] 6 | commands = make testlocal 7 | deps = 8 | -rrequirements/development 9 | whitelist_externals = 10 | make 11 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 1.0.0 3 | commit = True 4 | tag = True 5 | 6 | [flake8] 7 | max-line-length = 100 8 | exclude = tests/*,.tox/* 9 | 10 | [bumpversion:file:setup.py] 11 | 12 | [bumpversion:file:diu/__init__.py] 13 | 14 | [bdist_wheel] 15 | universal = 1 16 | 17 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: help test testlocal 2 | 3 | ifneq ($(strip $(TOX_RECREATE)),) 4 | TOX = tox --recreate 5 | else 6 | TOX = tox 7 | endif 8 | 9 | help: 10 | @echo "help: Print this help message" 11 | @echo "test: Run tests" 12 | @echo "testlocal: Run tests locally without using tox" 13 | 14 | test: 15 | @$(TOX) 16 | 17 | testlocal: 18 | @py.test --cov diu --cov-report term --cov-report html --cov-report xml -v 19 | @flake8 --show-source --statistics 20 | -------------------------------------------------------------------------------- /tests/test_legacy_file_arg.py: -------------------------------------------------------------------------------- 1 | from diu.main import Application as RealApplication 2 | 3 | 4 | class Application(RealApplication): 5 | def _load_config(self, *files): 6 | self.config = {} 7 | self.given_config_files = files 8 | 9 | 10 | class TestApplication(object): 11 | def test_deprecated_file_argument_takes_precedence(self): 12 | app = Application(["--file", "test.yaml"]) 13 | assert app.given_config_files == ("test.yaml",) 14 | 15 | def test_no_file_arg_uses_etc_docker_image_updater_yml(self): 16 | app = Application([]) 17 | assert app.given_config_files == ("/etc/docker-image-updater.yml",) 18 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | sudo: false 3 | python: 4 | - "2.7" 5 | - "3.3" 6 | - "3.4" 7 | - "3.5" 8 | - "pypy" 9 | - "pypy3" 10 | - "nightly" 11 | 12 | install: 13 | - pip install -U -r requirements/development python-coveralls 14 | # inspect.getargspec is removed with Python 3.6 onwards, not all libraries like this yet. 15 | - env | grep TRAVIS_PYTHON_VERSION 16 | - if [[ $TRAVIS_PYTHON_VERSION == 'nightly' ]]; then pip install -U 'coverage>=4.0.2'; fi 17 | # There's no pypi release for pep8 that works with 3.6 at the time of this writing 18 | - if [[ $TRAVIS_PYTHON_VERSION == 'nightly' ]]; then pip install -U https://github.com/PyCQA/pep8/archive/master.zip; fi 19 | script: 20 | - make testlocal 21 | after_success: 22 | - coveralls 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | *.egg-info/ 23 | .installed.cfg 24 | *.egg 25 | 26 | # PyInstaller 27 | # Usually these files are written by a python script from a template 28 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 29 | *.manifest 30 | *.spec 31 | 32 | # Installer logs 33 | pip-log.txt 34 | pip-delete-this-directory.txt 35 | 36 | # Unit test / coverage reports 37 | htmlcov/ 38 | .tox/ 39 | .coverage 40 | .cache 41 | nosetests.xml 42 | coverage.xml 43 | 44 | # Translations 45 | *.mo 46 | *.pot 47 | 48 | # Django stuff: 49 | *.log 50 | 51 | # Sphinx documentation 52 | docs/_build/ 53 | 54 | # PyBuilder 55 | target/ 56 | 57 | # IPython 58 | .ipynb_checkpoints/ 59 | 60 | # PyCharm IDE 61 | .idea/ 62 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Nick Groenen 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /diu/merge.py: -------------------------------------------------------------------------------- 1 | from copy import deepcopy 2 | 3 | 4 | def merge(a, b): 5 | """ 6 | Recursively merge datasets of a and b together so that: 7 | 8 | * Dicts are (recursively) merged 9 | * Lists are appended together 10 | * Other types take the value from b 11 | 12 | A ValueError is raised when trying to merge two items of 13 | a different type. 14 | """ 15 | if type(a) != type(b): 16 | raise ValueError( 17 | "Trying to merge two different types of data:\n" 18 | "Value A {a_type} = {a_value}\n\n" 19 | "Value B {b_type} = {b_value}\n".format( 20 | a_type=type(a), 21 | a_value=a, 22 | b_type=type(b), 23 | b_value=b, 24 | ) 25 | ) 26 | 27 | result = None 28 | if isinstance(b, list): 29 | result = list(set(a + b)) 30 | elif isinstance(b, dict): 31 | result = deepcopy(a) 32 | for k, v in b.items(): 33 | if k in result: 34 | result[k] = merge(result[k], v) 35 | else: 36 | result[k] = deepcopy(v) 37 | else: 38 | return b 39 | return result 40 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pkg_resources import parse_requirements 3 | from setuptools import setup 4 | 5 | 6 | def read(file): 7 | """Read file relative to the current directory and return it's contents""" 8 | return open(os.path.join(os.path.dirname(__file__), file)).read() 9 | 10 | 11 | def read_requirements(file): 12 | """Return a list of requirements from file relative to current directory""" 13 | with open(os.path.join(os.path.dirname(__file__), file)) as f: 14 | return [str(r) for r in parse_requirements(f)] 15 | 16 | 17 | setup( 18 | name="docker-image-updater", 19 | version="1.0.0", 20 | author="Nick Groenen", 21 | author_email="nick@groenen.me", 22 | description="Update docker images and trigger commands in response to updates", 23 | long_description=read('README.rst'), 24 | license="MIT", 25 | keywords="docker image update container", 26 | url="https://github.com/zoni/docker-image-updater", 27 | install_requires=read_requirements("requirements/base"), 28 | classifiers=[ 29 | "Development Status :: 5 - Production/Stable", 30 | "Topic :: Utilities", 31 | "Topic :: System :: Systems Administration", 32 | "Intended Audience :: System Administrators", 33 | "License :: OSI Approved :: MIT License", 34 | "Operating System :: POSIX", 35 | "Programming Language :: Python :: 2.7", 36 | "Programming Language :: Python :: 3", 37 | ], 38 | packages=['diu'], 39 | entry_points={ 40 | 'console_scripts': ['docker-image-updater=diu.main:main'] 41 | }, 42 | include_package_data=True, 43 | ) 44 | -------------------------------------------------------------------------------- /tests/test_merge.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from diu.merge import merge 3 | 4 | 5 | def test_shallow_dict_merge(): 6 | a = { 7 | 'one': [1, 2, 3] 8 | } 9 | b = { 10 | 'two': [2, 3, 4] 11 | } 12 | 13 | expected = { 14 | 'one': [1, 2, 3], 15 | 'two': [2, 3, 4], 16 | } 17 | assert merge(a, b) == expected 18 | 19 | a = { 20 | 'one': [1, 2, 3] 21 | } 22 | b = {} 23 | 24 | expected = { 25 | 'one': [1, 2, 3], 26 | } 27 | assert merge(a, b) == expected 28 | 29 | 30 | def test_deep_dict_merge(): 31 | a = { 32 | 'one': [1, 2, 3], 33 | 'three': {'one': 1}, 34 | } 35 | b = { 36 | 'two': [2, 3, 4], 37 | 'three': {'two': 2}, 38 | } 39 | 40 | expected = { 41 | 'one': [1, 2, 3], 42 | 'two': [2, 3, 4], 43 | 'three': { 44 | 'one': 1, 45 | 'two': 2, 46 | }, 47 | } 48 | assert merge(a, b) == expected 49 | 50 | 51 | def test_list_within_dict_merge(): 52 | a = { 53 | 'one': [1, 2], 54 | } 55 | b = { 56 | 'one': [2, 3], 57 | } 58 | 59 | expected = { 60 | 'one': [1, 2, 3], 61 | } 62 | assert merge(a, b) == expected 63 | 64 | 65 | def test_list_merge(): 66 | a = [1, 2] 67 | b = [2, 3] 68 | expected = [1, 2, 3] 69 | assert merge(a, b) == expected 70 | 71 | a = [1, 2] 72 | b = [] 73 | expected = [1, 2] 74 | assert merge(a, b) == expected 75 | 76 | 77 | def test_str_merge(): 78 | a = "some" 79 | b = "string" 80 | assert merge(a, b) == b 81 | 82 | 83 | def test_int_merge(): 84 | a = 1 85 | b = 2 86 | assert merge(a, b) == b 87 | 88 | 89 | def test_different_types_merge(): 90 | a = {} 91 | b = [1] 92 | with pytest.raises(ValueError): 93 | assert merge(a, b) 94 | -------------------------------------------------------------------------------- /tests/test_main.py: -------------------------------------------------------------------------------- 1 | import mock 2 | import pytest 3 | import yaml 4 | from diu.main import Application 5 | from diu.updater import ContainerSet, Updater 6 | 7 | 8 | class TestApplication(object): 9 | @pytest.fixture 10 | def app(self, tmpdir, config=None): 11 | f = tmpdir.join("config.yml") 12 | if config is None: 13 | config = { 14 | 'config': { 15 | 'docker': { 16 | 'version': '1.16', 17 | 'base_url': 'unix://var/run/docker.sock', 18 | } 19 | }, 20 | 'watch': { 21 | 'ubuntu': { 22 | 'images': ['ubuntu:latest', 'ubuntu:14.04'], 23 | 'commands': ['foo', 'bar'], 24 | }, 25 | } 26 | } 27 | yaml.dump(config, f.open('w')) 28 | return Application(args=[str(f)]) 29 | 30 | @pytest.fixture 31 | def multiconfig_app(self, tmpdir): 32 | f1 = tmpdir.join("config1.yml") 33 | f2 = tmpdir.join("config2.yml") 34 | 35 | config1 = { 36 | 'config': { 37 | 'docker': { 38 | 'version': '1.16', 39 | 'base_url': 'unix://var/run/docker.sock', 40 | } 41 | }, 42 | 'watch': { 43 | 'ubuntu': { 44 | 'images': ['ubuntu:latest', 'ubuntu:14.04'], 45 | 'commands': ['foo', 'bar'], 46 | }, 47 | } 48 | } 49 | config2 = { 50 | 'config': { 51 | 'docker': { 52 | 'version': '1.17', 53 | 'base_url': 'unix://var/run/docker.sock', 54 | } 55 | }, 56 | 'watch': { 57 | 'ubuntu': { 58 | 'images': ['ubuntu:latest', 'ubuntu:15.04'], 59 | 'commands': ['baz'], 60 | }, 61 | 'debian': { 62 | 'images': ['debian:squeeze'], 63 | 'commands': ['foo'] 64 | } 65 | } 66 | } 67 | 68 | yaml.dump(config1, f1.open('w')) 69 | yaml.dump(config2, f2.open('w')) 70 | 71 | return Application(args=[str(f1), str(f2)]) 72 | 73 | 74 | def test_config_is_loaded_from_config_file(self, app): 75 | assert app.config == { 76 | 'docker': { 77 | 'version': '1.16', 78 | 'base_url': 'unix://var/run/docker.sock', 79 | } 80 | } 81 | 82 | def test_containerset_is_loaded_from_config_file(self, app): 83 | assert len(app.containerset) == 1 84 | assert isinstance(app.containerset[0], ContainerSet) 85 | assert app.containerset[0].name == "ubuntu" 86 | 87 | def test_updater_instance_is_initialized_during_init(self, app): 88 | assert isinstance(app.updater, Updater) 89 | 90 | def test_run_calls_updater_do_updates(self, tmpdir): 91 | with mock.patch('diu.main.Updater') as m: 92 | app = self.app(tmpdir=tmpdir) 93 | app.updater.error_count = 0 94 | app.run() 95 | app.updater.do_updates.assert_called_once_with() 96 | 97 | def test_docker_client_initialized_with_params_from_config(self, tmpdir): 98 | with mock.patch('diu.main.DockerClient') as m: 99 | app = self.app(tmpdir=tmpdir) 100 | m.assert_called_once_with( 101 | version='1.16', 102 | base_url='unix://var/run/docker.sock' 103 | ) 104 | 105 | def test_config_loading_with_multiple_configs(self, multiconfig_app): 106 | app = multiconfig_app 107 | assert len(app.containerset) == 2 108 | assert isinstance(app.containerset[0], ContainerSet) 109 | assert isinstance(app.containerset[1], ContainerSet) 110 | 111 | if app.containerset[0].name == "debian": 112 | debian = app.containerset[0] 113 | ubuntu = app.containerset[1] 114 | else: 115 | debian = app.containerset[1] 116 | ubuntu = app.containerset[0] 117 | 118 | assert debian.name == "debian" 119 | assert debian.images == ['debian:squeeze'] 120 | assert debian.commands == ['foo'] 121 | assert ubuntu.name == "ubuntu" 122 | assert sorted(ubuntu.images) == sorted(['ubuntu:14.04', 'ubuntu:latest', 'ubuntu:15.04']) 123 | assert sorted(ubuntu.commands) == sorted(['baz', 'foo', 'bar']) 124 | 125 | def test_shady_configs(self, tmpdir): 126 | self.app(tmpdir=tmpdir, config={}) 127 | self.app(tmpdir=tmpdir, config={'watch': {}}) 128 | self.app(tmpdir=tmpdir, config={'watch': {"myapp": {}}}) 129 | with pytest.raises(SystemExit): 130 | self.app(tmpdir=tmpdir, config={'config': []}) 131 | with pytest.raises(SystemExit): 132 | self.app(tmpdir=tmpdir, config={'config': "foo"}) 133 | with pytest.raises(SystemExit): 134 | self.app(tmpdir=tmpdir, config={'watch': []}) 135 | with pytest.raises(SystemExit): 136 | self.app(tmpdir=tmpdir, config={'watch': "myapp"}) 137 | with pytest.raises(SystemExit): 138 | self.app(tmpdir=tmpdir, config={'watch': {"myapp": []}}) 139 | -------------------------------------------------------------------------------- /diu/updater.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function, absolute_import, unicode_literals, division 2 | import attr 3 | import logging 4 | import subprocess 5 | import sys 6 | from docker.errors import APIError 7 | 8 | 9 | @attr.s 10 | class ContainerSet(object): 11 | """ 12 | The definition of a set of Docker images which should be watched 13 | for updates with a set of actions to be executed when a newer version 14 | of the image is pulled. 15 | 16 | :param name: 17 | A unique name for this watcher. 18 | :param images: 19 | A list of Docker images to watch for updates. 20 | """ 21 | name = attr.ib() 22 | images = attr.ib(default=attr.Factory(list)) 23 | commands = attr.ib(default=attr.Factory(list)) 24 | 25 | 26 | class Updater(object): 27 | """ 28 | The docker image updater. 29 | """ 30 | 31 | def __init__(self, client, containerset): 32 | """ 33 | :param client: 34 | The Docker client to use (a docker.Client instance) 35 | :param containerset: 36 | A list of ContainerSet instances. 37 | """ 38 | self.client = client 39 | self.containerset = {x.name: x for x in containerset} 40 | self.logger = logging.getLogger(self.__class__.__name__) 41 | self._updated = [] # Tracks updated images 42 | self.error_count = 0 43 | 44 | def _pull_docker_image(self, image): 45 | """ 46 | Pull the given docker image, printing data to stdout to 47 | keep the user informed of progress. 48 | :param image: 49 | The name of the image to pull down. 50 | """ 51 | self.logger.info("Pulling image {}".format(image)) 52 | attached_to_tty = sys.stdout.isatty() 53 | 54 | for _ in self.client.pull(image, stream=True): 55 | if not attached_to_tty: 56 | continue 57 | sys.stdout.write('.') 58 | sys.stdout.flush() 59 | 60 | if attached_to_tty: 61 | sys.stdout.write("\n") 62 | 63 | def _update_image(self, image): 64 | """ 65 | Update the given docker image. 66 | 67 | :param image: 68 | The image to update, in the form of `ubuntu` or `ubuntu:latest`. 69 | :returns: 70 | True if the image is updated, False if it is already the latest version. 71 | """ 72 | try: 73 | self.logger.debug("Inspecting image {}".format(image)) 74 | image_id = self.client.inspect_image(image)['Id'] 75 | self.logger.debug("Image id: {}".format(image_id)) 76 | except APIError as e: 77 | if e.response.status_code == 404: 78 | self.logger.warning( 79 | "404 response from docker API, assuming image does not " 80 | "exist locally" 81 | ) 82 | image_id = None 83 | else: 84 | raise 85 | 86 | self._pull_docker_image(image) 87 | self.logger.debug("New image id: {}".format(image_id)) 88 | if image_id != self.client.inspect_image(image)['Id']: 89 | self.logger.debug("Image IDs differ before and after pull, image was updated") 90 | self._updated.append(image) 91 | return True 92 | 93 | self.logger.debug("Image IDs identical before and after pull") 94 | return False 95 | 96 | def _update(self, watcher): 97 | """ 98 | Update the containers configured by the supplied watcher and 99 | execute post-update actions as needed. 100 | 101 | :param watcher: 102 | An ContainerSet instance. 103 | """ 104 | updated = False 105 | for image in watcher.images: 106 | if image in self._updated: 107 | updated = True 108 | continue 109 | try: 110 | self.logger.info("Updating image {}".format(image)) 111 | updated_ = self._update_image(image) 112 | except Exception: 113 | self.logger.exception("Exception occurred during update of {}".format(image)) 114 | self.error_count += 1 115 | continue 116 | if updated_: 117 | self.logger.info("Image {} updated to latest version".format(image)) 118 | updated = True 119 | else: 120 | self.logger.info("Image {} already at latest version".format(image)) 121 | 122 | if not updated: 123 | self.logger.debug("No images in this set updated") 124 | return 125 | 126 | self.logger.debug("One or more images in this set updated") 127 | for command in watcher.commands: 128 | try: 129 | self._run_command(command) 130 | except Exception: 131 | self.logger.exception("Exception occurred during command execution") 132 | self.error_count += 1 133 | continue 134 | 135 | def _run_command(self, command): 136 | """ 137 | Run given command in a shell. 138 | """ 139 | self.logger.info("Running command: {}".format(command)) 140 | p = subprocess.Popen(command, shell=True) 141 | returncode = p.wait() 142 | if returncode == 0: 143 | self.logger.info("Command exited successfully") 144 | else: 145 | self.logger.error("Command exited with non-zero exit code {}".format(returncode)) 146 | self.error_count += 1 147 | 148 | def do_updates(self): 149 | """ 150 | Update the watched images. 151 | """ 152 | for watcher in self.containerset.values(): 153 | self.logger.info("Checking images in set {}".format(watcher.name)) 154 | self._update(watcher) 155 | -------------------------------------------------------------------------------- /diu/main.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function, absolute_import, unicode_literals, division 2 | 3 | import argparse 4 | import sys 5 | import logging 6 | import yaml 7 | 8 | from diu.merge import merge 9 | from diu.updater import ContainerSet, Updater 10 | from docker import Client as DockerClient 11 | 12 | try: 13 | from colorlog import ColoredFormatter 14 | except ImportError: 15 | ColoredFormatter = None 16 | 17 | 18 | class Application(object): 19 | """ 20 | The docker image updater application. 21 | """ 22 | 23 | def __init__(self, args=None): 24 | """ 25 | :param args: 26 | Program arguments. If not supplied, will use sys.argv 27 | """ 28 | self.logger = logging.getLogger(self.__class__.__name__) 29 | self.parser = self._create_parser() 30 | self.args = self.parser.parse_args(args=args) 31 | if self.args.debug: 32 | logging.getLogger().setLevel(logging.DEBUG) 33 | 34 | self.containerset = [] 35 | 36 | if self.args.deprecated_file is not None: 37 | self.logger.warning( 38 | "--file is deprecated, please migrate to using positional" 39 | " arguments instead" 40 | ) 41 | self._load_config(self.args.deprecated_file) 42 | else: 43 | self._load_config(*self.args.file) 44 | 45 | d = DockerClient(**self.config.get('docker', {})) 46 | self.updater = Updater(d, self.containerset) 47 | 48 | def _create_parser(self): 49 | """ 50 | Create the argparse argument parser. 51 | 52 | :returns: 53 | An instance of `argparse.ArgumentParser()` 54 | """ 55 | parser = argparse.ArgumentParser() 56 | parser.add_argument( 57 | "-f", "--file", 58 | dest="deprecated_file", 59 | metavar="FILE", 60 | default=None, 61 | help="deprecated - this flag will be removed in the future", 62 | ) 63 | parser.add_argument( 64 | "--debug", 65 | action="store_true", 66 | help="show debug messages" 67 | ) 68 | parser.add_argument( 69 | "file", 70 | help="configuration file(s) to use", 71 | nargs="*", 72 | default=("/etc/docker-image-updater.yml",), 73 | ) 74 | return parser 75 | 76 | def _load_config(self, *files): 77 | """ 78 | Load and parse the given configuration file. 79 | 80 | :param files: 81 | One or more sets of configuration files to load. 82 | """ 83 | final_config = {'config': {}, 'watch': {}} 84 | for f in files: 85 | try: 86 | data = yaml.safe_load(open(f)) 87 | except (IOError, yaml.parser.ParserError) as e: 88 | print("Error loading {f}: {e!s}".format(f=f, e=e), file=sys.stderr) 89 | sys.exit(1) 90 | try: 91 | final_config = merge(final_config, data) 92 | except ValueError as e: 93 | print( 94 | "You have an error in the configuration file {f}: {e!s}".format(f=f, e=e), 95 | file=sys.stderr 96 | ) 97 | sys.exit(1) 98 | 99 | self.config = final_config['config'] 100 | for key, value in final_config['watch'].items(): 101 | try: 102 | self._validate_watch_configuration(value) 103 | except ValueError as e: 104 | print( 105 | "You have an error in the configuration file {f}: {e!s}".format(f=f, e=e), 106 | file=sys.stderr 107 | ) 108 | sys.exit(1) 109 | self.containerset.append(ContainerSet( 110 | name=key, 111 | images=value.get('images', []), 112 | commands=value.get('commands', []), 113 | )) 114 | 115 | def _validate_watch_configuration(self, watch): 116 | """ 117 | Validate the structure of a 'watch' statement. 118 | """ 119 | if not isinstance(watch, dict): 120 | raise ValueError("Key 'watch' should be a dictionary") 121 | if not isinstance(watch.get('images', []), list): 122 | raise ValueError("Key 'images' should be of type list") 123 | if not isinstance(watch.get('commands', []), list): 124 | raise ValueError("Key 'commands' should be of type list") 125 | 126 | def run(self): 127 | """ 128 | Run the application. 129 | """ 130 | self.updater.do_updates() 131 | if self.updater.error_count > 0: 132 | sys.exit(1) 133 | 134 | 135 | def setup_logger(): 136 | logger = logging.getLogger() 137 | logger.setLevel(logging.INFO) 138 | console_handler = logging.StreamHandler(sys.stdout) 139 | if sys.stdout.isatty() and ColoredFormatter is not None: 140 | formatter = ColoredFormatter( 141 | "%(asctime)s %(log_color)s%(levelname)-8s%(reset)s " 142 | "%(cyan)s%(name)-10s%(reset)s %(white)s%(message)s%(reset)s", 143 | datefmt="%H:%M:%S", 144 | reset=True, 145 | log_colors={ 146 | 'DEBUG': 'blue', 147 | 'INFO': 'green', 148 | 'WARNING': 'yellow', 149 | 'ERROR': 'red', 150 | 'CRITICAL': 'red', 151 | } 152 | ) 153 | else: 154 | formatter = logging.Formatter( 155 | "%(asctime)s %(levelname)-8s %(name)-10s %(message)s", 156 | datefmt="%H:%M:%S", 157 | ) 158 | console_handler.setFormatter(formatter) 159 | logger.addHandler(console_handler) 160 | 161 | 162 | def main(): 163 | setup_logger() 164 | Application().run() 165 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Docker image updater 2 | ==================== 3 | 4 | *A utility to update docker images automatically. Supports execution of 5 | arbitrary commands when an image is updated, in order to restart running 6 | containers or trigger other custom behaviour.* 7 | 8 | .. image:: https://travis-ci.org/zoni/docker-image-updater.svg?branch=master 9 | :target: https://travis-ci.org/zoni/docker-image-updater 10 | .. image:: https://coveralls.io/repos/zoni/docker-image-updater/badge.svg?branch=master 11 | :target: https://coveralls.io/r/zoni/docker-image-updater 12 | 13 | 14 | Installation 15 | ------------ 16 | 17 | Docker image updater can be installed from the 18 | `Python Package Index `_ 19 | using:: 20 | 21 | pip install docker-image-updater 22 | 23 | Installation into a `virtualenv `_ 24 | is highly recommended! 25 | 26 | 27 | Usage 28 | ----- 29 | 30 | :: 31 | 32 | usage: docker-image-updater [-h] [-f FILE] [--debug] [file [file ...]] 33 | 34 | positional arguments: 35 | file configuration file(s) to use 36 | 37 | optional arguments: 38 | -h, --help show this help message and exit 39 | -f FILE, --file FILE deprecated - this flag will be removed in the future 40 | --debug show debug messages 41 | 42 | 43 | Docker image updater requires one or more configuration files which specify 44 | sets of images to watch and commands to execute. By default it will look 45 | for `/etc/docker-image-updater.yml` but you may give one or more alternate 46 | files on the command-line. 47 | 48 | When specifying more than one configuration file, the settings will be 49 | merged together with items from the latter configuration file(s) overwriting 50 | items from earlier files. 51 | 52 | Recommended usage is to run docker image updater from cron, using 53 | something like `cronic `_ to receive mail 54 | only in case of errors. 55 | 56 | 57 | Example output 58 | -------------- 59 | 60 | :: 61 | 62 | # docker-image-updater 63 | 22:13:04 INFO Updater Checking images in set jenkins 64 | 22:13:04 INFO Updater Updating image zoni/jenkins 65 | 22:13:04 INFO Updater Pulling image zoni/jenkins 66 | ........................................................................................................................................................................................................................................................................................................................... 67 | 22:14:50 INFO Updater Image zoni/jenkins updated to latest version 68 | 22:14:50 INFO Updater Running command: supervisorctl restart jenkins 69 | jenkins: stopped 70 | jenkins: started 71 | 22:14:54 INFO Updater Command exited successfully 72 | 73 | 74 | Configuration format 75 | -------------------- 76 | 77 | Configuration is expressed through a YAML file such as the following: 78 | 79 | :: 80 | 81 | config: 82 | docker: 83 | base_url: "unix://var/run/docker.sock" 84 | version: "1.16" 85 | watch: 86 | my-app: 87 | images: 88 | - my-app 89 | - redis 90 | commands: 91 | - restart my-app 92 | 93 | The item `watch` defines sets of images to watch. This is a dictionary where 94 | the keys (`my-app` in the example above) are arbitrary values for human 95 | reference. Under each of these keys a dictonary with the items `images` and 96 | `commands` is expected. 97 | 98 | `images` defines a list of docker images to check for updates. You can 99 | specify these as `image:tag` or simply as `image`, in which case Docker will 100 | use the *latest* tag automatically. 101 | 102 | `commands` defines a list of shell commands to execute whenever one of the 103 | listed images was updated. These will be run sequentially, in order. 104 | 105 | All items under `config.docker` are passed to the Docker client. 106 | For supported options, refer to the 107 | `docker-py documentation `_. 108 | 109 | 110 | Exit codes 111 | ---------- 112 | 113 | Docker image updater will exit with status 0 when everything went well, 114 | and there were either no updates or images were updated and all defined 115 | commands returned status code 0. 116 | 117 | If an image fails to update or one or more defined commands exits with 118 | a non-zero exit status then docker image updater will itself exit with 119 | status 1. 120 | 121 | 122 | Star me 123 | ------- 124 | 125 | If you use this software, please consider 126 | `starring `_ 127 | it on GitHub. This will give me some idea of how much it is used by 128 | other people. 129 | 130 | 131 | Related projects 132 | ---------------- 133 | 134 | * `docker-puller `_ 135 | * `DockerHub Webhook Listener `_ 136 | 137 | 138 | Changes 139 | ------- 140 | 141 | 1.0.0 (2015-11-10) 142 | ~~~~~~~~~~~~~~~~~~ 143 | 144 | * Allow multiple configuration files to be specified (settings will be merged in order) 145 | * Deprecated the `--file` argument 146 | * Print friendlier error messages in many cases of incorrect configurations 147 | 148 | 0.0.2 (2015-03-02) 149 | ~~~~~~~~~~~~~~~~~~ 150 | 151 | * Friendlier config file load error message 152 | * Change default config to `/etc/docker-image-updater.yml` 153 | 154 | 0.0.1 (2015-03-01) 155 | ~~~~~~~~~~~~~~~~~~ 156 | 157 | * Initial public release. 158 | 159 | 160 | License 161 | ------- 162 | 163 | The MIT License (MIT) 164 | 165 | Copyright (c) 2015 Nick Groenen 166 | 167 | Permission is hereby granted, free of charge, to any person obtaining a copy 168 | of this software and associated documentation files (the "Software"), to deal 169 | in the Software without restriction, including without limitation the rights 170 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 171 | copies of the Software, and to permit persons to whom the Software is 172 | furnished to do so, subject to the following conditions: 173 | 174 | The above copyright notice and this permission notice shall be included in 175 | all copies or substantial portions of the Software. 176 | 177 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 178 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 179 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 180 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 181 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 182 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 183 | THE SOFTWARE. 184 | -------------------------------------------------------------------------------- /tests/test_updater.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | import mock 3 | import pytest 4 | from copy import deepcopy 5 | from docker.errors import APIError 6 | from diu.updater import Updater, ContainerSet 7 | 8 | 9 | CONTAINERSET = [ 10 | ContainerSet( 11 | name="ubuntu", 12 | images=['ubuntu:latest', 'ubuntu:14.04'], 13 | commands=['foo', 'bar'], 14 | ) 15 | ] 16 | 17 | 18 | class TestUpdater(object): 19 | @pytest.fixture 20 | def default_image(self): 21 | return { 22 | 'Id': 'c3c3d842b8f7b00268f391ee65d57ffddb955cd8c7f5b330710b14bc9781d8f2', 23 | } 24 | 25 | @pytest.fixture(autouse=True) 26 | def updater(self, default_image): 27 | c = mock.MagicMock() 28 | c.inspect_image.return_value = default_image 29 | 30 | self.client = c 31 | self.updater = Updater(client=self.client, containerset=CONTAINERSET) 32 | return self.updater 33 | 34 | def test_init_sets_paramaters_on_self(self, updater): 35 | assert updater.client == self.client 36 | 37 | assert list(updater.containerset.keys()) == [w.name for w in CONTAINERSET] 38 | assert list(updater.containerset.values()) == CONTAINERSET 39 | 40 | def test_update_image_detects_whether_image_was_updated(self, updater, default_image): 41 | updated = updater._update_image('ubuntu:latest') 42 | assert not updated 43 | assert updater._updated == [] 44 | 45 | before = default_image 46 | after = deepcopy(before) 47 | after['Id'] = 'a-new-id' 48 | self.client.inspect_image.side_effect = [before, after] 49 | 50 | updated = updater._update_image('ubuntu:latest') 51 | assert updated 52 | assert updater._updated == ['ubuntu:latest'] 53 | 54 | def test_update_image_will_pull_if_image_not_found(self, updater, default_image): 55 | r = mock.MagicMock() 56 | r.status_code = 404 57 | e = APIError( 58 | "404 Client Error: Not Found", 59 | response=r, 60 | explanation="No such image: ubuntu:newtag" 61 | ) 62 | self.client.inspect_image.side_effect = [e, default_image] 63 | updated = updater._update_image('ubuntu:newtag') 64 | assert updated 65 | assert updater._updated == ['ubuntu:newtag'] 66 | 67 | def test_do_updates_calls_update_with_each_containerset(self, updater): 68 | with mock.patch.object(Updater, '_update') as m: 69 | updater.do_updates() 70 | 71 | expected_calls = [mock.call(w) for w in CONTAINERSET] 72 | assert m.call_args_list == expected_calls 73 | 74 | @mock.patch('diu.updater.Updater._run_command') 75 | def test_update_will_call_update_image_for_configured_images(self, run_command_mock, updater): 76 | with mock.patch.object(Updater, '_update_image') as m: 77 | updater.do_updates() 78 | 79 | expected_images = list(itertools.chain.from_iterable([x.images for x in CONTAINERSET])) 80 | expected_calls = [mock.call(i) for i in expected_images] 81 | assert expected_calls == m.call_args_list 82 | 83 | @mock.patch('diu.updater.Updater._run_command') 84 | def test_update_will_not_run_commands_if_no_images_updated(self, run_command_mock, updater): 85 | m = mock.MagicMock() 86 | m.return_value = False 87 | with mock.patch.object(Updater, '_update_image', new=m): 88 | updater.do_updates() 89 | assert not run_command_mock.called 90 | 91 | @mock.patch('diu.updater.Updater._run_command') 92 | def test_update_will_run_commands_if_images_updated(self, run_command_mock, updater): 93 | m = mock.MagicMock() 94 | m.return_value = True 95 | with mock.patch.object(Updater, '_update_image', new=m): 96 | updater.do_updates() 97 | assert run_command_mock.called 98 | 99 | expected_commands = list(itertools.chain.from_iterable([w.commands for w in CONTAINERSET])) 100 | expected_calls = [mock.call(x) for x in expected_commands] 101 | assert expected_calls == run_command_mock.call_args_list 102 | 103 | def test_update_continues_if_update_image_throws_exception(self, updater): 104 | m = mock.MagicMock() 105 | m.side_effect = Exception("Boom!") 106 | with mock.patch.object(Updater, '_update_image', new=m): 107 | updater.do_updates() 108 | 109 | def test_update_continues_if_run_command_throws_exception(self, updater): 110 | run_command_mock = mock.MagicMock() 111 | run_command_mock.side_effect = Exception("Boom!") 112 | update_image_mock = mock.MagicMock() 113 | update_image_mock.return_value = True 114 | with mock.patch.object(Updater, '_update_image', new=update_image_mock), \ 115 | mock.patch.object(Updater, '_run_command', new=run_command_mock): 116 | updater.do_updates() 117 | 118 | @mock.patch('diu.updater.Updater._run_command') 119 | def test_update_is_aware_of_images_updated_by_other_containerset(self, run_command_mock, updater): 120 | assert "ubuntu:latest" in CONTAINERSET[0].images 121 | updater._updated = ["ubuntu:latest"] 122 | 123 | updater._update(CONTAINERSET[0]) 124 | assert run_command_mock.called 125 | 126 | def test_error_count_is_incremented_if_updating_image_fails(self, updater): 127 | assert updater.error_count == 0 128 | m = mock.MagicMock() 129 | m.side_effect = Exception("Boom!") 130 | with mock.patch.object(Updater, '_update_image', new=m): 131 | updater.do_updates() 132 | assert updater.error_count == 2 133 | 134 | def test_error_count_is_incremented_if_running_command_fails(self, updater): 135 | run_command_mock = mock.MagicMock() 136 | run_command_mock.side_effect = Exception("Boom!") 137 | update_image_mock = mock.MagicMock() 138 | update_image_mock.return_value = True 139 | 140 | assert updater.error_count == 0 141 | with mock.patch.object(Updater, '_update_image', new=update_image_mock), \ 142 | mock.patch.object(Updater, '_run_command', new=run_command_mock): 143 | updater.do_updates() 144 | assert updater.error_count == 2 145 | 146 | def test_error_count_is_incremented_if_command_returns_non_zero_exit(self, updater): 147 | update_image_mock = mock.MagicMock() 148 | update_image_mock.return_value = True 149 | popen_return = mock.MagicMock() 150 | popen_return.wait.return_value = 1 151 | popen_mock = mock.MagicMock() 152 | popen_mock.return_value = popen_return 153 | assert updater.error_count == 0 154 | 155 | with mock.patch.object(Updater, '_update_image', new=update_image_mock), \ 156 | mock.patch('diu.updater.subprocess.Popen', new=popen_mock) as m: 157 | updater.do_updates() 158 | 159 | assert m.called 160 | assert updater.error_count == 2 161 | --------------------------------------------------------------------------------