├── tests ├── __init__.py ├── args_test.py ├── conftest.py ├── docker_autostop_test.py └── docker_gc_test.py ├── debian ├── compat ├── .gitignore ├── docker-custodian.links ├── control ├── rules └── changelog ├── .dockerignore ├── docker_custodian ├── __init__.py ├── __about__.py ├── args.py ├── docker_autostop.py └── docker_gc.py ├── .gitignore ├── tox.ini ├── Dockerfile ├── requirements.txt ├── Makefile ├── .github └── workflows │ ├── publish.yml │ ├── tests.yml │ └── docker.yml ├── setup.py ├── .pre-commit-config.yaml ├── .secrets.baseline ├── README.rst ├── CHANGELOG.md └── LICENSE /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /debian/compat: -------------------------------------------------------------------------------- 1 | 10 2 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | .tox 3 | tox.ini 4 | dist 5 | -------------------------------------------------------------------------------- /docker_custodian/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py? 2 | .*.swp 3 | .tox 4 | dist 5 | build/ 6 | *.pyc 7 | __pycache__ 8 | 9 | *.egg-info/ 10 | -------------------------------------------------------------------------------- /debian/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | !changelog 4 | !compat 5 | !control 6 | !copyright 7 | !rules 8 | !docker-custodian.links 9 | -------------------------------------------------------------------------------- /debian/docker-custodian.links: -------------------------------------------------------------------------------- 1 | opt/venvs/docker-custodian/bin/dcgc usr/bin/dcgc 2 | opt/venvs/docker-custodian/bin/dcstop usr/bin/dcstop 3 | -------------------------------------------------------------------------------- /docker_custodian/__about__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | 3 | __version_info__ = (0, 8, 1) 4 | __version__ = '%d.%d.%d' % __version_info__ 5 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py37,py38,py39 3 | 4 | [testenv] 5 | deps = 6 | -rrequirements.txt 7 | mock 8 | pre-commit 9 | pytest 10 | commands = 11 | py.test {posargs:tests} 12 | pre-commit run --all-files 13 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.9-alpine3.12 2 | 3 | COPY requirements.txt /code/requirements.txt 4 | RUN pip install -r /code/requirements.txt 5 | COPY docker_custodian/ /code/docker_custodian/ 6 | COPY setup.py /code/ 7 | RUN pip install --no-deps -e /code 8 | 9 | ENTRYPOINT ["dcgc"] 10 | -------------------------------------------------------------------------------- /debian/control: -------------------------------------------------------------------------------- 1 | Source: docker-custodian 2 | Maintainer: Daniel Nephin 3 | Build-Depends: 4 | dh-virtualenv, python3 (>= 3.7) | python3.7 5 | 6 | Depends: ${pythonRuntime:Depends} 7 | Package: docker-custodian 8 | Architecture: any 9 | Description: Remove old Docker containers and images that are no longer in use 10 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | callee==0.3.1 2 | certifi==2024.7.4 3 | chardet==5.1.0 4 | docker==5.0.3 5 | docker-pycreds==0.2.2 6 | future==0.16.0 7 | idna==2.6 8 | ipaddress==1.0.19 9 | python-dateutil==2.9.0 10 | pytimeparse==1.1.8 11 | requests==2.31.0 12 | setuptools==68.0.0 13 | six==1.11.0 14 | urllib3==1.24.2 15 | websocket-client==0.57.0 16 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: all clean tag test 2 | 3 | PACKAGE_VERSION = $(shell python setup.py --version) 4 | 5 | DOCKER_REPO ?= ${USER} 6 | BUILD_TAG ?= ${PACKAGE_VERSION} 7 | 8 | all: test 9 | 10 | clean: 11 | git clean -fdx -- debian 12 | rm -f ./dist 13 | find . -iname '*.pyc' -delete 14 | 15 | tag: 16 | git tag v${PACKAGE_VERSION} 17 | 18 | test: 19 | tox 20 | 21 | tests: test 22 | 23 | 24 | .PHONY: build 25 | build: 26 | docker build -t ${DOCKER_REPO}/docker-custodian:${BUILD_TAG} . 27 | -------------------------------------------------------------------------------- /docker_custodian/args.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | from dateutil import tz 4 | from pytimeparse import timeparse 5 | 6 | 7 | def timedelta_type(value): 8 | """Return the :class:`datetime.datetime.DateTime` for a time in the past. 9 | 10 | :param value: a string containing a time format supported by 11 | mod:`pytimeparse` 12 | """ 13 | if value is None: 14 | return None 15 | return datetime_seconds_ago(timeparse.timeparse(value)) 16 | 17 | 18 | def datetime_seconds_ago(seconds): 19 | now = datetime.datetime.now(tz.tzutc()) 20 | return now - datetime.timedelta(seconds=seconds) 21 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Publish on PyPI 3 | 4 | "on": 5 | push: 6 | tags: 7 | - v* 8 | 9 | jobs: 10 | publish: 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - name: Checkout Repo 15 | uses: actions/checkout@v2 16 | 17 | - name: Setup Python 18 | uses: actions/setup-python@v2 19 | with: 20 | python-version: 3.7 21 | 22 | - name: Install Python dependencies 23 | run: pip install wheel 24 | 25 | - name: Create a Wheel file and source distribution 26 | run: python setup.py sdist bdist_wheel 27 | 28 | - name: Publish distribution package to PyPI 29 | uses: pypa/gh-action-pypi-publish@v1.2.2 30 | with: 31 | password: ${{ secrets.pypi_password }} 32 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from setuptools import setup, find_packages 3 | from docker_custodian.__about__ import __version__ 4 | 5 | 6 | setup( 7 | name='docker_custodian', 8 | version=__version__, 9 | provides=['docker_custodian'], 10 | author='Daniel Nephin', 11 | author_email='dnephin@yelp.com', 12 | description='Keep docker hosts tidy.', 13 | url='https://github.com/Yelp/docker-custodian', 14 | packages=find_packages(exclude=['tests*']), 15 | include_package_data=True, 16 | install_requires=[ 17 | 'python-dateutil', 18 | 'docker', 19 | 'pytimeparse', 20 | ], 21 | license="Apache License 2.0", 22 | entry_points={ 23 | 'console_scripts': [ 24 | 'dcstop = docker_custodian.docker_autostop:main', 25 | 'dcgc = docker_custodian.docker_gc:main', 26 | ], 27 | }, 28 | ) 29 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v4.3.0 4 | hooks: 5 | - id: check-added-large-files 6 | - id: check-ast 7 | - id: check-builtin-literals 8 | - id: check-docstring-first 9 | - id: check-executables-have-shebangs 10 | - id: check-json 11 | - id: check-merge-conflict 12 | - id: check-symlinks 13 | - id: check-vcs-permalinks 14 | - id: check-yaml 15 | - id: debug-statements 16 | - id: end-of-file-fixer 17 | exclude: CHANGELOG.md 18 | - id: name-tests-test 19 | - id: requirements-txt-fixer 20 | - id: trailing-whitespace 21 | - repo: https://github.com/pycqa/flake8 22 | rev: 4.0.1 23 | hooks: 24 | - id: flake8 25 | args: ['--max-line-length=130'] 26 | - repo: https://github.com/Yelp/detect-secrets 27 | rev: v1.2.0 28 | hooks: 29 | - id: detect-secrets 30 | args: ['--baseline', '.secrets.baseline'] 31 | exclude: tests/.* 32 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Tests 3 | 4 | "on": 5 | push: 6 | branches: 7 | pull_request: 8 | branches: [master] 9 | 10 | jobs: 11 | build: 12 | 13 | runs-on: ubuntu-latest 14 | strategy: 15 | fail-fast: false 16 | matrix: 17 | include: 18 | - python-version: 3.7 19 | toxenv: py37 20 | - python-version: 3.8 21 | toxenv: py38 22 | - python-version: 3.9 23 | toxenv: py39 24 | 25 | steps: 26 | - uses: actions/checkout@v2 27 | - name: Set up Python ${{ matrix.python-version }} 28 | uses: actions/setup-python@v2 29 | with: 30 | python-version: ${{ matrix.python-version }} 31 | - name: Install dependencies 32 | run: | 33 | python -m pip install -U pip 34 | python -m pip install tox 35 | - name: Run tests 36 | env: 37 | TOXENV: ${{ matrix.toxenv }} 38 | run: | 39 | tox 40 | -------------------------------------------------------------------------------- /tests/args_test.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | try: 3 | from unittest import mock 4 | except ImportError: 5 | import mock 6 | 7 | from dateutil import tz 8 | 9 | from docker_custodian import args 10 | 11 | 12 | def test_datetime_seconds_ago(now): 13 | expected = datetime.datetime(2014, 1, 15, 10, 10, tzinfo=tz.tzutc()) 14 | with mock.patch( 15 | 'docker_custodian.args.datetime.datetime', 16 | autospec=True, 17 | ) as mock_datetime: 18 | mock_datetime.now.return_value = now 19 | assert args.datetime_seconds_ago(24 * 60 * 60 * 5) == expected 20 | 21 | 22 | def test_timedelta_type_none(): 23 | assert args.timedelta_type(None) is None 24 | 25 | 26 | def test_timedelta_type(now): 27 | expected = datetime.datetime(2014, 1, 15, 10, 10, tzinfo=tz.tzutc()) 28 | with mock.patch( 29 | 'docker_custodian.args.datetime.datetime', 30 | autospec=True, 31 | ) as mock_datetime: 32 | mock_datetime.now.return_value = now 33 | assert args.timedelta_type('5 days') == expected 34 | -------------------------------------------------------------------------------- /debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | # -*- makefile -*- 3 | 4 | export DH_OPTIONS 5 | 6 | ifeq ($(shell (. /etc/lsb-release && dpkg --compare-versions $$DISTRIB_RELEASE ge "24.04" && echo yes || echo no)),yes) 7 | python3_depends = python3.8, python3.8-distutils 8 | python3_runtime = python3.8 9 | else ifeq ($(shell (. /etc/lsb-release && dpkg --compare-versions $$DISTRIB_RELEASE ge "22.04" && echo yes || echo no)),yes) 10 | python3_depends = python3.7, python3.7-distutils 11 | python3_runtime = python3.7 12 | else 13 | python3_depends = python3.7 14 | python3_runtime = python3.7 15 | endif 16 | 17 | %: 18 | dh $@ --with python-virtualenv 19 | 20 | override_dh_gencontrol: 21 | dh_gencontrol -- -VpythonRuntime:Depends="$(python3_depends)" 22 | 23 | override_dh_virtualenv: 24 | dh_virtualenv --python $(python3_runtime) --extra-pip-arg --only-binary=:all: 25 | 26 | # do not call `make clean` as part of packaging 27 | override_dh_auto_clean: 28 | true 29 | 30 | # do not call `make` as part of packaging 31 | override_dh_auto_build: 32 | true 33 | 34 | override_dh_auto_test: 35 | true 36 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | from dateutil import tz 4 | import docker 5 | try: 6 | from unittest import mock 7 | except ImportError: 8 | import mock 9 | import pytest 10 | 11 | 12 | @pytest.fixture 13 | def container(): 14 | return { 15 | 'Id': 'abcdabcdabcdabcd', 16 | 'Created': '2013-12-20T17:00:00Z', 17 | 'Name': '/container_name', 18 | 'State': { 19 | 'Running': False, 20 | 'FinishedAt': '2014-01-01T17:30:00Z', 21 | 'StartedAt': '2014-01-01T17:01:00Z', 22 | } 23 | } 24 | 25 | 26 | @pytest.fixture 27 | def image(): 28 | return { 29 | 'Id': 'abcdabcdabcdabcd', 30 | 'Created': '2014-01-20T05:00:00Z', 31 | } 32 | 33 | 34 | @pytest.fixture 35 | def now(): 36 | return datetime.datetime(2014, 1, 20, 10, 10, tzinfo=tz.tzutc()) 37 | 38 | 39 | @pytest.fixture 40 | def earlier_time(): 41 | return datetime.datetime(2014, 1, 1, 0, 0, tzinfo=tz.tzutc()) 42 | 43 | 44 | @pytest.fixture 45 | def later_time(): 46 | return datetime.datetime(2014, 1, 20, 0, 10, tzinfo=tz.tzutc()) 47 | 48 | 49 | @pytest.fixture 50 | def mock_client(): 51 | client = mock.create_autospec(docker.APIClient) 52 | client._version = '1.21' 53 | return client 54 | -------------------------------------------------------------------------------- /.github/workflows/docker.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: push-docker-images 3 | 4 | "on": 5 | push: 6 | branches: 7 | - master 8 | tags: 9 | - 'v*' 10 | pull_request: 11 | branches: 12 | - master 13 | 14 | jobs: 15 | docker: 16 | runs-on: ubuntu-latest 17 | steps: 18 | 19 | - name: Checkout 20 | uses: actions/checkout@v2 21 | 22 | - name: Set up QEMU 23 | uses: docker/setup-qemu-action@v1 24 | 25 | - name: Set up Docker Buildx 26 | uses: docker/setup-buildx-action@v1 27 | 28 | - name: Docker meta 29 | id: meta 30 | uses: docker/metadata-action@v3 31 | with: 32 | images: ${{ github.repository }} 33 | flavor: | 34 | latest=auto 35 | tags: | 36 | type=ref,event=branch 37 | type=ref,event=pr 38 | type=semver,pattern={{version}} 39 | type=semver,pattern={{major}}.{{minor}} 40 | 41 | - name: Login to DockerHub 42 | if: github.event_name != 'pull_request' 43 | uses: docker/login-action@v1 44 | with: 45 | username: ${{ secrets.DOCKERHUB_USERNAME }} 46 | password: ${{ secrets.DOCKERHUB_TOKEN }} 47 | 48 | - name: Build and push 49 | uses: docker/build-push-action@v2 50 | with: 51 | context: . 52 | platforms: | 53 | linux/386 54 | linux/amd64 55 | linux/arm/v6 56 | linux/arm/v7 57 | linux/arm64 58 | linux/ppc64le 59 | linux/s390x 60 | push: ${{ github.event_name != 'pull_request' }} 61 | tags: ${{ steps.meta.outputs.tags }} 62 | labels: ${{ steps.meta.outputs.labels }} 63 | 64 | - name: Image digest 65 | run: echo ${{ steps.docker_build.outputs.digest }} 66 | -------------------------------------------------------------------------------- /.secrets.baseline: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1.2.0", 3 | "plugins_used": [ 4 | { 5 | "name": "ArtifactoryDetector" 6 | }, 7 | { 8 | "name": "AWSKeyDetector" 9 | }, 10 | { 11 | "name": "AzureStorageKeyDetector" 12 | }, 13 | { 14 | "name": "Base64HighEntropyString", 15 | "limit": 4.5 16 | }, 17 | { 18 | "name": "BasicAuthDetector" 19 | }, 20 | { 21 | "name": "CloudantDetector" 22 | }, 23 | { 24 | "name": "GitHubTokenDetector" 25 | }, 26 | { 27 | "name": "HexHighEntropyString", 28 | "limit": 3.0 29 | }, 30 | { 31 | "name": "IbmCloudIamDetector" 32 | }, 33 | { 34 | "name": "IbmCosHmacDetector" 35 | }, 36 | { 37 | "name": "JwtTokenDetector" 38 | }, 39 | { 40 | "name": "KeywordDetector", 41 | "keyword_exclude": "" 42 | }, 43 | { 44 | "name": "MailchimpDetector" 45 | }, 46 | { 47 | "name": "NpmDetector" 48 | }, 49 | { 50 | "name": "PrivateKeyDetector" 51 | }, 52 | { 53 | "name": "SendGridDetector" 54 | }, 55 | { 56 | "name": "SlackDetector" 57 | }, 58 | { 59 | "name": "SoftlayerDetector" 60 | }, 61 | { 62 | "name": "SquareOAuthDetector" 63 | }, 64 | { 65 | "name": "StripeDetector" 66 | }, 67 | { 68 | "name": "TwilioKeyDetector" 69 | } 70 | ], 71 | "filters_used": [ 72 | { 73 | "path": "detect_secrets.filters.allowlist.is_line_allowlisted" 74 | }, 75 | { 76 | "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", 77 | "min_level": 2 78 | }, 79 | { 80 | "path": "detect_secrets.filters.heuristic.is_indirect_reference" 81 | }, 82 | { 83 | "path": "detect_secrets.filters.heuristic.is_likely_id_string" 84 | }, 85 | { 86 | "path": "detect_secrets.filters.heuristic.is_lock_file" 87 | }, 88 | { 89 | "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string" 90 | }, 91 | { 92 | "path": "detect_secrets.filters.heuristic.is_potential_uuid" 93 | }, 94 | { 95 | "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign" 96 | }, 97 | { 98 | "path": "detect_secrets.filters.heuristic.is_sequential_string" 99 | }, 100 | { 101 | "path": "detect_secrets.filters.heuristic.is_swagger_file" 102 | }, 103 | { 104 | "path": "detect_secrets.filters.heuristic.is_templated_secret" 105 | } 106 | ], 107 | "results": {}, 108 | "generated_at": "2022-06-28T02:48:44Z" 109 | } 110 | -------------------------------------------------------------------------------- /tests/docker_autostop_test.py: -------------------------------------------------------------------------------- 1 | try: 2 | from unittest import mock 3 | except ImportError: 4 | import mock 5 | 6 | from docker_custodian.docker_autostop import ( 7 | build_container_matcher, 8 | get_opts, 9 | has_been_running_since, 10 | main, 11 | stop_container, 12 | stop_containers, 13 | ) 14 | 15 | 16 | def test_stop_containers(mock_client, container, now): 17 | matcher = mock.Mock() 18 | mock_client.containers.return_value = [container] 19 | mock_client.inspect_container.return_value = container 20 | 21 | stop_containers(mock_client, now, matcher, False) 22 | matcher.assert_called_once_with('container_name') 23 | mock_client.stop.assert_called_once_with(container['Id']) 24 | 25 | 26 | def test_stop_container(mock_client): 27 | id = 'asdb' 28 | stop_container(mock_client, id) 29 | mock_client.stop.assert_called_once_with(id) 30 | 31 | 32 | def test_build_container_matcher(): 33 | prefixes = ['one_', 'two_'] 34 | matcher = build_container_matcher(prefixes) 35 | 36 | assert matcher('one_container') 37 | assert matcher('two_container') 38 | assert not matcher('three_container') 39 | assert not matcher('one') 40 | 41 | 42 | def test_has_been_running_since_true(container, later_time): 43 | assert has_been_running_since(container, later_time) 44 | 45 | 46 | def test_has_been_running_since_false(container, earlier_time): 47 | assert not has_been_running_since(container, earlier_time) 48 | 49 | 50 | @mock.patch('docker_custodian.docker_autostop.build_container_matcher', 51 | autospec=True) 52 | @mock.patch('docker_custodian.docker_autostop.stop_containers', 53 | autospec=True) 54 | @mock.patch('docker_custodian.docker_autostop.get_opts', 55 | autospec=True) 56 | @mock.patch('docker_custodian.docker_autostop.docker', autospec=True) 57 | def test_main( 58 | mock_docker, 59 | mock_get_opts, 60 | mock_stop_containers, 61 | mock_build_matcher 62 | ): 63 | mock_get_opts.return_value.timeout = 30 64 | main() 65 | mock_get_opts.assert_called_once_with() 66 | mock_build_matcher.assert_called_once_with( 67 | mock_get_opts.return_value.prefix) 68 | mock_stop_containers.assert_called_once_with( 69 | mock.ANY, 70 | mock_get_opts.return_value.max_run_time, 71 | mock_build_matcher.return_value, 72 | mock_get_opts.return_value.dry_run) 73 | 74 | 75 | def test_get_opts_with_defaults(): 76 | opts = get_opts(args=['--prefix', 'one', '--prefix', 'two']) 77 | assert opts.timeout == 60 78 | assert opts.dry_run is False 79 | assert opts.prefix == ['one', 'two'] 80 | assert opts.max_run_time is None 81 | 82 | 83 | def test_get_opts_with_args(now): 84 | with mock.patch( 85 | 'docker_custodian.docker_autostop.timedelta_type', 86 | autospec=True 87 | ) as mock_timedelta_type: 88 | opts = get_opts(args=['--prefix', 'one', '--max-run-time', '24h']) 89 | assert opts.max_run_time == mock_timedelta_type.return_value 90 | mock_timedelta_type.assert_called_once_with('24h') 91 | -------------------------------------------------------------------------------- /docker_custodian/docker_autostop.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Stop docker container that have been running longer than the max_run_time and 4 | match some prefix. 5 | """ 6 | import argparse 7 | import logging 8 | import sys 9 | 10 | import dateutil.parser 11 | import docker 12 | import docker.errors 13 | import requests.exceptions 14 | 15 | from docker_custodian.args import timedelta_type 16 | from docker.utils import kwargs_from_env 17 | 18 | 19 | log = logging.getLogger(__name__) 20 | 21 | 22 | def stop_containers(client, max_run_time, matcher, dry_run): 23 | for container_summary in client.containers(): 24 | container = client.inspect_container(container_summary['Id']) 25 | name = container['Name'].lstrip('/') 26 | if ( 27 | matcher(name) and 28 | has_been_running_since(container, max_run_time) 29 | ): 30 | 31 | log.info("Stopping container %s %s: running since %s" % ( 32 | container['Id'][:16], 33 | name, 34 | container['State']['StartedAt'])) 35 | 36 | if not dry_run: 37 | stop_container(client, container['Id']) 38 | 39 | 40 | def stop_container(client, id): 41 | try: 42 | client.stop(id) 43 | except requests.exceptions.Timeout as e: 44 | log.warn("Failed to stop container %s: %s" % (id, e)) 45 | except docker.errors.APIError as ae: 46 | log.warn("Error stopping %s: %s" % (id, ae)) 47 | 48 | 49 | def build_container_matcher(prefixes): 50 | def matcher(name): 51 | return any(name.startswith(prefix) for prefix in prefixes) 52 | return matcher 53 | 54 | 55 | def has_been_running_since(container, min_time): 56 | started_at = container.get('State', {}).get('StartedAt') 57 | if not started_at: 58 | return False 59 | 60 | return dateutil.parser.parse(started_at) <= min_time 61 | 62 | 63 | def main(): 64 | logging.basicConfig( 65 | level=logging.INFO, 66 | format="%(message)s", 67 | stream=sys.stdout) 68 | 69 | opts = get_opts() 70 | client = docker.APIClient(version='auto', 71 | timeout=opts.timeout, 72 | **kwargs_from_env()) 73 | 74 | matcher = build_container_matcher(opts.prefix) 75 | stop_containers(client, opts.max_run_time, matcher, opts.dry_run) 76 | 77 | 78 | def get_opts(args=None): 79 | parser = argparse.ArgumentParser() 80 | parser.add_argument( 81 | '--max-run-time', 82 | type=timedelta_type, 83 | help="Maximum time a container is allows to run. Time may " 84 | "be specified in any pytimeparse supported format." 85 | ) 86 | parser.add_argument( 87 | '--prefix', action="append", default=[], 88 | help="Only stop containers which match one of the " 89 | "prefix." 90 | ) 91 | parser.add_argument( 92 | '--dry-run', action="store_true", 93 | help="Only log actions, don't stop anything." 94 | ) 95 | parser.add_argument( 96 | '-t', '--timeout', type=int, default=60, 97 | help="HTTP timeout in seconds for making docker API calls." 98 | ) 99 | opts = parser.parse_args(args=args) 100 | 101 | if not opts.prefix: 102 | parser.error("Running with no --prefix will match nothing.") 103 | 104 | return opts 105 | 106 | 107 | if __name__ == "__main__": 108 | main() 109 | -------------------------------------------------------------------------------- /debian/changelog: -------------------------------------------------------------------------------- 1 | docker-custodian (0.8.1) bionic; urgency=medium 2 | 3 | * Drop python 3.6 support 4 | * Upgrade pre-commit hooks 5 | * Use python 3.7 on Jammy (22.04) too 6 | 7 | -- Jason Perrin Tue, 28 Jun 2022 17:02:02 -0700 8 | 9 | docker-custodian (0.8.0) xenial; urgency=medium 10 | 11 | * Build for Python 3 12 | 13 | -- Sophie Matthews Tue, 26 Apr 2022 16:57:00 +0100 14 | 15 | docker-custodian (0.7.3) lucid; urgency=medium 16 | 17 | * Fix handling containers with null labels 18 | 19 | -- Matthew Mead-Briggs Thu, 25 Apr 2019 03:43:55 -0700 20 | 21 | docker-custodian (0.7.2) lucid; urgency=medium 22 | 23 | * Fix debian links and release 0.7.2 24 | 25 | -- Kyle Anderson Wed, 21 Mar 2018 15:48:42 -0700 26 | 27 | docker-custodian (0.7.1) lucid; urgency=medium 28 | 29 | * Release 0.7.1 30 | 31 | -- Kyle Anderson Wed, 21 Mar 2018 15:26:16 -0700 32 | 33 | docker-custodian (0.7.0) lucid; urgency=low 34 | 35 | * Delete volumes along with containers 36 | 37 | -- Paul O'Connor Wed, 05 Oct 2016 00:58:10 -0700 38 | 39 | docker-custodian (0.6.1) lucid; urgency=low 40 | 41 | * New release for pypi 42 | 43 | -- kwa Wed, 31 Aug 2016 09:49:37 -0700 44 | 45 | docker-custodian (0.6.0) lucid; urgency=low 46 | 47 | * Remove python 2.6 support 48 | * Remove argparse 49 | 50 | -- Daniel Hoherd Fri, 24 Jun 2016 13:55:49 -0700 51 | 52 | docker-custodian (0.5.3) lucid; urgency=low 53 | 54 | * Update docker-py 55 | 56 | -- Alex Dudko Mon, 4 Apr 2016 09:44:26 -0800 57 | 58 | docker-custodian (0.5.2) lucid; urgency=low 59 | 60 | * Fixed bug where never started containers that are not old were getting removed 61 | 62 | -- Semir Patel Tue, 15 Dec 2015 09:44:26 -0800 63 | 64 | docker-custodian (0.5.0) lucid; urgency=low 65 | 66 | * Add option to exclude images from removal by dcgc 67 | 68 | -- Daniel Nephin Tue, 21 Jul 2015 11:14:38 -0700 69 | 70 | docker-custodian (0.4.0) lucid; urgency=low 71 | 72 | * Renamed to docker-custodian 73 | * Changed defaults of dcgc to not remove anything 74 | 75 | -- Daniel Nephin Mon, 29 Jun 2015 18:48:22 -0700 76 | 77 | docker-custodian (0.3.3) lucid; urgency=low 78 | 79 | * Bug fixes for removing images by Id and with multiple tags 80 | 81 | -- Daniel Nephin Thu, 04 Jun 2015 13:24:14 -0700 82 | 83 | docker-custodian (0.3.2) lucid; urgency=low 84 | 85 | * docker-custodian should now remove image names before trying to remove 86 | by id, so that images tagged with more than one name are removed 87 | correctly 88 | 89 | -- Daniel Nephin Tue, 02 Jun 2015 13:26:56 -0700 90 | 91 | docker-custodian (0.3.1) lucid; urgency=low 92 | 93 | * Fix broken commands 94 | 95 | -- Daniel Nephin Mon, 09 Mar 2015 17:58:03 -0700 96 | 97 | docker-custodian (0.3.0) lucid; urgency=low 98 | 99 | * Change age and time options to support pytimeparse formats 100 | 101 | -- Daniel Nephin Fri, 06 Mar 2015 13:30:36 -0800 102 | 103 | docker-custodian (0.2.0) lucid; urgency=low 104 | 105 | * Add docker-autostop 106 | 107 | -- Daniel Nephin Wed, 28 Jan 2015 15:37:40 -0800 108 | 109 | docker-custodian (0.1.0) lucid; urgency=low 110 | 111 | * Initial release 112 | 113 | -- Daniel Nephin Thu, 02 Oct 2014 11:13:43 -0700 114 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Docker Custodian 2 | ================ 3 | 4 | .. |tests badge| image:: https://github.com/yelp/docker-custodian/actions/workflows/tests.yml/badge.svg 5 | .. |docker badge| image:: https://github.com/yelp/docker-custodian/actions/workflows/docker.yml/badge.svg 6 | .. |pypi badge| image:: https://github.com/yelp/docker-custodian/actions/workflows/publish.yml/badge.svg 7 | 8 | |tests badge| |docker badge| |pypi badge| 9 | 10 | Keep docker hosts tidy. 11 | 12 | 13 | .. contents:: 14 | :backlinks: none 15 | 16 | Install 17 | ------- 18 | 19 | There are three installation options 20 | 21 | Container 22 | ~~~~~~~~~ 23 | 24 | .. code:: 25 | 26 | docker pull yelp/docker-custodian 27 | docker run -ti \ 28 | -v /var/run/docker.sock:/var/run/docker.sock \ 29 | yelp/docker-custodian --help 30 | 31 | Debian/Ubuntu package 32 | ~~~~~~~~~~~~~~~~~~~~~ 33 | 34 | First build the package (requires `dh-virtualenv`) 35 | 36 | .. code:: sh 37 | 38 | dpkg-buildpackage -us -uc 39 | 40 | Then install it 41 | 42 | .. code:: sh 43 | 44 | dpkg -i ../docker-custodian_*.deb 45 | 46 | 47 | pypi.org 48 | ~~~~~~~~ 49 | 50 | .. code:: sh 51 | 52 | pip install docker-custodian 53 | 54 | 55 | dcgc 56 | ---- 57 | 58 | Remove old docker containers and docker images. 59 | 60 | ``dcgc`` will remove stopped containers and unused images that are older than 61 | "max age". Running containers, and images which are used by a container are 62 | never removed. 63 | 64 | Maximum age can be specificied with any format supported by 65 | `pytimeparse `_. 66 | 67 | Example: 68 | 69 | .. code:: sh 70 | 71 | dcgc --max-container-age 3days --max-image-age 30days 72 | 73 | 74 | Prevent images from being removed 75 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 76 | 77 | ``dcgc`` supports an image exclude list. If you have images that you'd like 78 | to keep around forever you can use the exclude list to prevent them from 79 | being removed. 80 | 81 | :: 82 | 83 | --exclude-image 84 | Never remove images with this tag. May be specified more than once. 85 | 86 | --exclude-image-file 87 | Path to a file which contains a list of images to exclude, one 88 | image tag per line. 89 | 90 | You also can use basic pattern matching to exclude images with generic tags. 91 | 92 | .. code:: 93 | 94 | user/repositoryA:* 95 | user/repositoryB:?.? 96 | user/repositoryC-*:tag 97 | 98 | 99 | Prevent containers and associated images from being removed 100 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 101 | 102 | ``dcgc`` also supports a container exclude list based on labels. If there are 103 | stopped containers that you'd like to keep, then you can check the labels to 104 | prevent them from being removed. 105 | 106 | :: 107 | 108 | --exclude-container-label 109 | Never remove containers that have the label key=value. =value can be 110 | omitted and in that case only the key is checked. May be specified 111 | more than once. 112 | 113 | You also can use basic pattern matching to exclude generic labels. 114 | 115 | .. code:: 116 | 117 | foo* 118 | com.docker.compose.project=test* 119 | com.docker*=*bar* 120 | 121 | 122 | dcstop 123 | ------ 124 | 125 | Stop containers that have been running for too long. 126 | 127 | ``dcstop`` will ``docker stop`` containers where the container name starts 128 | with `--prefix` and it has been running for longer than `--max-run-time`. 129 | 130 | 131 | Example: 132 | 133 | .. code:: sh 134 | 135 | dcstop --max-run-time 2days --prefix "projectprefix_" 136 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## [v0.7.4](https://github.com/yelp/docker-custodian/tree/v0.7.4) (2021-06-23) 4 | 5 | [Full Changelog](https://github.com/yelp/docker-custodian/compare/v0.7.3...v0.7.4) 6 | 7 | **Closed issues:** 8 | 9 | - Upgrade requests to version 2.20.0 or later \(CVE-2018-18074\) [\#50](https://github.com/Yelp/docker-custodian/issues/50) 10 | - --dangling-volumes runs into an error when there are no dangling volumes [\#41](https://github.com/Yelp/docker-custodian/issues/41) 11 | - Use kwargs\_from\_env\(\) to get Client kwargs [\#13](https://github.com/Yelp/docker-custodian/issues/13) 12 | 13 | **Merged pull requests:** 14 | 15 | - Move to GitHub Actions [\#61](https://github.com/Yelp/docker-custodian/pull/61) ([IamTheFij](https://github.com/IamTheFij)) 16 | - Remove pre-commit autoupdate in tox [\#60](https://github.com/Yelp/docker-custodian/pull/60) ([IamTheFij](https://github.com/IamTheFij)) 17 | - Enable multi-arch builds [\#59](https://github.com/Yelp/docker-custodian/pull/59) ([ViViDboarder](https://github.com/ViViDboarder)) 18 | - Simplify instructions for installing with pip [\#54](https://github.com/Yelp/docker-custodian/pull/54) ([dmerejkowsky](https://github.com/dmerejkowsky)) 19 | - Bump urllib and requests [\#52](https://github.com/Yelp/docker-custodian/pull/52) ([keymone](https://github.com/keymone)) 20 | 21 | ## [v0.7.3](https://github.com/yelp/docker-custodian/tree/v0.7.3) (2019-04-25) 22 | 23 | [Full Changelog](https://github.com/yelp/docker-custodian/compare/v0.7.2...v0.7.3) 24 | 25 | **Merged pull requests:** 26 | 27 | - Fix null labels [\#51](https://github.com/Yelp/docker-custodian/pull/51) ([mattmb](https://github.com/mattmb)) 28 | - adding .secrets.baseline [\#49](https://github.com/Yelp/docker-custodian/pull/49) ([domanchi](https://github.com/domanchi)) 29 | - Ignore CHANGELOG.md for the end-of-file precommit hook [\#48](https://github.com/Yelp/docker-custodian/pull/48) ([ATRAN2](https://github.com/ATRAN2)) 30 | 31 | ## [v0.7.2](https://github.com/yelp/docker-custodian/tree/v0.7.2) (2018-03-21) 32 | 33 | [Full Changelog](https://github.com/yelp/docker-custodian/compare/v0.7.1...v0.7.2) 34 | 35 | ## [v0.7.1](https://github.com/yelp/docker-custodian/tree/v0.7.1) (2018-03-21) 36 | 37 | [Full Changelog](https://github.com/yelp/docker-custodian/compare/v0.6.1...v0.7.1) 38 | 39 | **Implemented enhancements:** 40 | 41 | - Remove volumes when removing containers [\#36](https://github.com/Yelp/docker-custodian/pull/36) ([pauloconnor](https://github.com/pauloconnor)) 42 | - Add patterns support for exclude-image-file [\#35](https://github.com/Yelp/docker-custodian/pull/35) ([vshlapakov](https://github.com/vshlapakov)) 43 | 44 | **Fixed bugs:** 45 | 46 | - Fix broken filtering for images used by containers [\#43](https://github.com/Yelp/docker-custodian/pull/43) ([chekunkov](https://github.com/chekunkov)) 47 | 48 | **Closed issues:** 49 | 50 | - docker-custodian crashing when cleaning up. [\#38](https://github.com/Yelp/docker-custodian/issues/38) 51 | - backports.ssl-match-hostname\>=3.5 error when running in Docker [\#37](https://github.com/Yelp/docker-custodian/issues/37) 52 | - Publish on PyPI? [\#10](https://github.com/Yelp/docker-custodian/issues/10) 53 | 54 | **Merged pull requests:** 55 | 56 | - Add --exclude-container-label argument to dcgc [\#47](https://github.com/Yelp/docker-custodian/pull/47) ([ATRAN2](https://github.com/ATRAN2)) 57 | - Port to docker lib [\#46](https://github.com/Yelp/docker-custodian/pull/46) ([samiam](https://github.com/samiam)) 58 | - Correctly handle empty dangling volumes [\#45](https://github.com/Yelp/docker-custodian/pull/45) ([samiam](https://github.com/samiam)) 59 | - Remove dangling volumes [\#40](https://github.com/Yelp/docker-custodian/pull/40) ([ymilki](https://github.com/ymilki)) 60 | - Revert docker-py version change [\#39](https://github.com/Yelp/docker-custodian/pull/39) ([pauloconnor](https://github.com/pauloconnor)) 61 | 62 | ## [v0.6.1](https://github.com/yelp/docker-custodian/tree/v0.6.1) (2016-08-31) 63 | 64 | [Full Changelog](https://github.com/yelp/docker-custodian/compare/v0.5.1...v0.6.1) 65 | 66 | **Closed issues:** 67 | 68 | - Add ability to kill/delete running containers by age [\#25](https://github.com/Yelp/docker-custodian/issues/25) 69 | - Add tag to yelp/docker-custodian on the docker hub [\#21](https://github.com/Yelp/docker-custodian/issues/21) 70 | - Error while fetching server API version [\#20](https://github.com/Yelp/docker-custodian/issues/20) 71 | - Could It also clear mesos tmp files? [\#19](https://github.com/Yelp/docker-custodian/issues/19) 72 | 73 | **Merged pull requests:** 74 | 75 | - Bump to python \>=2.7 [\#33](https://github.com/Yelp/docker-custodian/pull/33) ([danielhoherd](https://github.com/danielhoherd)) 76 | - Overriding tox with /bin/true [\#32](https://github.com/Yelp/docker-custodian/pull/32) ([danielhoherd](https://github.com/danielhoherd)) 77 | - Bump debian version to 0.6.0 [\#31](https://github.com/Yelp/docker-custodian/pull/31) ([danielhoherd](https://github.com/danielhoherd)) 78 | - Remove py26 support, code cleanup [\#27](https://github.com/Yelp/docker-custodian/pull/27) ([kentwills](https://github.com/kentwills)) 79 | - Don't remove recently created containers that were never used [\#23](https://github.com/Yelp/docker-custodian/pull/23) ([analogue](https://github.com/analogue)) 80 | - Upgrade requirements to fix \#17 [\#18](https://github.com/Yelp/docker-custodian/pull/18) ([dnephin](https://github.com/dnephin)) 81 | 82 | ## [v0.5.1](https://github.com/yelp/docker-custodian/tree/v0.5.1) (2015-09-29) 83 | 84 | [Full Changelog](https://github.com/yelp/docker-custodian/compare/v0.4.0...v0.5.1) 85 | 86 | **Implemented enhancements:** 87 | 88 | - Use Client\(version='auto'\) [\#7](https://github.com/Yelp/docker-custodian/issues/7) 89 | 90 | **Fixed bugs:** 91 | 92 | - Automatically determining API version doesn't work? [\#17](https://github.com/Yelp/docker-custodian/issues/17) 93 | 94 | **Closed issues:** 95 | 96 | - Put on the docker hub using the yelp namespace [\#16](https://github.com/Yelp/docker-custodian/issues/16) 97 | - Is it production ready? [\#15](https://github.com/Yelp/docker-custodian/issues/15) 98 | 99 | **Merged pull requests:** 100 | 101 | - Automatically determine the Docker API version to use [\#12](https://github.com/Yelp/docker-custodian/pull/12) ([jschrantz](https://github.com/jschrantz)) 102 | - Add install instructions to the README [\#9](https://github.com/Yelp/docker-custodian/pull/9) ([dnephin](https://github.com/dnephin)) 103 | - Fix readme formatting [\#8](https://github.com/Yelp/docker-custodian/pull/8) ([dnephin](https://github.com/dnephin)) 104 | - Support excluding some images from removal [\#6](https://github.com/Yelp/docker-custodian/pull/6) ([dnephin](https://github.com/dnephin)) 105 | - Add .travis.yml [\#5](https://github.com/Yelp/docker-custodian/pull/5) ([dnephin](https://github.com/dnephin)) 106 | 107 | ## [v0.4.0](https://github.com/yelp/docker-custodian/tree/v0.4.0) (2015-07-08) 108 | 109 | [Full Changelog](https://github.com/yelp/docker-custodian/compare/d7d25053e09b7006d16125dd3b967b845c599eaf...v0.4.0) 110 | 111 | 112 | 113 | \* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)* 114 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /docker_custodian/docker_gc.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Remove old docker containers and images that are no longer in use. 4 | 5 | """ 6 | import argparse 7 | import fnmatch 8 | import logging 9 | import sys 10 | 11 | import dateutil.parser 12 | import docker 13 | import docker.errors 14 | import requests.exceptions 15 | 16 | from collections import namedtuple 17 | from docker_custodian.args import timedelta_type 18 | from docker.utils import kwargs_from_env 19 | 20 | log = logging.getLogger(__name__) 21 | 22 | 23 | # This seems to be something docker uses for a null/zero date 24 | YEAR_ZERO = "0001-01-01T00:00:00Z" 25 | 26 | ExcludeLabel = namedtuple('ExcludeLabel', ['key', 'value']) 27 | 28 | 29 | def cleanup_containers( 30 | client, 31 | max_container_age, 32 | dry_run, 33 | exclude_container_labels, 34 | ): 35 | all_containers = get_all_containers(client) 36 | filtered_containers = filter_excluded_containers( 37 | all_containers, 38 | exclude_container_labels, 39 | ) 40 | for container_summary in reversed(list(filtered_containers)): 41 | container = api_call( 42 | client.inspect_container, 43 | container=container_summary['Id'], 44 | ) 45 | if not container or not should_remove_container( 46 | container, 47 | max_container_age, 48 | ): 49 | continue 50 | 51 | log.info("Removing container %s %s %s" % ( 52 | container['Id'][:16], 53 | container.get('Name', '').lstrip('/'), 54 | container['State']['FinishedAt'])) 55 | 56 | if not dry_run: 57 | api_call( 58 | client.remove_container, 59 | container=container['Id'], 60 | v=True, 61 | ) 62 | 63 | 64 | def filter_excluded_containers(containers, exclude_container_labels): 65 | if not exclude_container_labels: 66 | return containers 67 | 68 | def include_container(container): 69 | if should_exclude_container_with_labels( 70 | container, 71 | exclude_container_labels, 72 | ): 73 | return False 74 | return True 75 | return filter(include_container, containers) 76 | 77 | 78 | def should_exclude_container_with_labels(container, exclude_container_labels): 79 | if container['Labels']: 80 | for exclude_label in exclude_container_labels: 81 | if exclude_label.value: 82 | matching_keys = fnmatch.filter( 83 | container['Labels'].keys(), 84 | exclude_label.key, 85 | ) 86 | label_values_to_check = [ 87 | container['Labels'][matching_key] 88 | for matching_key in matching_keys 89 | ] 90 | if fnmatch.filter(label_values_to_check, exclude_label.value): 91 | return True 92 | else: 93 | if fnmatch.filter( 94 | container['Labels'].keys(), 95 | exclude_label.key 96 | ): 97 | return True 98 | return False 99 | 100 | 101 | def should_remove_container(container, min_date): 102 | state = container.get('State', {}) 103 | 104 | if state.get('Running'): 105 | return False 106 | 107 | if state.get('Ghost'): 108 | return True 109 | 110 | # Container was created, but never started 111 | if state.get('FinishedAt') == YEAR_ZERO: 112 | created_date = dateutil.parser.parse(container['Created']) 113 | return created_date < min_date 114 | 115 | finished_date = dateutil.parser.parse(state['FinishedAt']) 116 | return finished_date < min_date 117 | 118 | 119 | def get_all_containers(client): 120 | log.info("Getting all containers") 121 | containers = client.containers(all=True) 122 | log.info("Found %s containers", len(containers)) 123 | return containers 124 | 125 | 126 | def get_all_images(client): 127 | log.info("Getting all images") 128 | images = client.images() 129 | log.info("Found %s images", len(images)) 130 | return images 131 | 132 | 133 | def get_dangling_volumes(client): 134 | log.info("Getting dangling volumes") 135 | volumes = client.volumes({'dangling': True})['Volumes'] or [] 136 | log.info("Found %s dangling volumes", len(volumes)) 137 | return volumes 138 | 139 | 140 | def cleanup_images(client, max_image_age, dry_run, exclude_set): 141 | # re-fetch container list so that we don't include removed containers 142 | 143 | containers = get_all_containers(client) 144 | images = get_all_images(client) 145 | if docker.utils.compare_version('1.21', client._version) < 0: 146 | image_tags_in_use = {container['Image'] for container in containers} 147 | images = filter_images_in_use(images, image_tags_in_use) 148 | else: 149 | # ImageID field was added in 1.21 150 | image_ids_in_use = {container['ImageID'] for container in containers} 151 | images = filter_images_in_use_by_id(images, image_ids_in_use) 152 | images = filter_excluded_images(images, exclude_set) 153 | 154 | for image_summary in reversed(list(images)): 155 | remove_image(client, image_summary, max_image_age, dry_run) 156 | 157 | 158 | def filter_excluded_images(images, exclude_set): 159 | def include_image(image_summary): 160 | image_tags = image_summary.get('RepoTags') 161 | if no_image_tags(image_tags): 162 | return True 163 | for exclude_pattern in exclude_set: 164 | if fnmatch.filter(image_tags, exclude_pattern): 165 | return False 166 | return True 167 | 168 | return filter(include_image, images) 169 | 170 | 171 | def filter_images_in_use(images, image_tags_in_use): 172 | def get_tag_set(image_summary): 173 | image_tags = image_summary.get('RepoTags') 174 | if no_image_tags(image_tags): 175 | # The repr of the image Id used by client.containers() 176 | return set(['%s:latest' % image_summary['Id'][:12]]) 177 | return set(image_tags) 178 | 179 | def image_not_in_use(image_summary): 180 | return not get_tag_set(image_summary) & image_tags_in_use 181 | 182 | return filter(image_not_in_use, images) 183 | 184 | 185 | def filter_images_in_use_by_id(images, image_ids_in_use): 186 | def image_not_in_use(image_summary): 187 | return image_summary['Id'] not in image_ids_in_use 188 | 189 | return filter(image_not_in_use, images) 190 | 191 | 192 | def is_image_old(image, min_date): 193 | return dateutil.parser.parse(image['Created']) < min_date 194 | 195 | 196 | def no_image_tags(image_tags): 197 | return not image_tags or image_tags == [':'] 198 | 199 | 200 | def remove_image(client, image_summary, min_date, dry_run): 201 | image = api_call(client.inspect_image, image=image_summary['Id']) 202 | if not image or not is_image_old(image, min_date): 203 | return 204 | 205 | log.info("Removing image %s" % format_image(image, image_summary)) 206 | if dry_run: 207 | return 208 | 209 | image_tags = image_summary.get('RepoTags') 210 | # If there are no tags, remove the id 211 | if no_image_tags(image_tags): 212 | api_call(client.remove_image, image=image_summary['Id']) 213 | return 214 | 215 | # Remove any repository tags so we don't hit 409 Conflict 216 | for image_tag in image_tags: 217 | api_call(client.remove_image, image=image_tag) 218 | 219 | 220 | def remove_volume(client, volume, dry_run): 221 | if not volume: 222 | return 223 | 224 | log.info("Removing volume %s" % volume['Name']) 225 | if dry_run: 226 | return 227 | 228 | api_call(client.remove_volume, name=volume['Name']) 229 | 230 | 231 | def cleanup_volumes(client, dry_run): 232 | dangling_volumes = get_dangling_volumes(client) 233 | 234 | for volume in reversed(dangling_volumes): 235 | log.info("Removing dangling volume %s", volume['Name']) 236 | remove_volume(client, volume, dry_run) 237 | 238 | 239 | def api_call(func, **kwargs): 240 | try: 241 | return func(**kwargs) 242 | except requests.exceptions.Timeout as e: 243 | params = ','.join('%s=%s' % item for item in kwargs.items()) 244 | log.warn("Failed to call %s %s %s" % (func.__name__, params, e)) 245 | except docker.errors.APIError as ae: 246 | params = ','.join('%s=%s' % item for item in kwargs.items()) 247 | log.warn("Error calling %s %s %s" % (func.__name__, params, ae)) 248 | 249 | 250 | def format_image(image, image_summary): 251 | def get_tags(): 252 | tags = image_summary.get('RepoTags') 253 | if not tags or tags == [':']: 254 | return '' 255 | return ', '.join(tags) 256 | 257 | return "%s %s" % (image['Id'][:16], get_tags()) 258 | 259 | 260 | def build_exclude_set(image_tags, exclude_file): 261 | exclude_set = set(image_tags or []) 262 | 263 | def is_image_tag(line): 264 | return line and not line.startswith('#') 265 | 266 | if exclude_file: 267 | lines = [line.strip() for line in exclude_file.read().split('\n')] 268 | exclude_set.update(filter(is_image_tag, lines)) 269 | return exclude_set 270 | 271 | 272 | def format_exclude_labels(exclude_label_args): 273 | exclude_labels = [] 274 | for exclude_label_arg in exclude_label_args: 275 | split_exclude_label = exclude_label_arg.split('=', 1) 276 | exclude_label_key = split_exclude_label[0] 277 | if len(split_exclude_label) == 2: 278 | exclude_label_value = split_exclude_label[1] 279 | else: 280 | exclude_label_value = None 281 | exclude_labels.append( 282 | ExcludeLabel( 283 | key=exclude_label_key, 284 | value=exclude_label_value, 285 | ) 286 | ) 287 | return exclude_labels 288 | 289 | 290 | def main(): 291 | logging.basicConfig( 292 | level=logging.INFO, 293 | format="%(message)s", 294 | stream=sys.stdout) 295 | 296 | args = get_args() 297 | client = docker.APIClient(version='auto', 298 | timeout=args.timeout, 299 | **kwargs_from_env()) 300 | 301 | exclude_container_labels = format_exclude_labels( 302 | args.exclude_container_label 303 | ) 304 | 305 | if args.max_container_age: 306 | cleanup_containers( 307 | client, 308 | args.max_container_age, 309 | args.dry_run, 310 | exclude_container_labels, 311 | ) 312 | 313 | if args.max_image_age: 314 | exclude_set = build_exclude_set( 315 | args.exclude_image, 316 | args.exclude_image_file) 317 | cleanup_images(client, args.max_image_age, args.dry_run, exclude_set) 318 | 319 | if args.dangling_volumes: 320 | cleanup_volumes(client, args.dry_run) 321 | 322 | 323 | def get_args(args=None): 324 | parser = argparse.ArgumentParser() 325 | parser.add_argument( 326 | '--max-container-age', 327 | type=timedelta_type, 328 | help="Maximum age for a container. Containers older than this age " 329 | "will be removed. Age can be specified in any pytimeparse " 330 | "supported format.") 331 | parser.add_argument( 332 | '--max-image-age', 333 | type=timedelta_type, 334 | help="Maxium age for an image. Images older than this age will be " 335 | "removed. Age can be specified in any pytimeparse supported " 336 | "format.") 337 | parser.add_argument( 338 | '--dangling-volumes', 339 | action="store_true", 340 | help="Dangling volumes will be removed.") 341 | parser.add_argument( 342 | '--dry-run', action="store_true", 343 | help="Only log actions, don't remove anything.") 344 | parser.add_argument( 345 | '-t', '--timeout', type=int, default=60, 346 | help="HTTP timeout in seconds for making docker API calls.") 347 | parser.add_argument( 348 | '--exclude-image', 349 | action='append', 350 | help="Never remove images with this tag.") 351 | parser.add_argument( 352 | '--exclude-image-file', 353 | type=argparse.FileType('r'), 354 | help="Path to a file which contains a list of images to exclude, one " 355 | "image tag per line.") 356 | parser.add_argument( 357 | '--exclude-container-label', 358 | action='append', type=str, default=[], 359 | help="Never remove containers with this label key or label key=value") 360 | 361 | return parser.parse_args(args=args) 362 | 363 | 364 | if __name__ == "__main__": 365 | main() 366 | -------------------------------------------------------------------------------- /tests/docker_gc_test.py: -------------------------------------------------------------------------------- 1 | from callee import String, Regex 2 | from six import StringIO 3 | import textwrap 4 | 5 | import docker.errors 6 | try: 7 | from unittest import mock 8 | except ImportError: 9 | import mock 10 | import requests.exceptions 11 | 12 | from docker_custodian import docker_gc 13 | 14 | 15 | class TestShouldRemoveContainer(object): 16 | 17 | def test_is_running(self, container, now): 18 | container['State']['Running'] = True 19 | assert not docker_gc.should_remove_container(container, now) 20 | 21 | def test_is_ghost(self, container, now): 22 | container['State']['Ghost'] = True 23 | assert docker_gc.should_remove_container(container, now) 24 | 25 | def test_old_never_run(self, container, now, earlier_time): 26 | container['Created'] = str(earlier_time) 27 | container['State']['FinishedAt'] = docker_gc.YEAR_ZERO 28 | assert docker_gc.should_remove_container(container, now) 29 | 30 | def test_not_old_never_run(self, container, now, earlier_time): 31 | container['Created'] = str(now) 32 | container['State']['FinishedAt'] = docker_gc.YEAR_ZERO 33 | assert not docker_gc.should_remove_container(container, now) 34 | 35 | def test_old_stopped(self, container, now): 36 | assert docker_gc.should_remove_container(container, now) 37 | 38 | def test_not_old(self, container, now): 39 | container['State']['FinishedAt'] = '2014-01-21T00:00:00Z' 40 | assert not docker_gc.should_remove_container(container, now) 41 | 42 | 43 | def test_cleanup_containers(mock_client, now): 44 | max_container_age = now 45 | mock_client.containers.return_value = [ 46 | {'Id': 'abcd'}, 47 | {'Id': 'abbb'}, 48 | ] 49 | mock_containers = [ 50 | { 51 | 'Id': 'abcd', 52 | 'Name': 'one', 53 | 'State': { 54 | 'Running': False, 55 | 'FinishedAt': '2014-01-01T01:01:01Z', 56 | }, 57 | }, 58 | { 59 | 'Id': 'abbb', 60 | 'Name': 'two', 61 | 'State': { 62 | 'Running': True, 63 | 'FinishedAt': '2014-01-01T01:01:01Z', 64 | }, 65 | }, 66 | ] 67 | mock_client.inspect_container.side_effect = iter(mock_containers) 68 | docker_gc.cleanup_containers(mock_client, max_container_age, False, None) 69 | mock_client.remove_container.assert_called_once_with(container='abcd', 70 | v=True) 71 | 72 | 73 | def test_filter_excluded_containers(): 74 | mock_containers = [ 75 | {'Labels': {'toot': ''}}, 76 | {'Labels': {'too': 'lol'}}, 77 | {'Labels': {'toots': 'lol'}}, 78 | {'Labels': {'foo': 'bar'}}, 79 | {'Labels': None}, 80 | ] 81 | result = docker_gc.filter_excluded_containers(mock_containers, None) 82 | assert mock_containers == list(result) 83 | exclude_labels = [ 84 | docker_gc.ExcludeLabel(key='too', value=None), 85 | docker_gc.ExcludeLabel(key='foo', value=None), 86 | ] 87 | result = docker_gc.filter_excluded_containers( 88 | mock_containers, 89 | exclude_labels, 90 | ) 91 | assert [ 92 | mock_containers[0], 93 | mock_containers[2], 94 | mock_containers[4] 95 | ] == list(result) 96 | exclude_labels = [ 97 | docker_gc.ExcludeLabel(key='too*', value='lol'), 98 | ] 99 | result = docker_gc.filter_excluded_containers( 100 | mock_containers, 101 | exclude_labels, 102 | ) 103 | assert [ 104 | mock_containers[0], 105 | mock_containers[3], 106 | mock_containers[4] 107 | ] == list(result) 108 | 109 | 110 | def test_cleanup_images(mock_client, now): 111 | max_image_age = now 112 | mock_client.images.return_value = images = [ 113 | {'Id': 'abcd'}, 114 | {'Id': 'abbb'}, 115 | ] 116 | mock_images = [ 117 | { 118 | 'Id': 'abcd', 119 | 'Created': '2014-01-01T01:01:01Z' 120 | }, 121 | { 122 | 'Id': 'abbb', 123 | 'Created': '2014-01-01T01:01:01Z' 124 | }, 125 | ] 126 | mock_client.inspect_image.side_effect = iter(mock_images) 127 | 128 | docker_gc.cleanup_images(mock_client, max_image_age, False, set()) 129 | assert mock_client.remove_image.mock_calls == [ 130 | mock.call(image=image['Id']) for image in reversed(images) 131 | ] 132 | 133 | 134 | def test_cleanup_volumes(mock_client): 135 | mock_client.volumes.return_value = volumes = { 136 | 'Volumes': [ 137 | { 138 | 'Mountpoint': 'unused', 139 | 'Labels': None, 140 | 'Driver': 'unused', 141 | 'Name': u'one' 142 | }, 143 | { 144 | 'Mountpoint': 'unused', 145 | 'Labels': None, 146 | 'Driver': 'unused', 147 | 'Name': u'two' 148 | }, 149 | ], 150 | 'Warnings': None, 151 | } 152 | 153 | docker_gc.cleanup_volumes(mock_client, False) 154 | assert mock_client.remove_volume.mock_calls == [ 155 | mock.call(name=volume['Name']) 156 | for volume in reversed(volumes['Volumes']) 157 | ] 158 | 159 | 160 | def test_filter_images_in_use(): 161 | image_tags_in_use = set([ 162 | 'user/one:latest', 163 | 'user/foo:latest', 164 | 'other:12345', 165 | '2471708c19be:latest', 166 | ]) 167 | images = [ 168 | { 169 | 'RepoTags': [':'], 170 | 'Id': '2471708c19beabababab' 171 | }, 172 | { 173 | 'RepoTags': [':'], 174 | 'Id': 'babababababaabababab' 175 | }, 176 | { 177 | 'RepoTags': ['user/one:latest', 'user/one:abcd'] 178 | }, 179 | { 180 | 'RepoTags': ['other:abcda'] 181 | }, 182 | { 183 | 'RepoTags': ['other:12345'] 184 | }, 185 | { 186 | 'RepoTags': ['new_image:latest', 'new_image:123'] 187 | }, 188 | ] 189 | expected = [ 190 | { 191 | 'RepoTags': [':'], 192 | 'Id': 'babababababaabababab' 193 | }, 194 | { 195 | 'RepoTags': ['other:abcda'] 196 | }, 197 | { 198 | 'RepoTags': ['new_image:latest', 'new_image:123'] 199 | }, 200 | ] 201 | actual = docker_gc.filter_images_in_use(images, image_tags_in_use) 202 | assert list(actual) == expected 203 | 204 | 205 | def test_filter_images_in_use_by_id(mock_client, now): 206 | mock_client._version = '1.21' 207 | mock_client.containers.return_value = [ 208 | {'Id': 'abcd', 'ImageID': '1'}, 209 | {'Id': 'abbb', 'ImageID': '2'}, 210 | ] 211 | mock_containers = [ 212 | { 213 | 'Id': 'abcd', 214 | 'Name': 'one', 215 | 'State': { 216 | 'Running': False, 217 | 'FinishedAt': '2014-01-01T01:01:01Z' 218 | } 219 | }, 220 | { 221 | 'Id': 'abbb', 222 | 'Name': 'two', 223 | 'State': { 224 | 'Running': True, 225 | 'FinishedAt': '2014-01-01T01:01:01Z' 226 | } 227 | } 228 | ] 229 | mock_client.inspect_container.side_effect = iter(mock_containers) 230 | mock_client.images.return_value = [ 231 | {'Id': '1', 'Created': '2014-01-01T01:01:01Z'}, 232 | {'Id': '2', 'Created': '2014-01-01T01:01:01Z'}, 233 | {'Id': '3', 'Created': '2014-01-01T01:01:01Z'}, 234 | {'Id': '4', 'Created': '2014-01-01T01:01:01Z'}, 235 | {'Id': '5', 'Created': '2014-01-01T01:01:01Z'}, 236 | {'Id': '6', 'Created': '2014-01-01T01:01:01Z'}, 237 | ] 238 | mock_client.inspect_image.side_effect = lambda image: { 239 | 'Id': image, 240 | 'Created': '2014-01-01T01:01:01Z' 241 | } 242 | docker_gc.cleanup_images(mock_client, now, False, set()) 243 | assert mock_client.remove_image.mock_calls == [ 244 | mock.call(image=id_) for id_ in ['6', '5', '4', '3'] 245 | ] 246 | 247 | 248 | def test_filter_excluded_images(): 249 | exclude_set = set([ 250 | 'user/one:latest', 251 | 'user/foo:latest', 252 | 'other:12345', 253 | ]) 254 | images = [ 255 | { 256 | 'RepoTags': [':'], 257 | 'Id': 'babababababaabababab' 258 | }, 259 | { 260 | 'RepoTags': ['user/one:latest', 'user/one:abcd'] 261 | }, 262 | { 263 | 'RepoTags': ['other:abcda'] 264 | }, 265 | { 266 | 'RepoTags': ['other:12345'] 267 | }, 268 | { 269 | 'RepoTags': ['new_image:latest', 'new_image:123'] 270 | }, 271 | ] 272 | expected = [ 273 | { 274 | 'RepoTags': [':'], 275 | 'Id': 'babababababaabababab' 276 | }, 277 | { 278 | 'RepoTags': ['other:abcda'] 279 | }, 280 | { 281 | 'RepoTags': ['new_image:latest', 'new_image:123'] 282 | }, 283 | ] 284 | actual = docker_gc.filter_excluded_images(images, exclude_set) 285 | assert list(actual) == expected 286 | 287 | 288 | def test_filter_excluded_images_advanced(): 289 | exclude_set = set([ 290 | 'user/one:*', 291 | 'user/foo:tag*', 292 | 'user/repo-*:tag', 293 | ]) 294 | images = [ 295 | { 296 | 'RepoTags': [':'], 297 | 'Id': 'babababababaabababab' 298 | }, 299 | { 300 | 'RepoTags': ['user/one:latest', 'user/one:abcd'] 301 | }, 302 | { 303 | 'RepoTags': ['user/foo:test'] 304 | }, 305 | { 306 | 'RepoTags': ['user/foo:tag123'] 307 | }, 308 | { 309 | 'RepoTags': ['user/repo-1:tag'] 310 | }, 311 | { 312 | 'RepoTags': ['user/repo-2:tag'] 313 | }, 314 | 315 | ] 316 | expected = [ 317 | { 318 | 'RepoTags': [':'], 319 | 'Id': 'babababababaabababab' 320 | }, 321 | { 322 | 'RepoTags': ['user/foo:test'], 323 | }, 324 | ] 325 | actual = docker_gc.filter_excluded_images(images, exclude_set) 326 | assert list(actual) == expected 327 | 328 | 329 | def test_is_image_old(image, now): 330 | assert docker_gc.is_image_old(image, now) 331 | 332 | 333 | def test_is_image_old_false(image, later_time): 334 | assert not docker_gc.is_image_old(image, later_time) 335 | 336 | 337 | def test_remove_image_no_tags(mock_client, image, now): 338 | image_id = 'abcd' 339 | image_summary = {'Id': image_id} 340 | mock_client.inspect_image.return_value = image 341 | docker_gc.remove_image(mock_client, image_summary, now, False) 342 | 343 | mock_client.remove_image.assert_called_once_with(image=image_id) 344 | 345 | 346 | def test_remove_image_new_image_not_removed(mock_client, image, later_time): 347 | image_id = 'abcd' 348 | image_summary = {'Id': image_id} 349 | mock_client.inspect_image.return_value = image 350 | docker_gc.remove_image(mock_client, image_summary, later_time, False) 351 | 352 | assert not mock_client.remove_image.mock_calls 353 | 354 | 355 | def test_remove_image_with_tags(mock_client, image, now): 356 | image_id = 'abcd' 357 | repo_tags = ['user/one:latest', 'user/one:12345'] 358 | image_summary = { 359 | 'Id': image_id, 360 | 'RepoTags': repo_tags 361 | } 362 | mock_client.inspect_image.return_value = image 363 | docker_gc.remove_image(mock_client, image_summary, now, False) 364 | 365 | assert mock_client.remove_image.mock_calls == [ 366 | mock.call(image=tag) for tag in repo_tags 367 | ] 368 | 369 | 370 | def test_api_call_success(): 371 | func = mock.Mock() 372 | container = "abcd" 373 | result = docker_gc.api_call(func, container=container) 374 | func.assert_called_once_with(container="abcd") 375 | assert result == func.return_value 376 | 377 | 378 | def test_api_call_with_timeout(): 379 | func = mock.Mock( 380 | side_effect=requests.exceptions.ReadTimeout("msg"), 381 | __name__="remove_image") 382 | image = "abcd" 383 | 384 | with mock.patch( 385 | 'docker_custodian.docker_gc.log', 386 | autospec=True) as mock_log: 387 | docker_gc.api_call(func, image=image) 388 | 389 | func.assert_called_once_with(image="abcd") 390 | mock_log.warn.assert_called_once_with('Failed to call remove_image ' 391 | + 'image=abcd msg' 392 | ) 393 | 394 | 395 | def test_api_call_with_api_error(): 396 | func = mock.Mock( 397 | side_effect=docker.errors.APIError( 398 | "Ooops", 399 | mock.Mock(status_code=409, reason="Conflict"), 400 | explanation="failed"), 401 | __name__="remove_image") 402 | image = "abcd" 403 | 404 | with mock.patch( 405 | 'docker_custodian.docker_gc.log', 406 | autospec=True) as mock_log: 407 | docker_gc.api_call(func, image=image) 408 | 409 | func.assert_called_once_with(image="abcd") 410 | mock_log.warn.assert_called_once_with(String() & Regex('Error calling remove_image image=abcd 409 Client Error .*')) 411 | 412 | 413 | def days_as_seconds(num): 414 | return num * 60 * 60 * 24 415 | 416 | 417 | def test_get_args_with_defaults(): 418 | opts = docker_gc.get_args(args=[]) 419 | assert opts.timeout == 60 420 | assert opts.dry_run is False 421 | assert opts.max_container_age is None 422 | assert opts.max_image_age is None 423 | 424 | 425 | def test_get_args_with_args(): 426 | with mock.patch( 427 | 'docker_custodian.docker_gc.timedelta_type', 428 | autospec=True 429 | ) as mock_timedelta_type: 430 | opts = docker_gc.get_args(args=[ 431 | '--max-image-age', '30 days', 432 | '--max-container-age', '3d', 433 | ]) 434 | assert mock_timedelta_type.mock_calls == [ 435 | mock.call('30 days'), 436 | mock.call('3d'), 437 | ] 438 | assert opts.max_container_age == mock_timedelta_type.return_value 439 | assert opts.max_image_age == mock_timedelta_type.return_value 440 | 441 | 442 | def test_get_all_containers(mock_client): 443 | count = 10 444 | mock_client.containers.return_value = [mock.Mock() for _ in range(count)] 445 | with mock.patch('docker_custodian.docker_gc.log', 446 | autospec=True) as mock_log: 447 | containers = docker_gc.get_all_containers(mock_client) 448 | assert containers == mock_client.containers.return_value 449 | mock_client.containers.assert_called_once_with(all=True) 450 | mock_log.info.assert_called_with("Found %s containers", count) 451 | 452 | 453 | def test_get_all_images(mock_client): 454 | count = 7 455 | mock_client.images.return_value = [mock.Mock() for _ in range(count)] 456 | with mock.patch('docker_custodian.docker_gc.log', 457 | autospec=True) as mock_log: 458 | images = docker_gc.get_all_images(mock_client) 459 | assert images == mock_client.images.return_value 460 | mock_log.info.assert_called_with("Found %s images", count) 461 | 462 | 463 | def test_get_dangling_volumes(mock_client): 464 | count = 4 465 | mock_client.volumes.return_value = { 466 | 'Volumes': [mock.Mock() for _ in range(count)] 467 | } 468 | with mock.patch('docker_custodian.docker_gc.log', 469 | autospec=True) as mock_log: 470 | volumes = docker_gc.get_dangling_volumes(mock_client) 471 | assert volumes == mock_client.volumes.return_value['Volumes'] 472 | mock_log.info.assert_called_with("Found %s dangling volumes", count) 473 | 474 | 475 | def test_build_exclude_set(): 476 | image_tags = [ 477 | 'some_image:latest', 478 | 'repo/foo:12345', 479 | 'duplicate:latest', 480 | ] 481 | exclude_image_file = StringIO(textwrap.dedent(""" 482 | # Exclude this one because 483 | duplicate:latest 484 | # Also this one 485 | repo/bar:abab 486 | """)) 487 | expected = set([ 488 | 'some_image:latest', 489 | 'repo/foo:12345', 490 | 'duplicate:latest', 491 | 'repo/bar:abab', 492 | ]) 493 | 494 | exclude_set = docker_gc.build_exclude_set(image_tags, exclude_image_file) 495 | assert exclude_set == expected 496 | 497 | 498 | def test_format_exclude_labels(): 499 | exclude_label_args = [ 500 | 'voo*', 501 | 'doo=poo', 502 | ] 503 | expected = [ 504 | docker_gc.ExcludeLabel(key='voo*', value=None), 505 | docker_gc.ExcludeLabel(key='doo', value='poo'), 506 | ] 507 | exclude_labels = docker_gc.format_exclude_labels(exclude_label_args) 508 | assert expected == exclude_labels 509 | 510 | 511 | def test_build_exclude_set_empty(): 512 | exclude_set = docker_gc.build_exclude_set(None, None) 513 | assert exclude_set == set() 514 | 515 | 516 | def test_main(mock_client): 517 | with mock.patch( 518 | 'docker_custodian.docker_gc.docker.APIClient', 519 | return_value=mock_client): 520 | 521 | with mock.patch( 522 | 'docker_custodian.docker_gc.get_args', 523 | autospec=True) as mock_get_args: 524 | mock_get_args.return_value = mock.Mock( 525 | max_image_age=100, 526 | max_container_age=200, 527 | exclude_image=[], 528 | exclude_image_file=None, 529 | exclude_container_label=[], 530 | ) 531 | docker_gc.main() 532 | --------------------------------------------------------------------------------