├── .dockercfg.enc ├── .gitignore ├── .travis.yml ├── .travis ├── build.sh ├── publish.sh └── test.sh ├── CHANGELOG.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── config └── config.go ├── handler └── handler.go ├── image └── image.go ├── images ├── metrics_not_updated.png ├── metrics_up_to_date.png ├── metrics_what.png ├── upkick.png └── upkick.svg ├── main.go └── metrics ├── metrics.go └── metrics_test.go /.dockercfg.enc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/camptocamp/upkick/ba28bb9b9d21dc15e39cdb5ca5e09d01195cf316/.dockercfg.enc -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | upkick 2 | upkick.1 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | sudo: required 3 | services: docker 4 | go: 5 | - 1.8 6 | install: 7 | - go get ./... 8 | - go get github.com/bradfitz/goimports 9 | - go get github.com/mattn/goveralls 10 | - go get golang.org/x/tools/cmd/cover 11 | script: 12 | - make 13 | - "./.travis/build.sh && ./.travis/test.sh && ./.travis/publish.sh" 14 | - make coverage 15 | - "$HOME/gopath/bin/goveralls -service=travis-ci -coverprofile=coverage.out" 16 | deploy: 17 | provider: releases 18 | api_key: 19 | secure: SYx7JMfXLd2RxG8WL+Yxxsvk7YE+7+HnHiYWt15Z1vrVC3nm1lYbBC8Et2FnQQAhKiCPrFs0uT7PzC8RbRoHGODc4rFcWjsvp+tmNqJ7JS/4j6EV6dBkJlXSljWcAfuq+qFJgOp13ZTBOzTimCUNNihqnepaRmeo+eb5rK/Iy1TUDfPOOIMGSSdkpbehedDL83iip38MK+ofrVGGiCfCDVVhI4bnCgCf4ShDo1Ck2O8rBhrhBME/ky2nkIZFM+RUNqS52aGXH25DjIRBl4KRylKA1S2+z7NGVYaSXJtwRJx0iDyMgZhsc7Dt15L9/HluZQLk+x9NGkleITQ2ayAoffNdCQYDTPRp9oyYovbLQu5+T9JVty97UBuw74xosMpRwRQ/6w4Wjv1kqMo3jZZ4MvBtWna4ptHe+qRKXxsbfKqhWzL3DvJVBioOBI4N9qQWfFptabel9Y+ILucJzDRIIFMmebbmT9BZ4bpMHOCgGUj9mWmAkgH/QlD1xL6ObRp3VWKljhPM3Nm6WDJld1RSBloOdbCcCQXRQVgMVl/yFL1aVtBltrR1dlxQgBFlakL6dJeXgCJe3+cv86mC5G9M0X2AvwcH2mVeiUXKVytOaHhIAsAkXx+Zk48iPwkx1m+RqP3/F5D5L1s1LfDD3MvLilLTZWtFphWXr7ytQwUFYcY= 20 | file: 21 | - upkick 22 | - upkick.1 23 | on: 24 | repo: camptocamp/upkick 25 | tags: true 26 | -------------------------------------------------------------------------------- /.travis/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | if [ "$TRAVIS_BRANCH" == "master" ]; then 3 | echo "Building image with tag latest" 4 | docker build -t ${TRAVIS_REPO_SLUG}:latest . 5 | elif [ ! -z "$TRAVIS_TAG" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then 6 | echo "Building image with tag ${TRAVIS_TAG}" 7 | docker build -t ${TRAVIS_REPO_SLUG}:$TRAVIS_TAG . 8 | elif [ ! -z "$TRAVIS_BRANCH" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then 9 | echo "Building image with tag ${TRAVIS_BRANCH}" 10 | docker build -t ${TRAVIS_REPO_SLUG}:$TRAVIS_BRANCH . 11 | else 12 | echo "Don't know how to build image" 13 | exit 1 14 | fi 15 | -------------------------------------------------------------------------------- /.travis/publish.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | openssl aes-256-cbc -K $encrypted_45b586f32ae3_key -iv $encrypted_45b586f32ae3_iv -in .dockercfg.enc -out ~/.dockercfg -d 4 | 5 | if [ "$TRAVIS_BRANCH" == "master" ]; then 6 | echo "Deploying image to docker hub for master (latest)" 7 | docker push "${TRAVIS_REPO_SLUG}:latest" 8 | elif [ ! -z "$TRAVIS_TAG" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then 9 | echo "Deploying image to docker hub for tag ${TRAVIS_TAG}" 10 | docker push "${TRAVIS_REPO_SLUG}:${TRAVIS_TAG}" 11 | elif [ ! -z "$TRAVIS_BRANCH" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then 12 | echo "Deploying image to docker hub for branch ${TRAVIS_BRANCH}" 13 | docker push "${TRAVIS_REPO_SLUG}:${TRAVIS_BRANCH}" 14 | else 15 | echo "Not deploying image" 16 | fi 17 | -------------------------------------------------------------------------------- /.travis/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | exit 0 4 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # [0.7.1](https://github.com/camptocamp/upkick/releases/tag/0.7.1) (2018-07-25) 2 | 3 | * Features: 4 | 5 | - Wait for the pull to be complete before continuing (GH #11) 6 | 7 | # [0.7.0](https://github.com/camptocamp/upkick/releases/tag/0.7.0) (2018-05-02) 8 | 9 | * Features: 10 | 11 | - Also check stopped containers (GH #10) 12 | 13 | # [0.6.0](https://github.com/camptocamp/upkick/releases/tag/0.6.0) (2016-11-21) 14 | 15 | * Metrics: 16 | 17 | - Add container opt-in label 18 | 19 | # [0.5.0](https://github.com/camptocamp/upkick/releases/tag/0.5.0) (2016-09-15) 20 | 21 | * Metrics: 22 | 23 | - Make all metrics per image, with an image label 24 | 25 | # [0.4.0](https://github.com/camptocamp/upkick/releases/tag/0.4.0) (2016-09-15) 26 | 27 | * Metrics: 28 | 29 | - Make counters global 30 | - Use one metric, several events 31 | - Add more metrics for blacklisted containers 32 | 33 | # [0.3.0](https://github.com/camptocamp/upkick/releases/tag/0.3.0) (2016-09-15) 34 | 35 | * Features: 36 | 37 | - Added metrics (fix #2) 38 | 39 | # [0.2.0](https://github.com/camptocamp/upkick/releases/tag/0.2.0) (2016-09-15) 40 | 41 | * Features: 42 | 43 | - Add -w|--warn-only flag 44 | - Add per-container opt-out with labels (fix #1) 45 | - Blacklist critical known tags (fix #3) 46 | 47 | * Documentation: 48 | 49 | - Add Docker documentation 50 | 51 | # [0.1.0](https://github.com/camptocamp/upkick/releases/tag/0.1.0) (2016-09-14) 52 | 53 | * Initial release 54 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM scratch 2 | ADD upkick / 3 | ENTRYPOINT ["/upkick"] 4 | CMD [""] 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | DEPS = $(wildcard */*.go) 2 | VERSION = $(shell git describe --always --dirty) 3 | 4 | all: test upkick upkick.1 5 | 6 | upkick: main.go $(DEPS) 7 | CGO_ENABLED=0 GOOS=linux \ 8 | go build -a \ 9 | -ldflags="-X main.version=$(VERSION)" \ 10 | -installsuffix cgo -o $@ $< 11 | strip $@ 12 | 13 | upkick.1: upkick 14 | ./upkick -m > $@ 15 | 16 | lint: 17 | @ go get -v github.com/golang/lint/golint 18 | @for file in $$(git ls-files '*.go' | grep -v '_workspace/'); do \ 19 | export output="$$(golint $${file} | grep -v 'type name will be used as docker.DockerInfo')"; \ 20 | [ -n "$${output}" ] && echo "$${output}" && export status=1; \ 21 | done; \ 22 | exit $${status:-0} 23 | 24 | vet: main.go 25 | go vet $< 26 | 27 | imports: main.go 28 | goimports -d $< 29 | 30 | test: lint vet imports 31 | go test -v ./... 32 | 33 | coverage: 34 | rm -rf *.out 35 | echo "mode: set" > coverage.out 36 | go test -coverprofile=coverage.out 37 | for i in config handler image metrics; do \ 38 | go test -coverprofile=$$i.coverage.out github.com/camptocamp/upkick/$$i; \ 39 | tail -n +2 $$i.coverage.out >> coverage.out; \ 40 | done 41 | 42 | clean: 43 | rm -f upkick upkick.1 44 | 45 | .PHONY: all lint vet imports test coverage clean 46 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Upkick 2 | ====== 3 | 4 | [![Docker Pulls](https://img.shields.io/docker/pulls/camptocamp/upkick.svg)](https://hub.docker.com/r/camptocamp/upkick/) 5 | [![Build Status](https://img.shields.io/travis/camptocamp/upkick/master.svg)](https://travis-ci.org/camptocamp/upkick) 6 | [![Coverage Status](https://img.shields.io/coveralls/camptocamp/upkick.svg)](https://coveralls.io/r/camptocamp/upkick?branch=master) 7 | [![Go Report Card](https://goreportcard.com/badge/github.com/camptocamp/upkick)](https://goreportcard.com/report/github.com/camptocamp/upkick) 8 | [![By Camptocamp](https://img.shields.io/badge/by-camptocamp-fb7047.svg)](http://www.camptocamp.com) 9 | 10 | 11 | Unattended upgrades for Docker containers, the hard way. 12 | 13 | ![Upkick](images/upkick.png) 14 | 15 | 16 | ## Installing 17 | 18 | ```shell 19 | $ go get github.com/camptocamp/upkick 20 | ``` 21 | 22 | ## What does it do? 23 | 24 | Upkick helps you keep your containers up-to-date. When you launch it, it performs the following actions: 25 | 26 | * list all running containers on the Docker socket 27 | * update (pull) all images used in the containers 28 | * unless `--warn-only` is used, stop/remove all containers using outdated images (you need an orchestrator such as Rancher to restart them) 29 | * if a Prometheus gateway is provided, push metrics to it 30 | 31 | 32 | ## Isn't that what catalog templates are for? 33 | 34 | No. Catalog templates usually use tagged versions of images. However, Docker tags are not fixed: they correspond more to branches actually. So while a new catalog template might upgrade the tag and force an upgrade of a container, there is never a garantee that the container runs on the latest hash for the given tag. This is a concern for security, as images should be rebuilt on a regular basis. 35 | 36 | Rancher allows to set containers to "Always pull image before creating". While this is useful, it only garantees that images are updated when containers are recreated. We want containers to be up-to-date all the time! 37 | 38 | 39 | ## Usage 40 | 41 | ```shell 42 | Usage: 43 | upkick [OPTIONS] 44 | 45 | Application Options: 46 | -V, --version Display version. 47 | -l, --loglevel= Set loglevel ('debug', 'info', 'warn', 'error', 'fatal', 'panic'). (default: info) [$UPKICK_LOG_LEVEL] 48 | -m, --manpage Output manpage. 49 | -j, --json Log as JSON (to stderr). [$UPKICK_JSON_OUTPUT] 50 | -w, --warn-only Only warn, do not kick out-of-date containers. [$UPKICK_WARN_ONLY] 51 | -H, --hostname-from-rancher Retrieve hostname from Rancher metadata. [$CONPLICITY_HOSTNAME_FROM_RANCHER] 52 | 53 | Docker Options: 54 | -e, --docker-endpoint= The Docker endpoint. (default: unix:///var/run/docker.sock) [$DOCKER_ENDPOINT] 55 | 56 | Metrics Options: 57 | -g, --gateway-url= The prometheus push gateway URL to use. [$PUSHGATEWAY_URL] 58 | 59 | Help Options: 60 | -h, --help Show this help message 61 | ``` 62 | 63 | ## Using the Docker image 64 | 65 | ```shell 66 | $ docker run -v /var/run/docker.sock:/var/run/docker.sock:ro --rm -ti camptocamp/upkick 67 | ``` 68 | 69 | ## Per container opt-out 70 | 71 | You can set containers to only warn if they are outdated by placing an `io.upkick.warn_only=true` label on them. 72 | 73 | ## Per container opt-in 74 | 75 | If the global `--warn-only` flag is passed, you can opt-in for container kicking by placing an `io.upkick.warn_only=false` label on them. 76 | 77 | 78 | ## Metrics 79 | 80 | Upkick can push metrics to a Prometheus gateway. The currently exported metrics are: 81 | 82 | * `upkick_containers{what="total",image=""}`: total number of containers using a given image tag 83 | * `upkick_containers{what="blacklisted_tag",image=""}`: number of containers using a given image that is blacklisted 84 | * `upkick_containers{what="blacklisted_container",image=""}`: number of containers that opted-out of update (using labels) 85 | * `upkick_containers{what="up_to_date",image=""}`: number of containers using a given image already up-to-date 86 | * `upkick_containers{what="updated",image=""}`: number of containers using a given image successfully updated 87 | * `upkick_containers{what="update_failed",image=""}`: number of containers using a given image whose update failed 88 | * `upkick_containers{what="not_updated",image=""}`: number of containers using a given image that were not updated (because `--warn-only` was used) 89 | 90 | 91 | Here are some examples of useful Prometheus queries using these metrics: 92 | 93 | ![Out-of-date containers](images/metrics_not_updated.png) 94 | ![All states](images/metrics_what.png) 95 | ![Up-to-date containers](images/metrics_up_to_date.png) 96 | 97 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/jessevdk/go-flags" 9 | ) 10 | 11 | // Config stores the handler's configuration and UI interface parameters 12 | type Config struct { 13 | Version bool `short:"V" long:"version" description:"Display version."` 14 | Loglevel string `short:"l" long:"loglevel" description:"Set loglevel ('debug', 'info', 'warn', 'error', 'fatal', 'panic')." env:"UPKICK_LOG_LEVEL" default:"info"` 15 | Manpage bool `short:"m" long:"manpage" description:"Output manpage."` 16 | JSON bool `short:"j" long:"json" description:"Log as JSON (to stderr)." env:"UPKICK_JSON_OUTPUT"` 17 | Warn bool `short:"w" long:"warn-only" description:"Only warn, do not kick out-of-date containers." env:"UPKICK_WARN_ONLY"` 18 | HostnameFromRancher bool `short:"H" long:"hostname-from-rancher" description:"Retrieve hostname from Rancher metadata." env:"CONPLICITY_HOSTNAME_FROM_RANCHER"` 19 | 20 | Docker struct { 21 | Endpoint string `short:"e" long:"docker-endpoint" description:"The Docker endpoint." env:"DOCKER_ENDPOINT" default:"unix:///var/run/docker.sock"` 22 | } `group:"Docker Options"` 23 | 24 | Metrics struct { 25 | PushgatewayURL string `short:"g" long:"gateway-url" description:"The prometheus push gateway URL to use." env:"PUSHGATEWAY_URL"` 26 | } `group:"Metrics Options"` 27 | } 28 | 29 | // LoadConfig loads the config from flags & environment 30 | func LoadConfig(version string) *Config { 31 | var c Config 32 | parser := flags.NewParser(&c, flags.Default) 33 | if _, err := parser.Parse(); err != nil { 34 | os.Exit(1) 35 | } 36 | 37 | if c.Version { 38 | fmt.Printf("Upkick v%v\n", version) 39 | os.Exit(0) 40 | } 41 | 42 | if c.Manpage { 43 | var buf bytes.Buffer 44 | parser.ShortDescription = "Unattended upgrades for Docker containers" 45 | parser.LongDescription = `Upkick pulls Docker images and removes containers using obsolete images. 46 | 47 | Make sure your Docker orchestrator is set to recreate the containers. 48 | ` 49 | parser.WriteManPage(&buf) 50 | fmt.Printf(buf.String()) 51 | os.Exit(0) 52 | } 53 | return &c 54 | } 55 | -------------------------------------------------------------------------------- /handler/handler.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "net/http" 9 | "os" 10 | "strconv" 11 | "strings" 12 | "time" 13 | 14 | log "github.com/Sirupsen/logrus" 15 | "github.com/pkg/errors" 16 | 17 | "github.com/docker/docker/api/types" 18 | docker "github.com/docker/docker/client" 19 | 20 | "github.com/camptocamp/upkick/config" 21 | "github.com/camptocamp/upkick/image" 22 | "github.com/camptocamp/upkick/metrics" 23 | ) 24 | 25 | var blacklist = []string{ 26 | "rancher/agent", 27 | "rancher/agent-instance", 28 | "camptocamp/upkick", 29 | } 30 | 31 | // Upkick is an upkick handler 32 | type Upkick struct { 33 | Client *docker.Client 34 | Config *config.Config 35 | Hostname string 36 | Metrics *metrics.PrometheusMetrics 37 | } 38 | 39 | // NewUpkick returns a new Upkick handler 40 | func NewUpkick(version string) (*Upkick, error) { 41 | u := &Upkick{} 42 | err := u.setup(version) 43 | return u, err 44 | } 45 | 46 | // GetImages returns a slice of Image 47 | func (u *Upkick) GetImages() (images map[string]*image.Image, err error) { 48 | log.Debug("Getting images") 49 | containers, err := u.Client.ContainerList(context.Background(), types.ContainerListOptions{ 50 | All: true, 51 | }) 52 | if err != nil { 53 | err = errors.Wrap(err, "failed to list containers") 54 | return 55 | } 56 | 57 | images = make(map[string]*image.Image) 58 | 59 | containersTotal := make(map[string]int) 60 | blacklistedTags := make(map[string]int) 61 | blacklistedContainers := make(map[string]int) 62 | 63 | for _, c := range containers { 64 | cont, err := u.Client.ContainerInspect(context.Background(), c.ID) 65 | if err != nil { 66 | log.Errorf("failed to inspect container %s: %v", c.ID, err) 67 | continue 68 | } 69 | 70 | tag := cont.Config.Image 71 | containersTotal[tag]++ 72 | 73 | if blacklistedTag(tag) { 74 | log.Debugf("Ignoring blacklisted image tag %s", tag) 75 | blacklistedTags[tag]++ 76 | continue 77 | } 78 | 79 | if blacklistedContainer(cont) { 80 | log.Debugf("Ignoring blacklisted container %s", cont.ID) 81 | blacklistedContainers[tag]++ 82 | continue 83 | } 84 | 85 | i, ok := images[c.Image] 86 | if !ok { 87 | images[tag] = &image.Image{ 88 | ID: tag, 89 | } 90 | i = images[tag] 91 | i.Hashes = make(map[string]*image.Hash) 92 | } 93 | h, ok := i.Hashes[c.ImageID] 94 | if !ok { 95 | i.Hashes[c.ImageID] = &image.Hash{} 96 | h = i.Hashes[c.ImageID] 97 | } 98 | log.Debugf("Adding %s with hash %s to %s", c.ID, c.ImageID, tag) 99 | h.Containers = append(h.Containers, c.ID) 100 | } 101 | 102 | var m *metrics.Metric 103 | for _, i := range images { 104 | tag := i.ID 105 | m = u.Metrics.NewMetric("upkick_containers", "gauge") 106 | m.NewEvent(&metrics.Event{ 107 | Value: strconv.Itoa(containersTotal[tag]), 108 | Labels: map[string]string{ 109 | "what": "total", 110 | "image": tag, 111 | }, 112 | }) 113 | m.NewEvent(&metrics.Event{ 114 | Value: strconv.Itoa(blacklistedTags[tag]), 115 | Labels: map[string]string{ 116 | "what": "blacklisted_tag", 117 | "image": tag, 118 | }, 119 | }) 120 | m.NewEvent(&metrics.Event{ 121 | Value: strconv.Itoa(blacklistedContainers[tag]), 122 | Labels: map[string]string{ 123 | "what": "blacklisted_container", 124 | "image": tag, 125 | }, 126 | }) 127 | } 128 | 129 | return 130 | } 131 | 132 | // Pull pulls the newest version of an image 133 | func (u *Upkick) Pull(i *image.Image) (err error) { 134 | log.Debugf("Pulling Image %s", i) 135 | 136 | var pullOut io.ReadCloser 137 | pullOut, err = u.Client.ImagePull(context.Background(), i.ID, types.ImagePullOptions{}) 138 | if err != nil { 139 | msg := fmt.Sprintf("failed to pull image %s", i.ID) 140 | return errors.Wrap(err, msg) 141 | } 142 | 143 | // Wait for the image to be fully pulled 144 | io.Copy(ioutil.Discard, pullOut); 145 | pullOut.Close() 146 | 147 | img, _, err := u.Client.ImageInspectWithRaw(context.Background(), i.ID) 148 | if err != nil { 149 | msg := fmt.Sprintf("failed to inspect image %s", i.ID) 150 | return errors.Wrap(err, msg) 151 | } 152 | 153 | i.Hash = img.ID 154 | log.Infof("Image %s updated to %v", i, i.Hash) 155 | 156 | return 157 | } 158 | 159 | // Kick stops and removes all containers 160 | // using an obsolete version of the Image 161 | func (u *Upkick) Kick(i *image.Image) (err error) { 162 | log.Debugf("Kicking containers for Image %s", i) 163 | 164 | var noup int 165 | var outWarn int 166 | var upOK int 167 | var upNOK int 168 | 169 | for hash, hashS := range i.Hashes { 170 | if hash == i.Hash { 171 | // Already up-to-date 172 | log.Debugf("Not kicking containers for up-to-date hash %s", hash) 173 | noup += len(hashS.Containers) 174 | continue 175 | } 176 | 177 | for _, c := range hashS.Containers { 178 | cont, err := u.Client.ContainerInspect(context.Background(), c) 179 | if err != nil { 180 | log.Errorf("failed to inspect container %s: %v", c, err) 181 | continue 182 | } 183 | 184 | warnOnly, warnLabel := cont.Config.Labels["io.upkick.warn_only"] 185 | 186 | if u.Config.Warn && !(warnLabel && warnOnly == "false") { 187 | log.Warnf("Container %s uses an out-of-date image", c) 188 | outWarn++ 189 | continue 190 | } 191 | if cont.State.Running { 192 | log.Infof("Stopping container %s", c) 193 | timeout := 10 * time.Second 194 | err = u.Client.ContainerStop(context.Background(), c, &timeout) 195 | if err != nil { 196 | upNOK++ 197 | log.Errorf("failed to stop container %s: %v", c, err) 198 | continue 199 | } 200 | } else { 201 | log.Infof("Container %s already stopped", c) 202 | } 203 | 204 | log.Infof("Removing container %s", c) 205 | err = u.Client.ContainerRemove(context.Background(), c, types.ContainerRemoveOptions{}) 206 | if err != nil { 207 | upNOK++ 208 | log.Errorf("failed to remove container %s: %v", c, err) 209 | continue 210 | } 211 | upOK++ 212 | } 213 | } 214 | 215 | m := u.Metrics.NewMetric("upkick_containers", "gauge") 216 | m.NewEvent(&metrics.Event{ 217 | Value: strconv.Itoa(noup), 218 | Labels: map[string]string{ 219 | "what": "up_to_date", 220 | "image": i.ID, 221 | }, 222 | }) 223 | m.NewEvent(&metrics.Event{ 224 | Value: strconv.Itoa(upOK), 225 | Labels: map[string]string{ 226 | "what": "updated", 227 | "image": i.ID, 228 | }, 229 | }) 230 | m.NewEvent(&metrics.Event{ 231 | Value: strconv.Itoa(upNOK), 232 | Labels: map[string]string{ 233 | "what": "update_failed", 234 | "image": i.ID, 235 | }, 236 | }) 237 | m.NewEvent(&metrics.Event{ 238 | Value: strconv.Itoa(outWarn), 239 | Labels: map[string]string{ 240 | "what": "not_updated", 241 | "image": i.ID, 242 | }, 243 | }) 244 | 245 | return 246 | } 247 | 248 | // PushMetrics pushes metrics for the handler 249 | func (u *Upkick) PushMetrics() { 250 | /* Metrics per node: 251 | * 252 | * - States (gauges): 253 | * - NOUP: up-to-date containers 254 | * - UP-OK: container successfully updated 255 | * - UP-NOK: container failed to update 256 | * - OUT-WARN: container is out-of-date but not updated 257 | * 258 | * - Image timestamp per hash (counter) 259 | */ 260 | u.Metrics.Push() 261 | } 262 | 263 | func (u *Upkick) setup(version string) (err error) { 264 | u.Config = config.LoadConfig(version) 265 | 266 | err = u.setupLoglevel() 267 | if err != nil { 268 | return errors.Wrap(err, "failed to setup log level") 269 | } 270 | 271 | err = u.getHostname() 272 | if err != nil { 273 | return errors.Wrap(err, "failed to get hostname") 274 | } 275 | 276 | err = u.setupDocker() 277 | if err != nil { 278 | return errors.Wrap(err, "failed to setup Docker") 279 | } 280 | 281 | err = u.setupMetrics() 282 | if err != nil { 283 | return errors.Wrap(err, "failed to setup metrics") 284 | } 285 | 286 | return 287 | } 288 | 289 | func (u *Upkick) setupLoglevel() (err error) { 290 | switch u.Config.Loglevel { 291 | case "debug": 292 | log.SetLevel(log.DebugLevel) 293 | case "info": 294 | log.SetLevel(log.InfoLevel) 295 | case "warn": 296 | log.SetLevel(log.WarnLevel) 297 | case "error": 298 | log.SetLevel(log.ErrorLevel) 299 | case "fatal": 300 | log.SetLevel(log.FatalLevel) 301 | case "panic": 302 | log.SetLevel(log.PanicLevel) 303 | default: 304 | errMsg := fmt.Sprintf("Wrong log level '%v'", u.Config.Loglevel) 305 | err = errors.New(errMsg) 306 | } 307 | 308 | if u.Config.JSON { 309 | log.SetFormatter(&log.JSONFormatter{}) 310 | } 311 | 312 | return 313 | } 314 | 315 | func (u *Upkick) getHostname() (err error) { 316 | if u.Config.HostnameFromRancher { 317 | resp, err := http.Get("http://rancher-metadata/latest/self/host/name") 318 | if err != nil { 319 | return err 320 | } 321 | defer resp.Body.Close() 322 | body, err := ioutil.ReadAll(resp.Body) 323 | if err != nil { 324 | return err 325 | } 326 | u.Hostname = string(body) 327 | } else { 328 | u.Hostname, err = os.Hostname() 329 | } 330 | return 331 | } 332 | 333 | func (u *Upkick) setupDocker() (err error) { 334 | u.Client, err = docker.NewClient(u.Config.Docker.Endpoint, "", nil, nil) 335 | return 336 | } 337 | 338 | func (u *Upkick) setupMetrics() (err error) { 339 | u.Metrics = metrics.NewMetrics(u.Hostname, u.Config.Metrics.PushgatewayURL) 340 | return 341 | } 342 | 343 | func blacklistedTag(tag string) bool { 344 | baseImage := strings.Split(tag, ":")[0] 345 | 346 | for _, b := range blacklist { 347 | if baseImage == b { 348 | return true 349 | } 350 | } 351 | 352 | return false 353 | } 354 | 355 | func blacklistedContainer(cont types.ContainerJSON) bool { 356 | if l, ok := cont.Config.Labels["io.upkick.warn_only"]; ok && l == "true" { 357 | return true 358 | } 359 | 360 | return false 361 | } 362 | -------------------------------------------------------------------------------- /image/image.go: -------------------------------------------------------------------------------- 1 | package image 2 | 3 | // Image is a Docker image 4 | type Image struct { 5 | ID string 6 | Hash string 7 | Hashes map[string]*Hash 8 | } 9 | 10 | // Hash is a specific Image 11 | type Hash struct { 12 | Containers []string 13 | } 14 | 15 | // String returns the string representation of an Image 16 | func (i *Image) String() string { 17 | return i.ID 18 | } 19 | -------------------------------------------------------------------------------- /images/metrics_not_updated.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/camptocamp/upkick/ba28bb9b9d21dc15e39cdb5ca5e09d01195cf316/images/metrics_not_updated.png -------------------------------------------------------------------------------- /images/metrics_up_to_date.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/camptocamp/upkick/ba28bb9b9d21dc15e39cdb5ca5e09d01195cf316/images/metrics_up_to_date.png -------------------------------------------------------------------------------- /images/metrics_what.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/camptocamp/upkick/ba28bb9b9d21dc15e39cdb5ca5e09d01195cf316/images/metrics_what.png -------------------------------------------------------------------------------- /images/upkick.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/camptocamp/upkick/ba28bb9b9d21dc15e39cdb5ca5e09d01195cf316/images/upkick.png -------------------------------------------------------------------------------- /images/upkick.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 20 | 22 | 32 | 36 | 43 | 49 | 57 | 61 | 66 | 72 | 77 | 85 | 89 | 90 | 96 | 106 | 113 | 118 | 119 | 129 | 133 | 140 | 146 | 154 | 158 | 163 | 169 | 174 | 182 | 186 | 187 | 193 | 203 | 210 | 215 | 216 | 217 | 235 | 237 | 238 | 240 | image/svg+xml 241 | 243 | 244 | 245 | 246 | 247 | 252 | 255 | 258 | 265 | 272 | 279 | 286 | 293 | 294 | 297 | 304 | 311 | 318 | 325 | 332 | 333 | 336 | 343 | 350 | 357 | 364 | 371 | 372 | 375 | 382 | 389 | 396 | 403 | 410 | 411 | 414 | 1148 | 1155 | 1162 | 1163 | 1166 | 1900 | 1907 | 1914 | 1915 | 1916 | 1917 | 1918 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | log "github.com/Sirupsen/logrus" 7 | "github.com/camptocamp/upkick/handler" 8 | ) 9 | 10 | var version = "undefined" 11 | var kicker *handler.Upkick 12 | 13 | func init() { 14 | var err error 15 | kicker, err = handler.NewUpkick(version) 16 | if err != nil { 17 | log.Fatal(err.Error()) 18 | } 19 | } 20 | 21 | func main() { 22 | var err error 23 | 24 | log.Infof("Upkick v%s starting", version) 25 | 26 | images, err := kicker.GetImages() 27 | if err != nil { 28 | log.Errorf(err.Error()) 29 | os.Exit(1) 30 | } 31 | 32 | for _, i := range images { 33 | err = kicker.Pull(i) 34 | if err != nil { 35 | log.Errorf("Failed to pull image %s: %v", i, err) 36 | } 37 | 38 | err = kicker.Kick(i) 39 | if err != nil { 40 | log.Errorf("Failed to kick containers for image %s: %v", i, err) 41 | } 42 | } 43 | 44 | kicker.PushMetrics() 45 | } 46 | -------------------------------------------------------------------------------- /metrics/metrics.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io/ioutil" 7 | "net/http" 8 | "strings" 9 | 10 | log "github.com/Sirupsen/logrus" 11 | ) 12 | 13 | // PrometheusMetrics is a struct to push metrics to Prometheus 14 | type PrometheusMetrics struct { 15 | Instance string 16 | PushgatewayURL string 17 | Metrics map[string]*Metric 18 | } 19 | 20 | // Metric is a Prometheus Metric 21 | type Metric struct { 22 | Name string 23 | Events []*Event 24 | Type string 25 | } 26 | 27 | // Event is a Prometheus Metric Event 28 | type Event struct { 29 | Name string 30 | Labels map[string]string 31 | Value string 32 | } 33 | 34 | // NewMetrics returns a new metrics struct 35 | func NewMetrics(instance, pushgatewayURL string) *PrometheusMetrics { 36 | return &PrometheusMetrics{ 37 | Instance: instance, 38 | PushgatewayURL: pushgatewayURL, 39 | Metrics: make(map[string]*Metric), 40 | } 41 | } 42 | 43 | // String formats an event for printing 44 | func (e *Event) String() string { 45 | var labels []string 46 | for l, v := range e.Labels { 47 | labels = append(labels, fmt.Sprintf("%s=\"%s\"", l, v)) 48 | } 49 | return fmt.Sprintf("%s{%s} %s", e.Name, strings.Join(labels, ","), e.Value) 50 | } 51 | 52 | // NewEvent adds an event to a Metric 53 | func (m *Metric) NewEvent(e *Event) { 54 | e.Name = m.Name 55 | m.Events = append(m.Events, e) 56 | } 57 | 58 | // NewMetric adds a new metric if it doesn't exist yet 59 | // or returns the existing matching metric otherwise 60 | func (p *PrometheusMetrics) NewMetric(name, mType string) (m *Metric) { 61 | m, ok := p.Metrics[name] 62 | if !ok { 63 | m = &Metric{ 64 | Name: name, 65 | } 66 | p.Metrics[name] = m 67 | } 68 | m.Type = mType 69 | return 70 | } 71 | 72 | // Push sends metrics to a Prometheus push gateway 73 | func (p *PrometheusMetrics) Push() (err error) { 74 | if p.PushgatewayURL == "" { 75 | log.Debug("No Pushgateway URL specified, not pushing metrics") 76 | return 77 | } 78 | metrics := p.Metrics 79 | url := p.PushgatewayURL + "/metrics/job/upkick/instance/" + p.Instance 80 | 81 | var data string 82 | for _, m := range metrics { 83 | if m.Type != "" { 84 | data += fmt.Sprintf("# TYPE %s %s\n", m.Name, m.Type) 85 | } 86 | for _, e := range m.Events { 87 | data += fmt.Sprintf("%s\n", e) 88 | } 89 | } 90 | data += "\n" 91 | 92 | log.WithFields(log.Fields{ 93 | "data": data, 94 | "url": url, 95 | }).Debug("Sending metrics to Prometheus Pushgateway") 96 | 97 | req, err := http.NewRequest("PUT", url, bytes.NewBufferString(data)) 98 | if err != nil { 99 | err = fmt.Errorf("failed to create HTTP request: %v", err) 100 | return 101 | } 102 | 103 | req.Header.Set("Content-Type", "text/plain; version=0.0.4") 104 | 105 | client := &http.Client{} 106 | resp, err := client.Do(req) 107 | if err != nil { 108 | err = fmt.Errorf("failed to get HTTP response: %v", err) 109 | return 110 | } 111 | 112 | defer resp.Body.Close() 113 | body, err := ioutil.ReadAll(resp.Body) 114 | if err != nil { 115 | err = fmt.Errorf("failed to read HTTP response: %v", err) 116 | return 117 | } 118 | 119 | log.WithFields(log.Fields{ 120 | "resp": string(body), 121 | }).Debug("Received Prometheus response") 122 | 123 | return 124 | } 125 | -------------------------------------------------------------------------------- /metrics/metrics_test.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import "testing" 4 | 5 | func TestEventString(t *testing.T) { 6 | e := &Event{ 7 | Name: "foo", 8 | Labels: map[string]string{ 9 | "volume": "baz", 10 | "instance": "qux", 11 | }, 12 | Value: "bar", 13 | } 14 | expected := "foo{volume=\"baz\",instance=\"qux\"} bar" 15 | expected2 := "foo{instance=\"qux\", volume=\"baz\"} bar" 16 | if e.String() != expected && e.String() != expected2 { 17 | t.Fatalf("Expected <%s>, got <%s>", expected, e.String()) 18 | } 19 | } 20 | 21 | func TestNewMetrics(t *testing.T) { 22 | p := NewMetrics("foo", "http://foo:9091") 23 | 24 | if p.Instance != "foo" { 25 | t.Fatalf("Expected instance to be foo, got %s", p.Instance) 26 | } 27 | 28 | if p.PushgatewayURL != "http://foo:9091" { 29 | t.Fatalf("Expected URL to be http://foo:9091, got %s", p.PushgatewayURL) 30 | } 31 | 32 | if len(p.Metrics) != 0 { 33 | t.Fatalf("Expected empty Metrics array, got size %v", len(p.Metrics)) 34 | } 35 | } 36 | 37 | func TestNewMetric(t *testing.T) { 38 | p := NewMetrics("foo", "http://foo:9091") 39 | m := p.NewMetric("bar", "qux") 40 | 41 | if len(p.Metrics) != 1 { 42 | t.Fatalf("Expected 1 metric, got %v", len(p.Metrics)) 43 | } 44 | 45 | if p.Metrics["bar"] != m { 46 | t.Fatal("Expected to find metric in handler") 47 | } 48 | 49 | if m.Name != "bar" { 50 | t.Fatalf("Expected name to be bar, got %s", m.Name) 51 | } 52 | 53 | if m.Type != "qux" { 54 | t.Fatalf("Expected type to be qux, got %s", m.Name) 55 | } 56 | } 57 | --------------------------------------------------------------------------------