├── .docker-make.yml ├── .gitignore ├── .travis.yml ├── Dockerfile ├── Dockerfile.test ├── LICENSE.txt ├── MANIFEST.in ├── README.md ├── __init__.py ├── dmake ├── __init__.py ├── build.py ├── cli.py ├── errors.py ├── template_args.py └── utils.py ├── docs └── yaml-configuration-reference.md ├── requirements.pip ├── setup.cfg ├── setup.py ├── test-requirements.pip ├── tests ├── __init__.py └── test_template_args.py └── tox.ini /.docker-make.yml: -------------------------------------------------------------------------------- 1 | tag-names: 2 | - name: datetime 3 | type: datetime 4 | value: '%Y%m%d%H%M' 5 | 6 | builds: 7 | test: 8 | context: / 9 | dockerfile: Dockerfile.test 10 | dockerignore: 11 | - .git 12 | 13 | docker-make: 14 | context: / 15 | dockerfile: Dockerfile 16 | pushes: 17 | - 'always=jizhilong/docker-make:{fcommitid}' 18 | - 'always=jizhilong/docker-make:{datetime}' 19 | - 'on_tag=jizhilong/docker-make:{git_tag}' 20 | - 'on_branch:master=jizhilong/docker-make:latest' 21 | labels: 22 | - 'com.dockermake.git.describe={git_describe}' 23 | dockerignore: 24 | - .git 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | *.pyc 3 | dist 4 | MANIFEST 5 | .venv 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | language: python 3 | python: 4 | - "2.7" 5 | services: 6 | - docker 7 | install: 8 | - pip install . 9 | script: 10 | - docker-make -d --no-push 11 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:2.7.11 2 | MAINTAINER Ji.Zhilong 3 | 4 | ADD requirements.pip /tmp/ 5 | RUN pip install -r /tmp/requirements.pip 6 | 7 | ADD . /usr/local/src/docker-make 8 | 9 | RUN pip install /usr/local/src/docker-make 10 | -------------------------------------------------------------------------------- /Dockerfile.test: -------------------------------------------------------------------------------- 1 | FROM python:2.7.11 2 | MAINTAINER Ji.Zhilong 3 | 4 | ADD . /usr/local/src/docker-make 5 | WORKDIR /usr/local/src/docker-make 6 | 7 | RUN pip install . &&\ 8 | pip install -r test-requirements.pip &&\ 9 | flake8 dmake && nosetests tests/ 10 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include requirements.pip test-requirements.pip 2 | exclude tests 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # docker-make 2 | [![Build Status](https://travis-ci.org/CtripCloud/docker-make.svg?branch=master)](https://travis-ci.org/CtripCloud/docker-make) [![Docker Pulls](https://img.shields.io/docker/pulls/jizhilong/docker-make.svg?maxAge=2592000)]() 3 | 4 | `docker-make` is a command line tool inspired by [docker-compose](https://www.docker.com/products/docker-compose), while `docker-compose` 5 | focus on managing the lifecycle of a bunch of related docker containers, `docker-make` aimes at simplify and automate the procedure of 6 | building,tagging,and pusing a bunch of related docker images. 7 | 8 | ## table of contents 9 | - [Install](#installation) 10 | * [via pip](#install-via-pip) 11 | * [via alias to docker run](#instanll-via-alias-to-docker-run) 12 | - [Quickstart](#quickstart) 13 | - [Use Cases](#typical-use-cases) 14 | * [single image-tag,push on condition](#single-image-tagpush-on-condition) 15 | * [two images-one for compile, one for deployment](#two-images-one-for-compile-one-for-deployment) 16 | * [two images-one for deploy, one for unit tests](#two-images-one-for-deploy-one-for-unit-tests) 17 | * [several images-one as base with tools, others for different applications](#several-images-one-as-base-with-tools-others-for-different-applications) 18 | - [CLI reference](#command-line-reference) 19 | - [.docker-make.yml reference](docs/yaml-configuration-reference.md) 20 | 21 | ## installation 22 | ### install via pip 23 | `docker-make` is a Python project, so you can install it via pip: 24 | 25 | ```bash 26 | pip install docker-make 27 | ``` 28 | 29 | ### install via alias to docker run 30 | or you can just execute it with a `docker run` command, and for convenience, you can set an alias: 31 | 32 | #### unixy 33 | 34 | ```bash 35 | alias docker-make="docker run --rm -w /usr/src/app \ 36 | -v ~/.docker:/root/.docker \ 37 | -v /var/run/docker.sock:/var/run/docker.sock \ 38 | -v "${PWD}:/usr/src/app" jizhilong/docker-make docker-make" 39 | ``` 40 | 41 | #### windows 42 | 43 | ```ps 44 | function docker-make { 45 | docker run --rm -w /build ` 46 | -v "${HOME}/.docker:/root/.docker" ` 47 | -v /var/run/docker.sock:/var/run/docker.sock ` 48 | -v "${PWD}:/build" ` 49 | jizhilong/docker-make docker-make 50 | } 51 | ``` 52 | 53 | ## quickstart 54 | `docker-make` is a user is itself, so you can build a image for it with the following command: 55 | 56 | ```bash 57 | git clone https://github.com/CtripCloud/docker-make.git 58 | cd docker-make 59 | docker-make --no-push 60 | ``` 61 | 62 | if all goes well, the console output would look like: 63 | 64 | ``` 65 | INFO 2016-06-19 01:21:49,513 docker-make(277) docker-make: building 66 | INFO 2016-06-19 01:21:49,657 docker-make(277) docker-make: attaching labels 67 | INFO 2016-06-19 01:21:49,748 docker-make(277) docker-make: label added: com.dockermake.git.commitid="3d97f0fc382e1f90f77584bbc8193509b835fce0" 68 | INFO 2016-06-19 01:21:49,748 docker-make(277) docker-make: build succeed: c4391b6110f6 69 | INFO 2016-06-19 01:21:49,756 docker-make(277) docker-make: tag added: jizhilong/docker-make:3d97f0fc382e1f90f77584bbc8193509b835fce0 70 | INFO 2016-06-19 01:21:49,760 docker-make(277) docker-make: tag added: jizhilong/docker-make:latest 71 | ``` 72 | 73 | ## how it works 74 | docker-make read and parse `.docker-make.yml`(configurable via command line) in the root of a git repo, 75 | in which you specify images to build, each build's Dockerfile, context, repo to push, rules for tagging, dependencies, etc. 76 | 77 | With information parsed from `.docker-make.yml`, `docker-make` will build, tag, push images in a appropriate order with 78 | regarding to dependency relations. 79 | 80 | ## typical use cases 81 | ### single image-tag,push on condition 82 | this is the most common use case, and `docker-compose` belongs to such case: 83 | 84 | ```yaml 85 | builds: 86 | docker-make: 87 | context: / 88 | dockerfile: Dockerfile 89 | pushes: 90 | - 'always=jizhilong/docker-make:{fcommitid}' 91 | - 'on_tag=jizhilong/docker-make:{git_tag}' 92 | - 'on_branch:master=jizhilong/docker-make:latest' 93 | labels: 94 | - 'com.dockermake.git.commitid={fcommitid}' 95 | dockerignore: 96 | - .git 97 | ``` 98 | 99 | ### two images-one for compile, one for deployment 100 | 101 | ```yaml 102 | builds: 103 | dwait: 104 | context: / 105 | dockerfile: Dockerfile.dwait 106 | extract: 107 | - /usr/src/dwait/bin/.:./dwait.bin.tar 108 | 109 | dresponse: 110 | context: / 111 | dockerfile: Dockerfile 112 | pushes: 113 | - 'on_branch:master=hub.privateregistry.com/dresponse:latest' 114 | - 'on_branch:master=hub.privateregistry.com/dresponse:{fcommitid}' 115 | depends_on: 116 | - dwait 117 | ``` 118 | 119 | In this case, golang codes of the project are compiled in `dwait`, the compiled binary is extracted out by `docker-make` 120 | and installed to `dresponse` with a `ADD` instruction in `dresponse`'s Dockerfile.Finally, `dresponse` is pushed to a private 121 | registry with two tags, git sha-1 commit id and 'latest'. 122 | With `docker-make`, you can do all of these steps properly via a single command； 123 | 124 | ``` 125 | $ docker-make 126 | INFO 2016-06-19 21:06:09,088 docker-make(278) dwait: building 127 | INFO 2016-06-19 21:06:09,440 docker-make(278) dwait: build succeed: ed169e889ecc 128 | INFO 2016-06-19 21:06:09,440 docker-make(278) dwait: extracting archives 129 | INFO 2016-06-19 21:06:09,599 docker-make(278) dwait: extracting archives succeed 130 | INFO 2016-06-19 21:06:09,600 docker-make(278) dresponse: building 131 | INFO 2016-06-19 21:06:10,305 docker-make(278) dresponse: build succeed: 671062910765 132 | INFO 2016-06-19 21:06:10,318 docker-make(278) dresponse: tag added: hub.privateregistry.com/dresponseagent:latest 133 | INFO 2016-06-19 21:06:10,325 docker-make(278) dresponse: tag added: hub.privateregistry.com/dresponseagent:a06fbc3a8af2f0fd12e89f539e481fe7d425c7c3 134 | INFO 2016-06-19 21:06:10,325 docker-make(278) dresponse: pushing to hub.privateregistry.com/dresponseagent:latest 135 | INFO 2016-06-19 21:06:17,576 docker-make(278) dresponse: pushed to hub.privateregistry.com/dresponseagent:latest 136 | INFO 2016-06-19 21:06:17,576 docker-make(278) dresponse: pushing to hub.privateregistry.com/dresponseagent:a06fbc3a8af2f0fd12e89f539e481fe7d425c7c3 137 | INFO 2016-06-19 21:06:18,505 docker-make(278) dresponse: pushed to hub.privateregistry.com/dresponseagent:a06fbc3a8af2f0fd12e89f539e481fe7d425c7c3 138 | ``` 139 | 140 | while the equivalent raw `docker` commands could be quite long, and require a careful atttension to the order of commands: 141 | 142 | ```bash 143 | $ docker build -t dmake -f Dockerfile.dwait 144 | $ tmp_container=`docker create dmake` 145 | $ docker cp $tmp_container:/usr/src/dwait/bin dwait.bin.tar 146 | $ docker build -t dresponse Dockerfile 147 | $ docker tag dresponsehu b.privateregistry.com/dresponse:latest 148 | $ docker tag dresponsehu b.privateregistry.com/dresponse:${git rev-parse HEAD} 149 | ``` 150 | 151 | ### two images-one for deploy, one for unit tests 152 | 153 | ```yaml 154 | builds: 155 | novadocker: 156 | context: / 157 | dockerfile: dockerfiles/Dockerfile 158 | pushes: 159 | - 'on_tag=hub.privateregistry.com/novadocker:{git_tag}' 160 | dockerignore: 161 | - .dmake.yml 162 | - .gitignore 163 | - tools 164 | - contrib 165 | - doc 166 | labels: 167 | - "com.dockermake.git.commitid={fcommitid}" 168 | 169 | novadocker-test: 170 | context: dockerfiles/ 171 | dockerfile: Dockerfile.test 172 | rewrite_from: novadocker 173 | depends_on: 174 | - novadocker 175 | ``` 176 | 177 | In this case, `novadocker` is built for deployment, `novadocker-teset` inherits from it via a 'FROM' instruction. 178 | The primary content of 'Dockerfile.test' includes installing testing dependencies and running the unit tests, if 179 | the tests pass, `docker build` will succeed, otherwise it will fail.Finally, `novadocker` will be pushed to a private 180 | registry, if the image is built on a git tag. 181 | 182 | The equivalent job could be expressed with some bash scripts. 183 | 184 | ```bash 185 | set -e 186 | fcommitid=`git rev-parse HEAD` 187 | docker build -t novadocker -f dockerfiles/Dockerfile --label com.dockermake.git.commitid=$fcommitid . 188 | docker build -t novadocker-test -f dockerfiles/Dockerfile.test . 189 | 190 | if git show-ref --tags|grep $fcommitid; then 191 | tagname=`git describe` 192 | docker tag novadocker hub.privateregistry.com/novadocker:$tagname 193 | docker push novadocker hub.privateregistry.com/novadocker:$tagname 194 | fi 195 | ``` 196 | 197 | ### several images-one as base with tools, others for different applications 198 | 199 | ```yaml 200 | builds: 201 | base: 202 | context: base 203 | dockerfile: Dockerfile 204 | 205 | java: 206 | context: java 207 | dockerfile: Dockerfile 208 | rewrite_from: base 209 | depends_on: 210 | - base 211 | 212 | php: 213 | context: php 214 | dockerfile: Dockerfile 215 | rewrite_from: base 216 | depends_on: 217 | - base 218 | 219 | java-app1: 220 | context: app1 221 | dockerfile: Dockerfile 222 | rewrite_from: java 223 | depends_on: 224 | - java 225 | 226 | php-app1: 227 | context: app2 228 | dockerfile: Dockerfile 229 | rewrite_from: php 230 | depends_on: 231 | - php 232 | ``` 233 | 234 | In such case, libraries like libc and monitoring tools are installed in base's Dockerfile. 235 | A java runtime environment and a php runtime environment are built by inheriting from the base image. 236 | With java and php as the base images, a java app and a php app are built. 237 | 238 | ## command line reference 239 | 240 | ```bash 241 | $ docker-make --help 242 | usage: docker-make [-h] [-f DMAKEFILE] [-d] [-rm] [--dry-run] [--no-push] 243 | [builds [builds ...]] 244 | 245 | build docker images in a simpler way. 246 | 247 | positional arguments: 248 | builds builds to execute. 249 | 250 | optional arguments: 251 | -h, --help show this help message and exit 252 | -f DMAKEFILE, --file DMAKEFILE 253 | path to docker-make configuration file. 254 | -d, --detailed print out detailed logs 255 | -rm, --remove remove intermediate containers 256 | --dry-run print docker commands only 257 | --no-push build only, dont push 258 | ``` 259 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ctripcloud/docker-make/d910ae53c3b4dd1e22a6900526f6fd8f60899f0c/__init__.py -------------------------------------------------------------------------------- /dmake/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ctripcloud/docker-make/d910ae53c3b4dd1e22a6900526f6fd8f60899f0c/dmake/__init__.py -------------------------------------------------------------------------------- /dmake/build.py: -------------------------------------------------------------------------------- 1 | import os 2 | import tempfile 3 | import logging 4 | 5 | import json 6 | from docker import utils as docker_utils 7 | 8 | from dmake import utils 9 | from dmake import template_args 10 | from dmake.errors import * # noqa 11 | 12 | 13 | LOG = logging.getLogger(__name__) 14 | 15 | 16 | class Build(object): 17 | def __init__(self, name, context, dockerfile, 18 | buildargs=None, dockerignore=None, labels=None, depends_on=None, 19 | extract=None, pushes=None, rewrite_from=None, 20 | remove_intermediate=None): 21 | self.name = name 22 | self.context = os.path.join(os.getcwd(), context.lstrip('/')) 23 | self.dockerfile = dockerfile 24 | self.dockerignore = dockerignore or [] 25 | self.buildargs = buildargs 26 | if '.dockerignore' not in self.dockerignore: 27 | self.dockerignore.append('.dockerignore') 28 | self.depends_on = depends_on or [] 29 | self.rewrite_from = rewrite_from 30 | self.remove_intermediate = remove_intermediate 31 | 32 | self.collect_pushes(pushes) 33 | self.collect_labels(labels) 34 | self.parse_extract(extract) 35 | 36 | @property 37 | def docker(self): 38 | return utils.docker_client() 39 | 40 | def collect_pushes(self, pushes): 41 | self.pushes = [] 42 | push_rules = pushes or [] 43 | 44 | for line in push_rules: 45 | try: 46 | push_mode, line = line.split('=', 1) 47 | repo, tag_template = line.rsplit(':', 1) 48 | self.pushes.append((push_mode, repo, tag_template)) 49 | except ValueError: 50 | raise ConfigurationError("wrong format for push %s" % line) 51 | 52 | def collect_labels(self, labels=None): 53 | self.labels = [] 54 | labels = labels or [] 55 | elements = template_args.label_template_args() 56 | for label_template in labels: 57 | try: 58 | key, value = label_template.split('=', 1) 59 | value = value.format(**elements) 60 | value = value.replace('"', r'\"') 61 | self.labels.append('%s="%s"' % (key, value)) 62 | except KeyError: 63 | LOG.warn('invalid label template: %s', label_template) 64 | except ValueError: 65 | raise ConfigurationError("invalid label template: %s" % 66 | label_template) 67 | 68 | def parse_extract(self, extract=None): 69 | extract = extract or [] 70 | self.extract = [] 71 | for item in extract: 72 | try: 73 | src, dst = item.split(':', 1) 74 | except ValueError: 75 | raise ConfigurationError('invalid extract rule: %s' % item) 76 | self.extract.append({ 77 | 'src': src, 78 | 'dst': os.path.join(self.context, dst) 79 | }) 80 | 81 | def dryrun(self): 82 | command = ["docker", "build", "-f", self.dockerfile] 83 | for label in self.labels: 84 | command.extend(["--label", label]) 85 | print ("%s: %s" % (self.name, " ".join(command))) 86 | 87 | def build(self): 88 | self._update_progress("building") 89 | self.non_labeled_image = self._build() 90 | 91 | if self.labels: 92 | self._update_progress("attaching labels") 93 | self.final_image = self._attach_labels() 94 | else: 95 | self.final_image = self.non_labeled_image 96 | self._update_progress("build succeed: %s" % self.final_image) 97 | 98 | if self.extract: 99 | self._update_progress("extracting archives") 100 | self._extract_contents(self.final_image, self.extract) 101 | self._update_progress("extracting archives succeed") 102 | 103 | def tag(self): 104 | template_kwargs = template_args.tag_template_args() 105 | for push_mode, repo, tag_template in self.pushes: 106 | # continue to next item if not needed 107 | need_push = self.need_push(push_mode) 108 | if not need_push: 109 | continue 110 | 111 | try: 112 | tag_name = tag_template.format(**template_kwargs) 113 | kwargs = {} 114 | if docker_utils.compare_version('1.22', self.docker._version) < 0: 115 | kwargs['force'] = True 116 | self.docker.tag(self.final_image, repo, tag_name, **kwargs) 117 | self._update_progress("tag added: %s:%s" % (repo, tag_name)) 118 | except KeyError as e: 119 | LOG.warn('invalid tag_template for this build: %s', e.message) 120 | 121 | def push(self): 122 | template_kwargs = template_args.tag_template_args() 123 | for push_mode, repo, tag_template in self.pushes: 124 | # continue to next item if not needed 125 | need_push = self.need_push(push_mode) 126 | if not need_push: 127 | continue 128 | 129 | try: 130 | tag_name = tag_template.format(**template_kwargs) 131 | except KeyError: 132 | raise PushFailed("can not get tag name for tag_template: %s" % tag_template) 133 | 134 | self._update_progress("pushing to %s:%s" % (repo, tag_name)) 135 | self._do_push(repo, tag_name) 136 | self._update_progress("pushed to %s:%s" % (repo, tag_name)) 137 | 138 | def need_push(self, push_mode): 139 | tag_template_args = template_args.tag_template_args() 140 | return { 141 | 'always': True, 142 | 'never': False, 143 | 'on_tag': tag_template_args.get('git_tag', False), 144 | 'on_branch:{0}'.format(tag_template_args.get('git_branch', '9x43d83')): True 145 | }.get(push_mode, False) 146 | 147 | def _update_progress(self, progress): 148 | self.progress = progress 149 | LOG.info("%s: %s", self.name, progress) 150 | 151 | def _extract_contents(self, img, paths): 152 | temp_container = self.docker.create_container(img, 'true') 153 | assert 'Id' in temp_container 154 | try: 155 | for path in paths: 156 | src, dst = path['src'], path['dst'] 157 | stream, stat = self.docker.get_archive(temp_container, src) 158 | with open(dst, 'w') as f: 159 | f.write(stream.data) 160 | utils.GarbageCleaner.register(dst) 161 | finally: 162 | self.docker.remove_container(temp_container) 163 | 164 | def _build(self): 165 | dockerfile = os.path.join(self.context, self.dockerfile) 166 | dockerignore = os.path.join(self.context, '.dockerignore') 167 | created_dockerignore = False 168 | if not os.path.exists(dockerignore): 169 | with open(dockerignore, 'w') as f: 170 | f.write("\n".join(self.dockerignore)) 171 | created_dockerignore = True 172 | utils.GarbageCleaner.register(dockerignore) 173 | 174 | if self.rewrite_from: 175 | original_lines = open(dockerfile).readlines() 176 | with open(dockerfile, 'w') as f: 177 | for line in original_lines: 178 | line = line.strip() 179 | if not line.startswith("FROM"): 180 | f.write("%s\n" % line) 181 | continue 182 | 183 | if " AS " not in line: 184 | f.write("FROM %s\n" % self.rewrite_from) 185 | else: 186 | from_text, intermediate_name = line.split(" AS ") 187 | f.write("FROM %s AS %s\n" % (self.rewrite_from, intermediate_name)) 188 | 189 | 190 | buildargs = {} 191 | if self.buildargs: 192 | buildargs = {k: os.path.expandvars(v) for k, v in [arg.split('=') for arg in self.buildargs]} 193 | 194 | params = { 195 | 'path': self.context, 196 | 'dockerfile': self.dockerfile, 197 | 'buildargs': buildargs, 198 | } 199 | 200 | if self.remove_intermediate: 201 | LOG.debug("Removing intermediate containers after each build") 202 | params['rm'] = self.remove_intermediate 203 | 204 | try: 205 | image_id = self._do_build(params) 206 | finally: 207 | if created_dockerignore: 208 | os.remove(dockerignore) 209 | if self.rewrite_from: 210 | with open(dockerfile, 'w') as f: 211 | f.write(''.join(original_lines)) 212 | return image_id 213 | 214 | def _attach_labels(self): 215 | pfile = tempfile.NamedTemporaryFile() 216 | pfile.write("FROM %s\n" % self.non_labeled_image) 217 | pfile.write("LABEL %s" % " ".join(self.labels)) 218 | pfile.seek(0) 219 | 220 | params = { 221 | 'fileobj': pfile, 222 | } 223 | 224 | if self.remove_intermediate: 225 | LOG.debug("Removing intermediate containers after each build") 226 | params['rm'] = self.remove_intermediate 227 | 228 | try: 229 | image_id = self._do_build(params) 230 | finally: 231 | pfile.close() 232 | for label in self.labels: 233 | self._update_progress("label added: %s" % label) 234 | return image_id 235 | 236 | def _do_build(self, params): 237 | response = self.docker.build(**params) 238 | image_id = None 239 | for data in response: 240 | for line in data.splitlines(): 241 | ret = json.loads(line) 242 | if 'stream' in ret: 243 | msg = ret['stream'] 244 | LOG.debug("%s: %s", self.name, msg) 245 | if 'errorDetail' in ret: 246 | raise BuildFailed(ret['errorDetail']['message']) 247 | if 'Successfully built' in ret.get('stream', ''): 248 | image_id = ret['stream'].strip().split()[-1] 249 | return image_id 250 | 251 | def _do_push(self, repo, tag): 252 | response = self.docker.push(repo, tag, stream=True) 253 | for line in response: 254 | LOG.debug("%s: %s", self.name, line) 255 | if 'errorDetail' in line: 256 | raise PushFailed("error in push %s:%s: %s" % (repo, tag, line)) 257 | 258 | def __repr__(self): 259 | return "Build: %s(%s)" % (self.name, self.progress) 260 | -------------------------------------------------------------------------------- /dmake/cli.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import logging 3 | 4 | from dotenv import load_dotenv 5 | 6 | from dmake.errors import * # noqa 7 | from dmake import utils 8 | from dmake import template_args 9 | import dmake.build 10 | 11 | LOG = logging.getLogger(__name__) 12 | 13 | 14 | def argparser(): 15 | parser = argparse.ArgumentParser(description="build docker images" 16 | " in a simpler way.") 17 | parser.add_argument('builds', type=str, nargs='*', 18 | help='builds to execute.') 19 | parser.add_argument('-f', '--file', dest='dmakefile', 20 | default='.docker-make.yml', 21 | help='path to docker-make configuration file.') 22 | parser.add_argument('-d', '--detailed', default=False, 23 | action='store_true', help='print out detailed logs') 24 | parser.add_argument('-rm', '--remove', default=False, 25 | action='store_true', 26 | help='remove intermediate containers') 27 | parser.add_argument('--dry-run', dest='dryrun', action='store_true', 28 | default=False, help='print docker commands only') 29 | parser.add_argument('--no-push', dest='nopush', action='store_true', 30 | default=False, help='build only, dont push images.') 31 | return parser 32 | 33 | 34 | def _main(): 35 | global LOG 36 | 37 | parser = argparser() 38 | args = parser.parse_args() 39 | 40 | log_format = '%(levelname)s %(name)s(%(lineno)s) %(asctime)s %(message)s' 41 | log_level = logging.DEBUG if args.detailed else logging.INFO 42 | logging.basicConfig(format=log_format, level=log_level) 43 | LOG = logging.getLogger("docker-make") 44 | 45 | load_dotenv() 46 | 47 | try: 48 | template_args.init_tag_names(args.dmakefile) 49 | builds_order, builds_dict = utils.get_sorted_build_dicts_from_yaml( 50 | args.dmakefile) 51 | except ConfigurationError as e: 52 | LOG.error("failed to parse %s: %s", args.dmakefile, e.message) 53 | return 1 54 | except ValidateError as e: 55 | LOG.error("wrong configuration: %s", e.message) 56 | return 1 57 | except DmakeError as e: 58 | LOG.eror(e.message) 59 | return 1 60 | 61 | builds = {} 62 | for name in builds_order: 63 | if (args.remove): 64 | builds_dict[name]['remove_intermediate'] = args.remove 65 | builds[name] = dmake.build.Build(name=name, **builds_dict[name]) 66 | 67 | if args.builds: 68 | try: 69 | wants = utils.expand_wants(builds, args.builds) 70 | except BuildUnDefined as e: 71 | LOG.error("No such build: %s", e.build) 72 | return 1 73 | else: 74 | wants = set(builds_order) 75 | 76 | if args.dryrun: 77 | for name in builds_order: 78 | if name not in wants: 79 | continue 80 | build = builds[name] 81 | build.dryrun() 82 | return 83 | 84 | for name in builds_order: 85 | if name not in wants: 86 | continue 87 | build = builds[name] 88 | if build.rewrite_from: 89 | build.rewrite_from = builds[build.rewrite_from].non_labeled_image 90 | try: 91 | build.build() 92 | build.tag() 93 | except BuildFailed as e: 94 | LOG.error("failed to build %s: %s", build.name, e.message) 95 | return 1 96 | except Exception: 97 | LOG.exception("failed to build %s", build.name) 98 | return 1 99 | 100 | if not args.nopush: 101 | try: 102 | build.push() 103 | except PushFailed as e: 104 | LOG.error("failed to push %s: %s", build.name, e.message) 105 | return 1 106 | except Exception as e: 107 | LOG.exception("failed to push %s", build.name) 108 | return 1 109 | 110 | 111 | def main(): 112 | try: 113 | return _main() 114 | finally: 115 | utils.GarbageCleaner.clean_all() 116 | -------------------------------------------------------------------------------- /dmake/errors.py: -------------------------------------------------------------------------------- 1 | class DmakeError(Exception): 2 | pass 3 | 4 | 5 | class ConfigurationError(DmakeError): 6 | pass 7 | 8 | 9 | class ValidateError(DmakeError): 10 | pass 11 | 12 | 13 | class DependencyError(DmakeError): 14 | pass 15 | 16 | 17 | class BuildFailed(DmakeError): 18 | pass 19 | 20 | 21 | class PushFailed(DmakeError): 22 | pass 23 | 24 | 25 | class BuildUnDefined(DmakeError): 26 | def __init__(self, build): 27 | self.build = build 28 | super(BuildUnDefined, self).__init__() 29 | -------------------------------------------------------------------------------- /dmake/template_args.py: -------------------------------------------------------------------------------- 1 | import re 2 | import datetime 3 | import logging 4 | import subprocess 5 | 6 | from dmake import utils 7 | 8 | 9 | LOG = logging.getLogger(__name__) 10 | TAG_NAME_PATTERN = re.compile(r'^[a-zA-Z0-9_][]a-zA-Z0-9_\.\-]{0,127}$') 11 | TAG_NAME_LEAD_PATTERN = re.compile(r'^[a-zA-Z0-9_]$') 12 | TAG_NAME_ELEMENT_PATTERN = re.compile(r'^[a-zA-Z0-9_\.\-]$') 13 | _tag_template_args = None 14 | _label_template_args = None 15 | 16 | 17 | class TemplateArgsGenerator(object): 18 | def gen_args(self): 19 | raise StopIteration 20 | # yield needed to make gen_args a generator function 21 | yield 22 | 23 | 24 | class DateTimeGenerator(TemplateArgsGenerator): 25 | def __init__(self, name, format): 26 | self.name = name 27 | self.format = format 28 | 29 | def gen_args(self): 30 | yield self.name, datetime.datetime.now().strftime(self.format) 31 | 32 | 33 | class DateGenerator(DateTimeGenerator): 34 | def __init__(self): 35 | super(DateGenerator, self).__init__('date', '%Y%m%d') 36 | 37 | 38 | class ExternalCmdGenerator(TemplateArgsGenerator): 39 | def __init__(self, key=None, cmd=None): 40 | self.key = key or self.__class__.key 41 | self.cmd = cmd or self.__class__.cmd 42 | 43 | def gen_args(self): 44 | try: 45 | value = subprocess.check_output(self.cmd, 46 | stderr=subprocess.STDOUT, 47 | shell=not isinstance(self.cmd, 48 | list)) 49 | value = value.strip() 50 | if value: 51 | yield self.key, value.strip() 52 | except subprocess.CalledProcessError as e: 53 | log_level = logging.WARNING 54 | 55 | # having 0 tags is not worthy of warning 56 | if (isinstance(self, GitDescribeGenerator) and 57 | "No names found" in e.output): 58 | log_level = logging.INFO 59 | LOG.log(log_level, "failed to run %s: %s", self.cmd, e) 60 | pass 61 | 62 | 63 | class GitCommitGenerator(ExternalCmdGenerator): 64 | key = 'fcommitid' 65 | cmd = 'git rev-parse HEAD' 66 | 67 | def gen_args(self): 68 | for k, v in super(GitCommitGenerator, self).gen_args(): 69 | yield k, v 70 | yield 'scommitid', v[:7] 71 | 72 | 73 | class GitCommitMsgGenerator(ExternalCmdGenerator): 74 | key = 'commitmsg' 75 | cmd = 'git log --oneline|head -1' 76 | 77 | 78 | class GitBranchGenerator(ExternalCmdGenerator): 79 | key = 'git_branch' 80 | cmd = 'git rev-parse --abbrev-ref HEAD' 81 | 82 | 83 | class GitTagGenerator(ExternalCmdGenerator): 84 | key = 'git_tag' 85 | cmd = 'git tag --contains HEAD|head -1' 86 | 87 | 88 | class GitDescribeGenerator(ExternalCmdGenerator): 89 | key = 'git_describe' 90 | cmd = 'git describe --tags' 91 | 92 | 93 | def _template_args(generators): 94 | result = {} 95 | for g in generators: 96 | for k, v in g.gen_args(): 97 | if not validate_tag_name(v): 98 | result[k] = correct_tag_name(v) 99 | LOG.warn("%s is not a valid docker tag name," 100 | "will be automatically corrected to %s", 101 | v, result[k]) 102 | else: 103 | result[k] = v 104 | return result 105 | 106 | 107 | def validate_tag_name(name): 108 | return TAG_NAME_PATTERN.match(name) is not None 109 | 110 | 111 | def correct_tag_name(name): 112 | if not name: 113 | return "null" 114 | tmp_lst = [] 115 | lead, suffix = name[0], name[1:] 116 | if TAG_NAME_LEAD_PATTERN.match(lead) is None: 117 | tmp_lst.append('_') 118 | else: 119 | tmp_lst.append(lead) 120 | 121 | for c in suffix: 122 | if TAG_NAME_ELEMENT_PATTERN.match(c) is None: 123 | tmp_lst.append('_') 124 | else: 125 | tmp_lst.append(c) 126 | return ''.join(tmp_lst)[:128] 127 | 128 | 129 | def tag_template_args(extra_generators=None): 130 | global _tag_template_args 131 | if _tag_template_args is not None: 132 | return _tag_template_args 133 | extra_generators = extra_generators or [] 134 | generators = [GitCommitGenerator(), GitCommitMsgGenerator(), 135 | GitBranchGenerator(), GitTagGenerator(), 136 | GitDescribeGenerator(), DateGenerator()] 137 | generators.extend(extra_generators) 138 | _tag_template_args = _template_args(generators) 139 | return _tag_template_args 140 | 141 | 142 | def label_template_args(extra_generators=None): 143 | global _label_template_args 144 | if _label_template_args is not None: 145 | return _label_template_args 146 | extra_generators = extra_generators or [] 147 | generators = [GitCommitGenerator(), GitCommitMsgGenerator(), 148 | GitBranchGenerator(), GitTagGenerator(), 149 | GitDescribeGenerator()] 150 | generators.extend(extra_generators) 151 | _label_template_args = _template_args(generators) 152 | return _label_template_args 153 | 154 | 155 | def init_tag_names(dmakefile): 156 | data = utils.load_yaml(dmakefile) 157 | configurations = data.get('tag-names', None) 158 | extra_generators = create_extra_generators(configurations) 159 | label_template_args(extra_generators) 160 | tag_template_args(extra_generators) 161 | 162 | 163 | def create_extra_generators(configurations): 164 | if configurations is None: 165 | return [] 166 | 167 | configurable_tag_name_generators = { 168 | 'datetime': DateTimeGenerator, 169 | 'cmd': ExternalCmdGenerator 170 | } 171 | 172 | tag_name_generators = [] 173 | 174 | for config in configurations: 175 | if not validate_tag_name_config(config): 176 | continue 177 | name, type_, value = config['name'], config['type'], config['value'] 178 | cls = configurable_tag_name_generators.get(type_, None) 179 | if cls is not None: 180 | tag_name_generators.append(cls(name, value)) 181 | return tag_name_generators 182 | 183 | 184 | def validate_tag_name_config(config): 185 | for key in ('name', 'type', 'value'): 186 | if key not in config: 187 | LOG.warn("%s absent in %s", key, config) 188 | return False 189 | return True 190 | -------------------------------------------------------------------------------- /dmake/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import logging 3 | 4 | import yaml 5 | import docker 6 | from distutils.version import LooseVersion 7 | from docker import utils as docker_utils 8 | 9 | from dmake.errors import * # noqa 10 | 11 | 12 | LOG = logging.getLogger(__name__) 13 | _docker = None 14 | 15 | 16 | class _GarbageCleaner(object): 17 | def __init__(self): 18 | self._files = set() 19 | 20 | def register(self, filename): 21 | self._files.add(filename) 22 | 23 | def clean(self, filename): 24 | LOG.debug("cleaning up %s", filename) 25 | if not os.path.exists(filename): 26 | return 27 | if os.path.isfile(filename) or os.path.islink(filename): 28 | os.remove(filename) 29 | if os.path.isdir(filename): 30 | os.rmdir(filename) 31 | 32 | def clean_all(self): 33 | for filename in self._files: 34 | self.clean(filename) 35 | 36 | 37 | GarbageCleaner = _GarbageCleaner() 38 | 39 | 40 | def docker_client(): 41 | global _docker 42 | if _docker is None: 43 | params = docker_utils.kwargs_from_env() 44 | params['version'] = 'auto' 45 | if LooseVersion(docker.__version__) < LooseVersion('2.0.0'): 46 | _docker = docker.client.Client(**params) 47 | else: 48 | _docker = docker.api.client.APIClient(**params) 49 | return _docker 50 | 51 | 52 | def load_yaml(filename='.docker-make.yml'): 53 | try: 54 | with open(filename) as f: 55 | return yaml.safe_load(f) 56 | except (IOError, yaml.YAMLError) as e: 57 | err_msg = getattr(e, '__module__', '') + '.' + e.__class__.__name__ 58 | raise ConfigurationError(u"{}: {}".format(err_msg, e)) 59 | 60 | 61 | def validate(config): 62 | builds = config.get('builds') 63 | if builds is None: 64 | raise ValidateError("no builds specified") 65 | if not isinstance(builds, dict): 66 | raise ValidateError("builds should be a dict") 67 | 68 | for name, build in builds.iteritems(): 69 | for dep in build.get('depends_on', []): 70 | if dep not in builds: 71 | raise ValidateError("%s depends on %s, which is not present in" 72 | "the current configuration." % (name, dep)) 73 | return True 74 | 75 | 76 | def sort_builds_dict(builds): 77 | # Topological sort (Cormen/Tarjan algorithm) 78 | unmarked = builds.keys() 79 | temporary_marked = set() 80 | sorted_builds = [] 81 | 82 | def visit(n): 83 | if n in temporary_marked: 84 | if n in builds[n].get('depends_on', []): 85 | raise DependencyError('A build can not' 86 | ' depend on itself: %s' % n['name']) 87 | raise DependencyError('Circular dependency between %s' % 88 | ' and '.join(temporary_marked)) 89 | 90 | if n in unmarked: 91 | temporary_marked.add(n) 92 | builds_dep_on_n = [name for name, build in builds.iteritems() 93 | if 94 | n in build.get('depends_on', [])] 95 | for m in builds_dep_on_n: 96 | visit(m) 97 | temporary_marked.remove(n) 98 | unmarked.remove(n) 99 | sorted_builds.insert(0, n) 100 | 101 | while unmarked: 102 | visit(unmarked[-1]) 103 | 104 | return sorted_builds 105 | 106 | 107 | def get_sorted_build_dicts_from_yaml(filename): 108 | config = load_yaml(filename) 109 | validate(config) 110 | builds = config["builds"] 111 | builds_order = sort_builds_dict(builds) 112 | return builds_order, builds 113 | 114 | 115 | def expand_wants(candidates, wants): 116 | ret = set() 117 | wants = set(wants) 118 | while wants: 119 | want = wants.pop() 120 | if want not in candidates: 121 | raise BuildUnDefined(want) 122 | ret.add(want) 123 | for dep in candidates[want].depends_on: 124 | if dep not in ret: 125 | wants.add(dep) 126 | return ret 127 | -------------------------------------------------------------------------------- /docs/yaml-configuration-reference.md: -------------------------------------------------------------------------------- 1 | # reference for .docker-make.yml 2 | 3 | ## tag-names(essential, list of dict, default: []) 4 | definition of customized tag names. 5 | 6 | ### `name` (essential, string) 7 | name of the new customized tag, which can be referred in a tag template. 8 | 9 | ### `type` (essential, string) 10 | type of the new customized tag which produces a tag name based on the value of `value` field, choices include: 11 | * `datetime` 12 | * `cmd` 13 | 14 | ### `value` (essential, string) 15 | argument passed to the tag name generator specified by the `type` field: 16 | * for `datetime` type, value is a Python datetime formatter, e.g '%Y%m%d%H%M'(ref [datetime.strftime](https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior)). 17 | * for `cmd` type, value is a shell command, e.g. `echo hello-world`. 18 | 19 | ## builds(essential, dict, default: {}) 20 | definition of `docker-builds` and their relationships. 21 | 22 | ### name of build(e.g, `dwait` and `dresponse`) (essential, string) 23 | names for your build. 24 | 25 | ### `context` (essential, string) 26 | path to build context, relative to the root of the repo. 27 | 28 | ### `dockerfile` (essential, string) 29 | Dockerfile for the build, relative to the context. 30 | 31 | ### `buildargs` (optional, [string], default: []) 32 | List of build arguments. 33 | Each argument should be provided in `"ARG=VALUE"` form. 34 | 35 | ### `pushes` (optional, [string], default: []) 36 | pushing rule for the built image, a single rule is composed in a form of '=<\repo>:', 37 | in which: 38 | * `push_mode` defines when to push, choices include: 39 | * `always`: always push the successfully built image. 40 | * `on_tag`: push if built on a git tag. 41 | * `on_branch:`: push if built on branch `branchname` 42 | 43 | * `repo` defines which repo to push to. 44 | 45 | * `tag_template` is a python formattable string for generating the image tag, available template variables include: 46 | * `date`: date of the built(e.g, 20160617) 47 | * `scommitid`: a 7-char trunc of the corresponding git sha-1 commit id. 48 | * `fcommitid`: full git commit id. 49 | * `git_tag`: git tag name (if built on a git tag) 50 | * `git_branch`: git branch name(if built on a git branch) 51 | 52 | ### `dockerignore` (optional, [string], default: []) 53 | files or directories you want ignore in the context, during `docker build` 54 | ref: [dockerignore](https://docs.docker.com/engine/reference/builder/#dockerignore-file) 55 | 56 | ### `labels` (optional, [string]) 57 | define labels applied to built image, each item should be with format '=""', with `` 58 | being a python template string, available template variables include: 59 | * `scommitid`: a 7-char trunc of the corresponding git sha-1 commit id. 60 | * `fcommitid`: full git commit id. 61 | * `git_tag`: git tag name (if built on a git tag) 62 | * `git_branch`: git branch name(if built on a git branch) 63 | 64 | ### `depends_on` (optional, [string], default: []) 65 | which builds this build depends on, `docker-make` will build the depends first. 66 | 67 | ### `extract` (optional, [string], default: []) 68 | define a list of source-destination pairs, with `source` point to a path of the newly built image, and `destination` being a filename on the host, `docker-make` will package `source` in a tar file, and copy the tar file to `destination`. Each item's syntax is similar to `docker run -v` 69 | 70 | ### `rewrite_from` (optional, string, default: '') 71 | a build's name which should be available in `.docker-make.yml`, if supplied, `docker-make` will build `rewrite_from` first, and replace current build's Dockerfile's `FROM` with `rewrite_from`'s fresh image id. 72 | -------------------------------------------------------------------------------- /requirements.pip: -------------------------------------------------------------------------------- 1 | PyYAML >= 3.10, < 4 2 | python-dotenv==0.10.3 3 | # 'docker-py >= 1.8.1, < 2' or 'docker >= 2.0.0, < 3' 4 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from distutils.version import LooseVersion 3 | from setuptools import setup, find_packages 4 | 5 | 6 | def get_docker_client_requirement(): 7 | DOCKER_PY_REQUIREMENT = 'docker-py >= 1.8.1, < 2' 8 | DOCKER_RRQUIREMENT = 'docker >= 2.0.0, < 3' 9 | docker_client_installed = True 10 | try: 11 | import docker 12 | except ImportError: 13 | docker_client_installed = False 14 | if docker_client_installed and\ 15 | LooseVersion(docker.__version__) < LooseVersion('2.0.0'): 16 | return DOCKER_PY_REQUIREMENT 17 | return DOCKER_RRQUIREMENT 18 | 19 | 20 | def find_requirements(fn): 21 | lines = [] 22 | with open(fn) as f: 23 | for line in f: 24 | line = line.strip() 25 | if not line.startswith('#'): 26 | lines.append(line) 27 | return lines 28 | 29 | 30 | setup( 31 | name='docker-make', 32 | description='build,tag,and push a bunch of related docker images via a single command', 33 | version='1.1.7', 34 | author='jizhilong', 35 | author_email='zhilongji@gmail.com', 36 | url='https://github.com/CtripCloud/docker-make', 37 | license='Apache', 38 | keywords=['docker', 'image',' build'], 39 | packages=find_packages(exclude=['tests']), 40 | entry_points={ 41 | 'console_scripts': [ 42 | 'docker-make = dmake.cli:main' 43 | ] 44 | }, 45 | install_requires=find_requirements('requirements.pip') +\ 46 | [get_docker_client_requirement()], 47 | tests_require=find_requirements('test-requirements.pip'), 48 | test_suite='nose.collector', 49 | classifiers=[], 50 | ) 51 | -------------------------------------------------------------------------------- /test-requirements.pip: -------------------------------------------------------------------------------- 1 | unittest2 2 | flake8==2.6.2 3 | nose==1.3.7 4 | mock==2.0.0 5 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ctripcloud/docker-make/d910ae53c3b4dd1e22a6900526f6fd8f60899f0c/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_template_args.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | from datetime import datetime 3 | 4 | import unittest2 5 | from mock import mock 6 | 7 | from dmake import template_args 8 | 9 | 10 | class TemplateArgsGeneratorTests(unittest2.TestCase): 11 | @mock.patch('datetime.datetime') 12 | def test_date_generator(self, mocked_datetime): 13 | mocked_datetime.now.return_value = datetime(2016, 7, 21) 14 | args_date = next(template_args.DateGenerator().gen_args(), None) 15 | self.assertIsInstance(args_date, tuple) 16 | k, v = args_date 17 | self.assertEqual(k, 'date') 18 | self.assertEqual(v, '20160721') 19 | mocked_datetime.now.assert_called_once() 20 | 21 | @mock.patch('datetime.datetime') 22 | def test_datetime_generator(self, mocked_datetime): 23 | mocked_datetime.now.return_value = datetime(2016, 7, 21, 12, 23) 24 | args_date = next(template_args.DateTimeGenerator( 25 | 'datetime', '%Y%m%d%H%M').gen_args(), None) 26 | self.assertIsInstance(args_date, tuple) 27 | k, v = args_date 28 | self.assertEqual(k, 'datetime') 29 | self.assertEqual(v, '201607211223') 30 | mocked_datetime.now.assert_called_once() 31 | 32 | def test_validate_tag_name(self): 33 | self.assertTrue(template_args.validate_tag_name('v1.0.0')) 34 | self.assertTrue(template_args.validate_tag_name('latest')) 35 | self.assertFalse(template_args.validate_tag_name('feature/123')) 36 | self.assertFalse(template_args.validate_tag_name('-master')) 37 | self.assertFalse(template_args.validate_tag_name('.test')) 38 | 39 | def test_correct_tag_name(self): 40 | self.assertEqual(template_args.correct_tag_name('feature/123'), 41 | 'feature_123') 42 | self.assertEqual(template_args.correct_tag_name('-master'), 43 | '_master') 44 | self.assertEqual(template_args.correct_tag_name('.test'), 45 | '_test') 46 | long_tag_name = ''.join(str(i) for i in xrange(128)) 47 | self.assertEqual(len(template_args.correct_tag_name(long_tag_name)), 48 | 128) 49 | 50 | 51 | class ExternalCmdGeneratorTests(unittest2.TestCase): 52 | 53 | @mock.patch('subprocess.check_output', return_value=' dummy ') 54 | def test_key_cmd_in_cls_attr(self, mocked_check_output): 55 | class DummyGenerator(template_args.ExternalCmdGenerator): 56 | key = 'dummy' 57 | cmd = 'echo dummy' 58 | args = next(DummyGenerator().gen_args(), None) 59 | self.assertIsInstance(args, tuple) 60 | k, v = args 61 | self.assertEqual(k, 'dummy') 62 | self.assertEqual(v, 'dummy') 63 | mocked_check_output.assert_called_once_with('echo dummy', shell=True, stderr=-2) 64 | 65 | @mock.patch('subprocess.check_output', return_value=' dummy ') 66 | def test_key_cmd_in_init(self, mocked_check_output): 67 | key, cmd = 'dummy', 'echo dummy' 68 | args = next(template_args.ExternalCmdGenerator(key, cmd).gen_args(), None) 69 | self.assertIsInstance(args, tuple) 70 | k, v = args 71 | self.assertEqual(k, 'dummy') 72 | self.assertEqual(v, 'dummy') 73 | mocked_check_output.assert_called_once_with('echo dummy', shell=True, stderr=-2) 74 | 75 | @mock.patch('subprocess.check_output', side_effect=subprocess.CalledProcessError(-1, 'echo dummy')) 76 | def test_raise_call_error(self, mocked_check_output): 77 | key, cmd = 'dummy', 'echo dummy' 78 | args = next(template_args.ExternalCmdGenerator(key, cmd).gen_args(), None) 79 | self.assertIsNone(args) 80 | mocked_check_output.assert_called_once_with('echo dummy', shell=True, stderr=-2) 81 | 82 | @mock.patch('subprocess.check_output', return_value=' ') 83 | def test_blank_output(self, mocked_check_output): 84 | key, cmd = 'dummy', 'echo dummy' 85 | args = next(template_args.ExternalCmdGenerator(key, cmd).gen_args(), None) 86 | self.assertIsNone(args) 87 | mocked_check_output.assert_called_once_with('echo dummy', shell=True, stderr=-2) 88 | 89 | 90 | class GitGeneratorsTests(unittest2.TestCase): 91 | @mock.patch('subprocess.check_output', return_value='56903369fd200ea021dbb75f357f94b7fb5e829e') 92 | def test_git_commit(self, mocked_check_output): 93 | pairs = template_args.GitCommitGenerator().gen_args() 94 | k1, v1 = next(pairs) 95 | self.assertEqual(k1, 'fcommitid') 96 | self.assertEqual(v1, '56903369fd200ea021dbb75f357f94b7fb5e829e') 97 | mocked_check_output.assert_called_once_with('git rev-parse HEAD', shell=True, stderr=-2) 98 | 99 | k2, v2 = next(pairs) 100 | self.assertEqual(k2, 'scommitid') 101 | self.assertEqual(v2, '5690336') 102 | 103 | @mock.patch('subprocess.check_output', return_value='5690336 refactor and add unit tests.') 104 | def test_git_commitmsg(self, mocked_check_output): 105 | k, v = next(template_args.GitCommitMsgGenerator().gen_args()) 106 | self.assertEqual(k, 'commitmsg') 107 | self.assertEqual(v, '5690336 refactor and add unit tests.') 108 | mocked_check_output.assert_called_once_with('git log --oneline|head -1', shell=True, 109 | stderr=-2) 110 | 111 | @mock.patch('subprocess.check_output', return_value='master') 112 | def test_git_branch(self, mocked_check_output): 113 | k, v = next(template_args.GitBranchGenerator().gen_args()) 114 | self.assertEqual(k, 'git_branch') 115 | self.assertEqual(v, 'master') 116 | mocked_check_output.assert_called_once_with('git rev-parse --abbrev-ref HEAD', shell=True, 117 | stderr=-2) 118 | 119 | @mock.patch('subprocess.check_output', return_value='1.11.3') 120 | def test_git_tag(self, mocked_check_output): 121 | k, v = next(template_args.GitTagGenerator().gen_args()) 122 | self.assertEqual(k, 'git_tag') 123 | self.assertEqual(v, '1.11.3') 124 | mocked_check_output.assert_called_once_with('git tag --contains HEAD|head -1', shell=True, 125 | stderr=-2) 126 | 127 | @mock.patch('subprocess.check_output', return_value='1.1.2-5-g5690336') 128 | def test_git_describe(self, mocked_check_output): 129 | k, v = next(template_args.GitDescribeGenerator().gen_args()) 130 | self.assertEqual(k, 'git_describe') 131 | self.assertEqual(v, '1.1.2-5-g5690336') 132 | mocked_check_output.assert_called_once_with('git describe --tags', shell=True, stderr=-2) 133 | 134 | 135 | class ArgsExportingFunctionTests(unittest2.TestCase): 136 | @mock.patch('datetime.datetime') 137 | def test__template_args(self, mocked_datetime): 138 | mocked_datetime.now.return_value = datetime(2016, 7, 21) 139 | generators = [template_args.DateGenerator()] 140 | ret = template_args._template_args(generators) 141 | self.assertEqual(ret, {'date': '20160721'}) 142 | mocked_datetime.now.assert_called_once() 143 | 144 | @mock.patch('dmake.template_args._template_args', return_value={}) 145 | def test_tag_template_args(self, mocked__template_args): 146 | self.assertIsNone(template_args._tag_template_args) 147 | ret = template_args.tag_template_args() 148 | self.assertEqual(ret, {}) 149 | self.assertEqual(template_args._tag_template_args, {}) 150 | ta = template_args 151 | generator_classes = [ta.GitCommitGenerator, ta.GitCommitMsgGenerator, 152 | ta.GitBranchGenerator, ta.GitTagGenerator, 153 | ta.GitDescribeGenerator, ta.DateGenerator] 154 | for obj, cls in zip(mocked__template_args.call_args[0][0], 155 | generator_classes): 156 | self.assertIsInstance(obj, cls) 157 | 158 | @mock.patch('dmake.template_args._template_args', return_value={}) 159 | def test_label_template_args(self, mocked__template_args): 160 | self.assertIsNone(template_args._label_template_args) 161 | ret = template_args.label_template_args() 162 | self.assertEqual(ret, {}) 163 | self.assertEqual(template_args._label_template_args, {}) 164 | ta = template_args 165 | generator_classes = [ta.GitCommitGenerator, ta.GitCommitMsgGenerator, 166 | ta.GitBranchGenerator, ta.GitTagGenerator, 167 | ta.GitDescribeGenerator] 168 | for obj, cls in zip(mocked__template_args.call_args[0][0], 169 | generator_classes): 170 | self.assertIsInstance(obj, cls) 171 | 172 | def test_validate_tag_name_config(self): 173 | func = template_args.validate_tag_name_config 174 | self.assertTrue(func({ 175 | 'type': 'cmd', 176 | 'name': 'dummy', 177 | 'value': 'echo dummt', 178 | })) 179 | self.assertFalse(func({ 180 | 'name': 'dummy', 181 | 'value': 'echo dummt', 182 | })) 183 | self.assertFalse(func({ 184 | 'type': 'cmd', 185 | 'value': 'echo dummt', 186 | })) 187 | self.assertFalse(func({ 188 | 'type': 'cmd', 189 | 'name': 'dummy', 190 | })) 191 | self.assertFalse(func({ 192 | })) 193 | 194 | def test_create_extra_generators(self): 195 | configurations = [ 196 | {'type': 'cmd', 197 | 'name': 'dummy', 198 | 'value': 'echo dummt'}, 199 | ] 200 | result = template_args.create_extra_generators(configurations) 201 | self.assertEqual(1, len(result)) 202 | self.assertIsInstance(result[0], 203 | template_args.ExternalCmdGenerator) 204 | configurations = [ 205 | {'type': 'datetime', 206 | 'name': 'time', 207 | 'value': '%H%M'}, 208 | {'type': 'notexist', 209 | 'name': 'dummy', 210 | 'value': 'dummy'} 211 | ] 212 | result = template_args.create_extra_generators(configurations) 213 | self.assertEqual(1, len(result)) 214 | self.assertIsInstance(result[0], 215 | template_args.DateTimeGenerator) 216 | 217 | @mock.patch('dmake.utils.load_yaml') 218 | @mock.patch('dmake.template_args.label_template_args') 219 | @mock.patch('dmake.template_args.tag_template_args') 220 | def test_init_tag_names(self, patched_tag_template_args, 221 | patched_label_template_args, 222 | patched_load_yaml): 223 | patched_load_yaml.return_value = {'tag-names': []} 224 | template_args.init_tag_names('.docker-make.yml') 225 | patched_load_yaml.assert_called_once_with('.docker-make.yml') 226 | patched_label_template_args.assert_called_once_with([]) 227 | patched_tag_template_args.assert_called_once_with([]) 228 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore=F405 3 | --------------------------------------------------------------------------------