├── .gitignore ├── DESCRIPTION.md ├── LICENSE.md ├── MANIFEST.in ├── README.md ├── docket ├── __init__.py └── command_line.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | build 3 | dist 4 | docket.egg-info 5 | 6 | -------------------------------------------------------------------------------- /DESCRIPTION.md: -------------------------------------------------------------------------------- 1 | Docker Build with secrets 2 | ========================= 3 | 4 | Build docker images with private files. 5 | 6 | Private files are stripped from the final image after the build is complete. 7 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Roman Shtylman 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include DESCRIPTION.md 2 | 3 | # Include the test suite (FIXME: does not work yet) 4 | # recursive-include tests * 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # docket 2 | 3 | Build docker images with secrets. Use it just like the `docker build` command. 4 | 5 | ## install 6 | 7 | ``` 8 | pip install git+git://github.com/defunctzombie/docket.git 9 | ``` 10 | 11 | ## Use 12 | 13 | Put some files into `$HOME/.docker/private`. They will be available during the build process. 14 | 15 | Use docket like you would use `docker build` 16 | 17 | ```shell 18 | docket -t foobar 19 | ``` 20 | 21 | ## Private Files 22 | 23 | Any files in `$HOME/.docker/private` will be available during the build process. The folder structure under this directory will mirror the folder structure under `/` in the container. 24 | 25 | These private files will not appear in any layer of the final image. 26 | 27 | ## How it works 28 | 29 | *Note*: You need to understand docker layered file system internals for this to make sense. 30 | 31 | Docket will examine your Dockerfile and locate the `FROM` image. It will create a new layer with the private files in `$HOME/.docker/private` and apply it to this base image. 32 | 33 | It will then create a new temporary Dockerfile copy of your original Dockerfile and alter the `FROM` entry to point to this newly created image (which contains the private layer). Docket will package up your original build context and this new Dockerfile (replacing your original in the context) and send it over to the docker daemon to build. 34 | 35 | After a successful build, docket will "download" (using the docker save feature) the image and unpackage it. This will result in a folder for every layer of the image. Docket will find the layer which references the "private" image it created and update the layer json to point to the original base image id you requested. It will then remove the private layer files and create a tarball to send back to docker (using the load feature). 36 | 37 | This final image will contain no history of the private layer. 38 | -------------------------------------------------------------------------------- /docket/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import logging 3 | import tarfile 4 | import os 5 | import json 6 | import sys 7 | import tempfile 8 | import shutil 9 | import re 10 | import hashlib 11 | 12 | from subprocess import Popen 13 | from os import walk 14 | from fnmatch import fnmatch 15 | 16 | logger = logging.getLogger('docket') 17 | 18 | def build(client, tag, buildpath, no_cache): 19 | logger.debug('building image with tag (%s)', tag) 20 | build_dockerfile = os.path.join(buildpath, 'Dockerfile') 21 | 22 | ## identify the ID of the FROM image we desired 23 | parent_id = None 24 | with open(build_dockerfile) as dockerfile: 25 | parent_id = base_image_id(client, dockerfile.read()) 26 | 27 | logger.info('base image id %s', parent_id) 28 | 29 | private_layer_id = build_private_layer(client, parent_id) 30 | 31 | context_tar = create_build_context(buildpath, private_layer_id) 32 | 33 | docker_build(client, tag, context_tar) 34 | 35 | context_tar.close() 36 | 37 | strip_private_layer(client, tag, parent_id, private_layer_id) 38 | 39 | 40 | def base_image_id(client, dockerfile): 41 | match = re.search('^FROM (?P.*)', dockerfile) 42 | image_name = match.group('image') 43 | 44 | logger.info('pulling base image %s', image_name) 45 | res = client.pull(image_name, stream=True) 46 | for line in res: 47 | print json.loads(line).get('status', '') 48 | 49 | image, tag = image_name.split(':', 1) 50 | image_list = client.images(name=image) 51 | 52 | image_id = None 53 | for image in image_list: 54 | try: 55 | if image['RepoTags'].index(image_name) >= 0: 56 | image_id = image['Id'] 57 | except ValueError: 58 | continue 59 | 60 | return image_id 61 | 62 | def build_private_layer(client, parent_id): 63 | private_home = os.path.join(os.path.expanduser('~'), '.docker', 'private') 64 | 65 | private_layer = tempfile.NamedTemporaryFile() 66 | private_layer_tar = tarfile.open(mode='w', fileobj=private_layer) 67 | private_layer_tar.add(private_home, arcname='') 68 | private_layer.seek(0) 69 | 70 | # create a unique ID for the new private layer 71 | private_layer_id = None 72 | md = hashlib.md5() 73 | md.update(parent_id) 74 | 75 | with open(private_layer.name, 'rb') as tar: 76 | for chunk in iter(lambda: tar.read(128), ''): 77 | md.update(chunk) 78 | 79 | private_layer_id = md.hexdigest() 80 | private_layer_id += private_layer_id 81 | private_layer.seek(0) 82 | 83 | logger.info('private layer id %s', private_layer_id) 84 | 85 | base_image_info = client.inspect_image(parent_id) 86 | private_image_info = base_image_info; 87 | 88 | private_image_info['Parent'] = parent_id 89 | private_image_info['Id'] = private_layer_id 90 | private_image_info['ContainerConfig']['Image'] = parent_id 91 | private_image_info['Config']['Image'] = parent_id 92 | 93 | private_image_info['parent'] = private_image_info['Parent'] 94 | private_image_info['id'] = private_image_info['Id'] 95 | private_image_info['container_config'] = private_image_info['ContainerConfig'] 96 | private_image_info['config'] = private_image_info['Config'] 97 | private_image_info['created'] = private_image_info['Created'] 98 | 99 | private_image = tempfile.NamedTemporaryFile() 100 | private_image_tar = tarfile.open(mode='w', fileobj=private_image) 101 | private_image_tar.add(private_layer.name, arcname=private_layer_id + '/layer.tar') 102 | 103 | with tempfile.NamedTemporaryFile() as tmp: 104 | tmp.write('1.0') 105 | tmp.seek(0) 106 | tarinfo = private_image_tar.gettarinfo(name='VERSION', arcname=private_layer_id + '/VERSION', fileobj=tmp) 107 | private_image_tar.addfile(tarinfo, tmp) 108 | 109 | with tempfile.NamedTemporaryFile() as tmp: 110 | json.dump(private_image_info, tmp) 111 | tmp.seek(0) 112 | tarinfo = private_image_tar.gettarinfo(name='json', arcname=private_layer_id + '/json', fileobj=tmp) 113 | private_image_tar.addfile(tarinfo, tmp) 114 | 115 | private_image_tar.close() 116 | 117 | need_load = True 118 | try: 119 | inspect = client.inspect_image(private_layer_id) 120 | need_load = False 121 | except: 122 | pass 123 | 124 | if need_load: 125 | private_image.seek(0) 126 | 127 | logger.info('loading private image', private_layer_id) 128 | try: 129 | res = client.load_image(private_image) 130 | except Exception as err: 131 | print err 132 | finally: 133 | private_layer.close() 134 | 135 | return private_layer_id 136 | 137 | def fnmatch_any(relpath, patterns): 138 | return any([fnmatch(relpath, pattern) for pattern in patterns]) 139 | 140 | def no_dockerfile(tarinfo): 141 | if tarinfo.name == 'Dockerfile': 142 | return None 143 | return tarinfo 144 | 145 | def create_build_context(buildpath, private_layer_id): 146 | 147 | build_dockerfile = os.path.join(buildpath, 'Dockerfile') 148 | 149 | tmp_dockerfile = tempfile.NamedTemporaryFile() 150 | 151 | with open(build_dockerfile) as dockerfile: 152 | content = dockerfile.read() 153 | content = re.sub(r'FROM (.*)\n', 'FROM ' + private_layer_id + '\n', content) 154 | tmp_dockerfile.write(content) 155 | 156 | tmp_dockerfile.seek(0) 157 | 158 | def tar(path, dockerfile, exclude=None): 159 | f = tempfile.NamedTemporaryFile() 160 | t = tarfile.open(mode='w', fileobj=f) 161 | for dirpath, dirnames, filenames in os.walk(path): 162 | relpath = os.path.relpath(dirpath, path) 163 | if relpath == '.': 164 | relpath = '' 165 | if exclude is None: 166 | fnames = filenames 167 | else: 168 | dirnames[:] = [d for d in dirnames 169 | if not fnmatch_any(os.path.join(relpath, d), 170 | exclude)] 171 | fnames = [name for name in filenames 172 | if not fnmatch_any(os.path.join(relpath, name), 173 | exclude)] 174 | for name in fnames: 175 | arcname = os.path.join(relpath, name) 176 | # ignore dockerfile because we will add our synthetic one 177 | t.add(os.path.join(path, arcname), arcname=arcname, filter=no_dockerfile) 178 | 179 | tarinfo = t.gettarinfo(name='Dockerfile', arcname='Dockerfile', fileobj=dockerfile) 180 | t.addfile(tarinfo, dockerfile) 181 | t.close() 182 | f.seek(0) 183 | return f 184 | 185 | dockerignore = os.path.join(buildpath, '.dockerignore') 186 | exclude = None 187 | if os.path.exists(dockerignore): 188 | with open(dockerignore, 'r') as f: 189 | exclude = list(filter(bool, f.read().split('\n'))) 190 | 191 | logger.info('creating context tar from %s', buildpath) 192 | context_tar = tar(buildpath, tmp_dockerfile, exclude=exclude) 193 | return context_tar 194 | 195 | def docker_build(client, tag, context, nocache=False): 196 | 197 | logger.info('building') 198 | res = client.build(fileobj=context, tag=tag, stream=True, custom_context=True, rm=True, nocache=nocache) 199 | 200 | for l in res: 201 | msg = json.loads(l) 202 | if msg.has_key('error'): 203 | print msg['error'], 204 | else: 205 | print msg['stream'], 206 | 207 | def strip_private_layer(client, tag, parent_id, private_layer_id): 208 | 209 | build_tar = tempfile.NamedTemporaryFile() 210 | logger.info('saving tar file from build %s', build_tar.name) 211 | 212 | # TODO save using client 213 | p_args = ['docker', 'save', '--output', build_tar.name, tag] 214 | p = Popen(p_args) 215 | 216 | res = p.wait() 217 | if res != 0: 218 | sys.exit(res) 219 | 220 | try: 221 | client.remove_image(tag) 222 | except Exception: 223 | pass 224 | 225 | extract_dir = tempfile.mkdtemp() 226 | logger.info('extract the build tar %s', extract_dir) 227 | 228 | try: 229 | with tarfile.open(mode='r', fileobj=build_tar) as tar: 230 | tar.extractall(path=extract_dir) 231 | 232 | # prune away image layers under private_id 233 | # we alreayd have them, don't need them again 234 | def prune(basepath, start_id): 235 | json_path = basepath + '/' + start_id + '/json' 236 | f = open(json_path, 'r+') 237 | content = json.load(f) 238 | f.close() 239 | if content.has_key('parent'): 240 | prune(basepath, content['parent']) 241 | elif content.has_key('Parent'): 242 | prune(basepath, content['Parent']) 243 | logger.debug('pruning %s', start_id) 244 | shutil.rmtree(basepath + '/' + start_id) 245 | 246 | logger.info('Splice out private layer id %s', private_layer_id) 247 | prune(extract_dir, private_layer_id) 248 | 249 | for (dirpath, dirnames, filenames) in walk(extract_dir): 250 | for dir in dirnames: 251 | json_path = extract_dir + '/' + dir + '/json' 252 | 253 | f = open(json_path, 'r+') 254 | content = json.load(f) 255 | if content.has_key('parent') and content['parent'] == private_layer_id: 256 | content['parent'] = parent_id 257 | content['Parent'] = parent_id 258 | content['config']['Image'] = parent_id 259 | content['container_config']['Image'] = parent_id 260 | f.seek(0) 261 | json.dump(content, f) 262 | f.truncate() 263 | elif content.has_key('Parent') and content['Parent'] == private_layer_id: 264 | content['parent'] = parent_id 265 | content['Parent'] = parent_id 266 | content['config']['Image'] = parent_id 267 | content['container_config']['Image'] = parent_id 268 | f.seek(0) 269 | json.dump(content, f) 270 | f.truncate() 271 | f.close() 272 | 273 | logger.info('make final tarball') 274 | 275 | tmp_fpath = tempfile.mkstemp() 276 | try: 277 | tmp_file = tmp_fpath[0] 278 | tmp_path = tmp_fpath[1] 279 | 280 | with tarfile.open(name=tmp_path, mode='w') as tar: 281 | tar.add(extract_dir, arcname='') 282 | 283 | os.fsync(tmp_file) 284 | 285 | logger.info('loading final image %s', tmp_path) 286 | p_args = ['docker', 'load', '--input', tmp_path] 287 | p = Popen(p_args) 288 | 289 | res = p.wait() 290 | if res != 0: 291 | sys.exit(res) 292 | finally: 293 | os.remove(tmp_fpath[1]) 294 | 295 | finally: 296 | shutil.rmtree(extract_dir) 297 | -------------------------------------------------------------------------------- /docket/command_line.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import docker 3 | import logging 4 | import os 5 | import docket 6 | 7 | logger = logging.getLogger('docket') 8 | logging.basicConfig() 9 | 10 | parser = argparse.ArgumentParser(description='') 11 | parser.add_argument('-t --tag', dest='tag', help='tag for final image') 12 | parser.add_argument('--verbose', dest='verbose', action='store_true', help='verbose output', default=False) 13 | parser.add_argument('--no-cache', dest='no_cache', action='store_true', help='Do not use cache when building the image', default=False) 14 | parser.add_argument('buildpath', nargs='*') 15 | 16 | args = parser.parse_args() 17 | 18 | if args.verbose: 19 | logger.setLevel(logging.DEBUG) 20 | 21 | cert_path = os.environ.get('DOCKER_CERT_PATH', '') 22 | tls_verify = os.environ.get('DOCKER_TLS_VERIFY', '0') 23 | 24 | base_url = os.environ.get('DOCKER_HOST', 'tcp://127.0.0.1:2375') 25 | base_url = base_url.replace('tcp:', 'https:') 26 | tls_config = None 27 | 28 | if cert_path: 29 | tls_config = docker.tls.TLSConfig(verify=tls_verify, 30 | client_cert=(os.path.join(cert_path, 'cert.pem'), os.path.join(cert_path, 'key.pem')), 31 | ca_cert=os.path.join(cert_path, 'ca.pem') 32 | ) 33 | 34 | client = docker.Client(base_url=base_url, version='1.15', timeout=10, tls=tls_config) 35 | 36 | tag = args.tag or None 37 | buildpath = args.buildpath[0] 38 | 39 | def main(): 40 | docket.build(client=client, tag=tag, buildpath=buildpath, no_cache=args.no_cache) 41 | exit() 42 | 43 | if __name__ == '__main__': 44 | main() 45 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | from codecs import open 3 | from os import path 4 | 5 | here = path.abspath(path.dirname(__file__)) 6 | 7 | # Get the long description from the relevant file 8 | with open(path.join(here, 'DESCRIPTION.md'), encoding='utf-8') as f: 9 | long_description = f.read() 10 | 11 | setup( 12 | name='docket', 13 | 14 | version='1.0.1', 15 | 16 | description='docker builder with private file support', 17 | long_description=long_description, 18 | 19 | url='https://github.com/defunctzombie/docker-build', 20 | 21 | author='Roman Shtylman', 22 | author_email='shtlyman@gmail.com', 23 | 24 | license='MIT', 25 | 26 | # See https://pypi.python.org/pypi?%3Aaction=list_classifiers 27 | classifiers=[ 28 | # How mature is this project? Common values are 29 | # 3 - Alpha 30 | # 4 - Beta 31 | # 5 - Production/Stable 32 | 'Development Status :: 3 - Alpha', 33 | 34 | 'Intended Audience :: Developers', 35 | 'Topic :: Software Development :: Build Tools', 36 | 37 | 'License :: OSI Approved :: MIT License', 38 | 39 | 'Programming Language :: Python :: 2', 40 | 'Programming Language :: Python :: 2.6', 41 | 'Programming Language :: Python :: 2.7' 42 | ], 43 | 44 | keywords='docker', 45 | 46 | # You can just specify the packages manually here if your project is 47 | # simple. Or you can use find_packages(). 48 | packages=find_packages(exclude=['contrib', 'docs', 'tests*']), 49 | 50 | install_requires=['docker-py'], 51 | 52 | extras_require = { 53 | 'dev': ['check-manifest'], 54 | 'test': ['coverage'], 55 | }, 56 | 57 | package_data={ 58 | }, 59 | 60 | entry_points={ 61 | 'console_scripts': [ 62 | 'docket=docket.command_line:main', 63 | ], 64 | }, 65 | ) 66 | --------------------------------------------------------------------------------