├── bob ├── docker.py ├── __init__.py ├── env.py ├── __main__.py └── builds.py ├── .gitignore ├── Pipfile ├── logme.ini ├── Dockerfile ├── README.md ├── setup.py └── Pipfile.lock /bob/docker.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /bob/__init__.py: -------------------------------------------------------------------------------- 1 | from . import __main__ as main -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | bob_builder.egg-info/* 2 | .vscode/* 3 | -------------------------------------------------------------------------------- /bob/env.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | HEROKUISH_IMAGE = os.environ.get("HEROKUISH_IMAGE", "gliderlabs/herokuish:latest") 4 | BUILD_TIMEOUT = 60 * 60 5 | -------------------------------------------------------------------------------- /Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | url = "https://pypi.org/simple" 3 | verify_ssl = true 4 | name = "pypi" 5 | 6 | [packages] 7 | "delegator.py" = "*" 8 | docopt = "*" 9 | logme = "*" 10 | requests = "*" 11 | 12 | [dev-packages] 13 | bob-builder = {editable = true, path = "."} 14 | 15 | [requires] 16 | python_version = "3.7" 17 | -------------------------------------------------------------------------------- /logme.ini: -------------------------------------------------------------------------------- 1 | [colors] 2 | CRITICAL = 3 | color: PURPLE 4 | style: BOLD 5 | ERROR = RED 6 | WARNING = YELLOW 7 | INFO = None 8 | DEBUG = GREEN 9 | 10 | [logme] 11 | level = DEBUG 12 | formatter = {asctime} - {name} - {levelname} - {message} 13 | stream = 14 | type: StreamHandler 15 | active: True 16 | level: DEBUG 17 | null = 18 | type: NullHandler 19 | active: False 20 | level: NOTSET 21 | -------------------------------------------------------------------------------- /bob/__main__.py: -------------------------------------------------------------------------------- 1 | """bob-builder: builds things. 2 | 3 | Usage: 4 | bob-builder [--buildpack=] [--push] [--username= --password=] [--allow-insecure] 5 | 6 | """ 7 | 8 | from docopt import docopt 9 | 10 | from .builds import Build 11 | 12 | 13 | def main(): 14 | args = docopt(__doc__) 15 | image_name = args[""] 16 | codepath = args[""] 17 | trigger_push = args["--push"] 18 | buildpack = args["--buildpack"] 19 | (username, password) = (args["--username"], args["--password"]) 20 | 21 | allow_insecure = args["--allow-insecure"] 22 | 23 | build = Build( 24 | image_name=image_name, 25 | codepath=codepath, 26 | buildpack=buildpack, 27 | username=username, 28 | password=password, 29 | allow_insecure=allow_insecure, 30 | trigger_push=trigger_push, 31 | ) 32 | 33 | 34 | if __name__ == "__main__": 35 | main() 36 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM heroku/heroku:18-build 2 | 3 | ENV DEBIAN_FRONTEND noninteractive 4 | ENV LC_ALL C.UTF-8 5 | ENV LANG C.UTF-8 6 | 7 | # -- Install Pipenv: 8 | RUN apt update && apt upgrade -y && apt install python3.7-dev -y 9 | RUN curl --silent https://bootstrap.pypa.io/get-pip.py | python3.7 10 | 11 | # Backwards compatility. 12 | RUN rm -fr /usr/bin/python3 && ln /usr/bin/python3.7 /usr/bin/python3 13 | 14 | RUN pip3 install pipenv 15 | 16 | # -- Install Application into container: 17 | RUN set -ex && mkdir /bob 18 | WORKDIR /bob 19 | 20 | # -- Adding Pipfiles 21 | COPY Pipfile Pipfile 22 | COPY Pipfile.lock Pipfile.lock 23 | 24 | # Install Docker. 25 | RUN apt install -y docker.io 26 | 27 | # Install daemontools 28 | # RUN apt-get update -qq && apt-get install -qq -y daemontools && apt-get -qq -y --allow-downgrades --allow-remove-essential --allow-change-held-packages dist-upgrade && apt-get clean && rm -rf /var/cache/apt/archives/* /var/lib/apt/lists/* /var/tmp/* 29 | 30 | # Install Herokuish. 31 | # RUN curl --location --silent https://github.com/gliderlabs/herokuish/releases/download/v0.4.4/herokuish_0.4.4_linux_x86_64.tgz | tar -xzC /bin 32 | 33 | # -- Install dependencies: 34 | RUN set -ex && pipenv install --deploy --system 35 | 36 | COPY . /bob 37 | RUN pip3 install -e . 38 | 39 | ENTRYPOINT ["bob-builder", "/app"] 40 | VOLUME /var/lib/docker 41 | VOLUME /app 42 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # bob-builder: builds images, from your code. 2 | 3 | UNDER DEVELOPMENT 4 | 5 | ## Usage 6 | 7 | This software is intended to be used with Docker. It requires Docker privlidges, as it runs Docker itself. 8 | 9 | First, we'll go through the basics of running this software in Docker, then I'll show you the basics of running it, pretending Docker isn't involved. 10 | 11 | 12 | ### Running with Docker 13 | 14 | Run a build of your current working directory: 15 | 16 | $ docker run --privileged -v $(pwd):/app kennethreitz/bob-builder some-imagename 17 | 18 | Run a build of your current working directory, using your native docker instance: 19 | 20 | $ docker run --privileged -v $(pwd):/app -v /var/run/docker.sock:/var/run/docker.sock kennethreitz/bob-builder some-imagename 21 | 22 | ### Using the Software 23 | 24 | # Build a Dockerfile-based image. 25 | $ 26 | Building with Docker. 27 | 28 | # Build a Buildpack-style repo. 29 | $ bob-builder 30 | Building with Heroku-ish. 31 | 32 | # Build a Buildpack-style repo with a custom buildpack. 33 | $ bob-builder --buildpack= 34 | Building with Heroku-ish, with custom buildpack. 35 | 36 | # Push to registry too. 37 | $ bob-builder --push 38 | 39 | # Push to registry (with credentials) too. 40 | $ bob-builder --username=username --password=password --push 41 | 42 | By default, each build will be tagged with a uuid4, unless you specify your own tag in the image name. 43 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Note: To use the 'upload' functionality of this file, you must: 5 | # $ pip install twine 6 | 7 | import io 8 | import os 9 | import sys 10 | import shutil 11 | 12 | from setuptools import find_packages, setup, Command 13 | 14 | # Package meta-data. 15 | NAME = "bob-builder" 16 | DESCRIPTION = "My short description for my project." 17 | URL = "https://github.com/kennethreitz/bob-the-builder" 18 | EMAIL = "me@example.com" 19 | AUTHOR = "Kenneth Reitz" 20 | REQUIRES_PYTHON = ">=3.6.0" 21 | VERSION = None 22 | 23 | # What packages are required for this module to be executed? 24 | REQUIRED = [ 25 | # 'requests', 'maya', 'records', 26 | ] 27 | 28 | # What packages are optional? 29 | EXTRAS = { 30 | # 'fancy feature': ['django'], 31 | } 32 | 33 | # The rest you shouldn't have to touch too much :) 34 | # ------------------------------------------------ 35 | # Except, perhaps the License and Trove Classifiers! 36 | # If you do change the License, remember to change the Trove Classifier for that! 37 | 38 | here = os.path.abspath(os.path.dirname(__file__)) 39 | 40 | 41 | # Import the README and use it as the long-description. 42 | # Note: this will only work if 'README.md' is present in your MANIFEST.in file! 43 | try: 44 | with io.open(os.path.join(here, "README.md"), encoding="utf-8") as f: 45 | long_description = "\n" + f.read() 46 | except FileNotFoundError: 47 | long_description = DESCRIPTION 48 | 49 | 50 | class UploadCommand(Command): 51 | """Support setup.py upload.""" 52 | 53 | description = "Build and publish the package." 54 | user_options = [] 55 | 56 | @staticmethod 57 | def status(s): 58 | """Prints things in bold.""" 59 | print("\033[1m{0}\033[0m".format(s)) 60 | 61 | def initialize_options(self): 62 | pass 63 | 64 | def finalize_options(self): 65 | pass 66 | 67 | def run(self): 68 | try: 69 | self.status("Removing previous builds…") 70 | shutil.rmtree(os.path.join(here, "dist")) 71 | except OSError: 72 | pass 73 | 74 | self.status("Building Source and Wheel (universal) distribution…") 75 | os.system("{0} setup.py sdist bdist_wheel --universal".format(sys.executable)) 76 | 77 | self.status("Uploading the package to PyPI via Twine…") 78 | os.system("twine upload dist/*") 79 | 80 | self.status("Pushing git tags…") 81 | os.system("git tag v{0}".format(about["__version__"])) 82 | os.system("git push --tags") 83 | 84 | sys.exit() 85 | 86 | 87 | # Add logme.ini 88 | dest_file = os.path.join(here, "bob/logme.ini") 89 | shutil.copy(os.path.join(here, "logme.ini"), dest_file) 90 | 91 | 92 | # Where the magic happens: 93 | setup( 94 | name=NAME, 95 | version=VERSION, 96 | description=DESCRIPTION, 97 | long_description=long_description, 98 | long_description_content_type="text/markdown", 99 | author=AUTHOR, 100 | author_email=EMAIL, 101 | python_requires=REQUIRES_PYTHON, 102 | url=URL, 103 | packages=find_packages(exclude=("tests",)), 104 | package_data={ 105 | "": [dest_file] 106 | }, 107 | # If your package is a single module, use this instead of 'packages': 108 | # py_modules=['mypackage'], 109 | entry_points={"console_scripts": ["bob-builder=bob.__main__:main"]}, 110 | install_requires=REQUIRED, 111 | extras_require=EXTRAS, 112 | include_package_data=True, 113 | license="Apache 2.0", 114 | classifiers=[ 115 | # Trove classifiers 116 | # Full list: https://pypi.python.org/pypi?%3Aaction=list_classifiers 117 | "License :: OSI Approved :: MIT License", 118 | "Programming Language :: Python", 119 | "Programming Language :: Python :: 3", 120 | "Programming Language :: Python :: 3.7", 121 | "Programming Language :: Python :: Implementation :: CPython", 122 | "Programming Language :: Python :: Implementation :: PyPy", 123 | ], 124 | # $ setup.py publish support. 125 | cmdclass={"upload": UploadCommand}, 126 | ) 127 | -------------------------------------------------------------------------------- /Pipfile.lock: -------------------------------------------------------------------------------- 1 | { 2 | "_meta": { 3 | "hash": { 4 | "sha256": "a979e95e78a9bfe4c0b687c394729a034f217db55334a81c10451afd436d1ca0" 5 | }, 6 | "pipfile-spec": 6, 7 | "requires": { 8 | "python_version": "3.7" 9 | }, 10 | "sources": [ 11 | { 12 | "name": "pypi", 13 | "url": "https://pypi.org/simple", 14 | "verify_ssl": true 15 | } 16 | ] 17 | }, 18 | "default": { 19 | "bnmutils": { 20 | "hashes": [ 21 | "sha256:09d4fe9ec208b7f244a7d1954fc36209b84a1b7bbb87d85dc2cc69cb2fbccaa0", 22 | "sha256:5ba31ba38f0aa99b559d2b5459edd69f30eb86a31d624b338e28cbdd8f2337df" 23 | ], 24 | "markers": "python_version >= '3'", 25 | "version": "==1.0.2" 26 | }, 27 | "certifi": { 28 | "hashes": [ 29 | "sha256:376690d6f16d32f9d1fe8932551d80b23e9d393a8578c5633a2ed39a64861638", 30 | "sha256:456048c7e371c089d0a77a5212fb37a2c2dce1e24146e3b7e0261736aaeaa22a" 31 | ], 32 | "version": "==2018.8.24" 33 | }, 34 | "chardet": { 35 | "hashes": [ 36 | "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", 37 | "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" 38 | ], 39 | "version": "==3.0.4" 40 | }, 41 | "click": { 42 | "hashes": [ 43 | "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", 44 | "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7" 45 | ], 46 | "markers": "python_version != '3.3.*' and python_version != '3.0.*' and python_version != '3.2.*' and python_version != '3.1.*' and python_version >= '2.7'", 47 | "version": "==7.0" 48 | }, 49 | "delegator.py": { 50 | "hashes": [ 51 | "sha256:814657d96b98a244c479e3d5f6e9e850ac333e85f807d6bc846e72bbb2537806", 52 | "sha256:e6cc9cedab9ae59b169ee0422e17231adedadb144e63c0b5a60e6ff8adf8521b" 53 | ], 54 | "index": "pypi", 55 | "version": "==0.1.1" 56 | }, 57 | "docopt": { 58 | "hashes": [ 59 | "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491" 60 | ], 61 | "index": "pypi", 62 | "version": "==0.6.2" 63 | }, 64 | "idna": { 65 | "hashes": [ 66 | "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", 67 | "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16" 68 | ], 69 | "version": "==2.7" 70 | }, 71 | "logme": { 72 | "hashes": [ 73 | "sha256:0a5b8415cd24d0d7d32f34082de471f000f0d93e5a26939f137536b9e38e7633", 74 | "sha256:ead1fa94c612bc50914224b5da80821a5cd5af03247245e18dd4980ad8cdfdfc" 75 | ], 76 | "index": "pypi", 77 | "version": "==1.3.1" 78 | }, 79 | "pexpect": { 80 | "hashes": [ 81 | "sha256:2a8e88259839571d1251d278476f3eec5db26deb73a70be5ed5dc5435e418aba", 82 | "sha256:3fbd41d4caf27fa4a377bfd16fef87271099463e6fa73e92a52f92dfee5d425b" 83 | ], 84 | "version": "==4.6.0" 85 | }, 86 | "ptyprocess": { 87 | "hashes": [ 88 | "sha256:923f299cc5ad920c68f2bc0bc98b75b9f838b93b599941a6b63ddbc2476394c0", 89 | "sha256:d7cc528d76e76342423ca640335bd3633420dc1366f258cb31d05e865ef5ca1f" 90 | ], 91 | "version": "==0.6.0" 92 | }, 93 | "requests": { 94 | "hashes": [ 95 | "sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1", 96 | "sha256:ec22d826a36ed72a7358ff3fe56cbd4ba69dd7a6718ffd450ff0e9df7a47ce6a" 97 | ], 98 | "index": "pypi", 99 | "version": "==2.19.1" 100 | }, 101 | "urllib3": { 102 | "hashes": [ 103 | "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf", 104 | "sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5" 105 | ], 106 | "markers": "python_version < '4' and python_version != '3.3.*' and python_version != '3.2.*' and python_version != '3.0.*' and python_version != '3.1.*' and python_version >= '2.6'", 107 | "version": "==1.23" 108 | } 109 | }, 110 | "develop": { 111 | "bob-builder": { 112 | "editable": true, 113 | "path": "." 114 | } 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /bob/builds.py: -------------------------------------------------------------------------------- 1 | import io 2 | import os 3 | import time 4 | import json 5 | import tempfile 6 | import tarfile 7 | 8 | from uuid import uuid4 9 | from pathlib import Path 10 | 11 | import logme 12 | import delegator 13 | from requests import Session 14 | 15 | from .env import HEROKUISH_IMAGE, BUILD_TIMEOUT 16 | 17 | delegator.TIMEOUT = BUILD_TIMEOUT 18 | requests = Session() 19 | 20 | 21 | @logme.log 22 | class Build: 23 | def __init__( 24 | self, 25 | *, 26 | image_name, 27 | codepath, 28 | allow_insecure=False, 29 | username=None, 30 | password=None, 31 | buildpack=None, 32 | trigger_build=True, 33 | trigger_push=True, 34 | ): 35 | self.uuid = uuid4().hex 36 | self.image_name = image_name 37 | self.codepath = Path(os.path.abspath(codepath)) 38 | self.username = username 39 | self.password = password 40 | self.allow_insecure = allow_insecure 41 | self.was_built = None 42 | 43 | self.buildpack = buildpack 44 | self.buildpack_dir = None 45 | 46 | assert os.path.exists(self.codepath) 47 | 48 | if self.buildpack: 49 | self.ensure_buildpack() 50 | 51 | if trigger_build: 52 | self.build() 53 | 54 | if trigger_push: 55 | self.push() 56 | 57 | @property 58 | def custom_buildpacks_path(self): 59 | if self.buildpack: 60 | if not self.buildpack_dir: 61 | self.buildpack_dir = Path(tempfile.gettempdir()) 62 | return self.buildpack_dir 63 | 64 | @property 65 | def custom_buildpack_path(self): 66 | if self.buildpack: 67 | dl_dir = (self.custom_buildpacks_path / "buildpack").resolve() 68 | 69 | # Ensure the download dir exists. 70 | os.makedirs(dl_dir, exist_ok=True) 71 | 72 | return dl_dir 73 | 74 | def ensure_buildpack(self): 75 | assert self.buildpack 76 | 77 | untargz = False 78 | clone = False 79 | 80 | if self.buildpack.endswith(".tgz") or self.buildpack.endswith(".tar.gz"): 81 | untargz = True 82 | else: 83 | clone = True 84 | 85 | if untargz: 86 | self.logger.info("Downloading buildpack...") 87 | r = requests.get(self.buildpack, stream=False) 88 | self.logger.info("Extracting buildpack...") 89 | b = io.BytesIO(r.content) 90 | t = tarfile.open(mode="r:gz", fileobj=b) 91 | t.extractall(path=self.custom_buildpack_path) 92 | 93 | elif unzip: 94 | r = requests.get(self.buildpack) 95 | elif clone: 96 | cmd = f"git clone {self.buildpack} {self.custom_buildpack_path}" 97 | self.logger.debug(f"$ {cmd}") 98 | c = delegator.run(cmd) 99 | assert c.ok 100 | 101 | def docker(self, cmd, assert_ok=True, fail=True): 102 | cmd = f"docker {cmd}" 103 | self.logger.debug(f"$ {cmd}") 104 | c = delegator.run(cmd) 105 | try: 106 | assert c.ok 107 | except AssertionError as e: 108 | self.logger.debug(c.out) 109 | self.logger.debug(c.err) 110 | 111 | if fail: 112 | raise e 113 | 114 | return c 115 | 116 | @property 117 | def requires_login(self): 118 | return all([self.username, self.password]) 119 | 120 | @property 121 | def docker_tag(self): 122 | if ":" in self.image_name: 123 | return self.image_name 124 | else: 125 | return f"{self.image_name}:{self.uuid}" 126 | 127 | @property 128 | def has_dockerfile(self): 129 | return os.path.isfile((self.codepath / "Dockerfile").resolve()) 130 | 131 | @property 132 | def registry_specified(self): 133 | if len(self.image_name.split("/")) > 1: 134 | return self.image_name.split("/")[0] 135 | 136 | def ensure_docker(self): 137 | 138 | if self.allow_insecure and self.registry_specified: 139 | logger.debug("Configuring docker service to allow our insecure registry...") 140 | # Configure our registry as insecure. 141 | try: 142 | with open("/etc/docker/daemon.json", "w") as f: 143 | data = {"insecure-registries": [self.registry_specified]} 144 | json.dump(data, f) 145 | # This fails when running on Windows... 146 | except FileNotFoundError: 147 | pass 148 | 149 | # Start docker service. 150 | self.logger.info("Starting docker") 151 | c = delegator.run("service docker start") 152 | # assert c.ok 153 | time.sleep(0.3) 154 | 155 | try: 156 | # Login to Docker. 157 | if self.requires_login: 158 | 159 | self.docker(f"login -u {self.username} -p {self.password}") 160 | c = self.docker("ps") 161 | assert c.ok 162 | except AssertionError: 163 | raise RuntimeError("Docker is not available.") 164 | 165 | def docker_build(self): 166 | self.logger.info(f"Using Docker to build {self.uuid!r} of {self.image_name!r}.") 167 | 168 | self.ensure_docker() 169 | 170 | c = self.docker(f"build {self.codepath} --tag {self.docker_tag}") 171 | self.logger.debug(c.out) 172 | self.logger.debug(c.err) 173 | 174 | def buildpack_build(self): 175 | self.logger.info(f"Using buildpacks to build {self.uuid!r}.") 176 | buildpacks = ( 177 | f"-v {self.custom_buildpacks_path}:/tmp/buildpacks" 178 | if self.buildpack 179 | else "" 180 | ) 181 | docker_cmd = ( 182 | f"run -i --name=build-{self.uuid} -v {self.codepath}:/tmp/app {buildpacks}" 183 | f" {HEROKUISH_IMAGE} /bin/herokuish buildpack build" 184 | ) 185 | c = self.docker(docker_cmd) 186 | self.logger.debug(c.out) 187 | self.logger.debug(c.err) 188 | 189 | # Commit the Docker build. 190 | commit = self.docker(f"commit build-{self.uuid}") 191 | commit_output = commit.out.strip() 192 | 193 | docker_cmd = ( 194 | f"create --expose 80 --env PORT=80 " 195 | f"--name={self.uuid} {commit_output} /bin/herokuish procfile start web" 196 | ) 197 | create = self.docker(docker_cmd) 198 | create_output = create.out.strip() 199 | self.logger.debug(create_output) 200 | 201 | # Commit service to Docker. 202 | commit = self.docker(f"commit {self.uuid}") 203 | commit_output = commit.out.strip() 204 | 205 | tag = self.docker(f"tag {commit_output} {self.docker_tag}") 206 | tag_output = tag.out.strip() 207 | 208 | def build(self): 209 | self.logger.info(f"Starting build {self.uuid!r} of {self.image_name!r}.") 210 | if self.has_dockerfile: 211 | self.docker_build() 212 | else: 213 | self.buildpack_build() 214 | 215 | self.was_built = True 216 | self.logger.info(f"{self.docker_tag} successfully built!") 217 | 218 | def push(self): 219 | assert self.was_built 220 | assert self.push 221 | 222 | c = self.docker(f"push {self.docker_tag}") 223 | self.logger.debug(c.out) 224 | self.logger.debug(c.err) 225 | assert c.ok 226 | --------------------------------------------------------------------------------