├── giftwrap ├── __init__.py ├── tests │ ├── __init__.py │ ├── test_util.py │ ├── test_builder.py │ ├── test_log.py │ └── test_settings.py ├── templates │ └── Dockerfile.jinja2 ├── color.py ├── build_spec.py ├── util.py ├── package.py ├── settings.py ├── shell.py ├── builders │ ├── package_builder.py │ ├── __init__.py │ └── docker_builder.py ├── openstack_git_repo.py ├── openstack_project.py ├── openstack_commit.py └── gerrit.py ├── CHANGELOG.md ├── giftwrap.png ├── test-requirements.txt ├── .travis.yml ├── .gitreview ├── .rubocop.yml ├── .gitignore ├── requirements.txt ├── scripts ├── prepare_redhat.sh └── prepare_debian.sh ├── Makefile ├── examples ├── manifest.yml └── docker-manifest.yml ├── Dockerfile ├── setup.py ├── tox.ini ├── setup.cfg ├── Vagrantfile ├── README.md └── LICENSE /giftwrap/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /giftwrap/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 2.1.0 2 | ----- 3 | - feature: add builder drivers using stevedore 4 | -------------------------------------------------------------------------------- /giftwrap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blueboxgroup/giftwrap/HEAD/giftwrap.png -------------------------------------------------------------------------------- /test-requirements.txt: -------------------------------------------------------------------------------- 1 | nose 2 | mock 3 | unittest2 4 | flake8 5 | pep8==1.5.7 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - 2.6 4 | - 2.7 5 | script: 6 | - make test 7 | -------------------------------------------------------------------------------- /.gitreview: -------------------------------------------------------------------------------- 1 | [gerrit] 2 | host=review.openstack.org 3 | port=29418 4 | project=openstack/giftwrap.git 5 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | AllCops: 2 | Include: 3 | - ./**/*.rb 4 | - Vagrantfile 5 | 6 | WordArray: 7 | MinSize: 3 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .tox/ 2 | *.egg-info/ 3 | AUTHORS 4 | ChangeLog 5 | *.csv 6 | .vagrant/ 7 | build 8 | pbr*.egg 9 | *.pyc 10 | *.sw? 11 | *.deb 12 | manifests/* 13 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | argparse 2 | colorama 3 | pbr 4 | GitPython==0.3.2.RC1 5 | giturlparse.py 6 | pyyaml 7 | jinja2 8 | requests 9 | pygerrit 10 | docker-py 11 | virtualenv 12 | stevedore 13 | -------------------------------------------------------------------------------- /scripts/prepare_redhat.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | yum install -y rubygems ruby-devel python-devel libffi-devel openssl-devel libxml2-devel libxslt-devel libmysqlclient-devel libpq-dev libsqlite3-devel 4 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: test 2 | 3 | deps: 4 | @echo "--> Installing dependencies" 5 | @pip install tox 6 | 7 | test: deps 8 | @echo "--> Running tests" 9 | @tox 10 | 11 | .PHONY: all deps test 12 | -------------------------------------------------------------------------------- /giftwrap/templates/Dockerfile.jinja2: -------------------------------------------------------------------------------- 1 | FROM {{ base_image }} 2 | {% if maintainer -%} 3 | MAINTAINER {{ maintainer }} 4 | {% endif %} 5 | 6 | {% for k,v in envvars.iteritems() -%} 7 | ENV {{ k }} {{ v }} 8 | {% endfor -%} 9 | 10 | RUN {% for command in commands[:-1] -%}{{ command|safe }} && {% endfor -%} {{ commands[-1]|safe }} 11 | -------------------------------------------------------------------------------- /examples/manifest.yml: -------------------------------------------------------------------------------- 1 | --- 2 | settings: 3 | package_name_format: 'openstack-{{ project.name }}-{{ settings.version }}' 4 | build_type: package 5 | version: '10.0-bbc1' 6 | base_path: '/opt/bbc/openstack-{{ settings.version }}' 7 | force_overwrite: true 8 | 9 | projects: 10 | - name: glance 11 | gitref: stable/juno 12 | -------------------------------------------------------------------------------- /scripts/prepare_debian.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | apt-get -yqq update && apt-get dist-upgrade -y 4 | apt-get -yqq update && apt-get install -yqq build-essential ruby1.9.1-dev git python-pip python-dev python-virtualenv libxml2-dev libxslt-dev libffi-dev libmysqlclient-dev libpq-dev libsqlite3-dev 5 | apt-get remove -y python-pip python-setuptools 6 | -------------------------------------------------------------------------------- /examples/docker-manifest.yml: -------------------------------------------------------------------------------- 1 | --- 2 | settings: 3 | package_name_format: 'openstack-{{ project.name }}-{{ settings.version }}' 4 | build_type: package 5 | version: '10.0-bbc1' 6 | base_path: '/openstack' 7 | force_overwrite: true 8 | 9 | projects: 10 | - name: glance 11 | gitref: stable/kilo 12 | - name: heat 13 | gitref: stable/kilo 14 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:precise 2 | 3 | RUN apt-get update && \ 4 | apt-get -yqq install build-essential ruby1.9.1 ruby1.9.1-dev python-dev \ 5 | rubygems1.9.1 git python-pip libxml2-dev libxslt-dev libffi-dev libmysqlclient-dev libpq-dev 6 | 7 | RUN gem install --no-ri --no-rdoc fpm 8 | 9 | ADD https://get.docker.com/builds/Linux/x86_64/docker-latest /usr/local/bin/docker 10 | 11 | RUN chmod +x /usr/local/bin/docker 12 | 13 | ADD . /giftwrap 14 | 15 | WORKDIR /giftwrap 16 | 17 | RUN python setup.py install 18 | 19 | CMD giftwrap build -m /tmp/manifest.yml 20 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # vim: tabstop=4 shiftwidth=4 softtabstop=4 2 | 3 | # Copyright 2014, John Dewey 4 | # All Rights Reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 7 | # not use this file except in compliance with the License. You may obtain 8 | # a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 15 | # License for the specific language governing permissions and limitations 16 | 17 | import setuptools 18 | 19 | setuptools.setup( 20 | setup_requires=['pbr'], 21 | pbr=True) 22 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | # Tox (http://tox.testrun.org/) is a tool for running tests 2 | # in multiple virtualenvs. This configuration file will run the 3 | # test suite on all supported python versions. To use it, "pip install tox" 4 | # and then run "tox" from this directory. 5 | 6 | [tox] 7 | envlist = py27,pep8 8 | 9 | [testenv] 10 | usedevelop = True 11 | setenv = VIRTUAL_ENV={envdir} 12 | deps = -r{toxinidir}/requirements.txt 13 | -r{toxinidir}/test-requirements.txt 14 | commands = nosetests 15 | sitepackages = False 16 | 17 | [testenv:pep8] 18 | commands = 19 | flake8 {posargs} --ignore=E402 {toxinidir}/giftwrap 20 | 21 | [testenv:venv] 22 | commands = {posargs} 23 | 24 | [testenv:pyflakes] 25 | deps = flake8 26 | commands = flake8 27 | 28 | [flake8] 29 | ignore = H301,H306 30 | builtins = _ 31 | exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg 32 | show-source = True 33 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = python-giftwrap 3 | version = 2.1.0 4 | summary = giftwrap - A tool to build full-stack native system packages. 5 | description-file = 6 | README.md 7 | author = Craig Tracey 8 | author-email = craigtracey@gmail.com 9 | home-page = https://github.com/blueboxgroup/giftwrap 10 | include_package_data = True 11 | classifier = 12 | Intended Audience :: Information Technology 13 | Intended Audience :: Developers 14 | License :: OSI Approved :: Apache Software License 15 | Operating System :: POSIX :: Linux 16 | Programming Language :: Python 17 | 18 | [entry_points] 19 | console_scripts = 20 | giftwrap = giftwrap.shell:main 21 | 22 | giftwrap.builder.drivers = 23 | package = giftwrap.builders.package_builder:PackageBuilder 24 | docker = giftwrap.builders.docker_builder:DockerBuilder 25 | 26 | [files] 27 | packages = 28 | giftwrap 29 | extra_files = 30 | giftwrap/templates/Dockerfile.jinja2 31 | 32 | [build_sphinx] 33 | all_files = 1 34 | build-dir = doc/build 35 | source-dir = doc/source 36 | -------------------------------------------------------------------------------- /giftwrap/tests/test_util.py: -------------------------------------------------------------------------------- 1 | # vim: tabstop=4 shiftwidth=4 softtabstop=4 2 | 3 | # Copyright 2014, John Dewey 4 | # All Rights Reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 7 | # not use this file except in compliance with the License. You may obtain 8 | # a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 15 | # License for the specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import unittest2 as unittest 19 | 20 | from giftwrap import util 21 | 22 | 23 | class TestUtil(unittest.TestCase): 24 | 25 | def test_execute_returns_stdout(self): 26 | cmd = 'echo stdout' 27 | out = util.execute(cmd) 28 | self.assertEquals('stdout\n', out) 29 | 30 | def test_execute_raises_exception_on_error(self): 31 | cmd = 'echo stderr >&2 && false' 32 | with self.assertRaises(Exception): 33 | util.execute(cmd) 34 | 35 | def test_nonzero_exit_code(self): 36 | cmd = 'echo stdout && false' 37 | out = util.execute(cmd, exit=1) 38 | self.assertEquals('stdout\n', out) 39 | -------------------------------------------------------------------------------- /giftwrap/tests/test_builder.py: -------------------------------------------------------------------------------- 1 | # vim: tabstop=4 shiftwidth=4 softtabstop=4 2 | 3 | # Copyright 2015, Craig Tracey 4 | # All Rights Reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 7 | # not use this file except in compliance with the License. You may obtain 8 | # a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 15 | # License for the specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import copy 19 | import unittest2 as unittest 20 | 21 | from giftwrap.builders import Builder, BUILDER_DRIVER_NAMESPACE 22 | from stevedore import extension 23 | 24 | BASE_DRIVERS = set(['docker', 'package']) 25 | 26 | 27 | class TestBuilder(unittest.TestCase): 28 | 29 | def test_default_drivers(self): 30 | drivers = set(Builder.builder_names()) 31 | self.assertEqual(drivers, BASE_DRIVERS) 32 | 33 | def test_additional_drivers(self): 34 | em = extension.ExtensionManager(BUILDER_DRIVER_NAMESPACE) 35 | em.extensions.append(extension.Extension('test', None, None, None)) 36 | drivers = set(Builder.builder_names(em)) 37 | base_drivers = copy.copy(BASE_DRIVERS) 38 | base_drivers.add('test') 39 | self.assertEqual(drivers, base_drivers) 40 | -------------------------------------------------------------------------------- /giftwrap/color.py: -------------------------------------------------------------------------------- 1 | # vim: tabstop=4 shiftwidth=4 softtabstop=4 2 | 3 | # Copyright 2014, Craig Tracey 4 | # All Rights Reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 7 | # not use this file except in compliance with the License. You may obtain 8 | # a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 15 | # License for the specific language governing permissions and limitations 16 | 17 | from colorama import AnsiToWin32, Fore, Style 18 | from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL, StreamHandler 19 | 20 | 21 | class ColorStreamHandler(StreamHandler): 22 | 23 | log_colors = { 24 | INFO: Fore.GREEN, 25 | DEBUG: Fore.CYAN, 26 | WARNING: Fore.YELLOW, 27 | ERROR: Fore.RED, 28 | CRITICAL: Fore.RED, 29 | } 30 | 31 | def __init__(self, stream): 32 | StreamHandler.__init__(self, AnsiToWin32(stream).stream) 33 | 34 | @property 35 | def is_tty(self): 36 | isatty = getattr(self.stream, 'isatty', None) 37 | return isatty and isatty() 38 | 39 | def format(self, record): 40 | msg = StreamHandler.format(self, record) 41 | if self.is_tty: 42 | msg = self.log_colors[record.levelno] + msg + Style.RESET_ALL 43 | return msg 44 | -------------------------------------------------------------------------------- /giftwrap/tests/test_log.py: -------------------------------------------------------------------------------- 1 | # vim: tabstop=4 shiftwidth=4 softtabstop=4 2 | 3 | # Copyright 2014, John Dewey 4 | # All Rights Reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 7 | # not use this file except in compliance with the License. You may obtain 8 | # a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 15 | # License for the specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import logging 19 | import unittest2 as unittest 20 | 21 | from nose.plugins.logcapture import LogCapture 22 | 23 | 24 | class TestLog(unittest.TestCase): 25 | 26 | def test_get_logger(self): 27 | lc = LogCapture() 28 | lc.begin() 29 | logger = logging.getLogger('giftwrap') 30 | logger.setLevel(logging.INFO) 31 | logger.debug('test-debug') 32 | logger.info('test-info') 33 | lc.end() 34 | 35 | self.assertEquals("giftwrap: INFO: test-info", lc.handler.buffer[0]) 36 | 37 | def test_get_logger_debug(self): 38 | lc = LogCapture() 39 | lc.begin() 40 | logger = logging.getLogger('giftwrap') 41 | logger.setLevel(logging.DEBUG) 42 | logger.info('test-info') 43 | logger.debug('test-debug') 44 | lc.end() 45 | 46 | self.assertEquals("giftwrap: INFO: test-info", lc.handler.buffer[0]) 47 | self.assertEquals("giftwrap: DEBUG: test-debug", lc.handler.buffer[1]) 48 | -------------------------------------------------------------------------------- /giftwrap/tests/test_settings.py: -------------------------------------------------------------------------------- 1 | # vim: tabstop=4 shiftwidth=4 softtabstop=4 2 | 3 | # Copyright 2014, John Dewey 4 | # All Rights Reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 7 | # not use this file except in compliance with the License. You may obtain 8 | # a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 15 | # License for the specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import unittest2 as unittest 19 | 20 | from giftwrap import settings 21 | 22 | SAMPLE_SETTINGS = { 23 | 'package_name_format': 'my-package-name', 24 | 'version': '1.2', 25 | 'base_path': '/basepath' 26 | } 27 | 28 | 29 | class TestSettings(unittest.TestCase): 30 | 31 | def test_factory(self): 32 | settings_dict = SAMPLE_SETTINGS 33 | s = settings.Settings.factory(settings_dict) 34 | 35 | self.assertEquals('my-package-name', s.package_name_format) 36 | self.assertEquals('1.2', s.version) 37 | self.assertEquals('/basepath', s.base_path) 38 | 39 | def test_factory_has_default_base_path(self): 40 | settings_dict = {'version': 'version'} 41 | s = settings.Settings.factory(settings_dict) 42 | 43 | self.assertEquals('/opt/openstack', s.base_path) 44 | 45 | def test_factory_raises_when_version_missing(self): 46 | settings_dict = SAMPLE_SETTINGS 47 | del(settings_dict['version']) 48 | 49 | with self.assertRaises(Exception): 50 | settings.Settings.factory(settings_dict) 51 | -------------------------------------------------------------------------------- /giftwrap/build_spec.py: -------------------------------------------------------------------------------- 1 | # vim: tabstop=4 shiftwidth=4 softtabstop=4 2 | 3 | # Copyright 2014, Craig Tracey 4 | # All Rights Reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 7 | # not use this file except in compliance with the License. You may obtain 8 | # a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 15 | # License for the specific language governing permissions and limitations 16 | 17 | import yaml 18 | 19 | from giftwrap.openstack_project import OpenstackProject 20 | from giftwrap.settings import Settings 21 | 22 | 23 | class BuildSpec(object): 24 | 25 | def __init__(self, manifest, version, build_type=None, parallel=True, 26 | limit_projects=None): 27 | self._manifest = yaml.load(manifest) 28 | self.version = version 29 | self.build_type = build_type 30 | manifest_settings = self._manifest['settings'] 31 | if version: 32 | manifest_settings['version'] = version 33 | if build_type: 34 | manifest_settings['build_type'] = build_type 35 | if build_type == 'docker': 36 | parallel = False 37 | manifest_settings['parallel_build'] = parallel 38 | self.settings = Settings.factory(manifest_settings) 39 | self.projects = self._render_projects(limit_projects) 40 | 41 | def _render_projects(self, limit_projects): 42 | projects = [] 43 | if 'projects' in self._manifest: 44 | for project in self._manifest['projects']: 45 | if limit_projects is None or project['name'] in limit_projects: 46 | projects.append(OpenstackProject.factory(self.settings, 47 | project, 48 | self.version)) 49 | return projects 50 | -------------------------------------------------------------------------------- /giftwrap/util.py: -------------------------------------------------------------------------------- 1 | # vim: tabstop=4 shiftwidth=4 softtabstop=4 2 | 3 | # Copyright 2014, John Dewey 4 | # All Rights Reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 7 | # not use this file except in compliance with the License. You may obtain 8 | # a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 15 | # License for the specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import logging 19 | import os 20 | import subprocess 21 | 22 | LOG = logging.getLogger(__name__) 23 | 24 | 25 | def execute(command, cwd=None, exit=0): 26 | """ 27 | Executes a command in a subprocess. Returns a tuple of 28 | (exitcode, out, err). 29 | 30 | :param command: Command string to execute. 31 | :param cwd: Directory to execute from. 32 | :param exit: The expected exit code. 33 | """ 34 | 35 | original_dir = None 36 | if cwd: 37 | original_dir = os.getcwd() 38 | os.chdir(cwd) 39 | LOG.debug("Changed directory to %s", cwd) 40 | 41 | LOG.info("Running: '%s'", command) 42 | process = subprocess.Popen(command, 43 | cwd=os.getcwd(), 44 | stdin=subprocess.PIPE, 45 | stdout=subprocess.PIPE, 46 | stderr=subprocess.PIPE, 47 | shell=True) 48 | (out, err) = process.communicate() 49 | exitcode = process.wait() 50 | 51 | LOG.debug("Command exited with rc: %s; STDOUT: %s; STDERR: %s" % 52 | (exitcode, out, err)) 53 | 54 | if cwd: 55 | os.chdir(original_dir) 56 | LOG.debug("Changed directory back to %s", original_dir) 57 | 58 | if exitcode != exit: 59 | raise Exception("Failed to run '%s': rc: %d, out: '%s', err: '%s'" % 60 | (command, exitcode, out, err)) 61 | 62 | return out 63 | 64 | 65 | def relative_pathify(path): 66 | if path.startswith('/'): 67 | return path[1:] 68 | return path 69 | -------------------------------------------------------------------------------- /giftwrap/package.py: -------------------------------------------------------------------------------- 1 | # vim: tabstop=4 shiftwidth=4 softtabstop=4 2 | 3 | # Copyright 2014, Craig Tracey 4 | # All Rights Reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 7 | # not use this file except in compliance with the License. You may obtain 8 | # a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 15 | # License for the specific language governing permissions and limitations 16 | 17 | import os 18 | import platform 19 | import re 20 | 21 | from giftwrap.util import execute 22 | 23 | SUPPORTED_DISTROS = { 24 | 'deb': ['Ubuntu'], 25 | 'rpm': ['Scientific Linux', 'CentOS.*'] 26 | } 27 | 28 | 29 | class Package(object): 30 | 31 | def __init__(self, name, version, install_path, output_dir, 32 | overwrite=False, dependencies=None): 33 | self.name = name 34 | self.version = version 35 | self.install_path = install_path 36 | self.output_dir = output_dir 37 | self.overwrite = overwrite 38 | self.dependencies = dependencies 39 | 40 | def _get_platform_target(self): 41 | current_distro = platform.linux_distribution()[0] 42 | for pkgtype, distros in SUPPORTED_DISTROS.iteritems(): 43 | for distro in distros: 44 | if re.match(distro, current_distro): 45 | return pkgtype 46 | raise Exception("Sorry, '%s' is an unsupported distribution" % 47 | current_distro) 48 | 49 | def build(self): 50 | target = self._get_platform_target() 51 | overwrite = '' 52 | if self.overwrite: 53 | overwrite = '-f' 54 | 55 | deps = '' 56 | if self.dependencies: 57 | deps = '-d %s' % (' -d '.join(self.dependencies)) 58 | 59 | if not os.path.exists(self.output_dir): 60 | os.makedirs(self.output_dir) 61 | 62 | # not wrapping in a try block - handled by caller 63 | execute("fpm %s -s dir -t %s -n %s -v %s %s %s" % (overwrite, 64 | target, self.name, self.version, deps, self.install_path), 65 | self.output_dir) 66 | -------------------------------------------------------------------------------- /giftwrap/settings.py: -------------------------------------------------------------------------------- 1 | # vim: tabstop=4 shiftwidth=4 softtabstop=4 2 | 3 | # Copyright 2014, Craig Tracey 4 | # All Rights Reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 7 | # not use this file except in compliance with the License. You may obtain 8 | # a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 15 | # License for the specific language governing permissions and limitations 16 | 17 | import os 18 | 19 | DEFAULT_BUILD_TYPE = 'package' 20 | 21 | 22 | class Settings(object): 23 | 24 | DEFAULTS = { 25 | 'package_name_format': 'openstack-{{ project.name }}', 26 | 'base_path': '/opt/openstack', 27 | 'install_path': '{{ settings.base_path }}/{{ project.name }}' 28 | } 29 | 30 | def __init__(self, build_type=DEFAULT_BUILD_TYPE, 31 | package_name_format=None, version=None, 32 | base_path=None, install_path=None, gerrit_dependencies=True, 33 | force_overwrite=False, output_dir=None, include_config=True, 34 | parallel_build=True): 35 | if not version: 36 | raise Exception("'version' is a required settings") 37 | self.build_type = build_type 38 | self._package_name_format = package_name_format 39 | self.version = version 40 | self._base_path = base_path 41 | self._install_path = install_path 42 | self.gerrit_dependencies = gerrit_dependencies 43 | self.force_overwrite = force_overwrite 44 | self._output_dir = output_dir 45 | self.include_config = include_config 46 | self.parallel_build = parallel_build 47 | 48 | @property 49 | def package_name_format(self): 50 | return self._get_setting('package_name_format') 51 | 52 | @property 53 | def base_path(self): 54 | return self._get_setting('base_path') 55 | 56 | @property 57 | def install_path(self): 58 | return self._get_setting('install_path') 59 | 60 | @property 61 | def output_dir(self): 62 | if not self._output_dir: 63 | self._output_dir = os.getcwd() 64 | return self._output_dir 65 | 66 | def _get_setting(self, setting_name): 67 | setting = object.__getattribute__(self, '_%s' % setting_name) 68 | if setting is None: 69 | setting = Settings.DEFAULTS[setting_name] 70 | return setting 71 | 72 | @staticmethod 73 | def factory(settings_dict): 74 | return Settings(**settings_dict) 75 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # encoding: UTF-8 2 | 3 | GIFTWRAP_MANIFEST = ENV['GIFTWRAP_MANIFEST'] || 'examples/manifest.yml' 4 | GIFTWRAP_ARGS = ENV['GIFTWRAP_ARGS'] || '-t package' 5 | GIFTWRAP_BUILDBOX_NAME = ENV['GIFTWRAP_BUILDBOX_NAME'] || 'ursula-precise' 6 | GIFTWRAP_BUILDBOX_URL = ENV['GIFTWRAP_BUILDBOX_URL'] || 'http://apt.openstack.blueboxgrid.com/vagrant/ursula-precise.box' 7 | 8 | # CentOS7 example 9 | #GIFTWRAP_BUILDBOX_NAME = ENV['GIFTWRAP_BUILDBOX_NAME'] || 'centos7' 10 | #GIFTWRAP_BUILDBOX_URL = ENV['GIFTWRAP_BUILDBOX_URL'] || 'https://f0fff3908f081cb6461b407be80daf97f07ac418.googledrive.com/host/0BwtuV7VyVTSkUG1PM3pCeDJ4dVE/centos7.box' 11 | 12 | GIFTWRAP_POSTBUILD_SCRIPT = ENV['GIFTWRAP_POSTBUILD_SCRIPT'] || "" 13 | 14 | GET_PIP_MD5 = ENV['GIFTWRAP_GET_PIP_MD5'] || 'add41078298d8111714c6b87636714f5' 15 | 16 | ENV['VAGRANT_DEFAULT_PROVIDER'] = 'virtualbox' 17 | 18 | Vagrant.configure('2') do |config| 19 | config.vm.box = GIFTWRAP_BUILDBOX_NAME 20 | config.vm.box_url = GIFTWRAP_BUILDBOX_URL 21 | 22 | config.vm.provider :openstack do |os, override| 23 | os.openstack_auth_url = "#{ENV['OS_AUTH_URL']}/tokens" 24 | os.username = ENV['OS_USERNAME'] 25 | os.password = ENV['OS_PASSWORD'] 26 | os.tenant_name = ENV['OS_TENANT_NAME'] 27 | os.openstack_network_url = ENV['OS_NEUTRON_URL'] 28 | 29 | os.flavor = ENV['GIFTWRAP_OS_FLAVOR'] || 'm1.small' 30 | os.image = ENV['GIFTWRAP_OS_IMAGE'] || 'ubuntu-12.04' 31 | 32 | if ENV['GIFTWRAP_OS_NETWORKS'] 33 | os.networks = ENV['GIFTWRAP_OS_NETWORKS'].split(",") 34 | else 35 | os.networks = ['internal'] 36 | end 37 | 38 | override.ssh.username = ENV['GIFTWRAP_OS_USERNAME'] || 'ubuntu' 39 | if ENV['GIFTWRAP_OS_FLOATING_IP_POOL'] 40 | os.floating_ip_pool = ENV['GIFTWRAP_OS_FLOATING_IP_POOL'] 41 | end 42 | if ENV['GIFTWRAP_OS_SECURITY_GROUPS'] 43 | os.security_groups = ENV['GIFTWRAP_SECURITY_GROUPS'].split(",") 44 | end 45 | os.rsync_exclude_paths = [] 46 | os.rsync_cvs_exclude = false 47 | end 48 | 49 | config.vm.provision 'shell', inline: <<-EOF 50 | #!/bin/bash 51 | set -x 52 | set -e 53 | 54 | if [ -f /etc/lsb-release ]; then 55 | . /etc/lsb-release 56 | OS=$DISTRIB_ID 57 | elif [ -f /etc/debian_version ]; then 58 | OS=Debian 59 | elif [ -f /etc/redhat-release ]; then 60 | OS=RedHat 61 | fi 62 | 63 | if [ "$OS" == "Debian" ] || [ "$OS" == "Ubuntu" ]; then 64 | /vagrant/scripts/prepare_debian.sh 65 | elif [ "$OS" == "RedHat" ]; then 66 | /vagrant/scripts/prepare_redhat.sh 67 | fi 68 | 69 | gem install --no-ri --no-rdoc fpm 70 | cd /vagrant 71 | 72 | wget -q -O /tmp/get-pip.py https://bootstrap.pypa.io/get-pip.py 73 | if ! md5sum /tmp/get-pip.py | grep -q $GET_PIP_MD5; then 74 | echo "pip installation could not be verified. Quitting" 75 | exit -1 76 | fi 77 | python /tmp/get-pip.py 78 | pip install -U setuptools 79 | 80 | export PATH=/usr/local/bin/:$PATH 81 | pip install . 82 | giftwrap build -m #{GIFTWRAP_MANIFEST} #{GIFTWRAP_ARGS} 83 | 84 | if [ ! -z "#{GIFTWRAP_POSTBUILD_SCRIPT}" ]; then 85 | echo "Running postbuild script: '#{GIFTWRAP_POSTBUILD_SCRIPT}'" 86 | #{GIFTWRAP_POSTBUILD_SCRIPT} 87 | fi 88 | 89 | EOF 90 | 91 | config.vm.define 'giftwrap' do |c| 92 | c.vm.host_name = 'giftwrap' 93 | end 94 | end 95 | -------------------------------------------------------------------------------- /giftwrap/shell.py: -------------------------------------------------------------------------------- 1 | # vim: tabstop=4 shiftwidth=4 softtabstop=4 2 | 3 | # Copyright 2014, Craig Tracey 4 | # All Rights Reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 7 | # not use this file except in compliance with the License. You may obtain 8 | # a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 15 | # License for the specific language governing permissions and limitations 16 | 17 | import argparse 18 | import logging 19 | import signal 20 | import sys 21 | 22 | from giftwrap.builders import Builder, BuilderFactory 23 | from giftwrap.build_spec import BuildSpec 24 | from giftwrap.color import ColorStreamHandler 25 | 26 | LOG = logging.getLogger(__name__) 27 | 28 | 29 | def _setup_logger(level=logging.INFO): 30 | logger = logging.getLogger() 31 | logger.setLevel(level) 32 | log_handler = ColorStreamHandler(sys.stdout) 33 | fmt = logging.Formatter(fmt='%(asctime)s %(threadName)s %(name)s ' 34 | '%(levelname)s: %(message)s', 35 | datefmt='%F %H:%M:%S') 36 | log_handler.setFormatter(fmt) 37 | logger.addHandler(log_handler) 38 | 39 | 40 | def build(args): 41 | """ the entry point for all build subcommand tasks """ 42 | builder = None 43 | fail = False 44 | try: 45 | manifest = None 46 | 47 | with open(args.manifest, 'r') as fh: 48 | manifest = fh.read() 49 | 50 | buildspec = BuildSpec(manifest, args.version, args.type, args.parallel, 51 | args.projects) 52 | builder = BuilderFactory.create_builder(args.type, buildspec) 53 | 54 | def _signal_handler(*args): 55 | LOG.info("Process interrrupted. Cleaning up.") 56 | builder.cleanup() 57 | sys.exit() 58 | signal.signal(signal.SIGINT, _signal_handler) 59 | 60 | rc = builder.build() 61 | except Exception as e: 62 | LOG.exception("Oops something went wrong: %s", e) 63 | fail = True 64 | 65 | builder.cleanup() 66 | if fail: 67 | sys.exit(-1) 68 | sys.exit(rc) 69 | 70 | 71 | def main(): 72 | """ the entry point for all things giftwrap """ 73 | parser = argparse.ArgumentParser() 74 | parser.add_argument('-d', '--debug', action='store_true', 75 | help='Set logging level to DEBUG') 76 | 77 | subparsers = parser.add_subparsers(title='subcommands', 78 | description='valid subcommands', 79 | help='additional help') 80 | build_subcmd = subparsers.add_parser('build', 81 | description='build giftwrap packages') 82 | build_subcmd.add_argument('-m', '--manifest', required=True) 83 | build_subcmd.add_argument('-v', '--version') 84 | build_subcmd.add_argument('-t', '--type', choices=Builder.builder_names(), 85 | required=True) 86 | build_subcmd.add_argument('-s', '--synchronous', dest='parallel', 87 | action='store_false') 88 | 89 | def csvarg(arg): 90 | if arg is not None: 91 | return arg.split(',') 92 | 93 | build_subcmd.add_argument('-p', '--projects', type=csvarg, dest='projects') 94 | build_subcmd.set_defaults(func=build) 95 | 96 | args = parser.parse_args() 97 | 98 | log_level = logging.INFO 99 | if args.debug: 100 | log_level = logging.DEBUG 101 | _setup_logger(log_level) 102 | 103 | args.func(args) 104 | 105 | 106 | if __name__ == '__main__': 107 | main() 108 | -------------------------------------------------------------------------------- /giftwrap/builders/package_builder.py: -------------------------------------------------------------------------------- 1 | # vim: tabstop=4 shiftwidth=4 softtabstop=4 2 | 3 | # Copyright 2014, Craig Tracey 4 | # All Rights Reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 7 | # not use this file except in compliance with the License. You may obtain 8 | # a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 15 | # License for the specific language governing permissions and limitations 16 | 17 | import distutils.dir_util 18 | import logging 19 | import os 20 | import shutil 21 | import tempfile 22 | 23 | from giftwrap.builders import Builder 24 | from giftwrap.openstack_git_repo import OpenstackGitRepo 25 | from giftwrap.package import Package 26 | from giftwrap.util import execute 27 | 28 | LOG = logging.getLogger(__name__) 29 | 30 | 31 | class PackageBuilder(Builder): 32 | 33 | def __init__(self, spec): 34 | self._temp_dir = None 35 | super(PackageBuilder, self).__init__(spec) 36 | 37 | def _execute(self, command, cwd=None, exit=0): 38 | return execute(command, cwd, exit) 39 | 40 | def _make_temp_dir(self, prefix='giftwrap'): 41 | return tempfile.mkdtemp(prefix) 42 | 43 | def _make_dir(self, path, mode=0777): 44 | os.makedirs(path, mode) 45 | 46 | def _prepare_build(self): 47 | return 48 | 49 | def _prepare_project_build(self, project): 50 | install_path = project.install_path 51 | 52 | LOG.info("Beginning to build '%s'", project.name) 53 | if os.path.exists(install_path): 54 | if self._spec.settings.force_overwrite: 55 | LOG.info("force_overwrite is set, so removing " 56 | "existing path '%s'" % install_path) 57 | shutil.rmtree(install_path) 58 | else: 59 | raise Exception("Install path '%s' already exists" % 60 | install_path) 61 | 62 | def _clone_project(self, giturl, name, gitref, depth, path): 63 | LOG.info("Fetching source code for '%s'", name) 64 | repo = OpenstackGitRepo(giturl, name, gitref, depth=depth) 65 | repo.clone(path) 66 | return repo 67 | 68 | def _create_virtualenv(self, venv_command, path): 69 | self._execute(venv_command, path) 70 | 71 | def _install_pip_dependencies(self, venv_path, dependencies): 72 | pip_path = self._get_venv_pip_path(venv_path) 73 | for dependency in dependencies: 74 | self._execute("%s install %s" % (pip_path, dependency)) 75 | 76 | def _copy_sample_config(self, src_clone_dir, project): 77 | src_config = os.path.join(src_clone_dir, 'etc') 78 | dest_config = os.path.join(project.install_path, 'etc') 79 | 80 | if not os.path.exists(src_config): 81 | LOG.warning("Project configuration does not seem to exist " 82 | "in source repo '%s'. Skipping.", project.name) 83 | else: 84 | LOG.debug("Copying config from '%s' to '%s'", src_config, 85 | dest_config) 86 | distutils.dir_util.copy_tree(src_config, dest_config) 87 | 88 | def _install_project(self, venv_path, src_clone_dir): 89 | pip_path = self._get_venv_pip_path(venv_path) 90 | self._execute("%s install %s" % (pip_path, src_clone_dir)) 91 | 92 | def _finalize_project_build(self, project): 93 | # build the package 94 | pkg = Package(project.package_name, project.version, 95 | project.install_path, self._spec.settings.output_dir, 96 | self._spec.settings.force_overwrite, 97 | project.system_dependencies) 98 | pkg.build() 99 | 100 | def _finalize_build(self): 101 | return 102 | 103 | def _cleanup_build(self): 104 | shutil.rmtree(self._temp_dir) 105 | -------------------------------------------------------------------------------- /giftwrap/openstack_git_repo.py: -------------------------------------------------------------------------------- 1 | # vim: tabstop=4 shiftwidth=4 softtabstop=4 2 | 3 | # Copyright 2014, Craig Tracey 4 | # All Rights Reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 7 | # not use this file except in compliance with the License. You may obtain 8 | # a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 15 | # License for the specific language governing permissions and limitations 16 | 17 | import datetime 18 | import logging 19 | import os 20 | import re 21 | import time 22 | import urlparse 23 | 24 | from giftwrap.openstack_commit import OpenstackCommit 25 | from git import Repo 26 | 27 | LOG = logging.getLogger(__name__) 28 | 29 | 30 | class OpenstackGitRepo(object): 31 | 32 | def __init__(self, url, project=None, branch='master', 33 | metadata_cache_dir=None, depth=None): 34 | self.url = url 35 | self._project = project 36 | self.branch = branch 37 | self._repo = None 38 | self._metadata_cache_dir = metadata_cache_dir 39 | self._head_commit = None 40 | self._depth = depth 41 | 42 | @property 43 | def cloned(self): 44 | return self._repo is not None 45 | 46 | @property 47 | def head(self): 48 | if not self._head_commit and self._repo: 49 | self._head_commit = OpenstackCommit(self._repo.head.commit, 50 | self.project, self.branch, 51 | self._cache_dir()) 52 | return self._head_commit 53 | 54 | @property 55 | def project(self): 56 | if not self._project: 57 | parsed_url = urlparse.urlparse(self.url) 58 | project = os.path.splitext(parsed_url.path)[0] 59 | self._project = re.sub(r'^/', '', project) 60 | return self._project 61 | 62 | def clone(self, outdir): 63 | LOG.debug("Cloning '%s' to '%s'", self.url, outdir) 64 | kwargs = {'recursive': True} 65 | if self._depth: 66 | LOG.debug("Cloning with depth=%d", self._depth) 67 | kwargs['depth'] = self._depth 68 | self._repo = Repo.clone_from(self.url, outdir, **kwargs) 69 | git = self._repo.git 70 | git.checkout(self.branch) 71 | self._invalidate_attrs() 72 | 73 | def checkout_branch(self, branch, update=True): 74 | if not self._repo: 75 | raise Exception("Cannot checkout on non-existent repo") 76 | LOG.debug("Checking out branch: %s (update: %s)", branch, update) 77 | self._repo.git.checkout(branch) 78 | self._invalidate_attrs() 79 | self.branch = branch 80 | if update: 81 | self._repo.git.pull('origin', branch) 82 | 83 | @property 84 | def branches(self): 85 | branches = [] 86 | for ref in self._repo.remotes.origin.refs: 87 | branches.append(re.sub('^\w*/', '', ref.name)) 88 | return branches 89 | 90 | def __iter__(self): 91 | if not self._repo: 92 | raise Exception("iterator called before clone") 93 | self._commit_iterator = self._repo.iter_commits() 94 | return self 95 | 96 | def next(self): 97 | return OpenstackCommit(next(self._commit_iterator), 98 | self.project, self.branch, 99 | self._cache_dir()) 100 | 101 | def _cache_dir(self): 102 | if self._metadata_cache_dir: 103 | return os.path.join(self._metadata_cache_dir, 104 | self.project, self.branch) 105 | return None 106 | 107 | def _invalidate_attrs(self): 108 | self._head_commit = None 109 | self._commit_iterator = None 110 | 111 | def reset_to_date(self, date): 112 | if self._repo: 113 | commit_date_sha = None 114 | for commit in self._repo.iter_commits(): 115 | if commit.committed_date >= date: 116 | continue 117 | elif commit.committed_date < date: 118 | commit_date_sha = commit.hexsha 119 | break 120 | if not commit_date_sha: 121 | raise Exception("Unable to find commit for date %s", 122 | datetime.datetime.fromtimestamp(date)) 123 | git = self._repo.git 124 | LOG.debug("Reset repo '%s' to commit at '%s'", self.url, 125 | time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(date))) 126 | git.checkout(commit_date_sha) 127 | -------------------------------------------------------------------------------- /giftwrap/openstack_project.py: -------------------------------------------------------------------------------- 1 | # vim: tabstop=4 shiftwidth=4 softtabstop=4 2 | 3 | # Copyright 2014, Craig Tracey 4 | # All Rights Reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 7 | # not use this file except in compliance with the License. You may obtain 8 | # a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 15 | # License for the specific language governing permissions and limitations 16 | 17 | import urlparse 18 | 19 | from jinja2 import Environment 20 | 21 | DEFAULT_GITREF = 'master' 22 | DEFAULT_GITURL = { 23 | 'openstack': 'https://git.openstack.org/openstack/', 24 | 'stackforge': 'https://github.com/stackforge/' 25 | } 26 | DEFAULT_VENV_COMMAND = "virtualenv --no-wheel ." 27 | DEFAULT_INSTALL_COMMAND = "./bin/pip install '%s'" # noqa 28 | 29 | TEMPLATE_VARS = ('name', 'version', 'gitref', 'stackforge') 30 | 31 | 32 | class OpenstackProject(object): 33 | 34 | def __init__(self, settings, name, version=None, gitref=None, giturl=None, 35 | gitdepth=None, venv_command=None, install_command=None, 36 | install_path=None, package_name=None, stackforge=False, 37 | system_dependencies=[], pip_dependencies=[], 38 | postinstall_dependencies=[]): 39 | self._settings = settings 40 | self.name = name 41 | self._version = version 42 | self._gitref = gitref 43 | self._giturl = giturl 44 | self.gitdepth = gitdepth 45 | self._venv_command = venv_command 46 | self._install_command = install_command 47 | self._install_path = install_path 48 | self._package_name = package_name 49 | self.stackforge = stackforge 50 | self._git_path = None 51 | self.system_dependencies = system_dependencies 52 | self.pip_dependencies = pip_dependencies 53 | self.postinstall_dependencies = postinstall_dependencies 54 | 55 | @property 56 | def version(self): 57 | if not self._version: 58 | self._version = self._settings.version 59 | return self._version 60 | 61 | @property 62 | def gitref(self): 63 | if not self._gitref: 64 | self._gitref = DEFAULT_GITREF 65 | return self._gitref 66 | 67 | @property 68 | def giturl(self): 69 | if not self._giturl: 70 | key = 'openstack' 71 | if self.stackforge: 72 | key = 'stackforge' 73 | self._giturl = urlparse.urljoin(DEFAULT_GITURL[key], self.name) 74 | return self._giturl 75 | 76 | @property 77 | def venv_command(self): 78 | if not self._venv_command: 79 | self._venv_command = DEFAULT_VENV_COMMAND 80 | return self._venv_command 81 | 82 | @property 83 | def package_name(self): 84 | if not self._package_name: 85 | self._package_name = \ 86 | self._render_from_settings('package_name_format') 87 | return self._package_name 88 | 89 | def _template_vars(self): 90 | template_vars = {'project': self} 91 | template_vars['settings'] = self._settings 92 | for var in TEMPLATE_VARS: 93 | template_vars[var] = object.__getattribute__(self, var) 94 | return template_vars 95 | 96 | @property 97 | def install_path(self): 98 | if not self._install_path: 99 | self._install_path = self._render_from_settings('install_path') 100 | return self._install_path 101 | 102 | @property 103 | def install_command(self): 104 | if not self._install_command: 105 | self._install_command = DEFAULT_INSTALL_COMMAND 106 | return self._install_command 107 | 108 | @property 109 | def git_path(self): 110 | if not self._git_path: 111 | gitorg = 'openstack' 112 | if self.stackforge: 113 | gitorg = 'stackforge' 114 | self._git_path = '%s/%s' % (gitorg, self.name) 115 | return self._git_path 116 | 117 | def _render_from_settings(self, setting_name): 118 | setting = getattr(self._settings, setting_name) 119 | env = Environment() 120 | env.add_extension('jinja2.ext.autoescape') 121 | result = setting 122 | while True: 123 | t = env.from_string(result) 124 | newresult = t.render(self._template_vars()) 125 | if newresult == result: 126 | break 127 | result = newresult 128 | return result 129 | 130 | @staticmethod 131 | def factory(settings, project_dict, version): 132 | return OpenstackProject(settings, version=version, **project_dict) 133 | -------------------------------------------------------------------------------- /giftwrap/openstack_commit.py: -------------------------------------------------------------------------------- 1 | # vim: tabstop=4 shiftwidth=4 softtabstop=4 2 | 3 | # Copyright 2014, Craig Tracey 4 | # All Rights Reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 7 | # not use this file except in compliance with the License. You may obtain 8 | # a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 15 | # License for the specific language governing permissions and limitations 16 | 17 | import logging 18 | import os 19 | import re 20 | import yaml 21 | 22 | from giftwrap.gerrit import GerritReview 23 | 24 | LOG = logging.getLogger(__name__) 25 | 26 | 27 | class OpenstackCommit(object): 28 | 29 | def __init__(self, commit, project, branch, meta_cache_dir=None): 30 | self.commit = commit 31 | self.project = project 32 | self.branch = branch 33 | self._change_id = None 34 | self._editable_dependencies = None 35 | self._pip_dependencies = None 36 | self._is_merge = None 37 | self._parent = None 38 | self._gerrit_review = None 39 | self._meta_cache_dir = meta_cache_dir 40 | 41 | @property 42 | def hexsha(self): 43 | return self.commit.hexsha 44 | 45 | @property 46 | def change_id(self): 47 | if not self._change_id: 48 | self._change_id = str(self._get_change_id()) 49 | return self._change_id 50 | 51 | @property 52 | def is_merge(self): 53 | if self._is_merge is None: 54 | self._is_merge = (len(self.commit.parents) == 2) 55 | return self._is_merge 56 | 57 | @property 58 | def parent(self): 59 | if self.is_merge: 60 | self._parent = OpenstackCommit(self.commit.parents[1], 61 | self.project, self.branch) 62 | return self._parent 63 | 64 | @property 65 | def gerrit_review(self): 66 | if not self._gerrit_review: 67 | self._gerrit_review = GerritReview(self.change_id, 68 | self.project, self.branch) 69 | return self._gerrit_review 70 | 71 | def _gather_dependencies(self): 72 | try: 73 | deps = self.gerrit_review.build_pip_dependencies() 74 | self._editable_dependencies = [] 75 | self._pip_dependencies = {} 76 | for dep in deps: 77 | if '-e' in dep: 78 | self._editable_dependencies.append(dep) 79 | else: 80 | parts = dep.split('==') 81 | self._pip_dependencies[parts[0]] = parts[1] 82 | except Exception as e: 83 | LOG.debug("Couldn't find dependencies for %s: %s", self.hexsha, e) 84 | 85 | @property 86 | def pip_dependencies(self): 87 | if not self._pip_dependencies: 88 | self._gather_dependencies() 89 | return self._pip_dependencies 90 | 91 | @property 92 | def editable_dependencies(self): 93 | if not self._editable_dependencies: 94 | self._gather_dependencies() 95 | return self._editable_dependencies 96 | 97 | def _get_change_id(self): 98 | commit = self.commit 99 | if self.is_merge: 100 | commit = self.parent.commit 101 | match = re.search('Change-Id:\s*(I\w+)', commit.message) 102 | if match: 103 | return match.group(1) 104 | 105 | def is_cached(self): 106 | return os.path.isfile(self.cache_file) 107 | 108 | @property 109 | def cache_file(self): 110 | return os.path.join(self._meta_cache_dir, self.hexsha) 111 | 112 | def _get_from_cache(self, key): 113 | if self.is_cached(): 114 | with open(self.cache_file, 'r') as fh: 115 | cached_data = yaml.load(fh) 116 | if key in cached_data: 117 | return cached_data[key] 118 | return None 119 | 120 | def is_cacheable(self): 121 | if self.pip_dependencies or self.editable_dependencies: 122 | return True 123 | return False 124 | 125 | def __dict__(self): 126 | data = {} 127 | data['pip_dependencies'] = self.pip_dependencies 128 | data['editable_dependencies'] = self.editable_dependencies 129 | data['change_id'] = self.change_id 130 | return data 131 | 132 | def persist_to_cache(self): 133 | if not self.is_cacheable(): 134 | LOG.debug("Not caching %s as there is no point", self.hexsha) 135 | return 136 | dirname = os.path.dirname(self.cache_file) 137 | if not os.path.exists(dirname): 138 | os.makedirs(dirname) 139 | with open(self.cache_file, 'w') as fh: 140 | fh.write(yaml.dump(self.__dict__())) 141 | -------------------------------------------------------------------------------- /giftwrap/gerrit.py: -------------------------------------------------------------------------------- 1 | # vim: tabstop=4 shiftwidth=4 softtabstop=4 2 | 3 | # Copyright 2014, Craig Tracey 4 | # All Rights Reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 7 | # not use this file except in compliance with the License. You may obtain 8 | # a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 15 | # License for the specific language governing permissions and limitations 16 | 17 | import re 18 | 19 | import requests 20 | 21 | from pygerrit.rest import GerritRestAPI 22 | 23 | DEFAULT_GERRIT_URL = 'https://review.openstack.org' 24 | 25 | 26 | class GerritReview(object): 27 | 28 | def __init__(self, changeid, project, gerrit_url=DEFAULT_GERRIT_URL): 29 | self.changeid = changeid 30 | self.project = project 31 | self._gerrit_url = gerrit_url 32 | self._restclient = None 33 | 34 | def build_pip_dependencies(self, py26=False, py27=True, string=False): 35 | url = self._get_gate_build_log_url(py26, py27) 36 | response = requests.get(url) 37 | 38 | if response.status_code != 200: 39 | raise Exception("Unable to get console log at %s. Error: %d" % 40 | (url, response.status_code)) 41 | 42 | log = response.text.encode('utf-8') 43 | 44 | freeze_found = False 45 | dependencies = [] 46 | for line in log.split('\n'): 47 | line = re.sub('.*\|\s*', '', line) 48 | if not freeze_found: 49 | if line.endswith("pip freeze") or line.endswith("pbr freeze"): 50 | freeze_found = True 51 | continue 52 | elif re.match('[\w\-]+==.+', line) and not line.startswith('-e'): 53 | dependency = line.split('#')[0].strip() # remove any comments 54 | dependencies.append(dependency) 55 | 56 | short_name = self.project.split('/')[1] 57 | dependencies = filter(lambda x: not x.startswith(short_name + "=="), 58 | dependencies) 59 | 60 | if string: 61 | return (' ').join(dependencies) 62 | return dependencies 63 | 64 | def _get_rest_client(self): 65 | if not self._restclient: 66 | self._restclient = GerritRestAPI(url=self._gerrit_url) 67 | return self._restclient 68 | 69 | def _get_review_detail(self): 70 | """ get review details for a given change ID """ 71 | restclient = self._get_rest_client() 72 | url = "/changes/?q=%s" % self.changeid 73 | changes = restclient.get(url) 74 | 75 | change = None 76 | for c in changes: 77 | if c['project'] == self.project: 78 | change = c 79 | break 80 | 81 | if not change: 82 | raise Exception("could not find change with ID: %s" % 83 | self.changeid) 84 | 85 | detail = restclient.get("/changes/%s/detail" % change['id']) 86 | return detail 87 | 88 | def _get_reveiew_messages(self): 89 | details = self._get_review_detail() 90 | return details['messages'] 91 | 92 | def _get_gate_build_log_url(self, py26, py27): 93 | messages = self._get_reveiew_messages() 94 | messages.reverse() 95 | 96 | mergemsg = None 97 | for message in messages: 98 | msgtext = message['message'] 99 | if re.search('Patch Set \d+: Verified', msgtext): 100 | mergemsg = msgtext 101 | break 102 | 103 | gate_info = self._parse_merge_message(mergemsg) 104 | url = None 105 | for gate in gate_info: 106 | if py26 and re.match('gate\-.+\-python26', gate['name']): 107 | url = gate['url'] 108 | if py27 and re.match('gate\-.+\-python27', gate['name']): 109 | url = gate['url'] 110 | 111 | # check if it is console.html or console.html.gz 112 | resp = requests.get(url) 113 | if resp.status_code != 200: 114 | raise Exception("Unable to find the build's console log for %s" % 115 | url) 116 | 117 | build_log = None 118 | if 'console.html.gz' in resp.text: 119 | build_log = 'console.html.gz' 120 | elif 'console.html' in resp.text: 121 | build_log = 'console.html' 122 | else: 123 | raise Exception("Didn't find a build log. Does one exist?") 124 | 125 | if url: 126 | return "%s/%s" % (url, build_log) 127 | return url 128 | 129 | def _parse_merge_message(self, msg): 130 | """ a function that parses a successful gate gerrit message """ 131 | gate_info = [] 132 | for line in msg.split('\n'): 133 | parts = re.split('\s+', line) 134 | if parts[0] == '-': 135 | gate = {} 136 | gate['name'] = parts[1] 137 | gate['url'] = parts[2] 138 | gate['result'] = parts[4] 139 | gate_info.append(gate) 140 | return gate_info 141 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
Giftwrap
2 | 3 | A tool for creating bespoke system-native OpenStack artifacts. 4 | 5 | Anyone running OpenStack at scale typically crafts their own software distribution mechanism. There may be many reasons for this, but chief among them seem to be the desire to ship security patches, deliver custom code, lock their releases at a revision of their choosing, or just generally stay closer to trunk. 6 | 7 | Unfortunately, until now, there has been no easy way to accomplish this. If one were to decide to utilize distribution packages they are now at the mercy of the distribution itself - who's release timeline may not be the same as yours. 8 | 9 | On the other hand, if one were to install directly from source, they will encounter a slightly different problem. Because the Python community is quite active, and because OpenStack, in many regards, does not strictly call out it's dependencies, one may build OpenStack with incompatible (or at least unknown) dependencies. Even worse, one may even find that OpenStack components may be different on different nodes in the cluster. 10 | 11 | Long story short, this sucks. 12 | 13 | Inspired by some of the work I had done to create [omnibus-openstack](https://github.com/craigtracey/omnibus-openstack), I decided to do things slightly differently. While omnibus-openstack met most of my needs there were a few problems. First, the project was written in Ruby. While this, in my opinion, is not a problem, this makes it somewhat unapproachable to a vast segment of OpenStack users and operators. Second, the packages are HUGE. Again, while this may not be a real problem for many, the reason they are huge is that they manage all of the system level dependencies as well: things like openssl, libvirt, etc. These are not things that many folks typically want to be responsible for managing; whether for security or even complexity reasons. 14 | 15 | With all of this in mind, it seemed to me that we already had all of the information that we already needed to create system-native (ie. rpm, deb, and even Docker) artifacts that had already been tested with the Gerrit CI infrastructure. Hence, giftwrap. 16 | 17 | Status 18 | ------ 19 | [![Build Status](https://api.travis-ci.org/blueboxgroup/giftwrap.png)](https://travis-ci.org/blueboxgroup/giftwrap) 20 | 21 | Usage 22 | ===== 23 | 24 | $ pip install . 25 | $ python setup.py install 26 | $ giftwrap -h 27 | 28 | Dependencies 29 | ------------ 30 | 31 | * `Vagrant` 32 | * `fpm` 33 | * `docker` (optional) 34 | 35 | Development 36 | ----------- 37 | 38 | $ git clone https://github.com/blueboxgroup/giftwrap.git 39 | $ vagrant up 40 | 41 | Testing 42 | ------- 43 | 44 | $ make test 45 | 46 | Supports 47 | -------- 48 | * Jinja2 templating - change your build by changing variables; not your manifest 49 | * versioned paths - this allows you to run services side by side; easing the upgrade process. 50 | 51 | How It Works 52 | ------------ 53 | giftwrap is pretty simple. The basic flow is something like this: 54 | 1. Create a YAML manifest with the packages you would like to build. See sample.yml 55 | 2. Run: 56 | ``` 57 | giftwrap build -m [-v ] 58 | ``` 59 | 3. giftwrap will clone the git repo and git ref that you specify for each of the OpenStack projects 60 | 4. giftwrap will find the closest Gerrit Change-Id and retrieve it's build logs 61 | 5. From the build logs, giftwrap will find the pip dependencies used for that build and record them. 62 | 6. A new virtualenv will be built with the pip dependencies found 63 | 7. The OpenStack project code will be installed into the same virtualenv; but with locked pip dependencies 64 | 8. giftwrap will check [devstack](https://devstack.org) for the system dependencies necessary for that project (to be done) 65 | 9. An [fpm](https://github.com/jordansissel/fpm) package will be built from the intersection of the python install and system dependencies 66 | 67 | DockerDockerDockerDockerDocker 68 | ------------------------------ 69 | 70 | This shows how to use docker-in-docker to build packages really fast. the example shows doing it within the provided vagrant instance. but this is for illustration purposes, it should be runnable from any machine with docker installed. 71 | 72 | 73 | Build a giftwrap docker container like this: 74 | 75 | ``` 76 | $ docker build -t bluebox/giftwrap . 77 | ``` 78 | 79 | Run the giftwrap docker image and map in your docker socket and your manifest file: 80 | 81 | ``` 82 | $ docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -v /vagrant/sample.yml:/tmp/manifest.yml bluebox/giftwrap 83 | $ docker images openstack-9.0 84 | REPOSITORY TAG IMAGE ID CREATED VIRTUAL SIZE 85 | openstack-9.0 bbc6 44c37c8d9672 10 minutes ago 499.5 MB 86 | 87 | ``` 88 | 89 | TODO 90 | ---- 91 | * Provide option for source removal; package only the executables 92 | * Allow for additional pip dependencies, alternate pip dependency versions, and even user-defined pip dependencies 93 | * Allow for additional/alternate system package dependencies 94 | 95 | License 96 | ------- 97 | | | | 98 | |:---------------------|:---------------------------------------------------| 99 | | **Authors** | John Dewey () | 100 | | | Craig Tracey () | 101 | | | Paul Czarkowski () | 102 | | | | 103 | | **Copyright** | Copyright (c) 2014, John Dewey | 104 | | | Copyright (c) 2014, Craig Tracey | 105 | | | Copyright (c) 2014, Paul Czarkowski | 106 | 107 | Licensed under the Apache License, Version 2.0 (the "License"); 108 | you may not use this file except in compliance with the License. 109 | You may obtain a copy of the License at 110 | 111 | http://www.apache.org/licenses/LICENSE-2.0 112 | 113 | Unless required by applicable law or agreed to in writing, software 114 | distributed under the License is distributed on an "AS IS" BASIS, 115 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 116 | See the License for the specific language governing permissions and 117 | limitations under the License. 118 | -------------------------------------------------------------------------------- /giftwrap/builders/__init__.py: -------------------------------------------------------------------------------- 1 | # vim: tabstop=4 shiftwidth=4 softtabstop=4 2 | 3 | # Copyright 2014, Craig Tracey 4 | # All Rights Reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 7 | # not use this file except in compliance with the License. You may obtain 8 | # a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 15 | # License for the specific language governing permissions and limitations 16 | 17 | import logging 18 | import os 19 | import threading 20 | 21 | from giftwrap.gerrit import GerritReview 22 | from stevedore.driver import DriverManager 23 | from stevedore.extension import ExtensionManager 24 | 25 | from abc import abstractmethod, ABCMeta 26 | 27 | BUILDER_DRIVER_NAMESPACE = 'giftwrap.builder.drivers' 28 | LOG = logging.getLogger(__name__) 29 | 30 | 31 | class Builder(object): 32 | __metaclass__ = ABCMeta 33 | 34 | def __init__(self, spec): 35 | self._temp_dir = None 36 | self._temp_src_dir = None 37 | self._spec = spec 38 | self._thread_exit = [] 39 | 40 | @staticmethod 41 | def builder_names(ext_mgr=None): 42 | if not ext_mgr: 43 | ext_mgr = ExtensionManager(BUILDER_DRIVER_NAMESPACE) 44 | return ext_mgr.names() 45 | 46 | def _get_venv_pip_path(self, venv_path): 47 | return os.path.join(venv_path, 'bin/pip') 48 | 49 | def _get_gerrit_dependencies(self, repo, project): 50 | try: 51 | review = GerritReview(repo.head.change_id, project.git_path) 52 | return review.build_pip_dependencies() 53 | except Exception as e: 54 | LOG.warning("Could not install gerrit dependencies!!! " 55 | "Error was: %s", e) 56 | return [] 57 | 58 | def _build_project(self, project): 59 | try: 60 | self._prepare_project_build(project) 61 | self._make_dir(project.install_path) 62 | 63 | # clone the source 64 | src_clone_dir = os.path.join(self._temp_src_dir, project.name) 65 | repo = self._clone_project(project.giturl, project.name, 66 | project.gitref, project.gitdepth, 67 | src_clone_dir) 68 | 69 | # create and build the virtualenv 70 | self._create_virtualenv(project.venv_command, project.install_path) 71 | dependencies = [] 72 | if project.pip_dependencies: 73 | dependencies = project.pip_dependencies 74 | if self._spec.settings.gerrit_dependencies: 75 | dependencies += self._get_gerrit_dependencies(repo, project) 76 | 77 | if len(dependencies): 78 | self._install_pip_dependencies(project.install_path, 79 | dependencies) 80 | 81 | if self._spec.settings.include_config: 82 | self._copy_sample_config(src_clone_dir, project) 83 | 84 | self._install_project(project.install_path, src_clone_dir) 85 | 86 | if project.postinstall_dependencies: 87 | dependencies = project.postinstall_dependencies 88 | self._install_pip_dependencies(project.install_path, 89 | dependencies) 90 | 91 | # finish up 92 | self._finalize_project_build(project) 93 | except Exception as e: 94 | LOG.error("Oops. Problem building %s: %s", project.name, e) 95 | self._thread_exit.append(-1) 96 | self._thread_exit.append(0) 97 | 98 | def build(self): 99 | spec = self._spec 100 | 101 | self._prepare_build() 102 | 103 | # Create a temporary directory for the source code 104 | self._temp_dir = self._make_temp_dir() 105 | self._temp_src_dir = os.path.join(self._temp_dir, 'src') 106 | LOG.debug("Temporary working directory: %s", self._temp_dir) 107 | 108 | threads = [] 109 | for project in spec.projects: 110 | if spec.settings.parallel_build: 111 | t = threading.Thread(target=self._build_project, 112 | name=project.name, args=(project,)) 113 | threads.append(t) 114 | t.start() 115 | else: 116 | self._build_project(project) 117 | 118 | rc = 0 119 | if spec.settings.parallel_build: 120 | for thread in threads: 121 | thread.join() 122 | 123 | for thread_exit in self._thread_exit: 124 | if thread_exit != 0: 125 | rc = thread_exit 126 | 127 | self._finalize_build() 128 | return rc 129 | 130 | def cleanup(self): 131 | self._cleanup_build() 132 | 133 | @abstractmethod 134 | def _execute(self, command, cwd=None, exit=0): 135 | return 136 | 137 | @abstractmethod 138 | def _make_temp_dir(self, prefix='giftwrap'): 139 | return 140 | 141 | @abstractmethod 142 | def _make_dir(self, path, mode=0777): 143 | return 144 | 145 | @abstractmethod 146 | def _prepare_build(self): 147 | return 148 | 149 | @abstractmethod 150 | def _prepare_project_build(self, project): 151 | return 152 | 153 | @abstractmethod 154 | def _clone_project(self, project): 155 | return 156 | 157 | @abstractmethod 158 | def _create_virtualenv(self, venv_command, path): 159 | return 160 | 161 | @abstractmethod 162 | def _install_pip_dependencies(self, venv_path, dependencies): 163 | return 164 | 165 | @abstractmethod 166 | def _copy_sample_config(self, src_clone_dir, project): 167 | return 168 | 169 | @abstractmethod 170 | def _install_project(self, venv_path, src_clone_dir): 171 | return 172 | 173 | @abstractmethod 174 | def _finalize_project_build(self, project): 175 | return 176 | 177 | @abstractmethod 178 | def _finalize_build(self): 179 | return 180 | 181 | @abstractmethod 182 | def _cleanup_build(self): 183 | return 184 | 185 | 186 | from giftwrap.builders.package_builder import PackageBuilder # noqa 187 | from giftwrap.builders.docker_builder import DockerBuilder # noqa 188 | 189 | 190 | class BuilderFactory: 191 | 192 | @staticmethod 193 | def create_builder(builder_type, build_spec): 194 | driver_mgr = DriverManager(namespace=BUILDER_DRIVER_NAMESPACE, 195 | name=builder_type, 196 | invoke_args=(build_spec,), 197 | invoke_on_load=True) 198 | return driver_mgr.driver 199 | -------------------------------------------------------------------------------- /giftwrap/builders/docker_builder.py: -------------------------------------------------------------------------------- 1 | # vim: tabstop=4 shiftwidth=4 softtabstop=4 2 | 3 | # Copyright 2014, Craig Tracey 4 | # All Rights Reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 7 | # not use this file except in compliance with the License. You may obtain 8 | # a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 15 | # License for the specific language governing permissions and limitations 16 | 17 | import docker 18 | import jinja2 19 | import json 20 | import logging 21 | import os 22 | import re 23 | import tempfile 24 | 25 | from giftwrap.builders import Builder 26 | 27 | LOG = logging.getLogger(__name__) 28 | 29 | DEFAULT_TEMPLATE_FILE = os.path.join(os.path.dirname( 30 | os.path.dirname(__file__)), 31 | 'templates/Dockerfile.jinja2') 32 | APT_REQUIRED_PACKAGES = [ 33 | 'libffi-dev', 34 | 'libxml2-dev', 35 | 'libxslt1-dev', 36 | 'git', 37 | 'wget', 38 | 'curl', 39 | 'libldap2-dev', 40 | 'libsasl2-dev', 41 | 'libssl-dev', 42 | 'python-dev', 43 | 'libmysqlclient-dev', 44 | 'python-pip', 45 | 'build-essential' 46 | ] 47 | DEFAULT_SRC_PATH = '/opt/openstack' 48 | 49 | 50 | class DockerBuilder(Builder): 51 | 52 | def __init__(self, spec): 53 | self.base_image = 'ubuntu:14.04' 54 | self.maintainer = 'maintainer@example.com' 55 | self.envvars = {'DEBIAN_FRONTEND': 'noninteractive'} 56 | self._commands = [] 57 | super(DockerBuilder, self).__init__(spec) 58 | 59 | def _execute(self, command, cwd=None, exit=0): 60 | if cwd: 61 | self._commands.append("cd %s" % (cwd)) 62 | self._commands.append(command) 63 | if cwd: 64 | self._commands.append("cd -") 65 | 66 | def _make_temp_dir(self, prefix='giftwrap'): 67 | return "/tmp/giftwrap" 68 | self._commands.append("mktemp -d -t %s.XXXXXXXXXX" % prefix) 69 | 70 | def _make_dir(self, path, mode=0777): 71 | self._commands.append("mkdir -p -m %o %s" % (mode, path)) 72 | 73 | def _prepare_project_build(self, project): 74 | self.image_name = "giftwrap/openstack:%s" % (project.version) 75 | return 76 | 77 | def _clone_project(self, giturl, name, gitref, depth, path): 78 | cmd = "git clone %s -b %s --depth=%d %s" % (giturl, gitref, 79 | depth, path) 80 | self._commands.append(cmd) 81 | 82 | def _create_virtualenv(self, venv_command, path): 83 | self._execute(venv_command, path) 84 | 85 | def _install_pip_dependencies(self, venv_path, dependencies): 86 | pip_path = self._get_venv_pip_path(venv_path) 87 | for dependency in dependencies: 88 | self._execute("%s install %s" % (pip_path, dependency)) 89 | 90 | def _copy_sample_config(self, src_clone_dir, project): 91 | src_config = os.path.join(src_clone_dir, 'etc') 92 | dest_config = os.path.join(project.install_path, 'etc') 93 | 94 | self._commands.append("if [ -d %s ]; then cp -R %s %s; fi" % ( 95 | src_config, src_config, dest_config)) 96 | 97 | def _install_project(self, venv_path, src_clone_dir): 98 | pip_path = self._get_venv_pip_path(venv_path) 99 | self._execute("%s install %s" % (pip_path, src_clone_dir)) 100 | 101 | def _finalize_project_build(self, project): 102 | self._commands.append("rm -rf %s" % self._temp_dir) 103 | for command in self._commands: 104 | print command 105 | 106 | def _finalize_build(self): 107 | template_vars = { 108 | 'commands': self._commands 109 | } 110 | print self._render_dockerfile(template_vars) 111 | self._build_image() 112 | 113 | def _cleanup_build(self): 114 | return 115 | 116 | def _prepare_build(self): 117 | self._commands.append('apt-get update && apt-get install -y %s' % 118 | ' '.join(APT_REQUIRED_PACKAGES)) 119 | self._commands.append("pip install -U pip virtualenv") 120 | 121 | def _set_path(self): 122 | path = ":".join(self._paths) 123 | self.envvars['PATH'] = "%s:$PATH" % path 124 | 125 | def _render_dockerfile(self, extra_vars): 126 | template_vars = self.__dict__ 127 | template_vars.update(extra_vars) 128 | template_loader = jinja2.FileSystemLoader(searchpath='/') 129 | template_env = jinja2.Environment(loader=template_loader) 130 | template = template_env.get_template(DEFAULT_TEMPLATE_FILE) 131 | return template.render(template_vars) 132 | 133 | def _build_image(self): 134 | template_vars = { 135 | 'commands': self._commands 136 | } 137 | dockerfile_contents = self._render_dockerfile(template_vars) 138 | 139 | tempdir = tempfile.mkdtemp() 140 | dockerfile = os.path.join(tempdir, 'Dockerfile') 141 | with open(dockerfile, "w") as w: 142 | w.write(dockerfile_contents) 143 | docker_client = docker.Client(base_url='unix://var/run/docker.sock', 144 | timeout=10) 145 | build_result = docker_client.build(path=tempdir, stream=True, 146 | tag=self.image_name) 147 | for line in build_result: 148 | LOG.info(line.strip()) 149 | 150 | # I borrowed this from docker/stackbrew, should cull it down 151 | # to be more sane. 152 | def _parse_result(self, build_result): 153 | build_success_re = r'^Successfully built ([a-f0-9]+)\n$' 154 | if isinstance(build_result, tuple): 155 | img_id, logs = build_result 156 | return img_id, logs 157 | else: 158 | lines = [line for line in build_result] 159 | try: 160 | parsed_lines = [json.loads(e).get('stream', '') for e in lines] 161 | except ValueError: 162 | # sometimes all the data is sent on a single line ???? 163 | # 164 | # ValueError: Extra data: line 1 column 87 - line 1 column 165 | # 33268 (char 86 - 33267) 166 | line = lines[0] 167 | # This ONLY works because every line is formatted as 168 | # {"stream": STRING} 169 | parsed_lines = [ 170 | json.loads(obj).get('stream', '') for obj in 171 | re.findall('{\s*"stream"\s*:\s*"[^"]*"\s*}', line) 172 | ] 173 | 174 | for line in parsed_lines: 175 | match = re.match(build_success_re, line) 176 | if match: 177 | return match.group(1), parsed_lines 178 | return None, lines 179 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | --------------------------------------------------------------------------------