├── lumper ├── __init__.py ├── modes │ ├── __init__.py │ ├── server.py │ ├── mailer.py │ └── worker.py ├── mailer │ ├── __init__.py │ └── on_build.py ├── worker │ ├── __init__.py │ ├── heartbeat.py │ └── build.py ├── server │ ├── handlers │ │ ├── __init__.py │ │ ├── heartbeat.py │ │ └── webhook.py │ ├── __init__.py │ └── json_handler.py └── log_setter.py ├── autorestart.sh ├── Dockerfile ├── setup.py ├── fig.yml ├── bin └── lumper ├── LICENSE └── README.rst /lumper/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 -------------------------------------------------------------------------------- /autorestart.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | while sleep 1; do $@; done 4 | -------------------------------------------------------------------------------- /lumper/modes/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 -------------------------------------------------------------------------------- /lumper/mailer/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | 4 | from on_build import on_build -------------------------------------------------------------------------------- /lumper/worker/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | 4 | from heartbeat import heartbeat 5 | from build import BuildHandler -------------------------------------------------------------------------------- /lumper/server/handlers/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | 4 | from heartbeat import HeartBeat 5 | from webhook import GitHubWebHookHandler -------------------------------------------------------------------------------- /lumper/worker/heartbeat.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | 4 | from crew.worker import context, Task 5 | from time import time 6 | 7 | @Task("heartbeat") 8 | def heartbeat(data): 9 | context.settings.heartbeat_counter += 1 10 | return context.settings.uuid, time(), context.settings.heartbeat_counter -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:14.04 2 | MAINTAINER Dmitry Orlov 3 | 4 | RUN apt-get update && \ 5 | apt-get install -y python-pip python-dev git && \ 6 | apt-get clean 7 | 8 | ADD . /tmp/build/ 9 | ADD autorestart.sh /usr/local/bin/autorestart.sh 10 | RUN pip install --upgrade --pre /tmp/build && rm -fr /tmp/build 11 | 12 | ENTRYPOINT ["/usr/local/bin/lumper"] 13 | -------------------------------------------------------------------------------- /lumper/server/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | from functools import wraps 4 | import logging 5 | 6 | log = logging.getLogger("handlers") 7 | 8 | HANDLERS = [] 9 | 10 | def register(*urls): 11 | def deco(cls): 12 | global HANDLERS 13 | 14 | for url in urls: 15 | HANDLERS.append((url, cls)) 16 | log.debug('Register URL %r as %r' % (url, cls)) 17 | 18 | return cls 19 | return deco 20 | 21 | import handlers -------------------------------------------------------------------------------- /lumper/server/handlers/heartbeat.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | from __future__ import division, absolute_import 4 | from ..json_handler import JSONRequest 5 | from .. import register 6 | import tornado.gen 7 | from time import time 8 | 9 | 10 | @register(r"/api/v1/heartbeat") 11 | class HeartBeat(JSONRequest): 12 | SUPPORTED_METHODS = ('GET') 13 | 14 | @tornado.gen.coroutine 15 | def get(self): 16 | start = time() 17 | uuid, result, counter = yield self.settings['crew'].call('heartbeat') 18 | self.response({ 19 | "delta": (result - start) * 1000, 20 | "uuid": uuid, 21 | "beats": counter 22 | }) -------------------------------------------------------------------------------- /lumper/modes/server.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | import os 4 | import tornado.ioloop 5 | from tornado.web import Application 6 | from tornado.httpserver import HTTPServer 7 | from tornado.log import app_log as log 8 | from lumper.server import HANDLERS 9 | from crew.master.tornado import Client 10 | from pika import PlainCredentials 11 | 12 | 13 | def run(args): 14 | crew_client = Client( 15 | host=args.rmq_address, port=args.rmq_port, virtualhost=args.rmq_vhost, 16 | credentials=PlainCredentials(username=args.rmq_user, password=args.rmq_password) 17 | ) 18 | 19 | app = Application( 20 | args=args, 21 | handlers=HANDLERS, 22 | xsrf_cookies=False, 23 | cookie_secret=args.cookie_secret, 24 | debug=args.debug, 25 | reload=args.debug, 26 | gzip=args.gzip, 27 | crew=crew_client, 28 | timeout=args.timeout 29 | ) 30 | 31 | http_server = HTTPServer(app, xheaders=True) 32 | http_server.listen(args.port, address=args.address) 33 | log.info('Server started {host}:{port}'.format(host=args.address, port=args.port)) 34 | 35 | try: 36 | tornado.ioloop.IOLoop.instance().start() 37 | except Exception as e: 38 | log.exception(e) 39 | log.fatal("Server aborted by error: %r", e) 40 | 41 | return 0 42 | -------------------------------------------------------------------------------- /lumper/log_setter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | import argparse 4 | import logging 5 | import sys 6 | 7 | 8 | class LogSetterAction(argparse.Action): 9 | FORMAT = u'[%(asctime)s] %(filename)s:%(lineno)d %(levelname)-6s %(message)s' 10 | 11 | def __init__(self, option_strings, dest, nargs=None, **kwargs): 12 | if nargs is not None: 13 | raise ValueError("nargs not allowed") 14 | 15 | logging.basicConfig(format=self.FORMAT, level=logging.INFO) 16 | super(self.__class__, self).__init__(option_strings, dest, **kwargs) 17 | 18 | def __call__(self, parser, namespace, values, option_string=None): 19 | self.setter(values) 20 | setattr(namespace, self.dest, values) 21 | 22 | def setter(self, level='info'): 23 | root = logging.getLogger() 24 | 25 | level = getattr(logging, level.upper(), logging.INFO) 26 | 27 | fmt = root.handlers[0].formatter 28 | if root.handlers: 29 | for handler in root.handlers: 30 | root.removeHandler(handler) 31 | 32 | handler = logging.StreamHandler(stream=sys.stderr) 33 | handler.setFormatter(fmt) 34 | 35 | lvl = root.level if not level else level 36 | 37 | root.addHandler(handler) 38 | root.setLevel(lvl) 39 | 40 | root.info('Logging level is "{0}"'.format(logging._levelNames[lvl])) -------------------------------------------------------------------------------- /lumper/modes/mailer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | from collections import namedtuple 4 | 5 | from crew.worker import Listener, Context, NODE_UUID, UUID 6 | from pika import PlainCredentials 7 | from crew.worker import context 8 | import logging 9 | import lumper.mailer 10 | 11 | def run(args): 12 | log = logging.getLogger("main") 13 | try: 14 | SMTPSettings = namedtuple("SmtpSettings", "host port user password tls sender") 15 | Listener( 16 | port=args.amqp_port, 17 | host=args.amqp_address, 18 | credentials=PlainCredentials(username=args.amqp_user, password=args.amqp_password) if args.amqp_user else None, 19 | virtual_host=args.amqp_vhost, 20 | handlers=context.handlers, 21 | set_context=Context( 22 | options=args, 23 | node_uuid=NODE_UUID, 24 | uuid=UUID, 25 | smtp=SMTPSettings( 26 | host=args.smtp_host, 27 | port=args.smtp_port, 28 | user=args.smtp_user, 29 | password=args.smtp_password, 30 | tls=args.smtp_tls, 31 | sender=args.smtp_sender 32 | ) 33 | ), 34 | ).loop() 35 | except Exception as e: 36 | if logging.getLogger().level < logging.INFO: 37 | log.exception(e) 38 | else: 39 | log.fatal("Exiting by fatal error: %s", e) 40 | return 0 41 | -------------------------------------------------------------------------------- /lumper/modes/worker.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | from crew.worker import Listener, Context, NODE_UUID, UUID 4 | from pika import PlainCredentials 5 | from crew.worker import context 6 | import logging 7 | import docker 8 | import docker.tls 9 | import lumper.worker 10 | 11 | 12 | def run(args): 13 | log = logging.getLogger("main") 14 | 15 | if args.docker_tls: 16 | tls = docker.tls.TLSConfig(client_cert=(args.docker_client_cert, args.docker_client_key), 17 | ca_cert=args.docker_ca_cert, assert_hostname=False) 18 | else: 19 | tls = False 20 | 21 | docker_client = docker.Client(base_url=args.docker_url, tls=tls, timeout=300) 22 | docker_client.verify = args.docker_tls_strict 23 | 24 | try: 25 | log.info('Testing docker connection: %s', args.docker_url) 26 | docker_client.info() 27 | 28 | Listener( 29 | port=args.amqp_port, 30 | host=args.amqp_address, 31 | credentials=PlainCredentials(username=args.amqp_user, password=args.amqp_password) if args.amqp_user else None, 32 | virtual_host=args.amqp_vhost, 33 | handlers=context.handlers, 34 | set_context=Context( 35 | options=args, 36 | node_uuid=NODE_UUID, 37 | uuid=UUID, 38 | heartbeat_counter=0, 39 | docker=docker_client 40 | ) 41 | ).loop() 42 | except Exception as e: 43 | if logging.getLogger().level < logging.INFO: 44 | log.exception(e) 45 | else: 46 | log.fatal("Exiting by fatal error: %s", e) 47 | return 0 48 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | from __future__ import absolute_import, print_function 4 | from setuptools import find_packages 5 | import os 6 | 7 | # def rpath(*start_paths): 8 | # packages = set() 9 | # for start_path in start_paths: 10 | # if os.path.exists(os.path.join(start_path, '__init__.py')): 11 | # packages.add(start_path) 12 | # for cdir, dirs, files in os.walk(start_path): 13 | # for d in dirs: 14 | # path = os.path.join(cdir, d) 15 | # if os.path.exists(os.path.join(path, '__init__.py')): 16 | # packages.add(path.replace(os.sep, ".")) 17 | # return list(packages) 18 | 19 | try: 20 | from setuptools import setup 21 | except ImportError: 22 | from distutils.core import setup 23 | 24 | 25 | __version__ = '0.3.0' 26 | __author__ = 'Dmitry Orlov ' 27 | 28 | 29 | supports = { 30 | 'install_requires': [ 31 | 'tornado', 32 | 'arconfig', 33 | 'crew>=0.8.9', 34 | 'gitpython', 35 | 'docker-py', 36 | 'arrow' 37 | ] 38 | } 39 | 40 | setup( 41 | name='lumper', 42 | version=__version__, 43 | author=__author__, 44 | author_email='me@mosquito.su', 45 | license="MIT", 46 | description="Containers builder for docker.", 47 | platforms="all", 48 | url="http://github.com/mosquito/lumper", 49 | classifiers=[ 50 | 'Environment :: Console', 51 | 'Programming Language :: Python', 52 | ], 53 | scripts=['bin/lumper'], 54 | # package_data={ 55 | # 'lumper.some': ['rc/*'], 56 | # }, 57 | long_description=open('README.rst').read(), 58 | packages=find_packages(), 59 | **supports 60 | ) 61 | -------------------------------------------------------------------------------- /fig.yml: -------------------------------------------------------------------------------- 1 | rabbitmq: 2 | image: dockerfile/rabbitmq:latest 3 | expose: 4 | - 5672 5 | ports: 6 | - 15672:15672 7 | 8 | registry: 9 | image: registry:latest 10 | environment: 11 | SETTINGS_FLAVOR: local 12 | SEARCH_BACKEND: sqlalchemy 13 | expose: 14 | - 5000 15 | volumes: 16 | - /storage/registry:/tmp/registry 17 | 18 | registryUI: 19 | image: mosquito/docker-registry-ui:latest 20 | ports: 21 | - 8081:80 22 | 23 | postfix: 24 | image: catatnight/postfix:latest 25 | hostname: lumper.example.com 26 | environment: 27 | maildomain: mail.lumper.example.com 28 | smtp_user: lumper:lumper 29 | expose: 30 | - 25 31 | 32 | mailer: 33 | image: mosquito/lumper:latest 34 | entrypoint: /usr/local/bin/autorestart.sh 35 | command: lumper mailer --admin-mail="admin@example.com" --smtp-sender="build@example.com" --smtp-host postfix --smtp-port 25 --smtp-user lumper --smtp-password lumper -a rabbitmq -U guest -P guest --logging=debug --mail-map /tmp/mailmap.json 36 | #volumes: 37 | # - /storage/mailmap.json:/tmp/mailmap.json 38 | links: 39 | - postfix 40 | - rabbitmq 41 | 42 | worker: 43 | image: mosquito/lumper:latest 44 | links: 45 | - rabbitmq 46 | - registry 47 | entrypoint: /usr/local/bin/autorestart.sh 48 | command: lumper worker --docker-url="tcp://172.18.0.1:4142" -a rabbitmq --docker-registry="127.0.0.1:5000" -U guest -P guest --logging=debug --docker-publish 49 | environment: 50 | HOME: /root 51 | volumes: 52 | # add ssh keys there .ssh/{known_hosts,id_rsa} and add key to git[hub|lab] 53 | - /storage/worker:/root 54 | 55 | backend: 56 | image: mosquito/lumper:latest 57 | expose: 58 | - 8000 59 | links: 60 | - rabbitmq 61 | command: server -p 8000 -a 0.0.0.0 -A rabbitmq --user=guest --password=guest --logging=debug -T 1800 62 | 63 | -------------------------------------------------------------------------------- /lumper/server/json_handler.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | import tornado.web 4 | import json 5 | import traceback 6 | import logging 7 | 8 | log = logging.getLogger("handlers.json") 9 | 10 | class JSONRequest(tornado.web.RequestHandler): 11 | INDENT = None 12 | ENCODING = 'utf-8' 13 | ENSURE_ASCII = False 14 | SORT_KEYS = False 15 | __json = None 16 | 17 | def options(self, *args, **kwargs): 18 | self.finish() 19 | 20 | def prepare(self): 21 | self.clear_header('Content-Type') 22 | self.set_header('Access-Control-Allow-Methods', ", ".join(self.SUPPORTED_METHODS)) 23 | self.set_header('Access-Control-Allow-Headers', "accept, origin, content-type, cookie") 24 | self.set_header('Access-Control-Allow-Credentials', 'true') 25 | self.set_header('Access-Control-Max-Age', 3600) 26 | 27 | if self.request.method == 'OPTIONS': 28 | return self.finish() 29 | 30 | self.content_type = 'application/json' 31 | self.set_header("Content-Type", "application/json; charset=%s" % self.ENCODING.lower()) 32 | 33 | @property 34 | def json(self): 35 | if self.__json: 36 | return self.__json 37 | else: 38 | if self.content_type in self.request.headers.get('Content-Type', ''): 39 | try: 40 | self.__json = json.loads(self.request.body) if self.request.body else {} 41 | return self.__json 42 | except Exception as e: 43 | log.debug(traceback.format_exc()) 44 | log.error(repr(e)) 45 | self.write_error(400) 46 | 47 | @staticmethod 48 | def default(obj): 49 | if hasattr(obj, '__dict__'): 50 | return obj.__dict__ 51 | elif hasattr(obj, '_to_json'): 52 | return obj._to_json() 53 | else: 54 | return str(obj) 55 | 56 | def _jsonify(self, data): 57 | default = self.default 58 | return json.dumps( 59 | data, 60 | indent=self.INDENT, 61 | sort_keys=self.SORT_KEYS, 62 | ensure_ascii=self.ENSURE_ASCII, 63 | encoding=self.ENCODING, 64 | default=default 65 | ) 66 | 67 | def response(self, data, finish=False): 68 | if not self._finished: 69 | self.write(self._jsonify(data)) 70 | if finish: 71 | self.finish() -------------------------------------------------------------------------------- /lumper/mailer/on_build.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | from datetime import datetime 4 | import json 5 | import urllib2 6 | from crew.worker import context, Task 7 | from email.mime.base import MIMEBase 8 | from email.mime.text import MIMEText 9 | from email.mime.multipart import MIMEMultipart 10 | from email import Encoders 11 | import logging 12 | import smtplib 13 | 14 | 15 | class Attachment(object): 16 | pass 17 | 18 | 19 | class FileAttachment(Attachment): 20 | def __init__(self, data, file_name, content_type="application/octet-stream"): 21 | self.part = MIMEBase('application', "octet-stream") 22 | self.part.set_payload(data.encode('utf-8')) 23 | Encoders.encode_base64(self.part) 24 | self.part.add_header('Content-Disposition', 'attachment; filename="{0}"'.format(file_name)) 25 | 26 | def attach(self, email): 27 | assert isinstance(email, Email) 28 | email.add_part(self.part) 29 | 30 | 31 | class Email(object): 32 | log = logging.getLogger('emailer.email') 33 | 34 | def __init__(self, sender, recipient, subject, headers={}): 35 | self.__sent = False 36 | self.msg = MIMEMultipart() 37 | self.msg['Subject'] = subject 38 | self.msg['From'] = sender 39 | self.msg['To'] = recipient 40 | self.log.debug("Building message from <%s> to <%s>", sender, recipient) 41 | for key, value in headers.iteritems(): 42 | self.log.debug('Add header "%s": "%s"', key, value) 43 | self.msg[key] = value 44 | 45 | def __setitem__(self, key, value): 46 | self.log.debug('Setting header "%s": "%s"', key, value) 47 | self.msg[key] = value 48 | 49 | def __getitem__(self, item): 50 | return self.msg[item] 51 | 52 | def get(self, key, default=None): 53 | if key in self.msg: 54 | return self.msg[key] 55 | else: 56 | return default 57 | 58 | def add_part(self, part): 59 | assert isinstance(part, MIMEBase) 60 | self.msg.attach(part) 61 | 62 | def append(self, part, mimetype='plain'): 63 | prt = unicode(part) 64 | self.log.debug('Adding "%s" (length: %s) part to message from <%s> to <%s>', mimetype, len(prt), self.msg['From'], self.msg['To']) 65 | self.msg.attach(MIMEText(prt, mimetype, _charset='utf-8')) 66 | 67 | def send(self, host='localhost', port=25, tls=False, user=None, password=None): 68 | self.log.info("Sending message for <%s>.", self.msg['To']) 69 | self.log.debug("Connecting to SMTP server %s:%d", host, port) 70 | if not self.__sent: 71 | connect = smtplib.SMTP(host, port=port) 72 | try: 73 | connect.ehlo() 74 | if tls: 75 | self.log.debug("Establishing TLS") 76 | connect.starttls() 77 | connect.ehlo() 78 | if user: 79 | connect.login(user=user, password=password) 80 | connect.sendmail(self.msg['From'], self.msg['To'], self.msg.as_string()) 81 | self.__sent = True 82 | return True 83 | except Exception as e: 84 | self.log.exception(e) 85 | self.log.error("Error: %r", e) 86 | self.__sent = False 87 | return False 88 | finally: 89 | connect.close() 90 | else: 91 | self.log.error("Message already sent") 92 | return False 93 | 94 | log = logging.getLogger('emailer.task') 95 | 96 | @Task("build.finished") 97 | def on_build(data): 98 | if isinstance(data, Exception): 99 | email = Email( 100 | sender=context.settings.smtp.sender, 101 | recipient=context.settings.options.admin_mail, 102 | subject="Build exception" 103 | ) 104 | 105 | email.append("Error: %r\n\nTraceback: %s\n\n" % (data, getattr(data, '_tb', "No traceback"))) 106 | 107 | FileAttachment( 108 | "\n".join(getattr(data, "log", ['No log',])), 109 | file_name='build_log.txt', 110 | content_type='text/plain' 111 | ).attach(email) 112 | 113 | else: 114 | if context.settings.options.mail_map: 115 | recepient = context.settings.options.mail_map.get(data['sender'], context.settings.options.admin_mail) 116 | else: 117 | recepient = context.settings.options.admin_mail 118 | 119 | email = Email( 120 | sender=context.settings.smtp.sender, 121 | recipient=recepient, 122 | subject="[%s] <%s> Build %s" % ( 123 | data.get('tag'), 124 | data.get('name'), 125 | 'successful' if data.get('status') else 'failed' 126 | ) 127 | ) 128 | 129 | 130 | email.append( 131 | "\n".join([ 132 | "Build %s %s" % (data.get('name'), 'successful' if data.get('status') else 'failed'), 133 | "\n", 134 | "Sender: %s" % data.get('sender'), 135 | "Repository: %s" % data.get('repo'), 136 | "Commit: %s" % data.get('commit'), 137 | "Commit message: %s" % data.get('message'), 138 | "Tag: %s" % data.get('tag'), 139 | "Build timestamp: %s" % data.get('timestamp'), 140 | "Build date: %s" % datetime.utcfromtimestamp(data['timestamp']) if data.get('timestamp') else None, 141 | "\n\n" 142 | ])) 143 | 144 | FileAttachment( 145 | "\n".join(data.get('build_log')), 146 | file_name='build_log.txt', 147 | content_type='text/plain' 148 | ).attach(email) 149 | 150 | if context.settings.options.build_hooks: 151 | hook_data = json.dumps(data, sort_keys=False, encoding="utf-8") 152 | for url in context.settings.options.build_hooks: 153 | try: 154 | log.info('Sending build data to: "%s"', url) 155 | req = urllib2.Request(url) 156 | req.add_header('Content-Type', 'application/json') 157 | response = urllib2.urlopen(req, hook_data, timeout=3) 158 | log.info('Build hook response for "%s": HTTP %d', url, response.code) 159 | except Exception as e: 160 | if log.getEffectiveLevel() <= logging.DEBUG: 161 | log.exception(e) 162 | log.error("Build hook error: %r", e) 163 | 164 | return email.send( 165 | host=context.settings.smtp.host, 166 | port=context.settings.smtp.port, 167 | user=context.settings.smtp.user, 168 | password=context.settings.smtp.password, 169 | tls=context.settings.smtp.tls 170 | ) 171 | -------------------------------------------------------------------------------- /lumper/server/handlers/webhook.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | 4 | from time import time 5 | import tornado.web 6 | import tornado.gen 7 | import hmac 8 | import hashlib 9 | import re 10 | import arrow 11 | 12 | from ..json_handler import JSONRequest 13 | from .. import register 14 | from tornado.gen import Future 15 | from tornado.log import app_log as log 16 | 17 | REXP={ 18 | "split_refs": re.compile("^refs\/(?P\S+)\/(?P\S+)") 19 | } 20 | 21 | @register("/github/webhook/") 22 | @register("/webhook/github") 23 | class GitHubWebHookHandler(JSONRequest): 24 | @tornado.web.asynchronous 25 | @tornado.gen.coroutine 26 | def post(self, *args, **kwargs): 27 | self.delivery = self.request.headers.get("X-Github-Delivery") 28 | self.event = self.request.headers.get("X-Github-Event") 29 | 30 | is_valid = self.event and self.delivery and self.verify() if self.settings['args'].github_secret else None 31 | 32 | if is_valid: 33 | handler = getattr(self, "event_%s" % self.event, lambda: self.response("OK")) 34 | resp = handler() 35 | 36 | if isinstance(resp, Future): 37 | ret = yield resp 38 | else: 39 | ret = resp 40 | 41 | if not self._finished: 42 | self.finish(ret) 43 | else: 44 | self.send_error(403) 45 | if not self._finished: 46 | self.finish() 47 | 48 | def verify(self): 49 | signature = self.request.headers.get("X-Hub-Signature") 50 | h = hmac.new(self.settings['args'].github_secret, self.request.body, hashlib.sha1) 51 | return "sha1=%s" % h.hexdigest() == signature 52 | 53 | def event_ping(self): 54 | """ Just ping. """ 55 | log.debug("Got PING request: %s", self.json) 56 | self.send_error(204) 57 | 58 | def event_push(self): 59 | """ Any Git push to a Repository, including editing tags or branches. Commits via API actions that update 60 | references are also counted. This is the default event.""" 61 | return self._process_tag() 62 | 63 | def _process_tag(self): 64 | matcher = REXP['split_refs'].match(self.json.get('ref', "")) 65 | if matcher: 66 | refs = matcher.groupdict() 67 | if refs['key'] == 'tags': 68 | tag = refs['value'] 69 | commit = self.json.get("head_commit") 70 | data = { 71 | "tag": tag, 72 | "repo": self.json['repository']['ssh_url'], 73 | "commit": commit['id'], 74 | "message": commit['message'], 75 | "timestamp": arrow.get(commit['timestamp']).timestamp, 76 | "name": self.json['repository']['full_name'], 77 | "sender": self.json['sender']['login'] 78 | } 79 | 80 | self.settings['crew'].call("build", data, routing_key="crew.tasks.build.finished", expiration=self.settings.get('timeout', 600)) 81 | self.response(True) 82 | else: 83 | self.response(False) 84 | 85 | # def event_commit_comment(self): 86 | # """ Any time a Commit is commented on. """ 87 | # def event_delete(self): 88 | # """ Any time a Branch or Tag is deleted. """ 89 | # def event_deployment(self): 90 | # """ Any time a Repository has a new deployment created from the API. """ 91 | # def event_deployment_status(self): 92 | # """ Any time a deployment for a Repository has a status update from the API. """ 93 | # def event_fork(self): 94 | # """ Any time a Repository is forked. """ 95 | # def event_gollum(self): 96 | # """ Any time a Wiki page is updated. """ 97 | # def event_issue_comment(self): 98 | # """ Any time an Issue is commented on. """ 99 | # def event_issues(self): 100 | # """ Any time an Issue is assigned, unassigned, labeled, unlabeled, opened, closed, or reopened. """ 101 | # def event_member(self): 102 | # """ Any time a User is added as a collaborator to a non-Organization Repository. """ 103 | # def event_membership(self): 104 | # """ Any time a User is added or removed from a team. Organization hooks only. """ 105 | # def event_page_build(self): 106 | # """ Any time a Pages site is built or results in a failed build. """ 107 | # def event_public(self): 108 | # """ Any time a Repository changes from private to public. """ 109 | # def event_pull_request_review_comment(self): 110 | # """ Any time a Commit is commented on while inside a Pull Request review (the Files Changed tab). """ 111 | # def event_pull_request(self): 112 | # """ Any time a Pull Request is assigned, unassigned, labeled, unlabeled, opened, closed, reopened, or 113 | # synchronized (updated due to a (selevent_f):new push in the branch that the pull request is tracking).""" 114 | # def event_repository(self): 115 | # """ Any time a Repository is created. Organization hooks only. """ 116 | # def event_release(self): 117 | # """ Any time a Release is published in a Repository. """ 118 | # def event_status(self): 119 | # """ Any time a Repository has a status update from the API """ 120 | # def event_team_add(self): 121 | # """ Any time a team is added or modified on a Repository. """ 122 | # def event_watch(self): 123 | # """ Any time a User watches a Repository. """ 124 | 125 | 126 | @register("/webhook/gitlab") 127 | class CommonWebHookHandler(JSONRequest): 128 | 129 | @tornado.web.asynchronous 130 | @tornado.gen.coroutine 131 | def post(self, *args, **kwargs): 132 | matcher = REXP['split_refs'].match(self.json.get('ref', "")) 133 | if matcher: 134 | refs = matcher.groupdict() 135 | if refs['key'] == 'tags': 136 | tag = refs['value'] 137 | commit = self.json.get('commits', []) 138 | commit = commit[0] if commit else {} 139 | 140 | repo_name = self.json['repository']['homepage'].split("/") 141 | repo_name = "%s/%s" % (repo_name.pop(-2), repo_name.pop()) 142 | 143 | data = { 144 | "tag": tag, 145 | "repo": self.json['repository']['url'], 146 | "commit": self.json['checkout_sha'], 147 | "message": commit.get('message', ''), 148 | "timestamp": arrow.get(commit.get('timestamp', time())).timestamp, 149 | "name": repo_name, 150 | "sender": str(self.json['user_id']), 151 | } 152 | 153 | self.settings['crew'].call("build", data, routing_key="crew.tasks.build.finished", expiration=self.settings.get('timeout', 600)) 154 | self.response(True) 155 | else: 156 | self.response(False) 157 | -------------------------------------------------------------------------------- /bin/lumper: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | import argparse 4 | import json 5 | import logging 6 | import uuid 7 | from socket import getfqdn 8 | from arconfig import GenConfigAction, LoadConfigAction 9 | from lumper.log_setter import LogSetterAction 10 | 11 | if __name__ == "__main__": 12 | parser = argparse.ArgumentParser(epilog='Notice: exec "%(prog)s --help" for command options') 13 | 14 | subparsers = parser.add_subparsers(dest='cmd') 15 | subparsers.required = True 16 | 17 | # Server mode 18 | subparser = subparsers.add_parser("server", help="Run http backend") 19 | subparser.add_argument("--config", action=LoadConfigAction) 20 | subparser.add_argument("--gen-config", action=GenConfigAction) 21 | 22 | group = subparser.add_argument_group("Server options") 23 | group.add_argument('-a', '--address', dest="address", help="Listen this address", default="localhost") 24 | group.add_argument('-p', '--port', dest="port", help="Listen this port", type=int, default=8228) 25 | 26 | group.add_argument('--secret', dest="cookie_secret", 27 | help="Cookie secret", default=str(uuid.uuid3(uuid.NAMESPACE_DNS, getfqdn()))) 28 | 29 | group.add_argument('--gzip', dest="gzip", help="Gzip HTTP responses", action='store_true', default=False) 30 | 31 | group.add_argument('--debug', dest="debug", help="Debugging mode", default=False, action="store_true") 32 | group.add_argument('--logging', dest="logging", help="Logging level", action=LogSetterAction) 33 | 34 | group.add_argument('--github-secret', dest="github_secret", help="Github webhook's secret", default=None) 35 | 36 | group.add_argument('-A', '--rmq-address', dest="rmq_address", help="RMQ host address", default="localhost") 37 | group.add_argument('-P', '--rmq-port', dest="rmq_port", help="RMQ host port", type=int, default=5672) 38 | group.add_argument('-H', '--vhost', dest="rmq_vhost", help="RMQ virtual host", default="/") 39 | group.add_argument('--user', dest="rmq_user", help="RMQ virtual host", default=None) 40 | group.add_argument('--password', dest="rmq_password", help="RMQ virtual host", default=None) 41 | 42 | group = subparser.add_argument_group("Task options") 43 | group.add_argument('-T', '--timeout', dest="timeout", help="Build timeout", type=int, default=600) 44 | 45 | # Worker mode 46 | subparser = subparsers.add_parser("worker", help="Run in worker mode") 47 | subparser.add_argument("--config", action=LoadConfigAction) 48 | subparser.add_argument("--gen-config", action=GenConfigAction) 49 | 50 | group = subparser.add_argument_group("Main options") 51 | group.add_argument('--logging', dest="logging", help="Logging level", action=LogSetterAction) 52 | 53 | group = subparser.add_argument_group("RabbitMQ options") 54 | group.add_argument('-a', '--address', dest="amqp_address", help="RMQ host address", default="localhost") 55 | group.add_argument('-p', '--port', dest="amqp_port", help="RMQ host port", type=int, default=5672) 56 | group.add_argument('-H', '--vhost', dest="amqp_vhost", help="RMQ virtual host", default="/") 57 | group.add_argument('-U', '--user', dest="amqp_user", help="RMQ username", default=None) 58 | group.add_argument('-P', '--password', dest="amqp_password", help="RMQ password", default=None) 59 | 60 | group = subparser.add_argument_group("Docker options") 61 | group.add_argument('--docker-url', dest="docker_url", 62 | help="Docker daemon url [\"unix:///var/run/docker.sock\"]", 63 | default="unix:///var/run/docker.sock") 64 | group.add_argument('--docker-tls', dest="docker_tls", help="Set when a docker daemon use TLS", action="store_true") 65 | group.add_argument('--docker-ca', dest="docker_ca_cert", help="TLS certificate authority", default="ca.crt") 66 | group.add_argument('--docker-cert', dest="docker_client_cert", help="TLS client certificate", default="client.crt") 67 | group.add_argument('--docker-key', dest="docker_client_key", help="TLS client private key", default="client.pem") 68 | group.add_argument('--docker-tls-strict', dest="docker_tls_strict", help="Strict verification server certificate", 69 | action="store_true") 70 | group.add_argument('--docker-registry', dest="docker_registry", 71 | help="Set if you have a private registry", default='localhost:5000') 72 | group.add_argument('--docker-ssl-registry', dest="docker_ssl_registry", 73 | help="The private registry use ssl", action='store_true') 74 | group.add_argument('--docker-publish', dest="docker_publish", 75 | help="Set if you want push images to registry", action="store_true") 76 | 77 | # Mailer mode 78 | subparser = subparsers.add_parser("mailer", help="Run as mailer delivery worker") 79 | subparser.add_argument("--config", action=LoadConfigAction) 80 | subparser.add_argument("--gen-config", action=GenConfigAction) 81 | 82 | group = subparser.add_argument_group("Main options") 83 | group.add_argument('--logging', dest="logging", help="Logging level", action=LogSetterAction) 84 | 85 | group = subparser.add_argument_group("RabbitMQ options") 86 | group.add_argument('-a', '--address', dest="amqp_address", help="RMQ host address", default="localhost") 87 | group.add_argument('-p', '--port', dest="amqp_port", help="RMQ host port", type=int, default=5672) 88 | group.add_argument('-H', '--vhost', dest="amqp_vhost", help="RMQ virtual host", default="/") 89 | group.add_argument('-U', '--user', dest="amqp_user", help="RMQ username", default=None) 90 | group.add_argument('-P', '--password', dest="amqp_password", help="RMQ password", default=None) 91 | 92 | group = subparser.add_argument_group("SMTP options") 93 | group.add_argument("--smtp-host", dest="smtp_host", help="Server host", default="localhost") 94 | group.add_argument("--smtp-port", dest="smtp_port", help="Server port", type=int, default=25) 95 | group.add_argument("--smtp-user", dest="smtp_user", help="Authentication username. Do auth if set.", default=None, type=str) 96 | group.add_argument("--smtp-password", dest="smtp_password", help="Password.", default=None, type=str) 97 | group.add_argument("--smtp-tls", dest="smtp_tls", help="Use TLS.", action='store_true') 98 | default_sender = "lumper@%s" % (getfqdn()) 99 | group.add_argument( 100 | "--smtp-sender", dest="smtp_sender", 101 | help="Sender of messages [default: %s]" % default_sender, 102 | default=default_sender 103 | ) 104 | 105 | group.add_argument( 106 | "--build-hook", 107 | dest="build_hooks", 108 | metavar="URL", 109 | help="The url address on which will send build-hook (Might be multiple).", 110 | action='append', 111 | default=[], 112 | ) 113 | 114 | default_user = "root@%s" % (getfqdn()) 115 | group = subparser.add_argument_group("Delivery options") 116 | group.add_argument("--mail-map", dest="mail_map", help="github user to E-mail map json file with hash.", default=None) 117 | group.add_argument( 118 | "--admin-mail", 119 | dest="admin_mail", 120 | help="admin email for unknown users [default: %s]" % default_user, 121 | default=default_user 122 | ) 123 | 124 | args = parser.parse_args() 125 | 126 | if args.cmd == 'server': 127 | from lumper.modes.server import run 128 | 129 | if not args.github_secret: 130 | logging.getLogger().warning("Github secret is not presented. Signature checking are disabled.") 131 | else: 132 | args.github_secret = str(args.github_secret) 133 | exit(run(args)) 134 | 135 | elif args.cmd == 'worker': 136 | from lumper.modes.worker import run 137 | exit(run(args)) 138 | 139 | elif args.cmd == 'mailer': 140 | from lumper.modes.mailer import run 141 | 142 | if args.mail_map: 143 | args.mail_map = json.load(open(args.mail_map, 'r')) 144 | else: 145 | args.mail_map = {} 146 | 147 | exit(run(args)) 148 | 149 | else: 150 | print ("Incompatible mode") 151 | exit(128) 152 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU LESSER GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | 9 | This version of the GNU Lesser General Public License incorporates 10 | the terms and conditions of version 3 of the GNU General Public 11 | License, supplemented by the additional permissions listed below. 12 | 13 | 0. Additional Definitions. 14 | 15 | As used herein, "this License" refers to version 3 of the GNU Lesser 16 | General Public License, and the "GNU GPL" refers to version 3 of the GNU 17 | General Public License. 18 | 19 | "The Library" refers to a covered work governed by this License, 20 | other than an Application or a Combined Work as defined below. 21 | 22 | An "Application" is any work that makes use of an interface provided 23 | by the Library, but which is not otherwise based on the Library. 24 | Defining a subclass of a class defined by the Library is deemed a mode 25 | of using an interface provided by the Library. 26 | 27 | A "Combined Work" is a work produced by combining or linking an 28 | Application with the Library. The particular version of the Library 29 | with which the Combined Work was made is also called the "Linked 30 | Version". 31 | 32 | The "Minimal Corresponding Source" for a Combined Work means the 33 | Corresponding Source for the Combined Work, excluding any source code 34 | for portions of the Combined Work that, considered in isolation, are 35 | based on the Application, and not on the Linked Version. 36 | 37 | The "Corresponding Application Code" for a Combined Work means the 38 | object code and/or source code for the Application, including any data 39 | and utility programs needed for reproducing the Combined Work from the 40 | Application, but excluding the System Libraries of the Combined Work. 41 | 42 | 1. Exception to Section 3 of the GNU GPL. 43 | 44 | You may convey a covered work under sections 3 and 4 of this License 45 | without being bound by section 3 of the GNU GPL. 46 | 47 | 2. Conveying Modified Versions. 48 | 49 | If you modify a copy of the Library, and, in your modifications, a 50 | facility refers to a function or data to be supplied by an Application 51 | that uses the facility (other than as an argument passed when the 52 | facility is invoked), then you may convey a copy of the modified 53 | version: 54 | 55 | a) under this License, provided that you make a good faith effort to 56 | ensure that, in the event an Application does not supply the 57 | function or data, the facility still operates, and performs 58 | whatever part of its purpose remains meaningful, or 59 | 60 | b) under the GNU GPL, with none of the additional permissions of 61 | this License applicable to that copy. 62 | 63 | 3. Object Code Incorporating Material from Library Header Files. 64 | 65 | The object code form of an Application may incorporate material from 66 | a header file that is part of the Library. You may convey such object 67 | code under terms of your choice, provided that, if the incorporated 68 | material is not limited to numerical parameters, data structure 69 | layouts and accessors, or small macros, inline functions and templates 70 | (ten or fewer lines in length), you do both of the following: 71 | 72 | a) Give prominent notice with each copy of the object code that the 73 | Library is used in it and that the Library and its use are 74 | covered by this License. 75 | 76 | b) Accompany the object code with a copy of the GNU GPL and this license 77 | document. 78 | 79 | 4. Combined Works. 80 | 81 | You may convey a Combined Work under terms of your choice that, 82 | taken together, effectively do not restrict modification of the 83 | portions of the Library contained in the Combined Work and reverse 84 | engineering for debugging such modifications, if you also do each of 85 | the following: 86 | 87 | a) Give prominent notice with each copy of the Combined Work that 88 | the Library is used in it and that the Library and its use are 89 | covered by this License. 90 | 91 | b) Accompany the Combined Work with a copy of the GNU GPL and this license 92 | document. 93 | 94 | c) For a Combined Work that displays copyright notices during 95 | execution, include the copyright notice for the Library among 96 | these notices, as well as a reference directing the user to the 97 | copies of the GNU GPL and this license document. 98 | 99 | d) Do one of the following: 100 | 101 | 0) Convey the Minimal Corresponding Source under the terms of this 102 | License, and the Corresponding Application Code in a form 103 | suitable for, and under terms that permit, the user to 104 | recombine or relink the Application with a modified version of 105 | the Linked Version to produce a modified Combined Work, in the 106 | manner specified by section 6 of the GNU GPL for conveying 107 | Corresponding Source. 108 | 109 | 1) Use a suitable shared library mechanism for linking with the 110 | Library. A suitable mechanism is one that (a) uses at run time 111 | a copy of the Library already present on the user's computer 112 | system, and (b) will operate properly with a modified version 113 | of the Library that is interface-compatible with the Linked 114 | Version. 115 | 116 | e) Provide Installation Information, but only if you would otherwise 117 | be required to provide such information under section 6 of the 118 | GNU GPL, and only to the extent that such information is 119 | necessary to install and execute a modified version of the 120 | Combined Work produced by recombining or relinking the 121 | Application with a modified version of the Linked Version. (If 122 | you use option 4d0, the Installation Information must accompany 123 | the Minimal Corresponding Source and Corresponding Application 124 | Code. If you use option 4d1, you must provide the Installation 125 | Information in the manner specified by section 6 of the GNU GPL 126 | for conveying Corresponding Source.) 127 | 128 | 5. Combined Libraries. 129 | 130 | You may place library facilities that are a work based on the 131 | Library side by side in a single library together with other library 132 | facilities that are not Applications and are not covered by this 133 | License, and convey such a combined library under terms of your 134 | choice, if you do both of the following: 135 | 136 | a) Accompany the combined library with a copy of the same work based 137 | on the Library, uncombined with any other library facilities, 138 | conveyed under the terms of this License. 139 | 140 | b) Give prominent notice with the combined library that part of it 141 | is a work based on the Library, and explaining where to find the 142 | accompanying uncombined form of the same work. 143 | 144 | 6. Revised Versions of the GNU Lesser General Public License. 145 | 146 | The Free Software Foundation may publish revised and/or new versions 147 | of the GNU Lesser General Public License from time to time. Such new 148 | versions will be similar in spirit to the present version, but may 149 | differ in detail to address new problems or concerns. 150 | 151 | Each version is given a distinguishing version number. If the 152 | Library as you received it specifies that a certain numbered version 153 | of the GNU Lesser General Public License "or any later version" 154 | applies to it, you have the option of following the terms and 155 | conditions either of that published version or of any later version 156 | published by the Free Software Foundation. If the Library as you 157 | received it does not specify a version number of the GNU Lesser 158 | General Public License, you may choose any version of the GNU Lesser 159 | General Public License ever published by the Free Software Foundation. 160 | 161 | If the Library as you received it specifies that a proxy can decide 162 | whether future versions of the GNU Lesser General Public License shall 163 | apply, that proxy's public statement of acceptance of any version is 164 | permanent authorization for you to choose that version for the 165 | Library. 166 | 167 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Lumper 2 | ====== 3 | 4 | Distributed building system for docker. Will pull repo from the github by tag-webhook and build it by Dockerfile. 5 | 6 | Features 7 | -------- 8 | 9 | #. Distributed system. Any part might be working on different hosts. 10 | #. Email notifications (success and errors). 11 | #. Pushing into public or private docker registry. 12 | #. Building in queue. 13 | #. Emailing reports to administrator about exceptions. 14 | #. TLS client auth for docker daemon. 15 | #. SMTP authentication 16 | #. Building by webhook tag from github (*You might be author of the extension for other services through pull-request ;-)*. 17 | #. Multiple installations (thanks to RMQ vhosts) 18 | 19 | Requirements 20 | ------------ 21 | 22 | * Python >=2.7 (>3.4 need testing). 23 | * RabbitMQ server. Provide communication for components. 24 | 25 | 26 | Parts 27 | ----- 28 | 29 | The system consists of 3 parts 30 | 31 | * WEB Server. Based on tornado http server for accepting webhooks 32 | * Worker. Building daemon listen AMQP 33 | * Mailer. Mailing daemon. Provides notifying about build results. 34 | 35 | 36 | Installation 37 | ------------ 38 | 39 | #. Install Rabbitmq Server 40 | #. pip install lumper 41 | 42 | Usage 43 | ----- 44 | 45 | The lumper provides one executable file **lumper**. You might run this with option --help (or -h):: 46 | 47 | $ lumper --help 48 | usage: lumper [-h] {server,worker,mailer} ... 49 | 50 | positional arguments: 51 | {server,worker,mailer} 52 | server Run http backend 53 | worker Run in worker mode 54 | mailer Run as mailer delivery worker 55 | 56 | optional arguments: 57 | -h, --help show this help message and exit 58 | 59 | Notice: exec "lumper --help" for command options 60 | 61 | 62 | Command line configuration 63 | ++++++++++++++++++++++++++ 64 | 65 | And you might see help about any modes. For web server:: 66 | 67 | $ lumper server --help 68 | usage: lumper server [-h] [--config CONFIG] [--gen-config] [-a ADDRESS] 69 | [-p PORT] [--secret COOKIE_SECRET] [--gzip] [--debug] 70 | [--logging LOGGING] [--github-secret GITHUB_SECRET] 71 | [-A RMQ_ADDRESS] [-P RMQ_PORT] [-H RMQ_VHOST] 72 | [--user RMQ_USER] [--password RMQ_PASSWORD] 73 | 74 | optional arguments: 75 | -h, --help show this help message and exit 76 | --config CONFIG Load configuration from file 77 | --gen-config Create example of the config_file.json 78 | 79 | Server options: 80 | -a ADDRESS, --address ADDRESS 81 | Listen this address 82 | -p PORT, --port PORT Listen this port 83 | --secret COOKIE_SECRET 84 | Cookie secret 85 | --gzip Gzip HTTP responses 86 | --debug Debugging mode 87 | --logging LOGGING Logging level 88 | --github-secret GITHUB_SECRET 89 | Github webhook's secret 90 | -A RMQ_ADDRESS, --rmq-address RMQ_ADDRESS 91 | RMQ host address 92 | -P RMQ_PORT, --rmq-port RMQ_PORT 93 | RMQ host port 94 | -H RMQ_VHOST, --vhost RMQ_VHOST 95 | RMQ virtual host 96 | --user RMQ_USER RMQ virtual host 97 | --password RMQ_PASSWORD 98 | RMQ virtual host 99 | 100 | 101 | For worker:: 102 | 103 | $ lumper worker --help 104 | usage: lumper worker [-h] [--config CONFIG] [--gen-config] [--logging LOGGING] 105 | [-a AMQP_ADDRESS] [-p AMQP_PORT] [-H AMQP_VHOST] 106 | [-U AMQP_USER] [-P AMQP_PASSWORD] 107 | [--docker-url DOCKER_URL] [--docker-tls] 108 | [--docker-ca DOCKER_CA_CERT] 109 | [--docker-cert DOCKER_CLIENT_CERT] 110 | [--docker-key DOCKER_CLIENT_KEY] [--docker-tls-strict] 111 | [--docker-registry DOCKER_REGISTRY] 112 | [--docker-ssl-registry] [--docker-publish] 113 | 114 | optional arguments: 115 | -h, --help show this help message and exit 116 | --config CONFIG Load configuration from file 117 | --gen-config Create example of the config_file.json 118 | 119 | Main options: 120 | --logging LOGGING Logging level 121 | 122 | RabbitMQ options: 123 | -a AMQP_ADDRESS, --address AMQP_ADDRESS 124 | RMQ host address 125 | -p AMQP_PORT, --port AMQP_PORT 126 | RMQ host port 127 | -H AMQP_VHOST, --vhost AMQP_VHOST 128 | RMQ virtual host 129 | -U AMQP_USER, --user AMQP_USER 130 | RMQ username 131 | -P AMQP_PASSWORD, --password AMQP_PASSWORD 132 | RMQ password 133 | 134 | Docker options: 135 | --docker-url DOCKER_URL 136 | Docker daemon url ["unix:///var/run/docker.sock"] 137 | --docker-tls Set when a docker daemon use TLS 138 | --docker-ca DOCKER_CA_CERT 139 | TLS certificate authority 140 | --docker-cert DOCKER_CLIENT_CERT 141 | TLS client certificate 142 | --docker-key DOCKER_CLIENT_KEY 143 | TLS client private key 144 | --docker-tls-strict Strict verification server certificate 145 | --docker-registry DOCKER_REGISTRY 146 | Set if you have a private registry 147 | --docker-ssl-registry 148 | The private registry use ssl 149 | --docker-publish Set if you want push images to registry 150 | 151 | And for mailer:: 152 | 153 | $ lumper mailer --help 154 | usage: lumper mailer [-h] [--config CONFIG] [--gen-config] [--logging LOGGING] 155 | [-a AMQP_ADDRESS] [-p AMQP_PORT] [-H AMQP_VHOST] 156 | [-U AMQP_USER] [-P AMQP_PASSWORD] [--smtp-host SMTP_HOST] 157 | [--smtp-port SMTP_PORT] [--smtp-user SMTP_USER] 158 | [--smtp-password SMTP_PASSWORD] [--smtp-tls] 159 | [--smtp-sender SMTP_SENDER] [--mail-map MAIL_MAP] 160 | [--admin-mail ADMIN_MAIL] 161 | 162 | optional arguments: 163 | -h, --help show this help message and exit 164 | --config CONFIG Load configuration from file 165 | --gen-config Create example of the config_file.json 166 | 167 | Main options: 168 | --logging LOGGING Logging level 169 | 170 | RabbitMQ options: 171 | -a AMQP_ADDRESS, --address AMQP_ADDRESS 172 | RMQ host address 173 | -p AMQP_PORT, --port AMQP_PORT 174 | RMQ host port 175 | -H AMQP_VHOST, --vhost AMQP_VHOST 176 | RMQ virtual host 177 | -U AMQP_USER, --user AMQP_USER 178 | RMQ username 179 | -P AMQP_PASSWORD, --password AMQP_PASSWORD 180 | RMQ password 181 | 182 | SMTP options: 183 | --smtp-host SMTP_HOST 184 | Server host 185 | --smtp-port SMTP_PORT 186 | Server port 187 | --smtp-user SMTP_USER 188 | Authentication username. Do auth if set. 189 | --smtp-password SMTP_PASSWORD 190 | Password. 191 | --smtp-tls Use TLS. 192 | --smtp-sender SMTP_SENDER 193 | Sender of messages [default: lumper@localhost] 194 | 195 | Delivery options: 196 | --mail-map MAIL_MAP github user to E-mail map json file with hash. 197 | --admin-mail ADMIN_MAIL 198 | admin email for unknown users [default: root@localhost] 199 | 200 | 201 | Config files 202 | ++++++++++++ 203 | 204 | You might generate and save configuration from the command line:: 205 | 206 | $ lumper mailer --gen-conf 207 | { 208 | "admin_mail": "root@localhost", 209 | "amqp_address": "localhost", 210 | "amqp_password": null, 211 | "amqp_port": 5672, 212 | "amqp_user": null, 213 | "amqp_vhost": "/", 214 | "logging": null, 215 | "mail_map": null, 216 | "smtp_host": "localhost", 217 | "smtp_password": null, 218 | "smtp_port": 25, 219 | "smtp_sender": "lumper@localhost", 220 | "smtp_tls": false, 221 | "smtp_user": null 222 | } 223 | 224 | And load it with --config option. E.g **lumper mailer --config /etc/lumper/mailer.json** 225 | 226 | And convert your command line to config-file:: 227 | 228 | $ lumper mailer --smtp-host mail.google.com --gen-conf 229 | { 230 | ... 231 | "smtp_host": "mail.google.com", 232 | ... 233 | } 234 | 235 | Notice: **Option --gen-conf must be defined in the end.** 236 | -------------------------------------------------------------------------------- /lumper/worker/build.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | 4 | import logging 5 | import json 6 | import os 7 | import shutil 8 | import traceback 9 | import git 10 | import re 11 | import requests 12 | 13 | from crew.worker import context, HandlerClass 14 | from uuid import uuid4 15 | from tempfile import gettempdir 16 | 17 | log = logging.getLogger("builder") 18 | 19 | 20 | class TemporaryFolder(object): 21 | 22 | def __init__(self): 23 | self._dir = os.path.join(gettempdir(), str(uuid4())) 24 | self._curdir = os.path.abspath(os.getcwd()) 25 | 26 | def __enter__(self): 27 | assert not os.path.exists(self._dir) 28 | 29 | log.debug('Making directory: "%s"', self._dir) 30 | os.makedirs(self._dir) 31 | 32 | log.debug('Changing directory: "%s"', self._dir) 33 | os.chdir(self._dir) 34 | return self._dir 35 | 36 | def __exit__(self, exc_type, exc_val, exc_tb): 37 | log.debug('Changing directory: "%s"', self._curdir) 38 | os.chdir(self._curdir) 39 | 40 | if os.path.exists(self._dir): 41 | log.debug('Deleting directory: "%s"', self._dir) 42 | shutil.rmtree(self._dir) 43 | 44 | 45 | class BuildHandler(HandlerClass): 46 | STREAM_EXPR = { 47 | "build_success": re.compile("^Successfully built\s+(?P\S+)\n?$") 48 | } 49 | 50 | def process(self): 51 | try: 52 | self.git = git.Git() 53 | self.docker = context.settings.docker 54 | 55 | with TemporaryFolder() as path: 56 | self.prepare(path) 57 | try: 58 | self.data.update({"id": self.build(path)}) 59 | except Exception as e: 60 | self.data.update({'error': e}) 61 | 62 | self.data.update({'build_log': self.build_log}) 63 | 64 | if self.data.get('status') and context.settings.options.docker_publish: 65 | self.push() 66 | 67 | return self.data 68 | except Exception as e: 69 | exc = Exception(repr(e)) 70 | exc._tb = traceback.format_exc(e) 71 | exc.log = getattr(self, 'build_log', []) 72 | return exc 73 | 74 | def push(self): 75 | registry = context.settings.options.docker_registry 76 | use_ssl = context.settings.options.docker_ssl_registry 77 | 78 | tag = self.data['tag'].lstrip("v") 79 | repo, name = ["".join(list(i)[::-1]).replace('/', '_') for i in self.data['name'].lower()[::-1].split('/', 1)][::-1] 80 | 81 | if not context.settings.options.docker_registry: 82 | log.warning("PUSHING TO PUBLIC DOCKER REGISTRY.") 83 | else: 84 | name = ("%s/%s" % (repo, name)).lower() 85 | repo = ("%s/%s" % (registry, name)).lower() 86 | 87 | try: 88 | self.docker.tag(self.data['id'], repo, tag) 89 | except Exception as e: 90 | log.error(e) 91 | 92 | log.info( 93 | "Preparing to push to the registry: %s://%s", 94 | 'https' if use_ssl else 'http', registry if registry else 'PUBLIC' 95 | ) 96 | 97 | response = self.docker.push( 98 | repo, tag=tag, 99 | insecure_registry=not use_ssl, 100 | stream=True 101 | ) 102 | 103 | self.build_log.append('') 104 | self.build_log.append( 105 | "Pushing into registry %s://%s" % ('https' if use_ssl else 'http', registry if registry else 'public') 106 | ) 107 | 108 | for line in response: 109 | chunk = json.loads(line) 110 | data = chunk.get("status") 111 | if chunk.get('error'): 112 | self.data['status'] = False 113 | details = chunk.get('errorDetail', {}).get('message') 114 | log.error(details) 115 | self.build_log.append(details) 116 | else: 117 | log.debug(data) 118 | 119 | if registry: 120 | url = "%s://%s" % ('https' if use_ssl else 'http', registry) 121 | try: 122 | log.debug("Trying to fetch image id") 123 | img_id = filter(lambda x: str(x[0]) == str(tag), requests.get("%s/v1/repositories/%s/tags" % (url, name)).json().items())[0][1] 124 | log.info('Pushing successful as %s', img_id) 125 | log.debug("Deleting tag: latest") 126 | resp = requests.delete("%s/v1/repositories/%s/tags/latest" % (url, name)) 127 | log.debug('%s', resp.json()) 128 | 129 | log.debug("Setting latest tag as %s", img_id) 130 | resp = requests.put( 131 | "%s/v1/repositories/%s/tags/latest" % (url, name), 132 | '"%s"' % img_id, 133 | headers={'Content-Type': 'application/json'} 134 | ) 135 | log.debug('%s', resp.json()) 136 | except Exception: 137 | self.build_log.append("ERROR: Can't fetch image id from registry \"%s\"" % url) 138 | 139 | def prepare(self, path): 140 | url = self.data['repo'] 141 | log.info('Cloning repo "%s" => "%s"', url, path) 142 | res = self.git.clone(url, path) 143 | log.debug("Cloning result: %s", res) 144 | 145 | commit_hash = self.data['commit'] 146 | log.info('Checkout commit "%s"', commit_hash) 147 | self.git.checkout(commit_hash) 148 | self.git.submodule("update", "--recursive", "--init") 149 | 150 | log.info('Updating submodules') 151 | for sm in git.Repo(path).submodules: 152 | log.info(' Updating submodule: "%s"', sm) 153 | sm.update(recursive=True, init=True) 154 | log.info(' Submodule "%s" updated', sm) 155 | 156 | self.restore_commit_times(path) 157 | 158 | log.info("Preparing complete") 159 | 160 | @staticmethod 161 | def restore_commit_times(path): 162 | 163 | log.info("Restoring file mtimes for path: %s", path) 164 | 165 | def walk(tree): 166 | ret = list() 167 | for i in tree: 168 | ret.append(i) 169 | if i.type == 'tree': 170 | ret.extend(walk(i)) 171 | return ret 172 | 173 | repo = git.Repo(path) 174 | 175 | def find_mtimes(repo): 176 | objects = walk(repo.tree()) 177 | t = repo.head.commit 178 | tt = t.traverse() 179 | ret = {} 180 | while objects: 181 | hashes = set(i.binsha for i in walk(t.tree)) 182 | # iterate over reversed list to be able to remove elements by index 183 | for n, i in reversed(list(enumerate(objects))): 184 | if i.binsha not in hashes: 185 | del objects[n] 186 | else: 187 | if i.path not in ret or t.authored_date < ret[i.path]: 188 | ret[i.path] = t.authored_date 189 | try: 190 | t = next(tt) 191 | except StopIteration: 192 | break 193 | return ret 194 | 195 | for i, mtime in find_mtimes(repo).items(): 196 | fname = os.path.join(path, i) 197 | log.debug("%s %s", mtime, fname) 198 | os.utime(fname.encode('utf-8'), (mtime, mtime)) 199 | 200 | for sm in repo.submodules: 201 | sm_internal_path = os.path.join(repo.git_dir, 'modules', sm.name) 202 | if os.path.exists(sm_internal_path): 203 | sm_repo = git.Repo(sm_internal_path) 204 | else: 205 | sm_repo = git.Repo(sm.path) 206 | 207 | for i, mtime in find_mtimes(sm_repo).items(): 208 | fname = os.path.join(path, sm.path, i) 209 | log.debug(u"%s %s", mtime, fname) 210 | os.utime(fname.encode('utf-8'), (mtime, mtime)) 211 | 212 | def build(self, path): 213 | log.debug("Start building...") 214 | tag = ("%s:%s" % (self.data['name'], self.data['tag'].lstrip("v"))).lower() 215 | log.debug("Selecting tag: %s", tag) 216 | 217 | self.build_log = [] 218 | log.debug('Building') 219 | try: 220 | for line in self.docker.build(path, rm=True, tag=tag): 221 | chunk = json.loads(line) 222 | stream = chunk.get("stream", "").rstrip("\n\r") 223 | if stream: 224 | success = self.STREAM_EXPR['build_success'].match(stream) 225 | self.build_log.append(stream) 226 | if success: 227 | self.data['status'] = True 228 | return success.groupdict()['id'] 229 | else: 230 | log.info(stream) 231 | 232 | elif chunk.get("error"): 233 | err = chunk['error'].strip("\n\r") 234 | log.error(err) 235 | self.build_log.append(err) 236 | log.error(chunk.get('error')) 237 | raise StandardError(chunk['error']) 238 | except Exception as e: 239 | log.exception(e) 240 | 241 | def resolve_image_id(self, image_id): 242 | self.docker.images() 243 | 244 | 245 | BuildHandler.bind("build") 246 | --------------------------------------------------------------------------------