├── lambdawebhook ├── __init__.py ├── hook.py └── sqs.py ├── requirements.txt ├── .travis.yml ├── test ├── data │ └── testevent.json └── unit_test.py ├── tox.ini ├── .gitignore ├── LICENSE ├── setup.py └── README.md /lambdawebhook/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | tox 2 | nose 3 | httpretty 4 | requests 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: python 3 | python: 4 | - "2.7" 5 | - "3.6" 6 | install: pip install tox-travis 7 | script: tox 8 | -------------------------------------------------------------------------------- /test/data/testevent.json: -------------------------------------------------------------------------------- 1 | { 2 | "x_github_delivery": "dafdsfs", 3 | "x_github_event": "push", 4 | "x_hub_signature": "sha1=426631b39a65784001d89e7818e1c609f6450872", 5 | "jenkins_url": "https://localhost/github-webhook/", 6 | "secret": "testsecret", 7 | "payload": { 8 | "ref": "refs/heads/master", 9 | "created": false, 10 | "deleted": false, 11 | "forced": false, 12 | "base_ref": null 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | # Tox (http://tox.testrun.org/) is a tool for running tests 2 | # in multiple virtualenvs. This configuration file will run the 3 | # test suite on all supported python versions. To use it, "pip install tox" 4 | # and then run "tox" from this directory. 5 | 6 | [tox] 7 | envlist = py27, py36, flake8 8 | 9 | [travis] 10 | python = 11 | 2.7: py27 12 | 3.6: py36, flake8 13 | 14 | [testenv] 15 | commands=nosetests 16 | setenv = 17 | PYTHONHASHSEED = 0 18 | deps= 19 | nose 20 | httpretty 21 | boto3 22 | requests 23 | 24 | [testenv:flake8] 25 | basepython = python3.6 26 | deps = 27 | flake8 28 | commands = 29 | flake8 lambdawebhook test --max-line-length=120 30 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | local_settings.py 55 | 56 | # Flask instance folder 57 | instance/ 58 | 59 | # Scrapy stuff: 60 | .scrapy 61 | 62 | # Sphinx documentation 63 | docs/_build/ 64 | 65 | # PyBuilder 66 | target/ 67 | 68 | # IPython Notebook 69 | .ipynb_checkpoints 70 | 71 | # pyenv 72 | .python-version 73 | 74 | # celery beat schedule file 75 | celerybeat-schedule 76 | 77 | # dotenv 78 | .env 79 | 80 | # Spyder project settings 81 | .spyderproject 82 | 83 | site-packages/ 84 | lambdawebhook.zip 85 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016, Pristine, Inc. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | * Neither the name of the {organization} nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | AWS Lambda function to receive GitHub webhooks from API gateway and relay them to an EC2 instance 3 | """ 4 | from setuptools import find_packages, setup 5 | 6 | dependencies = ['boto3', 'requests', 'httpretty'] 7 | 8 | setup( 9 | name='lambda-webhook', 10 | version='0.2.0', 11 | url='https://github.com/pristineio/lambda-webhook', 12 | license='BSD', 13 | author='John Schwinghammer', 14 | author_email='john+githubsource@pristine.io', 15 | description='AWS Lambda function to receive GitHub webhooks from API gateway and relay them to an EC2 instance', 16 | long_description=__doc__, 17 | packages=find_packages(exclude=['tests']), 18 | include_package_data=True, 19 | zip_safe=False, 20 | platforms='any', 21 | install_requires=dependencies, 22 | entry_points={ 23 | 'console_scripts': [ 24 | 'lambda-webhook-sqs=lambdawebhook.sqs:cmd', 25 | ], 26 | }, 27 | classifiers=[ 28 | # As from http://pypi.python.org/pypi?%3Aaction=list_classifiers 29 | # 'Development Status :: 1 - Planning', 30 | # 'Development Status :: 2 - Pre-Alpha', 31 | # 'Development Status :: 3 - Alpha', 32 | 'Development Status :: 4 - Beta', 33 | # 'Development Status :: 5 - Production/Stable', 34 | # 'Development Status :: 6 - Mature', 35 | # 'Development Status :: 7 - Inactive', 36 | 'Environment :: Other Environment', 37 | 'Intended Audience :: Developers', 38 | 'License :: OSI Approved :: BSD License', 39 | 'Operating System :: POSIX :: Linux', 40 | 'Operating System :: MacOS', 41 | 'Programming Language :: Python', 42 | 'Programming Language :: Python :: 2', 43 | 'Topic :: Software Development :: Build Tools', 44 | ] 45 | ) 46 | -------------------------------------------------------------------------------- /test/unit_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import lambdawebhook.hook 3 | import os 4 | import json 5 | import httpretty 6 | import base64 7 | from requests import HTTPError 8 | 9 | 10 | def load_test_event(): 11 | mypath = os.path.dirname(os.path.abspath(__file__)) 12 | with open(os.path.join(mypath, 'data/testevent.json'), 'r') as eventfile: 13 | githubevent = json.load(eventfile) 14 | githubevent['payload'] = base64.b64encode(json.dumps(githubevent['payload'], sort_keys=True).encode('ascii')) 15 | return githubevent 16 | 17 | 18 | class VerifySignatureTestCase(unittest.TestCase): 19 | def runTest(self): 20 | githubevent = load_test_event() 21 | 22 | # Match the conversion that happens in the beginning of lambda_handler() 23 | githubevent['payload'] = base64.b64decode(githubevent['payload']) 24 | 25 | # This signature is missing the 'sha1=' prefix and will fail validation 26 | self.assertFalse(lambdawebhook.hook.verify_signature(githubevent['secret'], 27 | '952548c8f23303f4925411b09b0c5d0c13d0cfb5', 28 | githubevent['payload'])) 29 | 30 | # This signature should validate the payload 31 | self.assertTrue(lambdawebhook.hook.verify_signature(githubevent['secret'], 32 | githubevent['x_hub_signature'], 33 | githubevent['payload'])) 34 | 35 | 36 | class LambdaHandlerTestCase(unittest.TestCase): 37 | @httpretty.activate 38 | def runTest(self): 39 | invalidevent = load_test_event() 40 | invalidevent['secret'] = 'invalidsecret' 41 | self.assertRaises(HTTPError, lambdawebhook.hook.lambda_handler, invalidevent, {}) 42 | 43 | # Check return codes 44 | httpretty.register_uri(httpretty.POST, 'https://localhost/github-webhook/', 45 | status=200) 46 | githubevent = load_test_event() 47 | self.assertIsNone(lambdawebhook.hook.lambda_handler(githubevent, {})) 48 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # lambda-webhook 2 | 3 | An AWS Lambda function to receive GitHub webhooks from API gateway and relay them to an EC2 instance. 4 | 5 | 6 | ## Usage 7 | 8 | 1. Create a Security Group for the Lambda function 9 | - Inbound: None 10 | - Outbound: Only allow HTTPS/HTTP to the receiving instance 11 | 12 | 1. Create a Lambda function: 13 | - Runtime: Python 2.7 14 | - Handler: hook.lambda_handler 15 | - Role: Basic With VPC 16 | - Memory: 128MB 17 | - Timeout: 30sec 18 | - VPC: The VPC of the receiving instance 19 | - Subnets: At least 2 private subnets within the VPC 20 | - Security Groups: The Security Group configured previously 21 | 22 | 1. Install dependencies locally: 23 | 24 | `$ pip install -r requirements.txt -t lambdawebhook/lib/` 25 | 26 | 1. Create a ZIP archive of the `lambdawebhook` directory: 27 | 28 | `$ cd lambdawebhook` 29 | `$ zip -r lambdawebhook.zip *` 30 | 31 | 1. Upload the zipped code to the Lambda function created previously 32 | 33 | 1. Create an API in API gateway 34 | 35 | 1. Create a resource for `/github` 36 | 37 | 1. Create a POST method for `/github` 38 | - Integration type: Lambda Function 39 | - Lambda Region: The region of the Lamba function created previously 40 | - Lambda Function: The Lambda function created previously 41 | 42 | 1. Integration Request -> Mapping Templates: 43 | - Content-Type: `application/json` 44 | - Mapping Template (replace `secret` and `jenkins_url` as appropriate for your configuration): 45 | 46 | { 47 | "x_github_delivery": "$util.escapeJavaScript($input.params().header.get('X-GitHub-Delivery'))", 48 | "x_github_event": "$util.escapeJavaScript($input.params().header.get('X-GitHub-Event'))", 49 | "x_hub_signature": "$util.escapeJavaScript($input.params().header.get('X-Hub-Signature'))", 50 | "secret": "some_secret", 51 | "jenkins_url": "https://jenkins/github-webhook/", 52 | "payload": "$util.base64Encode($input.body)" 53 | } 54 | 55 | 1. Method Response: 56 | - HTTP Status: `400` 57 | 58 | 1. Integration Response -> Add integration response: 59 | - Lambda Error Regex: `400 Client Error: Bad Request` 60 | - Method response status: `400` 61 | - Mapping Templates: 62 | - Content-Type: `application/json` 63 | - Template: 64 | 65 | { 66 | "message": $input.json('$.errorMessage') 67 | } 68 | 69 | 1. Deploy API 70 | 71 | 1. Configure the webhook and secret for the GitHub repository using the API URL provided in the previous step, and `secret` set above. 72 | 73 | 1. Test by pushing some code to the repository. 74 | 75 | ## Development 76 | 77 | Linting (flake8) and testing (unittest) are executed using `tox` in the root directory of this repository: 78 | 79 | $ pip install tox 80 | $ tox 81 | -------------------------------------------------------------------------------- /lambdawebhook/hook.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import print_function 3 | import os 4 | import sys 5 | import hashlib 6 | import hmac 7 | import json 8 | import base64 9 | import time 10 | 11 | import boto3 12 | 13 | # Add the lib directory to the path for Lambda to load our libs 14 | sys.path.append(os.path.join(os.path.dirname(__file__), 'lib')) 15 | from requests import Session, HTTPError # NOQA 16 | from requests.packages.urllib3.util.retry import Retry # NOQA 17 | from requests.adapters import HTTPAdapter # NOQA 18 | 19 | 20 | class StaticRetry(Retry): 21 | def sleep(self): 22 | time.sleep(3) 23 | 24 | 25 | def verify_signature(secret, signature, payload): 26 | computed_hash = hmac.new(secret.encode('ascii'), payload, hashlib.sha1) 27 | computed_signature = '='.join(['sha1', computed_hash.hexdigest()]) 28 | return hmac.compare_digest(computed_signature.encode('ascii'), signature.encode('ascii')) 29 | 30 | 31 | def relay_github(event, requests_session): 32 | verified = verify_signature(event['secret'], 33 | event['x_hub_signature'], 34 | event['payload']) 35 | print('Signature verified: {}'.format(verified)) 36 | 37 | if verified: 38 | response = requests_session.post(event['jenkins_url'], 39 | headers={ 40 | 'Content-Type': 'application/json', 41 | 'X-GitHub-Delivery': event['x_github_delivery'], 42 | 'X-GitHub-Event': event['x_github_event'], 43 | 'X-Hub-Signature': event['x_hub_signature'] 44 | }, 45 | data=event['payload']) 46 | response.raise_for_status() 47 | else: 48 | raise HTTPError('400 Client Error: Bad Request') 49 | 50 | 51 | def relay_quay(event, requests_session): 52 | response = requests_session.post(event['jenkins_url'], 53 | headers={ 54 | 'Content-Type': 'application/json' 55 | }, 56 | data=event['payload']) 57 | response.raise_for_status() 58 | 59 | 60 | def relay_sqs(event): 61 | sqs_queue = event.get('sqs_queue') 62 | sqs_region = event.get('sqs_region', 'us-west-2') 63 | assert sqs_queue 64 | 65 | sqs_obj = dict( 66 | timestamp=int(time.time()), 67 | jenkins_url=event.get('jenkins_url'), 68 | headers={ 69 | 'Content-Type': 'application/json', 70 | 'X-GitHub-Delivery': event['x_github_delivery'], 71 | 'X-GitHub-Event': event['x_github_event'], 72 | 'X-Hub-Signature': event['x_hub_signature'] 73 | }, 74 | data=event['payload'], 75 | ) 76 | 77 | sqs = boto3.client('sqs', sqs_region) 78 | queue_url = sqs.get_queue_url(QueueName=sqs_queue)['QueueUrl'] 79 | sqs.send_message( 80 | QueueUrl=queue_url, 81 | MessageBody=json.dumps(sqs_obj).decode(), 82 | ) 83 | 84 | 85 | def lambda_handler(event, context): 86 | print('Webhook received') 87 | event['payload'] = base64.b64decode(event['payload']) 88 | requests_session = Session() 89 | retries = StaticRetry(total=40) 90 | requests_session.mount(event['jenkins_url'], HTTPAdapter(max_retries=retries)) 91 | 92 | if event.get('service') == 'quay': 93 | relay_quay(event, requests_session) 94 | if event.get('service') == 'sqs': 95 | relay_sqs(event) 96 | else: 97 | relay_github(event, requests_session) 98 | print('Successfully relayed payload') 99 | 100 | 101 | if __name__ == '__main__': 102 | pass 103 | -------------------------------------------------------------------------------- /lambdawebhook/sqs.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import sys 4 | import threading 5 | import time 6 | 7 | import boto3 8 | import requests 9 | from requests.exceptions import ConnectionError 10 | 11 | 12 | class SqsReceiver(object): 13 | def __init__(self, options): 14 | self.options = options 15 | self.debug = options.debug 16 | 17 | def deliver_message(self, message): 18 | body = json.loads(message['Body']) 19 | url = self.options.webhook_url or body.get('jenkins_url') 20 | if self.debug: 21 | print("Posting message to webhook.") 22 | for t in [0, 1, 2, 4, 8, 16]: 23 | time.sleep(t) 24 | try: 25 | response = requests.post(url, headers=body.get('headers'), data=body.get('data')) 26 | if self.debug: 27 | print("Post result: {}".format(repr(response))) 28 | break 29 | except ConnectionError as e: 30 | print("ConnectionError: {}".format(e)) 31 | 32 | def run(self): 33 | sqs = boto3.client('sqs', self.options.region) 34 | queue_url = sqs.get_queue_url(QueueName=self.options.queue).get('QueueUrl') 35 | if self.debug: 36 | print("Using queue URL: {}".format(queue_url)) 37 | while True: 38 | receive = sqs.receive_message( 39 | QueueUrl=queue_url, 40 | MaxNumberOfMessages=10, 41 | VisibilityTimeout=300, 42 | WaitTimeSeconds=self.options.wait_time_seconds, 43 | ) 44 | message_list = receive.get('Messages', []) 45 | if message_list: 46 | deliver_message_threads = list() 47 | receipt_handles = list() 48 | for message in message_list: 49 | t = threading.Thread(target=self.deliver_message, args=[message]) 50 | deliver_message_threads.append(t) 51 | t.start() 52 | receipt_handles.append( 53 | {'Id': message['MessageId'], 'ReceiptHandle': message['ReceiptHandle']} 54 | ) 55 | 56 | for t in deliver_message_threads: 57 | t.join(timeout=120) 58 | 59 | result = sqs.delete_message_batch(QueueUrl=queue_url, Entries=receipt_handles) 60 | if self.debug: 61 | print("delete_message_batch() -> {}".format(repr(result))) 62 | 63 | if self.debug: 64 | print("Webhooks found on previous attempt. Checking for more.") 65 | 66 | elif self.options.run_forever: 67 | time.sleep(30) 68 | else: 69 | if self.debug: 70 | print("No messages received. Exiting.") 71 | break 72 | 73 | 74 | def cmd(): 75 | parser = argparse.ArgumentParser( 76 | description="Lambda webhook SQS relay. Receives and delivers webhooks placed in SQS by lambda-webhook.") 77 | 78 | parser.add_argument('queue', type=str, help='SQS Queue name') 79 | parser.add_argument('--region', type=str, default='us-west-2', help='SQS Queue region') 80 | parser.add_argument( 81 | '--webhook-url', type=str, 82 | help='Override webhook receiver URL. URL may be provided in queue message.' 83 | ) 84 | parser.add_argument('--wait-time-seconds', type=int, default=15, 85 | help='SQS receive_message WaitTimeSeconds') 86 | parser.add_argument('--run-forever', action='store_true', default=False, 87 | help="Run until process is interrupted. ") 88 | parser.add_argument('-d', dest='debug', action='store_true', default=False, help='Debug') 89 | 90 | options = parser.parse_args(sys.argv[1:]) 91 | if options.debug: 92 | print("Argparse options: {}".format(repr(options))) 93 | 94 | receiver = SqsReceiver(options) 95 | receiver.run() 96 | 97 | 98 | if __name__ == '__main__': 99 | cmd() 100 | --------------------------------------------------------------------------------