├── rest_hooks_delivery ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ └── retry_failed_hooks.py ├── migrations │ ├── __init__.py │ └── 0001_initial.py ├── __init__.py ├── apps.py ├── models.py ├── admin.py ├── templates │ └── admin │ │ └── rest_hooks_delivery │ │ └── failedhook │ │ └── change_form.html └── deliverers.py ├── CHANGELOG.txt ├── DESCRIPTION ├── MANIFEST.in ├── .gitignore ├── setup.py ├── LICENSE └── README.rst /rest_hooks_delivery/management/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /rest_hooks_delivery/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /rest_hooks_delivery/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /CHANGELOG.txt: -------------------------------------------------------------------------------- 1 | === 0.1.0 === 2 | Initial release 3 | 4 | * simple retry deliverer 5 | -------------------------------------------------------------------------------- /DESCRIPTION: -------------------------------------------------------------------------------- 1 | Various webhook deliverers for django-rest-hooks and django-rest-hooks-ng. 2 | -------------------------------------------------------------------------------- /rest_hooks_delivery/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # vim: ft=python:sw=4:ts=4:sts=4:et: 3 | 4 | VERSION = '0.2.1' 5 | default_app_config = 'rest_hooks_delivery.apps.RestHooksDeliveryConfig' 6 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include AUTHORS 2 | include LICENSE 3 | include DESCRIPTION 4 | include CHANGELOG.txt 5 | include README.rst 6 | recursive-include rest_hooks_delivery/templates * 7 | recursive-include rest_hooks_delivery/static * 8 | -------------------------------------------------------------------------------- /rest_hooks_delivery/apps.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # vim: ft=python:sw=4:ts=4:sts=4:et: 3 | 4 | from django.apps import AppConfig 5 | 6 | 7 | class RestHooksDeliveryConfig(AppConfig): 8 | name = 'rest_hooks_delivery' 9 | verbose_name = "Rest Hooks Delivery" 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | *.egg-info/ 23 | .installed.cfg 24 | *.egg 25 | 26 | # PyInstaller 27 | # Usually these files are written by a python script from a template 28 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 29 | *.manifest 30 | *.spec 31 | 32 | # Installer logs 33 | pip-log.txt 34 | pip-delete-this-directory.txt 35 | 36 | # Unit test / coverage reports 37 | htmlcov/ 38 | .tox/ 39 | .coverage 40 | .coverage.* 41 | .cache 42 | nosetests.xml 43 | coverage.xml 44 | 45 | # Translations 46 | *.mo 47 | *.pot 48 | 49 | # setuptools MANIFEST 50 | MANIFEST 51 | 52 | # Django stuff: 53 | *.log 54 | 55 | # Sphinx documentation 56 | docs/_build/ 57 | 58 | # PyBuilder 59 | target/ 60 | 61 | db.sqlite 62 | 63 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | version = __import__('rest_hooks_delivery').VERSION 4 | 5 | setup( 6 | name='django-rest-hooks-delivery', 7 | description=('Various webhook deliverers for django-rest-hooks and ' 8 | 'django-rest-hooks-ng.'), 9 | version=version, 10 | author='PressLabs', 11 | author_email='ping@presslabs.com', 12 | url='http://github.com/PressLabs/django-rest-hooks-delivery', 13 | install_requires=['Django>=1.7', 'requests', 'django-jsonfield'], 14 | packages=['rest_hooks_delivery'], 15 | include_package_data=True, 16 | classifiers = ['Development Status :: 4 - Beta', 17 | 'Environment :: Web Environment', 18 | 'Framework :: Django', 19 | 'Intended Audience :: Developers', 20 | 'License :: OSI Approved :: MIT License', 21 | 'Operating System :: OS Independent', 22 | 'Programming Language :: Python', 23 | 'Topic :: Utilities'], 24 | ) 25 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | Copyright (c) 2015 PressLabs 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | this software and associated documentation files (the "Software"), to deal in 6 | the Software without restriction, including without limitation the rights to 7 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 8 | of the Software, and to permit persons to whom the Software is furnished to do 9 | so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in all 12 | copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 20 | SOFTWARE. 21 | -------------------------------------------------------------------------------- /rest_hooks_delivery/management/commands/retry_failed_hooks.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # vim: ft=python:sw=4:ts=4:sts=4:et: 3 | from django.conf import settings 4 | from django.core.management.base import BaseCommand, CommandError 5 | from django.db.models import F 6 | from rest_hooks.utils import get_module 7 | 8 | from rest_hooks_delivery.models import FailedHook 9 | 10 | 11 | class Command(BaseCommand): 12 | def handle(self, *args, **options): 13 | deliverer = getattr(settings, 'HOOK_DELIVERER', None) 14 | if not deliverer: 15 | raise CommandError("No custom HOOK_DELIVERER set in settings.py") 16 | return 5 17 | deliverer = get_module(deliverer) 18 | count = 0 19 | for hook in FailedHook.objects.filter(target=F('hook__target'), 20 | event=F('hook__event'), 21 | user_id=F('hook__user_id')): 22 | deliverer(hook.target, hook.payload, hook=hook.hook, 23 | cleanup=True) 24 | count += 1 25 | self.stdout.write("Retried %d failed webhooks" % count) 26 | -------------------------------------------------------------------------------- /rest_hooks_delivery/models.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # vim: ft=python:sw=4:ts=4:sts=4:et: 3 | from django.conf import settings 4 | from django.db import models 5 | 6 | HOOK_EVENTS = getattr(settings, 'HOOK_EVENTS', None) 7 | if HOOK_EVENTS is None: 8 | raise Exception('You need to define settings.HOOK_EVENTS!') 9 | AUTH_USER_MODEL = getattr(settings, 'AUTH_USER_MODEL', 'auth.User') 10 | 11 | 12 | class FailedHook(models.Model): 13 | last_retry = models.DateTimeField(auto_now=True, editable=False, 14 | db_index=True) 15 | target = models.URLField('Original target URL', max_length=255, 16 | editable=False, db_index=True) 17 | event = models.CharField('Event', max_length=64, db_index=True, 18 | choices=[(e, e) for e in 19 | sorted(HOOK_EVENTS.keys())], 20 | editable=False) 21 | user = models.ForeignKey(AUTH_USER_MODEL, editable=False) 22 | payload = models.TextField(editable=False) 23 | response_headers = models.TextField(editable=False, max_length=65535) 24 | response_body = models.TextField(editable=False, max_length=65535) 25 | last_status = models.PositiveSmallIntegerField(editable=False, 26 | db_index=True) 27 | retries = models.PositiveIntegerField(editable=False, db_index=True, 28 | default=1) 29 | 30 | hook = models.ForeignKey('rest_hooks.Hook', editable=False) 31 | 32 | def __unicode__(self): 33 | return u'%s [%d]' % (self.target, self.last_status) 34 | 35 | class Meta: 36 | ordering = ('-last_retry',) 37 | unique_together = (('target', 'event', 'user', 'hook'),) 38 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Django REST Hooks Delivery 2 | ========================== 3 | 4 | Various deliverers for `django rest hooks 5 | `_ and `django rest hooks ng 6 | `_. 7 | 8 | Installation 9 | ------------ 10 | 11 | To get the latest stable release from PyPi 12 | 13 | .. code-block:: bash 14 | 15 | pip install django-rest-hooks-delivery 16 | 17 | To get the latest commit from GitHub 18 | 19 | .. code-block:: bash 20 | 21 | pip install -e git+git://github.com/PressLabs/django-rest-hooks-delivery.git#egg=rest_hooks_delivery 22 | 23 | Add ``rest_hooks_delivery`` to your ``INSTALLED_APPS`` 24 | 25 | .. code-block:: python 26 | 27 | INSTALLED_APPS = ( 28 | ..., 29 | 'rest_hooks_delivery', 30 | ) 31 | 32 | Don't forget to migrate your database 33 | 34 | .. code-block:: bash 35 | 36 | ./manage.py migrate rest_hooks_delivery # if you are using django > 1.7 37 | 38 | ./manage.py syncdb rest_hooks_delivery # if you are using django < 1.7 39 | 40 | 41 | Usage 42 | ----- 43 | 44 | Make sure you have added :code:`rest_hooks_delivery` to the list of 45 | :code:`INSTALLED_APPS` before :code:`django.contrib.admin` and that you have 46 | set :code:`HOOK_DELIVERER` to one of the available deliverers. Currently only 47 | :code:`rest_hooks_delivery.deliverers.retry` is available. 48 | 49 | .. code-block:: python 50 | 51 | ### settings.py ### 52 | 53 | INSTALLED_APPS = [ 54 | ... 55 | 'rest_hooks_delivery', 56 | 'django.contrib.admin', 57 | ] 58 | 59 | HOOK_DELIVERER = 'rest_hooks_delivery.deliverers.retry' 60 | 61 | It also provides a management command useful for retrying failed hooks. 62 | 63 | .. code-block:: bash 64 | 65 | ./manage.py retry_failed_hooks 66 | 67 | -------------------------------------------------------------------------------- /rest_hooks_delivery/admin.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # vim: ft=python:sw=4:ts=4:sts=4:et: 3 | from django.conf import settings 4 | from django.contrib import admin, messages 5 | from django.db.models import F 6 | from rest_hooks.utils import get_module 7 | 8 | from rest_hooks_delivery.models import FailedHook 9 | 10 | 11 | def retry_hook(modeladmin, request, queryset): 12 | deliverer = getattr(settings, 'HOOK_DELIVERER', None) 13 | if not deliverer: 14 | modeladmin.message_user(request, "No custom HOOK_DELIVERER set in " 15 | "settings.py", messages.ERROR) 16 | return 17 | 18 | deliverer = get_module(deliverer) 19 | count = 0 20 | for hook in queryset.filter(target=F('hook__target'), 21 | event=F('hook__event'), 22 | user_id=F('hook__user_id')): 23 | deliverer(hook.target, hook.payload, hook=hook.hook, 24 | cleanup=True) 25 | count += 1 26 | modeladmin.message_user(request, "Retried %d failed webhooks" % count) 27 | retry_hook.short_description = "Retry selected hooks" 28 | 29 | 30 | class FailedHookAdmin(admin.ModelAdmin): 31 | list_display = ('__unicode__', 'event', 'user', 'last_status', 32 | 'last_retry', 'retries', 'valid', 'hook') 33 | readonly_fields = ('target', 'event', 'user', 'last_status', 'last_retry', 34 | 'retries', 'payload', 'response_headers', 35 | 'response_body') 36 | 37 | actions = (retry_hook, ) 38 | 39 | def has_add_permission(self, request): 40 | return False 41 | 42 | def valid(self, obj): 43 | return (obj.target == obj.hook.target and obj.event == obj.hook.event 44 | and obj.user.pk == obj.hook.user.pk) 45 | valid.boolean = True 46 | 47 | 48 | admin.site.register(FailedHook, FailedHookAdmin) 49 | -------------------------------------------------------------------------------- /rest_hooks_delivery/templates/admin/rest_hooks_delivery/failedhook/change_form.html: -------------------------------------------------------------------------------- 1 | {% extends "admin/change_form.html" %} 2 | {% block field_sets %} 3 | {% for fieldset in adminform %} 4 |
5 | {% if fieldset.name %}

{{ fieldset.name }}

{% endif %} 6 | {% if fieldset.description %} 7 |
{{ fieldset.description|safe }}
8 | {% endif %} 9 | {% for line in fieldset %} 10 |
11 | {% if line.fields|length_is:'1' %}{{ line.errors }}{% endif %} 12 | {% for field in line %} 13 | 14 | {% if not line.fields|length_is:'1' and not field.is_readonly %}{{ field.errors }}{% endif %} 15 | {% if field.is_checkbox %} 16 | {{ field.field }}{{ field.label_tag }} 17 | {% else %} 18 | {{ field.label_tag }} 19 | {% if field.is_readonly %} 20 |
21 | 
{{ field.contents }}
22 |
23 | {% else %} 24 | {{ field.field }} 25 | {% endif %} 26 | {% endif %} 27 | {% if field.field.help_text %} 28 |

{{ field.field.help_text|safe }}

29 | {% endif %} 30 |
31 | {% endfor %} 32 | 33 | {% endfor %} 34 |
35 | {% endfor %} 36 | {% endblock %} 37 | -------------------------------------------------------------------------------- /rest_hooks_delivery/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | from django.conf import settings 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('rest_hooks', '0001_initial'), 12 | migrations.swappable_dependency(settings.AUTH_USER_MODEL), 13 | ] 14 | 15 | operations = [ 16 | migrations.CreateModel( 17 | name='FailedHook', 18 | fields=[ 19 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 20 | ('last_retry', models.DateTimeField(auto_now=True, db_index=True)), 21 | ('target', models.URLField(verbose_name=b'Original target URL', max_length=255, editable=False, db_index=True)), 22 | ('event', models.CharField(db_index=True, verbose_name=b'Event', max_length=64, editable=False, choices=[(b'customer.created', b'customer.created'), (b'customer.deleted', b'customer.deleted'), (b'customer.updated', b'customer.updated'), (b'invoice.created', b'invoice.created'), (b'invoice.deleted', b'invoice.deleted'), (b'invoice.updated', b'invoice.updated'), (b'plan.created', b'plan.created'), (b'plan.deleted', b'plan.deleted'), (b'plan.updated', b'plan.updated'), (b'proforma.created', b'proforma.created'), (b'proforma.deleted', b'proforma.deleted'), (b'proforma.updated', b'proforma.updated'), (b'provider.created', b'provider.created'), (b'provider.deleted', b'provider.deleted'), (b'provider.updated', b'provider.updated'), (b'subscription.created', b'subscription.created'), (b'subscription.deleted', b'subscription.deleted'), (b'subscription.updated', b'subscription.updated')])), 23 | ('payload', models.TextField(editable=False)), 24 | ('response_headers', models.TextField(max_length=65535, editable=False)), 25 | ('response_body', models.TextField(max_length=65535, editable=False)), 26 | ('last_status', models.PositiveSmallIntegerField(editable=False, db_index=True)), 27 | ('retries', models.PositiveIntegerField(default=1, editable=False, db_index=True)), 28 | ('hook', models.ForeignKey(editable=False, to='rest_hooks.Hook')), 29 | ('user', models.ForeignKey(editable=False, to=settings.AUTH_USER_MODEL)), 30 | ], 31 | options={ 32 | 'ordering': ('-last_retry',), 33 | }, 34 | ), 35 | migrations.AlterUniqueTogether( 36 | name='failedhook', 37 | unique_together=set([('target', 'event', 'user', 'hook')]), 38 | ), 39 | ] 40 | -------------------------------------------------------------------------------- /rest_hooks_delivery/deliverers.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # vim: ft=python:sw=4:ts=4:sts=4:et: 3 | import collections 4 | import json 5 | import threading 6 | 7 | import requests 8 | 9 | from django.db.models import F 10 | 11 | from rest_hooks_delivery.models import FailedHook 12 | 13 | 14 | class FlushThread(threading.Thread): 15 | def __init__(self, client): 16 | threading.Thread.__init__(self) 17 | self.client = client 18 | 19 | def run(self): 20 | self.client.sync_flush() 21 | 22 | 23 | class Client(object): 24 | """ 25 | Manages a simple pool of threads to flush the queue of requests. 26 | """ 27 | def __init__(self, num_threads=3): 28 | self.queue = collections.deque() 29 | 30 | self.flush_lock = threading.Lock() 31 | self.num_threads = num_threads 32 | self.flush_threads = [FlushThread(self) for _ in 33 | range(self.num_threads)] 34 | self.total_sent = 0 35 | 36 | def enqueue(self, method, *args, **kwargs): 37 | self.queue.append((method, args, kwargs)) 38 | self.refresh_threads() 39 | 40 | def get(self, *args, **kwargs): 41 | self.enqueue('get', *args, **kwargs) 42 | 43 | def post(self, *args, **kwargs): 44 | self.enqueue('post', *args, **kwargs) 45 | 46 | def put(self, *args, **kwargs): 47 | self.enqueue('put', *args, **kwargs) 48 | 49 | def delete(self, *args, **kwargs): 50 | self.enqueue('delete', *args, **kwargs) 51 | 52 | def refresh_threads(self): 53 | with self.flush_lock: 54 | # refresh if there are jobs to do and no threads are alive 55 | if len(self.queue) > 0: 56 | to_refresh = [index for index, thread in 57 | enumerate(self.flush_threads) 58 | if not thread.is_alive()] 59 | for index in to_refresh: 60 | self.flush_threads[index] = FlushThread(self) 61 | self.flush_threads[index].start() 62 | 63 | def sync_flush(self): 64 | while len(self.queue) > 0: 65 | method, args, kwargs = self.queue.pop() 66 | hook_id = kwargs.pop('_hook_id') 67 | hook_event = kwargs.pop('_hook_event') 68 | hook_user_id = kwargs.pop('_hook_user_id') 69 | cleanup = kwargs.pop('_cleanup') 70 | r = getattr(requests, method)(*args, **kwargs) 71 | payload = kwargs.get('data', '{}') 72 | if r.status_code > 299: 73 | try: 74 | failed_hook = FailedHook.objects.get(target=r.request.url, 75 | event=hook_event, 76 | user_id=hook_user_id, 77 | hook_id=hook_id) 78 | failed_hook.payload = payload 79 | failed_hook.response_headers = {k: r.headers[k] for k in 80 | r.headers.iterkeys()} 81 | failed_hook.response_body = r.content 82 | failed_hook.last_status = r.status_code 83 | failed_hook.retries = F('retries') + 1 84 | failed_hook.save() 85 | 86 | except FailedHook.DoesNotExist: 87 | FailedHook.objects.create( 88 | target=r.request.url, 89 | payload=payload, 90 | response_headers={k: r.headers[k] 91 | for k in r.headers.iterkeys()}, 92 | response_body=r.content, 93 | last_status=r.status_code, 94 | event=hook_event, 95 | user_id=hook_user_id, 96 | hook_id=hook_id 97 | ) 98 | elif cleanup: 99 | FailedHook.objects.filter(target=r.request.url, 100 | event=F('hook__event'), 101 | user_id=F('hook__user_id'), 102 | hook_id=hook_id).delete() 103 | 104 | self.total_sent += 1 105 | 106 | 107 | client = Client() 108 | 109 | 110 | def retry(target, payload, instance=None, hook=None, cleanup=False, **kwargs): 111 | client.post( 112 | url=target, 113 | data=json.dumps(payload) if not isinstance(payload, basestring) else 114 | payload, 115 | headers={'Content-Type': 'application/json'}, 116 | _hook_id=hook.pk, 117 | _hook_event=hook.event, 118 | _hook_user_id=hook.user.pk, 119 | _cleanup=cleanup 120 | ) 121 | --------------------------------------------------------------------------------