├── hookah ├── tests │ ├── __init__.py │ ├── test_storage.py │ └── test_queue.py ├── __init__.py ├── web.py ├── storage.py ├── queue.py └── dispatch.py ├── TODO ├── .gitignore ├── setup.py ├── bin └── hookah ├── twisted └── plugins │ └── hookah_plugin.py ├── LICENSE └── README.md /hookah/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | -tests 2 | -response callbacks 3 | -logging 4 | -config 5 | 6 | -channel management 7 | -egg dist 8 | 9 | 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *# 2 | *~ 3 | *.pyc 4 | _trial_temp 5 | *.egg-info 6 | twistd.log 7 | twistd.pid 8 | build 9 | twisted/plugins/dropin.cache 10 | -------------------------------------------------------------------------------- /hookah/__init__.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | from twisted.internet import reactor 3 | from twisted.web.server import Site 4 | from hookah.web import HookahResource 5 | from hookah import queue 6 | 7 | config = {} 8 | 9 | def configure(config_file): 10 | config = yaml.load(open(config_file, 'r').read()) 11 | queue.instance.startConsumers(1) 12 | reactor.listenTCP(config['port'], Site(HookahResource.setup())) 13 | 14 | 15 | class HookahRequest(object): 16 | def __init__(self, url, headers, body): 17 | self.url = url 18 | self.headers = headers 19 | self.body = body 20 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name = "hookah", 5 | version="0.1.0", 6 | description="Scalable async HTTP request dispatcher", 7 | 8 | author="Jeff Lindsay", 9 | author_email="progrium@gmail.com", 10 | url="http://github.com/progrium/hookah/tree/master", 11 | download_url="http://github.com/progrium/hookah/tarball/master", 12 | classifiers=[ 13 | ], 14 | packages=['hookah'], 15 | data_files=[('twisted/plugins', ['twisted/plugins/hookah_plugin.py'])], 16 | scripts=['bin/hookah'], 17 | install_requires = [ 18 | 'pyyaml>=3', 19 | ], 20 | ) 21 | -------------------------------------------------------------------------------- /bin/hookah: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | from twisted.internet import reactor 5 | 6 | from twisted.python import log 7 | from optparse import OptionParser 8 | 9 | import hookah 10 | 11 | parser = OptionParser() 12 | parser.add_option('--config', '-c', 13 | help="Config file for Hookah", 14 | default='shisha.conf', type='str') 15 | parser.add_option('--verbose', '-v', help="Show http activity", 16 | action="store_true") 17 | opts, args = parser.parse_args() 18 | 19 | if opts.verbose: 20 | log.startLogging(sys.stdout) 21 | 22 | hookah.configure(opts.config) 23 | reactor.run() 24 | 25 | -------------------------------------------------------------------------------- /hookah/web.py: -------------------------------------------------------------------------------- 1 | from twisted.python.util import sibpath 2 | from twisted.web import client, error, http, static 3 | from twisted.web.resource import Resource 4 | from twisted.internet import task 5 | 6 | 7 | 8 | 9 | class HookahResource(Resource): 10 | isLeaf = False 11 | 12 | def getChild(self, name, request): 13 | if name == '': 14 | return self 15 | return Resource.getChild(self, name, request) 16 | 17 | def render(self, request): 18 | path = '/'.join(request.prepath) 19 | 20 | if path in ['favicon.ico', 'robots.txt']: 21 | return 22 | 23 | return '' 24 | 25 | @classmethod 26 | def setup(cls): 27 | r = cls() 28 | from hookah import dispatch 29 | r.putChild('dispatch', dispatch.DispatchResource()) 30 | return r 31 | -------------------------------------------------------------------------------- /twisted/plugins/hookah_plugin.py: -------------------------------------------------------------------------------- 1 | from zope.interface import implements 2 | 3 | from twisted.python import usage 4 | from twisted.plugin import IPlugin 5 | from twisted.application.service import IServiceMaker 6 | from twisted.application import internet 7 | from twisted.web.server import Site 8 | 9 | from hookah.web import HookahResource 10 | 11 | 12 | class Options(usage.Options): 13 | optParameters = [["port", "p", 8080, "The port number to listen on."]] 14 | 15 | 16 | class HookahMaker(object): 17 | implements(IServiceMaker, IPlugin) 18 | tapname = "hookah" 19 | description = "Yeah. Hookah." 20 | options = Options 21 | 22 | def makeService(self, options): 23 | """ 24 | Construct a TCPServer from a factory defined in myproject. 25 | """ 26 | return internet.TCPServer(int(options["port"]), Site(HookahResource.setup())) 27 | 28 | serviceMaker = HookahMaker() 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) 2009 Jeff Lindsay 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. -------------------------------------------------------------------------------- /hookah/tests/test_storage.py: -------------------------------------------------------------------------------- 1 | from twisted.trial import unittest 2 | 3 | from hookah import storage 4 | 5 | class BaseStorageTest(object): 6 | 7 | storageFactory = None 8 | deletionMeaningful = True 9 | 10 | def setUp(self): 11 | self.s = self.storageFactory() 12 | 13 | self.oneKey = self.s.put("one") 14 | self.twoKey = self.s.put("two") 15 | 16 | def testGet(self): 17 | self.assertEquals("one", self.s[self.oneKey]) 18 | self.assertEquals("two", self.s[self.twoKey]) 19 | 20 | def testDelete(self): 21 | self.assertEquals("one", self.s[self.oneKey]) 22 | self.assertEquals("two", self.s[self.twoKey]) 23 | 24 | del self.s[self.oneKey] 25 | self.assertEquals("two", self.s[self.twoKey]) 26 | 27 | try: 28 | v = self.s[self.oneKey] 29 | if self.deletionMeaningful: 30 | self.fail("Expected failure, got " + v) 31 | except KeyError: 32 | pass 33 | 34 | def testRecent(self): 35 | self.assertEquals(['two', 'one'], self.s.recent()) 36 | self.assertEquals(['two'], self.s.recent(1)) 37 | 38 | class MemoryStorageTest(BaseStorageTest, unittest.TestCase): 39 | 40 | storageFactory = storage.MemoryStorage 41 | 42 | class InlineStorageTest(BaseStorageTest, unittest.TestCase): 43 | 44 | storageFactory = storage.InlineStorage 45 | deletionMeaningful = False 46 | -------------------------------------------------------------------------------- /hookah/storage.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import cPickle 3 | from collections import deque 4 | 5 | class LocalStorage(object): 6 | 7 | MAX_ITEMS = 20 8 | 9 | def __init__(self): 10 | self._recent = deque() 11 | 12 | def recent(self, n=10): 13 | return list(self._recent)[0:n] 14 | 15 | def recordLocal(self, ob): 16 | if len(self._recent) >= self.MAX_ITEMS: 17 | self._recent.pop() 18 | self._recent.appendleft(ob) 19 | 20 | class MemoryStorage(LocalStorage): 21 | 22 | def __init__(self): 23 | super(MemoryStorage, self).__init__() 24 | self._storage = {} 25 | self._sequence = 0 26 | 27 | def put(self, hookahRequest): 28 | self._sequence += 1 29 | self._storage[self._sequence] = hookahRequest 30 | self.recordLocal(hookahRequest) 31 | return self._sequence 32 | 33 | def __getitem__(self, key): 34 | return self._storage[key] 35 | 36 | def __delitem__(self, key): 37 | del self._storage[key] 38 | 39 | class InlineStorage(LocalStorage): 40 | 41 | def __init__(self): 42 | super(InlineStorage, self).__init__() 43 | self._recent = deque() 44 | 45 | def put(self, hookaRequest): 46 | self.recordLocal(hookaRequest) 47 | return base64.encodestring(cPickle.dumps(hookaRequest)) 48 | 49 | def __getitem__(self, key): 50 | return cPickle.loads(base64.decodestring(key)) 51 | 52 | def __delitem__(self, key): 53 | pass 54 | 55 | instance = MemoryStorage() 56 | 57 | -------------------------------------------------------------------------------- /hookah/tests/test_queue.py: -------------------------------------------------------------------------------- 1 | from hookah import queue, storage 2 | from time import sleep 3 | from twisted.trial import unittest 4 | from twisted.internet import defer 5 | 6 | class MockConsumer(object): 7 | 8 | def __init__(self, i=1): 9 | self.d = defer.Deferred() 10 | self.i = i 11 | 12 | def __call__(self, key): 13 | self.i -= 1 14 | if self.i == 0: 15 | self.d.callback(key) 16 | return defer.succeed("yay") 17 | 18 | class TestQueue(unittest.TestCase): 19 | 20 | def setUp(self): 21 | self.c = MockConsumer() 22 | self.q = queue.MemoryQueue(consumer=lambda: self.c) 23 | self.q.startConsumers(1) 24 | 25 | def testOneJob(self): 26 | k = storage.instance.put('Test Thing') 27 | self.q.submit(k) 28 | 29 | print "Checking..." 30 | self.c.d.addBoth(lambda x: self.q.shutDown()) 31 | return self.c.d 32 | 33 | def verifyMissing(self, *keys): 34 | for k in keys: 35 | try: 36 | self.storage.instance[k] 37 | self.fail("Found unexpected key", k) 38 | except KeyError: 39 | pass 40 | 41 | def testTwoJobs(self): 42 | k1 = storage.instance.put('Test Thing 1') 43 | self.q.submit(k1) 44 | k2 = storage.instance.put('Test Thing 2') 45 | self.q.submit(k2) 46 | 47 | 48 | self.c.d.addCallback(lambda x: self.verifyMissing(k1, k2)) 49 | self.c.d.addBoth(lambda x: self.q.shutDown()) 50 | return self.c.d 51 | -------------------------------------------------------------------------------- /hookah/queue.py: -------------------------------------------------------------------------------- 1 | from twisted.internet import defer, task 2 | 3 | import storage 4 | 5 | class Consumer(object): 6 | 7 | def __call__(self, key): 8 | request = storage.instance[key] 9 | # TODO: Work 10 | return defer.succeed("yay") 11 | 12 | class Queue(object): 13 | 14 | def submit(self, key): 15 | """Submit a job by work queue key.""" 16 | raise NotImplementedError 17 | 18 | def finish(self, key): 19 | """Mark a job as completed.""" 20 | raise NotImplementedError 21 | 22 | def retry(self, key): 23 | """Retry a job.""" 24 | raise NotImplementedError 25 | 26 | def startConsumers(self, n=5): 27 | """Start consumers working on the queue.""" 28 | raise NotImplementedError 29 | 30 | def shutDown(self): 31 | """Shut down the queue.""" 32 | raise NotImplementedError 33 | 34 | @defer.inlineCallbacks 35 | def doTask(self, c): 36 | key = yield self.q.get() 37 | # XXX: Indicate success/requeue/whatever 38 | try: 39 | worked = yield c(key) 40 | self.finish(key) 41 | except: 42 | self.retry(key) 43 | 44 | class MemoryQueue(Queue): 45 | 46 | def __init__(self, consumer=Consumer): 47 | self.q = defer.DeferredQueue() 48 | self.keepGoing = True 49 | self.consumer = consumer 50 | self.cooperator = task.Cooperator() 51 | 52 | def submit(self, key): 53 | self.q.put(key) 54 | 55 | def finish(self, key): 56 | del storage.instance[key] 57 | 58 | retry = submit 59 | 60 | def shutDown(self): 61 | self.keepGoing = False 62 | 63 | def startConsumers(self, n=5): 64 | def f(c): 65 | while self.keepGoing: 66 | yield self.doTask(c) 67 | 68 | for i in range(n): 69 | self.cooperator.coiterate(f(self.consumer())) 70 | 71 | instance = MemoryQueue() 72 | 73 | -------------------------------------------------------------------------------- /hookah/dispatch.py: -------------------------------------------------------------------------------- 1 | from twisted.internet import reactor 2 | from twisted.web import client, error, http 3 | from twisted.web.resource import Resource 4 | from hookah import queue 5 | import sys, os 6 | 7 | import base64 8 | 9 | from hookah import HookahRequest 10 | from hookah import storage, queue 11 | 12 | # TODO: Make these configurable 13 | RETRIES = 3 14 | DELAY_MULTIPLIER = 5 15 | 16 | def dispatch_request(request): 17 | key = storage.instance.put(request) 18 | queue.instance.submit(key) 19 | 20 | def post_and_retry(url, data, retry=0, content_type='application/x-www-form-urlencoded'): 21 | if type(data) is dict: 22 | print "Posting [%s] to %s with %s" % (retry, url, data) 23 | data = urllib.urlencode(data) 24 | else: 25 | print "Posting [%s] to %s with %s bytes of postdata" % (retry, url, len(data)) 26 | headers = { 27 | 'Content-Type': content_type, 28 | 'Content-Length': str(len(data)), 29 | } 30 | client.getPage(url, method='POST' if len(data) else 'GET', headers=headers, postdata=data if len(data) else None).addCallbacks( \ 31 | if_success, lambda reason: if_fail(reason, url, data, retry, content_type)) 32 | 33 | def if_success(page): pass 34 | def if_fail(reason, url, data, retry, content_type): 35 | if reason.getErrorMessage()[0:3] in ['301', '302', '303']: 36 | return # Not really a fail 37 | print reason.getErrorMessage() 38 | if retry < RETRIES: 39 | retry += 1 40 | reactor.callLater(retry * DELAY_MULTIPLIER, post_and_retry, url, data, retry, content_type) 41 | 42 | class DispatchResource(Resource): 43 | isLeaf = True 44 | 45 | def render(self, request): 46 | url = base64.b64decode(request.postpath[0]) 47 | 48 | if url: 49 | headers = {} 50 | for header in ['content-type', 'content-length']: 51 | value = request.getHeader(header) 52 | if value: 53 | headers[header] = value 54 | 55 | dispatch_request(HookahRequest(url, headers, request.content.read())) 56 | 57 | request.setResponseCode(http.ACCEPTED) 58 | return "202 Scheduled" 59 | else: 60 | request.setResponseCode(http.BAD_REQUEST) 61 | return "400 No destination URL" 62 | 63 | if __name__ == '__main__': 64 | from twisted.web.server import Request 65 | from cStringIO import StringIO 66 | class TestRequest(Request): 67 | postpath = ['aHR0cDovL3Byb2dyaXVtLmNvbT9ibGFo'] 68 | content = StringIO("BLAH") 69 | 70 | output = DispatchResource().render(TestRequest({}, True)) 71 | print output 72 | assert output == 'BLAH' -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Hookah 2 | ====== 3 | The HTTP event engine 4 | 5 | Current Features 6 | ---------------- 7 | * Interface is 100% HTTP (use easily from any language) 8 | * Dispatches POST requests (webhooks) asynchronously with retry 9 | * Provides publish/subscribe interface using [PubSubHubbub protocol](http://code.google.com/p/pubsubhubbub/) (experimental) 10 | * Provides "Twitter Stream API"-style long-polling interface for topics (super experimental) 11 | 12 | 13 | About 14 | ----- 15 | Hookah was originally created to ease the implementation of webhooks in your web systems. While webhooks are still at the core, it's becoming a scalable HTTP event engine with HTTP pubsub and long-polling event streaming. And of course, webhooks. Any system with webhooks or looking to implement webhooks will benefit from Hookah. 16 | 17 | Requirements 18 | ------------ 19 | Hookah currently depends on Twisted. 20 | 21 | Usage 22 | ----- 23 | Hookah is a simple, lightweight standalone web server that you run locally alongside your existing web stack. Starting it from the command line is simple: 24 | 25 | twistd hookah --port 8080 26 | 27 | Using the Dispatcher 28 | -------------------- 29 | Posting to /dispatch with a _url POST parameter will queue that POST request for that URL and return immediately. This allows you to use Hookah as an outgoing request queue that handles retries, etc. Using HTTP means you can do this easily from any language using a familiar API. 30 | 31 | Posting to /dispatch with a _topic POST parameter will broadcast that post to any callbacks subscribed to that topic (see following PubSub section), or any stream consumers with a long-running request on that topic. 32 | 33 | Using PubSub 34 | ------------ 35 | Refer to the [PubSubHubbub spec](http://pubsubhubbub.googlecode.com/svn/trunk/pubsubhubbub-core-0.1.html), as Hookah is currently quite compliant with this excellent protocol. The hub endpoint is at /hub, but this multiplexes (based on 'hub.mode' param) between /publish for publish pings, and /subscribe for subscription requests. 36 | 37 | **This feature is still very early** and as a result it is incomplete. The main caveat is that there is no permanent storage of subscription data or of the queues. This means if you were to restart Hookah, all subscriptions would have to be made again. 38 | 39 | Using Streams 40 | ------------- 41 | Hookah implements a long-running stream API, modeled after [Twitter's Stream API](http://apiwiki.twitter.com/Streaming-API-Documentation). Just do a GET request to /stream with a topic parameter, and you'll get a persistent, chunked HTTP connection that will send you messages published to that topic as they come in. Refer to the Twitter Stream API docs to get a better feel for this pragmatic Comet streaming technique. 42 | 43 | Todo 44 | ---- 45 | 46 | 1. Persistent storage (SQLite, MySQL, CouchDB) and queuing (in memory, Kestrel, RabbitMQ) backends 47 | 1. Configuration 48 | 1. Backlog/history with resend 49 | 1. "Errback" webhook 50 | 1. Async response handling 51 | 52 | License 53 | ------- 54 | 55 | Hookah is released under the MIT license, which can be found in the LICENSE file. 56 | 57 | Contributors 58 | ------------ 59 | * you? 60 | 61 | Author 62 | ------ 63 | Jeff Lindsay 64 | 65 | Learn more about web hooks 66 | -------------------------- 67 | http://webhooks.org --------------------------------------------------------------------------------