├── .gitignore ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── async_pubsub ├── __init__.py ├── base.py ├── constants.py ├── redis_pubsub.py └── zmq_pubsub.py ├── examples ├── redis_publish.py ├── redis_subscribe.py ├── zmq_publish.py ├── zmq_service.py └── zmq_subscribe.py ├── requirements.txt ├── setup.py └── tests ├── __init__.py ├── test_redis_pubsub.py └── test_zmq_pubsub.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .project 3 | .pydevproject 4 | .settings 5 | dist 6 | build 7 | async_pubsub.egg-info 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014 by Abhinav Singh and contributors. 2 | 3 | Some rights reserved. 4 | 5 | Redistribution and use in source and binary forms of the software as well 6 | as documentation, with or without modification, are permitted provided 7 | that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright 10 | notice, this list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above 13 | copyright notice, this list of conditions and the following 14 | disclaimer in the documentation and/or other materials provided 15 | with the distribution. 16 | 17 | * The names of the contributors may not be used to endorse or 18 | promote products derived from this software without specific 19 | prior written permission. 20 | 21 | THIS SOFTWARE AND DOCUMENTATION IS PROVIDED BY THE COPYRIGHT HOLDERS AND 22 | CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT 23 | NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER 25 | OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 26 | EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 27 | PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 28 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 29 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 30 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 31 | SOFTWARE AND DOCUMENTATION, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 32 | DAMAGE. 33 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include Makefile 3 | include README.md 4 | include requirements.txt 5 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: clean package release test 2 | 3 | all: clean test 4 | 5 | clean: 6 | find . -name '*.pyc' -exec rm -f {} + 7 | find . -name '*.pyo' -exec rm -f {} + 8 | find . -name '*~' -exec rm -f {} + 9 | 10 | package: 11 | python setup.py sdist 12 | 13 | release: 14 | python setup.py sdist register upload 15 | 16 | test: 17 | nosetests -v --with-coverage --cover-package=async_pubsub --cover-erase --cover-html --nocapture 18 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Asynchronous PubSub in Python using Redis, ZMQ -------------------------------------------------------------------------------- /async_pubsub/__init__.py: -------------------------------------------------------------------------------- 1 | VERSION = (0, 1, 1) 2 | __version__ = '.'.join(map(str, VERSION[0:3])) + ''.join(VERSION[3:]) 3 | __description__ = 'Asynchronous PubSub in Python using Redis, ZMQ' 4 | __author__ = 'Abhinav Singh' 5 | __author_email__ = 'mailsforabhinav@gmail.com' 6 | __homepage__ = 'https://github.com/abhinavsingh/async_pubsub' 7 | __license__ = 'BSD' 8 | -------------------------------------------------------------------------------- /async_pubsub/base.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from .constants import (CALLBACK_TYPE_CONNECTED, CALLBACK_TYPE_SUBSCRIBED, 3 | CALLBACK_TYPE_UNSUBSCRIBED, CALLBACK_TYPE_MESSAGE, 4 | CALLBACK_TYPE_DISCONNECTED) 5 | 6 | class PubSubBase(object): 7 | 8 | def __init__(self, callback=None): 9 | self.callback = callback 10 | 11 | ## 12 | ## Methods implementation must define 13 | ## 14 | 15 | def connect(self): 16 | raise NotImplementedError() 17 | 18 | def disconnect(self): 19 | raise NotImplementedError() 20 | 21 | def subscribe(self): 22 | raise NotImplementedError() 23 | 24 | def unsubscribe(self): 25 | raise NotImplementedError() 26 | 27 | @staticmethod 28 | def publish(channel_id, message): 29 | raise NotImplementedError() 30 | 31 | ## 32 | ## events implementation must emit 33 | ## user callback is appropriately called 34 | ## 35 | 36 | def connected(self): 37 | self.callback(CALLBACK_TYPE_CONNECTED) 38 | 39 | def disconnected(self): 40 | self.callback(CALLBACK_TYPE_DISCONNECTED) 41 | 42 | def subscribed(self, channel_id): 43 | self.callback(CALLBACK_TYPE_SUBSCRIBED, channel_id) 44 | 45 | def unsubscribed(self, channel_id): 46 | self.callback(CALLBACK_TYPE_UNSUBSCRIBED, channel_id) 47 | 48 | def on_message(self, channel_id, message): 49 | self.callback(CALLBACK_TYPE_MESSAGE, channel_id, message) 50 | -------------------------------------------------------------------------------- /async_pubsub/constants.py: -------------------------------------------------------------------------------- 1 | CALLBACK_TYPE_CONNECTED = 'CONNECTED' 2 | CALLBACK_TYPE_DISCONNECTED = 'DISCONNECTED' 3 | CALLBACK_TYPE_SUBSCRIBED = 'SUBSCRIBED' 4 | CALLBACK_TYPE_UNSUBSCRIBED = 'UNSUBSCRIBED' 5 | CALLBACK_TYPE_MESSAGE = 'MESSAGE' 6 | -------------------------------------------------------------------------------- /async_pubsub/redis_pubsub.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import io 3 | import socket 4 | import redis 5 | import hiredis 6 | from tornado.iostream import IOStream 7 | from .base import PubSubBase 8 | 9 | class RedisPubSub(PubSubBase): 10 | 11 | def __init__(self, host='127.0.0.1', port=6379, *args, **kwargs): 12 | self.host = host 13 | self.port = port 14 | super(RedisPubSub, self).__init__(*args, **kwargs) 15 | 16 | @staticmethod 17 | def get_redis(): 18 | return redis.StrictRedis( 19 | host = '127.0.0.1', 20 | port = 6379, 21 | db = 0 22 | ) 23 | 24 | ## 25 | ## pubsub api 26 | ## 27 | 28 | def connect(self): 29 | self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 30 | self.stream = IOStream(self.socket) 31 | self.stream.connect((self.host, self.port), self.on_connect) 32 | 33 | def disconnect(self): 34 | self.unsubscribe() 35 | self.stream.close() 36 | 37 | def subscribe(self, channel_id): 38 | self.send('SUBSCRIBE', channel_id) 39 | 40 | def unsubscribe(self, channel_id=None): 41 | if channel_id: 42 | self.send('UNSUBSCRIBE', channel_id) 43 | else: 44 | self.send('UNSUBSCRIBE') 45 | 46 | @staticmethod 47 | def publish(channel_id, message): 48 | r = RedisPubSub.get_redis() 49 | r.publish(channel_id, message) 50 | 51 | ## 52 | ## socket/stream callbacks 53 | ## 54 | 55 | def on_connect(self): 56 | self.stream.set_close_callback(self.on_close) 57 | self.stream.read_until_close(self.on_data, self.on_streaming_data) 58 | self.reader = hiredis.Reader() 59 | self.connected() 60 | 61 | def on_data(self, *args, **kwargs): 62 | pass 63 | 64 | def on_streaming_data(self, data): 65 | self.reader.feed(data) 66 | reply = self.reader.gets() 67 | while reply: 68 | if reply[0] == 'subscribe': 69 | self.subscribed(reply[1]) 70 | elif reply[0] == 'unsubscribe': 71 | self.unsubscribed(reply[1]) 72 | elif reply[0] == 'message': 73 | self.on_message(reply[1], reply[2]) 74 | else: 75 | raise Exception('Unhandled data from redis %s' % reply) 76 | reply = self.reader.gets() 77 | 78 | def on_close(self): 79 | self.socket = None 80 | self.stream = None 81 | self.disconnected() 82 | 83 | ## 84 | ## redis protocol parser (derived from redis-py) 85 | ## 86 | 87 | def encode(self, value): 88 | if isinstance(value, bytes): 89 | return value 90 | if isinstance(value, float): 91 | value = repr(value) 92 | if not isinstance(value, basestring): 93 | value = str(value) 94 | if isinstance(value, unicode): 95 | value = value.encode('utf-8', 'strict') 96 | return value 97 | 98 | def pack_command(self, *args): 99 | cmd = io.BytesIO() 100 | cmd.write('*') 101 | cmd.write(str(len(args))) 102 | cmd.write('\r\n') 103 | for arg in args: 104 | arg = self.encode(arg) 105 | cmd.write('$') 106 | cmd.write(str(len(arg))) 107 | cmd.write('\r\n') 108 | cmd.write(arg) 109 | cmd.write('\r\n') 110 | return cmd.getvalue() 111 | 112 | def send(self, *args): 113 | """Send redis command.""" 114 | cmd = self.pack_command(*args) 115 | self.stream.write(cmd) 116 | -------------------------------------------------------------------------------- /async_pubsub/zmq_pubsub.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import zmq 3 | from zmq.eventloop.zmqstream import ZMQStream 4 | import logging 5 | from .base import PubSubBase 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | class ZMQPubSub(PubSubBase): 10 | 11 | def __init__(self, device_ip='127.0.0.1', fport=5559, bport=5560, *args, **kwargs): 12 | self.channels = list() 13 | self.device_ip = '127.0.0.1' 14 | self.fport = fport 15 | self.bport = bport 16 | super(ZMQPubSub, self).__init__(*args, **kwargs) 17 | 18 | ## 19 | ## pubsub api 20 | ## 21 | 22 | def connect(self): 23 | self.context = zmq.Context() 24 | self.socket = self.context.socket(zmq.SUB) 25 | self.socket.connect('tcp://%s:%s' % (self.device_ip, self.bport)) 26 | self.stream = ZMQStream(self.socket) 27 | self.stream.on_recv(self.on_streaming_data) 28 | self.connected() 29 | 30 | def disconnect(self): 31 | self.disconnected() 32 | 33 | def subscribe(self, channel_id): 34 | self.socket.setsockopt(zmq.SUBSCRIBE, str(channel_id)) 35 | self.channels.append(channel_id) 36 | self.subscribed(channel_id) 37 | 38 | def unsubscribe(self, channel_id=None): 39 | channels = [channel_id] if channel_id else self.channels 40 | for channel_id in channels: 41 | self.socket.setsockopt(zmq.UNSUBSCRIBE, str(channel_id)) 42 | self.unsubscribed(channel_id) 43 | self.channels.remove(channel_id) 44 | 45 | @staticmethod 46 | def publish(channel_id, message, device_ip='127.0.0.1', fport=5559): 47 | context = zmq.Context() 48 | socket = context.socket(zmq.PUSH) 49 | socket.connect('tcp://%s:%s' % (device_ip, fport)) 50 | socket.send_unicode('%s %s' % (channel_id, message)) 51 | 52 | ## 53 | ## other methods 54 | ## 55 | 56 | def on_streaming_data(self, data): 57 | for l in data: 58 | reply = l.split(' ', 1) 59 | self.on_message(reply[0], reply[1]) 60 | 61 | @staticmethod 62 | def start_service(fport=5559, bport=5560): 63 | try: 64 | context = zmq.Context(1) 65 | 66 | frontend = context.socket(zmq.PULL) 67 | frontend.bind('tcp://*:%s' % fport) 68 | 69 | backend = context.socket(zmq.PUB) 70 | backend.bind('tcp://*:%s' % bport) 71 | 72 | logger.info('starting zmq device') 73 | zmq.device(zmq.FORWARDER, frontend, backend) 74 | except KeyboardInterrupt: 75 | pass 76 | except Exception as e: 77 | logger.exception(e) 78 | finally: 79 | frontend.close() 80 | backend.close() 81 | context.term() 82 | 83 | -------------------------------------------------------------------------------- /examples/redis_publish.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from async_pubsub.redis_pubsub import RedisPubSub 3 | 4 | if __name__ == '__main__': 5 | channel_id = sys.argv[1] 6 | message = sys.argv[2] 7 | RedisPubSub.publish(channel_id, message) 8 | -------------------------------------------------------------------------------- /examples/redis_subscribe.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from tornado.ioloop import IOLoop 3 | 4 | from async_pubsub.redis_pubsub import RedisPubSub 5 | from async_pubsub.constants import (CALLBACK_TYPE_CONNECTED, CALLBACK_TYPE_SUBSCRIBED, 6 | CALLBACK_TYPE_UNSUBSCRIBED, CALLBACK_TYPE_MESSAGE, 7 | CALLBACK_TYPE_DISCONNECTED) 8 | 9 | ioloop = IOLoop.instance() 10 | 11 | class Subscriber(object): 12 | 13 | def __init__(self, channel_id): 14 | self.channel_id = channel_id 15 | self.r = RedisPubSub(host='127.0.0.1', port=6379, callback=self.callback) 16 | self.r.connect() 17 | 18 | def callback(self, evtype, *args, **kwargs): 19 | if evtype == CALLBACK_TYPE_CONNECTED: 20 | print 'connected' 21 | self.r.subscribe(self.channel_id) 22 | elif evtype == CALLBACK_TYPE_SUBSCRIBED: 23 | print 'subscribed to channel_id %s' % args[0] 24 | elif evtype == CALLBACK_TYPE_MESSAGE: 25 | print 'received on channel_id %s message %s' % (args[0], args[1]) 26 | self.r.unsubscribe() 27 | elif evtype == CALLBACK_TYPE_UNSUBSCRIBED: 28 | print 'unsubscribed' 29 | self.r.disconnect() 30 | elif evtype == CALLBACK_TYPE_DISCONNECTED: 31 | print 'disconnected' 32 | ioloop.stop() 33 | 34 | if __name__ == '__main__': 35 | channel_id = sys.argv[1] 36 | subscriber = Subscriber(channel_id) 37 | ioloop.start() 38 | print 'done.' 39 | -------------------------------------------------------------------------------- /examples/zmq_publish.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from async_pubsub.zmq_pubsub import ZMQPubSub 3 | 4 | if __name__ == '__main__': 5 | channel_id = sys.argv[1] 6 | message = sys.argv[2] 7 | ZMQPubSub.publish(channel_id, message) 8 | -------------------------------------------------------------------------------- /examples/zmq_service.py: -------------------------------------------------------------------------------- 1 | from async_pubsub.zmq_pubsub import ZMQPubSub 2 | 3 | if __name__ == '__main__': 4 | ZMQPubSub.start_service() 5 | -------------------------------------------------------------------------------- /examples/zmq_subscribe.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from tornado.ioloop import IOLoop 3 | 4 | from async_pubsub.zmq_pubsub import ZMQPubSub 5 | from async_pubsub.constants import (CALLBACK_TYPE_CONNECTED, CALLBACK_TYPE_SUBSCRIBED, 6 | CALLBACK_TYPE_UNSUBSCRIBED, CALLBACK_TYPE_MESSAGE, 7 | CALLBACK_TYPE_DISCONNECTED) 8 | 9 | from zmq.eventloop import ioloop as zmq_ioloop 10 | zmq_ioloop.install() 11 | 12 | ioloop = IOLoop.instance() 13 | 14 | class Subscriber(object): 15 | 16 | def __init__(self, channel_id): 17 | self.channel_id = channel_id 18 | self.r = ZMQPubSub(callback=self.callback) 19 | self.r.connect() 20 | 21 | def callback(self, evtype, *args, **kwargs): 22 | if evtype == CALLBACK_TYPE_CONNECTED: 23 | print 'connected' 24 | self.r.subscribe(self.channel_id) 25 | elif evtype == CALLBACK_TYPE_SUBSCRIBED: 26 | print 'subscribed to channel_id %s' % args[0] 27 | elif evtype == CALLBACK_TYPE_MESSAGE: 28 | print 'received on channel_id %s message %s' % (args[0], args[1]) 29 | self.r.unsubscribe() 30 | elif evtype == CALLBACK_TYPE_UNSUBSCRIBED: 31 | print 'unsubscribed' 32 | self.r.disconnect() 33 | elif evtype == CALLBACK_TYPE_DISCONNECTED: 34 | print 'disconnected' 35 | ioloop.stop() 36 | 37 | if __name__ == '__main__': 38 | channel_id = sys.argv[1] 39 | subscriber = Subscriber(channel_id) 40 | ioloop.start() 41 | print 'done.' 42 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | hiredis==0.1.1 2 | redis==2.7.6 3 | tornado==3.2 4 | pyzmq==13.1.0 -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from setuptools import setup, find_packages 3 | import async_pubsub 4 | 5 | classifiers = [ 6 | 'Development Status :: 4 - Beta', 7 | 'Environment :: Console', 8 | 'Intended Audience :: Developers', 9 | 'Intended Audience :: System Administrators', 10 | 'License :: OSI Approved :: BSD License', 11 | 'Operating System :: MacOS', 12 | 'Operating System :: POSIX', 13 | 'Operating System :: Unix', 14 | 'Programming Language :: Python :: 2.7', 15 | 'Topic :: Utilities', 16 | ] 17 | 18 | install_requires = open('requirements.txt', 'rb').read().strip().split() 19 | 20 | setup( 21 | name = 'async_pubsub', 22 | version = async_pubsub.__version__, 23 | description = async_pubsub.__description__, 24 | long_description = open('README.md').read().strip(), 25 | author = async_pubsub.__author__, 26 | author_email = async_pubsub.__author_email__, 27 | url = async_pubsub.__homepage__, 28 | license = async_pubsub.__license__, 29 | packages = find_packages(), 30 | install_requires = install_requires, 31 | classifiers = classifiers 32 | ) -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/abhinavsingh/async_pubsub/aeea4e2451edf4e45b44d333eb4131353f78ac48/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_redis_pubsub.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from async_pubsub import RedisPubSub 3 | 4 | class TestRedisPubSub(unittest.TestCase): 5 | 6 | pass 7 | -------------------------------------------------------------------------------- /tests/test_zmq_pubsub.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from async_pubsub import ZMQPubSub 3 | 4 | class TestZMQPubSub(unittest.TestCase): 5 | 6 | pass 7 | --------------------------------------------------------------------------------