├── .gitignore
├── examples
├── pyzmq_pub.py
├── pyzmq_req_client.py
├── test.html
├── pyzmq_rep_server.py
├── test.py
└── test.js
├── stresstest
├── run_bridge.py
├── server.py
└── client.py
├── test
├── test_utils.py
├── basic_test.py
└── rpc_test.py
├── bridgeutils.py
├── geventbridgeutils.py
├── bridge.js
├── README.rst
└── bridge.py
/.gitignore:
--------------------------------------------------------------------------------
1 | *.log*
2 | *.pyc
3 | *.swp
4 | *~
5 | *\#*
6 | save
7 |
8 |
--------------------------------------------------------------------------------
/examples/pyzmq_pub.py:
--------------------------------------------------------------------------------
1 | import zmq
2 | import time
3 |
4 | c = zmq.Context()
5 | s = c.socket(zmq.PUB)
6 | s.bind('tcp://127.0.0.1:10002')
7 | while(True):
8 | for c in range(100):
9 | print c
10 | s.send(str(c))
11 | time.sleep(1)
12 |
13 |
14 |
--------------------------------------------------------------------------------
/examples/pyzmq_req_client.py:
--------------------------------------------------------------------------------
1 | import zmq
2 | import time
3 |
4 | c = zmq.Context()
5 | s = c.socket(zmq.REQ)
6 | s.connect('tcp://127.0.0.1:10003')
7 | while(True):
8 | for c in range(100):
9 | print 'sending %s' % c
10 | s.send(str(c))
11 | print s.recv();
12 | time.sleep(1)
13 |
14 |
15 |
--------------------------------------------------------------------------------
/examples/test.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
6 |
7 |
8 |
9 |
10 |
11 | Plot
12 |
13 |
14 |
15 |
16 |
--------------------------------------------------------------------------------
/stresstest/run_bridge.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import websocket
3 | from gevent_zeromq import zmq
4 | import gevent
5 | import gevent.monkey
6 | gevent.monkey.patch_all()
7 | from gevent import spawn
8 | from geventwebsocket.handler import WebSocketHandler
9 | import bridge
10 | from gevent import pywsgi
11 | import time
12 | import logging
13 | log = logging.getLogger(__name__)
14 | import simplejson
15 | logging.basicConfig(level=logging.INFO)
16 |
17 | app = bridge.WsgiHandler()
18 | server = pywsgi.WSGIServer(('0.0.0.0', 9000), app.wsgi_handle,
19 | handler_class=WebSocketHandler)
20 | server.serve_forever()
21 |
22 |
23 |
24 |
--------------------------------------------------------------------------------
/stresstest/server.py:
--------------------------------------------------------------------------------
1 | from gevent_zeromq import zmq
2 | import gevent
3 | import gevent.monkey
4 | gevent.monkey.patch_all()
5 | from gevent import spawn
6 | import time
7 | import logging
8 | log = logging.getLogger(__name__)
9 | logging.basicConfig(level=logging.INFO)
10 |
11 | num_reqrep = 20
12 | rr = []
13 | def rr_test(s):
14 | while True:
15 | msg = s.recv()
16 | log.debug(msg)
17 | s.send(msg)
18 | ctx = zmq.Context()
19 | for c in range(num_reqrep):
20 | s = ctx.socket(zmq.REP)
21 | s.setsockopt(zmq.HWM, 100)
22 | port = s.bind_to_random_port("tcp://127.0.0.1")
23 | t = spawn(rr_test, s)
24 | rr.append((s,t, port))
25 |
26 | with open("rrports.txt","w+") as f:
27 | f.write(",".join([str(x[-1]) for x in rr]))
28 |
29 | threads = [x[1] for x in rr]
30 | gevent.joinall(threads)
31 |
32 |
33 |
34 |
35 |
36 |
--------------------------------------------------------------------------------
/examples/pyzmq_rep_server.py:
--------------------------------------------------------------------------------
1 | import gevent.monkey
2 | gevent.monkey.patch_all()
3 | from gevent_zeromq import zmq
4 | import gevent
5 | import time
6 |
7 | c = zmq.Context()
8 | s = c.socket(zmq.XREP)
9 | s.bind('tcp://127.0.0.1:10001')
10 | ident = None
11 | def loop():
12 | global ident
13 | while True:
14 | print 'starting'
15 | msg = s.recv_multipart()
16 | print 'received', msg
17 | if 'IDENT' in msg[-1]:
18 | ident = msg[-1].split()[-1]
19 | s.send_multipart(msg)
20 | reploop = gevent.spawn(loop)
21 | s2 = c.socket(zmq.REQ)
22 | s2.connect('tcp://127.0.0.1:10003')
23 |
24 | import geventbridgeutils
25 | while True:
26 | if ident is None:
27 | gevent.sleep(1)
28 | else:
29 | rpcclient = geventbridgeutils.GeventRPCClient(s2, ident, timeout=1000.0)
30 | break
31 |
32 | while(True):
33 | for c in range(100):
34 | print 'sending %s' % c
35 | retval = rpcclient.rpc('echo', c);
36 | print 'got %s' % retval
37 | print type(retval)
38 | gevent.sleep(1)
39 |
40 |
41 |
42 |
43 |
44 |
--------------------------------------------------------------------------------
/examples/test.py:
--------------------------------------------------------------------------------
1 | import gevent
2 | from gevent import pywsgi
3 | from geventwebsocket.handler import WebSocketHandler
4 | import logging
5 | import bridge
6 | import simplejson
7 | logging.basicConfig(level=logging.DEBUG)
8 | class MyBridgeClass(bridge.BridgeWebProxyHandler):
9 | def zmq_allowed(self, params):
10 | params = simplejson.loads(params)
11 | zmq_conn_string = params['zmq_conn_string']
12 | socket_type = params['socket_type']
13 | print 'auth', params['username'], params['socket_type']
14 | return params['username'] == 'hugo'
15 |
16 |
17 | class MyWsgiHandler(bridge.WsgiHandler):
18 | bridge_class = MyBridgeClass
19 | def websocket_allowed(self, environ):
20 | #you can add logic here to do auth
21 | return True
22 |
23 |
24 | app = MyWsgiHandler()
25 | server = pywsgi.WSGIServer(('0.0.0.0', 8000), app.wsgi_handle,
26 | # keyfile='/etc/nginx/server.key',
27 | # certfile='/etc/nginx/server.crt',
28 | handler_class=WebSocketHandler)
29 | server.serve_forever()
30 |
31 |
--------------------------------------------------------------------------------
/examples/test.js:
--------------------------------------------------------------------------------
1 | context = new zmq.Context('ws://localhost:8000/data');
2 | reqrep = context.Socket(zmq.REQ);
3 |
4 | reqrep.connect('tcp://127.0.0.1:10001', {'username':'hugo'});
5 |
6 | reqrep2 = context.Socket(zmq.REQ);
7 | reqrep2.connect('tcp://127.0.0.1:10001', {'username':'hugo'});
8 | reqrep.send('hello', function(x){console.log(x)});
9 | reqrep2.send('hello', function(x){
10 | console.log(x + 'sdf');
11 | console.log(x + 'sfsf');
12 | });
13 | sub = context.Socket(zmq.SUB);
14 | sub.onmessage = function(x){
15 | console.log(['sub', x]);
16 | }
17 | sub.connect("tcp://127.0.0.1:10002", {'username':'hugo'});
18 |
19 | sub2 = context.Socket(zmq.SUB);
20 | sub2.onmessage = function(x){
21 | console.log(['not alowed!', x]);
22 | }
23 | sub2.connect("tcp://127.0.0.1:10002", {'username':'nonhugo'});
24 |
25 |
26 | rep = context.Socket(zmq.REP);
27 | reqrep.send('IDENT ' + rep.identity, function(x){console.log(x)});
28 | rep.connect("tcp://127.0.0.1:10003", {'username' : 'hugo'});
29 | // rep.onmessage = function(x){
30 | // console.log(x);
31 | // //echo
32 | // rep.send(x)
33 | // }
34 |
35 | RPC = function(socket){
36 | zmq.RPCServer.call(this, socket);
37 | }
38 | RPC.prototype = new zmq.RPCServer();
39 |
40 | RPC.prototype.echo = function(arg){
41 | return arg;
42 | }
43 |
44 | var rpc = new RPC(rep);
--------------------------------------------------------------------------------
/test/test_utils.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import websocket
3 | from gevent_zeromq import zmq
4 | import gevent
5 | import gevent.monkey
6 | gevent.monkey.patch_all()
7 | from gevent import spawn
8 | from geventwebsocket.handler import WebSocketHandler
9 | import bridge
10 | from gevent import pywsgi
11 | import time
12 | import logging
13 | log = logging.getLogger(__name__)
14 | import simplejson
15 |
16 | def wait_until(func, timeout=1.0, interval=0.01):
17 | st = time.time()
18 | while True:
19 | if func():
20 | return True
21 | if (time.time() - st) > interval:
22 | return False
23 | gevent.sleep(interval)
24 |
25 | def connect(server, ws_address, zmq_conn_string, sock_type):
26 | wait_until(lambda : server.started)
27 | sock = websocket.WebSocket()
28 | sock.io_sock.settimeout(1.0)
29 | sock.connect(ws_address)
30 | auth = {
31 | 'zmq_conn_string' : zmq_conn_string,
32 | 'socket_type' : sock_type
33 | }
34 | auth = simplejson.dumps(auth)
35 | sock.send(simplejson.dumps(
36 | {
37 | 'identity' : 'testidentity',
38 | 'msg_type' : 'connect',
39 | 'content' : auth
40 | }))
41 | msg = sock.recv()
42 | msgobj = simplejson.loads(msg)
43 | msgobj = simplejson.loads(msgobj['content'])
44 | assert msgobj['status']
45 | return sock
46 |
--------------------------------------------------------------------------------
/stresstest/client.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import websocket
3 | from gevent_zeromq import zmq
4 | import gevent
5 | import gevent.monkey
6 | gevent.monkey.patch_all()
7 | from gevent import spawn
8 | from geventwebsocket.handler import WebSocketHandler
9 | import bridge
10 | from gevent import pywsgi
11 | import time
12 | import logging
13 | log = logging.getLogger(__name__)
14 | import simplejson
15 | logging.basicConfig(level=logging.INFO)
16 | import uuid
17 |
18 | with open("rrports.txt","r") as f:
19 | ports = [int(x) for x in f.read().split(",")]
20 | results = {}
21 | def test(port, sock_type, num_reqs):
22 | identity = str(uuid.uuid4())
23 | results[port, identity] = 0
24 | sock = websocket.WebSocket()
25 | sock.io_sock.settimeout(1.0)
26 | zmq_conn_string = "tcp://127.0.0.1:" + str(port)
27 | sock.connect('ws://127.0.0.1:9000')
28 | auth = {
29 | 'zmq_conn_string' : zmq_conn_string,
30 | 'socket_type' : sock_type
31 | }
32 | auth = simplejson.dumps(auth)
33 | sock.send(simplejson.dumps(
34 | {
35 | 'identity' : identity,
36 | 'msg_type' : 'connect',
37 | 'content' : auth
38 | }))
39 | msg = sock.recv()
40 | for c in range(num_reqs):
41 | try:
42 | sock.send(simplejson.dumps(
43 | {'identity' : identity,
44 | 'msg_type' : 'user',
45 | 'content' : identity}))
46 | msg = sock.recv()
47 | log.debug(msg)
48 | assert simplejson.loads(msg)['content'] == identity
49 | #print identity
50 | results[port, identity] += 1
51 | except:
52 | pass
53 | sock.close()
54 |
55 | num_reqs = 100
56 | num_per_port = 10
57 | num_ports = len(ports)
58 | while True:
59 | threads = []
60 | for p in ports:
61 | for c in range(num_per_port):
62 | threads.append(spawn(test, p, zmq.REQ, num_reqs))
63 | #threads = [spawn(test, p, zmq.REQ, 100) for p in ports]
64 | def report():
65 | while(True):
66 | log.info("%s, %s, %s", len(results),
67 | np.sum(np.array(results.values())),
68 | num_reqs * num_per_port * num_ports)
69 | gevent.sleep(1)
70 | threads.append(spawn(report))
71 | gevent.sleep(5.0)
72 | [x.kill() for x in threads]
73 |
--------------------------------------------------------------------------------
/bridgeutils.py:
--------------------------------------------------------------------------------
1 | import zmq
2 | import simplejson as jsonapi
3 | import logging
4 | log = logging.getLogger(__name__)
5 |
6 | class RPCClient(object):
7 | def __init__(self, socket, ident, timeout=1000.0):
8 | self.socket = socket
9 | self.ident = ident
10 | self.timeout = timeout
11 | self.poller = zmq.Poller()
12 | self.poller.register(self.socket, zmq.POLLIN)
13 |
14 | def rpc(self, funcname, *args, **kwargs):
15 | msg = {'funcname' : funcname,
16 | 'args' : args}
17 | self.socket.send_multipart(['', jsonapi.dumps(msg), self.ident])
18 |
19 | socks = dict(self.poller.poll(timeout=self.timeout))
20 | if self.socket in socks:
21 | message = self.socket.recv_multipart()
22 | print message
23 | msgobj = jsonapi.loads(message[-1])
24 | return msgobj.get('returnval', None)
25 | else:
26 | return None
27 |
28 | class RPCServer(object):
29 | #none, means we can rpc any function
30 | #explicit iterable means, only those
31 | #functions in the iterable can be executed
32 | #(use a set)
33 |
34 | authorized_functions = None
35 |
36 | def __init__(self, reqrep_socket, timeout=1000.0):
37 | self.reqrep_socket = reqrep_socket
38 | self.poller = zmq.Poller()
39 | self.poller.register(self.reqrep_socket, zmq.POLLIN)
40 | self.kill = False
41 | self.timeout = timeout
42 |
43 | def run_rpc(self):
44 | while True:
45 | #the follow code must be wrapped in an exception handler
46 | #we don't know what we're getting
47 | socks = dict(self.poller.poll(timeout=self.timeout))
48 | if self.reqrep_socket in socks:
49 | try:
50 | msg = self.reqrep_socket.recv()
51 | msgobj = jsonapi.loads(msg)
52 | response_obj = self.get_response(msgobj)
53 | response = jsonapi.dumps(response_obj)
54 | except Exception as e:
55 | log.exception(e)
56 | response_obj = self.error_obj('unknown ooger')
57 | response = jsonapi.dumps(response_obj)
58 | self.reqrep_socket.send(jsonapi.dumps(response_obj))
59 | else:
60 | if self.kill:
61 | break
62 |
63 | def error_obj(self, error_msg):
64 | return {'status' : 'error',
65 | 'error_msg' : error_msg}
66 |
67 | def returnval_obj(self, returnval):
68 | return {'returnval' : returnval}
69 |
70 | def get_response(self, msgobj):
71 | funcname = msgobj['funcname']
72 | args = msgobj.get('args', [])
73 | kwargs = msgobj.get('kwargs', {})
74 | auth = False
75 | if self.authorized_functions is not None \
76 | and funcname not in self.authorized_functions:
77 | return self.error_obj('unauthorized access')
78 |
79 | if hasattr(self, 'can_' + funcname):
80 | auth = self.can_funcname(*args, **kwargs)
81 | if not auth:
82 | return self.error_obj('unauthorized access')
83 |
84 | func = getattr(self, funcname)
85 | retval = func(*args, **kwargs)
86 | return self.returnval_obj(retval)
87 |
--------------------------------------------------------------------------------
/geventbridgeutils.py:
--------------------------------------------------------------------------------
1 | import gevent
2 | import gevent.queue
3 | from gevent_zeromq import zmq
4 | import logging
5 | log = logging.getLogger(__name__)
6 | import simplejson
7 | from gevent import spawn
8 | import collections
9 | import logging
10 | import time
11 | log = logging.getLogger('__name__')
12 | jsonapi = simplejson
13 |
14 | class GeventZMQRPC(object):
15 | #none, means we can rpc any function
16 | #explicit iterable means, only those
17 | #functions in the iterable can be executed
18 | #(use a set)
19 |
20 | authorized_functions = None
21 |
22 | def __init__(self, reqrep_socket):
23 | self.reqrep_socket = reqrep_socket
24 |
25 | def run_rpc(self):
26 | while True:
27 | try:
28 | #the follow code must be wrapped in an exception handler
29 | #we don't know what we're getting
30 | msg = self.reqrep_socket.recv()
31 | msgobj = jsonapi.loads(msg)
32 | response_obj = self.get_response(msgobj)
33 | response = jsonapi.dumps(response_obj)
34 |
35 | except Exception as e:
36 | log.exception(e)
37 | response_obj = self.error_obj('unknown error')
38 | response = jsonapi.dumps(response_obj)
39 |
40 | self.reqrep_socket.send(jsonapi.dumps(response_obj))
41 |
42 | def error_obj(self, error_msg):
43 | return {'status' : 'error',
44 | 'error_msg' : error_msg}
45 |
46 | def returnval_obj(self, returnval):
47 | return {'returnval' : returnval}
48 |
49 | def get_response(self, msgobj):
50 | funcname = msgobj['funcname']
51 | args = msgobj.get('args', [])
52 | kwargs = msgobj.get('kwargs', {})
53 | auth = False
54 | if self.authorized_functions is not None \
55 | and funcname not in self.authorized_functions:
56 | return self.error_obj('unauthorized access')
57 |
58 | if hasattr(self, 'can_' + funcname):
59 | auth = self.can_funcname(*args, **kwargs)
60 | if not auth:
61 | return self.error_obj('unauthorized access')
62 |
63 | func = getattr(self, funcname)
64 | retval = func(*args, **kwargs)
65 | return self.returnval_obj(retval)
66 |
67 |
68 |
69 | class PubSubRPCClient(object):
70 | def __init__(self, socket):
71 | self.socket = socket
72 | self.queue = gevent.queue.Queue()
73 |
74 | def rpc(self, funcname, *args, **kwargs):
75 | msg = {'funcname' : funcname,
76 | 'args' : args}
77 | self.queue.put(jsonapi.dumps(msg))
78 |
79 | def run_pub(self):
80 | while True:
81 | msg = self.queue.get()
82 | self.socket.send(msg)
83 |
84 |
85 | class GeventRPCClient(object):
86 | def __init__(self, socket, ident, timeout=1.0):
87 | self.socket = socket
88 | self.ident = ident
89 | self.queue = gevent.queue.Queue()
90 | self.timeout = timeout
91 |
92 | def rpc(self, funcname, *args, **kwargs):
93 | msg = {'funcname' : funcname,
94 | 'args' : args}
95 | self.socket.send_multipart([jsonapi.dumps(msg), self.ident])
96 | data = []
97 | def recv():
98 | val = self.socket.recv()
99 | data.append(val)
100 | recv_t = gevent.spawn(recv)
101 | recv_t.join(timeout=self.timeout)
102 | recv_t.kill()
103 | if len(data) == 1:
104 | return jsonapi.loads(data[0])['returnval']
105 | else:
106 | return None
107 |
108 | def run_send(self):
109 | while True:
110 | msg = self.queue.get()
111 | self.socket.send(msg)
112 |
113 |
114 |
--------------------------------------------------------------------------------
/test/basic_test.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import websocket
3 | from gevent_zeromq import zmq
4 | import gevent
5 | import gevent.monkey
6 | gevent.monkey.patch_all()
7 | from gevent import spawn
8 | from geventwebsocket.handler import WebSocketHandler
9 | import bridge
10 | from gevent import pywsgi
11 | import time
12 | import logging
13 | log = logging.getLogger(__name__)
14 | import simplejson
15 | import test_utils
16 | wait_until = test_utils.wait_until
17 | connect = test_utils.connect
18 | port = 10010
19 | class ReqRepTest(unittest.TestCase):
20 | def setUp(self):
21 | self.ctx = zmq.Context()
22 | self.reqrep = self.ctx.socket(zmq.REP)
23 | self.rr_port = self.reqrep.bind_to_random_port("tcp://127.0.0.1")
24 | self.app = bridge.WsgiHandler()
25 | self.server = pywsgi.WSGIServer(('0.0.0.0', port), self.app.wsgi_handle,
26 | handler_class=WebSocketHandler)
27 | self.bridge_thread = spawn(self.server.serve_forever)
28 | self.rr_thread = spawn(self.rr_func)
29 |
30 | def rr_func(self):
31 | while True:
32 | msg = self.reqrep.recv()
33 | self.reqrep.send(msg)
34 |
35 | def tearDown(self):
36 | self.rr_thread.kill()
37 | self.bridge_thread.kill()
38 |
39 |
40 | def test_reqrep(self):
41 | sock = connect(self.server, "ws://127.0.0.1:" + str(port),
42 | 'tcp://127.0.0.1:' + str(self.rr_port),
43 | zmq.REQ)
44 | sock.send(simplejson.dumps(
45 | {
46 | 'identity' : 'testidentity',
47 | 'msg_type' : 'user',
48 | 'content' : 'MYMSG'
49 | }))
50 | msg = sock.recv()
51 | msgobj = simplejson.loads(msg)
52 | assert msgobj['content'] == 'MYMSG'
53 |
54 |
55 | class SubTest(unittest.TestCase):
56 | def setUp(self):
57 | self.ctx = zmq.Context()
58 | self.pub = self.ctx.socket(zmq.PUB)
59 | self.pub_port = self.pub.bind_to_random_port("tcp://127.0.0.1")
60 | self.app = bridge.WsgiHandler()
61 | self.server = pywsgi.WSGIServer(('0.0.0.0', port), self.app.wsgi_handle,
62 | handler_class=WebSocketHandler)
63 | self.bridge_thread = spawn(self.server.serve_forever)
64 |
65 | def tearDown(self):
66 | self.bridge_thread.kill()
67 |
68 | def test_sub(self):
69 | sock = connect(self.server, "ws://127.0.0.1:" + str(port),
70 | 'tcp://127.0.0.1:' + str(self.pub_port),
71 | zmq.SUB)
72 | self.pub.send('hellohello')
73 | msg = sock.recv()
74 | msgobj = simplejson.loads(msg)
75 | assert msgobj['identity'] == 'testidentity'
76 | assert msgobj['content'] == 'hellohello'
77 | self.pub.send('boingyboingy')
78 | msg = sock.recv()
79 | msgobj = simplejson.loads(msg)
80 | assert msgobj['identity'] == 'testidentity'
81 | assert msgobj['content'] == 'boingyboingy'
82 |
83 | class ClientRepTest(unittest.TestCase):
84 | def setUp(self):
85 | self.ctx = zmq.Context()
86 | self.reqrep = self.ctx.socket(zmq.REQ)
87 | self.reqrep.connect("tcp://127.0.0.1:101010")
88 | self.req_port = 101010
89 | self.app = bridge.WsgiHandler()
90 | self.server = pywsgi.WSGIServer(('0.0.0.0', port), self.app.wsgi_handle,
91 | handler_class=WebSocketHandler)
92 | self.bridge_thread = spawn(self.server.serve_forever)
93 | self.ws_thread = spawn(self.ws_reqrep)
94 |
95 |
96 | def tearDown(self):
97 | self.bridge_thread.kill()
98 | self.ws_thread.kill()
99 |
100 | def ws_reqrep(self):
101 | sock = connect(self.server, "ws://127.0.0.1:" + str(port),
102 | 'tcp://127.0.0.1:' + str(self.req_port),
103 | zmq.REP)
104 | while True:
105 | msg = sock.recv()
106 | log.debug(msg)
107 | msgobj = simplejson.loads(msg)
108 | sock.send(msg)
109 |
110 | def test_req_rep(self):
111 | self.reqrep.send_multipart(['hello', 'testidentity'])
112 | a = self.reqrep.recv_multipart()
113 | assert a[0] == 'hello'
114 |
--------------------------------------------------------------------------------
/test/rpc_test.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import websocket
3 | from gevent_zeromq import zmq
4 | import gevent
5 | import gevent.monkey
6 | gevent.monkey.patch_all()
7 | from gevent import spawn
8 | from geventwebsocket.handler import WebSocketHandler
9 | import bridge
10 | from gevent import pywsgi
11 | import time
12 | import logging
13 | log = logging.getLogger(__name__)
14 | import simplejson
15 | import test_utils
16 | wait_until = test_utils.wait_until
17 | connect = test_utils.connect
18 | import bridgeutils
19 | import geventbridgeutils
20 |
21 | port = 10020
22 |
23 | class TestRPC(geventbridgeutils.GeventZMQRPC):
24 | def echo(self, msg):
25 | return msg
26 |
27 | class ReqRepTest(unittest.TestCase):
28 | def setUp(self):
29 | self.ctx = zmq.Context()
30 | self.reqrep = self.ctx.socket(zmq.REP)
31 | self.rr_port = self.reqrep.bind_to_random_port("tcp://127.0.0.1")
32 | self.app = bridge.WsgiHandler()
33 | self.server = pywsgi.WSGIServer(('0.0.0.0', 9999), self.app.wsgi_handle,
34 | handler_class=WebSocketHandler)
35 | self.bridge_thread = spawn(self.server.serve_forever)
36 | self.rpc = TestRPC(self.reqrep)
37 | self.rr_thread = spawn(self.rpc.run_rpc)
38 |
39 | def tearDown(self):
40 | self.rr_thread.kill()
41 | self.bridge_thread.kill()
42 |
43 | def test_reqrep(self):
44 | sock = connect(self.server, "ws://127.0.0.1:9999",
45 | 'tcp://127.0.0.1:' + str(self.rr_port),
46 | zmq.REQ)
47 |
48 | rpc_request_obj = {'funcname' : 'echo',
49 | 'args' : ['echome'],
50 | 'kwargs' : {}}
51 | rpc_request_msg = simplejson.dumps(rpc_request_obj)
52 | sock.send(simplejson.dumps(
53 | {
54 | 'identity' : 'testidentity',
55 | 'msg_type' : 'user',
56 | 'content' : rpc_request_msg
57 | }))
58 | msg = sock.recv()
59 | msgobj = simplejson.loads(msg)
60 | payload = msgobj['content']
61 | payload = simplejson.loads(payload)
62 | payload = payload['returnval']
63 | assert payload == 'echome'
64 |
65 |
66 | class SubTest(unittest.TestCase):
67 | def setUp(self):
68 | self.ctx = zmq.Context()
69 | self.pub = self.ctx.socket(zmq.PUB)
70 | self.pub_port = self.pub.bind_to_random_port("tcp://127.0.0.1")
71 | self.app = bridge.WsgiHandler()
72 | self.server = pywsgi.WSGIServer(('0.0.0.0', 9999), self.app.wsgi_handle,
73 | handler_class=WebSocketHandler)
74 | self.bridge_thread = spawn(self.server.serve_forever)
75 |
76 | def tearDown(self):
77 | self.bridge_thread.kill()
78 |
79 | def test_sub(self):
80 | sock = connect(self.server, "ws://127.0.0.1:9999",
81 | 'tcp://127.0.0.1:' + str(self.pub_port),
82 | zmq.SUB)
83 | self.pub.send('hellohello')
84 | msg = sock.recv()
85 | msgobj = simplejson.loads(msg)
86 | assert msgobj['identity'] == 'testidentity'
87 | assert msgobj['content'] == 'hellohello'
88 | self.pub.send('boingyboingy')
89 | msg = sock.recv()
90 | msgobj = simplejson.loads(msg)
91 | assert msgobj['identity'] == 'testidentity'
92 | assert msgobj['content'] == 'boingyboingy'
93 |
94 |
95 |
96 | class ClientRepTest(unittest.TestCase):
97 | def setUp(self):
98 | self.ctx = zmq.Context()
99 | self.reqrep = self.ctx.socket(zmq.REQ)
100 | self.reqrep.connect("tcp://127.0.0.1:9010")
101 | self.req_port = 9010
102 | self.app = bridge.WsgiHandler()
103 | self.server = pywsgi.WSGIServer(('0.0.0.0', port), self.app.wsgi_handle,
104 | handler_class=WebSocketHandler)
105 | self.bridge_thread = spawn(self.server.serve_forever)
106 | self.ws_thread = spawn(self.ws_reqrep)
107 |
108 |
109 | def tearDown(self):
110 | self.bridge_thread.kill()
111 | self.ws_thread.kill()
112 |
113 | def ws_reqrep(self):
114 | sock = connect(self.server, "ws://127.0.0.1:" + str(port),
115 | 'tcp://127.0.0.1:' + str(self.req_port),
116 | zmq.REP)
117 | while True:
118 | msg = sock.recv()
119 | log.debug(msg)
120 | msgobj = simplejson.loads(msg)
121 | sock.send(msg)
122 |
123 | def test_req_rep(self):
124 | self.reqrep.send_multipart(['hello', 'testidentity'])
125 | a = self.reqrep.recv_multipart()
126 | assert a[0] == 'hello'
127 |
--------------------------------------------------------------------------------
/bridge.js:
--------------------------------------------------------------------------------
1 |
2 | zmq = {}
3 | zmq.SUB = 2
4 | zmq.REQ = 3
5 | zmq.REP = 4
6 | zmq.uuid = function () {
7 | //from ipython project
8 | // http://www.ietf.org/rfc/rfc4122.txt
9 | var s = [];
10 | var hexDigits = "0123456789ABCDEF";
11 | for (var i = 0; i < 32; i++) {
12 | s[i] = hexDigits.substr(Math.floor(Math.random() * 0x10), 1);
13 | }
14 | s[12] = "4"; // bits 12-15 of the time_hi_and_version field to 0010
15 | s[16] = hexDigits.substr((s[16] & 0x3) | 0x8, 1); // bits 6-7 of the clock_seq_hi_and_reserved to 01
16 |
17 | var uuid = s.join("");
18 | return uuid;
19 | };
20 |
21 | zmq.Context = function(ws_conn_string){
22 | //zmq context proxy. contains a websocketconnection
23 | //routes messages to fake zmq sockets on the js side
24 | this.sockets = {}
25 | var that = this;
26 | try {
27 | this.s = new WebSocket(ws_conn_string);
28 | }
29 | catch (e) {
30 | this.s = new MozWebSocket(ws_conn_string);
31 | }
32 | this.s.onmessage = function(msg){
33 | var msgobj = JSON.parse(msg.data);
34 | var socket = that.sockets[msgobj['identity']]
35 | if (socket.connected){
36 | socket._handle(msgobj['content']);
37 | }else if (msgobj['msg_type'] === 'connection_reply'){
38 | that.onconnect(socket, msgobj['content']);
39 | }
40 | }
41 | this.s.onopen = function(){
42 | that.connected = true;
43 | $.map(that.send_buffer, function(x){
44 | that.s.send(x);
45 | });
46 | that.send_buffer = [];
47 | }
48 | this.connected = false;
49 | this.send_buffer = [];
50 | }
51 |
52 | zmq.Context.prototype.connect = function(socket, zmq_conn_string, auth){
53 | auth['zmq_conn_string'] = zmq_conn_string;
54 | auth['socket_type'] = socket.socket_type;
55 | var msg = JSON.stringify(auth)
56 | var msgobj = socket.construct_message(msg, 'connect')
57 | msg = JSON.stringify(msgobj);
58 | this.send(msg);
59 | }
60 |
61 | zmq.Context.prototype.onconnect = function(socket, msg){
62 | var msgobj = JSON.parse(msg);
63 | if (msgobj['status'] === 'success'){
64 | socket.connected = true;
65 | }else{
66 | socket.connected = false;
67 | }
68 | }
69 |
70 | zmq.Context.prototype.Socket = function(socket_type){
71 | //contexts are also a factory for sockets, just like
72 | //in normal zeromq
73 | var fakesocket;
74 | if (socket_type === zmq.SUB){
75 | fakesocket = new zmq.SubSocket(this);
76 | }else if (socket_type === zmq.REQ){
77 | fakesocket = new zmq.ReqSocket(this);
78 | }else if (socket_type === zmq.REP){
79 | fakesocket = new zmq.RepSocket(this);
80 | }
81 | this.sockets[fakesocket.identity] = fakesocket;
82 | return fakesocket;
83 | }
84 |
85 | zmq.Context.prototype.send = function(msg){
86 | if (this.connected){
87 | this.s.send(msg);
88 | }else{
89 | this.send_buffer.push(msg);
90 | }
91 | }
92 |
93 | zmq.Socket = function(ctx){
94 | this.ctx = ctx
95 | this.identity = zmq.uuid();
96 | }
97 |
98 | zmq.Socket.prototype.construct_message = function(msg, msg_type){
99 | //your message should be a string
100 | //constructs a message object, as json
101 | //this will be serialized before it goes to the wire
102 | if(!msg_type){
103 | msg_type = 'userlevel'
104 | }
105 | return {
106 | 'identity' : this.identity,
107 | 'content' : msg,
108 | 'msg_type' : msg_type
109 | }
110 | }
111 |
112 | zmq.Socket.prototype.connect = function(zmq_conn_string, auth){
113 | this.ctx.connect(this, zmq_conn_string, auth);
114 | }
115 |
116 | zmq.ReqSocket = function(ctx){
117 | zmq.Socket.call(this, ctx);
118 | this.reqrep_buffer = [];
119 | this.busy = false;
120 | this.socket_type = zmq.REQ;
121 | }
122 | zmq.ReqSocket.prototype = new zmq.Socket();
123 | zmq.ReqSocket.prototype.send = function(msg, callback, msg_type){
124 | var msgobj = this.construct_message(msg, msg_type)
125 | this._send(JSON.stringify(msgobj), callback);
126 | }
127 | zmq.ReqSocket.prototype._send = function(msg, callback){
128 | this.reqrep_buffer.push([msg, callback]);
129 | if (this.busy){
130 | return
131 | }else{
132 | this._send_buffer();
133 | }
134 | }
135 | zmq.ReqSocket.prototype._send_buffer = function(){
136 | if (this.busy || this.reqrep_buffer.length == 0){
137 | return
138 | }else{
139 | this.busy = true;
140 | this.ctx.send(this.reqrep_buffer[0][0]);
141 | return
142 | }
143 | }
144 |
145 | zmq.ReqSocket.prototype._handle = function(msg){
146 | this.busy = false;
147 | var callback = this.reqrep_buffer[0][1]
148 | this.reqrep_buffer = this.reqrep_buffer.slice(1);
149 | callback(msg);
150 | this._send_buffer();
151 | }
152 |
153 |
154 | zmq.SubSocket = function(ctx){
155 | zmq.ReqSocket.call(this, ctx);
156 | this.socket_type = zmq.SUB;
157 | }
158 |
159 | //prototype from req socket, because we need the auth functionality
160 | zmq.SubSocket.prototype = new zmq.Socket();
161 | zmq.SubSocket.prototype._handle = function(msg){
162 | this.onmessage(msg);
163 | }
164 |
165 | zmq.RepSocket = function(ctx){
166 | zmq.ReqSocket.call(this, ctx);
167 | this.socket_type = zmq.REP;
168 | this.in_buffer = [];
169 | },
170 |
171 | zmq.RepSocket.prototype = new zmq.Socket();
172 |
173 | zmq.RepSocket.prototype.send = function(msg){
174 | //layer adressing information
175 | var msg = [this.address, '', msg]
176 | msg = JSON.stringify(msg);
177 | var msgobj = this.construct_message(msg)
178 | this.ctx.send(JSON.stringify(msgobj));
179 | //this is a reply, so now we are no longer busy
180 | this.busy = false;
181 | //try to process in_buffer
182 | this._recv_buffer();
183 | }
184 | zmq.RepSocket.prototype._recv_buffer = function(){
185 | if (this.busy || this.in_buffer.length == 0){
186 | return
187 | }else{
188 | var msg = this.in_buffer[0];
189 | this.in_buffer = this.in_buffer.slice(1);
190 |
191 | this.busy = true;
192 | var msgobj = JSON.parse(msg);
193 | var address = msgobj[0];
194 | var content = msgobj[msgobj.length - 1];
195 | this.address = address;
196 | this.onmessage(content);
197 | }
198 | }
199 |
200 | zmq.RepSocket.prototype._handle = function(msg){
201 | this.in_buffer.push(msg);
202 | this._recv_buffer();
203 | }
204 |
205 | zmq.RPCClient = function(socket){
206 | this.socket = socket;
207 | }
208 |
209 | zmq.RPCClient.prototype.rpc = function(funcname, args, kwargs, callback){
210 | msg = {'funcname' : funcname,
211 | 'args' : args,
212 | 'kwargs' : kwargs}
213 | var wrapped_callback = function (msg){
214 | var msgobj = JSON.parse(msg);
215 | callback(msgobj['returnval']);
216 | }
217 | this.socket.send(JSON.stringify(msg), wrapped_callback);
218 | }
219 |
220 | zmq.PubRPCServer = function(socket){
221 | this.socket = socket;
222 | var that = this;
223 | if (socket){
224 | socket.onmessage = function(msg){
225 | that.handle_pub(msg);
226 | }
227 | }
228 | }
229 | zmq.PubRPCServer.prototype.handle_pub = function(msg){
230 | var msgobj = JSON.parse(msg)
231 | var funcname = msgobj['funcname']
232 | var args = msgobj['args'] || [];
233 | this[funcname].apply(this, args);
234 | }
235 |
236 | zmq.RPCServer = function(socket){
237 | this.socket = socket;
238 | var that = this;
239 | if (socket){
240 | socket.onmessage = function(msg){
241 | that.handle(msg);
242 | }
243 | }
244 | }
245 |
246 | zmq.RPCServer.prototype.handle = function(msg){
247 | var msgobj = JSON.parse(msg)
248 | var funcname = msgobj['funcname']
249 | var args = msgobj['args'] || [];
250 | try{
251 | retval = this[funcname].apply(this, args);
252 | }catch(err){
253 | retval = err;
254 | }
255 | this.socket.send(JSON.stringify({'returnval' : retval}));
256 | }
--------------------------------------------------------------------------------
/README.rst:
--------------------------------------------------------------------------------
1 | ============
2 | ZmqWebBridge
3 | ============
4 |
5 | The point of this project is to allow javascript client code to interact with arbitrary zeromq services in the backend, with minimal server side configuration. In general, when people write websocket code, they write alot of application specific web socket code in whichever web framework they are using. We want to push most of the application logic into generic zmq processes, with the only application code running in the web server being the code necessary to authenticate users and decide who can access which resources.
6 |
7 | Zmq Web Bridge, is a project that allows javascript running in the browser to connect to zmq.REP and zmq.PUB sockets running on a server, via a python gevent proxy.
8 | The javascript client code sets up a websocket, which communicates with the python gevent proxy, which forwards communications to various zmq backends. This package also provides a basic framework for authentication, to determine who can access which zmq resources.
9 |
10 | * you need to know what you're doing before you use this in production, zeromq isn't written for public facing apps, the way most web frameworks are created some possible problems that aren't apparent at first glance
11 |
12 | 1. if a client sends a malformed zmq connection string, your bridge will fail (this failure occurs on the native code side, so we can't catch it in python)
13 | 2. if you use a REP socket, and you call socket.recv_json(), and some random client in the world is NOT sending you json, it'll bust your REQ REP ping pong cycle. so you have to handle those types of things accordingly.
14 |
15 | * I recently tried to deploy this on another machine, and it would not work with the pyzmq installed there, I tracked it down to some versioning issues which caused XREQs and XREPs not to function properly, reinstalling fresh pyzmq from git (and stable zeromq 2.1) solved the issue. I'll try adding a test (there are no tests yet) which will tell you if your XREQs and XREPs are working properly
16 |
17 | Syntax
18 | ------
19 |
20 | js reqrep
21 | ----------
22 | ::
23 |
24 | context = new zmq.Context('wss://localhost:8000/data');
25 | reqrep = context.Socket(zmq.REQ);
26 | reqrep.connect('tcp://127.0.0.1:10001', {});
27 |
28 | reqrep2 = context.Socket(zmq.REQ);
29 | reqrep2.connect('tcp://127.0.0.1:10001', {});
30 | reqrep.send('hello', function(x){
31 | //callback for request
32 | console.log(x)
33 | }
34 | );
35 | reqrep2.send('hello', function(x){
36 | //callback for request
37 | console.log(x + 'sdf');
38 | console.log(x + 'sfsf');
39 | });
40 |
41 |
42 | js pub sub:
43 | -----------
44 | ::
45 |
46 | sub = context.Socket(zmq.SUB);
47 | sub.onmessage = function(x){
48 | console.log(['sub', x]);
49 | }
50 | sub.connect("tcp://127.0.0.1:10002", {});
51 |
52 | sever side python:
53 | generic, zmq code, which is not apart of this framework:
54 |
55 | python : your application bridge
56 | --------------------------------
57 | ::
58 |
59 |
60 | from gevent import pywsgi
61 | from geventwebsocket.handler import WebSocketHandler
62 | import bridge
63 | import simplejson
64 |
65 | logging.basicConfig(level=logging.DEBUG)
66 | class MyBridgeClass(bridge.BridgeWebProxyHandler):
67 | def zmq_allowed(self, params):
68 | params = simplejson.loads(params)
69 | zmq_conn_string = params['zmq_conn_string']
70 | socket_type = params['socket_type']
71 | return params['username'] == 'hugo'
72 |
73 |
74 | class MyWsgiHandler(bridge.WsgiHandler):
75 | bridge_class = MyBridgeClass
76 | HWM = 100 #zmq HWM must be set here, and on your server
77 | def websocket_allowed(self, environ):
78 | #you can add logic here to do auth
79 | return True
80 |
81 | app = MyWsgiHandler()
82 | server = pywsgi.WSGIServer(('0.0.0.0', 8000), app.wsgi_handle,
83 | # keyfile='/etc/nginx/server.key',
84 | # certfile='/etc/nginx/server.crt',
85 | handler_class=WebSocketHandler)
86 | server.serve_forever()
87 |
88 | python req rep server, and pub server
89 | -------------------------------------
90 | ::
91 |
92 | (this has nothing to do with the bridge, it's just a generic zmq process)
93 | import zmq
94 | import time
95 |
96 | c = zmq.Context()
97 | s = c.socket(zmq.REP)
98 | s.bind('tcp://127.0.0.1:10001')
99 |
100 | while True:
101 | msg = s.recv_multipart()
102 | print 'received', msg
103 | s.send_multipart(['goober'])
104 |
105 | ------------------
106 | import zmq
107 | import time
108 |
109 | c = zmq.Context()
110 | s = c.socket(zmq.PUB)
111 | s.bind('tcp://127.0.0.1:10002')
112 | while(True):
113 | for c in range(100):
114 | print c
115 | s.send(str(c))
116 | time.sleep(1)
117 |
118 |
119 |
120 | bridge code structure
121 | ---------------------
122 | ZmqGatewayFactory - returns an existing zeromq gateway if we have one, otherwise
123 | constructs a new onew
124 |
125 | WebProxyHandler - generic handler which works with proxy objects, proxies can
126 | register with WebProxyHandler, and deregister with them
127 |
128 | * ZmqGateway - proxy handler which handles the zeromq side of things.
129 |
130 | 1. SubGateway - sub socket version
131 |
132 | 2. ReqGateway - request socket version
133 |
134 | * BridgeWebProxyHandler - proxy handler which handles the web socket side of things.
135 | you have one of these per web socket connection. it listens on the web
136 | socket, and when a connection request is received, grabs the appropriate
137 | zeromq gateway from the factory. It also registers the proxy with this
138 | object nad the zeromq gateway
139 |
140 | * SocketProxy
141 |
142 | 1. ReqSocketProxy
143 |
144 | 2. SubSocketProxy
145 |
146 | these proxy objects below are dumb objects. all they do is manage
147 | relationships with their reqpective websocket and zeromq gateways.
148 | the gateways use this object to get to the appropriate opposing gateway
149 | you have one instance of this, for every fake zeromq socket you have on the
150 | js side
151 |
152 | =============
153 | RPC Interface
154 | =============
155 |
156 |
157 | We've also built in an RPC interface
158 |
159 | Request Reply
160 | -------------
161 |
162 | * Python
163 |
164 | You define functions that you want to be able to call from js. the functions
165 | can be called with args, and kwargs. the return value is passed down
166 | to the JS callback. a list of authorized functions is specified for each
167 | RPC server, if it is None then all functions are fair game. the can_function
168 | prefixed function is called, if it exists, and if it returns false,
169 | we don't execute the main function. This is where you would
170 | build in any method level
171 | authentication
172 |
173 | ::
174 |
175 | class TestRPC(bridgeutils.GeventZMQRPC):
176 | authorized_functions = ['echo', 'add_user']
177 | def echo(self, msg):
178 | return msg
179 |
180 | def add_user(self, username, password, type='boy'):
181 | return {'status' : "success"}
182 |
183 | def can_add_user(self, username, password, type='boy'):
184 | return True
185 |
186 | * JS
187 |
188 | Javascript RPC client is instantiated with an instance of the socket.
189 | rpc takes 4 arguments, function name, args, kwargs, and the callback.
190 | the callback gets the object that was returned - any JSON object is
191 | supported
192 |
193 | ::
194 |
195 | rpc_client = new zmq.RPCClient(socket);
196 | rpc_client.rpc('echo', 'hello!', {}, function(response){
197 | console.log(response);
198 | });
199 | rpc_client.rpc('echo', 'hello!',
200 | {type : 'girl'},
201 | function(response){
202 | console.log(response['status']);
203 | });
204 |
205 |
206 | Pub Sub
207 | -------
208 |
209 | Pub sub allows the server, to remotely call functions on the client - though
210 | since the communciation is one way, there is no return value. Also since
211 | JS does not support key word args, we only support positional arguments
212 |
213 | * Python
214 |
215 | ::
216 |
217 | rpc_client = bridgeutils.PubSubRPCCLient(socket)
218 | rpc_client.rpc('add', 1, 2)
219 |
220 | * JS
221 |
222 | ::
223 |
224 | rpc_server = zmq.PubRPCServer()
225 | rpc_server.prototype.add = function(first, second){
226 | console.log(first + second);
227 | }
228 |
229 |
230 |
--------------------------------------------------------------------------------
/bridge.py:
--------------------------------------------------------------------------------
1 | import gevent
2 | import gevent.monkey
3 | gevent.monkey.patch_all()
4 | from geventwebsocket.handler import WebSocketHandler
5 | from gevent import pywsgi
6 | from gevent_zeromq import zmq
7 | import logging
8 | log = logging.getLogger(__name__)
9 | import simplejson
10 | from gevent import spawn
11 | import Queue
12 | import hashlib
13 |
14 | # demo app
15 | class ZmqGatewayFactory(object):
16 | """ factory returns an existing gateway if we have one,
17 | or creates a new one and starts it if we don't
18 | """
19 | def __init__(self, HWM=100):
20 | self.gateways = {}
21 | self.ctx = zmq.Context()
22 | self.HWM = HWM
23 |
24 | def get(self, socket_type, zmq_conn_string):
25 | if (socket_type, zmq_conn_string) in self.gateways:
26 | gateway = self.gateways[socket_type, zmq_conn_string]
27 | return gateway
28 | else:
29 | if socket_type == zmq.REQ:
30 | log.debug("spawning req socket %s" ,zmq_conn_string)
31 | self.gateways[socket_type, zmq_conn_string] = \
32 | ReqGateway(zmq_conn_string,
33 | self.HWM,
34 | ctx=self.ctx)
35 | elif socket_type == zmq.REP:
36 | log.debug("spawning rep socket %s" ,zmq_conn_string)
37 | self.gateways[socket_type, zmq_conn_string] = \
38 | RepGateway(zmq_conn_string,
39 | self.HWM,
40 | ctx=self.ctx)
41 | else:
42 | log.debug("spawning sub socket %s" ,zmq_conn_string)
43 | self.gateways[socket_type, zmq_conn_string] = \
44 | SubGateway(zmq_conn_string,
45 | self.HWM,
46 | ctx=self.ctx)
47 | self.gateways[socket_type, zmq_conn_string].start()
48 | return self.gateways[socket_type, zmq_conn_string]
49 |
50 | def shutdown(self):
51 | """
52 | Close all sockets associated with this context, and then
53 | terminate the context.
54 | """
55 | self.ctx.destroy()
56 |
57 | class WebProxyHandler(object):
58 | """ generic handler which works with proxy objects, proxies can
59 | register with WebProxyHandler, and deregister with them
60 | """
61 | def __init__(self):
62 | self.proxies = {}
63 |
64 | def register(self, identity, proxy):
65 | self.proxies[identity] = proxy
66 |
67 | def deregister(self, identity):
68 | try:
69 | self.proxies.pop(identity)
70 | except KeyError as e:
71 | pass
72 |
73 | def close(self):
74 | for v in self.proxies.values():
75 | v.deregister()
76 |
77 | class ZmqGateway(WebProxyHandler):
78 | """ proxy handler which handles the zeromq side of things.
79 | """
80 | def __init__(self, zmq_conn_string, ctx=None):
81 | super(ZmqGateway, self).__init__()
82 | self.zmq_conn_string = zmq_conn_string
83 | self.ctx = ctx
84 |
85 | def send_proxy(self, identity, msg):
86 | try:
87 | self.proxies[identity].send_web(msg)
88 | #what exception is thrown here?
89 | except Exception as e:
90 | log.exception(e)
91 | self.deregister(identity)
92 |
93 | class SubGateway(ZmqGateway):
94 | def __init__(self, zmq_conn_string, HWM, ctx=None):
95 | super(SubGateway, self).__init__(zmq_conn_string, ctx=ctx)
96 | self.s = ctx.socket(zmq.SUB)
97 | self.s.setsockopt(zmq.SUBSCRIBE, '');
98 | if HWM:
99 | self.s.setsockopt(zmq.HWM, HWM);
100 | self.s.connect(zmq_conn_string)
101 |
102 | def run(self):
103 | while(True):
104 | msg = self.s.recv(copy=True)
105 | try:
106 | log.debug('subgateway, received %s', msg)
107 | for k in self.proxies.keys():
108 | if self.proxies[k].msgfilter in msg:
109 | self.send_proxy(k, msg)
110 | except Exception as e:
111 | log.exception(e)
112 | continue
113 |
114 | def start(self):
115 | self.thread = spawn(self.run)
116 |
117 | class RepGateway(ZmqGateway):
118 | def __init__(self, zmq_conn_string, HWM, ctx=None):
119 | super(RepGateway, self).__init__(zmq_conn_string, ctx=ctx)
120 | self.s = ctx.socket(zmq.XREP)
121 | if HWM:
122 | self.s.setsockopt(zmq.HWM, 100);
123 | self.s.bind(zmq_conn_string)
124 | self.queue = Queue.Queue()
125 | self.addresses = {}
126 |
127 | def send(self, identity, msg):
128 | self.queue.put(msg)
129 |
130 | def _send(self, multipart_msg):
131 | multipart_msg = [str(x) for x in multipart_msg]
132 | log.debug('sending %s', multipart_msg)
133 | self.s.send_multipart(multipart_msg)
134 |
135 | def run_recv_zmq(self):
136 | while True:
137 | msg = self.s.recv_multipart(copy=True)
138 | log.debug('received %s', msg)
139 | try:
140 | target_ident = msg[-1]
141 | address_idx = msg.index('')
142 | address_data = msg[:address_idx]
143 | hashval = hashlib.sha1(str(address_data)).hexdigest()
144 | self.addresses[hashval] = address_data
145 | newmsg = [hashval] + [str(x) for x in \
146 | msg[address_idx:-1]]
147 | msg = simplejson.dumps(newmsg)
148 | self.send_proxy(target_ident, msg)
149 | except:
150 | pass
151 |
152 | def run_send_zmq(self):
153 | while True:
154 | try:
155 | obj = self.queue.get()
156 | log.debug('ws received %s', obj)
157 | obj = simplejson.loads(obj)
158 | address_data = self.addresses[obj[0]]
159 | self._send(address_data + obj[1:])
160 | except:
161 | pass
162 |
163 | def start(self):
164 | self.thread_recv = spawn(self.run_recv_zmq)
165 | self.thread_send = spawn(self.run_send_zmq)
166 |
167 | class ReqGateway(ZmqGateway):
168 | def __init__(self, zmq_conn_string, HWM, ctx=None):
169 | super(ReqGateway, self).__init__(zmq_conn_string, ctx=ctx)
170 | self.s = ctx.socket(zmq.XREQ)
171 | if HWM:
172 | self.s.setsockopt(zmq.HWM, 100);
173 | self.s.connect(zmq_conn_string)
174 | self.queue = Queue.Queue()
175 |
176 | def send(self, identity, msg):
177 | self.queue.put((identity, msg))
178 |
179 | def _send(self, identity, msg):
180 | #append null string to front of message, just like REQ
181 | #embed identity the same way
182 | self.s.send_multipart([str(identity), '', str(msg)])
183 | #log.debug('reqgateway, sent %s', msg)
184 |
185 | def handle_request(self, msg):
186 | #strip off the trailing string
187 | identity = msg[0]
188 | msg = msg[-1]
189 | self.send_proxy(identity, msg)
190 |
191 | def start(self):
192 | self.thread_recv = spawn(self.run_recv_zmq)
193 | self.thread_send = spawn(self.run_send_zmq)
194 |
195 | def run_recv_zmq(self):
196 | while True:
197 | msg = self.s.recv_multipart(copy=True)
198 | try:
199 | log.debug('reqgateway, received %s', msg)
200 | self.handle_request(msg)
201 | except Exception as e:
202 | log.exception(e)
203 | continue
204 |
205 | def run_send_zmq(self):
206 | while True:
207 | try:
208 | obj = self.queue.get()
209 | identity, msg = obj
210 | self._send(identity, msg)
211 | except:
212 | pass
213 |
214 | class BridgeWebProxyHandler(WebProxyHandler):
215 | """
216 | should rename this to BridgeWebSocketGateway
217 | proxy handler which handles the web socket side of things.
218 | you have one of these per web socket connection. it listens on the web
219 | socket, and when a connection request is received, grabs the appropriate
220 | zeromq gateway from the factory. It also registers the proxy with this
221 | object nad the zeromq gateway
222 | """
223 |
224 | def __init__(self, ws, gateway_factory):
225 | super(BridgeWebProxyHandler, self).__init__()
226 | self.ws = ws
227 | self.gateway_factory = gateway_factory
228 |
229 | def zmq_allowed(self, options):
230 | return True
231 |
232 | def connect(self, identity, content):
233 | content = simplejson.loads(content);
234 | zmq_conn_string = content['zmq_conn_string']
235 | socket_type = content['socket_type']
236 | if socket_type == zmq.REQ:
237 | proxy = ReqSocketProxy(identity)
238 | elif socket_type == zmq.REP:
239 | proxy = RepSocketProxy(identity)
240 | else:
241 | proxy = SubSocketProxy(identity, content.get('msgfilter', ''))
242 | gateway = self.gateway_factory.get(socket_type, zmq_conn_string)
243 | proxy.register(self, gateway)
244 |
245 | def handle_request(self, msg):
246 | msg = simplejson.loads(msg)
247 |
248 | msg_type = msg.get('msg_type')
249 | identity = msg.get('identity')
250 | content = msg.get('content')
251 |
252 | if msg_type == 'connect':
253 | if self.zmq_allowed(content):
254 | self.connect(identity, content)
255 | content = simplejson.dumps({'status' : 'success'})
256 | self.send(identity, content, msg_type='connection_reply')
257 | else:
258 | content = simplejson.dumps({'status' : 'error'})
259 | self.send(identity, content, msg_type='connection_reply')
260 | else:
261 | self.send_proxy(identity, content)
262 |
263 | def send_proxy(self, identity, content):
264 | try:
265 | self.proxies[identity].send_zmq(content)
266 | #what exception is thrown here?
267 | except Exception as e:
268 | log.exception(e)
269 | self.deregister(identity)
270 |
271 | def send(self, identity, msg, msg_type=None):
272 | json_msg = {'identity' : identity,
273 | 'content' : msg}
274 | if msg_type is not None:
275 | json_msg['msg_type'] = msg_type
276 | log.debug('ws sent %s', json_msg)
277 | self.ws.send(simplejson.dumps(json_msg))
278 |
279 | def run(self):
280 | while True:
281 | msg = self.ws.receive()
282 | #log.debug('ws received %s', msg)
283 | if msg is None:
284 | self.close()
285 | break
286 | self.handle_request(msg)
287 |
288 | """these proxy objects below are dumb objects. all they do is manage
289 | relationships with their reqpective websocket and zeromq gateways.
290 | the gateways use this object to get to the appropriate opposing gateway
291 | SocketProxy
292 | ReqSocketProxy
293 | SubSocketProxy
294 | you have one instance of this, for every fake zeromq socket you have on the
295 | js side
296 | """
297 | class SocketProxy(object):
298 |
299 | def __init__(self, identity):
300 | self.identity = identity
301 |
302 | def register(self, wsgateway, zmqgateway):
303 | self.wsgateway = wsgateway
304 | self.zmqgateway = zmqgateway
305 | wsgateway.register(self.identity, self)
306 | zmqgateway.register(self.identity, self)
307 |
308 | def deregister(self):
309 | self.wsgateway.deregister(self.identity)
310 | self.zmqgateway.deregister(self.identity)
311 |
312 | def send_web(self, msg):
313 | self.wsgateway.send(self.identity, msg)
314 |
315 | def send_zmq(self, msg):
316 | self.zmqgateway.send(self.identity, msg)
317 |
318 | class ReqSocketProxy(SocketProxy):
319 | socket_type = zmq.REQ
320 |
321 | class RepSocketProxy(SocketProxy):
322 | socket_type = zmq.REP
323 |
324 |
325 | class SubSocketProxy(SocketProxy):
326 | socket_type = zmq.SUB
327 | def __init__(self, identity, msgfilter):
328 | super(SubSocketProxy, self).__init__(identity)
329 | self.msgfilter = msgfilter
330 |
331 |
332 |
333 | """
334 | Gevent wsgi handler - the main server. once instnace of this per process
335 | """
336 | class WsgiHandler(object):
337 | bridge_class = BridgeWebProxyHandler
338 | HWM = 100
339 | def __init__(self):
340 | self.zmq_gateway_factory = ZmqGatewayFactory(self.HWM)
341 |
342 | def websocket_allowed(self, environ):
343 | return True
344 |
345 | def wsgi_handle(self, environ, start_response):
346 | if 'wsgi.websocket' in environ and self.websocket_allowed(environ):
347 | handler = self.bridge_class(environ['wsgi.websocket'],
348 | self.zmq_gateway_factory)
349 | handler.run()
350 | else:
351 | start_response("404 Not Found", [])
352 | return []
353 |
354 | def __del__(self):
355 | """
356 | Upon destruction shut down any open sockets, don't rely
357 | on the garbage collector which can leave sockets
358 | dangling open.
359 | """
360 | self.zmq_gateway_factory.shutdown()
361 |
362 | if __name__ == "__main__":
363 | logging.basicConfig(level=logging.DEBUG)
364 | app = WsgiHandler()
365 | server = pywsgi.WSGIServer(('127.0.0.1', 8000), app.wsgi_handle,
366 | # keyfile='/etc/nginx/server.key',
367 | # certfile='/etc/nginx/server.crt',
368 | handler_class=WebSocketHandler)
369 | try:
370 | server.serve_forever()
371 | except KeyboardInterrupt:
372 | print 'Shutting down gracefully.'
373 | server.zmq_gateway_factory.shutdown()
374 |
--------------------------------------------------------------------------------