├── whizzer ├── rpc │ ├── __init__.py │ ├── dispatch.py │ ├── proxy.py │ ├── service.py │ ├── msgpackrpc.py │ └── picklerpc.py ├── test │ ├── __init__.py │ ├── common.py │ ├── test_process.py │ ├── test_protocol.py │ ├── mocks.py │ ├── test_server.py │ ├── test_transport.py │ ├── test_client.py │ ├── test_defer.py │ ├── test_rpc_pickle.py │ └── test_rpc_msgpack.py ├── __init__.py ├── protocol.py ├── debug.py ├── process.py ├── transport.py ├── server.py ├── client.py └── defer.py ├── .gitignore ├── MANIFEST.in ├── .hgignore ├── docs ├── index.rst ├── concepts.rst ├── make.bat ├── Makefile └── conf.py ├── benchmarks └── iteration.py ├── LICENSE ├── README ├── examples ├── httpserver.py ├── pingpongweb.py ├── server.py ├── client.py ├── servicefork.py └── forked.py └── setup.py /whizzer/rpc/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /whizzer/test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.pyo 3 | *.un~ 4 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | recursive-include whizzer *.py 3 | 4 | -------------------------------------------------------------------------------- /.hgignore: -------------------------------------------------------------------------------- 1 | syntax: glob 2 | build 3 | .lock-wscript 4 | *.swp 5 | *.pyc 6 | *.pyo 7 | *~ 8 | *_build 9 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Whizzer Reference Documentation 2 | =============================== 3 | 4 | Whizzer is a framework for asynchronous programming with python. 5 | 6 | Introduction to Whizzer 7 | ---------------------------- 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | 12 | gettingstarted 13 | concepts 14 | 15 | .. toctree:: 16 | 17 | 18 | 19 | Module Reference 20 | ---------------- 21 | 22 | Indices and tables 23 | ================== 24 | 25 | * :ref:`genindex` 26 | * :ref:`modindex` 27 | * :ref:`search` 28 | -------------------------------------------------------------------------------- /benchmarks/iteration.py: -------------------------------------------------------------------------------- 1 | import pyev 2 | import time 3 | 4 | def stop(watcher, events): 5 | watcher.loop.stop() 6 | 7 | 8 | def timed(watcher, events): 9 | pass 10 | 11 | 12 | 13 | 14 | loop = pyev.default_loop() 15 | t1 = pyev.Timer(10.0, 0.0, loop, stop) 16 | t2 = pyev.Timer(0.00000001, 0.00000001, loop, timed) 17 | t1.start() 18 | t2.start() 19 | 20 | before = time.time() 21 | loop.start() 22 | after = time.time() 23 | 24 | diff = after-before 25 | 26 | ips = loop.iteration/diff 27 | 28 | print("iterations per second: %f" % ips) 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2010 Tom Burdick 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | 21 | 22 | -------------------------------------------------------------------------------- /whizzer/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2010 Tom Burdick 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /whizzer/test/common.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2010 Tom Burdick 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in 11 | # all copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | # THE SOFTWARE. 20 | 21 | import pyev 22 | 23 | loop = None 24 | 25 | if not loop: 26 | loop = pyev.default_loop() 27 | -------------------------------------------------------------------------------- /whizzer/test/test_process.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2010 Tom Burdick 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in 11 | # all copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | # THE SOFTWARE. 20 | 21 | import unittest 22 | 23 | import pyev 24 | 25 | from whizzer.defer import Deferred 26 | from whizzer.process import Process 27 | from mocks import * 28 | from common import loop 29 | 30 | def run(): 31 | print("ran") 32 | 33 | 34 | class TestProcess(unittest.TestCase): 35 | def test_create(self): 36 | p = Process(loop, run) 37 | 38 | if __name__ == '__main__': 39 | unittest.main() 40 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | Whizzer 2 | ======= 3 | 4 | Whizzer is a python library to help write fast non-blocking socket servers. 5 | 6 | Whizzer uses class templates to write callback driven programs. Callbacks 7 | are initiated from the C event loop library libev and its python wrapper 8 | pyev. The event loop itself is as fast as any other modern C event loop using 9 | epoll and heap timers. It is much faster than an event loop written purely 10 | in python. 11 | 12 | Whizzer supports limited coroutines as well without any form of stack storing 13 | by recursively calling in to the main loop. Coroutines are limited by the 14 | size of the stack and the stack depth limit set in python both of which are 15 | relatively simple to adjust though limited. It does not attempt to store task 16 | state between task switching in any novel ways that may break C extensions. 17 | 18 | Whizzer is similiar in style to twisted in that it provides class templates 19 | which may be derived and further defined to implement only the necessary 20 | functionality. Implementing a protocol in Whizzer means deriving the Protocol 21 | class and implementing a few methods. 22 | 23 | Whizzer strives to be very fast, tested, and relatively simple. 24 | 25 | Whizzer attempts to have the following features packaged with it. 26 | 27 | * Python 2/3 compatible 28 | * Namely Python 2.7 and python 3.2 or better 29 | * Fast RPC protocol implementations (msgpack-rpc, json-rpc, pickle-rpc) 30 | * 100k+ notifies/s (one way calls, no responses) 31 | * 20k+ calls/s (request/response rpc calls) 32 | 33 | In the future it would be nice to have a WSGI handler based on Ryan Dahl's C 34 | http protocol parser. 35 | 36 | 37 | Ad-Hoc Benchmarks 38 | ================= 39 | 40 | On my Athlon II X4 I can get the examples/servicefork.py to show close to 41 | 400k notifications a second. 42 | -------------------------------------------------------------------------------- /examples/httpserver.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2010 Tom Burdick 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | import sys 23 | import time 24 | import signal 25 | import logbook 26 | from logbook.more import ColorizedStderrHandler 27 | 28 | import pyev 29 | 30 | sys.path.insert(0, '..') 31 | 32 | from whizzer.server import TcpServer 33 | from whizzer.http import HTTPProtocol 34 | 35 | logger = logbook.Logger('http server') 36 | 37 | 38 | def main(): 39 | loop = pyev.default_loop() 40 | 41 | signal_handler = whizzer.signal_handler(loop) 42 | signal_handler.start() 43 | 44 | factory = HTTPProtocolFactory() 45 | server = whizzer.TcpServer(loop, factory, "127.0.0.1", 2000, 256) 46 | 47 | signal_handler.start() 48 | server.start() 49 | 50 | loop.loop() 51 | 52 | if __name__ == "__main__": 53 | stderr_handler = ColorizedStderrHandler(level='DEBUG') 54 | with stderr_handler.applicationbound(): 55 | main() 56 | -------------------------------------------------------------------------------- /docs/concepts.rst: -------------------------------------------------------------------------------- 1 | ======== 2 | Concepts 3 | ======== 4 | 5 | *********************** 6 | Socket Programming 7 | *********************** 8 | 9 | Sockets are a way to stream bytes of data between programs, computers, even 10 | outside of our very planet. However they have pitfalls that often times a 11 | newcomer may miss. I know I certainly did. 12 | 13 | Socket programming can be somewhat frustrating at times. For example 14 | consider the two pieces of code below, about the simplest client and server 15 | possible in python. 16 | 17 | .. code-block:: python 18 | 19 | import socket 20 | 21 | mysock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 22 | mysock.connect(("10.10.10.10", 2000)) 23 | mysock.send("Hello!") 24 | print("Sent Hello!") 25 | 26 | .. code-block:: python 27 | 28 | import socket 29 | 30 | mysock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 31 | mysock.bind(("0.0.0.0", 2000)) 32 | mysock.listen(1) 33 | 34 | (connection,address) = mysock.accept() 35 | print(connection.recv(6)) 36 | 37 | 38 | Depending on the kind of connection the mysock.send and connection.recv could 39 | a long time to complete. This leaves both programs simply waiting around doing 40 | nothing particularly useful. 41 | 42 | ********************** 43 | Two Rivaling Solutions 44 | ********************** 45 | 46 | There are two rivaling solutions to the problem of waiting for sockets and in 47 | the more general sense input/output operations. Fork, threads, and select. All 48 | of the solutions have their uses and using them together in some instances can 49 | lead to greater performance. In the case of python the best solution is really 50 | select. Python's GIL tends to scare most of us away from threads as a solution 51 | to parallel execution. 52 | 53 | ********************** 54 | Pyev 55 | ********************** 56 | 57 | Whizzer is heavily based on pyev. Pyev acts as a loop with a set of event 58 | watchers. When the event the watcher is looking for happens the callback 59 | given to it when constructed is called. Pyev is very fast. 60 | 61 | -------------------------------------------------------------------------------- /examples/pingpongweb.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2010 Tom Burdick 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | """Ping Pong Web, Nicholas Piel's Asynchronous Web Server Test. 23 | 24 | This ping pong web server compares extremely well to everything else 25 | listed on Nicholas's website. 26 | 27 | http://nichol.as/asynchronous-servers-in-python 28 | 29 | """ 30 | 31 | import sys 32 | 33 | import pyev 34 | 35 | sys.path.insert(0, '..') 36 | 37 | import whizzer 38 | 39 | class PongProtocol(whizzer.Protocol): 40 | def connection_made(self): 41 | self.transport.write("HTTP/1.0 200 OK\r\nContent-Length: 5 \r\n\r\nPong!\r\n") 42 | self.lose_connection() 43 | 44 | loop = pyev.default_loop() 45 | sigwatcher = whizzer.signal_handler(loop) 46 | factory = whizzer.ProtocolFactory() 47 | factory.protocol = PongProtocol 48 | server = whizzer.TcpServer(loop, factory, "0.0.0.0", 8000, 500) 49 | 50 | sigwatcher.start() 51 | server.start() 52 | loop.loop() 53 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2010 Tom Burdick 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | from setuptools import setup, find_packages 23 | 24 | CLASSIFIERS = filter(None, map(str.strip, 25 | """ 26 | Intended Audience :: Developers 27 | License :: OSI Approved :: MIT License 28 | Programming Language :: Python 29 | Programming Language :: Python :: 2 30 | Programming Language :: Python :: 3 31 | Topic :: Software Development :: Libraries :: Python Modules 32 | Topic :: System :: Networking 33 | Topic :: Internet 34 | """.splitlines())) 35 | 36 | 37 | setup(name='whizzer', 38 | version='0.4.0', 39 | description='Fast event driven socket server framework based on pyev', 40 | author='Tom Burdick', 41 | author_email='thomas.burdick@gmail.com', 42 | url='http://github.com/bfrog/whizzer', 43 | packages=find_packages(), 44 | install_requires=['pyev>=0.8', 'msgpack-python', 'logbook'], 45 | classifiers=CLASSIFIERS, 46 | ) 47 | -------------------------------------------------------------------------------- /whizzer/protocol.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2010 Tom Burdick 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | class Protocol(object): 23 | """Basis of all client handling functionality.""" 24 | def __init__(self, loop): 25 | self.loop = loop 26 | 27 | def make_connection(self, transport, address): 28 | """Called externally when the transport is ready.""" 29 | self.connected = True 30 | self.transport = transport 31 | self.connection_made(address) 32 | 33 | def connection_made(self, address): 34 | """Called when the connection is ready to use.""" 35 | 36 | def connection_lost(self, reason): 37 | """Called when the connection has been lost.""" 38 | 39 | def data(self, data): 40 | """Handle an incoming stream of data.""" 41 | 42 | def lose_connection(self): 43 | self.transport.close() 44 | 45 | class ProtocolFactory(object): 46 | """Protocol factory.""" 47 | def build(self, loop): 48 | """Build and return a Protocol object.""" 49 | return self.protocol(loop) 50 | 51 | -------------------------------------------------------------------------------- /whizzer/test/test_protocol.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2010 Tom Burdick 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in 11 | # all copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | # THE SOFTWARE. 20 | 21 | import sys 22 | import time 23 | import unittest 24 | import pyev 25 | 26 | from whizzer.protocol import Protocol, ProtocolFactory 27 | from mocks import * 28 | from common import loop 29 | 30 | class TestProtocol(unittest.TestCase): 31 | def test_protocol(self): 32 | protocol = Protocol(loop) 33 | 34 | def test_factory(self): 35 | factory = ProtocolFactory() 36 | 37 | def test_factory_build(self): 38 | factory = ProtocolFactory() 39 | factory.protocol = Protocol 40 | p = factory.build(loop) 41 | self.assertTrue(isinstance(p, Protocol)) 42 | 43 | def test_lose_connection(self): 44 | factory = ProtocolFactory() 45 | factory.protocol = Protocol 46 | p = factory.build(loop) 47 | self.assertTrue(isinstance(p, Protocol)) 48 | t = MockTransport() 49 | p.make_connection(t, 'test') 50 | p.lose_connection() 51 | self.assertTrue(t.closes==1) 52 | 53 | 54 | 55 | if __name__ == '__main__': 56 | unittest.main() 57 | -------------------------------------------------------------------------------- /whizzer/debug.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2010 Tom Burdick 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | import gc 23 | import pyev 24 | import logbook 25 | 26 | logger = logbook.Logger(__name__) 27 | 28 | class ObjectWatcher(object): 29 | def __init__(self, loop, classes=[]): 30 | """Watches the garbage collector and prints out stats 31 | periodically with how many objects are in the gc for a given type. 32 | 33 | Useful for debugging situations where referrences are being held 34 | when they shouldn't be. A rather annoying problem in callback 35 | oriented code (that I've personally found anyways). 36 | 37 | """ 38 | gc.set_debug(gc.DEBUG_STATS) 39 | self.classes = classes 40 | self.timer = pyev.Timer(5.0, 5.0, loop, self.print_stats) 41 | self.timer.start() 42 | 43 | def count(self, cls): 44 | return len([obj for obj in gc.get_objects() if isinstance(obj, cls)]) 45 | 46 | def print_stats(self, watcher, events): 47 | gc.collect() 48 | logger.debug("Object Stats") 49 | for cls in self.classes: 50 | logger.debug(" %s : %d" % (cls.__name__, self.count(cls))) 51 | -------------------------------------------------------------------------------- /examples/server.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2010 Tom Burdick 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | import sys 23 | import time 24 | import signal 25 | import logbook 26 | from logbook.more import ColorizedStderrHandler 27 | 28 | import pyev 29 | 30 | sys.path.insert(0, '..') 31 | 32 | import whizzer 33 | from whizzer import protocol 34 | 35 | logger = logbook.Logger('echo server') 36 | 37 | 38 | class EchoProtocolFactory(protocol.ProtocolFactory): 39 | def __init__(self): 40 | self.echoes = 0 41 | 42 | def build(self, loop): 43 | return EchoProtocol(loop, self) 44 | 45 | 46 | class EchoProtocol(protocol.Protocol): 47 | def __init__(self, loop, factory): 48 | self.loop = loop 49 | self.factory = factory 50 | 51 | def data(self, data): 52 | self.transport.write(data) 53 | self.factory.echoes += 1 54 | 55 | class EchoStatistics(object): 56 | def __init__(self, loop, factory): 57 | self.loop = loop 58 | self.factory = factory 59 | self.timer = pyev.Timer(2.0, 2.0, loop, self._print_stats) 60 | 61 | def start(self): 62 | self.timer.start() 63 | 64 | def _print_stats(self, watcher, events): 65 | logger.error('echoes per seconds %f' % (self.factory.echoes/2.0)) 66 | self.factory.echoes = 0 67 | 68 | def main(): 69 | loop = pyev.default_loop() 70 | 71 | signal_handler = whizzer.signal_handler(loop) 72 | signal_handler.start() 73 | 74 | factory = EchoProtocolFactory() 75 | server = whizzer.TcpServer(loop, factory, "127.0.0.1", 2000, 256) 76 | stats = EchoStatistics(loop, factory) 77 | 78 | signal_handler.start() 79 | server.start() 80 | stats.start() 81 | 82 | loop.loop() 83 | 84 | if __name__ == "__main__": 85 | stderr_handler = ColorizedStderrHandler(level='DEBUG') 86 | with stderr_handler.applicationbound(): 87 | main() 88 | -------------------------------------------------------------------------------- /whizzer/process.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2010 Tom Burdick 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | import signal 23 | import time 24 | import os 25 | import sys 26 | import logbook 27 | import pyev 28 | 29 | logger = logbook.Logger(__name__) 30 | 31 | 32 | class Process(object): 33 | """Acts as a wrapper around multiprocessing.Process/os.fork that provides logging 34 | and process watching using pyev. 35 | 36 | """ 37 | def __init__(self, loop, run, *args, **kwargs): 38 | """Setup a process object with the above arguments.""" 39 | self.loop = loop 40 | self.run = run 41 | self.args = args 42 | self.kwargs = kwargs 43 | self.watcher = None 44 | 45 | def start(self): 46 | """Start the process, essentially forks and calls target function.""" 47 | logger.info("starting process") 48 | process = os.fork() 49 | time.sleep(0.01) 50 | if process != 0: 51 | logger.debug('starting child watcher') 52 | self.loop.reset() 53 | self.child_pid = process 54 | self.watcher = pyev.Child(self.child_pid, False, self.loop, self._child) 55 | self.watcher.start() 56 | else: 57 | self.loop.reset() 58 | logger.debug('running main function') 59 | self.run(*self.args, **self.kwargs) 60 | logger.debug('quitting') 61 | sys.exit(0) 62 | 63 | def stop(self): 64 | """Stop the process.""" 65 | logger.info("stopping process") 66 | self.watcher.stop() 67 | os.kill(self.child_pid, signal.SIGTERM) 68 | 69 | def _child(self, watcher, events): 70 | """Handle child watcher callback.""" 71 | watcher.stop() 72 | self.crashed() 73 | 74 | def crashed(self): 75 | """Handle a process crash. 76 | 77 | This should be overridden if you wish to do 78 | anything more than log that the process died. 79 | 80 | """ 81 | self.logger.error("%s crashed" % self.child_pid) 82 | -------------------------------------------------------------------------------- /whizzer/rpc/dispatch.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2010 Tom Burdick 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | 23 | class Dispatch(object): 24 | """Remote call dispatcher.""" 25 | 26 | def __init__(self): 27 | """Instantiate a basic dispatcher.""" 28 | self.functions = dict() 29 | 30 | def call(self, function, args=(), kwargs={}): 31 | """Call a method given some args and kwargs. 32 | 33 | function -- string containing the method name to call 34 | args -- arguments, either a list or tuple 35 | 36 | returns the result of the method. 37 | 38 | May raise an exception if the method isn't in the dict. 39 | 40 | """ 41 | return self.functions[function](*args, **kwargs) 42 | 43 | def add(self, fn, name=None): 44 | """Add a function that the dispatcher will know about. 45 | 46 | fn -- a callable object 47 | name -- optional alias for the function 48 | 49 | """ 50 | if not name: 51 | name = fn.__name__ 52 | self.functions[name] = fn 53 | 54 | def remote(fn, name=None, types=None): 55 | """Decorator that adds a remote attribute to a function. 56 | 57 | fn -- function being decorated 58 | name -- aliased name of the function, used for remote proxies 59 | types -- a argument type specifier, can be used to ensure 60 | arguments are of the correct type 61 | """ 62 | if not name: 63 | name = fn.__name__ 64 | 65 | fn.remote = {"name": name, "types": types} 66 | return fn 67 | 68 | 69 | class ObjectDispatch(Dispatch): 70 | """Remote call dispatch 71 | """ 72 | def __init__(self, obj): 73 | """Instantiate a object dispatcher, takes an object 74 | with methods marked using the remote decorator 75 | 76 | obj -- Object with methods decorated by the remote decorator. 77 | 78 | """ 79 | Dispatch.__init__(self) 80 | self.obj = obj 81 | attrs = dir(self.obj) 82 | for attr in attrs: 83 | a = getattr(self.obj, attr) 84 | if hasattr(a, 'remote'): 85 | self.add(a, a.remote['name']) 86 | 87 | 88 | -------------------------------------------------------------------------------- /whizzer/test/mocks.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2010 Tom Burdick 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in 11 | # all copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | # THE SOFTWARE. 20 | 21 | import sys 22 | 23 | from whizzer.protocol import Protocol, ProtocolFactory 24 | 25 | class MockTransport(object): 26 | def __init__(self): 27 | self.closes = 0 28 | self.writes = 0 29 | 30 | def close(self): 31 | print("close") 32 | self.closes += 1 33 | 34 | def write(self): 35 | print("write") 36 | self.writes += 1 37 | 38 | class MockLogger(object): 39 | def __init__(self): 40 | self.warns = [] 41 | self.errors = [] 42 | self.debugs = [] 43 | self.infos = [] 44 | 45 | def warn(self, message): 46 | print("warn: " + message) 47 | self.warns.append(message) 48 | 49 | def info(self, message): 50 | print("info: " + message) 51 | self.infos.append(message) 52 | 53 | def debug(self, message): 54 | print("debug: " + message) 55 | self.debugs.append(message) 56 | 57 | def error(self, message): 58 | print("error: " + message) 59 | self.errors.append(message) 60 | 61 | class MockProtocol(Protocol): 62 | def __init__(self, loop): 63 | Protocol.__init__(self, loop) 64 | self.reads = 0 65 | self.errors = 0 66 | self.connections = 0 67 | self.losses = 0 68 | self.connected = False 69 | self.transport = None 70 | self._data = [] 71 | self.reason = None 72 | 73 | def data(self, d): 74 | self.reads += 1 75 | self._data.append(d) 76 | print("reads " + str(self.reads)) 77 | 78 | def connection_made(self, address): 79 | self.connections += 1 80 | print("connections " + str(self.connections)) 81 | 82 | def connection_lost(self, reason=None): 83 | self.losses += 1 84 | self.reason = reason 85 | print("losses " + str(self.losses)) 86 | 87 | class MockFactory(ProtocolFactory): 88 | def __init__(self): 89 | ProtocolFactory.__init__(self) 90 | self.builds = 0 91 | 92 | def build(self, loop): 93 | self.builds += 1 94 | print("builds " + str(self.builds)) 95 | return self.protocol(loop) 96 | 97 | 98 | -------------------------------------------------------------------------------- /whizzer/rpc/proxy.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2010 Tom Burdick 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | import logbook 23 | 24 | from whizzer.defer import Deferred 25 | 26 | logger = logbook.Logger(__name__) 27 | 28 | 29 | class Proxy(object): 30 | """Proxy object that performs remote calls.""" 31 | 32 | def __init__(self, loop, protocol): 33 | """MsgPackProxy 34 | 35 | loop -- A pyev loop. 36 | protocol -- An instance of RPCProtocol. 37 | 38 | """ 39 | self.loop = loop 40 | self.protocol = protocol 41 | self.request_num = 0 42 | self.requests = dict() 43 | 44 | #: synchronous call() timeout 45 | self.timeout = None 46 | 47 | def call(self, method, *args): 48 | """Perform a synchronous remote call where the returned value is given immediately. 49 | 50 | This may block for sometime in certain situations. If it takes more than the Proxies 51 | set timeout then a TimeoutError is raised. 52 | 53 | Any exceptions the remote call raised that can be sent over the wire are raised. 54 | 55 | Internally this calls begin_call(method, *args).result(timeout=self.timeout) 56 | 57 | """ 58 | return self.begin_call(method, *args).result(self.timeout) 59 | 60 | def notify(self, method, *args): 61 | """Perform a synchronous remote call where no return value is desired.""" 62 | 63 | self.protocol.send_notification(method, args) 64 | 65 | def begin_call(self, method, *args): 66 | """Perform an asynchronous remote call where the return value is not known yet. 67 | 68 | This returns immediately with a Deferred object. The Deferred object may then be 69 | used to attach a callback, force waiting for the call, or check for exceptions. 70 | 71 | """ 72 | d = Deferred(self.loop, logger=logger) 73 | d.request = self.request_num 74 | self.requests[self.request_num] = d 75 | self.protocol.send_request(d.request, method, args) 76 | self.request_num += 1 77 | return d 78 | 79 | def response(self, msgid, error, result): 80 | """Handle a results message given to the proxy by the protocol object.""" 81 | if error: 82 | self.requests[msgid].errback(Exception(str(error))) 83 | else: 84 | self.requests[msgid].callback(result) 85 | del self.requests[msgid] 86 | 87 | 88 | -------------------------------------------------------------------------------- /examples/client.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2010 Tom Burdick 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | import sys 23 | import signal 24 | import logbook 25 | from logbook import NullHandler 26 | from logbook.more import ColorizedStderrHandler 27 | import pyev 28 | 29 | sys.path.insert(0, '..') 30 | 31 | import whizzer 32 | from whizzer import protocol 33 | 34 | logger = logbook.Logger('echo_client') 35 | 36 | 37 | class EchoClientProtocol(protocol.Protocol): 38 | def connection_made(self, address): 39 | """When the connection is made, send something.""" 40 | logger.info("connection made to {}".format(address)) 41 | self.count = 0 42 | self.connected = True 43 | self.transport.write(b'Echo Me') 44 | 45 | def data(self, data): 46 | logger.info("echo'd " + data.decode('ASCII')) 47 | if self.count < 10000: 48 | self.count += 1 49 | self.transport.write(b'Echo Me') 50 | else: 51 | self.lose_connection() 52 | self.connected = False 53 | 54 | def interrupt(watcher, events): 55 | watcher.loop.unloop() 56 | 57 | 58 | class EchoClient(object): 59 | def __init__(self, id, loop, factory): 60 | self.id = id 61 | self.loop = loop 62 | self.factory = factory 63 | self.connect_client() 64 | 65 | def connect_client(self): 66 | client = whizzer.TcpClient(self.loop, self.factory, "127.0.0.1", 2000) 67 | logger.info('client calling connect') 68 | d = client.connect() 69 | logger.info('client called connect') 70 | d.add_callback(self.connect_success) 71 | d.add_errback(self.connect_failed) 72 | self.timer = pyev.Timer(1.0, 0.0, self.loop, self.timeout, None) 73 | self.timer.start() 74 | 75 | def connect_success(self, result): 76 | logger.info('connect success, protocol is {}'.format(result.connected)) 77 | self.connect_client() 78 | self.timer.stop() 79 | 80 | def connect_failed(self, error): 81 | logger.error('client {} connecting failed, reason {}'.format(id, error)) 82 | 83 | def timeout(self, watcher, events): 84 | logger.error('timeout') 85 | 86 | def main(): 87 | loop = pyev.default_loop() 88 | 89 | signal_handler = whizzer.signal_handler(loop) 90 | 91 | factory = whizzer.ProtocolFactory() 92 | factory.protocol = EchoClientProtocol 93 | 94 | clients = [] 95 | # number of parallel clients 96 | for x in range(0, 2): 97 | clients.append(EchoClient(x, loop, factory)) 98 | 99 | signal_handler.start() 100 | loop.loop() 101 | 102 | if __name__ == "__main__": 103 | stderr_handler = ColorizedStderrHandler(level='INFO') 104 | null_handler = NullHandler() 105 | with null_handler.applicationbound(): 106 | with stderr_handler.applicationbound(): 107 | main() 108 | -------------------------------------------------------------------------------- /whizzer/test/test_server.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2010 Tom Burdick 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in 11 | # all copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | # THE SOFTWARE. 20 | 21 | 22 | import gc 23 | import os 24 | import sys 25 | import socket 26 | import select 27 | import unittest 28 | import pyev 29 | 30 | from whizzer.protocol import Protocol, ProtocolFactory 31 | from whizzer.server import UnixServer, TcpServer 32 | from mocks import * 33 | from common import loop 34 | 35 | fpath = os.path.dirname(__file__) 36 | 37 | class TestServerCreation(unittest.TestCase): 38 | def test_tcp_server(self): 39 | factory = ProtocolFactory() 40 | factory.protocol = Protocol 41 | server = TcpServer(loop, factory, "0.0.0.0", 2000) 42 | server = None 43 | 44 | def test_unix_server(self): 45 | factory = ProtocolFactory() 46 | factory.protocol = Protocol 47 | server = UnixServer(loop, factory, "bogus") 48 | server = None 49 | # path should be cleaned up as soon as garbage collected 50 | gc.collect() 51 | self.assertTrue(not os.path.exists("bogus")) 52 | 53 | class TestUnixServer(unittest.TestCase): 54 | def setUp(self): 55 | self.factory = MockFactory() 56 | self.factory.protocol = MockProtocol 57 | self.path = fpath + "/test_socket" 58 | self.server = UnixServer(loop, self.factory, self.path) 59 | 60 | def tearDown(self): 61 | self.server.shutdown() 62 | self.server = None 63 | self.server = None 64 | self.factory = None 65 | gc.collect() 66 | 67 | def c_sock(self): 68 | csock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) 69 | csock.setblocking(False) 70 | return csock 71 | 72 | def c_connect(self, csock): 73 | csock.connect(self.path) 74 | 75 | def c_isconnected(self, csock, testmsg="testmsg"): 76 | (rlist, wlist, xlist) = select.select([], [csock], [], 0.0) 77 | if csock in wlist: 78 | try: 79 | csock.send(testmsg) 80 | return True 81 | except IOError as e: 82 | return False 83 | else: 84 | return False 85 | 86 | def test_start(self): 87 | self.server.start() 88 | csock = self.c_sock() 89 | self.c_connect(csock) 90 | loop.start(pyev.EVRUN_ONCE) 91 | self.assertTrue(self.c_isconnected(csock)) 92 | 93 | def test_stop(self): 94 | self.server.start() 95 | csock = self.c_sock() 96 | self.c_connect(csock) 97 | loop.start(pyev.EVRUN_ONCE) 98 | self.assertTrue(self.c_isconnected(csock)) 99 | self.server.stop() 100 | loop.start(pyev.EVRUN_ONCE) 101 | self.assertTrue(self.c_isconnected(csock)) 102 | csock.close() 103 | csock = None 104 | loop.start(pyev.EVRUN_NOWAIT) 105 | csock = self.c_sock() 106 | self.c_connect(csock) 107 | self.assertTrue(self.factory.builds == 1) 108 | 109 | if __name__ == '__main__': 110 | unittest.main() 111 | -------------------------------------------------------------------------------- /whizzer/rpc/service.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2010 Tom Burdick 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | import signal 23 | 24 | import pyev 25 | 26 | import logbook 27 | 28 | from whizzer.process import Process 29 | from whizzer.server import UnixServer 30 | from whizzer.client import UnixClient 31 | 32 | from whizzer.rpc.dispatch import remote, ObjectDispatch 33 | from whizzer.rpc.msgpackrpc import MsgPackProtocolFactory 34 | 35 | logger = logbook.Logger(__name__) 36 | 37 | 38 | def spawn(cls, loop, name, path, *args, **kwargs): 39 | p = Process(loop, service, cls, loop, name, path, *args, **kwargs) 40 | p.start() 41 | return p 42 | 43 | def service(cls, loop, name, path, *args, **kwargs): 44 | instance = cls(loop, name, path, *args, **kwargs) 45 | instance.run() 46 | 47 | 48 | class ServiceProxy(object): 49 | """Proxy to a service.""" 50 | 51 | def __init__(self, loop, path): 52 | self.loop = loop 53 | self.path = path 54 | self.proxy = None 55 | 56 | def connect(self): 57 | self.factory = MsgPackProtocolFactory() 58 | self.client = UnixClient(self.loop, self.factory, self.path) 59 | d = self.client.connect() 60 | d.add_callback(self.connected) 61 | return d 62 | 63 | def connected(self, result=None): 64 | d = self.factory.proxy(0) 65 | d.add_callback(self.set_proxy) 66 | 67 | def set_proxy(self, proxy): 68 | self.proxy = proxy 69 | 70 | def call(self, method, *args): 71 | return self.proxy.call(method, *args) 72 | 73 | def notify(self, method, *args): 74 | return self.proxy.notify(method, *args) 75 | 76 | def begin_call(self, method, *args): 77 | return self.proxy.begin_call(method, *args) 78 | 79 | 80 | class Service(object): 81 | """A generic service class meant to be run in a process on its own and 82 | handle requests using RPC. 83 | 84 | """ 85 | 86 | def __init__(self, loop, name, path): 87 | """Create a service with a name.""" 88 | self.loop = loop 89 | self.name = name 90 | self.path = path 91 | self.logger = logbook.Logger(self.name) 92 | 93 | def signal_init(self): 94 | self.sigintwatcher = pyev.Signal(signal.SIGINT, self.loop, self._stop) 95 | self.sigintwatcher.start() 96 | self.sigtermwatcher = pyev.Signal(signal.SIGTERM, self.loop, self._terminate) 97 | self.sigtermwatcher.start() 98 | 99 | def _stop(self, watcher, events): 100 | self.stop(signal.SIGINT) 101 | 102 | def _terminate(self, watcher, events): 103 | self.terminate(signal.SIGTERM) 104 | 105 | def listen_init(self): 106 | """Setup the service to listen for clients.""" 107 | self.dispatcher = ObjectDispatch(self) 108 | self.factory = MsgPackProtocolFactory(self.dispatcher) 109 | self.server = UnixServer(self.loop, self.factory, self.path) 110 | self.server.start() 111 | 112 | def run(self): 113 | """Run the event loop.""" 114 | self.signal_init() 115 | self.listen_init() 116 | self.logger.info('starting') 117 | self.loop.start() 118 | 119 | @remote 120 | def stop(self, reason=None): 121 | """Shutdown the service with a reason.""" 122 | self.logger.info('stopping') 123 | self.loop.stop(pyev.EVBREAK_ALL) 124 | 125 | @remote 126 | def terminate(self, reason=None): 127 | """Terminate the service with a reason.""" 128 | self.logger.info('terminating') 129 | self.loop.unloop(pyev.EVUNLOOP_ALL) 130 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | if NOT "%PAPER%" == "" ( 11 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 12 | ) 13 | 14 | if "%1" == "" goto help 15 | 16 | if "%1" == "help" ( 17 | :help 18 | echo.Please use `make ^` where ^ is one of 19 | echo. html to make standalone HTML files 20 | echo. dirhtml to make HTML files named index.html in directories 21 | echo. singlehtml to make a single large HTML file 22 | echo. pickle to make pickle files 23 | echo. json to make JSON files 24 | echo. htmlhelp to make HTML files and a HTML help project 25 | echo. qthelp to make HTML files and a qthelp project 26 | echo. devhelp to make HTML files and a Devhelp project 27 | echo. epub to make an epub 28 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 29 | echo. text to make text files 30 | echo. man to make manual pages 31 | echo. changes to make an overview over all changed/added/deprecated items 32 | echo. linkcheck to check all external links for integrity 33 | echo. doctest to run all doctests embedded in the documentation if enabled 34 | goto end 35 | ) 36 | 37 | if "%1" == "clean" ( 38 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 39 | del /q /s %BUILDDIR%\* 40 | goto end 41 | ) 42 | 43 | if "%1" == "html" ( 44 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 45 | echo. 46 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 47 | goto end 48 | ) 49 | 50 | if "%1" == "dirhtml" ( 51 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 52 | echo. 53 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 54 | goto end 55 | ) 56 | 57 | if "%1" == "singlehtml" ( 58 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 59 | echo. 60 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 61 | goto end 62 | ) 63 | 64 | if "%1" == "pickle" ( 65 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 66 | echo. 67 | echo.Build finished; now you can process the pickle files. 68 | goto end 69 | ) 70 | 71 | if "%1" == "json" ( 72 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 73 | echo. 74 | echo.Build finished; now you can process the JSON files. 75 | goto end 76 | ) 77 | 78 | if "%1" == "htmlhelp" ( 79 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 80 | echo. 81 | echo.Build finished; now you can run HTML Help Workshop with the ^ 82 | .hhp project file in %BUILDDIR%/htmlhelp. 83 | goto end 84 | ) 85 | 86 | if "%1" == "qthelp" ( 87 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 88 | echo. 89 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 90 | .qhcp project file in %BUILDDIR%/qthelp, like this: 91 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\whizzer.qhcp 92 | echo.To view the help file: 93 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\whizzer.ghc 94 | goto end 95 | ) 96 | 97 | if "%1" == "devhelp" ( 98 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 99 | echo. 100 | echo.Build finished. 101 | goto end 102 | ) 103 | 104 | if "%1" == "epub" ( 105 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 106 | echo. 107 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 108 | goto end 109 | ) 110 | 111 | if "%1" == "latex" ( 112 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 113 | echo. 114 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 115 | goto end 116 | ) 117 | 118 | if "%1" == "text" ( 119 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 120 | echo. 121 | echo.Build finished. The text files are in %BUILDDIR%/text. 122 | goto end 123 | ) 124 | 125 | if "%1" == "man" ( 126 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 127 | echo. 128 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 129 | goto end 130 | ) 131 | 132 | if "%1" == "changes" ( 133 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 134 | echo. 135 | echo.The overview file is in %BUILDDIR%/changes. 136 | goto end 137 | ) 138 | 139 | if "%1" == "linkcheck" ( 140 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 141 | echo. 142 | echo.Link check complete; look for any errors in the above output ^ 143 | or in %BUILDDIR%/linkcheck/output.txt. 144 | goto end 145 | ) 146 | 147 | if "%1" == "doctest" ( 148 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 149 | echo. 150 | echo.Testing of doctests in the sources finished, look at the ^ 151 | results in %BUILDDIR%/doctest/output.txt. 152 | goto end 153 | ) 154 | 155 | :end 156 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | 15 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest 16 | 17 | help: 18 | @echo "Please use \`make ' where is one of" 19 | @echo " html to make standalone HTML files" 20 | @echo " dirhtml to make HTML files named index.html in directories" 21 | @echo " singlehtml to make a single large HTML file" 22 | @echo " pickle to make pickle files" 23 | @echo " json to make JSON files" 24 | @echo " htmlhelp to make HTML files and a HTML help project" 25 | @echo " qthelp to make HTML files and a qthelp project" 26 | @echo " devhelp to make HTML files and a Devhelp project" 27 | @echo " epub to make an epub" 28 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 29 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 30 | @echo " text to make text files" 31 | @echo " man to make manual pages" 32 | @echo " changes to make an overview of all changed/added/deprecated items" 33 | @echo " linkcheck to check all external links for integrity" 34 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 35 | 36 | clean: 37 | -rm -rf $(BUILDDIR)/* 38 | 39 | html: 40 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 41 | @echo 42 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 43 | 44 | dirhtml: 45 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 46 | @echo 47 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 48 | 49 | singlehtml: 50 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 51 | @echo 52 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 53 | 54 | pickle: 55 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 56 | @echo 57 | @echo "Build finished; now you can process the pickle files." 58 | 59 | json: 60 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 61 | @echo 62 | @echo "Build finished; now you can process the JSON files." 63 | 64 | htmlhelp: 65 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 66 | @echo 67 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 68 | ".hhp project file in $(BUILDDIR)/htmlhelp." 69 | 70 | qthelp: 71 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 72 | @echo 73 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 74 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 75 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/whizzer.qhcp" 76 | @echo "To view the help file:" 77 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/whizzer.qhc" 78 | 79 | devhelp: 80 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 81 | @echo 82 | @echo "Build finished." 83 | @echo "To view the help file:" 84 | @echo "# mkdir -p $$HOME/.local/share/devhelp/whizzer" 85 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/whizzer" 86 | @echo "# devhelp" 87 | 88 | epub: 89 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 90 | @echo 91 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 92 | 93 | latex: 94 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 95 | @echo 96 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 97 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 98 | "(use \`make latexpdf' here to do that automatically)." 99 | 100 | latexpdf: 101 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 102 | @echo "Running LaTeX files through pdflatex..." 103 | make -C $(BUILDDIR)/latex all-pdf 104 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 105 | 106 | text: 107 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 108 | @echo 109 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 110 | 111 | man: 112 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 113 | @echo 114 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 115 | 116 | changes: 117 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 118 | @echo 119 | @echo "The overview file is in $(BUILDDIR)/changes." 120 | 121 | linkcheck: 122 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 123 | @echo 124 | @echo "Link check complete; look for any errors in the above output " \ 125 | "or in $(BUILDDIR)/linkcheck/output.txt." 126 | 127 | doctest: 128 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 129 | @echo "Testing of doctests in the sources finished, look at the " \ 130 | "results in $(BUILDDIR)/doctest/output.txt." 131 | -------------------------------------------------------------------------------- /examples/servicefork.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2010 Tom Burdick 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | 23 | """ This example starts up an adder service and 4 adder clients that are themselves 24 | benchmarking sevices (which can have bench_notify remotely called) 25 | 26 | At the moment the benchmarking services simply do 1000 notifies and a single 27 | rpc call every second or so, you can easily modify this to really push the 28 | adder service on your machine to see what is possible. 29 | 30 | """ 31 | 32 | 33 | 34 | import sys 35 | import signal 36 | import time 37 | 38 | import pyev 39 | 40 | sys.path.insert(0, '..') 41 | 42 | from whizzer.rpc.dispatch import remote 43 | from whizzer.rpc.service import Service, ServiceProxy, spawn 44 | 45 | 46 | class Adder(Service): 47 | 48 | def stats_init(self): 49 | self.add_calls = 0 50 | self.last_stats = time.time() 51 | self.stats_timer = pyev.Timer(2.0, 2.0, self.loop, self.stats) 52 | self.stats_timer.start() 53 | 54 | def stats(self, watcher, events): 55 | diff = time.time() - self.last_stats 56 | self.logger.info("%f calls in %f seconds, %f calls per second" % (self.add_calls, diff, self.add_calls/diff)) 57 | self.add_calls = 0 58 | self.last_stats = time.time() 59 | 60 | def run(self): 61 | self.signal_init() 62 | self.listen_init() 63 | self.stats_init() 64 | self.logger.info("starting") 65 | self.loop.start() 66 | 67 | @remote 68 | def add(self, a, b): 69 | self.add_calls += 1 70 | return a+b 71 | 72 | 73 | class AdderBench(Service): 74 | def __init__(self, loop, name, path, adder_path): 75 | Service.__init__(self, loop, name, path) 76 | self.adder_path = adder_path 77 | 78 | def stats_init(self): 79 | self.add_calls = 0 80 | self.last_stats = time.time() 81 | self.stats_timer = pyev.Timer(2.0, 2.0, self.loop, self.stats) 82 | self.stats_timer.start() 83 | 84 | def stats(self, watcher, events): 85 | self.bench_notify(20000) 86 | diff = time.time() - self.last_stats 87 | #self.logger.info("{} calls in {} seconds, {} calls per second".format( 88 | # self.add_calls, diff, self.add_calls/diff)) 89 | self.add_calls = 0 90 | self.last_stats = time.time() 91 | 92 | def proxy_init(self): 93 | self.proxy = ServiceProxy(self.loop, self.adder_path) 94 | connected = False 95 | while not connected: 96 | try: 97 | self.proxy.connect().result() 98 | connected = True 99 | except: 100 | time.sleep(0.1) 101 | 102 | def run(self): 103 | self.signal_init() 104 | self.listen_init() 105 | self.stats_init() 106 | self.proxy_init() 107 | self.logger.info("starting") 108 | self.loop.start() 109 | 110 | @remote 111 | def bench_notify(self, calls): 112 | #self.logger.info("notify") 113 | start = time.time() 114 | for x in range(calls): 115 | self.proxy.notify('add', 1, 1) 116 | self.proxy.call('add', 1, 1) 117 | end = time.time() 118 | #self.logger.info("took {} to perform {} notifies, {} notifies per second".format( 119 | # end-start, calls, calls/(end-start))) 120 | 121 | 122 | def main(): 123 | path = "adder_service" 124 | name = "adder" 125 | 126 | loop = pyev.default_loop() 127 | 128 | sigwatcher = pyev.Signal(signal.SIGINT, loop, lambda watcher, events: watcher.loop.stop(pyev.EVBREAK_ALL)) 129 | sigwatcher.start() 130 | 131 | service = spawn(Adder, loop, name, path) 132 | sproxy = ServiceProxy(loop, path) 133 | 134 | sproxy.connect() 135 | 136 | clients = [] 137 | proxies = [] 138 | 139 | # to push the server further (to see how fast it will really go...) 140 | # just add more clients! 141 | for x in range(30): 142 | bpath = "adder_bench_%i" % x 143 | client = spawn(AdderBench, loop, bpath, bpath, path) 144 | bproxy = ServiceProxy(loop, "adder_bench_1") 145 | bproxy.connect() 146 | clients.append(client) 147 | proxies.append(bproxy) 148 | 149 | loop.start() 150 | 151 | if __name__ == "__main__": 152 | main() 153 | -------------------------------------------------------------------------------- /examples/forked.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2010 Tom Burdick 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | import sys 23 | import time 24 | import signal 25 | 26 | import logbook 27 | from logbook import NullHandler 28 | from logbook.more import ColorizedStderrHandler 29 | 30 | import pyev 31 | 32 | 33 | sys.path.insert(0, '..') 34 | 35 | from whizzer.process import Process 36 | from whizzer.server import UnixServer 37 | from whizzer.client import UnixClient 38 | from whizzer.rpc.dispatch import remote, ObjectDispatch 39 | from whizzer.rpc.picklerpc import PickleProtocolFactory 40 | from whizzer.rpc.msgpackrpc import MsgPackProtocolFactory 41 | 42 | logger = logbook.Logger('forked!') 43 | 44 | 45 | class AdderService(object): 46 | def __init__(self): 47 | logger.info('creating an adder service') 48 | 49 | @remote 50 | def add(self, a, b): 51 | return a+b 52 | 53 | def server_stop(watcher, events): 54 | logger.debug('got shutdown') 55 | watcher.loop.unloop(pyev.EVUNLOOP_ALL) 56 | 57 | def server_main(loop, path): 58 | """Run in the client after the fork.""" 59 | loop.fork() 60 | logger.debug('forked function') 61 | sigintwatcher = pyev.Signal(signal.SIGINT, loop, lambda watcher, events: logger.info('interrupt ignored')) 62 | sigintwatcher.start() 63 | sigtermwatcher = pyev.Signal(signal.SIGTERM, loop, server_stop) 64 | sigtermwatcher.start() 65 | adder = AdderService() 66 | dispatcher = ObjectDispatch(adder) 67 | pickle_factory = PickleProtocolFactory(dispatcher) 68 | pickle_server = UnixServer(loop, pickle_factory, path) 69 | pickle_server.start() 70 | msgpack_factory = MsgPackProtocolFactory(dispatcher) 71 | msgpack_server = UnixServer(loop, msgpack_factory, path + '_mp') 72 | msgpack_server.start() 73 | 74 | logger.debug('running server loop') 75 | 76 | import cProfile 77 | cProfile.runctx('loop.loop()', None, {'loop':loop}, 'server_profile') 78 | 79 | logger.debug('server unlooped') 80 | 81 | 82 | def main(): 83 | path = 'adder_socket' 84 | loop = pyev.default_loop() 85 | 86 | sigwatcher = pyev.Signal(signal.SIGINT, loop, lambda watcher, events: watcher.loop.unloop(pyev.EVUNLOOP_ALL)) 87 | sigwatcher.start() 88 | 89 | p = Process(loop, server_main, loop, 'adder_socket') 90 | p.start() 91 | 92 | pickle_factory = PickleProtocolFactory() 93 | pickle_client = UnixClient(loop, pickle_factory, path) 94 | 95 | retries = 10 96 | while retries: 97 | try: 98 | pickle_client.connect().result() 99 | retries = 0 100 | except Exception as e: 101 | time.sleep(0.1) 102 | retries -= 1 103 | 104 | proxy = pickle_factory.proxy(0).result() 105 | 106 | start = time.time() 107 | s = 0 108 | for i in range(10000): 109 | s = proxy.call('add', 1, s) 110 | stop = time.time() 111 | 112 | logger.info('pickle-rpc took {} seconds to perform {} calls, {} calls per second', stop-start, s, s/(stop-start)) 113 | 114 | start = time.time() 115 | for i in range(10000): 116 | proxy.notify('add', 1, s) 117 | proxy.call('add', 1, s) 118 | stop = time.time() 119 | 120 | logger.info('pickle-rpc took {} seconds to perform {} notifications, {} notifies per second', stop-start, 10000, 10000/(stop-start)) 121 | 122 | msgpack_factory = MsgPackProtocolFactory() 123 | msgpack_client = UnixClient(loop, msgpack_factory, path + '_mp') 124 | 125 | retries = 10 126 | while retries: 127 | try: 128 | msgpack_client.connect().result() 129 | retries = 0 130 | except Exception as e: 131 | time.sleep(0.1) 132 | retries -= 1 133 | 134 | proxy = msgpack_factory.proxy(0).result() 135 | 136 | start = time.time() 137 | s = 0 138 | for i in range(10000): 139 | s = proxy.call('add', 1, s) 140 | stop = time.time() 141 | 142 | logger.info('msgpack-rpc took {} seconds to perform {} calls, {} calls per second', stop-start, s, s/(stop-start)) 143 | 144 | start = time.time() 145 | for i in range(10000): 146 | proxy.notify('add', 1, s) 147 | proxy.call('add', 1, s) 148 | stop = time.time() 149 | 150 | logger.info('msgpack-rpc took {} seconds to perform {} notifications, {} notifies per second', stop-start, 10000, 10000/(stop-start)) 151 | 152 | p.stop() 153 | 154 | if __name__ == "__main__": 155 | stderr_handler = ColorizedStderrHandler(level='DEBUG') 156 | null_handler = NullHandler() 157 | with null_handler.applicationbound(): 158 | with stderr_handler.applicationbound(): 159 | main() 160 | -------------------------------------------------------------------------------- /whizzer/test/test_transport.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2010 Tom Burdick 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | 23 | import os 24 | import sys 25 | import socket 26 | import unittest 27 | import pyev 28 | 29 | from whizzer.transport import SocketTransport, ConnectionClosed, BufferOverflowError 30 | from common import loop 31 | 32 | fpath = os.path.dirname(__file__) 33 | 34 | class TestSocketTransport(unittest.TestCase): 35 | def setUp(self): 36 | # setup some blocking sockets to test the transport with 37 | self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) 38 | self.sock.bind(fpath + "/test_sock") 39 | self.sock.listen(1) 40 | self.csock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) 41 | self.csock.connect(fpath + "/test_sock") 42 | self.ssock, self.saddr = self.sock.accept() 43 | self.data = [] 44 | self.reason = None 45 | 46 | def tearDown(self): 47 | self.sock = None 48 | self.csock = None 49 | self.ssock = None 50 | self.saddr = None 51 | os.remove(fpath + "/test_sock") 52 | self.data = [] 53 | self.reason = None 54 | 55 | def read(self, data): 56 | print("got " + str(data)) 57 | self.data = data 58 | 59 | def close(self, reason): 60 | self.reason = reason 61 | 62 | def test_creation(self): 63 | t = SocketTransport(loop, self.ssock, self.read, self.close) 64 | 65 | def test_write(self): 66 | msg = b'hello' 67 | t = SocketTransport(loop, self.ssock, self.read, self.close) 68 | t.write(msg) 69 | loop.start(pyev.EVRUN_NOWAIT) 70 | rmsg = self.csock.recv(len(msg)) 71 | self.assertEqual(rmsg, msg) 72 | 73 | def test_close(self): 74 | t = SocketTransport(loop, self.ssock, self.read, self.close) 75 | t.close() 76 | self.assertTrue(t.closed) 77 | self.assertTrue(isinstance(self.reason, ConnectionClosed)) 78 | 79 | def test_closed_write(self): 80 | t = SocketTransport(loop, self.ssock, self.read, self.close) 81 | t.close() 82 | self.assertRaises(ConnectionClosed, t.write, ("hello")) 83 | 84 | def test_closed_start(self): 85 | t = SocketTransport(loop, self.ssock, self.read, self.close) 86 | t.close() 87 | self.assertRaises(ConnectionClosed, t.start) 88 | 89 | def test_closed_stop(self): 90 | t = SocketTransport(loop, self.ssock, self.read, self.close) 91 | t.close() 92 | self.assertRaises(ConnectionClosed, t.stop) 93 | 94 | def test_stop(self): 95 | t = SocketTransport(loop, self.ssock, self.read, self.close) 96 | t.stop() 97 | 98 | def test_stop(self): 99 | t = SocketTransport(loop, self.ssock, self.read, self.close) 100 | t.stop() 101 | 102 | def test_stop_buffered_write(self): 103 | t = SocketTransport(loop, self.ssock, self.read, self.close) 104 | count = 0 105 | msg = b'hello' 106 | while(t.write != t.buffered_write): 107 | count += 1 108 | t.write(msg) 109 | t.write(msg) 110 | t.stop() 111 | self.csock.recv(count*len(msg)) 112 | t.start() 113 | loop.start(pyev.EVRUN_NOWAIT) 114 | self.assertTrue(t.write == t.unbuffered_write) 115 | 116 | def test_close_buffered_write(self): 117 | t = SocketTransport(loop, self.ssock, self.read, self.close) 118 | count = 0 119 | msg = b'hello' 120 | while(t.write != t.buffered_write): 121 | count += 1 122 | t.write(msg) 123 | t.write(msg) 124 | t.close() 125 | self.assertRaises(ConnectionClosed, t.write, msg) 126 | 127 | def test_buffered_write(self): 128 | t = SocketTransport(loop, self.ssock, self.read, self.close) 129 | count = 0 130 | msg = b'hello' 131 | while(t.write != t.buffered_write): 132 | count += 1 133 | t.write(msg) 134 | t.write(msg) 135 | self.csock.recv(count*len(msg)) 136 | loop.start(pyev.EVRUN_NOWAIT) 137 | self.assertTrue(t.write == t.unbuffered_write) 138 | 139 | def test_overflow_write(self): 140 | t = SocketTransport(loop, self.ssock, self.read, self.close) 141 | self.assertRaises(BufferOverflowError, t.write, bytes([1 for x in range(0, 1024*1024)])) 142 | 143 | def test_read(self): 144 | t = SocketTransport(loop, self.ssock, self.read, self.close) 145 | t.start() 146 | self.csock.send(b'hello') 147 | loop.start(pyev.EVRUN_NOWAIT) 148 | self.assertEqual(self.data, b'hello') 149 | 150 | def test_error(self): 151 | t = SocketTransport(loop, self.ssock, self.read, self.close) 152 | self.csock.close() 153 | t.write(b'hello') 154 | loop.start(pyev.EVRUN_NOWAIT) 155 | self.assertTrue(self.reason is not None) 156 | 157 | if __name__ == '__main__': 158 | unittest.main() 159 | -------------------------------------------------------------------------------- /whizzer/test/test_client.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2010 Tom Burdick 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in 11 | # all copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | # THE SOFTWARE. 20 | 21 | import os 22 | import sys 23 | import time 24 | import unittest 25 | import socket 26 | 27 | import pyev 28 | 29 | from whizzer.defer import Deferred 30 | from whizzer.protocol import Protocol, ProtocolFactory 31 | from whizzer.client import TcpClient, UnixClient 32 | from mocks import * 33 | from common import loop 34 | 35 | class TestClientCreation(unittest.TestCase): 36 | def test_tcp_client(self): 37 | factory = ProtocolFactory() 38 | factory.protocol = Protocol 39 | client = TcpClient(loop, MockFactory(), "0.0.0.0", 2000) 40 | 41 | def test_unix_client(self): 42 | factory = ProtocolFactory() 43 | factory.protocol = Protocol 44 | client = UnixClient(loop, MockFactory(), "bogus") 45 | 46 | class TestUnixClient(unittest.TestCase): 47 | """Functional test for UnixClient.""" 48 | def setUp(self): 49 | self.path = "test" 50 | self.ssock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) 51 | self.ssock.bind(self.path) 52 | self.ssock.listen(5) 53 | 54 | self.factory = MockFactory() 55 | self.factory.protocol = MockProtocol 56 | self.client = UnixClient(loop, self.factory, self.path) 57 | 58 | self._connected = False 59 | 60 | def tearDown(self): 61 | self.ssock.close() 62 | os.remove(self.path) 63 | self.client = None 64 | self.factory = None 65 | self.ssock = None 66 | 67 | self._connected = False 68 | 69 | def connected(self, protocol): 70 | self.assertTrue(isinstance(protocol, MockProtocol)) 71 | self.protocol = protocol 72 | self._connected = True 73 | 74 | def test_connect(self): 75 | d = self.client.connect() 76 | self.assertTrue(isinstance(d, Deferred)) 77 | d.add_callback(self.connected) 78 | (csock, addr) = self.ssock.accept() 79 | 80 | def test_lose_connection(self): 81 | d = self.client.connect() 82 | self.assertTrue(isinstance(d, Deferred)) 83 | d.add_callback(self.connected) 84 | (csock, addr) = self.ssock.accept() 85 | d.result() 86 | self.protocol.lose_connection() 87 | 88 | def test_interrupt(self): 89 | d = self.client.connect() 90 | self.assertTrue(isinstance(d, Deferred)) 91 | d.add_callback(self.connected) 92 | (csock, addr) = self.ssock.accept() 93 | d.result() 94 | self.client._interrupt(None, None) 95 | self.assertTrue(self.client.connection is None) 96 | 97 | def test_disconnect(self): 98 | d = self.client.connect() 99 | self.assertTrue(isinstance(d, Deferred)) 100 | d.add_callback(self.connected) 101 | (csock, addr) = self.ssock.accept() 102 | d.result() 103 | self.client.disconnect() 104 | self.assertTrue(self.client.connection is None) 105 | 106 | class TestTcpClient(unittest.TestCase): 107 | """Functional test for TcpClient.""" 108 | def setUp(self): 109 | self.port = 6000 110 | self.ssock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 111 | 112 | while True: 113 | try: 114 | self.ssock.bind(("0.0.0.0", self.port)) 115 | break 116 | except IOError as e: 117 | self.port += 1 118 | 119 | self.ssock.listen(5) 120 | 121 | self.factory = MockFactory() 122 | self.factory.protocol = MockProtocol 123 | self.client = TcpClient(loop, self.factory, "0.0.0.0", self.port) 124 | self._connected = False 125 | 126 | def tearDown(self): 127 | self.ssock.close() 128 | self.client = None 129 | self.factory = None 130 | self.ssock = None 131 | self._connected = False 132 | 133 | def connected(self, protocol): 134 | self.assertTrue(isinstance(protocol, MockProtocol)) 135 | self.protocol = protocol 136 | self._connected = True 137 | 138 | def test_connect(self): 139 | d = self.client.connect() 140 | self.assertTrue(isinstance(d, Deferred)) 141 | d.add_callback(self.connected) 142 | (csock, addr) = self.ssock.accept() 143 | 144 | def test_lose_connection(self): 145 | d = self.client.connect() 146 | self.assertTrue(isinstance(d, Deferred)) 147 | d.add_callback(self.connected) 148 | (csock, addr) = self.ssock.accept() 149 | d.result() 150 | self.protocol.lose_connection() 151 | 152 | def test_interrupt(self): 153 | d = self.client.connect() 154 | self.assertTrue(isinstance(d, Deferred)) 155 | d.add_callback(self.connected) 156 | (csock, addr) = self.ssock.accept() 157 | d.result() 158 | self.client._interrupt(None, None) 159 | self.assertTrue(self.client.connection is None) 160 | 161 | def test_disconnect(self): 162 | d = self.client.connect() 163 | self.assertTrue(isinstance(d, Deferred)) 164 | d.add_callback(self.connected) 165 | (csock, addr) = self.ssock.accept() 166 | d.result() 167 | self.client.disconnect() 168 | self.assertTrue(self.client.connection is None) 169 | 170 | if __name__ == '__main__': 171 | unittest.main() 172 | -------------------------------------------------------------------------------- /whizzer/test/test_defer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2010 Tom Burdick 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | import sys 23 | import time 24 | 25 | import unittest 26 | import pyev 27 | 28 | from whizzer.defer import Deferred, CancelledError, AlreadyCalledError, TimeoutError 29 | 30 | from common import loop 31 | 32 | def throw_always(result): 33 | raise Exception("success") 34 | 35 | def one_always(result): 36 | return 1 37 | 38 | def add(a, b): 39 | return a+b 40 | 41 | class TestDeferred(unittest.TestCase): 42 | def setUp(self): 43 | self.deferred = Deferred(loop) 44 | self.result = None 45 | 46 | def tearDown(self): 47 | self.deferred = None 48 | self.result = None 49 | 50 | def set_result(self, result): 51 | self.result = result 52 | 53 | def set_exception(self, exception): 54 | self.exception = exception 55 | 56 | def call_later(self, delay, func, *args, **kwargs): 57 | timer = pyev.Timer(delay, 0.0, loop, self._do_later, (func, args, kwargs)) 58 | timer.start() 59 | return timer 60 | 61 | def _do_later(self, watcher, events): 62 | (func, args, kwargs) = watcher.data 63 | func(*args, **kwargs) 64 | watcher.stop() 65 | 66 | def test_callback(self): 67 | self.deferred.add_callback(self.set_result) 68 | self.deferred.callback(5) 69 | self.assertTrue(self.result==5) 70 | 71 | def test_callback_chain(self): 72 | d = self.deferred 73 | d.add_callback(add, 1) 74 | d.add_callback(self.set_result) 75 | self.deferred.callback(5) 76 | self.assertTrue(self.result==6) 77 | 78 | def test_log_error(self): 79 | """Unhandled exceptions should be logged if the deferred is deleted.""" 80 | self.deferred.add_callback(throw_always) 81 | self.deferred.callback(None) 82 | self.deferred = None # delete it 83 | 84 | def test_errback(self): 85 | self.deferred.add_errback(self.set_result) 86 | self.deferred.errback(Exception()) 87 | self.assertTrue(isinstance(self.result, Exception)) 88 | 89 | def test_callback_skips(self): 90 | """When a callback raises an exception 91 | all callbacks without errbacks are skipped until the next 92 | errback is found. 93 | 94 | """ 95 | self.deferred.add_callback(throw_always) 96 | self.deferred.add_callback(one_always) 97 | self.deferred.add_callback(add, 2) 98 | self.deferred.add_errback(one_always) 99 | self.deferred.add_callback(self.set_result) 100 | self.deferred.callback(None) 101 | self.assertTrue(self.result==1) 102 | 103 | def test_errback_reraised(self): 104 | """If an errback raises, then the next errback is called.""" 105 | self.deferred.add_errback(throw_always) 106 | self.deferred.add_errback(self.set_result) 107 | self.deferred.errback(Exception()) 108 | self.assertTrue(isinstance(self.result, Exception)) 109 | 110 | def test_cancelled(self): 111 | self.deferred.cancel() 112 | self.assertRaises(CancelledError, self.deferred.errback, Exception("testcancelled")) 113 | self.assertRaises(CancelledError, self.deferred.callback, None) 114 | self.assertRaises(CancelledError, self.deferred.result) 115 | 116 | def test_already_called(self): 117 | self.deferred.callback(None) 118 | self.assertRaises(AlreadyCalledError, self.deferred.errback, Exception("testalreadycalled")) 119 | self.assertRaises(AlreadyCalledError, self.deferred.callback, None) 120 | self.assertRaises(AlreadyCalledError, self.deferred.cancel) 121 | 122 | def test_cancel_callback(self): 123 | self.deferred = Deferred(loop, cancelled_cb=self.set_result) 124 | self.deferred.cancel() 125 | self.assertTrue(self.result == self.deferred) 126 | 127 | def test_result_chain(self): 128 | self.deferred.callback(5) 129 | self.assertTrue(self.deferred.result()==5) 130 | self.deferred.add_callback(add, 2) 131 | self.assertTrue(self.deferred.result()==7) 132 | self.deferred.add_callback(throw_always) 133 | self.assertRaises(Exception, self.deferred.result) 134 | 135 | def test_result(self): 136 | self.deferred.callback(5) 137 | self.assertTrue(self.deferred.result()==5) 138 | 139 | def test_result_exceptioned(self): 140 | self.deferred.errback(Exception("exceptioned result")) 141 | self.assertRaises(Exception, self.deferred.result) 142 | 143 | def test_delayed_result(self): 144 | now = time.time() 145 | t1 = self.call_later(0.5, self.deferred.callback, 5) 146 | self.assertTrue(self.deferred.result() == 5) 147 | self.assertTrue(time.time() - now > 0.4) 148 | 149 | def test_delayed_result_chained(self): 150 | now = time.time() 151 | t1 = self.call_later(0.5, self.deferred.callback, 5) 152 | self.deferred.add_callback(add, 4) 153 | self.assertTrue(self.deferred.result() == 9) 154 | self.assertTrue(time.time() - now > 0.4) 155 | 156 | def test_delayed_result_timeout(self): 157 | t1 = self.call_later(0.5, self.deferred.callback, 5) 158 | self.assertRaises(TimeoutError, self.deferred.result, 0.1) 159 | 160 | def test_delayed_result_cancelled(self): 161 | t1 = self.call_later(0.5, self.deferred.callback, 5) 162 | t2 = self.call_later(0.2, self.deferred.cancel) 163 | self.assertRaises(CancelledError, self.deferred.result, 0.3) 164 | 165 | if __name__ == '__main__': 166 | unittest.main() 167 | -------------------------------------------------------------------------------- /whizzer/test/test_rpc_pickle.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2010 Tom Burdick 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | 23 | import os 24 | import sys 25 | import socket 26 | import unittest 27 | import pyev 28 | 29 | from whizzer import server, client, protocol 30 | from whizzer.rpc import dispatch, proxy, picklerpc 31 | 32 | from common import loop 33 | 34 | class TestDispatch(unittest.TestCase): 35 | def setUp(self): 36 | self.count = 0 37 | 38 | def tearDown(self): 39 | self.count = 0 40 | 41 | def func(self): 42 | self.count += 1 43 | 44 | def add(self, a, b): 45 | self.count += 1 46 | return a+b 47 | 48 | def test_add_noname(self): 49 | a = dispatch.Dispatch() 50 | a.add(self.func) 51 | self.assertEqual(len(a.functions), 1) 52 | self.assertTrue(self.func in a.functions.values()) 53 | self.assertTrue(self.func.__name__ in a.functions.keys()) 54 | 55 | def test_add_with_name(self): 56 | a = dispatch.Dispatch() 57 | a.add(self.func, "somebogusname") 58 | self.assertEqual(len(a.functions), 1) 59 | self.assertTrue(self.func in a.functions.values()) 60 | self.assertTrue("somebogusname" in a.functions.keys()) 61 | 62 | def test_call_noargs(self): 63 | a = dispatch.Dispatch() 64 | a.add(self.func) 65 | a.call(self.func.__name__, ()) 66 | self.assertEqual(self.count, 1) 67 | 68 | def test_call_args(self): 69 | a = dispatch.Dispatch() 70 | a.add(self.add) 71 | self.assertEqual(a.call(self.add.__name__, (1, 2)), 3) 72 | self.assertEqual(self.count, 1) 73 | 74 | 75 | class MockService(object): 76 | def __init__(self): 77 | self.last_called = None 78 | 79 | @dispatch.remote 80 | def add(self, a, b): 81 | self.last_called = self.add 82 | return a+b 83 | 84 | @dispatch.remote 85 | def exception(self): 86 | self.last_called = self.exception 87 | raise Exception() 88 | 89 | @dispatch.remote 90 | def rpc_error(self): 91 | self.last_called = self.rpc_error 92 | raise Exception() 93 | 94 | @dispatch.remote 95 | def tuple_ret(self, a, b, c): 96 | self.last_called = self.tuple_ret 97 | return (a,b,c) 98 | 99 | @dispatch.remote 100 | def dict_ret(self, a, b, c): 101 | self.last_called = self.dict_ret 102 | return {'a':a,'b':b,'c':c} 103 | 104 | @dispatch.remote 105 | def list_ret(self, a, b, c): 106 | self.last_called = self.list_ret 107 | return [a,b,c] 108 | 109 | @dispatch.remote 110 | def not_set_future_ret(self): 111 | self.last_called = self.not_set_future_ret 112 | return future.Future() 113 | 114 | @dispatch.remote 115 | def set_future_ret(self): 116 | self.last_called = self.set_future_ret 117 | f = future.Future() 118 | f.set_result(True) 119 | return f 120 | 121 | def local_only(self): 122 | self.last_called = self.local_only 123 | pass 124 | 125 | class TestPickleProtocol(unittest.TestCase): 126 | """A functional test against the pickle rpc protocol.""" 127 | def setUp(self): 128 | self.factory = picklerpc.PickleProtocolFactory(dispatch.ObjectDispatch(MockService())) 129 | self.protocol = self.factory.build(loop) 130 | 131 | def tearDown(self): 132 | self.factory = None 133 | self.protocol = None 134 | 135 | def runTest(self): 136 | unittest.TestCase.runTest(self) 137 | 138 | def mock_send_response(self, msgid, result): 139 | """Mock send response to make testing narrowed down and simpler.""" 140 | self.response = (msgid, result) 141 | print "response was " + str(self.response) 142 | 143 | def mock_send_error(self, msgid, error): 144 | """Mock send response to make testing narrowed down and simpler.""" 145 | self.error = (msgid, error) 146 | print "error was " + str(self.error) 147 | 148 | 149 | def test_connection_made(self): 150 | future_proxy = self.protocol.proxy() 151 | self.protocol.connection_made(None) 152 | self.assertTrue(isinstance(future_proxy.result(), proxy.Proxy)) 153 | 154 | def test_handle_request(self): 155 | self.protocol.send_response = self.mock_send_response 156 | self.protocol.send_error = self.mock_send_error 157 | self.protocol.handle_request(0, 0, "add", (1, 2), {}) 158 | self.assertTrue(self.response == (0, 3)) 159 | 160 | def test_handle_unknown_request(self): 161 | self.protocol.send_response = self.mock_send_response 162 | self.protocol.send_error = self.mock_send_error 163 | self.protocol.handle_request(0, 0, "blah_add", (1, 2), {}) 164 | self.assertTrue(self.error[0] == 0) 165 | self.assertTrue(isinstance(self.error[1], KeyError)) 166 | 167 | def test_handle_badargs_request(self): 168 | self.protocol.send_response = self.mock_send_response 169 | self.protocol.send_error = self.mock_send_error 170 | self.protocol.handle_request(0, 0, "add", (1, 2, 3), {}) 171 | self.assertTrue(self.error[0] == 0) 172 | self.assertTrue(isinstance(self.error[1], TypeError)) 173 | 174 | def test_notify(self): 175 | pass 176 | 177 | def test_unknown_notify(self): 178 | pass 179 | 180 | def test_badargs_notify(self): 181 | pass 182 | 183 | def test_exceptioned_notify(self): 184 | pass 185 | 186 | def test_rpcexceptioned_notify(self): 187 | pass 188 | 189 | if __name__ == '__main__': 190 | unittest.main() 191 | -------------------------------------------------------------------------------- /whizzer/test/test_rpc_msgpack.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2010 Tom Burdick 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | 23 | import os 24 | import sys 25 | import socket 26 | import unittest 27 | import pyev 28 | 29 | from whizzer import server, client, protocol 30 | from whizzer.rpc import dispatch, proxy, msgpackrpc 31 | 32 | from common import loop 33 | 34 | class TestDispatch(unittest.TestCase): 35 | def setUp(self): 36 | self.count = 0 37 | 38 | def tearDown(self): 39 | self.count = 0 40 | 41 | def func(self): 42 | self.count += 1 43 | 44 | def add(self, a, b): 45 | self.count += 1 46 | return a+b 47 | 48 | def test_add_noname(self): 49 | a = dispatch.Dispatch() 50 | a.add(self.func) 51 | self.assertEqual(len(a.functions), 1) 52 | self.assertTrue(self.func in a.functions.values()) 53 | self.assertTrue(self.func.__name__ in a.functions.keys()) 54 | 55 | def test_add_with_name(self): 56 | a = dispatch.Dispatch() 57 | a.add(self.func, "somebogusname") 58 | self.assertEqual(len(a.functions), 1) 59 | self.assertTrue(self.func in a.functions.values()) 60 | self.assertTrue("somebogusname" in a.functions.keys()) 61 | 62 | def test_call_noargs(self): 63 | a = dispatch.Dispatch() 64 | a.add(self.func) 65 | a.call(self.func.__name__, ()) 66 | self.assertEqual(self.count, 1) 67 | 68 | def test_call_args(self): 69 | a = dispatch.Dispatch() 70 | a.add(self.add) 71 | self.assertEqual(a.call(self.add.__name__, (1, 2)), 3) 72 | self.assertEqual(self.count, 1) 73 | 74 | 75 | class MockService(object): 76 | def __init__(self): 77 | self.last_called = None 78 | 79 | @dispatch.remote 80 | def add(self, a, b): 81 | self.last_called = self.add 82 | return a+b 83 | 84 | @dispatch.remote 85 | def exception(self): 86 | self.last_called = self.exception 87 | raise Exception() 88 | 89 | @dispatch.remote 90 | def rpc_error(self): 91 | self.last_called = self.rpc_error 92 | raise Exception() 93 | 94 | @dispatch.remote 95 | def tuple_ret(self, a, b, c): 96 | self.last_called = self.tuple_ret 97 | return (a,b,c) 98 | 99 | @dispatch.remote 100 | def dict_ret(self, a, b, c): 101 | self.last_called = self.dict_ret 102 | return {'a':a,'b':b,'c':c} 103 | 104 | @dispatch.remote 105 | def list_ret(self, a, b, c): 106 | self.last_called = self.list_ret 107 | return [a,b,c] 108 | 109 | @dispatch.remote 110 | def not_set_future_ret(self): 111 | self.last_called = self.not_set_future_ret 112 | return future.Future() 113 | 114 | @dispatch.remote 115 | def set_future_ret(self): 116 | self.last_called = self.set_future_ret 117 | f = future.Future() 118 | f.set_result(True) 119 | return f 120 | 121 | def local_only(self): 122 | self.last_called = self.local_only 123 | pass 124 | 125 | class TestMsgPackProtocol(unittest.TestCase): 126 | """A functional test against the msgpack rpc protocol.""" 127 | def setUp(self): 128 | self.factory = msgpackrpc.MsgPackProtocolFactory(dispatch.ObjectDispatch(MockService())) 129 | self.protocol = self.factory.build(loop) 130 | 131 | def tearDown(self): 132 | self.factory = None 133 | self.protocol = None 134 | 135 | def runTest(self): 136 | unittest.TestCase.runTest(self) 137 | 138 | def mock_send_response(self, msgid, error, result): 139 | """Mock send response to make testing narrowed down and simpler.""" 140 | self.response = (msgid, error, result) 141 | print "response was " + str(self.response) 142 | 143 | def test_connection_made(self): 144 | future_proxy = self.protocol.proxy() 145 | self.protocol.connection_made(None) 146 | self.assertTrue(isinstance(future_proxy.result(), proxy.Proxy)) 147 | 148 | def test_handle_request(self): 149 | self.protocol.send_response = self.mock_send_response 150 | self.protocol.request(0, 0, "add", (1, 2)) 151 | self.assertTrue(self.response == (0, None, 3)) 152 | 153 | def test_handle_unknown_request(self): 154 | self.protocol.send_response = self.mock_send_response 155 | self.protocol.request(0, 0, "blah_add", (1, 2)) 156 | self.assertTrue(self.response[0] == 0) 157 | self.assertTrue(self.response[1][0] == KeyError.__name__) 158 | self.assertTrue(self.response[2] == None) 159 | 160 | def test_handle_badargs_request(self): 161 | self.protocol.send_response = self.mock_send_response 162 | self.protocol.request(0, 0, "add", (1, 2, 3)) 163 | self.assertTrue(self.response[0] == 0) 164 | self.assertTrue(self.response[1][0] == TypeError.__name__) 165 | self.assertTrue(self.response[2] == None) 166 | 167 | def test_handle_exceptioned_request(self): 168 | self.assertRaises(Exception, self.protocol.request, 0, 0, "exception") 169 | 170 | def test_handle_rpcexceptioned_request(self): 171 | self.protocol.send_response = self.mock_send_response 172 | self.protocol.request(0, 0, "rpc_error") 173 | self.assertTrue(self.response[1] != None) 174 | 175 | def test_notify(self): 176 | pass 177 | 178 | def test_unknown_notify(self): 179 | pass 180 | 181 | def test_badargs_notify(self): 182 | pass 183 | 184 | def test_exceptioned_notify(self): 185 | pass 186 | 187 | def test_rpcexceptioned_notify(self): 188 | pass 189 | 190 | if __name__ == '__main__': 191 | unittest.main() 192 | -------------------------------------------------------------------------------- /whizzer/transport.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2010 Tom Burdick 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | import socket 23 | import errno 24 | import pyev 25 | 26 | 27 | class ConnectionClosed(Exception): 28 | """Signifies the connection is no longer valid.""" 29 | 30 | 31 | class BufferOverflowError(Exception): 32 | """Signifies something would cause a buffer overflow.""" 33 | 34 | 35 | class SocketTransport(object): 36 | """A buffered writtable transport.""" 37 | 38 | def __init__(self, loop, sock, read_cb, close_cb, max_size=1024 * 512): 39 | """Creates a socket transport that will perform the given functions 40 | whenever the socket is readable or has an error. Writting to the 41 | transport by default simply calls the send() function and checks for 42 | errors. If the error happens to be that the socket is unavailable 43 | (its buffer is full) the write is buffered until the max_size limit is 44 | reached writting out of the buffer whenever the socket is writtable. 45 | 46 | loop -- pyev loop 47 | sock -- python socket object 48 | read_cb -- read function (callback when the socket is read) 49 | close_cb -- closed function (callback when the socket has been closed) 50 | max_size -- maximum user space buffer 51 | 52 | """ 53 | self.loop = loop 54 | self.sock = sock 55 | self.read_cb = read_cb 56 | self.close_cb = close_cb 57 | self.max_size = max_size 58 | self.sock.setblocking(False) 59 | self.read_watcher = pyev.Io(self.sock, pyev.EV_READ, self.loop, 60 | self._readable) 61 | self.write_watcher = pyev.Io(self.sock, pyev.EV_WRITE, self.loop, 62 | self._writtable) 63 | self.write_buffer = bytearray() 64 | self.closed = False 65 | 66 | self.write = self.unbuffered_write 67 | 68 | def start(self): 69 | """Start watching the socket.""" 70 | if self.closed: 71 | raise ConnectionClosed() 72 | 73 | self.read_watcher.start() 74 | if self.write == self.buffered_write: 75 | self.write_watcher.start() 76 | 77 | def stop(self): 78 | """Stop watching the socket.""" 79 | if self.closed: 80 | raise ConnectionClosed() 81 | 82 | if self.read_watcher.active: 83 | self.read_watcher.stop() 84 | if self.write_watcher.active: 85 | self.write_watcher.stop() 86 | 87 | def write(self, buf): 88 | """Write data to a non-blocking socket. 89 | 90 | This function is aliased depending on the state of the socket. 91 | 92 | It may either be unbuffered_write or buffered_write, the caller 93 | should not care. 94 | 95 | buf -- bytes to send 96 | 97 | """ 98 | 99 | def unbuffered_write(self, buf): 100 | """Performs an unbuffered write, the default unless socket.send does 101 | not send everything, in which case an unbuffered write is done and the 102 | write method is set to be a buffered write until the buffer is empty 103 | once again. 104 | 105 | buf -- bytes to send 106 | 107 | """ 108 | if self.closed: 109 | raise ConnectionClosed() 110 | 111 | result = 0 112 | try: 113 | result = self.sock.send(buf) 114 | except EnvironmentError as e: 115 | # if the socket is simply backed up ignore the error 116 | if e.errno != errno.EAGAIN: 117 | self._close(e) 118 | return 119 | 120 | # when the socket buffers are full/backed up then we need to poll to see 121 | # when we can write again 122 | if result != len(buf): 123 | self.write = self.buffered_write 124 | self.write_watcher.start() 125 | self.write(buf[result:]) 126 | 127 | def buffered_write(self, buf): 128 | """Appends a bytes like object to the transport write buffer. 129 | 130 | Raises BufferOverflowError if buf would cause the buffer to grow beyond 131 | the specified maximum. 132 | 133 | buf -- bytes to send 134 | 135 | """ 136 | if self.closed: 137 | raise ConnectionClosed() 138 | 139 | if len(buf) + len(self.write_buffer) > self.max_size: 140 | raise BufferOverflowError() 141 | else: 142 | self.write_buffer.extend(buf) 143 | 144 | def _writtable(self, watcher, events): 145 | """Called by the pyev watcher (self.write_watcher) whenever the socket 146 | is writtable. 147 | 148 | Calls send using the userspace buffer (self.write_buffer) and checks 149 | for errors. If there are no errors then continue on as before. 150 | Otherwise closes the socket and calls close_cb with the error. 151 | 152 | """ 153 | try: 154 | sent = self.sock.send(bytes(self.write_buffer)) 155 | self.write_buffer = self.write_buffer[sent:] 156 | if len(self.write_buffer) == 0: 157 | self.write_watcher.stop() 158 | self.write = self.unbuffered_write 159 | except EnvironmentError as e: 160 | self._close(e) 161 | 162 | def _readable(self, watcher, events): 163 | """Called by the pyev watcher (self.read_watcher) whenever the socket 164 | is readable. 165 | 166 | Calls recv and checks for errors. If there are no errors then read_cb 167 | is called with the newly arrived bytes. Otherwise closes the socket 168 | and calls close_cb with the error. 169 | 170 | """ 171 | try: 172 | data = self.sock.recv(4096) 173 | if len(data) == 0: 174 | self._close(ConnectionClosed()) 175 | else: 176 | self.read_cb(data) 177 | except IOError as e: 178 | self._close(e) 179 | 180 | def _close(self, e): 181 | """Really close the transport with a reason. 182 | 183 | e -- reason the socket is being closed. 184 | 185 | """ 186 | self.stop() 187 | self.sock.close() 188 | self.closed = True 189 | self.close_cb(e) 190 | 191 | def close(self): 192 | """Close the transport.""" 193 | self._close(ConnectionClosed()) 194 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # whizzer documentation build configuration file, created by 4 | # sphinx-quickstart on Tue Aug 24 19:23:30 2010. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | import sys, os 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | sys.path.insert(0, os.path.abspath('..')) 20 | 21 | # -- General configuration ----------------------------------------------------- 22 | 23 | # If your documentation needs a minimal Sphinx version, state it here. 24 | #needs_sphinx = '1.0' 25 | 26 | # Add any Sphinx extension module names here, as strings. They can be extensions 27 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 28 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.viewcode'] 29 | 30 | # Add any paths that contain templates here, relative to this directory. 31 | templates_path = ['_templates'] 32 | 33 | # The suffix of source filenames. 34 | source_suffix = '.rst' 35 | 36 | # The encoding of source files. 37 | #source_encoding = 'utf-8-sig' 38 | 39 | # The master toctree document. 40 | master_doc = 'index' 41 | 42 | # General information about the project. 43 | project = u'whizzer' 44 | copyright = u'2010, Tom Burdick' 45 | 46 | # The version info for the project you're documenting, acts as replacement for 47 | # |version| and |release|, also used in various other places throughout the 48 | # built documents. 49 | # 50 | # The short X.Y version. 51 | version = '0.1' 52 | # The full version, including alpha/beta/rc tags. 53 | release = '0.1' 54 | 55 | # The language for content autogenerated by Sphinx. Refer to documentation 56 | # for a list of supported languages. 57 | #language = None 58 | 59 | # There are two options for replacing |today|: either, you set today to some 60 | # non-false value, then it is used: 61 | #today = '' 62 | # Else, today_fmt is used as the format for a strftime call. 63 | #today_fmt = '%B %d, %Y' 64 | 65 | # List of patterns, relative to source directory, that match files and 66 | # directories to ignore when looking for source files. 67 | exclude_patterns = ['_build'] 68 | 69 | # The reST default role (used for this markup: `text`) to use for all documents. 70 | #default_role = None 71 | 72 | # If true, '()' will be appended to :func: etc. cross-reference text. 73 | #add_function_parentheses = True 74 | 75 | # If true, the current module name will be prepended to all description 76 | # unit titles (such as .. function::). 77 | #add_module_names = True 78 | 79 | # If true, sectionauthor and moduleauthor directives will be shown in the 80 | # output. They are ignored by default. 81 | #show_authors = False 82 | 83 | # The name of the Pygments (syntax highlighting) style to use. 84 | pygments_style = 'sphinx' 85 | 86 | # A list of ignored prefixes for module index sorting. 87 | #modindex_common_prefix = [] 88 | 89 | 90 | # -- Options for HTML output --------------------------------------------------- 91 | 92 | # The theme to use for HTML and HTML Help pages. See the documentation for 93 | # a list of builtin themes. 94 | html_theme = 'default' 95 | 96 | # Theme options are theme-specific and customize the look and feel of a theme 97 | # further. For a list of options available for each theme, see the 98 | # documentation. 99 | #html_theme_options = {} 100 | 101 | # Add any paths that contain custom themes here, relative to this directory. 102 | #html_theme_path = [] 103 | 104 | # The name for this set of Sphinx documents. If None, it defaults to 105 | # " v documentation". 106 | #html_title = None 107 | 108 | # A shorter title for the navigation bar. Default is the same as html_title. 109 | #html_short_title = None 110 | 111 | # The name of an image file (relative to this directory) to place at the top 112 | # of the sidebar. 113 | #html_logo = None 114 | 115 | # The name of an image file (within the static path) to use as favicon of the 116 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 117 | # pixels large. 118 | #html_favicon = None 119 | 120 | # Add any paths that contain custom static files (such as style sheets) here, 121 | # relative to this directory. They are copied after the builtin static files, 122 | # so a file named "default.css" will overwrite the builtin "default.css". 123 | html_static_path = ['_static'] 124 | 125 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 126 | # using the given strftime format. 127 | #html_last_updated_fmt = '%b %d, %Y' 128 | 129 | # If true, SmartyPants will be used to convert quotes and dashes to 130 | # typographically correct entities. 131 | #html_use_smartypants = True 132 | 133 | # Custom sidebar templates, maps document names to template names. 134 | #html_sidebars = {} 135 | 136 | # Additional templates that should be rendered to pages, maps page names to 137 | # template names. 138 | #html_additional_pages = {} 139 | 140 | # If false, no module index is generated. 141 | #html_domain_indices = True 142 | 143 | # If false, no index is generated. 144 | #html_use_index = True 145 | 146 | # If true, the index is split into individual pages for each letter. 147 | #html_split_index = False 148 | 149 | # If true, links to the reST sources are added to the pages. 150 | #html_show_sourcelink = True 151 | 152 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 153 | #html_show_sphinx = True 154 | 155 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 156 | #html_show_copyright = True 157 | 158 | # If true, an OpenSearch description file will be output, and all pages will 159 | # contain a tag referring to it. The value of this option must be the 160 | # base URL from which the finished HTML is served. 161 | #html_use_opensearch = '' 162 | 163 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 164 | #html_file_suffix = None 165 | 166 | # Output file base name for HTML help builder. 167 | htmlhelp_basename = 'whizzerdoc' 168 | 169 | 170 | # -- Options for LaTeX output -------------------------------------------------- 171 | 172 | # The paper size ('letter' or 'a4'). 173 | #latex_paper_size = 'letter' 174 | 175 | # The font size ('10pt', '11pt' or '12pt'). 176 | #latex_font_size = '10pt' 177 | 178 | # Grouping the document tree into LaTeX files. List of tuples 179 | # (source start file, target name, title, author, documentclass [howto/manual]). 180 | latex_documents = [ 181 | ('index', 'whizzer.tex', u'whizzer Documentation', 182 | u'Tom Burdick', 'manual'), 183 | ] 184 | 185 | # The name of an image file (relative to this directory) to place at the top of 186 | # the title page. 187 | #latex_logo = None 188 | 189 | # For "manual" documents, if this is true, then toplevel headings are parts, 190 | # not chapters. 191 | #latex_use_parts = False 192 | 193 | # If true, show page references after internal links. 194 | #latex_show_pagerefs = False 195 | 196 | # If true, show URL addresses after external links. 197 | #latex_show_urls = False 198 | 199 | # Additional stuff for the LaTeX preamble. 200 | #latex_preamble = '' 201 | 202 | # Documents to append as an appendix to all manuals. 203 | #latex_appendices = [] 204 | 205 | # If false, no module index is generated. 206 | #latex_domain_indices = True 207 | 208 | 209 | # -- Options for manual page output -------------------------------------------- 210 | 211 | # One entry per manual page. List of tuples 212 | # (source start file, name, description, authors, manual section). 213 | man_pages = [ 214 | ('index', 'whizzer', u'whizzer Documentation', 215 | [u'Tom Burdick'], 1) 216 | ] 217 | -------------------------------------------------------------------------------- /whizzer/rpc/msgpackrpc.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2010 Tom Burdick 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | 23 | import msgpack 24 | 25 | from whizzer.protocol import Protocol, ProtocolFactory 26 | from whizzer.defer import Deferred 27 | 28 | from whizzer.rpc.proxy import Proxy 29 | from whizzer.rpc.dispatch import Dispatch 30 | 31 | 32 | class MsgPackProxy(Proxy): 33 | """A MessagePack-RPC Proxy.""" 34 | 35 | def __init__(self, loop, protocol): 36 | """MsgPackProxy 37 | 38 | loop -- A pyev loop. 39 | protocol -- An instance of MsgPackProtocol. 40 | 41 | """ 42 | self.loop = loop 43 | self.protocol = protocol 44 | self.request_num = 0 45 | self.requests = dict() 46 | self.timeout = None 47 | 48 | def set_timeout(self, timeout): 49 | """Set the timeout of blocking calls, None means block forever. 50 | 51 | timeout -- seconds after which to raise a TimeoutError for blocking calls. 52 | 53 | """ 54 | self.timeout = timeout 55 | 56 | def call(self, method, *args): 57 | """Perform a synchronous remote call where the returned value is given immediately. 58 | 59 | This may block for sometime in certain situations. If it takes more than the Proxies 60 | set timeout then a TimeoutError is raised. 61 | 62 | Any exceptions the remote call raised that can be sent over the wire are raised. 63 | 64 | Internally this calls begin_call(method, *args).result(timeout=self.timeout) 65 | 66 | """ 67 | return self.begin_call(method, *args).result(self.timeout) 68 | 69 | def notify(self, method, *args): 70 | """Perform a synchronous remote call where value no return value is desired. 71 | 72 | While faster than call it still blocks until the remote callback has been sent. 73 | 74 | This may block for sometime in certain situations. If it takes more than the Proxies 75 | set timeout then a TimeoutError is raised. 76 | 77 | """ 78 | self.protocol.send_notification(method, args) 79 | 80 | def begin_call(self, method, *args): 81 | """Perform an asynchronous remote call where the return value is not known yet. 82 | 83 | This returns immediately with a Deferred object. The Deferred object may then be 84 | used to attach a callback, force waiting for the call, or check for exceptions. 85 | 86 | """ 87 | d = Deferred(self.loop) 88 | d.request = self.request_num 89 | self.requests[self.request_num] = d 90 | self.protocol.send_request(d.request, method, args) 91 | self.request_num += 1 92 | return d 93 | 94 | def response(self, msgid, error, result): 95 | """Handle a results message given to the proxy by the protocol object.""" 96 | if error: 97 | self.requests[msgid].errback(Exception(str(error))) 98 | else: 99 | self.requests[msgid].callback(result) 100 | del self.requests[msgid] 101 | 102 | class MsgPackProtocol(Protocol): 103 | def __init__(self, loop, factory, dispatch=Dispatch()): 104 | Protocol.__init__(self, loop) 105 | self.factory = factory 106 | self.dispatch = dispatch 107 | self._proxy = None 108 | self._proxy_deferreds = [] 109 | self.handlers = {0:self.request, 1:self.response, 2:self.notify} 110 | self.unpacker = msgpack.Unpacker() 111 | 112 | def connection_made(self, address): 113 | """When a connection is made the proxy is available.""" 114 | self._proxy = MsgPackProxy(self.loop, self) 115 | for d in self._proxy_deferreds: 116 | d.callback(self._proxy) 117 | 118 | def response(self, msgtype, msgid, error, result): 119 | """Handle an incoming response.""" 120 | self._proxy.response(msgid, error, result) 121 | 122 | def notify(self, msgtype, method, params): 123 | """Handle an incoming notify request.""" 124 | self.dispatch.call(method, params) 125 | 126 | def request(self, msgtype, msgid, method, params=[]): 127 | """Handle an incoming call request.""" 128 | result = None 129 | error = None 130 | exception = None 131 | 132 | try: 133 | result = self.dispatch.call(method, params) 134 | except Exception as e: 135 | error = (e.__class__.__name__, str(e)) 136 | exception = e 137 | 138 | if isinstance(result, Deferred): 139 | result.add_callback(self._result, msgid) 140 | result.add_errback(self._error, msgid) 141 | else: 142 | self.send_response(msgid, error, result) 143 | 144 | def data(self, data): 145 | """Use msgpack's streaming feed feature to build up a set of lists. 146 | 147 | The lists should then contain the messagepack-rpc specified items. 148 | 149 | This should be outrageously fast. 150 | 151 | """ 152 | self.unpacker.feed(data) 153 | for msg in self.unpacker: 154 | self.handlers[msg[0]](*msg) 155 | 156 | def _result(self, result, msgid): 157 | self.send_response(msgid, None, result) 158 | 159 | def _error(self, exception, msgid): 160 | self.send_response(msgid, exception, None) 161 | 162 | def send_request(self, msgid, method, params): 163 | msg = msgpack.packb([0, msgid, method, params]) 164 | self.transport.write(msg) 165 | 166 | def send_response(self, msgid, error, result): 167 | msg = msgpack.packb([1, msgid, error, result]) 168 | self.transport.write(msg) 169 | 170 | def send_notification(self, method, params): 171 | msg = msgpack.packb([2, method, params]) 172 | self.transport.write(msg) 173 | 174 | def proxy(self): 175 | """Return a Deferred that will result in a proxy object in the future.""" 176 | d = Deferred(self.loop) 177 | self._proxy_deferreds.append(d) 178 | 179 | if self._proxy: 180 | d.callback(self._proxy) 181 | 182 | return d 183 | 184 | def connection_lost(self, reason=None): 185 | """Tell the factory we lost our connection.""" 186 | self.factory.lost_connection(self) 187 | self.factory = None 188 | 189 | 190 | class MsgPackProtocolFactory(ProtocolFactory): 191 | def __init__(self, dispatch=Dispatch()): 192 | ProtocolFactory.__init__(self) 193 | self.dispatch = dispatch 194 | self.protocol = MsgPackProtocol 195 | self.protocols = [] 196 | 197 | def proxy(self, conn_number): 198 | """Return a proxy for a given connection number.""" 199 | return self.protocols[conn_number].proxy() 200 | 201 | def build(self, loop): 202 | p = self.protocol(loop, self, self.dispatch) 203 | self.protocols.append(p) 204 | return p 205 | 206 | def lost_connection(self, p): 207 | """Called by the rpc protocol whenever it loses a connection.""" 208 | self.protocols.remove(p) 209 | -------------------------------------------------------------------------------- /whizzer/server.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2010 Tom Burdick 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | import os 23 | import signal 24 | import socket 25 | import logbook 26 | import pyev 27 | 28 | from whizzer.transport import SocketTransport, ConnectionClosed 29 | 30 | logger = logbook.Logger(__name__) 31 | 32 | 33 | class Connection(object): 34 | """A connection to the server from a remote client.""" 35 | def __init__(self, loop, sock, address, protocol, server): 36 | """Create a server connection.""" 37 | self.loop = loop 38 | self.sock = sock 39 | self.address = address 40 | self.protocol = protocol 41 | self.server = server 42 | self.transport = SocketTransport(self.loop, self.sock, self.protocol.data, self.closed) 43 | 44 | def make_connection(self): 45 | self.transport.start() 46 | self.protocol.make_connection(self.transport, self.address) 47 | 48 | def closed(self, reason): 49 | """Callback performed when the transport is closed.""" 50 | self.server.remove_connection(self) 51 | self.protocol.connection_lost(reason) 52 | if not isinstance(reason, ConnectionClosed): 53 | logger.warn("connection closed, reason: %s" % str(reason)) 54 | else: 55 | logger.info("connection closed") 56 | 57 | def close(self): 58 | """Close the connection.""" 59 | self.transport.close() 60 | 61 | 62 | class ShutdownError(Exception): 63 | """Error signifying the server has already been shutdown and cannot be 64 | used further.""" 65 | 66 | class SocketServer(object): 67 | """A socket server.""" 68 | def __init__(self, loop, factory, sock, address): 69 | """Socket server listens on a given socket for incoming connections. 70 | When a new connection is available it accepts it and creates a new 71 | Connection and Protocol to handle reading and writting data. 72 | 73 | loop -- pyev loop 74 | factory -- protocol factory (object with build(loop) method that returns a protocol object) 75 | sock -- socket to listen on 76 | 77 | """ 78 | self.loop = loop 79 | self.factory = factory 80 | self.sock = sock 81 | self.address = address 82 | self.connections = set() 83 | self._closing = False 84 | self._shutdown = False 85 | self.interrupt_watcher = pyev.Signal(signal.SIGINT, self.loop, self._interrupt) 86 | self.interrupt_watcher.start() 87 | self.read_watcher = pyev.Io(self.sock, pyev.EV_READ, self.loop, self._readable) 88 | 89 | def start(self): 90 | """Start the socket server. 91 | 92 | The socket server will begin accepting incoming connections. 93 | 94 | """ 95 | if self._shutdown: 96 | raise ShutdownError() 97 | 98 | self.read_watcher.start() 99 | logger.info("server started listening on {}".format(self.address)) 100 | 101 | def stop(self): 102 | """Stop the socket server. 103 | 104 | The socket server will stop accepting incoming connections. 105 | 106 | The connections already made will continue to exist. 107 | 108 | """ 109 | if self._shutdown: 110 | raise ShutdownError() 111 | 112 | self.read_watcher.stop() 113 | logger.info("server stopped listening on {}".format(self.address)) 114 | 115 | def shutdown(self, reason = ConnectionClosed()): 116 | """Shutdown the socket server. 117 | 118 | The socket server will stop accepting incoming connections. 119 | 120 | All connections will be dropped. 121 | 122 | """ 123 | if self._shutdown: 124 | raise ShutdownError() 125 | 126 | self.stop() 127 | 128 | self._closing = True 129 | for connection in self.connections: 130 | connection.close() 131 | self.connections = set() 132 | self._shutdown = True 133 | if isinstance(reason, ConnectionClosed): 134 | logger.info("server shutdown") 135 | else: 136 | logger.warn("server shutdown, reason %s" % str(reason)) 137 | 138 | def _interrupt(self, watcher, events): 139 | """Handle the interrupt signal sanely.""" 140 | self.shutdown() 141 | 142 | def _readable(self, watcher, events): 143 | """Called by the pyev watcher (self.read_watcher) whenever the socket 144 | is readable. 145 | 146 | This means either the socket has been closed or there is a new 147 | client connection waiting. 148 | 149 | """ 150 | protocol = self.factory.build(self.loop) 151 | try: 152 | sock, address = self.sock.accept() 153 | connection = Connection(self.loop, sock, address, protocol, self) 154 | self.connections.add(connection) 155 | connection.make_connection() 156 | logger.debug("added connection") 157 | except IOError as e: 158 | self.shutdown(e) 159 | 160 | def remove_connection(self, connection): 161 | """Called by the connections themselves when they have been closed.""" 162 | if not self._closing: 163 | self.connections.remove(connection) 164 | logger.debug("removed connection") 165 | 166 | class _PathRemoval(object): 167 | """Remove a path when the object dies. 168 | 169 | Used by UnixServer so that a __del__ method is not needed for UnixServer. 170 | 171 | """ 172 | def __init__(self, path): 173 | self.path = path 174 | 175 | def __del__(self): 176 | if os.path.exists(self.path): 177 | os.unlink(self.path) 178 | 179 | class UnixServer(SocketServer): 180 | """A unix server is a socket server that listens on a domain socket.""" 181 | def __init__(self, loop, factory, path, backlog=256): 182 | self.address = path 183 | self.path_removal = _PathRemoval(self.address) 184 | self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) 185 | self.sock.bind(path) 186 | self.sock.listen(backlog) 187 | self.sock.setblocking(False) 188 | SocketServer.__init__(self, loop, factory, self.sock, self.address) 189 | 190 | def shutdown(self): 191 | """Shutdown the socket unix socket server ensuring the unix socket is 192 | removed. 193 | 194 | """ 195 | err = None 196 | SocketServer.shutdown(self) 197 | 198 | class TcpServer(SocketServer): 199 | """A tcp server is a socket server that listens on a internet socket.""" 200 | def __init__(self, loop, factory, host, port, backlog=256): 201 | self.address = (host, port) 202 | self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 203 | self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 204 | self.sock.bind((host, port)) 205 | self.sock.listen(backlog) 206 | self.sock.setblocking(False) 207 | SocketServer.__init__(self, loop, factory, self.sock, self.address) 208 | -------------------------------------------------------------------------------- /whizzer/client.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2010 Tom Burdick 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | import socket 23 | 24 | import signal 25 | import logbook 26 | import pyev 27 | 28 | from whizzer.transport import SocketTransport, ConnectionClosed 29 | from whizzer.defer import Deferred 30 | 31 | logger = logbook.Logger(__name__) 32 | 33 | 34 | class TimeoutError(Exception): 35 | pass 36 | 37 | 38 | class Connection(object): 39 | """Represents a connection to a server from a client.""" 40 | 41 | def __init__(self, loop, sock, addr, protocol, client): 42 | """Create a client connection.""" 43 | self.loop = loop 44 | self.sock = sock 45 | self.addr = addr 46 | self.protocol = protocol 47 | self.client = client 48 | logger.debug('making transport') 49 | self.transport = SocketTransport(self.loop, self.sock, 50 | self.protocol.data, self.closed) 51 | logger.debug('protocol.make_connection') 52 | self.protocol.make_connection(self.transport, self.addr) 53 | logger.debug('transport.start()') 54 | self.transport.start() 55 | logger.debug('transport started') 56 | 57 | def closed(self, reason): 58 | """Callback performed when the transport is closed.""" 59 | self.client.remove_connection(self) 60 | self.protocol.connection_lost(reason) 61 | if not isinstance(reason, ConnectionClosed): 62 | logger.warn("connection closed, reason {}".format(reason)) 63 | else: 64 | logger.info("connection closed") 65 | 66 | def close(self): 67 | """Close the connection.""" 68 | self.transport.close() 69 | 70 | 71 | class ConnectorStartedError(Exception): 72 | """Connectors may only be started once.""" 73 | 74 | 75 | class Connector(object): 76 | """State machine for a connection to a remote socket.""" 77 | 78 | def __init__(self, loop, sock, addr, timeout): 79 | self.loop = loop 80 | self.sock = sock 81 | self.addr = addr 82 | self.timeout = timeout 83 | self.connect_watcher = pyev.Io(self.sock, pyev.EV_WRITE, self.loop, self._connected) 84 | self.timeout_watcher = pyev.Timer(self.timeout, 0.0, self.loop, self._timeout) 85 | self.deferred = Deferred(self.loop) 86 | self.started = False 87 | self.connected = False 88 | self.timedout = False 89 | self.errored = False 90 | 91 | def start(self): 92 | """Start the connector state machine.""" 93 | if self.started: 94 | raise ConnectorStartedError() 95 | 96 | self.started = True 97 | 98 | try: 99 | self.connect_watcher.start() 100 | self.timeout_watcher.start() 101 | self.sock.connect(self.addr) 102 | except IOError as e: 103 | self.errored = True 104 | self._finish() 105 | self.deferred.errback(e) 106 | 107 | return self.deferred 108 | 109 | def cancel(self): 110 | """Cancel a connector from completing.""" 111 | if self.started and not self.connected and not self.timedout: 112 | self.connect_watcher.stop() 113 | self.timeout_watcher.stop() 114 | 115 | def _connected(self, watcher, events): 116 | """Connector is successful, return the socket.""" 117 | self.connected = True 118 | self._finish() 119 | self.deferred.callback(self.sock) 120 | 121 | def _timeout(self, watcher, events): 122 | """Connector timed out, raise a timeout error.""" 123 | self.timedout = True 124 | self._finish() 125 | self.deferred.errback(TimeoutError()) 126 | 127 | def _finish(self): 128 | """Finalize the connector.""" 129 | self.connect_watcher.stop() 130 | self.timeout_watcher.stop() 131 | 132 | 133 | class SocketClientConnectedError(object): 134 | """Raised when a client is already connected.""" 135 | 136 | 137 | class SocketClientConnectingError(object): 138 | """Raised when a client is already connecting.""" 139 | 140 | 141 | class SocketClient(object): 142 | """A simple socket client.""" 143 | def __init__(self, loop, factory): 144 | self.loop = loop 145 | self.factory = factory 146 | self.connector = None 147 | self.connection = None 148 | self.connect_deferred = None 149 | 150 | self.sigint_watcher = pyev.Signal(signal.SIGINT, self.loop, 151 | self._interrupt) 152 | self.sigint_watcher.start() 153 | 154 | self.connector = None 155 | self.sock = None 156 | self.addr = None 157 | 158 | def _interrupt(self, watcher, events): 159 | if self.connection: 160 | self.connection.close() 161 | 162 | def _connect(self, sock, addr, timeout): 163 | """Start watching the socket for it to be writtable.""" 164 | if self.connection: 165 | raise SocketClientConnectedError() 166 | 167 | if self.connector: 168 | raise SocketClientConnectingError() 169 | 170 | self.connect_deferred = Deferred(self.loop) 171 | self.sock = sock 172 | self.addr = addr 173 | self.connector = Connector(self.loop, sock, addr, timeout) 174 | self.connector.deferred.add_callback(self._connected) 175 | self.connector.deferred.add_errback(self._connect_failed) 176 | self.connector.start() 177 | 178 | return self.connect_deferred 179 | 180 | def _connected(self, sock): 181 | """When the socket is writtable, the socket is ready to be used.""" 182 | logger.debug('socket connected, building protocol') 183 | self.protocol = self.factory.build(self.loop) 184 | self.connection = Connection(self.loop, self.sock, self.addr, 185 | self.protocol, self) 186 | self.connector = None 187 | self.connect_deferred.callback(self.protocol) 188 | 189 | def _connect_failed(self, reason): 190 | """Connect failed.""" 191 | self.connector = None 192 | self.connect_deferred.errback(reason) 193 | 194 | def _disconnect(self): 195 | """Disconnect from a socket.""" 196 | if self.connection: 197 | self.connection.close() 198 | self.connection = None 199 | 200 | def connect(self, timeout=5): 201 | """Should be overridden to create a socket and connect it. 202 | 203 | Once the socket is connected it should be passed to _connect. 204 | 205 | """ 206 | 207 | def remove_connection(self, connection): 208 | self.connection = None 209 | 210 | 211 | class UnixClient(SocketClient): 212 | """A unix client is a socket client that connects to a domain socket.""" 213 | def __init__(self, loop, factory, path): 214 | SocketClient.__init__(self, loop, factory) 215 | self.path = path 216 | 217 | def connect(self, timeout=5.0): 218 | sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) 219 | return self._connect(sock, self.path, timeout) 220 | 221 | def disconnect(self): 222 | return self._disconnect() 223 | 224 | 225 | class TcpClient(SocketClient): 226 | """A unix client is a socket client that connects to a domain socket.""" 227 | def __init__(self, loop, factory, host, port): 228 | SocketClient.__init__(self, loop, factory) 229 | self.host = host 230 | self.port = port 231 | 232 | def connect(self, timeout=5.0): 233 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 234 | return self._connect(sock, (self.host, self.port), timeout) 235 | 236 | def disconnect(self): 237 | return self._disconnect() 238 | -------------------------------------------------------------------------------- /whizzer/rpc/picklerpc.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2010 Tom Burdick 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | 23 | try: 24 | import cPickle as pickle 25 | except: 26 | import pickle 27 | 28 | import struct 29 | 30 | import logbook 31 | 32 | from whizzer.protocol import Protocol, ProtocolFactory 33 | from whizzer.defer import Deferred 34 | 35 | from whizzer.rpc.proxy import Proxy 36 | from whizzer.rpc.dispatch import Dispatch 37 | 38 | logger = logbook.Logger(__name__) 39 | 40 | 41 | def dumps(obj): 42 | return pickle.dumps(obj, pickle.HIGHEST_PROTOCOL) 43 | 44 | def loads(string): 45 | return pickle.loads(string) 46 | 47 | 48 | class PickleProxy(Proxy): 49 | """A MessagePack-RPC Proxy.""" 50 | 51 | def __init__(self, loop, protocol): 52 | """PickleProxy 53 | 54 | loop -- A pyev loop. 55 | protocol -- An instance of PickleProtocol. 56 | 57 | """ 58 | self.loop = loop 59 | self.protocol = protocol 60 | self.request_num = 0 61 | self.requests = dict() 62 | self.timeout = None 63 | 64 | def set_timeout(self, timeout): 65 | """Set the timeout of blocking calls, None means block forever. 66 | 67 | timeout -- seconds after which to raise a TimeoutError for blocking calls. 68 | 69 | """ 70 | self.timeout = timeout 71 | 72 | def call(self, method, *args, **kwargs): 73 | """Perform a synchronous remote call where the returned value is given immediately. 74 | 75 | This may block for sometime in certain situations. If it takes more than the Proxies 76 | set timeout then a TimeoutError is raised. 77 | 78 | Any exceptions the remote call raised that can be sent over the wire are raised. 79 | 80 | Internally this calls begin_call(method, *args).result(timeout=self.timeout) 81 | 82 | """ 83 | return self.begin_call(method, *args, **kwargs).result(self.timeout) 84 | 85 | def notify(self, method, *args, **kwargs): 86 | """Perform a synchronous remote call where value no return value is desired. 87 | 88 | While faster than call it still blocks until the remote callback has been sent. 89 | 90 | This may block for sometime in certain situations. If it takes more than the Proxies 91 | set timeout then a TimeoutError is raised. 92 | 93 | """ 94 | self.protocol.send_notification(method, args, kwargs) 95 | 96 | def begin_call(self, method, *args, **kwargs): 97 | """Perform an asynchronous remote call where the return value is not known yet. 98 | 99 | This returns immediately with a Deferred object. The Deferred object may then be 100 | used to attach a callback, force waiting for the call, or check for exceptions. 101 | 102 | """ 103 | d = Deferred(self.loop) 104 | d.request = self.request_num 105 | self.requests[self.request_num] = d 106 | self.protocol.send_request(d.request, method, args, kwargs) 107 | self.request_num += 1 108 | return d 109 | 110 | def response(self, msgid, response): 111 | """Handle a response message.""" 112 | self.requests[msgid].callback(response) 113 | del self.requests[msgid] 114 | 115 | def error(self, msgid, error): 116 | """Handle a error message.""" 117 | self.requests[msgid].errback(error) 118 | del self.requests[msgid] 119 | 120 | class PickleProtocol(Protocol): 121 | def __init__(self, loop, factory, dispatch=Dispatch()): 122 | Protocol.__init__(self, loop) 123 | self.factory = factory 124 | self.dispatch = dispatch 125 | self._proxy = None 126 | self._proxy_deferreds = [] 127 | self.handlers = {0:self.handle_request, 1:self.handle_notification, 128 | 2:self.handle_response, 3:self.handle_error} 129 | self._buffer = bytes() 130 | self._data_handler = self.data_length 131 | 132 | def connection_made(self, address): 133 | """When a connection is made the proxy is available.""" 134 | self._proxy = PickleProxy(self.loop, self) 135 | for d in self._proxy_deferreds: 136 | d.callback(self._proxy) 137 | 138 | def data(self, data): 139 | """Use a length prefixed protocol to give the length of a pickled 140 | message. 141 | 142 | """ 143 | self._buffer = self._buffer + data 144 | 145 | while self._data_handler(): 146 | pass 147 | 148 | def connection_lost(self, reason=None): 149 | """Tell the factory we lost our connection.""" 150 | self.factory.lost_connection(self) 151 | self.factory = None 152 | 153 | def data_length(self): 154 | if len(self._buffer) >= 4: 155 | self._msglen = struct.unpack('!I', self._buffer[:4])[0] 156 | self._buffer = self._buffer[4:] 157 | self._data_handler = self.data_message 158 | return True 159 | return False 160 | 161 | def data_message(self): 162 | if len(self._buffer) >= self._msglen: 163 | msg = loads(self._buffer[:self._msglen]) 164 | self.handlers[msg[0]](*msg) 165 | self._buffer = self._buffer[self._msglen:] 166 | self._data_handler = self.data_length 167 | return True 168 | return False 169 | 170 | def handle_request(self, msgtype, msgid, method, args, kwargs): 171 | """Handle a request.""" 172 | response = None 173 | error = None 174 | exception = None 175 | 176 | try: 177 | response = self.dispatch.call(method, args, kwargs) 178 | except Exception as e: 179 | error = (e.__class__.__name__, str(e)) 180 | exception = e 181 | 182 | if isinstance(response, Deferred): 183 | response.add_callback(self.send_response, msgid) 184 | response.add_errback(self.send_error, msgid) 185 | else: 186 | if exception is None: 187 | self.send_response(msgid, response) 188 | else: 189 | self.send_error(msgid, exception) 190 | 191 | def handle_notification(self, msgtype, method, args, kwargs): 192 | """Handle a notification.""" 193 | self.dispatch.call(method, args, kwargs) 194 | 195 | def handle_response(self, msgtype, msgid, response): 196 | """Handle a response.""" 197 | self._proxy.response(msgid, response) 198 | 199 | def handle_error(self, msgtype, msgid, error): 200 | """Handle an error.""" 201 | self._proxy.error(msgid, error) 202 | 203 | def send(self, msg): 204 | length = struct.pack('!I', len(msg)) 205 | self.transport.write(length) 206 | self.transport.write(msg) 207 | 208 | def send_request(self, msgid, method, args, kwargs): 209 | """Send a request.""" 210 | msg = dumps([0, msgid, method, args, kwargs]) 211 | self.send(msg) 212 | 213 | def send_notification(self, method, args, kwargs): 214 | """Send a notification.""" 215 | msg = dumps([1, method, args, kwargs]) 216 | self.send(msg) 217 | 218 | def send_response(self, msgid, response): 219 | """Send a response.""" 220 | msg = dumps([2, msgid, response]) 221 | self.send(msg) 222 | 223 | def send_error(self, msgid, error): 224 | """Send an error.""" 225 | msg = dumps([3, msgid, error]) 226 | self.send(msg) 227 | 228 | def proxy(self): 229 | """Return a Deferred that will result in a proxy object in the future.""" 230 | d = Deferred(self.loop) 231 | self._proxy_deferreds.append(d) 232 | 233 | if self._proxy: 234 | d.callback(self._proxy) 235 | 236 | return d 237 | 238 | 239 | class PickleProtocolFactory(ProtocolFactory): 240 | def __init__(self, dispatch=Dispatch()): 241 | ProtocolFactory.__init__(self) 242 | self.dispatch = dispatch 243 | self.protocol = PickleProtocol 244 | self.protocols = [] 245 | 246 | def proxy(self, conn_number): 247 | """Return a proxy for a given connection number.""" 248 | return self.protocols[conn_number].proxy() 249 | 250 | def build(self, loop): 251 | p = self.protocol(loop, self, self.dispatch) 252 | self.protocols.append(p) 253 | return p 254 | 255 | def lost_connection(self, p): 256 | """Called by the rpc protocol whenever it loses a connection.""" 257 | self.protocols.remove(p) 258 | -------------------------------------------------------------------------------- /whizzer/defer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2010 Tom Burdick 3 | # Copyright (c) 2001-2010 4 | # Allen Short 5 | # Andy Gayton 6 | # Andrew Bennetts 7 | # Antoine Pitrou 8 | # Apple Computer, Inc. 9 | # Benjamin Bruheim 10 | # Bob Ippolito 11 | # Canonical Limited 12 | # Christopher Armstrong 13 | # David Reid 14 | # Donovan Preston 15 | # Eric Mangold 16 | # Eyal Lotem 17 | # Itamar Shtull-Trauring 18 | # James Knight 19 | # Jason A. Mobarak 20 | # Jean-Paul Calderone 21 | # Jessica McKellar 22 | # Jonathan Jacobs 23 | # Jonathan Lange 24 | # Jonathan D. Simms 25 | # Jürgen Hermann 26 | # Kevin Horn 27 | # Kevin Turner 28 | # Mary Gardiner 29 | # Matthew Lefkowitz 30 | # Massachusetts Institute of Technology 31 | # Moshe Zadka 32 | # Paul Swartz 33 | # Pavel Pergamenshchik 34 | # Ralph Meijer 35 | # Sean Riley 36 | # Software Freedom Conservancy 37 | # Travis B. Hartwell 38 | # Thijs Triemstra 39 | # Thomas Herve 40 | # Timothy Allen 41 | # 42 | # Permission is hereby granted, free of charge, to any person obtaining a copy 43 | # of this software and associated documentation files (the "Software"), to deal 44 | # in the Software without restriction, including without limitation the rights 45 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 46 | # copies of the Software, and to permit persons to whom the Software is 47 | # furnished to do so, subject to the following conditions: 48 | # 49 | # The above copyright notice and this permission notice shall be included in 50 | # all copies or substantial portions of the Software. 51 | # 52 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 53 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 54 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 55 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 56 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 57 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 58 | # THE SOFTWARE. 59 | 60 | import traceback 61 | import signal 62 | import sys 63 | import collections 64 | import logbook 65 | import pyev 66 | 67 | logger = logbook.Logger(__name__) 68 | 69 | """An implementation of Twisted Deferred class and helpers with some add ons 70 | that make Deferred also look similiar to pythonfutures.Future object. 71 | 72 | It attempts to make writting callback oriented or serial coroutine style 73 | relatively easy to write. 74 | 75 | """ 76 | 77 | 78 | class AlreadyCalledError(Exception): 79 | """Error raised if a Deferred has already had errback or callback called.""" 80 | 81 | 82 | class TimeoutError(Exception): 83 | """Error raised if a Deferred first(), every(), or final() call timeout.""" 84 | 85 | 86 | class CancelledError(Exception): 87 | """Error raised if a Deferred first(), every(), or final() call timeout.""" 88 | 89 | 90 | class LastException(object): 91 | """When this object dies if there is something set to self.exception a 92 | traceback is printed. 93 | 94 | The reason this object exists is that printing the stacktrace before a 95 | deferred is deleted is incorrect. An errback can be added at any time 96 | that would then trap the exception. So until the Deferred is collected 97 | the only sane thing to do is nothing. 98 | 99 | """ 100 | def __init__(self): 101 | """LastException. 102 | 103 | logger -- optional logger object, excepted to have an error method. 104 | 105 | """ 106 | self.exception = None 107 | self.tb_info = None 108 | 109 | def __del__(self): 110 | if self.exception: 111 | 112 | logger.error("Unhandled Exception " + str(self.exception) + " of type " + str(type(self.exception))) 113 | if self.tb_info: 114 | logger.error("Traceback: \n" + str(self.tb_info)) 115 | else: 116 | logger.error("Traceback: Unavailable") 117 | 118 | self.exception = None 119 | self.tb_info = None 120 | 121 | 122 | class Deferred(object): 123 | """Deferred result handling. 124 | 125 | Follows the Twisted Deferred interface with the exception of camel case. 126 | 127 | """ 128 | 129 | warnings = False 130 | 131 | def __init__(self, loop, cancelled_cb=None): 132 | """Deferred. 133 | 134 | loop -- a pyev loop instance 135 | cancelled_cb -- an optional callable given this deferred as its 136 | argument when cancel() is called. 137 | 138 | """ 139 | self.loop = loop 140 | self.called = False 141 | self._done = False 142 | self._cancelled = False 143 | self._cancelled_cb = cancelled_cb 144 | self._wait = False 145 | self._result = None 146 | self._exception = False 147 | self._tb_info = None 148 | self._callbacks = collections.deque() 149 | self._last_exception = LastException() 150 | 151 | def add_callbacks(self, callback, errback=None, callback_args=None, 152 | callback_kwargs=None, errback_args=None, 153 | errback_kwargs=None): 154 | """Add a callback and errback to the callback chain. 155 | 156 | If the previous callback succeeds the return value is passed as the 157 | first argument to the next callback in the chain. 158 | 159 | If the previous callback raises an exception the exception is passed as 160 | the first argument to the next errback in the chain. 161 | 162 | """ 163 | self._callbacks.appendleft((callback, errback, callback_args, 164 | callback_kwargs, errback_args, 165 | errback_kwargs)) 166 | 167 | if self.called: 168 | self._do_callbacks() 169 | 170 | return self 171 | 172 | def add_callback(self, callback, *callback_args, **callback_kwargs): 173 | """Add a callback without an associated errback.""" 174 | return self.add_callbacks(callback, callback_args=callback_args, 175 | callback_kwargs=callback_kwargs) 176 | 177 | def add_errback(self, errback, *errback_args, **errback_kwargs): 178 | """Add a errback without an associated callback.""" 179 | return self.add_callbacks(None, errback=errback, errback_args=errback_args, 180 | errback_kwargs=errback_kwargs) 181 | 182 | def callback(self, result): 183 | """Begin the callback chain with the first callback.""" 184 | self._start_callbacks(result, False) 185 | 186 | def errback(self, result): 187 | """Begin the callback chain with the first errback. 188 | 189 | result -- A BaseException derivative. 190 | 191 | """ 192 | 193 | assert(isinstance(result, BaseException)) 194 | self._start_callbacks(result, True) 195 | 196 | def result(self, timeout=None): 197 | """Return the last result of the callback chain or raise the last 198 | exception thrown and not caught by an errback. 199 | 200 | This will block until the result is available. 201 | 202 | If a timeout is given and the call times out raise a TimeoutError 203 | 204 | If SIGINT is caught while waiting raises CancelledError. 205 | 206 | If cancelled while waiting raises CancelledError 207 | 208 | This acts much like a pythonfutures.Future.result() call 209 | except the entire callback processing chain is performed first. 210 | 211 | """ 212 | 213 | self._do_wait(timeout) 214 | 215 | if self._exception: 216 | self._last_exception.exception = None 217 | self._last_exception.tb_info = None 218 | raise self._result 219 | else: 220 | return self._result 221 | 222 | def cancel(self): 223 | """Cancel the deferred.""" 224 | if self.called: 225 | raise AlreadyCalledError() 226 | self._cancel() 227 | 228 | def _cancel(self): 229 | if not self._cancelled: 230 | self._cancelled = True 231 | if self._cancelled_cb: 232 | self._cancelled_cb(self) 233 | 234 | def _clear_wait(self, watcher, events): 235 | """Clear the wait flag if an interrupt is caught.""" 236 | self._wait = False 237 | 238 | def _do_wait(self, timeout): 239 | """Wait for the deferred to be completed for a period of time 240 | 241 | Raises TimeoutError if the wait times out before the future is done. 242 | Raises CancelledError if the future is cancelled before the 243 | timeout is done. 244 | 245 | """ 246 | 247 | if self._cancelled: 248 | raise CancelledError() 249 | 250 | if not self._done: 251 | self._wait = True 252 | 253 | self._sigint = pyev.Signal(signal.SIGINT, self.loop, lambda watcher, events: self._cancel(), None) 254 | self._sigint.start() 255 | 256 | if timeout and timeout > 0.0: 257 | self._timer = pyev.Timer(timeout, 0.0, self.loop, 258 | self._clear_wait, None) 259 | self._timer.start() 260 | 261 | while self._wait and not self._done and not self._cancelled: 262 | self.loop.start(pyev.EVRUN_ONCE) 263 | 264 | if self._cancelled: 265 | raise CancelledError() 266 | elif not self._done: 267 | raise TimeoutError() 268 | 269 | def _start_callbacks(self, result, exception): 270 | """Perform the callback chain going back and forth between the callback 271 | and errback as needed. 272 | 273 | If an exception is raised and the entire chain is gone through without a valid 274 | errback then its simply logged. 275 | 276 | """ 277 | if self._cancelled: 278 | raise CancelledError() 279 | if self.called: 280 | raise AlreadyCalledError() 281 | 282 | self._result = result 283 | self._exception = exception 284 | if self._exception: 285 | self._tb_info = ''.join(traceback.format_tb(sys.exc_info()[2])) 286 | self.called = True 287 | self._do_callbacks() 288 | 289 | def _do_callbacks(self): 290 | """Perform the callbacks.""" 291 | self._done = False 292 | 293 | while self._callbacks and not self._cancelled: 294 | cb, eb, cb_args, cb_kwargs, eb_args, eb_kwargs = self._callbacks.pop() 295 | if cb and not self._exception: 296 | try: 297 | self._result = cb(self._result, *cb_args, **cb_kwargs) 298 | self._exception = False 299 | except Exception as e: 300 | #logger.exception('deferred got exception while doing callback') 301 | self._exception = True 302 | self._result = e 303 | self._tb_info = ''.join(traceback.format_tb(sys.exc_info()[2])) 304 | elif eb and self._exception: 305 | try: 306 | self._result = eb(self._result, *eb_args, **eb_kwargs) 307 | self._exception = False 308 | except Exception as e: 309 | #logger.exception('deferred got exception while doing callback') 310 | self._exception = True 311 | self._result = e 312 | self._tb_info = ''.join(traceback.format_tb(sys.exc_info()[2])) 313 | 314 | 315 | if self._exception: 316 | if Deferred.warnings: 317 | logger.warn('Unhandled Exception: ' + str(self._result)) 318 | self._last_exception.exception = self._result 319 | self._last_exception.tb_info = self._tb_info 320 | else: 321 | self._last_exception.exception = None 322 | self._last_exception.tb_info = None 323 | 324 | self._done = True 325 | 326 | 327 | class DeferredList(Deferred): 328 | def __init__(self, loop, cancelled_cb=None): 329 | Deferred.__init__(loop, cancelled_cb) 330 | self._deferreds = set() 331 | 332 | def add(self, deferred): 333 | self._deferreds.add(deferred) 334 | deferred.add_callback(self._markoff, deferred) 335 | 336 | def _markoff(self, deferred): 337 | self._deferreds.remove(deferred) 338 | --------------------------------------------------------------------------------