├── .gitignore ├── .travis.yml ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.rst ├── TODO ├── TOTEST ├── diesel ├── __init__.py ├── app.py ├── buffer.py ├── client.py ├── console.py ├── convoy │ ├── __init__.py │ ├── consensus │ │ ├── __init__.py │ │ ├── client.py │ │ └── server.py │ ├── convoy_env.proto │ └── messagenet.py ├── core.py ├── dnosetests.py ├── events.py ├── hub.py ├── interactive.py ├── logmod.py ├── pipeline.py ├── protocols │ ├── DNS.py │ ├── __init__.py │ ├── dreadlock.py │ ├── http │ │ ├── __init__.py │ │ ├── core.py │ │ └── pool.py │ ├── irc.py │ ├── mongodb.py │ ├── nitro.py │ ├── pg.py │ ├── redis.py │ ├── riak.proto │ ├── riak.py │ ├── websockets.py │ ├── wsgi.py │ └── zeromq.py ├── resolver.py ├── runtime.py ├── security.py ├── util │ ├── __init__.py │ ├── debugtools.py │ ├── event.py │ ├── lock.py │ ├── patches │ │ ├── __init__.py │ │ └── requests_lib.py │ ├── pool.py │ ├── process.py │ ├── queue.py │ ├── stats.py │ └── streams.py └── web.py ├── doc ├── Makefile ├── _build │ └── .empty ├── _static │ └── .empty ├── _templates │ └── .empty ├── api │ ├── diesel.protocols.http.rst │ ├── diesel.protocols.rst │ ├── diesel.rst │ ├── diesel.util.rst │ └── modules.rst ├── conf.py ├── debugging.rst ├── firststeps.rst ├── index.rst ├── intro.rst ├── patterns.rst └── testing.rst ├── examples ├── chat-redis.py ├── chat.py ├── child_test.py ├── clocker.py ├── combined.py ├── combined_tls.py ├── consolechat.py ├── convoy.py ├── crawler.py ├── dhttpd ├── dispatch.py ├── dreadlocks.py ├── echo.py ├── event.py ├── fanout.py ├── fire.py ├── forker.py ├── http.py ├── http_client.py ├── http_pool.py ├── keep_alive.py ├── kv.proto ├── newwait.py ├── nitro.py ├── nitro_echo_service.py ├── queue.py ├── queue_fairness_and_speed.py ├── redis_lock.py ├── redispub.py ├── resolve_names.py ├── santa.py ├── signals.py ├── sleep_server.py ├── snakeoil-cert.pem ├── snakeoil-key.pem ├── stdin.py ├── stdin_stream.py ├── synchronized.py ├── test_dnosetest.py ├── thread.py ├── thread_pool.py ├── timer.py ├── timer_bench.py ├── udp_echo.py ├── web.py ├── zeromq_echo_service.py ├── zeromq_first.py ├── zeromq_receiver.py ├── zeromq_sender.py └── zeromq_test.py ├── setup.py ├── tests ├── Makefile ├── integration │ ├── test_event_ordering.py │ ├── test_fanout.py │ ├── test_fire.py │ ├── test_fork.py │ ├── test_os_signals.py │ ├── test_queue.py │ ├── test_sleep.py │ └── test_wait.py ├── protocol │ ├── test_mongodb.py │ ├── test_redis.py │ └── test_zmq_service.py └── unit │ ├── test_buffer.py │ ├── test_pipeline.py │ └── test_waiters.py └── tox.ini /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.egg-info 3 | .*.swp 4 | *_palm.py 5 | build/ 6 | dist/ 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "2.7" 4 | - "2.6" 5 | # command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors 6 | install: python setup.py install 7 | # command to run tests, e.g. python setup.py test 8 | script: dnosetests tests/unit tests/integration 9 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Copyright (c) 2011 Boomplex, LLC, Bump Technologies, Inc, and 3 | authors and contributors 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions 7 | are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright 10 | notice, this list of conditions and the following disclaimer. 11 | 2. Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the distribution. 14 | 3. The names of the authors and copyright holders may not be used to 15 | endorse or promote products derived from this software without 16 | specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, 19 | INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY 20 | AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL 21 | THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 22 | EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 23 | PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 24 | OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 26 | OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 27 | ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | recursive-include examples *.py 3 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PYSETUP=python setup.py 2 | 3 | default: install test 4 | 5 | install: 6 | $(PYSETUP) install 7 | 8 | test: test-basic 9 | 10 | test-basic: 11 | $(MAKE) -C tests test-basic 12 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | 2 | Why Diesel? 3 | =========== 4 | 5 | You should write your next network application using diesel_. 6 | 7 | Thanks to Python_ the syntax is clean and the development pace is rapid. Thanks 8 | to non-blocking I/O it's fast and scalable. Thanks to greenlets_ there's 9 | unwind(to(callbacks(no))). Thanks to nose_ it's trivial to test. Thanks to 10 | Flask_ you don't need to write a new web framework using it. 11 | 12 | It provides a clean API for writing network clients and servers. TCP and UDP 13 | supported. It bundles battle-tested clients for HTTP, DNS, Redis, Riak and 14 | MongoDB. It makes writing network applications fun. 15 | 16 | Read the documentation, browse the API and join the community in #diesel on 17 | freenode. 18 | 19 | Prerequisites 20 | ============= 21 | 22 | You'll need the `python-dev` package as well as libffi-dev, or your 23 | platform's equivalents. 24 | 25 | Installation 26 | ============ 27 | 28 | Diesel is an active project. Your best bet to stay up with the latest at this 29 | point is to clone from github.:: 30 | 31 | git clone git://github.com/jamwt/diesel.git 32 | 33 | Once you have a clone, `cd` to the `diesel` directory and install it.:: 34 | 35 | pip install . 36 | 37 | or:: 38 | 39 | python setup.py install 40 | 41 | or:: 42 | 43 | python setup.py develop 44 | 45 | 46 | For More Information 47 | ==================== 48 | 49 | Documentation and more can be found on the diesel_ website. 50 | 51 | 52 | Python 3? 53 | ========= 54 | 55 | Not yet. Here are dependencies blocking the transition: 56 | 57 | .. image:: https://caniusepython3.com/project/diesel.svg 58 | :target: https://caniusepython3.com/project/diesel 59 | 60 | 61 | .. _Python: http://www.python.org/ 62 | .. _greenlets: http://readthedocs.org/docs/greenlet/en/latest/ 63 | .. _nose: http://readthedocs.org/docs/nose/en/latest/ 64 | .. _Flask: http://flask.pocoo.org/ 65 | .. _diesel: http://diesel.io/ 66 | -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | 3.0 2 | * Convoy: 3 | - broadcast 4 | - test failure modes 5 | - make incoming messages concurrent! back with new util. thread pools 6 | - step up/step down transitions on Role objects 7 | - i_own call-in to the convoy 8 | - write up some use cases and scenarios 9 | - distributed key/value system? 10 | - more logging and cleaner logging for consensus server routing table 11 | movements 12 | * diesel + quickstart-based launcher (glowplug?) consolidation 13 | (deprecate the Application object and make everything 14 | fork_ etc based including keep-alive stuff) 15 | * get dowski's queue back? 16 | * merge in timdoug's UDP support from 2.x 17 | * native resolver using UDP code 18 | * finish tests 19 | * docs 20 | * website 21 | 22 | 3.X 23 | * community features (nice process for protocol extension development 24 | and distribution?) 25 | * postgres bindings 26 | * memcache bindings 27 | * operation now in progress broken on connect 28 | 29 | 4.0 30 | * Modify palm to be ctypes-based.. becauuuse.. 31 | * Integrate diesel with PyPy once it has first-class 32 | JIT-enabled greenlets 33 | -------------------------------------------------------------------------------- /TOTEST: -------------------------------------------------------------------------------- 1 | * Basic hub functions: 2 | * Write-ready, ready ready 3 | * EAGAIN behavior with mocks 4 | * Socket close 5 | * Timers and callbacks 6 | 7 | * Client: 8 | 9 | * Util 10 | * Pool, client failure, release vs. reclaim 11 | * Queue, callback patterns, iterator, locks etc 12 | 13 | * Core 14 | * bytes(), until() behavior and combinations 15 | * @call into 16 | * defer to thread behavior 17 | * connections vs. loops 18 | * loops not using connection-specific things 19 | * run tests over tls too--both client and server 20 | -------------------------------------------------------------------------------- /diesel/__init__.py: -------------------------------------------------------------------------------- 1 | # vim:ts=4:sw=4:expandtab 2 | from logmod import log, levels as loglevels, set_log_level 3 | import events 4 | from core import sleep, Loop, wait, fire, thread, until, Connection, UDPSocket, ConnectionClosed, ClientConnectionClosed, signal 5 | from core import until_eol, send, receive, call, first, fork, fork_child, label, fork_from_thread 6 | from core import ParentDiedException, ClientConnectionError, TerminateLoop, datagram 7 | from app import Application, Service, UDPService, quickstart, quickstop, Thunk 8 | from client import Client, UDPClient 9 | from resolver import resolve_dns_name, DNSResolutionError 10 | from runtime import is_running 11 | from hub import ExistingSignalHandler 12 | -------------------------------------------------------------------------------- /diesel/buffer.py: -------------------------------------------------------------------------------- 1 | # vim:ts=4:sw=4:expandtab 2 | class BufAny(object): 3 | pass 4 | 5 | class Buffer(object): 6 | '''An input buffer. 7 | 8 | Allows socket data to be read immediately and buffered, but 9 | fine-grained byte-counting or sentinel-searching to be 10 | specified by consumers of incoming data. 11 | ''' 12 | def __init__(self): 13 | self._atinbuf = [] 14 | self._atterm = None 15 | self._atmark = 0 16 | 17 | def set_term(self, term): 18 | '''Set the current sentinel. 19 | 20 | `term` is either an int, for a byte count, or 21 | a string, for a sequence of characters that needs 22 | to occur in the byte stream. 23 | ''' 24 | self._atterm = term 25 | 26 | def feed(self, data): 27 | '''Feed some data into the buffer. 28 | 29 | The buffer is appended, and the check() is run in case 30 | this append causes the sentinel to be satisfied. 31 | ''' 32 | self._atinbuf.append(data) 33 | self._atmark += len(data) 34 | return self.check() 35 | 36 | def clear_term(self): 37 | self._atterm = None 38 | 39 | def check(self): 40 | '''Look for the next message in the data stream based on 41 | the current sentinel. 42 | ''' 43 | ind = None 44 | all = None 45 | if self._atterm is BufAny: 46 | if self.has_data: 47 | return self.pop() 48 | return None 49 | if type(self._atterm) is int: 50 | if self._atmark >= self._atterm: 51 | ind = self._atterm 52 | elif self._atterm is None: 53 | return None 54 | else: 55 | all = ''.join(self._atinbuf) 56 | res = all.find(self._atterm) 57 | if res != -1: 58 | ind = res + len(self._atterm) 59 | if ind is None: 60 | return None 61 | self._atterm = None # this terminator was used 62 | if all is None: 63 | all = ''.join(self._atinbuf) 64 | use = all[:ind] 65 | new_all = all[ind:] 66 | self._atinbuf = [new_all] 67 | self._atmark = len(new_all) 68 | 69 | return use 70 | 71 | def pop(self): 72 | b = ''.join(self._atinbuf) 73 | self._atinbuf = [] 74 | self._atmark = 0 75 | return b 76 | 77 | @property 78 | def has_data(self): 79 | return bool(self._atinbuf) 80 | -------------------------------------------------------------------------------- /diesel/client.py: -------------------------------------------------------------------------------- 1 | # vim:ts=4:sw=4:expandtab 2 | import errno 3 | import socket 4 | 5 | class Client(object): 6 | '''An agent that connects to an external host and provides an API to 7 | return data based on a protocol across that host. 8 | ''' 9 | def __init__(self, addr, port, ssl_ctx=None, timeout=None, source_ip=None): 10 | self.ssl_ctx = ssl_ctx 11 | self.connected = False 12 | self.conn = None 13 | self.addr = addr 14 | self.port = port 15 | 16 | ip = self._resolve(self.addr) 17 | self._setup_socket(ip, timeout, source_ip) 18 | 19 | def _resolve(self, addr): 20 | from resolver import resolve_dns_name 21 | return resolve_dns_name(addr) 22 | 23 | def _setup_socket(self, ip, timeout, source_ip=None): 24 | 25 | from core import _private_connect 26 | remote_addr = (ip, self.port) 27 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 28 | sock.setblocking(0) 29 | 30 | if source_ip: 31 | sock.bind((source_ip, 0)) 32 | 33 | try: 34 | sock.connect(remote_addr) 35 | except socket.error, e: 36 | if e.args[0] == errno.EINPROGRESS: 37 | _private_connect(self, ip, sock, self.addr, self.port, timeout=timeout) 38 | else: 39 | raise 40 | 41 | def on_connect(self): 42 | pass 43 | 44 | def __enter__(self): 45 | return self 46 | 47 | def __exit__(self, *args, **kw): 48 | self.close() 49 | 50 | def close(self): 51 | '''Close the socket to the remote host. 52 | ''' 53 | if not self.is_closed: 54 | self.conn.close() 55 | self.conn = None 56 | self.connected = True 57 | 58 | @property 59 | def is_closed(self): 60 | return not self.conn or self.conn.closed 61 | 62 | class UDPClient(Client): 63 | def __init__(self, addr, port, source_ip=None): 64 | super(UDPClient, self).__init__(addr, port, source_ip = source_ip) 65 | 66 | def _setup_socket(self, ip, timeout, source_ip=None): 67 | from core import UDPSocket 68 | sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 69 | sock.setblocking(0) 70 | 71 | if source_ip: 72 | sock.bind((source_ip, 0)) 73 | 74 | self.conn = UDPSocket(self, sock, ip, self.port) 75 | self.connected = True 76 | 77 | def _resolve(self, addr): 78 | return addr 79 | 80 | class remote_addr(object): 81 | def __get__(self, inst, other): 82 | return (inst.addr, inst.port) 83 | 84 | def __set__(self, inst, value): 85 | inst.addr, inst.port = value 86 | remote_addr = remote_addr() 87 | -------------------------------------------------------------------------------- /diesel/console.py: -------------------------------------------------------------------------------- 1 | """A remote console into running diesel applications. 2 | 3 | With the functions and classes in this module you can open remote Python 4 | console sessions into your diesel applications. Technically, they aren't 5 | remote because it is hardcoded to run over localhost. But they are remote 6 | from a process point of view. 7 | 8 | An application that wishes to provide a remote console only needs to import 9 | and call the `install_console_signal_handler` function. That sets a handler 10 | for the SIGTRAP signal that attempts to make a connection to a certain port 11 | on localhost. 12 | 13 | Running there should be this module's `main` function. It sends the SIGTRAP 14 | to a specified PID and then waits for a connection. 15 | 16 | This inversion of the typical client/server roles is to allow for easily 17 | getting a console into one of many processes running on a host without having 18 | to configure a persistent remote console port or service for each one. 19 | 20 | The code also handles redirecting stdout for the console so that the results of 21 | `print` statements and the like are sent to the connected console and not the 22 | local stdout of the process. All other output to stdout will be directed to the 23 | process's normal stdout. 24 | 25 | """ 26 | import code 27 | import optparse 28 | import os 29 | import readline # for history feature side-effect 30 | import signal 31 | import struct 32 | import sys 33 | 34 | from cStringIO import StringIO 35 | 36 | 37 | import diesel 38 | 39 | from diesel.util import debugtools 40 | 41 | 42 | port = 4299 43 | 44 | def install_console_signal_handler(): 45 | """Call this function to provide a remote console in your app.""" 46 | def connect_to_user_console(sig, frame): 47 | diesel.fork_from_thread(application_console_endpoint) 48 | signal.signal(signal.SIGTRAP, connect_to_user_console) 49 | 50 | class LocalConsole(code.InteractiveConsole): 51 | """A modified Python interpreter UI that talks to a remote console.""" 52 | def runsource(self, source, filename=None): 53 | self.current_source = source.encode('utf-8') 54 | return code.InteractiveConsole.runsource(self, source, filename) 55 | 56 | def runcode(self, ignored_codeobj): 57 | if self.current_source: 58 | sz = len(self.current_source) 59 | header = struct.pack('>Q', sz) 60 | diesel.send("%s%s" % (header, self.current_source)) 61 | self.current_source = None 62 | header = diesel.receive(8) 63 | (sz,) = struct.unpack('>Q', header) 64 | if sz: 65 | data = diesel.receive(sz) 66 | print data.rstrip() 67 | 68 | def console_for(pid): 69 | """Sends a SIGTRAP to the pid and returns a console UI handler. 70 | 71 | The return value is meant to be passed to a diesel.Service. 72 | 73 | """ 74 | os.kill(pid, signal.SIGTRAP) 75 | banner = "Remote console PID=%d" % pid 76 | def interactive(addr): 77 | remote_console = LocalConsole() 78 | remote_console.interact(banner) 79 | diesel.quickstop() 80 | return interactive 81 | 82 | 83 | class RemoteConsoleService(diesel.Client): 84 | """Runs the backend console.""" 85 | def __init__(self, *args, **kw): 86 | self.interpreter = BackendInterpreter({ 87 | 'diesel':diesel, 88 | 'debugtools':debugtools, 89 | }) 90 | super(RemoteConsoleService, self).__init__(*args, **kw) 91 | 92 | @diesel.call 93 | def handle_command(self): 94 | header = diesel.receive(8) 95 | (sz,) = struct.unpack('>Q', header) 96 | data = diesel.receive(sz) 97 | stdout_patch = StdoutDispatcher() 98 | with stdout_patch: 99 | self.interpreter.runsource(data) 100 | output = stdout_patch.contents 101 | outsz = len(output) 102 | outheader = struct.pack('>Q', outsz) 103 | diesel.send("%s%s" % (outheader, output)) 104 | 105 | class BackendInterpreter(code.InteractiveInterpreter): 106 | def write(self, data): 107 | sys.stdout.write(data) 108 | 109 | def application_console_endpoint(): 110 | """Connects to the console UI and runs until disconnected.""" 111 | diesel.sleep(1) 112 | try: 113 | session = RemoteConsoleService('localhost', port) 114 | except diesel.ClientConnectionError: 115 | diesel.log.error('Failed to connect to local console') 116 | else: 117 | diesel.log.warning('Connected to local console') 118 | with session: 119 | while True: 120 | try: 121 | session.handle_command() 122 | except diesel.ClientConnectionClosed: 123 | diesel.log.warning('Disconnected from local console') 124 | break 125 | 126 | class StdoutDispatcher(object): 127 | """Dispatches calls to stdout to fake or real file-like objects. 128 | 129 | The creator of an instance will receive the fake file-like object and 130 | all others will receive the original stdout instance. 131 | 132 | """ 133 | def __init__(self): 134 | self.owning_loop = diesel.core.current_loop.id 135 | self._orig_stdout = sys.stdout 136 | self._fake_stdout = StringIO() 137 | 138 | def __getattr__(self, name): 139 | if diesel.core.current_loop.id == self.owning_loop: 140 | return getattr(self._fake_stdout, name) 141 | else: 142 | return getattr(self._orig_stdout, name) 143 | 144 | def __enter__(self): 145 | sys.stdout = self 146 | return self 147 | 148 | def __exit__(self, *args): 149 | sys.stdout = self._orig_stdout 150 | 151 | @property 152 | def contents(self): 153 | return self._fake_stdout.getvalue() 154 | 155 | def main(): 156 | parser = optparse.OptionParser("Usage: %prog PID") 157 | parser.add_option( 158 | '-p', '--port', default=port, type="int", 159 | help="The port to listen on for console connections", 160 | ) 161 | options, args = parser.parse_args() 162 | if not args: 163 | parser.print_usage() 164 | raise SystemExit(1) 165 | if args[0] == 'dummy': 166 | print "PID", os.getpid() 167 | def wait_for_signal(): 168 | log = diesel.log.name('dummy') 169 | log.min_level = diesel.loglevels.INFO 170 | install_console_signal_handler() 171 | while True: 172 | log.info("sleeping") 173 | diesel.sleep(5) 174 | diesel.quickstart(wait_for_signal) 175 | else: 176 | pid = int(args[0]) 177 | svc = diesel.Service(console_for(pid), options.port) 178 | diesel.quickstart(svc) 179 | 180 | if __name__ == '__main__': 181 | main() 182 | -------------------------------------------------------------------------------- /diesel/convoy/consensus/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dieseldev/diesel/8d48371fce0b79d6631053594bce06e4b9628499/diesel/convoy/consensus/__init__.py -------------------------------------------------------------------------------- /diesel/convoy/consensus/client.py: -------------------------------------------------------------------------------- 1 | from uuid import uuid4 2 | from itertools import chain 3 | from diesel import Client, fork, call, send, until_eol 4 | from diesel.util.queue import Queue 5 | import random 6 | 7 | nodeid = str(uuid4()) 8 | 9 | class ConvoyGetRequest(object): 10 | def __init__(self, key): 11 | self.key = key 12 | 13 | class ConvoySetRequest(object): 14 | def __init__(self, key, value, cap, timeout, lock): 15 | self.key = key 16 | self.value = value 17 | self.cap = cap 18 | self.timeout = timeout 19 | self.lock = lock 20 | 21 | class ConvoyWaitRequest(object): 22 | def __init__(self, timeout, clocks): 23 | self.timeout = timeout 24 | self.clocks = clocks 25 | 26 | class ConvoyAliveRequest(object): 27 | pass 28 | 29 | class ConvoyNameService(object): 30 | def __init__(self, servers): 31 | self.servers = servers 32 | self.request_queue = Queue() 33 | self.pool_locks = {} 34 | 35 | def __call__(self): 36 | while True: 37 | server = random.choice(self.servers) 38 | with ConvoyConsensusClient(*server) as client: 39 | while True: 40 | req, rq = self.request_queue.get() 41 | if type(req) is ConvoyGetRequest: 42 | resp = client.get(req.key) 43 | elif type(req) is ConvoySetRequest: 44 | resp = client.add_to_set(req.key, req.value, req.cap, req.timeout, req.lock) 45 | elif type(req) is ConvoyWaitRequest: 46 | resp = client.wait(req.timeout, req.clocks) 47 | elif type(req) is ConvoyAliveRequest: 48 | resp = client.keep_alive() 49 | else: 50 | assert 0 51 | rq.put(resp) 52 | 53 | def lookup(self, key): 54 | rq = Queue() 55 | self.request_queue.put((ConvoyGetRequest(key), rq)) 56 | return rq.get() 57 | 58 | def clear(self, key): 59 | rq = Queue() 60 | self.request_queue.put((ConvoySetRequest(key, None, 0, 5, 0), rq)) 61 | return rq.get() 62 | 63 | def set(self, key, value): 64 | rq = Queue() 65 | self.request_queue.put((ConvoySetRequest(key, value, 0, 5, 0), rq)) 66 | return rq.get() 67 | 68 | def add(self, key, value, cap, to=0): 69 | rq = Queue() 70 | self.request_queue.put((ConvoySetRequest(key, value, cap, to, 1), rq)) 71 | return rq.get() 72 | 73 | def wait(self, timeout, clocks): 74 | rq = Queue() 75 | self.request_queue.put((ConvoyWaitRequest(timeout, clocks), rq)) 76 | return rq.get() 77 | 78 | def alive(self): 79 | rq = Queue() 80 | self.request_queue.put((ConvoyAliveRequest(), rq)) 81 | return rq.get() 82 | 83 | class ConsensusSet(object): 84 | def __init__(self, l, clock=None): 85 | self.members = set(l) 86 | self.clock = clock 87 | 88 | def __repr__(self): 89 | return "consensus-set <%s @ %s>" % ( 90 | ','.join(self.members), self.clock) 91 | 92 | class ConvoySetFailed(object): 93 | def __init__(self, set=None): 94 | self.set = set 95 | 96 | class ConvoySetTimeout(ConvoySetFailed): 97 | pass 98 | 99 | class ConvoyWaitTimeout(object): 100 | pass 101 | 102 | class ConvoyWaitDone(object): 103 | def __init__(self, key): 104 | self.key = key 105 | 106 | class ConvoyConsensusClient(Client): 107 | '''low-level client; use the cluster abstraction''' 108 | @call 109 | def on_connect(self): 110 | send("CLIENT\r\n") 111 | send("HI %s\r\n" % nodeid) 112 | assert until_eol().strip().upper() == "HI-HOLA" 113 | 114 | @call 115 | def wait(self, timeout, clocks): 116 | parts = chain([timeout], 117 | *clocks.iteritems()) 118 | rest = ' '.join(map(str, parts)) 119 | send("BLOCKON " + rest + "\r\n") 120 | 121 | response = until_eol().strip() 122 | parts = response.split() 123 | result, rest = parts[0].upper(), parts[1:] 124 | if result == "BLOCKON-DONE": 125 | return ConvoyWaitDone(rest[0]) 126 | assert result == "BLOCKON-TIMEOUT" 127 | return ConvoyWaitTimeout() 128 | 129 | @call 130 | def get(self, key): 131 | send("GET %s\r\n" % key) 132 | response = until_eol().strip() 133 | parts = response.split() 134 | result, rest = parts[0].upper(), parts[1:] 135 | if result == "GET-MISSING": 136 | return ConsensusSet([]) 137 | elif result == "GET-NULL": 138 | clock = rest[0] 139 | return ConsensusSet([], clock) 140 | else: 141 | assert result == "GET-VALUE" 142 | clock = rest[0] 143 | values = rest[1:] 144 | return ConsensusSet(values, clock) 145 | 146 | @call 147 | def add_to_set(self, key, value, cap, timeout, lock): 148 | send("SET %s %s %s %s %s\r\n" % ( 149 | key, value or '_', cap, timeout, int(lock))) 150 | response = until_eol().strip() 151 | 152 | parts = response.split() 153 | result, rest = parts[0].upper(), parts[1:] 154 | 155 | if result == 'SET-TIMEOUT': 156 | if timeout == 0: 157 | cls = ConvoySetFailed 158 | else: 159 | cls = ConvoySetTimeout 160 | if rest: 161 | clock = rest[0] 162 | values = rest[1:] 163 | return cls(ConsensusSet(values, clock)) 164 | else: 165 | return cls() 166 | else: 167 | assert result == "SET-OKAY" 168 | clock = rest[0] 169 | values = rest[1:] 170 | return ConsensusSet(values, clock) 171 | 172 | @call 173 | def keep_alive(self): 174 | send("KEEPALIVE\r\n") 175 | assert until_eol().strip().upper() == "KEEPALIVE-OKAY" 176 | 177 | cargo = ConvoyNameService([('localhost', 1111), ('localhost', 1112), ('localhost', 1113)]) 178 | 179 | if __name__ == '__main__': 180 | def run(): 181 | print cargo.clear("foo") 182 | print cargo.set("foo", "bar") 183 | print cargo.lookup("foo") 184 | quickstop() 185 | from diesel import quickstart, quickstop 186 | quickstart(cargo, run) 187 | 188 | -------------------------------------------------------------------------------- /diesel/convoy/convoy_env.proto: -------------------------------------------------------------------------------- 1 | message MessageEnvelope { 2 | required bytes body = 1; 3 | required string type = 2; 4 | required string req_id = 3; 5 | required string node_id = 4; 6 | required bool wants_result = 5; 7 | } 8 | 9 | message MessageResponse { 10 | required string in_response_to = 1; 11 | 12 | enum MessageDeliveryStatus { 13 | REFUSED = 1; 14 | ACCEPTED = 2; 15 | FINISHED = 3; 16 | } 17 | 18 | enum MessageResult { 19 | EXCEPTION = 1; 20 | RESULT = 2; 21 | NULL = 3; 22 | } 23 | 24 | optional MessageResult result = 2; 25 | 26 | optional string error_message = 3; 27 | 28 | repeated MessageEnvelope responses = 4; 29 | 30 | required MessageDeliveryStatus delivered = 5; 31 | } 32 | -------------------------------------------------------------------------------- /diesel/convoy/messagenet.py: -------------------------------------------------------------------------------- 1 | import os 2 | from struct import pack, unpack 3 | 4 | from .convoy_env_palm import MessageResponse, MessageEnvelope 5 | from diesel import Client, call, send, receive, Service 6 | import traceback 7 | 8 | MESSAGE_OUT = 1 9 | MESSAGE_RES = 2 10 | 11 | class ConvoyId(object): 12 | def __init__(self): 13 | id = None 14 | me = ConvoyId() 15 | 16 | def host_loop(host, q): 17 | h, p = host.split('/') 18 | p = int(p) 19 | client = None 20 | while True: 21 | env, typ, cb = q.get() 22 | try: 23 | if not client: 24 | client = MessageClient(h, p) 25 | client.send_message(env, typ) 26 | except: 27 | traceback.print_exc() 28 | client.close() 29 | client = None 30 | if cb: 31 | cb() 32 | 33 | class MessageClient(Client): 34 | @call 35 | def send_message(self, env, typ): 36 | out = env.dumps() 37 | send(pack('=II', typ, len(out))) 38 | send(out) 39 | 40 | def handle_conn(*args): 41 | from diesel.convoy import convoy 42 | while True: 43 | head = receive(8) 44 | typ, size = unpack('=II', head) 45 | body = receive(size) 46 | if typ == MESSAGE_OUT: 47 | env = MessageEnvelope(body) 48 | convoy.local_dispatch(env) 49 | else: 50 | resp = MessageResponse(body) 51 | convoy.local_response(resp) 52 | 53 | class ConvoyService(Service): 54 | def __init__(self): 55 | Service.__init__(self, handle_conn, 0) 56 | 57 | def bind_and_listen(self): 58 | Service.bind_and_listen(self) 59 | 60 | me.id = '%s/%s' % (os.uname()[1], self.port) 61 | -------------------------------------------------------------------------------- /diesel/dnosetests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Run nosetests in the diesel event loop. 3 | 4 | You can pass the same command-line arguments that you can pass to the 5 | `nosetests` command to this script and it will execute the tests in the 6 | diesel event loop. This is a great way to test interactions between various 7 | diesel green threads and network-based applications built with diesel. 8 | 9 | """ 10 | import diesel 11 | import nose 12 | 13 | from diesel.logmod import levels, set_log_level 14 | 15 | def main(): 16 | set_log_level(levels.CRITICAL) 17 | diesel.quickstart(nose.main) 18 | 19 | if __name__ == '__main__': 20 | main() 21 | -------------------------------------------------------------------------------- /diesel/events.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | 3 | class StopWaitDispatch(Exception): pass 4 | class StaticValue(object): 5 | def __init__(self, value): 6 | self.value = value 7 | 8 | class EarlyValue(object): 9 | def __init__(self, val): 10 | self.val = val 11 | 12 | class Waiter(object): 13 | @property 14 | def wait_id(self): 15 | return str(hash(self)) 16 | 17 | def process_fire(self, given): 18 | return StaticValue(given) 19 | 20 | def ready_early(self): 21 | return False 22 | 23 | class StringWaiter(str, Waiter): 24 | @property 25 | def wait_id(self): 26 | return str(self) 27 | 28 | 29 | class WaitPool(object): 30 | '''A structure that manages all `wait`ers, makes sure fired events 31 | get to the right places. 32 | ''' 33 | def __init__(self): 34 | self.waits = defaultdict(set) 35 | self.loop_refs = defaultdict(set) 36 | 37 | def wait(self, who, what): 38 | if isinstance(what, basestring): 39 | what = StringWaiter(what) 40 | 41 | if what.ready_early(): 42 | return EarlyValue(what.process_fire(None)) 43 | 44 | self.waits[what.wait_id].add(who) 45 | self.loop_refs[who].add(what) 46 | return what.wait_id 47 | 48 | def fire(self, what, value): 49 | if isinstance(what, basestring): 50 | what = StringWaiter(what) 51 | 52 | static = False 53 | for handler in self.waits[what.wait_id]: 54 | if handler.fire_due: 55 | continue 56 | if not static: 57 | try: 58 | value = what.process_fire(value) 59 | except StopWaitDispatch: 60 | break 61 | if type(value) == StaticValue: 62 | static = True 63 | value = value.value 64 | handler.fire_in(what.wait_id, value) 65 | 66 | def clear(self, who): 67 | for what in self.loop_refs[who]: 68 | self.waits[what.wait_id].remove(who) 69 | if not self.waits[what.wait_id]: 70 | del self.waits[what.wait_id] 71 | del self.loop_refs[who] 72 | -------------------------------------------------------------------------------- /diesel/interactive.py: -------------------------------------------------------------------------------- 1 | """An interactive interpreter inside of a diesel event loop. 2 | 3 | It's useful for importing and interacting with code that expects to run 4 | inside of a diesel event loop. It works especially well for interactive 5 | sessions with diesel's various network protocol clients. 6 | 7 | Supports both the standard Python interactive interpreter and IPython (if 8 | installed). 9 | 10 | """ 11 | import code 12 | import sys 13 | sys.path.insert(0, '.') 14 | 15 | import diesel 16 | from diesel.util.streams import create_line_input_stream 17 | 18 | try: 19 | from IPython.Shell import IPShell 20 | IPYTHON_AVAILABLE = True 21 | except ImportError: 22 | try: 23 | # Support changes made in iPython 0.11 24 | from IPython.frontend.terminal.ipapp import TerminalInteractiveShell as IPShell 25 | IPYTHON_AVAILABLE = True 26 | except ImportError: 27 | IPYTHON_AVAILABLE = False 28 | 29 | 30 | # Library Functions: 31 | # ================== 32 | 33 | def interact_python(): 34 | """Runs an interactive interpreter; halts the diesel app when finished.""" 35 | globals_ = globals() 36 | env = { 37 | '__builtins__':globals_['__builtins__'], 38 | '__doc__':globals_['__doc__'], 39 | '__name__':globals_['__name__'], 40 | 'diesel':diesel, 41 | } 42 | inp = create_line_input_stream(sys.stdin) 43 | 44 | def diesel_input(prompt): 45 | sys.stdout.write(prompt) 46 | sys.stdout.flush() 47 | return inp.get().rstrip('\n') 48 | 49 | code.interact(None, diesel_input, env) 50 | diesel.quickstop() 51 | 52 | def interact_ipython(): 53 | """Starts an IPython instance; halts the diesel app when finished.""" 54 | IPShell(user_ns={'diesel':diesel}).mainloop() 55 | diesel.quickstop() 56 | 57 | # Interpreter entry points: 58 | # ========================= 59 | 60 | def python(): 61 | diesel.quickstart(interact_python) 62 | 63 | def ipython(): 64 | if not IPYTHON_AVAILABLE: 65 | print >> sys.stderr, "IPython not found." 66 | raise SystemExit(1) 67 | diesel.quickstart(interact_ipython) 68 | 69 | -------------------------------------------------------------------------------- /diesel/logmod.py: -------------------------------------------------------------------------------- 1 | # vim:ts=4:sw=4:expandtab 2 | '''A simple logging module that supports various verbosity 3 | levels and component-specific subloggers. 4 | ''' 5 | 6 | import sys 7 | import time 8 | from twiggy import log as olog, levels, outputs, formats, emitters 9 | try: 10 | from twiggy import add_emitters 11 | except ImportError: 12 | from twiggy import addEmitters as add_emitters 13 | from functools import partial 14 | 15 | diesel_format = formats.line_format 16 | diesel_format.traceback_prefix = '\n' 17 | diesel_format.conversion = formats.ConversionTable() 18 | diesel_format.conversion.add("time", partial(time.strftime, "%Y/%m/%d %H:%M:%S"), "[{1}]".format) 19 | diesel_format.conversion.add("name", str, "{{{1}}}".format) 20 | diesel_format.conversion.add("level", str, "{1}".format) 21 | diesel_format.conversion.aggregate = " ".join 22 | diesel_format.conversion.genericValue = str 23 | diesel_format.conversion.genericItem = lambda _1, _2: "%s=%s" % (_1, _2) 24 | 25 | diesel_output = outputs.StreamOutput(diesel_format) 26 | 27 | def set_log_level(level=levels.INFO): 28 | emitters.clear() 29 | 30 | add_emitters( 31 | ('*', level, None, diesel_output) 32 | ) 33 | 34 | log = olog.name("diesel") 35 | 36 | set_log_level() 37 | -------------------------------------------------------------------------------- /diesel/pipeline.py: -------------------------------------------------------------------------------- 1 | # vim:ts=4:sw=4:expandtab 2 | '''An outgoing pipeline that can handle 3 | strings or files. 4 | ''' 5 | try: 6 | import cStringIO 7 | except ImportError: 8 | raise ImportError, "cStringIO is required" 9 | 10 | from bisect import bisect_right 11 | 12 | _obj_SIO = cStringIO.StringIO 13 | _type_SIO = cStringIO.OutputType 14 | def make_SIO(d): 15 | t = _obj_SIO() 16 | t.write(d) 17 | t.seek(0) 18 | return t 19 | 20 | def get_file_length(f): 21 | m = f.tell() 22 | f.seek(0, 2) 23 | r = f.tell() 24 | f.seek(m) 25 | return r 26 | 27 | class PipelineCloseRequest(Exception): pass 28 | class PipelineClosed(Exception): pass 29 | 30 | class PipelineItem(object): 31 | def __init__(self, d): 32 | if type(d) is str: 33 | self.f = make_SIO(d) 34 | self.length = len(d) 35 | self.is_sio = True 36 | self.f.seek(0, 2) 37 | elif hasattr(d, 'seek'): 38 | self.f = d 39 | self.length = get_file_length(d) 40 | self.is_sio = False 41 | else: 42 | raise ValueError("argument to add() must be either a str or a file-like object") 43 | self.read = self.f.read 44 | 45 | def merge(self, s): 46 | self.f.write(s) 47 | self.length += len(s) 48 | 49 | def reset(self): 50 | if self.is_sio: 51 | self.is_sio = False 52 | self.f.seek(0, 0) 53 | 54 | @property 55 | def done(self): 56 | return self.f.tell() == self.length 57 | 58 | def __cmp__(self, other): 59 | if other is PipelineStandIn: 60 | return -1 61 | return cmp(self, other) 62 | 63 | class PipelineStandIn(object): pass 64 | 65 | class Pipeline(object): 66 | '''A pipeline that supports appending strings or 67 | files and can read() transparently across object 68 | boundaries in the outgoing buffer. 69 | ''' 70 | def __init__(self): 71 | self.line = [] 72 | self.current = None 73 | self.want_close = False 74 | 75 | def add(self, d, priority=5): 76 | '''Add object `d` to the pipeline. 77 | ''' 78 | if self.want_close: 79 | raise PipelineClosed 80 | 81 | priority *= -1 82 | 83 | dummy = (priority, PipelineStandIn) 84 | ind = bisect_right(self.line, dummy) 85 | if ind > 0 and type(d) is str and self.line[ind - 1][-1].is_sio: 86 | a_pri, adjacent = self.line[ind - 1] 87 | if adjacent.is_sio and a_pri == priority: 88 | adjacent.merge(d) 89 | else: 90 | self.line.insert(ind, (priority, PipelineItem(d))) 91 | else: 92 | self.line.insert(ind, (priority, PipelineItem(d))) 93 | 94 | def close_request(self): 95 | '''Add a close request to the outgoing pipeline. 96 | 97 | No more data will be allowed in the pipeline, and, when 98 | it is emptied, PipelineCloseRequest will be raised. 99 | ''' 100 | self.want_close = True 101 | 102 | def read(self, amt): 103 | '''Read up to `amt` bytes off the pipeline. 104 | 105 | May raise PipelineCloseRequest if the pipeline is 106 | empty and the connected stream should be closed. 107 | ''' 108 | if not self.current and not self.line: 109 | if self.want_close: 110 | raise PipelineCloseRequest() 111 | return '' 112 | 113 | if not self.current: 114 | _, self.current = self.line.pop(0) 115 | self.current.reset() 116 | 117 | out = '' 118 | while len(out) < amt: 119 | try: 120 | data = self.current.read(amt - len(out)) 121 | except ValueError: 122 | data = '' 123 | if data == '': 124 | if not self.line: 125 | self.current = None 126 | break 127 | _, self.current = self.line.pop(0) 128 | self.current.reset() 129 | else: 130 | out += data 131 | 132 | # eagerly evict and EOF that's been read _just_ short of 133 | # the EOF '' read() call.. so that we know we're empty, 134 | # and we don't bother with useless iterations 135 | if self.current and self.current.done: 136 | self.current = None 137 | 138 | return out 139 | 140 | def backup(self, d): 141 | '''Pop object d back onto the front the pipeline. 142 | 143 | Used in cases where not all data is sent() on the socket, 144 | for example--the remainder will be placed back in the pipeline. 145 | ''' 146 | cur = self.current 147 | self.current = PipelineItem(d) 148 | self.current.reset() 149 | if cur: 150 | self.line.insert(0, (-1000000, cur)) 151 | 152 | @property 153 | def empty(self): 154 | '''Is the pipeline empty? 155 | 156 | A close request is "data" that needs to be consumed, 157 | too. 158 | ''' 159 | return self.want_close == False and not self.line and not self.current 160 | -------------------------------------------------------------------------------- /diesel/protocols/DNS.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import time 4 | from collections import deque 5 | 6 | from diesel import UDPClient, call, send, first, datagram 7 | 8 | from dns.message import make_query, from_wire 9 | from dns.rdatatype import A 10 | from dns.resolver import Resolver as ResolvConf 11 | 12 | 13 | class NotFound(Exception): 14 | pass 15 | 16 | class Timeout(Exception): 17 | pass 18 | 19 | _resolv_conf = ResolvConf() 20 | _local_nameservers = _resolv_conf.nameservers 21 | _search_domains = [] 22 | if _resolv_conf.domain: 23 | _search_domains.append(str(_resolv_conf.domain)[:-1]) 24 | _search_domains.extend(map(lambda n: str(n)[:-1], _resolv_conf.search)) 25 | 26 | del _resolv_conf 27 | 28 | class DNSClient(UDPClient): 29 | """A DNS client. 30 | 31 | Uses nameservers from /etc/resolv.conf if none are supplied. 32 | 33 | """ 34 | def __init__(self, servers=None, port=53): 35 | if servers is None: 36 | self.nameservers = servers = _local_nameservers 37 | self.primary = self.nameservers[0] 38 | super(DNSClient, self).__init__(servers[0], port) 39 | 40 | @call 41 | def resolve(self, name, orig_timeout=5): 42 | """Try to resolve name. 43 | 44 | Returns: 45 | A list of IP addresses for name. 46 | 47 | Raises: 48 | * Timeout if the request to all servers times out. 49 | * NotFound if we get a response from a server but the name 50 | was not resolved. 51 | 52 | """ 53 | names = deque([name]) 54 | for n in _search_domains: 55 | names.append(('%s.%s' % (name, n))) 56 | start = time.time() 57 | timeout = orig_timeout 58 | r = None 59 | while names: 60 | n = names.popleft() 61 | try: 62 | r = self._actually_resolve(n, timeout) 63 | except: 64 | timeout = orig_timeout - (time.time() - start) 65 | if timeout <= 0 or not names: 66 | raise 67 | else: 68 | break 69 | assert r is not None 70 | return r 71 | 72 | def _actually_resolve(self, name, timeout): 73 | timeout = timeout / float(len(self.nameservers)) 74 | try: 75 | for server in self.nameservers: 76 | # Try each nameserver in succession. 77 | self.addr = server 78 | query = make_query(name, A) 79 | send(query.to_wire()) 80 | start = time.time() 81 | remaining = timeout 82 | while True: 83 | # Handle the possibility of responses that are not to our 84 | # original request - they are ignored and we wait for a 85 | # response that matches our query. 86 | item, data = first(datagram=True, sleep=remaining) 87 | if item == 'datagram': 88 | response = from_wire(data) 89 | if query.is_response(response): 90 | if response.answer: 91 | a_records = [r for r in response.answer if r.rdtype == A] 92 | return [item.address for item in a_records[0].items] 93 | raise NotFound 94 | else: 95 | # Not a response to our query - continue waiting for 96 | # one that is. 97 | remaining = remaining - (time.time() - start) 98 | elif item == 'sleep': 99 | break 100 | else: 101 | raise Timeout(name) 102 | finally: 103 | self.addr = self.primary 104 | 105 | -------------------------------------------------------------------------------- /diesel/protocols/__init__.py: -------------------------------------------------------------------------------- 1 | # vim:ts=4:sw=4:expandtab 2 | -------------------------------------------------------------------------------- /diesel/protocols/dreadlock.py: -------------------------------------------------------------------------------- 1 | from contextlib import contextmanager 2 | from diesel import send, until_eol, Client, call 3 | from diesel.util.pool import ConnectionPool 4 | 5 | class DreadlockTimeout(Exception): pass 6 | class DreadlockError(Exception): pass 7 | 8 | class DreadlockService(object): 9 | def __init__(self, host, port, pool=10): 10 | self.pool = ConnectionPool( 11 | lambda: DreadlockClient(host, port), 12 | lambda c: c.close()) 13 | 14 | @contextmanager 15 | def hold(self, key, timeout=10.0): 16 | with self.pool.connection as conn: 17 | conn.lock(key, timeout) 18 | try: 19 | yield None 20 | finally: 21 | conn.unlock(key) 22 | 23 | class DreadlockClient(Client): 24 | @call 25 | def lock(self, key, timeout=10.0): 26 | ms_timeout = int(timeout * 1000) 27 | send("lock %s %d\r\n" % (key, ms_timeout)) 28 | response = until_eol() 29 | if response[0] == 'l': 30 | return 31 | if response[0] == 't': 32 | raise DreadlockTimeout(key) 33 | if response[0] == 'e': 34 | raise DreadlockError(response[2:].strip()) 35 | assert False, response 36 | 37 | @call 38 | def unlock(self, key): 39 | send("unlock %s\r\n" % (key,)) 40 | response = until_eol() 41 | if response[0] == 'u': 42 | return 43 | if response[0] == 'e': 44 | raise DreadlockError(response[2:].strip()) 45 | assert False, response 46 | 47 | if __name__ == '__main__': 48 | locker = DreadlockService('localhost', 6001) 49 | import uuid 50 | from diesel import sleep, quickstart 51 | def f(): 52 | with locker.hold("foo"): 53 | id = uuid.uuid4() 54 | print "start!", id 55 | sleep(5) 56 | print "end!", id 57 | 58 | quickstart(f, f) 59 | -------------------------------------------------------------------------------- /diesel/protocols/http/__init__.py: -------------------------------------------------------------------------------- 1 | from .core import * 2 | -------------------------------------------------------------------------------- /diesel/protocols/http/pool.py: -------------------------------------------------------------------------------- 1 | import urllib 2 | import urlparse 3 | 4 | import diesel 5 | import diesel.protocols.http.core as http 6 | import diesel.util.pool as pool 7 | 8 | 9 | # XXX This dictionary can currently grow without bounds. A pool entry gets 10 | # created for every (host, port) key. Don't use this for a web crawler. 11 | _pools = {} 12 | 13 | VERSION = '3.0' 14 | USER_AGENT = 'diesel.protocols.http.pool v%s' % VERSION 15 | POOL_SIZE = 10 16 | 17 | class InvalidUrlScheme(Exception): 18 | pass 19 | 20 | def request(url, method='GET', timeout=60, body=None, headers=None): 21 | if body and (not isinstance(body, basestring)): 22 | body_bytes = urllib.urlencode(body) 23 | else: 24 | body_bytes = body 25 | req_url = urlparse.urlparse(url) 26 | if not headers: 27 | headers = {} 28 | headers.update({ 29 | 'Connection': 'keep-alive', 30 | }) 31 | if 'Host' not in headers: 32 | host = req_url.netloc.split(':')[0] 33 | headers['Host'] = host 34 | if 'User-Agent' not in headers: 35 | headers['User-Agent'] = USER_AGENT 36 | if req_url.query: 37 | req_path = '%s?%s' % (req_url.path, req_url.query) 38 | else: 39 | req_path = req_url.path 40 | encoded_path = req_path.encode('utf-8') 41 | # Loop to retry if the connection was closed. 42 | for i in xrange(POOL_SIZE): 43 | try: 44 | with http_pool_for_url(req_url).connection as conn: 45 | resp = conn.request(method, encoded_path, headers, timeout=timeout, body=body_bytes) 46 | break 47 | except diesel.ClientConnectionClosed, e: 48 | # try again with another pool connection 49 | continue 50 | else: 51 | raise e 52 | return resp 53 | 54 | def http_pool_for_url(req_url): 55 | host, port = host_and_port_from_url(req_url) 56 | if (host, port) not in _pools: 57 | make_client = ClientFactory(req_url.scheme, host, port) 58 | close_client = lambda c: c.close() 59 | conn_pool = pool.ConnectionPool(make_client, close_client, POOL_SIZE) 60 | _pools[(host, port)] = conn_pool 61 | return _pools[(host, port)] 62 | 63 | def host_and_port_from_url(req_url): 64 | if req_url.scheme == 'http': 65 | default_port = 80 66 | elif req_url.scheme == 'https': 67 | default_port = 443 68 | else: 69 | raise InvalidUrlScheme(req_url.scheme) 70 | if ':' not in req_url.netloc: 71 | host = req_url.netloc 72 | port = default_port 73 | else: 74 | host, port_ = req_url.netloc.split(':') 75 | port = int(port_) 76 | return host, port 77 | 78 | class ClientFactory(object): 79 | def __init__(self, scheme, host, port): 80 | if scheme == 'http': 81 | self.ClientClass = http.HttpClient 82 | elif scheme == 'https': 83 | self.ClientClass = http.HttpsClient 84 | else: 85 | raise InvalidUrlScheme(scheme) 86 | self.host = host 87 | self.port = port 88 | 89 | def __call__(self): 90 | return self.ClientClass(self.host, self.port) 91 | 92 | -------------------------------------------------------------------------------- /diesel/protocols/irc.py: -------------------------------------------------------------------------------- 1 | '''Experimental support for Internet Relay Chat''' 2 | 3 | from diesel import Client, call, sleep, send, until_eol, receive, first, Loop, Application, ConnectionClosed, quickstop 4 | from OpenSSL import SSL 5 | import os, pwd 6 | from types import GeneratorType 7 | 8 | LOCAL_HOST = os.uname()[1] 9 | DEFAULT_REAL_NAME = "Diesel IRC" 10 | DEFAULT_USER = pwd.getpwuid(os.getuid()).pw_name 11 | 12 | class IrcError(Exception): pass 13 | 14 | class IrcCommand(object): 15 | def __init__(self, prefix, command, raw_params): 16 | self.from_server = None 17 | self.from_nick = None 18 | self.from_host = None 19 | self.from_user = None 20 | if prefix: 21 | if '!' in prefix: 22 | self.from_nick, rest = prefix.split('!') 23 | self.from_host, self.from_user = rest.split('@') 24 | else: 25 | self.from_server = prefix 26 | 27 | self.prefix = prefix 28 | if command.isdigit(): 29 | command = int(command) 30 | self.command = command 31 | self.params = [] 32 | 33 | while raw_params: 34 | if raw_params.startswith(":"): 35 | self.params.append(raw_params[1:]) 36 | break 37 | parts = raw_params.split(' ', 1) 38 | if len(parts) == 1: 39 | self.params.append(raw_params) 40 | break 41 | p, raw_params = parts 42 | self.params.append(p) 43 | 44 | def __str__(self): 45 | return "%s (%s) %r" % (self.command, self.from_nick, self.params) 46 | 47 | class IrcClient(Client): 48 | def __init__(self, nick, host='localhost', port=6667, 49 | user=DEFAULT_USER, name=DEFAULT_REAL_NAME, password=None, 50 | **kw): 51 | self.nick = nick 52 | self.user = user 53 | self.name = name 54 | self.host = host 55 | self.logged_in = False 56 | self.password = password 57 | Client.__init__(self, host, port, **kw) 58 | 59 | @call 60 | def on_connect(self): 61 | self.do_login() 62 | self.on_logged_in() 63 | 64 | def on_logged_in(self): 65 | pass 66 | 67 | @call 68 | def do_login(self): 69 | if self.password: 70 | self.send_command("PASS", self.password) 71 | self.send_command("NICK", self.nick) 72 | self.send_command("USER", 73 | '%s@%s' % (self.user, LOCAL_HOST), 74 | 8, '*', self.name) 75 | self.logged_in = True 76 | 77 | @call 78 | def send_command(self, cmd, *args): 79 | if self.logged_in: 80 | send(":%s " % self.nick) 81 | acc = [cmd] 82 | for x, a in enumerate(args): 83 | ax = str(a) 84 | if ' ' in ax: 85 | assert (x == (len(args) - 1)), "no spaces except in final param" 86 | acc.append(':' + ax) 87 | else: 88 | acc.append(ax) 89 | send(' '.join(acc) + "\r\n") 90 | 91 | @call 92 | def recv_command(self): 93 | cmd = None 94 | while True: 95 | raw = until_eol().rstrip() 96 | if raw.startswith(':'): 97 | prefix, raw = raw[1:].split(' ', 1) 98 | else: 99 | prefix = None 100 | command, raw = raw.split(' ', 1) 101 | cmd = IrcCommand(prefix, command, raw) 102 | if cmd.command == 'PING': 103 | self.send_command('PONG', *cmd.params) 104 | elif cmd.command == 'ERROR': 105 | raise IrcError(cmd.params[0]) 106 | elif type(cmd.command) is int and cmd.command == 433: 107 | self.nick += '_' 108 | self.do_login() 109 | else: 110 | break 111 | 112 | return cmd 113 | 114 | class SSLIrcClient(IrcClient): 115 | def __init__(self, *args, **kw): 116 | kw['ssl_ctx'] = SSL.Context(SSL.SSLv23_METHOD) 117 | IrcClient.__init__(self, *args, **kw) 118 | 119 | 120 | class IrcBot(IrcClient): 121 | def __init__(self, *args, **kw): 122 | if 'channels' in kw: 123 | self.chans = set(kw.pop('channels')) 124 | else: 125 | self.chans = set() 126 | 127 | IrcClient.__init__(self, *args, **kw) 128 | 129 | def on_logged_in(self): 130 | for c in self.chans: 131 | assert not c.startswith('#') 132 | c = '#' + c 133 | self.send_command( 134 | 'JOIN', c) 135 | 136 | def on_message(self, channel, nick, content): 137 | pass 138 | 139 | def run(self): 140 | while True: 141 | cmd = self.recv_command() 142 | if cmd.command == "PRIVMSG": 143 | if cmd.from_nick and len(cmd.params) == 2: 144 | mchan, content = cmd.params 145 | if not content.startswith('\x01') and mchan.startswith('#') and mchan[1:] in self.chans: 146 | content = content.decode('utf-8') 147 | r = self.on_message(mchan, cmd.from_nick, content) 148 | self._handle_return(r, mchan) 149 | 150 | def _handle_return(self, r, chan): 151 | if r is None: 152 | pass 153 | elif type(r) is str: 154 | if r.startswith("/me"): 155 | r = "\x01ACTION " + r[4:] + "\x01" 156 | assert '\r' not in r 157 | assert '\n' not in r 158 | assert '\0' not in r 159 | self.send_command('PRIVMSG', chan, r) 160 | elif type(r) is unicode: 161 | self._handle_return(r.encode('utf-8'), chan) 162 | elif type(r) is tuple: 163 | chan, r = r 164 | self._handle_return(r, chan) 165 | elif type(r) is GeneratorType: 166 | for i in r: 167 | self._handle_return(i, chan) 168 | else: 169 | print 'Hmm, unknown type returned from message handler:', type(r) 170 | 171 | class SSLIrcBot(IrcBot): 172 | def __init__(self, *args, **kw): 173 | kw['ssl_ctx'] = SSL.Context(SSL.SSLv23_METHOD) 174 | IrcBot.__init__(self, *args, **kw) 175 | -------------------------------------------------------------------------------- /diesel/protocols/wsgi.py: -------------------------------------------------------------------------------- 1 | # vim:ts=4:sw=4:expandtab 2 | """A minimal WSGI implementation to hook into 3 | diesel's HTTP module. 4 | 5 | Note: not well-tested. Contributions welcome. 6 | """ 7 | from diesel import Application, Service 8 | from diesel.protocols.http import HttpServer, Response 9 | 10 | import functools 11 | 12 | class WSGIRequestHandler(object): 13 | '''The request_handler for the HttpServer that 14 | bootsraps the WSGI environment and hands it off to the 15 | WSGI callable. This is the key coupling. 16 | ''' 17 | def __init__(self, wsgi_callable, port=80): 18 | self.port = port 19 | self.wsgi_callable = wsgi_callable 20 | 21 | def _start_response(self, env, status, response_headers, exc_info=None): 22 | if exc_info: 23 | raise exc_info[0], exc_info[1], exc_info[2] 24 | else: 25 | r = env['diesel.response'] 26 | r.status = status 27 | for k, v in response_headers: 28 | r.headers.add(k, v) 29 | return r.response.append 30 | 31 | def __call__(self, req): 32 | env = req.environ 33 | buf = [] 34 | r = Response() 35 | env['diesel.response'] = r 36 | for output in self.wsgi_callable(env, 37 | functools.partial(self._start_response, env)): 38 | r.response.append(output) 39 | del env['diesel.response'] 40 | return r 41 | 42 | class WSGIApplication(Application): 43 | '''A WSGI application that takes over both `Service` 44 | setup, `request_handler` spec for the HTTPServer, 45 | and the app startup itself. 46 | 47 | 48 | Just pass it a wsgi_callable and port information, and 49 | it should do the rest. 50 | ''' 51 | def __init__(self, wsgi_callable, port=80, iface=''): 52 | Application.__init__(self) 53 | self.port = port 54 | self.wsgi_callable = wsgi_callable 55 | http_service = Service(HttpServer(WSGIRequestHandler(wsgi_callable, port)), port, iface) 56 | self.add_service(http_service) 57 | 58 | if __name__ == '__main__': 59 | def simple_app(environ, start_response): 60 | """Simplest possible application object""" 61 | status = '200 OK' 62 | response_headers = [('Content-type','text/plain')] 63 | start_response(status, response_headers) 64 | return ["Hello World!"] 65 | app = WSGIApplication(simple_app, port=7080) 66 | app.run() 67 | -------------------------------------------------------------------------------- /diesel/resolver.py: -------------------------------------------------------------------------------- 1 | '''Fetches the A record for a given name in a green thread, keeps 2 | a cache. 3 | ''' 4 | 5 | import os 6 | import random 7 | import time 8 | import socket 9 | from diesel.protocols.DNS import DNSClient, NotFound, Timeout 10 | from diesel.util.pool import ConnectionPool 11 | from diesel.util.lock import synchronized 12 | 13 | DNS_CACHE_TIME = 60 * 5 # five minutes 14 | 15 | cache = {} 16 | 17 | class DNSResolutionError(Exception): pass 18 | 19 | _pool = ConnectionPool(lambda: DNSClient(), lambda c: c.close()) 20 | 21 | hosts = {} 22 | 23 | def load_hosts(): 24 | if os.path.isfile("/etc/hosts"): 25 | for line in open("/etc/hosts"): 26 | parts = line.split() 27 | ip = None 28 | for p in parts: 29 | if p.startswith("#"): 30 | break 31 | if not ip: 32 | if ':' in p: 33 | break 34 | ip = p 35 | else: 36 | hosts[p] = ip 37 | 38 | load_hosts() 39 | 40 | def resolve_dns_name(name): 41 | '''Uses a pool of DNSClients to resolve name to an IP address. 42 | 43 | Keep a cache. 44 | ''' 45 | 46 | # Is name an IP address? 47 | try: 48 | socket.inet_pton(socket.AF_INET, name) 49 | return name 50 | except socket.error: 51 | # Not a valid IP address resolve it 52 | pass 53 | 54 | if name in hosts: 55 | return hosts[name] 56 | 57 | with synchronized('__diesel__.dns.' + name): 58 | try: 59 | ips, tm = cache[name] 60 | if time.time() - tm > DNS_CACHE_TIME: 61 | del cache[name] 62 | cache[name] 63 | except KeyError: 64 | try: 65 | with _pool.connection as conn: 66 | ips = conn.resolve(name) 67 | except (NotFound, Timeout): 68 | raise DNSResolutionError("could not resolve A record for %s" % name) 69 | cache[name] = ips, time.time() 70 | return random.choice(ips) 71 | -------------------------------------------------------------------------------- /diesel/runtime.py: -------------------------------------------------------------------------------- 1 | current_app = None 2 | 3 | def is_running(): 4 | return current_app is not None 5 | -------------------------------------------------------------------------------- /diesel/security.py: -------------------------------------------------------------------------------- 1 | from OpenSSL import SSL 2 | import traceback 3 | import sys 4 | 5 | def ssl_async_handshake(sock, hub, next): 6 | def shake(): 7 | try: 8 | sock.do_handshake() 9 | except SSL.WantReadError: 10 | hub.disable_write(sock) 11 | except SSL.WantWriteError: 12 | hub.enable_write(sock) 13 | except SSL.WantX509LookupError: 14 | pass 15 | except Exception, e: 16 | hub.unregister(sock) 17 | next(e) 18 | else: 19 | hub.unregister(sock) 20 | next() 21 | hub.register(sock, shake, shake, shake) 22 | shake() 23 | -------------------------------------------------------------------------------- /diesel/util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dieseldev/diesel/8d48371fce0b79d6631053594bce06e4b9628499/diesel/util/__init__.py -------------------------------------------------------------------------------- /diesel/util/debugtools.py: -------------------------------------------------------------------------------- 1 | import collections 2 | import gc 3 | import re 4 | import traceback 5 | 6 | from operator import itemgetter 7 | 8 | import diesel 9 | 10 | 11 | address_stripper = re.compile(r' at 0x[0-9a-f]+') 12 | 13 | def print_greenlet_stacks(): 14 | """Prints the stacks of greenlets from running loops. 15 | 16 | The number of greenlets at the same position in the stack is displayed 17 | on the line before the stack dump along with a simplified label for the 18 | loop callable. 19 | 20 | """ 21 | stacks = collections.defaultdict(int) 22 | loops = {} 23 | for obj in gc.get_objects(): 24 | if not isinstance(obj, diesel.Loop) or not obj.running: 25 | continue 26 | if obj.id == diesel.core.current_loop.id: 27 | continue 28 | fr = obj.coroutine.gr_frame 29 | stack = ''.join(traceback.format_stack(fr)) 30 | stacks[stack] += 1 31 | loops[stack] = obj 32 | for stack, count in sorted(stacks.iteritems(), key=itemgetter(1)): 33 | loop = loops[stack] 34 | loop_id = address_stripper.sub('', str(loop.loop_callable)) 35 | print '[%d] === %s ===' % (count, loop_id) 36 | print stack 37 | -------------------------------------------------------------------------------- /diesel/util/event.py: -------------------------------------------------------------------------------- 1 | from diesel import fire, first, signal 2 | from diesel.events import Waiter, StopWaitDispatch 3 | 4 | class EventTimeout(Exception): pass 5 | 6 | class Event(Waiter): 7 | def __init__(self): 8 | self.is_set = False 9 | 10 | def set(self): 11 | if not self.is_set: 12 | self.is_set = True 13 | fire(self) 14 | 15 | def clear(self): 16 | self.is_set = False 17 | 18 | def ready_early(self): 19 | return self.is_set 20 | 21 | def process_fire(self, value): 22 | if not self.is_set: 23 | raise StopWaitDispatch() 24 | return value 25 | 26 | def wait(self, timeout=None): 27 | kw = dict(waits=[self]) 28 | if timeout: 29 | kw['sleep'] = timeout 30 | mark, data = first(**kw) 31 | if mark != self: 32 | raise EventTimeout() 33 | 34 | class Countdown(Event): 35 | def __init__(self, count): 36 | self.remaining = count 37 | Event.__init__(self) 38 | 39 | def tick(self): 40 | self.remaining -= 1 41 | if self.remaining <= 0: 42 | self.set() 43 | 44 | class Signal(Event): 45 | """Used for waiting on an OS-level signal.""" 46 | 47 | def __init__(self, signum): 48 | """Create a Signal instance waiting on signum. 49 | 50 | It will be triggered whenever the provided signum is sent to the 51 | process. A Loop can be off doing other tasks when the signal arrives 52 | and it will still trigger this event (the Loop won't know until the 53 | next time it waits on this event though). 54 | 55 | After the event has been triggered, it must be rearmed before it can 56 | be waited on again. Otherwise, like a base Event, it will remain in 57 | the triggered state and thus waiting on it will immediately return. 58 | 59 | """ 60 | Event.__init__(self) 61 | self.signum = signum 62 | self.rearm() 63 | 64 | def rearm(self): 65 | """Prepares the Signal for use again. 66 | 67 | This must be called before waiting on a Signal again after it has 68 | been triggered. 69 | 70 | """ 71 | self.clear() 72 | signal(self.signum, self.set) 73 | -------------------------------------------------------------------------------- /diesel/util/lock.py: -------------------------------------------------------------------------------- 1 | from uuid import uuid4 2 | from diesel import wait, fire 3 | from collections import defaultdict 4 | from diesel.events import Waiter, StopWaitDispatch 5 | 6 | class Lock(Waiter): 7 | def __init__(self, count=1): 8 | self.count = count 9 | 10 | def acquire(self): 11 | if self.count == 0: 12 | wait(self) 13 | else: 14 | self.count -= 1 15 | 16 | def release(self): 17 | self.count += 1 18 | fire(self) 19 | 20 | def __enter__(self): 21 | self.acquire() 22 | 23 | def __exit__(self, *args, **kw): 24 | self.release() 25 | 26 | @property 27 | def is_locked(self): 28 | return self.count == 0 29 | 30 | def ready_early(self): 31 | return not self.is_locked 32 | 33 | def process_fire(self, value): 34 | if self.count == 0: 35 | raise StopWaitDispatch() 36 | 37 | self.count -= 1 38 | return value 39 | 40 | class SynchronizeDefault(object): pass 41 | 42 | _sync_locks = defaultdict(Lock) 43 | 44 | def synchronized(key=SynchronizeDefault): 45 | return _sync_locks[key] 46 | -------------------------------------------------------------------------------- /diesel/util/patches/__init__.py: -------------------------------------------------------------------------------- 1 | from requests_lib import enable_requests 2 | -------------------------------------------------------------------------------- /diesel/util/patches/requests_lib.py: -------------------------------------------------------------------------------- 1 | """Through the magic of monkeypatching, requests works with diesel. 2 | 3 | It's a hack. 4 | 5 | """ 6 | import httplib 7 | 8 | import diesel 9 | from diesel.resolver import DNSResolutionError 10 | from OpenSSL import SSL 11 | 12 | try: 13 | from requests.packages.urllib3 import connectionpool 14 | import requests 15 | except ImportError: 16 | connectionpool = None 17 | 18 | 19 | class SocketLike(diesel.Client): 20 | """A socket-like diesel Client. 21 | 22 | At least enough to satisfy the requests test suite. Its primary job is 23 | to return a FileLike instance when `makefile` is called. 24 | 25 | """ 26 | def __init__(self, host, port, **kw): 27 | super(SocketLike, self).__init__(host, port, **kw) 28 | self._timeout = None 29 | 30 | def makefile(self, mode, buffering): 31 | return FileLike(self, mode, buffering, self._timeout) 32 | 33 | def settimeout(self, n): 34 | self._timeout = n 35 | 36 | @diesel.call 37 | def sendall(self, data): 38 | diesel.send(data) 39 | 40 | def fileno(self): 41 | return id(self) 42 | 43 | class FileLike(object): 44 | """Gives you a file-like interface from a diesel Client.""" 45 | def __init__(self, client, mode, buffering, timeout): 46 | self._client = client 47 | self.mode = mode 48 | self.buffering = buffering 49 | self._extra = "" 50 | self._timeout = timeout 51 | 52 | # Properties To Stand In For diesel Client 53 | # ---------------------------------------- 54 | 55 | @property 56 | def conn(self): 57 | return self._client.conn 58 | 59 | @property 60 | def connected(self): 61 | return self._client.connected 62 | 63 | @property 64 | def is_closed(self): 65 | return self._client.is_closed 66 | 67 | # File-Like API 68 | # ------------- 69 | 70 | @diesel.call 71 | def read(self, size=None): 72 | assert size is not None, "Sorry, have to pass a size to read()" 73 | if size == 0: 74 | return '' 75 | evt, data = diesel.first(sleep=self._timeout, receive=size) 76 | if evt == 'sleep': 77 | self._timeout = None 78 | raise requests.exceptions.Timeout 79 | return data 80 | 81 | @diesel.call 82 | def readline(self, max_size=None): 83 | evt, line = diesel.first(sleep=self._timeout, until='\n') 84 | if evt == 'sleep': 85 | self._timeout = None 86 | raise requests.exceptions.Timeout 87 | if max_size: 88 | line = "".join([self._extra, line]) 89 | nl = line.find('\n') + 1 90 | if nl > max_size: 91 | nl = max_size 92 | line, self._extra = line[:nl], line[nl:] 93 | return line 94 | 95 | @diesel.call 96 | def write(self, data): 97 | diesel.send(data) 98 | 99 | @diesel.call 100 | def next(self): 101 | data = self.readline() 102 | if not data: 103 | raise StopIteration() 104 | return data 105 | 106 | def __iter__(self): 107 | return self 108 | 109 | def close(self): 110 | self._client.close() 111 | 112 | class HTTPConnection(httplib.HTTPConnection): 113 | def connect(self): 114 | try: 115 | self.sock = SocketLike(self.host, self.port) 116 | except DNSResolutionError: 117 | raise requests.ConnectionError 118 | 119 | class HTTPSConnection(httplib.HTTPSConnection): 120 | def connect(self): 121 | try: 122 | kw = {'ssl_ctx': SSL.Context(SSL.SSLv23_METHOD)} 123 | self.sock = SocketLike(self.host, self.port, **kw) 124 | except DNSResolutionError: 125 | raise requests.ConnectionError 126 | 127 | class RequestsLibNotFound(Exception):pass 128 | 129 | def enable_requests(): 130 | """This is monkeypatching.""" 131 | if not connectionpool: 132 | msg = "You need to install requests (http://python-requests.org)" 133 | raise RequestsLibNotFound(msg) 134 | connectionpool.HTTPConnection = HTTPConnection 135 | connectionpool.HTTPSConnection = HTTPSConnection 136 | 137 | -------------------------------------------------------------------------------- /diesel/util/pool.py: -------------------------------------------------------------------------------- 1 | '''Simple connection pool for asynchronous code. 2 | ''' 3 | from collections import deque 4 | from diesel import * 5 | from diesel.util.queue import Queue, QueueTimeout 6 | from diesel.util.event import Event 7 | 8 | 9 | class ConnectionPoolFull(Exception): pass 10 | 11 | class InfiniteQueue(object): 12 | def get(self, timeout): 13 | pass 14 | 15 | def put(self): 16 | pass 17 | 18 | class ConnectionPool(object): 19 | '''A connection pool that holds `pool_size` connected instances, 20 | calls init_callable() when it needs more, and passes 21 | to close_callable() connections that will not fit on the pool. 22 | ''' 23 | 24 | def __init__(self, init_callable, close_callable, pool_size=5, pool_max=None, poll_max_timeout=5): 25 | self.init_callable = init_callable 26 | self.close_callable = close_callable 27 | self.pool_size = pool_size 28 | self.poll_max_timeout = poll_max_timeout 29 | if pool_max: 30 | self.remaining_conns = Queue() 31 | for _ in xrange(pool_max): 32 | self.remaining_conns.inp.append(None) 33 | else: 34 | self.remaining_conns = InfiniteQueue() 35 | self.connections = deque() 36 | 37 | def get(self): 38 | try: 39 | self.remaining_conns.get(timeout=self.poll_max_timeout) 40 | except QueueTimeout: 41 | raise ConnectionPoolFull() 42 | 43 | if not self.connections: 44 | self.connections.append(self.init_callable()) 45 | conn = self.connections.pop() 46 | 47 | if not conn.is_closed: 48 | return conn 49 | else: 50 | self.remaining_conns.put() 51 | return self.get() 52 | 53 | def release(self, conn, error=False): 54 | self.remaining_conns.put() 55 | if not conn.is_closed: 56 | if not error and len(self.connections) < self.pool_size: 57 | self.connections.append(conn) 58 | else: 59 | self.close_callable(conn) 60 | 61 | @property 62 | def connection(self): 63 | return ConnContextWrapper(self, self.get()) 64 | 65 | class ConnContextWrapper(object): 66 | '''Context wrapper for try/finally behavior using the 67 | "with" statement. 68 | 69 | Ensures that connections return to the pool when the 70 | code block that requires them has ended. 71 | ''' 72 | def __init__(self, pool, conn): 73 | self.pool = pool 74 | self.conn = conn 75 | 76 | def __enter__(self): 77 | return self.conn 78 | 79 | def __exit__(self, type, value, tb): 80 | error = type is not None 81 | self.pool.release(self.conn, error) 82 | 83 | class ThreadPoolDie(object): pass 84 | 85 | class ThreadPool(object): 86 | def __init__(self, concurrency, handler, generator, finalizer=None): 87 | self.concurrency = concurrency 88 | self.handler = handler 89 | self.generator = generator 90 | self.finalizer = finalizer 91 | 92 | def handler_wrap(self): 93 | try: 94 | label("thread-pool-%s" % self.handler) 95 | while True: 96 | self.waiting += 1 97 | if self.waiting == 1: 98 | self.trigger.set() 99 | i = self.q.get() 100 | self.waiting -= 1 101 | if i == ThreadPoolDie: 102 | return 103 | self.handler(i) 104 | finally: 105 | self.running -=1 106 | if self.waiting == 0: 107 | self.trigger.set() 108 | if self.running == 0: 109 | self.finished.set() 110 | 111 | def __call__(self): 112 | self.q = Queue() 113 | self.trigger = Event() 114 | self.finished = Event() 115 | self.waiting = 0 116 | self.running = 0 117 | try: 118 | while True: 119 | for x in xrange(self.concurrency - self.running): 120 | self.running += 1 121 | fork(self.handler_wrap) 122 | 123 | if self.waiting == 0: 124 | self.trigger.wait() 125 | self.trigger.clear() 126 | 127 | try: 128 | n = self.generator() 129 | except StopIteration: 130 | break 131 | 132 | self.q.put(n) 133 | sleep() 134 | finally: 135 | for x in xrange(self.concurrency): 136 | self.q.put(ThreadPoolDie) 137 | if self.finalizer: 138 | self.finished.wait() 139 | fork(self.finalizer) 140 | 141 | class TerminalThreadPool(ThreadPool): 142 | def __call__(self, *args, **kw): 143 | try: 144 | ThreadPool.__call__(self, *args, **kw) 145 | finally: 146 | while self.running: 147 | self.trigger.wait() 148 | self.trigger.clear() 149 | log.warning("TerminalThreadPool's producer exited; issuing quickstop()") 150 | fork(quickstop) 151 | -------------------------------------------------------------------------------- /diesel/util/process.py: -------------------------------------------------------------------------------- 1 | """diesel's async I/O event hub meets multiprocessing. 2 | 3 | Let's you run CPU intensive work in subprocesses and not block the event hub 4 | while doing so. 5 | 6 | """ 7 | import multiprocessing as mp 8 | import traceback 9 | 10 | from diesel import runtime 11 | from diesel import core 12 | from diesel.util.queue import Queue 13 | 14 | 15 | def spawn(func): 16 | """Spawn a new process that will run func. 17 | 18 | The returned Process instance can be called just like func. 19 | 20 | The spawned OS process lives until it is term()ed or otherwise dies. Each 21 | call to the returned Process instance results in another iteration of 22 | the remote loop. This way a single process can handle multiple calls to 23 | func. 24 | 25 | """ 26 | return Process(func) 27 | 28 | 29 | def term(proc): 30 | """Terminate the given proc. 31 | 32 | That is all. 33 | """ 34 | proc.cleanup() 35 | proc.proc.terminate() 36 | 37 | 38 | class ConflictingCall(Exception): 39 | pass 40 | 41 | class Process(object): 42 | """A subprocess that cooperates with diesel's event hub. 43 | 44 | Communication with the spawned process happens over a pipe. Data that 45 | is to be sent to or received from the process is dispatched by the 46 | event hub. This makes it easy to run CPU intensive work in a non-blocking 47 | fashion and utilize multiple CPU cores. 48 | 49 | """ 50 | def __init__(self, func): 51 | """Creates a new Process instance that will call func. 52 | 53 | The returned instance can be called as if it were func. The following 54 | code will run ``time.sleep`` in a subprocess and execution will resume 55 | when the remote call completes. Other green threads can run in the 56 | meantime. 57 | 58 | >>> time_sleep = Process(time.sleep) 59 | >>> time_sleep(4.2) 60 | >>> do_other_stuff() 61 | 62 | """ 63 | self.func = func 64 | self.proc = None 65 | self.caller = None 66 | self.args = None 67 | self.params = None 68 | self.pipe = None 69 | self.in_call = False 70 | self.launch() 71 | 72 | def launch(self): 73 | """Starts a subprocess and connects it to diesel's plumbing. 74 | 75 | A pipe is created, registered with the event hub and used to 76 | communicate with the subprocess. 77 | 78 | """ 79 | self.pipe, remote_pipe = mp.Pipe() 80 | runtime.current_app.hub.register( 81 | self.pipe, 82 | self.handle_return_value, 83 | self.send_arguments_to_process, 84 | runtime.current_app.global_bail('Process error!'), 85 | ) 86 | def wrapper(pipe): 87 | while True: 88 | try: 89 | args, params = pipe.recv() 90 | pipe.send(self.func(*args, **params)) 91 | except (SystemExit, KeyboardInterrupt): 92 | pipe.close() 93 | break 94 | except Exception, e: 95 | e.original_traceback = traceback.format_exc() 96 | pipe.send(e) 97 | 98 | self.proc = mp.Process(target=wrapper, args=(remote_pipe,)) 99 | self.proc.daemon = True 100 | self.proc.start() 101 | 102 | def cleanup(self): 103 | runtime.current_app.hub.unregister(self.pipe) 104 | 105 | def handle_return_value(self): 106 | """Wakes up the caller with the return value of the subprocess func. 107 | 108 | Called by the event hub when data is ready. 109 | 110 | """ 111 | try: 112 | result = self.pipe.recv() 113 | except EOFError: 114 | self.pipe.close() 115 | self.proc.terminate() 116 | else: 117 | self.in_call = False 118 | self.caller.wake(result) 119 | 120 | def send_arguments_to_process(self): 121 | """Sends the arguments to the function to the remote process. 122 | 123 | Called by the event hub after the instance has been called. 124 | 125 | """ 126 | runtime.current_app.hub.disable_write(self.pipe) 127 | self.pipe.send((self.args, self.params)) 128 | 129 | def __call__(self, *args, **params): 130 | """Trigger the execution of self.func in the subprocess. 131 | 132 | Switches control back to the event hub, letting other loops run until 133 | the subprocess finishes computation. Returns the result of the 134 | subprocess's call to self.func. 135 | 136 | """ 137 | if self.in_call: 138 | msg = "Another loop (%r) is executing this process." % self.caller 139 | raise ConflictingCall(msg) 140 | runtime.current_app.hub.enable_write(self.pipe) 141 | self.args = args 142 | self.params = params 143 | self.caller = core.current_loop 144 | self.in_call = True 145 | return self.caller.dispatch() 146 | 147 | class NoSubProcesses(Exception): 148 | pass 149 | 150 | class ProcessPool(object): 151 | """A bounded pool of subprocesses. 152 | 153 | An instance is callable, just like a Process, and will return the result 154 | of executing the function in a subprocess. If all subprocesses are busy, 155 | the caller will wait in a queue. 156 | 157 | """ 158 | def __init__(self, concurrency, handler): 159 | """Creates a new ProcessPool with subprocesses that run the handler. 160 | 161 | Args: 162 | concurrency (int): The number of subprocesses to spawn. 163 | handler (callable): A callable that the subprocesses will execute. 164 | 165 | """ 166 | self.concurrency = concurrency 167 | self.handler = handler 168 | self.available_procs = Queue() 169 | self.all_procs = [] 170 | 171 | def __call__(self, *args, **params): 172 | """Gets a process from the pool, executes it, and returns the results. 173 | 174 | This call will block until there is a process available to handle it. 175 | 176 | """ 177 | if not self.all_procs: 178 | raise NoSubProcesses("Did you forget to start the pool?") 179 | try: 180 | p = self.available_procs.get() 181 | result = p(*args, **params) 182 | return result 183 | finally: 184 | self.available_procs.put(p) 185 | 186 | def pool(self): 187 | """A callable that starts the processes in the pool. 188 | 189 | This is useful as the callable to pass to a diesel.Loop when adding a 190 | ProcessPool to your application. 191 | 192 | """ 193 | for i in xrange(self.concurrency): 194 | proc = spawn(self.handler) 195 | self.available_procs.put(proc) 196 | self.all_procs.append(proc) 197 | 198 | if __name__ == '__main__': 199 | import diesel 200 | 201 | def sleep_and_return(secs): 202 | import time 203 | start = time.time() 204 | time.sleep(secs) 205 | return time.time() - start 206 | sleep_pool = ProcessPool(2, sleep_and_return) 207 | 208 | def main(): 209 | def waiting(ident): 210 | print ident, "waiting ..." 211 | t = sleep_pool(4) 212 | print ident, "woken up after", t 213 | 214 | diesel.fork(waiting, 'a') 215 | diesel.fork(waiting, 'b') 216 | diesel.fork(waiting, 'c') 217 | for i in xrange(11): 218 | print "busy!" 219 | diesel.sleep(1) 220 | div = spawn(lambda x,y: x/y) 221 | try: 222 | div(1,0) 223 | except ZeroDivisionError, e: 224 | diesel.log.error(e.original_traceback) 225 | print '^^ That was an intentional exception.' 226 | term(div) 227 | psleep = spawn(sleep_and_return) 228 | diesel.fork(psleep, 0.5) 229 | diesel.fork(psleep, 0.5) 230 | diesel.sleep(1) 231 | print '^^ That was an intentional exception.' 232 | diesel.quickstop() 233 | 234 | diesel.quickstart(sleep_pool.pool, main) 235 | -------------------------------------------------------------------------------- /diesel/util/queue.py: -------------------------------------------------------------------------------- 1 | from uuid import uuid4 2 | import random 3 | from collections import deque 4 | from contextlib import contextmanager 5 | 6 | from diesel import fire, sleep, first 7 | from diesel.events import Waiter, StopWaitDispatch 8 | 9 | class QueueEmpty(Exception): pass 10 | class QueueTimeout(Exception): pass 11 | 12 | class Queue(Waiter): 13 | def __init__(self): 14 | self.inp = deque() 15 | 16 | def put(self, i=None): 17 | self.inp.append(i) 18 | fire(self) 19 | 20 | def get(self, waiting=True, timeout=None): 21 | if self.inp: 22 | val = self.inp.popleft() 23 | sleep() 24 | return val 25 | mark = None 26 | 27 | if waiting: 28 | kw = dict(waits=[self]) 29 | if timeout: 30 | kw['sleep'] = timeout 31 | mark, val = first(**kw) 32 | if mark == self: 33 | return val 34 | else: 35 | raise QueueTimeout() 36 | 37 | raise QueueEmpty() 38 | 39 | def __iter__(self): 40 | return self 41 | 42 | def next(self): 43 | return self.get() 44 | 45 | @property 46 | def is_empty(self): 47 | return not bool(self.inp) 48 | 49 | def process_fire(self, value): 50 | if self.inp: 51 | return self.inp.popleft() 52 | else: 53 | raise StopWaitDispatch() 54 | 55 | def ready_early(self): 56 | return not self.is_empty 57 | 58 | class Fanout(object): 59 | def __init__(self): 60 | self.subs = set() 61 | 62 | def pub(self, m): 63 | for s in self.subs: 64 | s.put(m) 65 | 66 | @contextmanager 67 | def sub(self): 68 | q = Queue() 69 | self.subs.add(q) 70 | try: 71 | yield q 72 | finally: 73 | self.subs.remove(q) 74 | 75 | class Dispatcher(object): 76 | def __init__(self): 77 | self.subs = {} 78 | self.keys = [] 79 | self.backlog = [] 80 | 81 | def dispatch(self, m): 82 | if self.subs: 83 | k = random.choice(self.keys) 84 | self.subs[k].put(m) 85 | else: 86 | self.backlog.append(m) 87 | 88 | @contextmanager 89 | def accept(self): 90 | q = Queue() 91 | if self.backlog: 92 | for b in self.backlog: 93 | q.put(b) 94 | self.backlog = [] 95 | id = uuid4() 96 | self.subs[id] = q 97 | self.keys = list(self.subs) 98 | try: 99 | yield q 100 | finally: 101 | del self.subs[id] 102 | self.keys = list(self.subs) 103 | while not q.is_empty: 104 | self.dispatch(q.get()) 105 | -------------------------------------------------------------------------------- /diesel/util/stats.py: -------------------------------------------------------------------------------- 1 | 2 | from diesel import core 3 | 4 | class CPUStats(object): 5 | def __init__(self): 6 | self.caller = core.current_loop 7 | self.cpu_seconds = 0.0 8 | 9 | def __enter__(self): 10 | self.start_clock = self.caller.clocktime() 11 | return self 12 | 13 | def __exit__(self, exc_type, exc_val, exc_tb): 14 | if not (exc_type and exc_val and exc_tb): 15 | end_clock = self.caller.clocktime() 16 | self.cpu_seconds = end_clock - self.start_clock 17 | 18 | -------------------------------------------------------------------------------- /diesel/util/streams.py: -------------------------------------------------------------------------------- 1 | import thread 2 | from diesel.util.queue import Queue 3 | from diesel import fork_from_thread 4 | 5 | def put_stream_token(q, line): 6 | q.put(line) 7 | 8 | def consume_stream(stream, q): 9 | while True: 10 | line = stream.readline() 11 | fork_from_thread(put_stream_token, q, line) 12 | if line == '': 13 | break 14 | 15 | def create_line_input_stream(fileobj): 16 | q = Queue() 17 | thread.start_new_thread(consume_stream, (fileobj, q)) 18 | return q 19 | -------------------------------------------------------------------------------- /diesel/web.py: -------------------------------------------------------------------------------- 1 | '''Slight wrapper around flask to fit the diesel 2 | 3 | mold. 4 | ''' 5 | import traceback 6 | 7 | from flask import * # we're essentially republishing 8 | from werkzeug.debug import tbtools 9 | from diesel.protocols.websockets import WebSocketServer 10 | 11 | from app import Application, Service, quickstart 12 | from diesel import log, set_log_level, loglevels 13 | 14 | 15 | class _FlaskTwiggyLogProxy(object): 16 | """Proxies to a Twiggy Logger. 17 | 18 | Nearly all attribute access is proxied to a twiggy Logger, with the 19 | exception of the `name` attribute. This one change brings it closer in 20 | line with the API of the Python standard library `logging` module which 21 | Flask expects. 22 | 23 | """ 24 | def __init__(self, name): 25 | self.__dict__['_logger'] = log.name(name) 26 | self.__dict__['name'] = name 27 | 28 | def __getattr__(self, name): 29 | return getattr(self._logger, name) 30 | 31 | def __setattr__(self, name, value): 32 | return setattr(self._logger, name, value) 33 | 34 | class DieselFlask(Flask): 35 | def __init__(self, name, *args, **kw): 36 | self.jobs = [] 37 | self.diesel_app = self.make_application() 38 | Flask.__init__(self, name, *args, **kw) 39 | 40 | use_x_sendfile = True 41 | 42 | def request_class(self, environ): 43 | return environ # `id` -- environ IS the existing request. no need to make another 44 | 45 | @classmethod 46 | def make_application(cls): 47 | return Application() 48 | 49 | def make_logger(self, level): 50 | # Flask expects a _logger attribute which we set here. 51 | self._logger = _FlaskTwiggyLogProxy(self.logger_name) 52 | self._logger.min_level = level 53 | 54 | def log_exception(self, exc_info): 55 | """A replacement for Flask's default. 56 | 57 | The default passed an exc_info parameter to logger.error(), which 58 | diesel doesn't support. 59 | 60 | """ 61 | self._logger.trace().error('Exception on {0} [{1}]', 62 | request.path, 63 | request.method 64 | ) 65 | 66 | def schedule(self, *args): 67 | self.jobs.append(args) 68 | 69 | def handle_request(self, req): 70 | with self.request_context(req): 71 | try: 72 | response = self.full_dispatch_request() 73 | except Exception, e: 74 | self.log_exception(e) 75 | try: 76 | response = self.make_response(self.handle_exception(e)) 77 | except: 78 | tb = tbtools.get_current_traceback(skip=1) 79 | response = Response(tb.render_summary(), headers={'Content-Type' : 'text/html'}) 80 | 81 | return response 82 | 83 | def make_service(self, port=8080, iface='', verbosity=loglevels.DEBUG, debug=True): 84 | self.make_logger(verbosity) 85 | if debug: 86 | self.debug = True 87 | 88 | from diesel.protocols.http import HttpServer 89 | http_service = Service(HttpServer(self.handle_request), port, iface) 90 | 91 | return http_service 92 | 93 | def websocket(self, f): 94 | def no_web(req): 95 | assert 0, "Only `Upgrade` HTTP requests on a @websocket" 96 | ws = WebSocketServer(no_web, f) 97 | def ws_call(*args, **kw): 98 | assert not args and not kw, "No arguments allowed to websocket routes" 99 | return ws.do_upgrade(request) 100 | return ws_call 101 | 102 | def run(self, *args, **params): 103 | http_service = self.make_service(*args, **params) 104 | self.schedule(http_service) 105 | quickstart(*self.jobs, __app=self.diesel_app) 106 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | # the i18n builder cannot share the environment and doctrees with the others 15 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 16 | 17 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 18 | 19 | help: 20 | @echo "Please use \`make ' where is one of" 21 | @echo " html to make standalone HTML files" 22 | @echo " dirhtml to make HTML files named index.html in directories" 23 | @echo " singlehtml to make a single large HTML file" 24 | @echo " pickle to make pickle files" 25 | @echo " json to make JSON files" 26 | @echo " htmlhelp to make HTML files and a HTML help project" 27 | @echo " qthelp to make HTML files and a qthelp project" 28 | @echo " devhelp to make HTML files and a Devhelp project" 29 | @echo " epub to make an epub" 30 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 31 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 32 | @echo " text to make text files" 33 | @echo " man to make manual pages" 34 | @echo " texinfo to make Texinfo files" 35 | @echo " info to make Texinfo files and run them through makeinfo" 36 | @echo " gettext to make PO message catalogs" 37 | @echo " changes to make an overview of all changed/added/deprecated items" 38 | @echo " linkcheck to check all external links for integrity" 39 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 40 | 41 | clean: 42 | -rm -rf $(BUILDDIR)/* 43 | 44 | html: 45 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 46 | @echo 47 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 48 | 49 | dirhtml: 50 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 51 | @echo 52 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 53 | 54 | singlehtml: 55 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 56 | @echo 57 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 58 | 59 | pickle: 60 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 61 | @echo 62 | @echo "Build finished; now you can process the pickle files." 63 | 64 | json: 65 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 66 | @echo 67 | @echo "Build finished; now you can process the JSON files." 68 | 69 | htmlhelp: 70 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 71 | @echo 72 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 73 | ".hhp project file in $(BUILDDIR)/htmlhelp." 74 | 75 | qthelp: 76 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 77 | @echo 78 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 79 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 80 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Diesel.qhcp" 81 | @echo "To view the help file:" 82 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Diesel.qhc" 83 | 84 | devhelp: 85 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 86 | @echo 87 | @echo "Build finished." 88 | @echo "To view the help file:" 89 | @echo "# mkdir -p $$HOME/.local/share/devhelp/Diesel" 90 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Diesel" 91 | @echo "# devhelp" 92 | 93 | epub: 94 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 95 | @echo 96 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 97 | 98 | latex: 99 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 100 | @echo 101 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 102 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 103 | "(use \`make latexpdf' here to do that automatically)." 104 | 105 | latexpdf: 106 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 107 | @echo "Running LaTeX files through pdflatex..." 108 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 109 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 110 | 111 | text: 112 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 113 | @echo 114 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 115 | 116 | man: 117 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 118 | @echo 119 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 120 | 121 | texinfo: 122 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 123 | @echo 124 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 125 | @echo "Run \`make' in that directory to run these through makeinfo" \ 126 | "(use \`make info' here to do that automatically)." 127 | 128 | info: 129 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 130 | @echo "Running Texinfo files through makeinfo..." 131 | make -C $(BUILDDIR)/texinfo info 132 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 133 | 134 | gettext: 135 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 136 | @echo 137 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 138 | 139 | changes: 140 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 141 | @echo 142 | @echo "The overview file is in $(BUILDDIR)/changes." 143 | 144 | linkcheck: 145 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 146 | @echo 147 | @echo "Link check complete; look for any errors in the above output " \ 148 | "or in $(BUILDDIR)/linkcheck/output.txt." 149 | 150 | doctest: 151 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 152 | @echo "Testing of doctests in the sources finished, look at the " \ 153 | "results in $(BUILDDIR)/doctest/output.txt." 154 | -------------------------------------------------------------------------------- /doc/_build/.empty: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dieseldev/diesel/8d48371fce0b79d6631053594bce06e4b9628499/doc/_build/.empty -------------------------------------------------------------------------------- /doc/_static/.empty: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dieseldev/diesel/8d48371fce0b79d6631053594bce06e4b9628499/doc/_static/.empty -------------------------------------------------------------------------------- /doc/_templates/.empty: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dieseldev/diesel/8d48371fce0b79d6631053594bce06e4b9628499/doc/_templates/.empty -------------------------------------------------------------------------------- /doc/api/diesel.protocols.http.rst: -------------------------------------------------------------------------------- 1 | http Package 2 | ============ 3 | 4 | :mod:`http` Package 5 | ------------------- 6 | 7 | .. automodule:: diesel.protocols.http 8 | :members: HttpServer, HttpClient, HttpsClient, HttpRequestTimeout 9 | :undoc-members: 10 | :show-inheritance: 11 | 12 | :mod:`pool` Module 13 | ------------------ 14 | 15 | .. automodule:: diesel.protocols.http.pool 16 | :members: 17 | :undoc-members: 18 | :show-inheritance: 19 | 20 | -------------------------------------------------------------------------------- /doc/api/diesel.protocols.rst: -------------------------------------------------------------------------------- 1 | :mod:`protocols` Package 2 | ======================== 3 | 4 | .. automodule:: diesel.protocols 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | 9 | :mod:`DNS` Module 10 | ----------------- 11 | 12 | .. automodule:: diesel.protocols.DNS 13 | :members: 14 | :undoc-members: 15 | :show-inheritance: 16 | 17 | :mod:`dreadlock` Module 18 | ----------------------- 19 | 20 | .. automodule:: diesel.protocols.dreadlock 21 | :members: 22 | :undoc-members: 23 | :show-inheritance: 24 | 25 | :mod:`irc` Module 26 | ----------------- 27 | 28 | .. automodule:: diesel.protocols.irc 29 | :members: 30 | :undoc-members: 31 | :show-inheritance: 32 | 33 | :mod:`mongodb` Module 34 | --------------------- 35 | 36 | .. automodule:: diesel.protocols.mongodb 37 | :members: 38 | :undoc-members: 39 | :show-inheritance: 40 | 41 | :mod:`nitro` Module 42 | ------------------- 43 | 44 | .. automodule:: diesel.protocols.nitro 45 | :members: 46 | :undoc-members: 47 | :show-inheritance: 48 | 49 | :mod:`pg` Module 50 | ---------------- 51 | 52 | .. automodule:: diesel.protocols.pg 53 | :members: 54 | :undoc-members: 55 | :show-inheritance: 56 | 57 | :mod:`redis` Module 58 | ------------------- 59 | 60 | .. automodule:: diesel.protocols.redis 61 | :members: 62 | :undoc-members: 63 | :show-inheritance: 64 | 65 | :mod:`riak` Module 66 | ------------------ 67 | 68 | .. automodule:: diesel.protocols.riak 69 | :members: 70 | :undoc-members: 71 | :show-inheritance: 72 | 73 | :mod:`websockets` Module 74 | ------------------------ 75 | 76 | .. automodule:: diesel.protocols.websockets 77 | :members: 78 | :undoc-members: 79 | :show-inheritance: 80 | 81 | :mod:`wsgi` Module 82 | ------------------ 83 | 84 | .. automodule:: diesel.protocols.wsgi 85 | :members: 86 | :undoc-members: 87 | :show-inheritance: 88 | 89 | :mod:`zeromq` Module 90 | -------------------- 91 | 92 | .. automodule:: diesel.protocols.zeromq 93 | :members: 94 | :undoc-members: 95 | :show-inheritance: 96 | 97 | Subpackages 98 | ----------- 99 | 100 | .. toctree:: 101 | 102 | diesel.protocols.http 103 | 104 | -------------------------------------------------------------------------------- /doc/api/diesel.rst: -------------------------------------------------------------------------------- 1 | diesel Package 2 | ============== 3 | 4 | :mod:`diesel` Package 5 | --------------------- 6 | 7 | .. automodule:: diesel 8 | :members: Application, Loop, Service, UDPService, quickstart, quickstop, Client, UDPClient, sleep, wait, fire, until_eol, send, receive, call, first, fork, fork_child, thread, until, label, fork_from_thread, loglevels, set_log_level, Connection, UDPSocket, ConnectionClosed, ClientConnectionClosed, ParentDiedException, ClientConnectionError, TerminateLoop, datagram, resolve_dns_name, DNSResolutionError 9 | :undoc-members: 10 | :show-inheritance: 11 | 12 | :mod:`web` Module 13 | ----------------- 14 | 15 | .. automodule:: diesel.web 16 | :members: 17 | :undoc-members: 18 | :show-inheritance: 19 | 20 | Protocols and Utilities 21 | ----------------------- 22 | 23 | .. toctree:: 24 | 25 | diesel.protocols 26 | diesel.util 27 | 28 | -------------------------------------------------------------------------------- /doc/api/diesel.util.rst: -------------------------------------------------------------------------------- 1 | util Package 2 | ============ 3 | 4 | :mod:`debugtools` Module 5 | ------------------------ 6 | 7 | .. automodule:: diesel.util.debugtools 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | 12 | :mod:`event` Module 13 | ------------------- 14 | 15 | .. automodule:: diesel.util.event 16 | :members: 17 | :undoc-members: 18 | :show-inheritance: 19 | 20 | :mod:`lock` Module 21 | ------------------ 22 | 23 | .. automodule:: diesel.util.lock 24 | :members: 25 | :undoc-members: 26 | :show-inheritance: 27 | 28 | :mod:`pool` Module 29 | ------------------ 30 | 31 | .. automodule:: diesel.util.pool 32 | :members: 33 | :undoc-members: 34 | :show-inheritance: 35 | 36 | :mod:`process` Module 37 | --------------------- 38 | 39 | .. automodule:: diesel.util.process 40 | :members: 41 | :undoc-members: 42 | :show-inheritance: 43 | 44 | :mod:`queue` Module 45 | ------------------- 46 | 47 | .. automodule:: diesel.util.queue 48 | :members: 49 | :undoc-members: 50 | :show-inheritance: 51 | 52 | :mod:`stats` Module 53 | ------------------- 54 | 55 | .. automodule:: diesel.util.stats 56 | :members: 57 | :undoc-members: 58 | :show-inheritance: 59 | 60 | :mod:`streams` Module 61 | --------------------- 62 | 63 | .. automodule:: diesel.util.streams 64 | :members: 65 | :undoc-members: 66 | :show-inheritance: 67 | 68 | -------------------------------------------------------------------------------- /doc/api/modules.rst: -------------------------------------------------------------------------------- 1 | diesel 2 | ====== 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | diesel 8 | -------------------------------------------------------------------------------- /doc/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Diesel documentation build configuration file, created by 4 | # sphinx-quickstart on Wed Mar 13 21:22:42 2013. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | import sys, os 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | #sys.path.insert(0, os.path.abspath('.')) 20 | 21 | # -- General configuration ----------------------------------------------------- 22 | 23 | # If your documentation needs a minimal Sphinx version, state it here. 24 | #needs_sphinx = '1.0' 25 | 26 | # Add any Sphinx extension module names here, as strings. They can be extensions 27 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 28 | extensions = ['sphinx.ext.autodoc'] 29 | 30 | # Add any paths that contain templates here, relative to this directory. 31 | templates_path = ['_templates'] 32 | 33 | # The suffix of source filenames. 34 | source_suffix = '.rst' 35 | 36 | # The encoding of source files. 37 | #source_encoding = 'utf-8-sig' 38 | 39 | # The master toctree document. 40 | master_doc = 'index' 41 | 42 | # General information about the project. 43 | project = u'Diesel' 44 | copyright = u'2013, Jamie Turner' 45 | 46 | # The version info for the project you're documenting, acts as replacement for 47 | # |version| and |release|, also used in various other places throughout the 48 | # built documents. 49 | # 50 | # The short X.Y version. 51 | version = '3.0' 52 | # The full version, including alpha/beta/rc tags. 53 | release = '3.0' 54 | 55 | # The language for content autogenerated by Sphinx. Refer to documentation 56 | # for a list of supported languages. 57 | #language = None 58 | 59 | # There are two options for replacing |today|: either, you set today to some 60 | # non-false value, then it is used: 61 | #today = '' 62 | # Else, today_fmt is used as the format for a strftime call. 63 | #today_fmt = '%B %d, %Y' 64 | 65 | # List of patterns, relative to source directory, that match files and 66 | # directories to ignore when looking for source files. 67 | exclude_patterns = ['_build'] 68 | 69 | # The reST default role (used for this markup: `text`) to use for all documents. 70 | #default_role = None 71 | 72 | # If true, '()' will be appended to :func: etc. cross-reference text. 73 | #add_function_parentheses = True 74 | 75 | # If true, the current module name will be prepended to all description 76 | # unit titles (such as .. function::). 77 | #add_module_names = True 78 | 79 | # If true, sectionauthor and moduleauthor directives will be shown in the 80 | # output. They are ignored by default. 81 | #show_authors = False 82 | 83 | # The name of the Pygments (syntax highlighting) style to use. 84 | pygments_style = 'sphinx' 85 | 86 | # A list of ignored prefixes for module index sorting. 87 | #modindex_common_prefix = [] 88 | 89 | 90 | # -- Options for HTML output --------------------------------------------------- 91 | 92 | # The theme to use for HTML and HTML Help pages. See the documentation for 93 | # a list of builtin themes. 94 | html_theme = 'default' 95 | 96 | # Theme options are theme-specific and customize the look and feel of a theme 97 | # further. For a list of options available for each theme, see the 98 | # documentation. 99 | #html_theme_options = {} 100 | 101 | # Add any paths that contain custom themes here, relative to this directory. 102 | #html_theme_path = [] 103 | 104 | # The name for this set of Sphinx documents. If None, it defaults to 105 | # " v documentation". 106 | #html_title = None 107 | 108 | # A shorter title for the navigation bar. Default is the same as html_title. 109 | #html_short_title = None 110 | 111 | # The name of an image file (relative to this directory) to place at the top 112 | # of the sidebar. 113 | #html_logo = None 114 | 115 | # The name of an image file (within the static path) to use as favicon of the 116 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 117 | # pixels large. 118 | #html_favicon = None 119 | 120 | # Add any paths that contain custom static files (such as style sheets) here, 121 | # relative to this directory. They are copied after the builtin static files, 122 | # so a file named "default.css" will overwrite the builtin "default.css". 123 | html_static_path = ['_static'] 124 | 125 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 126 | # using the given strftime format. 127 | #html_last_updated_fmt = '%b %d, %Y' 128 | 129 | # If true, SmartyPants will be used to convert quotes and dashes to 130 | # typographically correct entities. 131 | #html_use_smartypants = True 132 | 133 | # Custom sidebar templates, maps document names to template names. 134 | #html_sidebars = {} 135 | 136 | # Additional templates that should be rendered to pages, maps page names to 137 | # template names. 138 | #html_additional_pages = {} 139 | 140 | # If false, no module index is generated. 141 | #html_domain_indices = True 142 | 143 | # If false, no index is generated. 144 | #html_use_index = True 145 | 146 | # If true, the index is split into individual pages for each letter. 147 | #html_split_index = False 148 | 149 | # If true, links to the reST sources are added to the pages. 150 | #html_show_sourcelink = True 151 | 152 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 153 | #html_show_sphinx = True 154 | 155 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 156 | #html_show_copyright = True 157 | 158 | # If true, an OpenSearch description file will be output, and all pages will 159 | # contain a tag referring to it. The value of this option must be the 160 | # base URL from which the finished HTML is served. 161 | #html_use_opensearch = '' 162 | 163 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 164 | #html_file_suffix = None 165 | 166 | # Output file base name for HTML help builder. 167 | htmlhelp_basename = 'Dieseldoc' 168 | 169 | 170 | # -- Options for LaTeX output -------------------------------------------------- 171 | 172 | latex_elements = { 173 | # The paper size ('letterpaper' or 'a4paper'). 174 | #'papersize': 'letterpaper', 175 | 176 | # The font size ('10pt', '11pt' or '12pt'). 177 | #'pointsize': '10pt', 178 | 179 | # Additional stuff for the LaTeX preamble. 180 | #'preamble': '', 181 | } 182 | 183 | # Grouping the document tree into LaTeX files. List of tuples 184 | # (source start file, target name, title, author, documentclass [howto/manual]). 185 | latex_documents = [ 186 | ('index', 'Diesel.tex', u'Diesel Documentation', 187 | u'Jamie Turner', 'manual'), 188 | ] 189 | 190 | # The name of an image file (relative to this directory) to place at the top of 191 | # the title page. 192 | #latex_logo = None 193 | 194 | # For "manual" documents, if this is true, then toplevel headings are parts, 195 | # not chapters. 196 | #latex_use_parts = False 197 | 198 | # If true, show page references after internal links. 199 | #latex_show_pagerefs = False 200 | 201 | # If true, show URL addresses after external links. 202 | #latex_show_urls = False 203 | 204 | # Documents to append as an appendix to all manuals. 205 | #latex_appendices = [] 206 | 207 | # If false, no module index is generated. 208 | #latex_domain_indices = True 209 | 210 | 211 | # -- Options for manual page output -------------------------------------------- 212 | 213 | # One entry per manual page. List of tuples 214 | # (source start file, name, description, authors, manual section). 215 | man_pages = [ 216 | ('index', 'diesel', u'Diesel Documentation', 217 | [u'Jamie Turner'], 1) 218 | ] 219 | 220 | # If true, show URL addresses after external links. 221 | #man_show_urls = False 222 | 223 | 224 | # -- Options for Texinfo output ------------------------------------------------ 225 | 226 | # Grouping the document tree into Texinfo files. List of tuples 227 | # (source start file, target name, title, author, 228 | # dir menu entry, description, category) 229 | texinfo_documents = [ 230 | ('index', 'Diesel', u'Diesel Documentation', 231 | u'Jamie Turner', 'Diesel', 'One line description of project.', 232 | 'Miscellaneous'), 233 | ] 234 | 235 | # Documents to append as an appendix to all manuals. 236 | #texinfo_appendices = [] 237 | 238 | # If false, no module index is generated. 239 | #texinfo_domain_indices = True 240 | 241 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 242 | #texinfo_show_urls = 'footnote' 243 | -------------------------------------------------------------------------------- /doc/debugging.rst: -------------------------------------------------------------------------------- 1 | Introspecting & Debugging Diesel Applications 2 | ============================================= 3 | 4 | Interactive Interpreter 5 | ----------------------- 6 | 7 | Python developers are used to opening the interactive interpreter to try 8 | various ideas and operations. However, most diesel functionality counts on a 9 | running event loop which you don't have by default when you type ``python`` and 10 | start mashing away at the ``>>>`` prompt. The experience is disappointing. 11 | 12 | :: 13 | 14 | $ python 15 | >>> from diesel.protocols.http import HttpsClient 16 | >>> c = HttpsClient('www.google.com', 443) 17 | Traceback (most recent call last): 18 | File "", line 1, in 19 | File "/home/christian/src/bump/server/contrib/diesel/diesel/protocols/http.py", line 262, in __init__ 20 | HttpClient.__init__(self, *args, **kw) 21 | 22 | [SNIP LOTS OF TRACEBACK] 23 | 24 | File "/home/christian/src/bump/server/contrib/diesel/diesel/core.py", line 92, in fire 25 | return current_loop.fire(*args, **kw) 26 | AttributeError: 'NoneType' object has no attribute 'fire' 27 | 28 | diesel includes a command called ``dpython`` that solves this problem. It uses a 29 | few modules from the standard library to provide an interactive interpreter 30 | inside of a running diesel event loop. 31 | 32 | :: 33 | 34 | $ dpython 35 | >>> from diesel.protocols.http import HttpsClient 36 | >>> c = HttpsClient('google.com', 443)>>> resp = c.request('GET', '/', {'Host':'www.google.com'}) 37 | >>> resp 38 | 39 | >>> 40 | 41 | If you prefer IPython and have it installed, you can use ``idpython`` and get a 42 | similar experience. 43 | 44 | Live Console 45 | ------------ 46 | 47 | Sometimes it is useful to do deep investigation into running processes. diesel 48 | supports doing this via the ``dconsole`` command. 49 | 50 | The console feature is enabled when you install a special signal handler in 51 | your application using ``diesel.console.install_console_signal_handler``. The 52 | signal handler enables the console feature without introducing lots of 53 | complexity into the application. There is no need for the application to listen 54 | for TCP connections on its own or to do anything out of the ordinary for 55 | 99.9999% of the time when it is not being inspected via ``dconsole``. 56 | 57 | When you run ``dconsole`` command, the following happens: 58 | 59 | 1. It opens a socket listening on a configurable port on ``localhost``. 60 | 2. It sends a ``SIGTRAP`` signal to the ``PID`` specified on the command line. 61 | 62 | Meanwhile, your application handles that signal by opening a client connection 63 | to the signaling ``dconsole`` process. Once the connection is established, you 64 | are presented with the familiar ``>>>`` Python interactive prompt and can 65 | freely investigate the running process. 66 | 67 | To try the feature out, open two shells. In the first, run:: 68 | 69 | $ dconsole dummy 70 | PID 28908 71 | [2013/03/20 04:26:38] {diesel} WARNING:Starting diesel 72 | [2013/03/20 04:26:38] {dummy} INFO:sleeping 73 | [2013/03/20 04:26:43] {dummy} INFO:sleeping 74 | [2013/03/20 04:26:48] {dummy} INFO:sleeping 75 | 76 | That incarnation of the command launches a "dummy" application and prints the 77 | ``PID`` so you can easily inspect it in another terminal:: 78 | 79 | $ dconsole 28908 80 | [2013/03/20 04:28:46] {diesel} WARNING:Starting diesel 81 | Remote console PID=28908 82 | >>> import os 83 | >>> os.getpid() 84 | 28908 85 | >>> import gc 86 | >>> gc.get_count() 87 | (184, 8, 2) 88 | 89 | The console access is also logged in the application's own log:: 90 | 91 | [2013/03/20 04:28:43] {dummy} INFO:sleeping 92 | [2013/03/20 04:28:47] {diesel} WARNING:Connected to local console 93 | [2013/03/20 04:28:48] {dummy} INFO:sleeping 94 | 95 | When you land at the console prompt, a module called ``debugtools`` is in the 96 | current namespace. It contains a function called ``print_greenlet_stacks``. It 97 | dumps out a list of stack traces for all currently running greenlets. This is 98 | useful to see if your application is blocked up in any particular place and to 99 | see how many of what type of greenlets are running, amongst others. 100 | 101 | You can also import your own modules and libraries to inspect your particular 102 | application environment. 103 | -------------------------------------------------------------------------------- /doc/firststeps.rst: -------------------------------------------------------------------------------- 1 | First Steps with Diesel 2 | ======================= 3 | 4 | To learn the basics of how to install `diesel`, `diesel` terminology and 5 | components, and writing basic `diesel` programs, head on over to 6 | `the tutorial `_. 7 | 8 | Then report back here for more depth. 9 | -------------------------------------------------------------------------------- /doc/index.rst: -------------------------------------------------------------------------------- 1 | .. Diesel documentation master file, created by 2 | sphinx-quickstart on Wed Mar 13 21:22:42 2013. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Diesel - Scalable Networking for Python 7 | ======================================= 8 | 9 | Contents: 10 | 11 | .. toctree:: 12 | :maxdepth: 2 13 | 14 | intro 15 | firststeps 16 | patterns 17 | debugging 18 | testing 19 | API Reference 20 | 21 | Indices and tables 22 | ================== 23 | 24 | * :ref:`genindex` 25 | * :ref:`modindex` 26 | * :ref:`search` 27 | 28 | -------------------------------------------------------------------------------- /doc/intro.rst: -------------------------------------------------------------------------------- 1 | Introduction 2 | ============ 3 | 4 | `diesel` is a set of libraries used for making networked systems in Python. 5 | Networked systems are applications or collections of applications that 6 | communicate across networks in order to achieve some goal using the collective 7 | resources of the network's machines; typically, the productions of 8 | these systems are also made avaiable to various users via the network. 9 | 10 | It is heavily influenced by the design of Erlang/OTP. It employs a 11 | "green thread" approach that uses coroutines to abstract away 12 | asynchronous (or "evented") I/O from the network programmer. It 13 | also encourages heavy use of queues and message-passing for component 14 | interfaces. 15 | 16 | Goals 17 | ----- 18 | 19 | * Provide useful, intuitive interconnections between applications on 20 | hosts to facilitate component-oriented design 21 | * Provide primitives for building robust systems; that is, systems in which 22 | some subset of components can fail and the larger operation of the system 23 | appears uninterrupted to users 24 | * Support high-concurreny and scalable systems that can support many users 25 | per CPU core and per GB of RAM, but *not at the expense of code clarity* 26 | * Minimize the amount of code the network programmer must write to 27 | achieve these goals, since `code == bugs` 28 | * Be somewhat opinionated about how best to achieve the above goals, 29 | to guide network programmers toward best practices 30 | 31 | Who is it for? 32 | -------------- 33 | 34 | * People with a network of computers to apply to a problem 35 | * Programmers who need to support many thousands of users and scale 36 | out horizontally 37 | * Programmers with large-scale service complexity, many components that work 38 | on many machines with differing roles and interfaces, managed by 39 | different teams 40 | * Those who need to maximize uptime by minimizing single-points-of-failure 41 | 42 | Who isn't it for? 43 | ----------------- 44 | 45 | * Someone building the first version of a simple website that needs to 46 | support only a few users (overcoming the learning curve is unlikely 47 | to be worth it vs. more familiar frameworks) 48 | * Someone with heavily reliance on existing blocking I/O libraries 49 | (typically, ones which communicate with a particular database 50 | system) 51 | * Numeric programming, or other computationally intensive jobs 52 | (cooperative multitasking does poorly here) 53 | * Someone that wants to just write networked systems "the way [they] 54 | always have" rather than have a framework impose new ideas and 55 | patterns 56 | -------------------------------------------------------------------------------- /doc/testing.rst: -------------------------------------------------------------------------------- 1 | Testing Diesel Applications 2 | =========================== 3 | 4 | dnosetests 5 | -------------------------------------------------------------------------------- /examples/chat-redis.py: -------------------------------------------------------------------------------- 1 | import time, cgi 2 | 3 | from diesel import Service, Application, sleep, first, Loop 4 | 5 | from diesel.web import DieselFlask 6 | from diesel.protocols.websockets import WebSocketDisconnect 7 | 8 | from diesel.util.queue import Fanout 9 | from diesel.protocols.redis import RedisSubHub, RedisClient 10 | from simplejson import dumps, loads, JSONDecodeError 11 | 12 | 13 | 14 | content = ''' 15 | 16 | 17 | 18 | 50 | 51 | 73 | 74 | 75 | 76 | 77 |

Diesel WebSocket Chat

78 | 79 |
80 | Nick:    81 | Message:    82 | 83 |
84 | 85 |
86 |
87 | 88 | 89 | 90 | ''' 91 | 92 | 93 | #f = Fanout() 94 | 95 | app = DieselFlask(__name__) 96 | 97 | hub = RedisSubHub(host="localhost") 98 | 99 | @app.route("/") 100 | def web_handler(): 101 | return content 102 | 103 | @app.route("/ws") 104 | @app.websocket 105 | def pubsub_socket(req, inq, outq): 106 | c = hub.make_client() 107 | with hub.subq('foo') as group: 108 | while True: 109 | q, v = first(waits=[inq, group]) 110 | if q == inq: # getting message from client 111 | print "(inq) %s" % v 112 | cmd = v.get("cmd", "") 113 | if cmd=="": 114 | print "published message to %i subscribers" % c.publish("foo", dumps({ 115 | 'nick' : cgi.escape(v['nick'].strip()), 116 | 'message' : cgi.escape(v['message'].strip()), 117 | })) 118 | else: 119 | outq.put(dict(message="test bot")) 120 | elif q == group: # getting message for broadcasting 121 | chan, msg_str = v 122 | try: 123 | msg = loads(msg_str) 124 | data = dict(message=msg['message'], nick=msg['nick']) 125 | print "(outq) %s" % data 126 | outq.put(data) 127 | except JSONDecodeError: 128 | print "error decoding message %s" % msg_str 129 | elif isinstance(v, WebSocketDisconnect): # getting a disconnect signal 130 | return 131 | else: 132 | print "oops %s" % v 133 | 134 | app.diesel_app.add_loop(Loop(hub)) 135 | app.run() 136 | 137 | -------------------------------------------------------------------------------- /examples/chat.py: -------------------------------------------------------------------------------- 1 | import time, cgi 2 | 3 | from diesel import Service, Application, sleep, first 4 | from diesel.web import DieselFlask 5 | from diesel.protocols.websockets import WebSocketDisconnect 6 | from diesel.util.queue import Fanout 7 | 8 | app = DieselFlask(__name__) 9 | 10 | content = ''' 11 | 12 | 13 | 14 | 46 | 47 | 69 | 70 | 71 | 72 | 73 |

Diesel WebSocket Chat

74 | 75 |
76 | Nick:    77 | Message:    78 | 79 |
80 | 81 |
82 |
83 | 84 | 85 | 86 | ''' 87 | 88 | 89 | f = Fanout() 90 | 91 | @app.route("/") 92 | def web_handler(): 93 | return content 94 | 95 | @app.route("/ws") 96 | @app.websocket 97 | def socket_handler(req, inq, outq): 98 | with f.sub() as group: 99 | while True: 100 | q, v = first(waits=[inq, group]) 101 | if q == group: 102 | outq.put(dict(message=v['message'], nick=v['nick'])) 103 | elif isinstance(v, WebSocketDisconnect): 104 | return 105 | elif v.get('nick', '').strip() and v.get('message', '').strip(): 106 | f.pub({ 107 | 'nick' : cgi.escape(v['nick'].strip()), 108 | 'message' : cgi.escape(v['message'].strip()), 109 | }) 110 | 111 | app.run() 112 | -------------------------------------------------------------------------------- /examples/child_test.py: -------------------------------------------------------------------------------- 1 | from diesel import fork_child, sleep, ParentDiedException, quickstart, quickstop 2 | 3 | def end_the_app_when_my_parent_dies(): 4 | try: 5 | while True: 6 | print "child: weeee!" 7 | sleep(1) 8 | except ParentDiedException: 9 | print "child: ack, woe is me, I'm an orphan. goodbye cruel world" 10 | quickstop() 11 | 12 | 13 | def parent(): 14 | print "parent: okay, parent here." 15 | sleep(1) 16 | print "parent: I'm so excited, about to become a parent" 17 | sleep(1) 18 | fork_child(end_the_app_when_my_parent_dies) 19 | sleep(1) 20 | print "parent: and, there he goes. I'm so proud" 21 | sleep(4) 22 | print "parent: okay, I'm outta here" 23 | 24 | 25 | if __name__ == '__main__': 26 | quickstart(parent) 27 | -------------------------------------------------------------------------------- /examples/clocker.py: -------------------------------------------------------------------------------- 1 | import os 2 | from diesel import Loop, fork, Application, sleep 3 | from diesel.util.stats import CPUStats 4 | 5 | def not_always_busy_worker(): 6 | with CPUStats() as stats: 7 | for _ in xrange(12): 8 | for i in xrange(10000000): # do some work to forward cpu seconds 9 | pass 10 | sleep(0.1) # give up control 11 | 12 | print "cpu seconds ", stats.cpu_seconds 13 | 14 | def spawn_busy_workers(): 15 | for _ in xrange(0,3): 16 | fork(not_always_busy_worker) 17 | 18 | a = Application() 19 | a.add_loop(Loop(spawn_busy_workers), track=True) 20 | a.run() 21 | -------------------------------------------------------------------------------- /examples/combined.py: -------------------------------------------------------------------------------- 1 | import time 2 | from diesel import Service, Client, send, quickstart, quickstop 3 | from diesel import until, call, log 4 | 5 | def handle_echo(remote_addr): 6 | while True: 7 | message = until('\r\n') 8 | send("you said: %s" % message) 9 | 10 | class EchoClient(Client): 11 | @call 12 | def echo(self, message): 13 | send(message + '\r\n') 14 | back = until("\r\n") 15 | return back 16 | 17 | log = log.name('echo-system') 18 | 19 | def do_echos(): 20 | client = EchoClient('localhost', 8000) 21 | t = time.time() 22 | for x in xrange(5000): 23 | msg = "hello, world #%s!" % x 24 | echo_result = client.echo(msg) 25 | assert echo_result.strip() == "you said: %s" % msg 26 | log.info('5000 loops in {0:.2f}s', time.time() - t) 27 | quickstop() 28 | 29 | quickstart(Service(handle_echo, port=8000), do_echos) 30 | -------------------------------------------------------------------------------- /examples/combined_tls.py: -------------------------------------------------------------------------------- 1 | from OpenSSL import SSL 2 | import time 3 | from diesel import Service, Client, send, quickstart, quickstop 4 | from diesel import until, call, log 5 | 6 | server_ctx = SSL.Context(SSL.TLSv1_METHOD) 7 | server_ctx.use_privatekey_file('snakeoil-key.pem') 8 | server_ctx.use_certificate_file('snakeoil-cert.pem') 9 | 10 | def handle_echo(remote_addr): 11 | while True: 12 | message = until('\r\n') 13 | send("you said: %s" % message) 14 | 15 | class EchoClient(Client): 16 | @call 17 | def echo(self, message): 18 | send(message + '\r\n') 19 | back = until("\r\n") 20 | return back 21 | 22 | log = log.name('echo-system') 23 | 24 | def do_echos(): 25 | with EchoClient('localhost', 8000, ssl_ctx=SSL.Context(SSL.TLSv1_METHOD)) as client: 26 | t = time.time() 27 | for x in xrange(5000): 28 | msg = "hello, world #%s!" % x 29 | echo_result = client.echo(msg) 30 | assert echo_result.strip() == "you said: %s" % msg 31 | log.info('5000 loops in {0:.2f}s', time.time() - t) 32 | quickstop() 33 | 34 | quickstart(Service(handle_echo, port=8000, ssl_ctx=server_ctx), do_echos) 35 | -------------------------------------------------------------------------------- /examples/consolechat.py: -------------------------------------------------------------------------------- 1 | # vim:ts=4:sw=4:expandtab 2 | '''Simple chat server. 3 | 4 | telnet, type your name, hit enter, then chat. Invite 5 | a friend to do the same. 6 | ''' 7 | import sys 8 | from diesel import ( 9 | Application, Service, until_eol, fire, first, send, Client, call, thread, 10 | fork, Loop, 11 | ) 12 | from diesel.util.queue import Queue 13 | 14 | def chat_server(addr): 15 | my_nick = until_eol().strip() 16 | while True: 17 | evt, data = first(until_eol=True, waits=['chat_message']) 18 | if evt == 'until_eol': 19 | fire('chat_message', (my_nick, data.strip())) 20 | else: 21 | nick, message = data 22 | send("<%s> %s\r\n" % (nick, message)) 23 | 24 | class ChatClient(Client): 25 | def __init__(self, *args, **kw): 26 | Client.__init__(self, *args, **kw) 27 | self.input = Queue() 28 | 29 | def read_chat_message(self, prompt): 30 | msg = raw_input(prompt) 31 | return msg 32 | 33 | def input_handler(self): 34 | nick = thread(self.read_chat_message, "nick: ").strip() 35 | self.nick = nick 36 | self.input.put(nick) 37 | while True: 38 | msg = thread(self.read_chat_message, "").strip() 39 | self.input.put(msg) 40 | 41 | @call 42 | def chat(self): 43 | fork(self.input_handler) 44 | nick = self.input.get() 45 | send("%s\r\n" % nick) 46 | while True: 47 | evt, data = first(until_eol=True, waits=[self.input]) 48 | if evt == "until_eol": 49 | print data.strip() 50 | else: 51 | send("%s\r\n" % data) 52 | 53 | def chat_client(): 54 | with ChatClient('localhost', 8000) as c: 55 | c.chat() 56 | 57 | app = Application() 58 | if sys.argv[1] == "server": 59 | app.add_service(Service(chat_server, 8000)) 60 | elif sys.argv[1] == "client": 61 | app.add_loop(Loop(chat_client)) 62 | else: 63 | print "USAGE: python %s [server|client]" % sys.argv[0] 64 | raise SystemExit(1) 65 | app.run() 66 | -------------------------------------------------------------------------------- /examples/convoy.py: -------------------------------------------------------------------------------- 1 | from diesel.convoy import convoy, ConvoyRole 2 | import kv_palm 3 | convoy.register(kv_palm) 4 | 5 | from kv_palm import ( GetRequest, GetOkay, GetMissing, 6 | SetRequest, SetOkay ) 7 | 8 | class KvNode(ConvoyRole): 9 | limit = 1 10 | def __init__(self): 11 | self.values = {} 12 | ConvoyRole.__init__(self) 13 | 14 | def handle_GetRequest(self, sender, request): 15 | if request.key in self.values: 16 | sender.respond(GetOkay(value=self.values[request.key])) 17 | else: 18 | sender.respond(GetMissing()) 19 | 20 | def handle_SetRequest(self, sender, request): 21 | self.values[request.key] = request.value 22 | sender.respond(SetOkay()) 23 | 24 | def run_sets(): 25 | print "I am here!" 26 | convoy.send(SetRequest(key="foo", value="bar")) 27 | print "I am here 2!" 28 | convoy.send(SetRequest(key="foo", value="bar")) 29 | print "I am here 3!" 30 | print convoy.rpc(GetRequest(key="foo")).single 31 | print "I am here 4!" 32 | 33 | import time 34 | t = time.time() 35 | for x in xrange(5000): 36 | convoy.send(SetRequest(key="foo", value="bar")) 37 | r = convoy.rpc(GetRequest(key="foo")).single 38 | print 5000.0 / (time.time() - t), "/ s" 39 | print '' 40 | print r 41 | 42 | if __name__ == '__main__': 43 | convoy.run_with_nameserver("localhost:11111", ["localhost:11111"], KvNode(), run_sets) 44 | #import cProfile 45 | #cProfile.run('convoy.run_with_nameserver("localhost:11111", ["localhost:11111"], KvNode(), run_sets)') 46 | -------------------------------------------------------------------------------- /examples/crawler.py: -------------------------------------------------------------------------------- 1 | # vim:ts=4:sw=4:expandtab 2 | '''A very simple, flawed web crawler--demonstrates 3 | Clients + Loops 4 | ''' 5 | 6 | import sys, time, re, os 7 | from urlparse import urlparse, urljoin 8 | 9 | url, folder = sys.argv[1:] 10 | 11 | schema, host, path, _, _, _ = urlparse(url) 12 | path = path or '/' 13 | base_dir = path if path.endswith('/') else os.path.dirname(path) 14 | if not base_dir.endswith('/'): 15 | base_dir += '/' 16 | 17 | assert schema == 'http', 'http only' 18 | 19 | from diesel import log as glog, quickstart, quickstop 20 | from diesel.protocols.http import HttpClient 21 | from diesel.util.pool import ThreadPool, ConnectionPool 22 | 23 | CONCURRENCY = 10 # go easy on those apache instances! 24 | 25 | url_exp = re.compile(r'(src|href)="([^"]+)', re.MULTILINE | re.IGNORECASE) 26 | 27 | heads = {'Host' : host} 28 | 29 | def get_links(s): 30 | for mo in url_exp.finditer(s): 31 | lpath = mo.group(2) 32 | if ':' not in lpath and '..' not in lpath: 33 | if lpath.startswith('/'): 34 | yield lpath 35 | else: 36 | yield urljoin(base_dir, lpath) 37 | 38 | conn_pool = ConnectionPool(lambda: HttpClient(host, 80), lambda c: c.close(), pool_size=CONCURRENCY) 39 | 40 | def ensure_dirs(lpath): 41 | def g(lpath): 42 | while len(lpath) > len(folder): 43 | lpath = os.path.dirname(lpath) 44 | yield lpath 45 | for d in reversed(list(g(lpath))): 46 | if not os.path.isdir(d): 47 | os.mkdir(d) 48 | 49 | def write_file(lpath, body): 50 | bytes.append(len(body)) 51 | lpath = (lpath if not lpath.endswith('/') else (lpath + 'index.html')).lstrip('/') 52 | lpath = os.path.join(folder, lpath) 53 | ensure_dirs(lpath) 54 | open(lpath, 'w').write(body) 55 | 56 | def follow_loop(lpath): 57 | log.info(" -> %s" % lpath) 58 | with conn_pool.connection as client: 59 | resp = client.request('GET', lpath, heads) 60 | write_file(lpath, resp.data) 61 | 62 | bytes = [] 63 | count = None 64 | log = glog.name('http-crawler') 65 | 66 | def req_loop(): 67 | global count 68 | 69 | log.info(path) 70 | with conn_pool.connection as client: 71 | resp = client.request('GET', path, heads) 72 | body = resp.data 73 | write_file(path, body) 74 | links = set(get_links(body)) 75 | for l in links: 76 | yield l 77 | count = len(links) + 1 78 | 79 | def stop(): 80 | log.info("Fetched %s files (%s bytes) in %.3fs with concurrency=%s" % (count, sum(bytes), time.time() - t, CONCURRENCY)) 81 | quickstop() 82 | 83 | t = time.time() 84 | 85 | pool = ThreadPool(CONCURRENCY, follow_loop, req_loop().next, stop) 86 | 87 | quickstart(pool) 88 | -------------------------------------------------------------------------------- /examples/dhttpd: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # vim:ts=4:sw=4:expandtab 3 | 4 | '''Very simple static web server. 5 | 6 | Takes a few thttpd-like arguments (-h for help) 7 | ''' 8 | import os 9 | import mimetypes 10 | 11 | from diesel import Application, Service, log 12 | from diesel.protocols import http 13 | from diesel.web import send_from_directory, DieselFlask 14 | 15 | log = log.name("dhttpd") 16 | 17 | BASE = '.' 18 | PORT = 8080 19 | 20 | app = DieselFlask(__file__) 21 | 22 | @app.route("/", methods=["GET"]) 23 | def static_http(file): 24 | log.info("GET {0}", file) 25 | return send_from_directory(BASE, file) 26 | 27 | def run(): 28 | app.run(port=PORT) 29 | 30 | def cli(): 31 | global BASE 32 | global PORT 33 | 34 | from optparse import OptionParser 35 | 36 | parser = OptionParser() 37 | parser.add_option("-d", "--directory", dest="directory", 38 | help="serve files from DIRECTORY", 39 | metavar="DIRECTORY", default=BASE) 40 | parser.add_option("-p", "--port", dest="port", 41 | help="listen on port PORT", 42 | metavar="PORT", default=PORT, type="int") 43 | 44 | (options, args) = parser.parse_args() 45 | 46 | if args: 47 | parser.error("dhttpd takes no positional arguments") 48 | 49 | if options.directory: 50 | BASE = options.directory 51 | if options.port: 52 | PORT = options.port 53 | 54 | run() 55 | 56 | if __name__ == '__main__': 57 | cli() 58 | -------------------------------------------------------------------------------- /examples/dispatch.py: -------------------------------------------------------------------------------- 1 | from diesel.util.queue import Dispatcher 2 | from diesel import quickstart, quickstop, sleep, log 3 | from functools import partial 4 | 5 | d = Dispatcher() 6 | WORKERS = 10 7 | 8 | r = [0] * WORKERS 9 | def worker(x): 10 | with d.accept() as q: 11 | while True: 12 | q.get() 13 | r[x] += 1 14 | 15 | def maker(): 16 | for x in xrange(500000): 17 | d.dispatch(x) 18 | if x % 10000 == 0: 19 | sleep() 20 | log.info("values: {0}", r) 21 | quickstop() 22 | 23 | quickstart(maker, *(partial(worker, x) for x in xrange(WORKERS))) 24 | -------------------------------------------------------------------------------- /examples/dreadlocks.py: -------------------------------------------------------------------------------- 1 | '''Test for client to dreadlock network lock service. 2 | ''' 3 | 4 | import uuid 5 | from diesel import sleep, quickstart 6 | from diesel.protocols.dreadlock import DreadlockService 7 | 8 | locker = DreadlockService('localhost', 6001) 9 | def f(): 10 | with locker.hold("foo", 30): 11 | id = uuid.uuid4() 12 | print "start!", id 13 | sleep(2) 14 | print "end!", id 15 | 16 | quickstart(f, f, f, f, f) 17 | -------------------------------------------------------------------------------- /examples/echo.py: -------------------------------------------------------------------------------- 1 | # vim:ts=4:sw=4:expandtab 2 | '''Simple echo server. 3 | ''' 4 | from diesel import Application, Service, until_eol, send 5 | 6 | def hi_server(addr): 7 | while 1: 8 | inp = until_eol() 9 | if inp.strip() == "quit": 10 | break 11 | send("you said %s" % inp) 12 | 13 | app = Application() 14 | app.add_service(Service(hi_server, 8013)) 15 | app.run() 16 | -------------------------------------------------------------------------------- /examples/event.py: -------------------------------------------------------------------------------- 1 | from diesel.util.event import Event 2 | from diesel import quickstart, quickstop, sleep 3 | 4 | pistol = Event() 5 | 6 | def racer(): 7 | pistol.wait() 8 | print "CHAAARGE!" 9 | 10 | def starter(): 11 | print "Ready..." 12 | sleep(1) 13 | print "Set..." 14 | sleep(1) 15 | print " ~~~~~~~~~ BANG ~~~~~~~~~ " 16 | pistol.set() 17 | sleep(1) 18 | print " ~~~~~~~~~ RACE OVER ~~~~~~~~~ " 19 | quickstop() 20 | 21 | quickstart(starter, [racer for x in xrange(8)]) 22 | -------------------------------------------------------------------------------- /examples/fanout.py: -------------------------------------------------------------------------------- 1 | from diesel import quickstart, quickstop, sleep 2 | from diesel.util.queue import Fanout 3 | from diesel.util.event import Countdown 4 | 5 | LISTENERS = 10 6 | EVENTS = 5 7 | 8 | cd = Countdown(LISTENERS * EVENTS) 9 | 10 | f = Fanout() 11 | 12 | def listener(x): 13 | with f.sub() as q: 14 | while True: 15 | v = q.get() 16 | print '%s <- %s' % (x, v) 17 | cd.tick() 18 | 19 | def teller(): 20 | for x in xrange(EVENTS): 21 | sleep(2) 22 | f.pub(x) 23 | 24 | def killer(): 25 | cd.wait() 26 | quickstop() 27 | 28 | from functools import partial 29 | quickstart(killer, teller, 30 | *[partial(listener, x) for x in xrange(LISTENERS)]) 31 | -------------------------------------------------------------------------------- /examples/fire.py: -------------------------------------------------------------------------------- 1 | # vim:ts=4:sw=4:expandtab 2 | '''Example of event firing. 3 | ''' 4 | import time 5 | import random 6 | from diesel import (quickstart, quickstop, sleep, 7 | fire, wait, log, loglevels, 8 | set_log_level) 9 | 10 | set_log_level(loglevels.DEBUG) 11 | 12 | def gunner(): 13 | x = 1 14 | while True: 15 | fire('bam', x) 16 | x += 1 17 | sleep() 18 | 19 | def sieged(): 20 | t = time.time() 21 | while True: 22 | n = wait('bam') 23 | if n % 10000 == 0: 24 | log.info(str(n)) 25 | if n == 50000: 26 | delt = time.time() - t 27 | log.debug("50,000 messages in {0:.3f}s {1:.1f}/s)", delt, 50000 / delt) 28 | quickstop() 29 | 30 | log = log.name('fire-system') 31 | quickstart(gunner, sieged) 32 | -------------------------------------------------------------------------------- /examples/forker.py: -------------------------------------------------------------------------------- 1 | from diesel import Loop, fork, Application, sleep 2 | 3 | def sleep_and_print(num): 4 | sleep(1) 5 | print num 6 | sleep(1) 7 | a.halt() 8 | 9 | 10 | def forker(): 11 | for x in xrange(5): 12 | fork(sleep_and_print, x) 13 | 14 | a = Application() 15 | a.add_loop(Loop(forker)) 16 | a.run() 17 | -------------------------------------------------------------------------------- /examples/http.py: -------------------------------------------------------------------------------- 1 | # vim:ts=4:sw=4:expandtab 2 | '''The oh-so-canonical "Hello, World!" http server. 3 | ''' 4 | from diesel import Application, Service 5 | from diesel.protocols import http 6 | 7 | def hello_http(req): 8 | return http.Response("Hello, World!") 9 | 10 | app = Application() 11 | app.add_service(Service(http.HttpServer(hello_http), 8088)) 12 | import cProfile 13 | cProfile.run('app.run()') 14 | -------------------------------------------------------------------------------- /examples/http_client.py: -------------------------------------------------------------------------------- 1 | # vim:ts=4:sw=4:expandtab 2 | '''Simple http client example. 3 | 4 | Check out crawler.py for more advanced behaviors involving 5 | many concurrent clients. 6 | ''' 7 | 8 | from diesel import Application, Loop, log, quickstart, quickstop 9 | from diesel.protocols.http import HttpClient 10 | 11 | def req_loop(): 12 | for path in ['/Py-TOC', '/']: 13 | with HttpClient('www.jamwt.com', 80) as client: 14 | heads = {'Host' : 'www.jamwt.com'} 15 | log.info(str(client.request('GET', path, heads))) 16 | quickstop() 17 | 18 | log = log.name('http-client') 19 | quickstart(req_loop) 20 | -------------------------------------------------------------------------------- /examples/http_pool.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from time import time 3 | 4 | from diesel import quickstart, quickstop 5 | from diesel.protocols.http.pool import request 6 | 7 | def f(): 8 | t1 = time() 9 | print request("http://example.iana.org/missing"), 'is missing?' 10 | t2 = time() 11 | print request("http://example.iana.org/missing"), 'is missing?' 12 | t3 = time() 13 | print request("http://example.iana.org/missing"), 'is missing?' 14 | t4 = time() 15 | print request("http://example.iana.org/"), 'is found?' 16 | t5 = time() 17 | 18 | print 'First request should (probably) have been longer (tcp handshake) than subsequent 3 requests:' 19 | reduce(lambda t1, t2: sys.stdout.write("%.4f\n" % (t2 - t1)) or t2, (t1, t2, t3, t4, t5)) 20 | 21 | quickstop() 22 | 23 | quickstart(f) 24 | -------------------------------------------------------------------------------- /examples/keep_alive.py: -------------------------------------------------------------------------------- 1 | from diesel import Application, Loop, sleep 2 | import time 3 | 4 | def restart(): 5 | print "I should restart" 6 | a = b 7 | 8 | a = Application() 9 | a.add_loop(Loop(restart), keep_alive=True) 10 | a.run() 11 | -------------------------------------------------------------------------------- /examples/kv.proto: -------------------------------------------------------------------------------- 1 | message GetRequest { 2 | required string key = 1; 3 | } 4 | 5 | message GetOkay { 6 | required bytes value = 1; 7 | } 8 | 9 | message GetMissing { 10 | } 11 | 12 | message SetRequest { 13 | required string key = 1; 14 | required bytes value = 2; 15 | } 16 | 17 | message SetOkay { 18 | } 19 | -------------------------------------------------------------------------------- /examples/newwait.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | from diesel import quickstart, first, sleep, fork 4 | from diesel.util.queue import Queue 5 | 6 | def fire_random(queues): 7 | while True: 8 | sleep(1) 9 | random.choice(queues).put(None) 10 | 11 | def make_and_wait(): 12 | q1 = Queue() 13 | q2 = Queue() 14 | both = [q1, q2] 15 | 16 | fork(fire_random, both) 17 | 18 | while True: 19 | q, v = first(waits=both) 20 | assert v is None 21 | if q == q1: 22 | print 'q1' 23 | elif q == q2: 24 | print 'q2' 25 | else: 26 | assert 0 27 | 28 | quickstart(make_and_wait) 29 | -------------------------------------------------------------------------------- /examples/nitro.py: -------------------------------------------------------------------------------- 1 | from pynitro import NitroFrame 2 | from diesel.protocols.nitro import DieselNitroSocket 3 | from diesel import quickstart, quickstop 4 | 5 | #loc = "tcp://127.0.0.1:4444" 6 | loc = "inproc://foobar" 7 | 8 | def server(): 9 | with DieselNitroSocket(bind=loc) as sock: 10 | while True: 11 | m = sock.recv() 12 | sock.send(NitroFrame("you said: " + m.data)) 13 | 14 | def client(): 15 | with DieselNitroSocket(connect=loc) as sock: 16 | for x in xrange(100000): 17 | sock.send(NitroFrame("Hello, dude!")) 18 | m = sock.recv() 19 | assert m.data == "you said: Hello, dude!" 20 | 21 | quickstop() 22 | 23 | quickstart(server, client) 24 | -------------------------------------------------------------------------------- /examples/nitro_echo_service.py: -------------------------------------------------------------------------------- 1 | import diesel 2 | from pynitro import NitroFrame 3 | from diesel.protocols.nitro import ( 4 | DieselNitroService, DieselNitroSocket, 5 | ) 6 | import uuid 7 | 8 | NUM_CLIENTS = 300 9 | cids = range(NUM_CLIENTS) 10 | dead = 0 11 | 12 | def echo_client(): 13 | global dead 14 | id = str(uuid.uuid4()) 15 | s = DieselNitroSocket(connect='tcp://127.0.0.1:4321') 16 | for i in xrange(50): 17 | s.send(NitroFrame('%s|m%d' % (id, i))) 18 | r = s.recv() 19 | assert r.data == 'm%d:%d' % (i, i + 1) 20 | dead += 1 21 | print 'done!', dead 22 | 23 | class EchoService(DieselNitroService): 24 | def handle_client_packet(self, packet, ctx): 25 | count = ctx.setdefault('count', 0) + 1 26 | ctx['count'] = count 27 | return '%s:%d' % (packet, count) 28 | 29 | def parse_message(self, raw): 30 | return raw.split('|') 31 | 32 | def cleanup_client(self, client): 33 | print 'client timed out', client.identity 34 | 35 | echo_svc = EchoService('tcp://*:4321') 36 | diesel.quickstart(echo_svc.run, *(echo_client for i in xrange(NUM_CLIENTS))) 37 | 38 | -------------------------------------------------------------------------------- /examples/queue.py: -------------------------------------------------------------------------------- 1 | from diesel.util.queue import Queue, QueueTimeout 2 | from diesel.util.event import Countdown 3 | from diesel import log as glog, sleep, quickstart, quickstop 4 | 5 | q = Queue() 6 | cd = Countdown(4) 7 | 8 | def putter(): 9 | log = glog.name("putter") 10 | 11 | log.info("putting 100000 things on queue") 12 | for x in xrange(100000): 13 | q.put(x) 14 | sleep() 15 | 16 | def getter(): 17 | log = glog.name("getter") 18 | got = 0 19 | while got < 25000: 20 | try: 21 | s = q.get(timeout=3) 22 | sleep() 23 | except QueueTimeout: 24 | log.warning("timeout before getting a value, retrying...") 25 | continue 26 | got += 1 27 | 28 | log.info("SUCCESS! got all 25,000") 29 | cd.tick() 30 | 31 | def manage(): 32 | cd.wait() 33 | quickstop() 34 | 35 | quickstart(manage, putter, [getter for x in xrange(4)]) 36 | -------------------------------------------------------------------------------- /examples/queue_fairness_and_speed.py: -------------------------------------------------------------------------------- 1 | import time 2 | import uuid 3 | 4 | import diesel 5 | import diesel.core 6 | from diesel.util.queue import Queue 7 | 8 | NUM_ITEMS = 100000 9 | NUM_WORKERS = 10 10 | 11 | shutdown = uuid.uuid4().hex 12 | q = Queue() 13 | dones = Queue() 14 | 15 | def worker(): 16 | num_processed = 0 17 | while True: 18 | val = diesel.wait(q) 19 | if val == shutdown: 20 | break 21 | num_processed += 1 22 | fmt_args = (diesel.core.current_loop, num_processed) 23 | print "%s, worker done (processed %d items)" % fmt_args 24 | dones.put('done') 25 | 26 | def main(): 27 | start = time.time() 28 | 29 | for i in xrange(NUM_ITEMS): 30 | q.put('item %d' % i) 31 | for i in xrange(NUM_WORKERS): 32 | q.put(shutdown) 33 | 34 | for i in xrange(NUM_WORKERS): 35 | diesel.fork_child(worker) 36 | for i in xrange(NUM_WORKERS): 37 | dones.get() 38 | 39 | print 'all workers done in %.2f secs' % (time.time() - start) 40 | diesel.quickstop() 41 | 42 | if __name__ == '__main__': 43 | diesel.quickstart(main) 44 | -------------------------------------------------------------------------------- /examples/redis_lock.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | from diesel import fork, quickstop, quickstart, sleep 4 | from diesel.protocols.redis import RedisClient, RedisTransactionError, RedisLock, LockNotAcquired 5 | 6 | 7 | """Implement the Redis INCR command using a lock. Obviously this is inefficient, but it's a good 8 | example of how to use the RedisLock class""" 9 | 10 | key = 'test-lock-key' 11 | incr_key = 'test-incr-key' 12 | counter = 0 13 | 14 | 15 | """If sleep_factor > lock_timeout you are exercising the timeout loop, otherwise, that loop should be a noop""" 16 | lock_timeout = 3 17 | sleep_factor = 1 18 | 19 | 20 | 21 | def take_lock(): 22 | global counter 23 | client = RedisClient('localhost', 6379) 24 | try: 25 | with RedisLock(client, key, timeout=lock_timeout) as lock: 26 | v = client.get(incr_key) 27 | sleep(random.random() * sleep_factor) 28 | client.set(incr_key, int(v) + 1) 29 | counter += 1 30 | except LockNotAcquired: 31 | pass 32 | 33 | def main(): 34 | client = RedisClient('localhost', 6379) 35 | client.delete(key) 36 | client.set(incr_key, 0) 37 | 38 | for _ in xrange(500): 39 | fork(take_lock) 40 | if random.random() > 0.1: 41 | sleep(random.random() / 10) 42 | sleep(2) 43 | assert counter == int(client.get(incr_key)), 'Incr failed!' 44 | quickstop() 45 | 46 | 47 | quickstart(main) 48 | -------------------------------------------------------------------------------- /examples/redispub.py: -------------------------------------------------------------------------------- 1 | # vim:ts=4:sw=4:expandtab 2 | '''Simple RedisSubHub client example. 3 | ''' 4 | 5 | from diesel import Application, Loop, sleep 6 | from diesel.protocols.redis import RedisSubHub, RedisClient 7 | import time, sys 8 | 9 | def send_loop(): 10 | c = RedisClient() 11 | sleep(1) 12 | 13 | print 'SEND S', time.time() 14 | 15 | for x in xrange(500): 16 | c.publish("foo", "bar") 17 | 18 | print 'SEND E', time.time() 19 | 20 | hub = RedisSubHub() 21 | 22 | def recv_loop(): 23 | print 'RECV S', time.time() 24 | with hub.sub('foo') as poll: 25 | for x in xrange(500): 26 | q, content = poll.fetch() 27 | print 'RECV E', time.time() 28 | 29 | a = Application() 30 | a.add_loop(Loop(hub)) # start up the sub loop 31 | if 'send' in sys.argv: 32 | a.add_loop(Loop(send_loop)) 33 | if 'recv' in sys.argv: 34 | a.add_loop(Loop(recv_loop)) 35 | a.add_loop(Loop(recv_loop)) 36 | a.run() 37 | -------------------------------------------------------------------------------- /examples/resolve_names.py: -------------------------------------------------------------------------------- 1 | from diesel import resolve_dns_name, DNSResolutionError 2 | from diesel import Application, Loop, sleep 3 | 4 | def resolve_the_google(): 5 | print 'started resolution!' 6 | g_ip = resolve_dns_name("www.google.com") 7 | print "www.google.com's ip is %s" % g_ip 8 | try: 9 | bad_host = "www.g8asdf21oogle.com" 10 | print "now checking %s" % bad_host 11 | resolve_dns_name(bad_host) 12 | except DNSResolutionError: 13 | print "yep, it failed as expected" 14 | else: 15 | raise RuntimeError("The bad host resolved. That's unexpected.") 16 | g_ip = resolve_dns_name("www.google.com") 17 | g_ip = resolve_dns_name("www.google.com") 18 | g_ip = resolve_dns_name("www.google.com") 19 | g_ip = resolve_dns_name("www.google.com") 20 | a.halt() 21 | 22 | def stuff(): 23 | while True: 24 | print "doing stuff!" 25 | sleep(0.01) 26 | 27 | a = Application() 28 | a.add_loop(Loop(stuff)) 29 | a.add_loop(Loop(resolve_the_google)) 30 | a.run() 31 | -------------------------------------------------------------------------------- /examples/santa.py: -------------------------------------------------------------------------------- 1 | import random 2 | from diesel import Application, Loop, fire, wait, sleep 3 | 4 | deer_group = [] 5 | elf_group = [] 6 | 7 | def santa(): 8 | while True: 9 | deer_ready, elves_ready = len(deer_group) == 9, len(elf_group) == 3 10 | if deer_ready: 11 | work_with_group('deer', deer_group, 'deliver toys') 12 | if elves_ready: 13 | work_with_group('elf', elf_group, 'meet in my study') 14 | sleep(random.random() * 1) 15 | 16 | def actor(name, type, group, task, max_group, max_sleep): 17 | def actor_event_loop(): 18 | while True: 19 | sleep(random.random() * max_sleep) 20 | if len(group) < max_group: 21 | group.append(name) 22 | wait('%s-group-started' % type) 23 | print "%s %s" % (name, task) 24 | wait('%s-group-done' % type) 25 | return actor_event_loop 26 | 27 | def work_with_group(name, group, message): 28 | print "Ho! Ho! Ho! Let's", message 29 | fire('%s-group-started' % name) 30 | sleep(random.random() * 3) 31 | excuse_group(name, group) 32 | 33 | def excuse_group(name, group): 34 | group[:] = [] 35 | fire('%s-group-done' % name, True) 36 | 37 | def main(): 38 | app = Application() 39 | app.add_loop(Loop(santa)) 40 | 41 | elf_do = "meets in study" 42 | for i in xrange(10): 43 | app.add_loop(Loop(actor("Elf %d" % i, 'elf', elf_group, elf_do, 3, 3))) 44 | 45 | deer_do = "delivers toys" 46 | for name in [ 47 | 'Dasher', 'Dancer', 'Prancer', 48 | 'Vixen', 'Comet', 'Cupid', 49 | 'Donner', 'Blitzen', 'Rudolph', 50 | ]: 51 | app.add_loop(Loop(actor(name, 'deer', deer_group, deer_do, 9, 9))) 52 | 53 | app.run() 54 | 55 | if __name__ == '__main__': 56 | main() 57 | -------------------------------------------------------------------------------- /examples/signals.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from signal import SIGUSR1 4 | 5 | import diesel 6 | 7 | from diesel.util.event import Signal 8 | 9 | 10 | log = diesel.log.name('signal-example') 11 | 12 | def main(): 13 | usr1 = Signal(SIGUSR1) 14 | ticks = 0 15 | log.fields(pid=os.getpid()).info('started') 16 | while True: 17 | evt, _ = diesel.first(sleep=1, waits=[usr1]) 18 | if evt == 'sleep': 19 | ticks += 1 20 | elif evt == usr1: 21 | log.fields(ticks=ticks).info('stats') 22 | # must rearm() to use again 23 | evt.rearm() 24 | 25 | diesel.quickstart(main) 26 | -------------------------------------------------------------------------------- /examples/sleep_server.py: -------------------------------------------------------------------------------- 1 | # vim:ts=4:sw=4:expandtab 2 | '''Demonstrate sleep-type behavior server-side. 3 | ''' 4 | from diesel import Application, Service, until_eol, sleep, send 5 | 6 | def delay_echo_server(addr): 7 | inp = until_eol() 8 | 9 | for x in xrange(4): 10 | sleep(2) 11 | send(str(x) + '\r\n') 12 | send("you said %s" % inp) 13 | 14 | app = Application() 15 | app.add_service(Service(delay_echo_server, 8013)) 16 | app.run() 17 | -------------------------------------------------------------------------------- /examples/snakeoil-cert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDWTCCAsKgAwIBAgIJAPGlGeeUYQ+wMA0GCSqGSIb3DQEBBQUAMHwxCzAJBgNV 3 | BAYTAlVTMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQHEw1Nb3VudGFpbiBW 4 | aWV3MRswGQYDVQQKExJEaWVzZWwgU25ha2VvaWwgQ28xIzAhBgkqhkiG9w0BCQEW 5 | FHNuYWtlb2lsQGV4YW1wbGUuY29tMB4XDTEwMDIwMzA0MTAyMFoXDTEzMTAzMDA0 6 | MTAyMFowfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCkNhbGlmb3JuaWExFjAUBgNV 7 | BAcTDU1vdW50YWluIFZpZXcxGzAZBgNVBAoTEkRpZXNlbCBTbmFrZW9pbCBDbzEj 8 | MCEGCSqGSIb3DQEJARYUc25ha2VvaWxAZXhhbXBsZS5jb20wgZ8wDQYJKoZIhvcN 9 | AQEBBQADgY0AMIGJAoGBAMAZVbm24PlSmKNK1BI1aRmGhFEdKwM+af+U+dv9olF2 10 | lqt2QzJtt2oPcpV4iakKbVhil/oMxNZdrMfJpVyrn77+18hL1WTyyI0sZ9PrKMXk 11 | 3atl3HsHX7ouoUflN6gPbomjo+6lyO51cRFm+rm5ShB/k716D8H9WNpW/av9GwHT 12 | AgMBAAGjgeIwgd8wHQYDVR0OBBYEFPVvvi3e5AWBzGbefCTWjRal8xKiMIGvBgNV 13 | HSMEgacwgaSAFPVvvi3e5AWBzGbefCTWjRal8xKioYGApH4wfDELMAkGA1UEBhMC 14 | VVMxEzARBgNVBAgTCkNhbGlmb3JuaWExFjAUBgNVBAcTDU1vdW50YWluIFZpZXcx 15 | GzAZBgNVBAoTEkRpZXNlbCBTbmFrZW9pbCBDbzEjMCEGCSqGSIb3DQEJARYUc25h 16 | a2VvaWxAZXhhbXBsZS5jb22CCQDxpRnnlGEPsDAMBgNVHRMEBTADAQH/MA0GCSqG 17 | SIb3DQEBBQUAA4GBALh5Bf3qZ4BL0HsTC0+koIZqlb13IeA9gtT1vsGqgPyHPSH/ 18 | mKsjaODLqLqufpz9OyaX4meQCwwd2Ax0cx2BZ6xuImSszoKgXmK1WiQN1qhJMrTi 19 | MYjLbXkypLfpP9g33ig+YgoVsq+UC472jIiAC8D3nnxzLBPcDe3LOVB4/t2K 20 | -----END CERTIFICATE----- 21 | -------------------------------------------------------------------------------- /examples/snakeoil-key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIICXAIBAAKBgQDAGVW5tuD5UpijStQSNWkZhoRRHSsDPmn/lPnb/aJRdpardkMy 3 | bbdqD3KVeImpCm1YYpf6DMTWXazHyaVcq5++/tfIS9Vk8siNLGfT6yjF5N2rZdx7 4 | B1+6LqFH5TeoD26Jo6PupcjudXERZvq5uUoQf5O9eg/B/VjaVv2r/RsB0wIDAQAB 5 | AoGBAIGPVMMJtdhSPcI8UKXrQfRGRm2St5TbfpAzQQV/nf9FdT81ZwLW/tJYktZ+ 6 | 0pGhB7iJ3qh1/jf6O/MPbCkBU55HmUJXean3dtkomrud2Ypbqua/e1sDpLc3/Af6 7 | FnLM1pChzJzCzcUQrQcmHwlEjd7OX7DJyYeCG+cwpux+5lKpAkEA+KKArpBCJgbN 8 | SFcovK0++4t4xgSqsn2DrqJDajbK19WNB+2wWZgJUwANxdsZEEcPtx+iAMor2hgT 9 | diOviMQzDQJBAMXKGTN0ieRLpCDm6jQoUmcF7g9K5fU3rqI3z7Yd9l/wzc8bMhYU 10 | 5gwLWOZross+7Y2W4UCmbCHms8YlLGe4UF8CQG7Y9AnfYr5VVfwkb5L+og+/dI7D 11 | 0d5VuvmGegvGddSX7pJUU8T91VpdscY+EgSBye3Yen9jov1Oso5/BmkxH5kCQAKn 12 | 5A680d0u5tVKRHrOz8xyV+/8oXnZdY7YEQHxBQ3kvd66DsIdJbmjrV3qtBTmk2oD 13 | TKMBmRIbdu6CMUZzQyECQAexc664/1pMYUW/NLrcwHc6DVGytjPoOhFeg+x64vHz 14 | Hj5q8LkY24XCp73OVcQYF99b171CuzTzlmvRFqnFy8A= 15 | -----END RSA PRIVATE KEY----- 16 | -------------------------------------------------------------------------------- /examples/stdin.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | from diesel import quickstart, fork_from_thread 4 | from diesel.util.queue import Queue 5 | from thread import start_new_thread 6 | 7 | q = Queue() 8 | 9 | def consume(): 10 | while True: 11 | v = q.get() 12 | print 'DIESEL GOT', v 13 | 14 | def put(line): 15 | q.put(line) 16 | 17 | def create(): 18 | while True: 19 | line = sys.stdin.readline() 20 | print 'iter!', line 21 | fork_from_thread(put, line) 22 | 23 | start_new_thread(create, ()) 24 | quickstart(consume) 25 | -------------------------------------------------------------------------------- /examples/stdin_stream.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | from diesel import quickstart 4 | from diesel.util.streams import create_line_input_stream 5 | 6 | def consume(): 7 | q = create_line_input_stream(sys.stdin) 8 | while True: 9 | v = q.get() 10 | print 'DIESEL GOT', v 11 | 12 | quickstart(consume) 13 | -------------------------------------------------------------------------------- /examples/synchronized.py: -------------------------------------------------------------------------------- 1 | from diesel import Application, Loop, sleep 2 | from diesel.util.lock import synchronized 3 | import random 4 | free = 0 5 | sync = 0 6 | 7 | def free_loop(): 8 | global free 9 | free += 1 10 | sleep(random.random()) 11 | free -= 1 12 | print 'FREE', free 13 | 14 | def sync_loop(): 15 | global sync 16 | with synchronized(): 17 | sync += 1 18 | sleep(random.random()) 19 | sync -= 1 20 | print 'SYNC', sync 21 | 22 | def manage(): 23 | sleep(10) 24 | a.halt() 25 | 26 | a = Application() 27 | for l in (free_loop, sync_loop): 28 | for x in xrange(10): 29 | a.add_loop(Loop(l)) 30 | a.add_loop(Loop(manage)) 31 | a.run() 32 | 33 | -------------------------------------------------------------------------------- /examples/test_dnosetest.py: -------------------------------------------------------------------------------- 1 | """Here is an example test module that can be run with `dnosetests`. 2 | 3 | It is written like a standard nose test module but the test functions are 4 | executed within the diesel event loop. That means they can fork other 5 | green threads, do network I/O and other diesel-ish things. Very handy for 6 | writing integration tests against diesel services. 7 | 8 | """ 9 | import time 10 | 11 | import diesel 12 | 13 | 14 | def test_sleeps_then_passes(): 15 | diesel.sleep(1) 16 | assert True 17 | 18 | def test_sleeps_then_fails(): 19 | diesel.sleep(1) 20 | assert False, "OH NOES!" 21 | -------------------------------------------------------------------------------- /examples/thread.py: -------------------------------------------------------------------------------- 1 | # vim:ts=4:sw=4:expandtab 2 | '''Example of deferring blocking calls to threads 3 | ''' 4 | from diesel import log, thread, quickstart 5 | import time 6 | from functools import partial 7 | 8 | def blocker(taskid, sleep_time): 9 | while True: 10 | def f(): 11 | time.sleep(sleep_time) 12 | thread(f) 13 | log.info('yo! {0} from {1} task', time.time(), taskid) 14 | 15 | quickstart(partial(blocker, 'fast', 1), partial(blocker, 'slow', 5)) 16 | -------------------------------------------------------------------------------- /examples/thread_pool.py: -------------------------------------------------------------------------------- 1 | from diesel import quickstart, sleep, quickstop 2 | from diesel.util.pool import ThreadPool 3 | import random 4 | 5 | def handle_it(i): 6 | print 'S', i 7 | sleep(random.random()) 8 | print 'E', i 9 | 10 | def c(): 11 | for x in xrange(0, 20): 12 | yield x 13 | 14 | make_it = c().next 15 | 16 | def stop_it(): 17 | quickstop() 18 | 19 | threads = ThreadPool(10, handle_it, make_it, stop_it) 20 | 21 | quickstart(threads) 22 | -------------------------------------------------------------------------------- /examples/timer.py: -------------------------------------------------------------------------------- 1 | from diesel import Application, Loop, sleep 2 | import time 3 | 4 | def l(): 5 | for x in xrange(2): 6 | print "hi" 7 | time.sleep(1) 8 | sleep(5) 9 | a.halt() 10 | 11 | a = Application() 12 | a.add_loop(Loop(l)) 13 | a.add_loop(Loop(l)) 14 | a.run() 15 | -------------------------------------------------------------------------------- /examples/timer_bench.py: -------------------------------------------------------------------------------- 1 | """A benchmark for diesel's internal timers. 2 | 3 | Try something like: 4 | 5 | $ python examples/timer_bench.py 10 6 | $ python examples/timer_bench.py 100 7 | $ python examples/timer_bench.py 1000 8 | 9 | The script will output the total time to run with the given number of 10 | producer/consumer pairs and a sample of CPU time while the benchmark was 11 | running. 12 | 13 | """ 14 | import os 15 | import subprocess 16 | import sys 17 | import time 18 | 19 | import diesel 20 | from diesel.util.event import Countdown 21 | from diesel.util.queue import Queue 22 | 23 | OPERATIONS = 60 24 | cpustats = [] 25 | 26 | 27 | def producer(q): 28 | for i in xrange(OPERATIONS): 29 | diesel.sleep(0.5) 30 | q.put(i) 31 | 32 | def consumer(q, done): 33 | for i in xrange(OPERATIONS): 34 | evt, data = diesel.first(waits=[q], sleep=10000) 35 | if evt == "sleep": 36 | print "sleep was triggered!" 37 | break 38 | done.tick() 39 | 40 | def pair(done): 41 | q = Queue() 42 | diesel.fork(producer, q) 43 | diesel.fork(consumer, q, done) 44 | 45 | def track_cpu_stats(): 46 | pid = os.getpid() 47 | def append_stats(): 48 | rawstats = subprocess.Popen(['ps -p %d -f' % pid], shell=True, stdout=subprocess.PIPE).communicate()[0] 49 | header, data = rawstats.split('\n', 1) 50 | procstats = [d for d in data.split(' ') if d] 51 | cpustats.append(int(procstats[3])) 52 | while True: 53 | diesel.sleep(1) 54 | diesel.thread(append_stats) 55 | 56 | def main(): 57 | diesel.fork(track_cpu_stats) 58 | actor_pairs = int(sys.argv[1]) 59 | done = Countdown(actor_pairs) 60 | for i in xrange(actor_pairs): 61 | pair(done) 62 | start = time.time() 63 | done.wait() 64 | print "done in %.2f secs" % (time.time() - start) 65 | diesel.sleep(1) 66 | diesel.quickstop() 67 | 68 | if __name__ == '__main__': 69 | diesel.set_log_level(diesel.loglevels.ERROR) 70 | diesel.quickstart(main) 71 | print cpustats 72 | -------------------------------------------------------------------------------- /examples/udp_echo.py: -------------------------------------------------------------------------------- 1 | # vim:ts=4:sw=4:expandtab 2 | '''Simple udp echo server and client. 3 | ''' 4 | import sys 5 | from diesel import ( 6 | UDPService, UDPClient, call, send, datagram, quickstart, receive, 7 | ) 8 | 9 | 10 | class EchoClient(UDPClient): 11 | """A UDPClient example. 12 | 13 | Very much like a normal Client but it can only receive datagrams 14 | from the wire. 15 | 16 | """ 17 | @call 18 | def say(self, msg): 19 | send(msg) 20 | return receive(datagram) 21 | 22 | def echo_server(): 23 | """The UDPService callback. 24 | 25 | Unlike a standard Service callback that represents a connection and takes 26 | the remote addr as the first function, a UDPService callback takes no 27 | arguments. It is responsible for receiving datagrams from the wire and 28 | acting upon them. 29 | 30 | """ 31 | while True: 32 | data = receive(datagram) 33 | send("you said %s" % data) 34 | 35 | def echo_client(): 36 | client = EchoClient('localhost', 8013) 37 | while True: 38 | msg = raw_input("> ") 39 | print client.say(msg) 40 | 41 | if len(sys.argv) == 2: 42 | if 'client' in sys.argv[1]: 43 | quickstart(echo_client) 44 | raise SystemExit 45 | elif 'server' in sys.argv[1]: 46 | quickstart(UDPService(echo_server, 8013)) 47 | raise SystemExit 48 | print 'usage: python %s (server|client)' % sys.argv[0] 49 | -------------------------------------------------------------------------------- /examples/web.py: -------------------------------------------------------------------------------- 1 | from diesel.web import DieselFlask, request 2 | 3 | app = DieselFlask(__name__) 4 | 5 | @app.route("/") 6 | def hello(): 7 | name = request.args.get('name', 'world') 8 | return "hello, %s!" % name 9 | 10 | @app.route("/err") 11 | def err(): 12 | a = b 13 | return "never happens.." 14 | 15 | if __name__ == '__main__': 16 | import diesel 17 | def t(): 18 | while True: 19 | diesel.sleep(1) 20 | print "also looping.." 21 | app.diesel_app.add_loop(diesel.Loop(t)) 22 | app.run(debug=True) 23 | -------------------------------------------------------------------------------- /examples/zeromq_echo_service.py: -------------------------------------------------------------------------------- 1 | import diesel 2 | from diesel.protocols.zeromq import ( 3 | DieselZMQService, zmq, zctx, DieselZMQSocket, 4 | ) 5 | 6 | 7 | NUM_CLIENTS = 100 8 | cids = range(NUM_CLIENTS) 9 | 10 | def echo_client(): 11 | sock = zctx.socket(zmq.DEALER) 12 | sock.identity = "client:%d" % cids.pop() 13 | s = DieselZMQSocket(sock, connect='tcp://127.0.0.1:4321') 14 | for i in xrange(10): 15 | s.send('msg:%d' % i) 16 | r = s.recv() 17 | assert r == 'msg:%d' % i 18 | print sock.identity, 'received', r 19 | 20 | class EchoService(DieselZMQService): 21 | def handle_client_packet(self, packet, ctx): 22 | return packet 23 | 24 | echo_svc = EchoService('tcp://*:4321') 25 | diesel.quickstart(echo_svc.run, *(echo_client for i in xrange(NUM_CLIENTS))) 26 | 27 | -------------------------------------------------------------------------------- /examples/zeromq_first.py: -------------------------------------------------------------------------------- 1 | from diesel import quickstart, quickstop, sleep, first 2 | from diesel.protocols.zeromq import DieselZMQSocket, zctx, zmq 3 | import time 4 | 5 | def get_messages(): 6 | outsock = DieselZMQSocket(zctx.socket(zmq.DEALER), bind="tcp://127.0.0.1:5000") 7 | 8 | for x in xrange(1000): 9 | t, m = first(sleep=1.0, waits=[outsock]) 10 | 11 | if t == 'sleep': 12 | print "sleep timeout!" 13 | else: 14 | print "zmq:", m 15 | 16 | quickstop() 17 | 18 | quickstart(get_messages) 19 | -------------------------------------------------------------------------------- /examples/zeromq_receiver.py: -------------------------------------------------------------------------------- 1 | from diesel import quickstart, quickstop, sleep 2 | from diesel.protocols.zeromq import DieselZMQSocket, zctx, zmq 3 | import time 4 | 5 | def get_messages(): 6 | outsock = DieselZMQSocket(zctx.socket(zmq.DEALER), bind="tcp://127.0.0.1:5000") 7 | 8 | t = time.time() 9 | for x in xrange(500000): 10 | msg = outsock.recv() 11 | assert msg == "yo dawg %s" % x 12 | if x % 1000 == 0: 13 | sleep() 14 | 15 | delt = time.time() - t 16 | print "500000 messages in %ss (%.1f/s)" % (delt, 500000.0 / delt) 17 | quickstop() 18 | 19 | def tick(): 20 | while True: 21 | print "Other diesel stuff" 22 | sleep(1) 23 | 24 | quickstart(get_messages, tick) 25 | -------------------------------------------------------------------------------- /examples/zeromq_sender.py: -------------------------------------------------------------------------------- 1 | from diesel import quickstart, quickstop, sleep 2 | from diesel.protocols.zeromq import DieselZMQSocket, zctx, zmq 3 | import time 4 | 5 | def send_message(): 6 | outsock = DieselZMQSocket(zctx.socket(zmq.DEALER), connect="tcp://127.0.0.1:5000") 7 | 8 | for x in xrange(500000): 9 | outsock.send("yo dawg %s" % x) 10 | if x % 1000 == 0: 11 | sleep() 12 | 13 | def tick(): 14 | while True: 15 | print "Other diesel stuff" 16 | sleep(1) 17 | 18 | quickstart(send_message, tick) 19 | -------------------------------------------------------------------------------- /examples/zeromq_test.py: -------------------------------------------------------------------------------- 1 | from diesel import quickstart, quickstop, sleep 2 | from diesel.protocols.zeromq import DieselZMQSocket, zctx, zmq 3 | import time 4 | 5 | def handle_messages(): 6 | insock = DieselZMQSocket(zctx.socket(zmq.DEALER), bind="inproc://foo") 7 | 8 | for x in xrange(500000): 9 | msg = insock.recv() 10 | assert msg == "yo dawg %s" % x 11 | delt = time.time() - t 12 | print "500000 messages in %ss (%.1f/s)" % (delt, 500000.0 / delt) 13 | quickstop() 14 | 15 | def send_message(): 16 | global t 17 | outsock = DieselZMQSocket(zctx.socket(zmq.DEALER), connect="inproc://foo") 18 | t = time.time() 19 | 20 | for x in xrange(500000): 21 | outsock.send("yo dawg %s" % x) 22 | if x % 1000 == 0: 23 | sleep() 24 | 25 | def tick(): 26 | while True: 27 | print "Other diesel stuff" 28 | sleep(1) 29 | 30 | quickstart(handle_messages, send_message, tick) 31 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import select, sys, os 2 | assert sys.version_info >= (2, 6), \ 3 | "Diesel requires python 2.6 (or greater 2.X release)" 4 | 5 | from setuptools import setup 6 | 7 | if os.system("which palmc > /dev/null 2>&1") == 0: 8 | os.system("palmc ./diesel/protocols ./diesel/protocols") 9 | 10 | additional_requires = [] 11 | 12 | if (os.environ.get('DIESEL_LIBEV') or 13 | os.environ.get('DIESEL_NO_EPOLL') or 14 | not hasattr(select, 'epoll')): 15 | additional_requires.append('pyev') 16 | 17 | VERSION = "3.0.24" 18 | 19 | setup(name="diesel", 20 | version=VERSION, 21 | author="Jamie Turner/Boomplex LLC/Bump Technologies, Inc/Various Contributors", 22 | author_email="jamie@bu.mp", 23 | description="Diesel is a coroutine-based networking library for Python", 24 | long_description=''' 25 | diesel is a framework for easily writing reliable and scalable network 26 | applications in Python. It uses the greenlet library layered atop 27 | asynchronous socket I/O in Python to achieve benefits of both 28 | the threaded-style (linear, blocking-ish code flow) and evented-style 29 | (no locking, low overhead per connection) concurrency paradigms. It's 30 | design is heavily inspired by the Erlang/OTP platform. 31 | 32 | It contains high-quality buffering, queuing and synchronization primitives, 33 | procedure supervision and supervision trees, connection pools, seamless 34 | thread integration, and more. 35 | 36 | An HTTP/1.1+WSGI+WebSockets implementation is included, as well as tight 37 | integration with the Flask web framework. 38 | 39 | Other bundled protocols include MongoDB, Riak, and Redis client libraries. 40 | ''', 41 | url="http://diesel.io", 42 | download_url="https://github.com/dieseldev/diesel/archive/diesel-%s.tar.gz" % VERSION, 43 | packages=[ 44 | "diesel", 45 | "diesel.protocols", 46 | "diesel.util", 47 | "diesel.util.patches", 48 | "diesel.protocols.http", 49 | ], 50 | scripts=["examples/dhttpd"], 51 | entry_points={ 52 | 'console_scripts': [ 53 | 'dpython = diesel.interactive:python', 54 | 'idpython = diesel.interactive:ipython', 55 | 'dnosetests = diesel.dnosetests:main', 56 | 'dconsole = diesel.console:main', 57 | ], 58 | }, 59 | install_requires=([ 60 | "greenlet", 61 | "twiggy", 62 | "pyopenssl", 63 | "flask", 64 | "http-parser >= 0.7.12", 65 | "dnspython", 66 | ] + additional_requires), 67 | ) 68 | -------------------------------------------------------------------------------- /tests/Makefile: -------------------------------------------------------------------------------- 1 | default: test-core 2 | 3 | test-all: 4 | dnosetests unit/ integration/ protocol/ 5 | 6 | test-core: 7 | dnosetests unit/ integration/ 8 | 9 | -------------------------------------------------------------------------------- /tests/integration/test_event_ordering.py: -------------------------------------------------------------------------------- 1 | import diesel 2 | 3 | from diesel import core 4 | from diesel.util.queue import Queue 5 | 6 | 7 | def test_pending_events_dont_break_ordering_when_handling_early_values(): 8 | 9 | # This test confirms that "early values" returned from a Waiter do 10 | # not give other pending event sources the chance to switch their 11 | # values into the greenlet while it context switches to give other 12 | # greenlets a chance to run. 13 | 14 | # First we setup a fake connection. It mimics a connection that does 15 | # not have data waiting in the buffer, and has to wait for the system 16 | # to call it back when data is ready on the socket. The delay argument 17 | # specifies how long the test should wait before simulating that data 18 | # is ready. 19 | 20 | conn1 = FakeConnection(1, delay=[None, 0.1]) 21 | 22 | # Next we setup a Queue instance and prime it with a value, so it will 23 | # be ready early and return an EarlyValue. 24 | 25 | q = Queue() 26 | q.put(1) 27 | 28 | # Force our fake connection into the connection stack for the current 29 | # loop so we can make network calls (like until_eol). 30 | 31 | loop = core.current_loop 32 | loop.connection_stack.append(conn1) 33 | 34 | try: 35 | 36 | # OK, this first() call does two things. 37 | # 1) It calls until_eol, finds that no data is ready, and sets up a 38 | # callback to be triggered when data is ready (which our 39 | # FakeConnection will simulate). 40 | # 2) Fetches from the 'q' which will result in an EarlyValue. 41 | 42 | source, value = diesel.first(until_eol=True, waits=[q]) 43 | assert source == q, source 44 | 45 | # What must happen is that the callback registered to handle data 46 | # from the FakeConnection when it arrives MUST BE CANCELED/DISCARDED/ 47 | # FORGOTTEN/NEVER CALLED. If it gets called, it will muck with 48 | # internal state, and possibly switch back into the running greenlet 49 | # with an unexpected value, which will throw off the ordering of 50 | # internal state and basically break everything. 51 | 52 | v = diesel.until_eol() 53 | assert v == 'expected value 1\r\n', 'actual value == %r !!!' % (v,) 54 | 55 | finally: 56 | loop.connection_stack = [] 57 | 58 | 59 | class FakeConnection(object): 60 | closed = False 61 | waiting_callback = None 62 | 63 | def __init__(self, conn_id, delay=None): 64 | self.conn_id = conn_id 65 | self.delay = delay 66 | 67 | def check_incoming(self, condition, callback): 68 | diesel.fork(self.delayed_value) 69 | return None 70 | 71 | def cleanup(self): 72 | self.waiting_callback = None 73 | 74 | def delayed_value(self): 75 | assert self.delay, \ 76 | "This connection requires more items in its delay list for further requests." 77 | delay = self.delay.pop(0) 78 | if delay is not None: 79 | print diesel.sleep(delay) 80 | if self.waiting_callback: 81 | self.waiting_callback('expected value %s\r\n' % self.conn_id) 82 | 83 | 84 | -------------------------------------------------------------------------------- /tests/integration/test_fanout.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | 3 | import diesel 4 | 5 | from diesel.util.queue import Fanout 6 | from diesel.util.event import Countdown 7 | 8 | class FanoutHarness(object): 9 | def setup(self): 10 | self.done = Countdown(10) 11 | self.fan = Fanout() 12 | self.subscriber_data = {} 13 | for x in xrange(10): 14 | diesel.fork(self.subscriber) 15 | diesel.sleep() 16 | for i in xrange(10): 17 | self.fan.pub(i) 18 | self.done.wait() 19 | 20 | def subscriber(self): 21 | self.subscriber_data[uuid.uuid4()] = data = [] 22 | with self.fan.sub() as q: 23 | for i in xrange(10): 24 | data.append(q.get()) 25 | self.done.tick() 26 | 27 | class TestFanout(FanoutHarness): 28 | def test_all_subscribers_get_the_published_messages(self): 29 | assert len(self.subscriber_data) == 10 30 | for values in self.subscriber_data.itervalues(): 31 | assert values == range(10), values 32 | 33 | def test_sub_is_removed_after_it_is_done(self): 34 | assert not self.fan.subs 35 | 36 | -------------------------------------------------------------------------------- /tests/integration/test_fire.py: -------------------------------------------------------------------------------- 1 | from diesel import fork, fire, wait, sleep, first 2 | from diesel.util.event import Event 3 | 4 | def test_basic_fire(): 5 | done = Event() 6 | v = [0] 7 | def w(): 8 | while True: 9 | wait("boom!") 10 | v[0] += 1 11 | 12 | def f(): 13 | sleep(0.05) 14 | fire("boom!") 15 | sleep(0.05) 16 | fire("boom!") 17 | done.set() 18 | 19 | fork(f) 20 | fork(w) 21 | ev, _ = first(sleep=1, waits=[done]) 22 | assert v[0] == 2 23 | 24 | def test_fire_multiple(): 25 | done = Event() 26 | v = [0] 27 | def w(): 28 | while True: 29 | wait("boom!") 30 | v[0] += 1 31 | 32 | def f(): 33 | sleep(0.05) 34 | fire("boom!") 35 | sleep(0.05) 36 | fire("boom!") 37 | done.set() 38 | 39 | fork(f) 40 | fork(w) 41 | fork(w) 42 | ev, _ = first(sleep=1, waits=[done]) 43 | assert v[0] == 4 44 | 45 | 46 | def test_fire_miss(): 47 | done = Event() 48 | v = [0] 49 | def w(): 50 | while True: 51 | wait("boom!") 52 | v[0] += 1 53 | 54 | def f(): 55 | sleep(0.05) 56 | fire("fizz!") 57 | done.set() 58 | 59 | fork(f) 60 | fork(w) 61 | fork(w) 62 | 63 | ev, _ = first(sleep=1, waits=[done]) 64 | assert v[0] == 0 # should not have woken up! 65 | 66 | -------------------------------------------------------------------------------- /tests/integration/test_fork.py: -------------------------------------------------------------------------------- 1 | from diesel import fork, sleep, first, Loop, fork_child, ParentDiedException 2 | from diesel.util.event import Event 3 | from diesel import runtime 4 | 5 | def tottering_child(v): 6 | v[0] += 1 7 | sleep(10) 8 | 9 | def test_basic_fork(): 10 | v = [0] 11 | done = Event() 12 | def parent(): 13 | fork(tottering_child, v) 14 | sleep(0.1) 15 | done.set() 16 | runtime.current_app.add_loop(Loop(parent)) 17 | ev, _ = first(sleep=1, waits=[done]) 18 | if ev == 'sleep': 19 | assert 0, "timed out" 20 | assert (v[0] == 1) 21 | 22 | def test_fork_many(): 23 | v = [0] 24 | COUNT = 10000 25 | def parent(): 26 | for x in xrange(COUNT): 27 | fork(tottering_child, v) 28 | 29 | runtime.current_app.add_loop(Loop(parent)) 30 | 31 | for i in xrange(16): 32 | if v[0] == COUNT: 33 | break 34 | sleep(0.5) # cumulative is long enough in core 2-era 35 | else: 36 | assert 0, "didn't reach expected COUNT soon enough" 37 | 38 | 39 | def dependent_child(got_exception): 40 | try: 41 | sleep(50) 42 | except ParentDiedException: 43 | got_exception[0] = 1 44 | else: 45 | got_exception[0] = 0 46 | 47 | def test_fork_child_normal_death(): 48 | got_exception = [0] 49 | def parent(): 50 | fork_child(dependent_child, got_exception) 51 | sleep(0.1) 52 | # implied, I end.. 53 | 54 | l = Loop(parent) 55 | runtime.current_app.add_loop(l) 56 | sleep() # give the Loop a chance to start 57 | while l.running: 58 | sleep() 59 | assert got_exception[0], "child didn't die when parent died!" 60 | 61 | 62 | def test_fork_child_exception(): 63 | got_exception = [0] 64 | def parent(): 65 | fork_child(dependent_child, got_exception) 66 | sleep(0.1) 67 | a = b # undef 68 | 69 | l = Loop(parent) 70 | runtime.current_app.add_loop(l) 71 | sleep() # give the Loop a chance to start 72 | while l.running: 73 | sleep() 74 | assert got_exception[0], "child didn't die when parent died!" 75 | 76 | def test_loop_keep_alive_normal_death(): 77 | v = [0] 78 | def l(): 79 | v[0] += 1 80 | 81 | def p(): 82 | sleep(0.7) 83 | 84 | runtime.current_app.add_loop(Loop(l), keep_alive=True) 85 | lp = Loop(p) 86 | runtime.current_app.add_loop(lp) 87 | sleep() 88 | while lp.running: 89 | sleep() 90 | assert (v[0] > 1) 91 | 92 | def test_loop_keep_alive_exception(): 93 | v = [0] 94 | def l(): 95 | v[0] += 1 96 | a = b # exception! 97 | 98 | def p(): 99 | sleep(0.7) 100 | 101 | runtime.current_app.add_loop(Loop(l), keep_alive=True) 102 | lp = Loop(p) 103 | runtime.current_app.add_loop(lp) 104 | sleep() 105 | while lp.running: 106 | sleep() 107 | assert (v[0] > 1) 108 | 109 | def flapping_child(): 110 | sleep(0.1) 111 | 112 | def strong_child(): 113 | sleep(20) 114 | 115 | def protective_parent(child, children): 116 | child = fork_child(child) 117 | child.keep_alive = True 118 | children[0] = child 119 | sleep(2) 120 | 121 | def test_child_keep_alive_retains_parent(): 122 | children = [None] 123 | parent = Loop(protective_parent, flapping_child, children) 124 | runtime.current_app.add_loop(parent) 125 | sleep() 126 | while parent.running: 127 | # Deaths of the child are interspersed here. 128 | assert parent.children 129 | assert children[0].parent is parent 130 | sleep() 131 | # The child should have died a bunch of times during the parent's life. 132 | assert children[0].deaths > 1, children[0].deaths 133 | 134 | def test_child_keep_alive_dies_with_parent(): 135 | # Starts a child that attempts to run for a long time, with a parent 136 | # that has a short lifetime. The rule is that children are always 137 | # subordinate to their parents. 138 | children = [None] 139 | parent = Loop(protective_parent, strong_child, children) 140 | runtime.current_app.add_loop(parent) 141 | sleep() 142 | while parent.running: 143 | sleep() 144 | 145 | # Wait here because a child could respawn after 0.5 seconds if the 146 | # child-always-dies-with-parent rule is being violated. 147 | sleep(1) 148 | 149 | # Once the parent is dead, the child (even thought it is keep-alive) 150 | # should be dead as well. 151 | assert not children[0].running 152 | assert not parent.children 153 | 154 | -------------------------------------------------------------------------------- /tests/integration/test_os_signals.py: -------------------------------------------------------------------------------- 1 | import os 2 | import signal 3 | import time 4 | 5 | import diesel 6 | 7 | from diesel.util.event import Countdown, Event, Signal 8 | 9 | 10 | state = {'triggered':False} 11 | 12 | def waiter(): 13 | diesel.signal(signal.SIGUSR1) 14 | state['triggered'] = True 15 | 16 | def test_can_wait_on_os_signals(): 17 | # Start our Loop that will wait on USR1 18 | diesel.fork(waiter) 19 | 20 | # Let execution switch to the newly spawned loop 21 | diesel.sleep() 22 | 23 | # We haven't sent the signal, so the state should not be triggered 24 | assert not state['triggered'] 25 | 26 | # Send the USR1 signal 27 | os.kill(os.getpid(), signal.SIGUSR1) 28 | 29 | # Again, force a switch so the waiter can act on the signal 30 | diesel.sleep() 31 | 32 | # Now that we're back, the waiter should have triggered the state 33 | assert state['triggered'] 34 | 35 | def test_multiple_signal_waiters(): 36 | N_WAITERS = 5 37 | c = Countdown(N_WAITERS) 38 | def mwaiter(): 39 | diesel.signal(signal.SIGUSR1) 40 | c.tick() 41 | for i in xrange(N_WAITERS): 42 | diesel.fork(mwaiter) 43 | diesel.sleep() 44 | os.kill(os.getpid(), signal.SIGUSR1) 45 | evt, data = diesel.first(sleep=1, waits=[c]) 46 | assert evt is c, "all waiters were not triggered!" 47 | 48 | def test_overwriting_custom_signal_handler_fails(): 49 | readings = [] 50 | success = Event() 51 | failure = Event() 52 | 53 | def append_reading(sig, frame): 54 | readings.append(sig) 55 | signal.signal(signal.SIGUSR1, signal.SIG_DFL) 56 | signal.signal(signal.SIGUSR1, append_reading) 57 | 58 | def overwriter(): 59 | try: 60 | diesel.signal(signal.SIGUSR1) 61 | except diesel.ExistingSignalHandler: 62 | success.set() 63 | else: 64 | failure.set() 65 | diesel.fork(overwriter) 66 | diesel.sleep() 67 | os.kill(os.getpid(), signal.SIGUSR1) 68 | evt, _ = diesel.first(waits=[success, failure]) 69 | assert evt is success 70 | assert readings 71 | 72 | def test_signals_are_handled_while_event_loop_is_blocked(): 73 | done = Event() 74 | 75 | def handler(): 76 | diesel.signal(signal.SIGUSR1) 77 | done.set() 78 | 79 | def killer(): 80 | time.sleep(0.1) 81 | os.kill(os.getpid(), signal.SIGUSR1) 82 | 83 | diesel.fork(handler) 84 | diesel.thread(killer) 85 | diesel.sleep() 86 | evt, _ = diesel.first(sleep=1, waits=[done]) 87 | assert evt is done 88 | 89 | def test_signal_captured_by_Signal_instance(): 90 | usr1 = Signal(signal.SIGUSR1) 91 | os.kill(os.getpid(), signal.SIGUSR1) 92 | evt, _ = diesel.first(sleep=0.1, waits=[usr1]) 93 | assert evt is usr1, evt 94 | 95 | def test_Signal_instances_trigger_multiple_times(): 96 | usr1 = Signal(signal.SIGUSR1) 97 | for i in xrange(5): 98 | os.kill(os.getpid(), signal.SIGUSR1) 99 | evt, _ = diesel.first(sleep=0.1, waits=[usr1]) 100 | assert evt is usr1, evt 101 | usr1.rearm() 102 | 103 | -------------------------------------------------------------------------------- /tests/integration/test_queue.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | from collections import defaultdict 4 | 5 | import diesel 6 | 7 | from diesel.util.queue import Queue, QueueTimeout 8 | from diesel.util.event import Countdown, Event 9 | 10 | 11 | N = 500 12 | W = 50 13 | TIMEOUT = 1.0 14 | 15 | class QueueHarness(object): 16 | def setup(self): 17 | self.queue = Queue() 18 | self.done = Countdown(N) 19 | self.results = [] 20 | self.handled = defaultdict(int) 21 | self.populate() 22 | self.consume() 23 | self.trigger() 24 | 25 | def consume(self): 26 | def worker(myid): 27 | while True: 28 | # Test both queue.get and wait() on queue (both are valid 29 | # APIs for getting items from the queue). The results should 30 | # be the same. 31 | if random.random() > 0.5: 32 | v = self.queue.get() 33 | else: 34 | v = diesel.wait(self.queue) 35 | self.results.append(v) 36 | self.handled[myid] += 1 37 | self.done.tick() 38 | for i in xrange(W): 39 | diesel.fork(worker, i) 40 | 41 | def trigger(self): 42 | ev, val = diesel.first(sleep=TIMEOUT, waits=[self.done]) 43 | if ev == 'sleep': 44 | assert 0, "timed out" 45 | 46 | def test_results_are_ordered_as_expected(self): 47 | assert self.results == range(N), self.results 48 | 49 | def test_results_are_balanced(self): 50 | for wid, count in self.handled.iteritems(): 51 | assert count == N/W, count 52 | 53 | class TestConsumersOnFullQueue(QueueHarness): 54 | def populate(self): 55 | for i in xrange(N): 56 | self.queue.put(i) 57 | 58 | class TestConsumersOnEmptyQueue(QueueHarness): 59 | def populate(self): 60 | def go(): 61 | diesel.wait('ready') 62 | for i in xrange(N): 63 | self.queue.put(i) 64 | diesel.fork(go) 65 | 66 | def trigger(self): 67 | diesel.sleep() 68 | diesel.fire('ready') 69 | super(TestConsumersOnEmptyQueue, self).trigger() 70 | 71 | class TestQueueTimeouts(object): 72 | def setup(self): 73 | self.result = Event() 74 | self.queue = Queue() 75 | self.timeouts = 0 76 | diesel.fork(self.consumer, 0.01) 77 | diesel.fork(self.producer, 0.05) 78 | diesel.fork(self.consumer, 0.10) 79 | ev, val = diesel.first(sleep=TIMEOUT, waits=[self.result]) 80 | if ev == 'sleep': 81 | assert 0, 'timed out' 82 | 83 | def consumer(self, timeout): 84 | try: 85 | self.queue.get(timeout=timeout) 86 | self.result.set() 87 | except QueueTimeout: 88 | self.timeouts += 1 89 | 90 | def producer(self, delay): 91 | diesel.sleep(delay) 92 | self.queue.put('test') 93 | 94 | def test_a_consumer_timed_out(self): 95 | assert self.timeouts == 1 96 | 97 | def test_a_consumer_got_a_value(self): 98 | assert self.result.is_set 99 | -------------------------------------------------------------------------------- /tests/integration/test_sleep.py: -------------------------------------------------------------------------------- 1 | from time import time 2 | from diesel import fork, fork_child, sleep, quickstart, quickstop, ParentDiedException 3 | from diesel.hub import Timer 4 | 5 | def test_basic_sleep(): 6 | delt = [None] 7 | STIME = 0.3 8 | def l(): 9 | t = time() 10 | sleep(STIME) 11 | delt[0] = time() - t 12 | l_ = fork(l) 13 | sleep() 14 | while l_.running: 15 | sleep() 16 | min_bound = (STIME - Timer.ALLOWANCE) 17 | max_bound = (STIME + Timer.ALLOWANCE) 18 | assert (delt[0] > min_bound and delt[0] < max_bound), delt[0] 19 | 20 | def test_sleep_independence(): 21 | v = [0] 22 | def i(): 23 | v[0] += 1 24 | sleep(0.1) 25 | v[0] += 1 26 | 27 | fork(i) 28 | fork(i) 29 | sleep(0.05) 30 | assert (v[0] == 2) 31 | sleep(0.1) 32 | assert (v[0] == 4) 33 | 34 | def test_sleep_zero(): 35 | '''Sleep w/out argument allows other loops to run 36 | ''' 37 | 38 | v = [0] 39 | def i(): 40 | for i in xrange(10000): 41 | v[0] += 1 42 | sleep() 43 | 44 | fork(i) 45 | 46 | sleep(0.05) 47 | cur = v[0] 48 | sleep() # allow i to get scheduled 49 | now = v[0] 50 | assert (now == cur + 1) 51 | -------------------------------------------------------------------------------- /tests/integration/test_wait.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | 3 | import diesel 4 | import diesel.runtime 5 | 6 | 7 | def waiting_green_thread(wait_id, done): 8 | diesel.wait(wait_id) 9 | done.append(True) 10 | 11 | def test_wait_tokens_dont_accumulate_forever(): 12 | """Wait tokens and related structures should be disposed of after use. 13 | 14 | They are tracked in a dictionary in the internal 15 | diesel.events.WaitPool. If a wait_id has no more objects waiting on it, 16 | it should be removed from that dictionary along with the set of waiting 17 | objects. 18 | 19 | """ 20 | done = [] 21 | wait_ids = [] 22 | expected_length = len(diesel.runtime.current_app.waits.waits) 23 | for i in xrange(50): 24 | wait_id = uuid.uuid4().hex 25 | diesel.fork(waiting_green_thread, wait_id, done) 26 | diesel.sleep() 27 | wait_ids.append(wait_id) 28 | for wait_id in wait_ids: 29 | diesel.fire(wait_id) 30 | diesel.sleep() 31 | while len(done) != 50: 32 | diesel.sleep(0.1) 33 | actual_length = len(diesel.runtime.current_app.waits.waits) 34 | assert actual_length == expected_length, actual_length 35 | -------------------------------------------------------------------------------- /tests/protocol/test_mongodb.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | 3 | import diesel 4 | from diesel.protocols.mongodb import * 5 | 6 | from diesel.util.queue import Fanout 7 | 8 | class MongoDbHarness(object): 9 | def setup(self): 10 | self.client = MongoClient() 11 | self.client.drop_database('dieseltest') 12 | self.db = self.client.dieseltest 13 | 14 | def filt(self, d): 15 | del d['_id'] 16 | return d 17 | 18 | class TestMongoDB(MongoDbHarness): 19 | def test_empty(self): 20 | assert self.db.cempty.find().all() == [] 21 | assert self.db.cempty.find().one() == None 22 | 23 | # INSERT 24 | def test_insert(self): 25 | d = {'one' : 'two'} 26 | assert self.db.ionly.insert({'one' : 'two'})['err'] == None 27 | 28 | def test_insert_many(self): 29 | d1 = {'one' : 'two'} 30 | d2 = {'three' : 'four'} 31 | inp = [d1, d2] 32 | inp.sort() 33 | assert self.db.imult.insert(inp)['err'] == None 34 | all = self.db.imult.find().all() 35 | map(self.filt, all) 36 | all.sort() 37 | assert all == inp 38 | 39 | # UPDATE 40 | def test_update_basic(self): 41 | d = {'one' : 'two', 'three' : 'four'} 42 | assert self.db.up1.insert(d)['err'] == None 43 | assert self.filt(self.db.up1.find().one()) == d 44 | assert self.db.up1.update({'one' : 'two'}, 45 | {'three' : 'five'})['err'] == None 46 | new = self.filt(self.db.up1.find().one()) 47 | assert 'one' not in new 48 | assert new['three'] == 'five' 49 | 50 | def test_update_set(self): 51 | d = {'one' : 'two', 'three' : 'four'} 52 | assert self.db.up2.insert(d)['err'] == None 53 | assert self.filt(self.db.up2.find().one()) == d 54 | assert self.db.up2.update({'one' : 'two'}, 55 | {'$set' : {'three' : 'five'}})['err'] == None 56 | new = self.filt(self.db.up2.find().one()) 57 | assert new['one'] == 'two' 58 | assert new['three'] == 'five' 59 | 60 | def test_update_not_multi(self): 61 | d = {'one' : 'two', 'three' : 'four'} 62 | assert self.db.up3.insert(d)['err'] == None 63 | assert self.db.up3.insert(d)['err'] == None 64 | assert self.db.up3.update({'one' : 'two'}, 65 | {'$set' : {'three' : 'five'}})['err'] == None 66 | 67 | threes = set() 68 | for r in self.db.up3.find(): 69 | threes.add(r['three']) 70 | assert threes == set(['four', 'five']) 71 | 72 | def test_update_multi(self): 73 | d = {'one' : 'two', 'three' : 'four'} 74 | assert self.db.up4.insert(d)['err'] == None 75 | assert self.db.up4.insert(d)['err'] == None 76 | assert self.db.up4.update({'one' : 'two'}, 77 | {'$set' : {'three' : 'five'}}, 78 | multi=True)['err'] == None 79 | 80 | threes = set() 81 | for r in self.db.up4.find(): 82 | threes.add(r['three']) 83 | assert threes == set(['five']) 84 | 85 | def test_update_miss(self): 86 | d = {'one' : 'two', 'three' : 'four'} 87 | assert self.db.up5.insert(d)['err'] == None 88 | snap = self.db.up5.find().all() 89 | assert self.db.up5.update({'one' : 'nottwo'}, 90 | {'$set' : {'three' : 'five'}}, 91 | multi=True)['err'] == None 92 | 93 | assert snap == self.db.up5.find().all() 94 | 95 | def test_update_all(self): 96 | d = {'one' : 'two', 'three' : 'four'} 97 | assert self.db.up6.insert(d)['err'] == None 98 | assert self.db.up6.insert(d)['err'] == None 99 | assert self.db.up6.update({}, 100 | {'$set' : {'three' : 'five'}}, 101 | multi=True)['err'] == None 102 | 103 | for r in self.db.up6.find().all(): 104 | assert r['three'] == 'five' 105 | 106 | def test_update_upsert(self): 107 | d = {'one' : 'two', 'three' : 'four'} 108 | assert self.db.up7.insert(d)['err'] == None 109 | assert self.db.up7.insert(d)['err'] == None 110 | assert len(self.db.up7.find().all()) == 2 111 | assert self.db.up7.update({'not' : 'there'}, 112 | {'this is' : 'good'}, upsert=True)['err'] == None 113 | 114 | assert len(self.db.up7.find().all()) == 3 115 | 116 | # DELETE 117 | def test_delete_miss(self): 118 | d = {'one' : 'two'} 119 | assert self.db.del1.insert({'one' : 'two'})['err'] == None 120 | assert len(self.db.del1.find().all()) == 1 121 | assert self.db.del1.delete({'not' : 'me'})['err'] == None 122 | assert len(self.db.del1.find().all()) == 1 123 | 124 | def test_delete_target(self): 125 | d = {'one' : 'two'} 126 | assert self.db.del2.insert({'one' : 'two'})['err'] == None 127 | assert self.db.del2.insert({'three' : 'four'})['err'] == None 128 | assert len(self.db.del2.find().all()) == 2 129 | assert self.db.del2.delete({'one' : 'two'})['err'] == None 130 | assert len(self.db.del2.find().all()) == 1 131 | 132 | def test_delete_all(self): 133 | d = {'one' : 'two'} 134 | assert self.db.del3.insert({'one' : 'two'})['err'] == None 135 | assert self.db.del3.insert({'three' : 'four'})['err'] == None 136 | assert len(self.db.del3.find().all()) == 2 137 | assert self.db.del3.delete({})['err'] == None 138 | assert len(self.db.del3.find().all()) == 0 139 | 140 | # QUERY 141 | def test_query_basic(self): 142 | d = {'one' : 'two'} 143 | assert self.db.onerec.insert(d)['err'] == None 144 | assert map(self.filt, self.db.onerec.find().all()) == [d] 145 | assert self.filt(self.db.onerec.find().one()) == d 146 | 147 | x = 0 148 | for r in self.db.onerec.find(): 149 | assert self.filt(r) == d 150 | x += 1 151 | assert x == 1 152 | 153 | def test_query_one(self): 154 | d1 = {'one' : 'two'} 155 | d2 = {'three' : 'four'} 156 | inp = [d1, d2] 157 | assert self.db.q0.insert(inp)['err'] == None 158 | assert len(self.db.q0.find().all()) == 2 159 | assert len(self.db.q0.find({'one' : 'two'}).all()) == 1 160 | 161 | def test_query_miss(self): 162 | d = {'one' : 'two'} 163 | assert self.db.q1.insert(d)['err'] == None 164 | assert self.db.q1.find({'one' : 'nope'}).all() == [] 165 | 166 | def test_query_subfields(self): 167 | d = {'one' : 'two', 'three' : 'four'} 168 | assert self.db.q2.insert(d)['err'] == None 169 | assert map(self.filt, 170 | self.db.q2.find({'one' : 'two'}, ['three']).all()) \ 171 | == [{'three' : 'four'}] 172 | 173 | def test_deterministic_order_when_static(self): 174 | for x in xrange(500): 175 | self.db.q3.insert({'x' : x}) 176 | 177 | snap = self.db.q3.find().all() 178 | 179 | for x in xrange(100): 180 | assert snap == self.db.q3.find().all() 181 | 182 | def test_skip(self): 183 | for x in xrange(500): 184 | self.db.q4.insert({'x' : x}) 185 | 186 | snap = self.db.q4.find().all() 187 | 188 | with_skip = self.db.q4.find(skip=250).all() 189 | 190 | assert snap[250:] == with_skip 191 | 192 | def test_limit(self): 193 | for x in xrange(500): 194 | self.db.q5.insert({'x' : x}) 195 | 196 | snap = self.db.q5.find().all() 197 | 198 | with_skip = self.db.q5.find(limit=150).all() 199 | 200 | assert snap[:150] == with_skip 201 | 202 | def test_skip_limit(self): 203 | for x in xrange(500): 204 | self.db.q6.insert({'x' : x}) 205 | 206 | snap = self.db.q6.find().all() 207 | 208 | with_skip = self.db.q6.find(skip=300, limit=150).all() 209 | 210 | assert snap[300:300+150] == with_skip 211 | 212 | # CURSOR PROPERTIES 213 | def test_cursor_count(self): 214 | for x in xrange(500): 215 | self.db.c1.insert({'x' : x}) 216 | 217 | c = self.db.c1.find() 218 | assert c.count() == 500 219 | assert len(c.all()) == 500 220 | 221 | c = self.db.c1.find(skip=100, limit=150) 222 | assert c.count() == 150 223 | assert len(c.all()) == 150 224 | 225 | def test_cursor_sort(self): 226 | for x in xrange(500): 227 | self.db.c2.insert({'x' : x}) 228 | 229 | snap = self.db.c2.find().sort('x', 1).all() 230 | 231 | print snap 232 | assert map(lambda d: d['x'], snap) == range(500) 233 | 234 | snap = self.db.c2.find().sort('x', -1).all() 235 | assert map(lambda d: d['x'], snap) == range(499, -1, -1) 236 | -------------------------------------------------------------------------------- /tests/protocol/test_redis.py: -------------------------------------------------------------------------------- 1 | import diesel 2 | from diesel.protocols.redis import * 3 | 4 | class RedisHarness(object): 5 | def setup(self): 6 | self.client = RedisClient() 7 | self.client.select(11) 8 | self.client.flushdb() 9 | 10 | class TestRedis(RedisHarness): 11 | def test_basic(self): 12 | r = self.client 13 | assert r.get('newdb') == None 14 | r.set('newdb', '1') 15 | 16 | r.set('foo3', 'bar') 17 | assert r.exists('foo3') 18 | r.delete('foo3') 19 | assert not r.exists('foo3') 20 | 21 | for x in xrange(5000): 22 | r.set('foo', 'bar') 23 | 24 | assert r.get('foo') == 'bar' 25 | assert r.get('foo2') == None 26 | 27 | assert r.exists('foo') == True 28 | assert r.exists('foo2') == False 29 | 30 | assert r.type('foo') == 'string' 31 | assert r.type('foo2') == 'none' 32 | assert r.keys('fo*') == set(['foo']) 33 | assert r.keys('bo*') == set() 34 | 35 | assert r.randomkey() 36 | 37 | r.rename('foo', 'bar') 38 | assert r.get('foo') == None 39 | assert r.get('bar') == 'bar' 40 | 41 | r.rename('bar', 'foo') 42 | assert r.get('foo') == 'bar' 43 | assert r.get('bar') == None 44 | assert r.dbsize() 45 | 46 | assert r.ttl('foo') == None 47 | 48 | r.setex("gonesoon", 3, "whatever") 49 | assert 0 < r.ttl("gonesoon") <= 3.0 50 | 51 | r.set("phrase", "to be or ") 52 | r.append("phrase", "not to be") 53 | assert r.get("phrase") == "to be or not to be" 54 | assert r.substr('phrase', 3, 11) == 'be or not' 55 | 56 | r.set("one", "two") 57 | assert r.mget(["one", "foo"]) == ['two', 'bar'] 58 | r.mset({"one" : "three", "foo": "four"}) 59 | assert r.mget(["one", "foo"]) == ['three', 'four'] 60 | 61 | def test_incr(self): 62 | r = self.client 63 | assert r.incr("counter") == 1 64 | assert r.get('counter') == '1' 65 | assert r.incr("counter") == 2 66 | assert r.get('counter') == '2' 67 | assert r.incrby("counter", 2) == 4 68 | assert r.get('counter') == '4' 69 | 70 | def test_decr(self): 71 | r = self.client 72 | r.set('counter', '4') 73 | assert r.decr("counter") == 3 74 | assert r.decr("counter") == 2 75 | assert r.decrby("counter", 2) == 0 76 | 77 | def test_lists(self): 78 | r = self.client 79 | r.rpush("ml", 5) 80 | r.lpush("ml", 1) 81 | assert r.lrange("ml", 0, 500) == ['1', '5'] 82 | assert r.llen("ml") == 2 83 | 84 | r.ltrim("ml", 1, 3) 85 | 86 | assert r.lrange("ml", 0, 500) == ['5'] 87 | 88 | r.lset("ml", 0, 'nifty!') 89 | assert r.lrange("ml", 0, 500) == ['nifty!'] 90 | assert r.lindex("ml", 0) == 'nifty!' 91 | 92 | r.lrem("ml", 'nifty!') 93 | 94 | assert r.lrange("ml", 0, 500) == [] 95 | 96 | r.rpush("ml", 'yes!') 97 | r.rpush("ml", 'no!') 98 | assert r.lrange("ml", 0, 500) == ["yes!", "no!"] 99 | 100 | assert r.lpop("ml") == 'yes!' 101 | assert r.rpop("ml") == 'no!' 102 | 103 | t = time.time() 104 | r.blpop(['ml'], 3) 105 | delt = time.time() - t 106 | assert 2.5 < delt < 10 107 | 108 | r.rpush("ml", 'yes!') 109 | r.rpush("ml", 'no!') 110 | assert r.blpop(['ml'], 3) == ('ml', 'yes!') 111 | assert r.blpop(['ml'], 3) == ('ml', 'no!') 112 | 113 | r.rpush("ml", 'yes!') 114 | r.rpush("ml", 'no!') 115 | r.rpush("ml2", 'one!') 116 | r.rpush("ml2", 'two!') 117 | 118 | r.rpoplpush("ml", "ml2") 119 | assert r.lrange("ml", 0, 500) == ['yes!'] 120 | assert r.lrange("ml2", 0, 500) == ['no!', 'one!', 'two!'] 121 | 122 | def test_sets(self): 123 | r = self.client 124 | r.sadd("s1", "one") 125 | r.sadd("s1", "two") 126 | r.sadd("s1", "three") 127 | 128 | assert r.smembers("s1") == set(["one", "two", "three"]) 129 | 130 | r.srem("s1", "three") 131 | 132 | assert r.smembers("s1") == set(["one", "two"]) 133 | 134 | r.smove("s1", "s2", "one") 135 | assert r.spop("s2") == 'one' 136 | assert r.scard("s1") == 1 137 | 138 | assert r.sismember("s1", "two") == True 139 | assert r.sismember("s1", "one") == False 140 | 141 | r.sadd("s1", "four") 142 | r.sadd("s2", "four") 143 | 144 | assert r.sinter(["s1", "s2"]) == set(['four']) 145 | r.sinterstore("s3", ["s1", "s2"]) 146 | assert r.smembers('s3') == r.sinter(["s1", "s2"]) 147 | assert r.sunion(["s1", "s2"]) == set(['two', 'four']) 148 | r.sunionstore("s3", ["s1", "s2"]) 149 | assert r.smembers('s3') == r.sunion(["s1", "s2"]) 150 | 151 | assert r.srandmember("s3") in r.smembers("s3") 152 | 153 | def test_zsets(self): 154 | r = self.client 155 | r.zadd("z1", 10, "ten") 156 | r.zadd("z1", 1, "one") 157 | r.zadd("z1", 2, "two") 158 | r.zadd("z1", 0, "zero") 159 | 160 | 161 | assert r.zrange("z1", 0, -1) == ['zero', 'one', 'two', 'ten'] 162 | r.zrem("z1", "two") 163 | assert r.zrange("z1", 0, -1) == ['zero', 'one', 'ten'] 164 | assert r.zrevrange("z1", 0, -1) == list(reversed(r.zrange("z1", 0, -1))) 165 | 166 | r.zrem("z1", (r.zrange("z1", 0, 0))[0]) # remove 'zero'? 167 | assert r.zrange("z1", 0, -1) == ['one', 'ten'] 168 | assert r.zcard("z1") == 2 169 | 170 | assert r.zscore("z1", "one") == 1.0 171 | 172 | r.zincrby("z1", -2, "one") 173 | 174 | assert r.zscore("z1", "one") == -1.0 175 | 176 | r.zadd("z1", 2, "two") 177 | r.zadd("z1", 3, "three") 178 | assert r.zrangebyscore("z1", -5, 15) == ['one', 'two', 'three', 'ten'] 179 | assert r.zrangebyscore("z1", 2, 15) == ['two', 'three', 'ten'] 180 | assert r.zrangebyscore("z1", 2, 15, 1, 50) == ['three', 'ten'] 181 | assert r.zrangebyscore("z1", 2, 15, 1, 1) == ['three'] 182 | assert r.zrangebyscore("z1", 2, 15, 1, 50, with_scores=True) == [('three', 3.0), ('ten', 10.0)] 183 | 184 | assert r.zcount("z1", 2, 15) == 3 185 | 186 | assert r.zremrangebyrank('z1', 1, 1) 187 | assert r.zrangebyscore("z1", -5, 15) == ['one', 'three', 'ten'] 188 | assert r.zremrangebyscore('z1', 2, 4) 189 | assert r.zrangebyscore("z1", -5, 15) == ['one', 'ten'] 190 | 191 | def test_hashes(self): 192 | r = self.client 193 | r.hset("h1", "bahbah", "black sheep") 194 | assert r.hget("h1", "bahbah") == "black sheep" 195 | 196 | r.hmset("h1", {"foo" : "bar", "baz" : "bosh"}) 197 | assert r.hmget("h1", ["foo", "bahbah", "baz"]) == {'foo' : 'bar', 'baz' : 'bosh', 'bahbah' : 'black sheep'} 198 | 199 | assert r.hincrby("h1", "count", 3) == 3 200 | assert r.hincrby("h1", "count", 4) == 7 201 | 202 | assert r.hmget("h1", ["foo", "count"]) == {'foo' : 'bar', 'count' : '7'} 203 | 204 | assert r.hexists("h1", "bahbah") == True 205 | assert r.hexists("h1", "nope") == False 206 | 207 | r.hdel("h1", "bahbah") 208 | assert r.hexists("h1", "bahbah") == False 209 | assert r.hlen("h1") == 3 210 | 211 | assert r.hkeys("h1") == set(['foo', 'baz', 'count']) 212 | assert set(r.hvals("h1")) == set(['bar', 'bosh', '7']) 213 | assert r.hgetall("h1") == {'foo' : 'bar', 'baz' : 'bosh', 'count' : '7'} 214 | 215 | def test_transactions(self): 216 | r = self.client 217 | r.set('t2', 1) 218 | # Try a successful transaction. 219 | with r.transaction() as t: 220 | t.incr('t1') 221 | t.incr('t2') 222 | assert r.get('t1') == '1' 223 | assert r.get('t2') == '2' 224 | 225 | # Try a failing transaction. 226 | try: 227 | with r.transaction() as t: 228 | t.incr('t1') 229 | t.icnr('t2') # typo! 230 | except AttributeError: 231 | # t1 should not get incremented since the transaction body 232 | # raised an exception. 233 | assert r.get('t1') == '1' 234 | assert t.aborted 235 | else: 236 | assert 0, "DID NOT RAISE" 237 | 238 | # Try watching keys in a transaction. 239 | r.set('w1', 'watch me') 240 | transaction = r.transaction(watch=['w1']) 241 | w1 = r.get('w1') 242 | with transaction: 243 | transaction.set('w2', w1 + ' if you can!') 244 | assert transaction.value == ['OK'] 245 | assert r.get('w2') == 'watch me if you can!' 246 | 247 | # Try changing watched keys. 248 | r.set('w1', 'watch me') 249 | transaction = r.transaction(watch=['w1']) 250 | r.set('w1', 'changed!') 251 | w1 = r.get('w1') 252 | try: 253 | with transaction: 254 | transaction.set('w2', w1 + ' if you can!') 255 | except RedisTransactionError: 256 | assert transaction.aborted 257 | else: 258 | assert 0, "DID NOT RAISE" 259 | -------------------------------------------------------------------------------- /tests/protocol/test_zmq_service.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | import diesel 4 | 5 | from collections import namedtuple 6 | from diesel.protocols.zeromq import DieselZMQService 7 | 8 | 9 | # Test Cases 10 | # ========== 11 | 12 | def test_incoming_message_loop_is_kept_alive(): 13 | def stop_after_10_sends(sock): 14 | if sock.send_calls == 10: 15 | raise StopIteration 16 | 17 | svc = MisbehavingService('something', max_ticks=10) 18 | loop = diesel.fork(svc.run) 19 | diesel.sleep() 20 | start = time.time() 21 | maxtime = 0.5 22 | while loop.running and time.time() - start < maxtime: 23 | diesel.sleep(0.1) 24 | if loop.running: 25 | loop.reschedule_with_this_value(diesel.TerminateLoop()) 26 | diesel.sleep() 27 | assert not loop.running 28 | assert svc.zmq_socket.exceptions > 1, svc.zmq_socket.exceptions 29 | 30 | 31 | # Stubs and Utilities For the Tests Above 32 | # ======================================= 33 | 34 | envelope = namedtuple('envelope', ['more', 'bytes']) 35 | body = namedtuple 36 | 37 | class MisbehavingSocket(object): 38 | """Stub for DieselZMQSocket.""" 39 | def __init__(self): 40 | self.recv_calls = 0 41 | self.send_calls = 0 42 | self.exceptions = 0 43 | 44 | def recv(self, copy=True): 45 | # raises an Exception every 5 calls 46 | self.recv_calls += 1 47 | if (self.recv_calls % 5) == 0: 48 | self.exceptions += 1 49 | raise Exception("aaaahhhhh") 50 | if not copy: 51 | return envelope(more=True, bytes="foobarbaz") 52 | return "this is the data you are looking for" 53 | 54 | def send(self, *args): 55 | self.send_calls += 1 56 | 57 | class MisbehavingService(DieselZMQService): 58 | """A DieselZMQService with a MisbehavingSocket. 59 | 60 | It also stops running after a number of iterations (controlled via a 61 | `max_ticks` keyword argument). 62 | 63 | """ 64 | def __init__(self, *args, **kw): 65 | self._test_ticks = 0 66 | self._max_ticks = kw.pop('max_ticks') 67 | super(MisbehavingService, self).__init__(*args, **kw) 68 | 69 | def _create_zeromq_server_socket(self): 70 | self.zmq_socket = MisbehavingSocket() 71 | 72 | @property 73 | def should_run(self): 74 | if self._test_ticks >= self._max_ticks: 75 | return False 76 | self._test_ticks += 1 77 | return True 78 | 79 | -------------------------------------------------------------------------------- /tests/unit/test_buffer.py: -------------------------------------------------------------------------------- 1 | from diesel.buffer import Buffer 2 | 3 | def test_feed(): 4 | b = Buffer() 5 | assert b.feed("rock") == None 6 | assert b.feed("rockandroll") == None 7 | 8 | def test_read_bytes(): 9 | b = Buffer() 10 | b.set_term(10) 11 | assert b.feed("rock") == None 12 | assert b.feed("rockandroll") == "rockrockan" 13 | # buffer left.. droll 14 | assert b.check() == None 15 | b.set_term(10) 16 | assert b.check() == None 17 | assert b.feed("r") == None 18 | assert b.feed("r") == None 19 | assert b.feed("r") == None 20 | assert b.feed("r") == None 21 | assert b.feed("r") == "drollrrrrr" 22 | assert b.feed("x" * 10000) == None # no term (re-)established 23 | 24 | def test_read_sentinel(): 25 | b = Buffer() 26 | assert b.feed("rock and") == None 27 | b.set_term("\r\n") 28 | assert b.feed(" roll\r") == None 29 | assert b.feed("\nrock ") == "rock and roll\r\n" 30 | assert b.feed("and roll 2\r\n") == None 31 | b.set_term("\r\n") 32 | assert b.check() == "rock and roll 2\r\n" 33 | 34 | def test_read_hybrid(): 35 | b = Buffer() 36 | assert b.feed("rock and") == None 37 | b.set_term("\r\n") 38 | assert b.feed(" roll\r") == None 39 | assert b.feed("\n012345678") == "rock and roll\r\n" 40 | b.set_term(16) 41 | assert b.check() == None 42 | assert b.feed("9abcdefgh") == "0123456789abcdef" 43 | 44 | -------------------------------------------------------------------------------- /tests/unit/test_pipeline.py: -------------------------------------------------------------------------------- 1 | #12345678 2 | # That comment above matters (used in the test!) 3 | from diesel.pipeline import Pipeline, PipelineClosed, PipelineCloseRequest 4 | from cStringIO import StringIO 5 | 6 | FILE = __file__ 7 | if FILE.endswith('.pyc') or FILE.endswith('.pyo'): 8 | FILE = FILE[:-1] 9 | 10 | def test_add_string(): 11 | p = Pipeline() 12 | assert (p.add("foo") == None) 13 | assert (not p.empty) 14 | 15 | def test_add_file(): 16 | p = Pipeline() 17 | assert (p.add(open(FILE)) == None) 18 | assert (not p.empty) 19 | 20 | def test_add_filelike(): 21 | p = Pipeline() 22 | sio = StringIO() 23 | assert (p.add(sio) == None) 24 | assert (not p.empty) 25 | 26 | def test_add_badtypes(): 27 | p = Pipeline() 28 | class Whatever(object): pass 29 | for item in [3, [], Whatever()]: 30 | try: 31 | p.add(item) 32 | except ValueError: 33 | pass 34 | assert (p.empty) 35 | 36 | 37 | def test_read_empty(): 38 | p = Pipeline() 39 | assert (p.read(500) == '') 40 | 41 | def test_read_string(): 42 | p = Pipeline() 43 | p.add("foo") 44 | assert (p.read(3) == "foo") 45 | assert (p.empty) 46 | 47 | def test_read_file(): 48 | p = Pipeline() 49 | p.add(open(FILE)) 50 | assert (p.read(5) == "#1234") 51 | 52 | def test_read_filelike(): 53 | p = Pipeline() 54 | p.add(StringIO('abcdef')) 55 | assert (p.read(5) == 'abcde') 56 | 57 | def test_read_twice(): 58 | p = Pipeline() 59 | p.add("foo") 60 | assert (p.read(2) == "fo") 61 | assert (p.read(2) == "o") 62 | 63 | def test_read_twice_empty(): 64 | p = Pipeline() 65 | p.add("foo") 66 | assert (p.read(2) == "fo") 67 | assert (p.read(2) == "o") 68 | assert (p.read(2) == "") 69 | 70 | def test_read_backup(): 71 | p = Pipeline() 72 | p.add("foo") 73 | assert (p.read(2) == "fo") 74 | p.backup("fo") 75 | assert (p.read(2) == "fo") 76 | assert (p.read(2) == "o") 77 | 78 | def test_read_backup_extra(): 79 | p = Pipeline() 80 | p.add("foo") 81 | assert (p.read(2) == "fo") 82 | p.backup("foobar") 83 | assert (p.read(500) == "foobaro") 84 | 85 | def test_read_hybrid_objects(): 86 | p = Pipeline() 87 | p.add("foo,") 88 | p.add(StringIO("bar,")) 89 | p.add(open(FILE)) 90 | 91 | assert (p.read(10) == "foo,bar,#1") 92 | assert (p.read(4) == "2345") 93 | p.backup("rock") # in the middle of the "file" 94 | assert (p.read(6) == "rock67") 95 | 96 | def test_close(): 97 | p = Pipeline() 98 | p.add("foo") 99 | p.add(StringIO("bar")) 100 | p.close_request() 101 | assert (p.read(1000) == "foobar") 102 | try: 103 | p.read(1000) 104 | except PipelineCloseRequest: 105 | pass 106 | 107 | def test_long_1(): 108 | p = Pipeline() 109 | p.add("foo") 110 | assert (p.read(2) == "fo") 111 | p.add("bar") 112 | assert (p.read(3) == "oba") 113 | p.backup("rocko") 114 | p.add(StringIO("soma")) 115 | assert (p.read(1000) == "rockorsoma") 116 | assert (p.read(1000) == "") 117 | assert (p.empty) 118 | p.add("X" * 10000) 119 | p.close_request() 120 | assert (p.read(5000) == 'X' * 5000) 121 | p.backup('XXX') 122 | try: 123 | p.add("newstuff") 124 | except PipelineClosed: 125 | pass 126 | assert (not p.empty) 127 | assert (p.read(100000) == 'X' * 5003) 128 | assert (not p.empty) 129 | try: 130 | p.read(1000) 131 | except PipelineCloseRequest: 132 | pass 133 | assert (not p.empty) 134 | try: 135 | p.read(1000) 136 | except PipelineCloseRequest: 137 | pass 138 | 139 | def test_pri_clean(): 140 | p = Pipeline() 141 | p.add("two") 142 | p.add("three") 143 | p.add("one") 144 | assert (p.read(18) == "twothreeone") 145 | 146 | p.add("two", 2) 147 | p.add("three", 3) 148 | p.add("six", 2) 149 | p.add("one", 1) 150 | assert (p.read(18) == "threetwosixone") 151 | -------------------------------------------------------------------------------- /tests/unit/test_waiters.py: -------------------------------------------------------------------------------- 1 | from diesel.events import ( 2 | Waiter, WaitPool, EarlyValue, StringWaiter, 3 | ) 4 | 5 | 6 | class EarlyReadyWaiter(Waiter): 7 | def ready_early(self): 8 | return True 9 | 10 | def process_fire(self, value): 11 | return "foo" 12 | 13 | class Who(object): 14 | """Someone who is waiting ...""" 15 | pass 16 | 17 | class TestStringWaiters(object): 18 | def test_string_waiters_for_the_same_string_have_the_same_wait_id(self): 19 | s1 = StringWaiter("asdf") 20 | s2 = StringWaiter("asdf") 21 | assert s1.wait_id == s2.wait_id 22 | 23 | def test_string_waiters_for_different_strings_have_different_wait_ids(self): 24 | s1 = StringWaiter("asdf") 25 | s2 = StringWaiter("jkl;") 26 | assert s1.wait_id != s2.wait_id 27 | 28 | class TestWaitPoolWithEarlyReturn(object): 29 | def setup(self): 30 | self.pool = WaitPool() 31 | self.who = Who() 32 | self.waiter = EarlyReadyWaiter() 33 | self.result = self.pool.wait(self.who, self.waiter) 34 | 35 | def test_results_is_EarlyValue(self): 36 | assert isinstance(self.result, EarlyValue) 37 | assert self.result.val == "foo" 38 | 39 | def test_waiter_wait_id_not_added_to_wait_pool(self): 40 | assert self.waiter.wait_id not in self.pool.waits 41 | 42 | def test_who_not_added_to_wait_pool(self): 43 | assert self.who not in self.pool.loop_refs 44 | 45 | class TestWaitPoolWithStringWaiter(object): 46 | def setup(self): 47 | self.pool = WaitPool() 48 | self.who = Who() 49 | self.wait_for = "a string" 50 | self.result = self.pool.wait(self.who, self.wait_for) 51 | 52 | def test_the_waiting_entity_is_added_to_wait_pool(self): 53 | assert self.pool.waits[self.wait_for] 54 | w = self.pool.waits[self.wait_for].pop() 55 | assert w is self.who 56 | 57 | def test_StringWaiter_added_to_wait_pool(self): 58 | v = self.pool.loop_refs[self.who].pop() 59 | assert isinstance(v, StringWaiter) 60 | 61 | def test_result_is_wait_id(self): 62 | assert self.result == self.wait_for 63 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py26, py27 3 | 4 | [testenv] 5 | deps = 6 | nose 7 | commands = 8 | dnosetests tests/unit/ tests/integration/ 9 | --------------------------------------------------------------------------------