├── .gitignore ├── LICENSE ├── README.md ├── examples ├── echo-coro.py ├── heartbeat.py └── ircbot │ ├── basebot.py │ ├── cmdbot.py │ ├── logbot.py │ └── opbot.py ├── gogreen ├── __init__.py ├── backdoor.py ├── btree.py ├── cache.py ├── coro.py ├── corocurl.py ├── corofile.py ├── corohttpd.py ├── coromysql.py ├── coromysqlerr.py ├── coroqueue.py ├── corowork.py ├── coutil.py ├── dqueue.py ├── emulate.py ├── fileobject.py ├── purepydns.py ├── pyinfo.py ├── sendfd.py ├── start.py ├── statistics.py ├── ultramini.py └── wsgi.py ├── pavement.py ├── paver-minilib.zip ├── setup.py └── tests └── test_ultramini.py /.gitignore: -------------------------------------------------------------------------------- 1 | .*.swp 2 | *.pyc 3 | *.pyo 4 | gogreen.egg-info/* 5 | MANIFEST.in 6 | build/* 7 | docs/build/* 8 | dist/* 9 | html/* 10 | 11 | examples/ircbot/run.py 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 1999, 2000 by eGroups, Inc. 2 | Copyright (c) 2005-2010 Slide, Inc 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are 7 | met: 8 | 9 | * Redistributions of source code must retain the above copyright 10 | notice, this list of conditions and the following disclaimer. 11 | * Redistributions in binary form must reproduce the above 12 | copyright notice, this list of conditions and the following 13 | disclaimer in the documentation and/or other materials provided 14 | with the distribution. 15 | * Neither the name of the author nor the names of other 16 | contributors may be used to endorse or promote products derived 17 | from this software without specific prior written permission. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Check out the examples/ directory for usage examples, or bring questions and suggestions to IRC or the mailing list 2 | 3 | \#gogreen on freenode IRC 4 | 5 | gogreen@librelist.com 6 | -------------------------------------------------------------------------------- /examples/echo-coro.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- Mode: Python; tab-width: 4 -*- 3 | 4 | # Copyright (c) 2005-2010 Slide, Inc. 5 | # All rights reserved. 6 | # 7 | # Redistribution and use in source and binary forms, with or without 8 | # modification, are permitted provided that the following conditions are 9 | # met: 10 | # 11 | # * Redistributions of source code must retain the above copyright 12 | # notice, this list of conditions and the following disclaimer. 13 | # * Redistributions in binary form must reproduce the above 14 | # copyright notice, this list of conditions and the following 15 | # disclaimer in the documentation and/or other materials provided 16 | # with the distribution. 17 | # * Neither the name of the author nor the names of other 18 | # contributors may be used to endorse or promote products derived 19 | # from this software without specific prior written permission. 20 | # 21 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | 33 | import socket 34 | import sys 35 | import os 36 | import signal 37 | 38 | from gogreen import coro 39 | from gogreen import start 40 | 41 | ECHO_PORT = 5580 42 | 43 | class EchoClient(coro.Thread): 44 | def __init__(self, *args, **kwargs): 45 | super(EchoClient, self).__init__(*args, **kwargs) 46 | 47 | self.sock = kwargs['sock'] 48 | self.addr = kwargs['addr'] 49 | self.exit = False 50 | 51 | def run(self): 52 | self.info('Accepted connection: %r', (self.addr,)) 53 | 54 | while not self.exit: 55 | try: 56 | buf = self.sock.recv(1024) 57 | except coro.CoroutineSocketWake: 58 | continue 59 | except socket.error: 60 | buf = '' 61 | 62 | if not buf: 63 | break 64 | 65 | try: 66 | self.sock.send(buf) 67 | except coro.CoroutineSocketWake: 68 | continue 69 | except socket.error: 70 | break 71 | 72 | self.sock.close() 73 | self.info('Connection closed: %r', (self.addr,)) 74 | 75 | def shutdown(self): 76 | self.exit = True 77 | self.sock.wake() 78 | 79 | class EchoServer(coro.Thread): 80 | def __init__(self, *args, **kwargs): 81 | super(EchoServer, self).__init__(*args, **kwargs) 82 | 83 | self.addr = kwargs['addr'] 84 | self.sock = None 85 | self.exit = False 86 | 87 | def run(self): 88 | self.sock = coro.make_socket(socket.AF_INET, socket.SOCK_STREAM) 89 | self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 90 | self.sock.bind(self.addr) 91 | self.sock.listen(128) 92 | 93 | self.info('Listening to address: %r' % (self.addr,)) 94 | 95 | while not self.exit: 96 | try: 97 | conn, addr = self.sock.accept() 98 | except coro.CoroutineSocketWake: 99 | continue 100 | except Exception, e: 101 | self.error('Exception from accept: %r', e) 102 | break 103 | 104 | eclnt = EchoClient(addr = addr, sock = conn) 105 | eclnt.start() 106 | 107 | self.sock.close() 108 | 109 | for child in self.child_list(): 110 | child.shutdown() 111 | 112 | self.child_wait(30) 113 | self.info('Server exit.') 114 | 115 | def shutdown(self): 116 | self.exit = True 117 | self.sock.wake() 118 | 119 | def run(here, log, loglevel, logdir, **kwargs): 120 | eserv = EchoServer(addr = ('', ECHO_PORT)) 121 | eserv.start() 122 | 123 | def shutdown_handler(signum, frame): 124 | eserv.shutdown() 125 | 126 | signal.signal(signal.SIGUSR2, shutdown_handler) 127 | 128 | try: 129 | coro.event_loop() 130 | except KeyboardInterrupt: 131 | pass 132 | 133 | return None 134 | 135 | if __name__ == '__main__': 136 | conf = { 137 | 0 : {'lockport' : 5581, 'echo_port' : 5580}, 138 | } 139 | value = start.main( 140 | conf, 141 | run, 142 | name = 'echoserver', 143 | ) 144 | sys.exit(value) 145 | -------------------------------------------------------------------------------- /examples/heartbeat.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # 3 | '''heartbeat 4 | 5 | a common pattern is to do some work every N seconds. this example demestrates 6 | how to do that with gogreen. 7 | ''' 8 | import signal 9 | import sys 10 | 11 | from gogreen import coro 12 | from gogreen import emulate 13 | from gogreen import start 14 | from gogreen import backdoor 15 | 16 | # do monkeypatching of various things like sockets, etc. 17 | # 18 | emulate.init() 19 | 20 | class Heartbeat(coro.Thread): 21 | 22 | def __init__(self, period, *args, **kwargs): 23 | super(Heartbeat, self).__init__(*args, **kwargs) 24 | self._period = period 25 | self._cond = coro.coroutine_cond() 26 | self._exit = False 27 | 28 | def run(self): 29 | self.info('Heartbeat: start') 30 | while not self._exit: 31 | self.info('bah-bump') 32 | self._cond.wait(self._period) 33 | self.info('Heartbeat: end') 34 | 35 | def shutdown(self): 36 | self._exit = True 37 | self._cond.wake_all() 38 | 39 | def run(here, log, loglevel, logdir, **kwargs): 40 | heart = Heartbeat( 41 | 1.2, # every 1.2 seconds 42 | log = log, 43 | ) 44 | heart.set_log_level(loglevel) 45 | heart.start() 46 | 47 | if 'backport' in here: 48 | back = backdoor.BackDoorServer( 49 | args = (here['backport'],), 50 | log = log, 51 | ) 52 | back.set_log_level(loglevel) 53 | back.start() 54 | 55 | def shutdown_handler(signum, frame): 56 | heart.shutdown() 57 | back.shutdown() 58 | 59 | signal.signal(signal.SIGUSR2, shutdown_handler) 60 | 61 | try: 62 | coro.event_loop() 63 | except KeyboardInterrupt: 64 | pass 65 | 66 | return None 67 | 68 | if __name__ == '__main__': 69 | conf = { 70 | 0 : {'lockport' : 6581, 'backport' : 5500, }, 71 | } 72 | value = start.main( 73 | conf, 74 | run, 75 | name = 'heartbeat', 76 | ) 77 | sys.exit(value) 78 | -------------------------------------------------------------------------------- /examples/ircbot/basebot.py: -------------------------------------------------------------------------------- 1 | import socket 2 | 3 | from gogreen import coro 4 | 5 | 6 | class Bot(coro.Thread): 7 | def __init__(self, 8 | server_address, 9 | nick, 10 | username=None, 11 | realname=None, 12 | password=None, 13 | nickserv=None, 14 | rooms=None): 15 | super(Bot, self).__init__() 16 | self.server_address = server_address 17 | self.nick = nick 18 | self.username = username or nick 19 | self.realname = realname or nick 20 | self.password = password 21 | self.nickserv = nickserv 22 | self.rooms = rooms or [] 23 | self.in_rooms = set() 24 | self.connected = self.registered = False 25 | 26 | def cmd(self, cmd, *args): 27 | args = map(str, args) 28 | if args and " " in args[-1] and not args[-1].startswith(":"): 29 | args[-1] = ":" + args[-1] 30 | 31 | # lock this so separate outgoing commands can't overlap 32 | self.write_lock.lock() 33 | try: 34 | self.sock.sendall(" ".join([cmd.upper()] + args) + "\n") 35 | finally: 36 | self.write_lock.unlock() 37 | 38 | def connect(self): 39 | raw_sock = socket.socket() 40 | self.sock = coro.coroutine_socket(raw_sock) 41 | self.sock.connect(self.server_address) 42 | self.write_lock = coro.coroutine_lock() 43 | self.connected = True 44 | 45 | def register(self): 46 | self.cmd("nick", self.nick) 47 | self.cmd("user", self.username, 0, 0, self.realname) 48 | if self.password: 49 | if self.nickserv: 50 | self.message(self.nickserv, "identify %s" % self.password) 51 | else: 52 | self.cmd("pass", self.password) 53 | self.registered = True 54 | 55 | def change_nick(self, nick): 56 | self.nick = nick 57 | self.cmd("nick", nick) 58 | 59 | def join_rooms(self): 60 | for room in self.rooms: 61 | if room in self.in_rooms: 62 | continue 63 | if not hasattr(room, "__iter__"): 64 | room = (room,) 65 | self.join_room(*room) 66 | 67 | def join_room(self, room, password=None): 68 | if password is None: 69 | self.cmd("join", room) 70 | else: 71 | self.cmd("join", room, password) 72 | self.in_rooms.add(room) 73 | 74 | def message(self, target, message): 75 | self.cmd("privmsg", target, message) 76 | 77 | def quit(self, message=None): 78 | self.child_wait() 79 | if message is None: 80 | self.cmd("quit") 81 | else: 82 | self.cmd("quit", message) 83 | 84 | def run(self): 85 | self.running = True 86 | 87 | if not self.connected: 88 | self.connect() 89 | if not self.registered: 90 | self.register() 91 | self.join_rooms() 92 | 93 | sockfile = self.sock.makefile() 94 | 95 | while self.running: 96 | line = sockfile.readline() 97 | 98 | if not line: 99 | break 100 | 101 | worker = BotWorker(self, args=(line,)) 102 | worker.start() 103 | 104 | def on_ping(self, cmd, args, prefix): 105 | self.cmd("pong", args[0]) 106 | 107 | def default_handler(self, cmd, args, prefix): 108 | pass 109 | 110 | def default_on_reply(self, code, args, prefix): 111 | pass 112 | 113 | def post_register(self): 114 | pass 115 | 116 | 117 | class BotWorker(coro.Thread): 118 | def __init__(self, bot, *args, **kwargs): 119 | super(BotWorker, self).__init__(*args, **kwargs) 120 | self.bot = bot 121 | 122 | def _parse_line(self, line): 123 | prefix = None 124 | if line.startswith(":"): 125 | prefix, line = line.split(" ", 1) 126 | prefix = prefix[1:] 127 | 128 | cmd, line = line.split(" ", 1) 129 | 130 | if line.startswith(":"): 131 | args = [line[1:]] 132 | elif " :" in line: 133 | args, final_arg = line.split(" :", 1) 134 | args = args.split(" ") 135 | args.append(final_arg) 136 | else: 137 | args = line.split(" ") 138 | 139 | return cmd, args, prefix 140 | 141 | def run(self, line): 142 | cmd, args, prefix = self._parse_line(line.rstrip('\r\n')) 143 | if cmd.isdigit(): 144 | cmd = int(cmd) 145 | if _code_is_error(cmd): 146 | raise IRCServerError(cmd, REPLY_CODES[cmd], *args) 147 | handler = getattr( 148 | self.bot, "on_reply_%d" % cmd, self.bot.default_on_reply) 149 | else: 150 | handler = getattr( 151 | self.bot, "on_%s" % cmd.lower(), self.bot.default_handler) 152 | 153 | handler(cmd, args, prefix) 154 | 155 | 156 | class IRCError(Exception): 157 | pass 158 | 159 | 160 | class IRCServerError(IRCError): 161 | pass 162 | 163 | 164 | def _code_is_error(code): 165 | return code >= 400 166 | 167 | # http://irchelp.org/irchelp/rfc/chapter6.html 168 | REPLY_CODES = { 169 | 200: "RPL_TRACELINK", 170 | 201: "RPL_TRACECONNECTING", 171 | 202: "RPL_TRACEHANDSHAKE", 172 | 203: "RPL_TRACEUNKNOWN", 173 | 204: "RPL_TRACEOPERATOR", 174 | 205: "RPL_TRACEUSER", 175 | 206: "RPL_TRACESERVER", 176 | 208: "RPL_TRACENEWTYPE", 177 | 211: "RPL_STATSLINKINFO", 178 | 212: "RPL_STATSCOMMANDS", 179 | 213: "RPL_STATSCLINE", 180 | 214: "RPL_STATSNLINE", 181 | 215: "RPL_STATSILINE", 182 | 216: "RPL_STATSKLINE", 183 | 218: "RPL_STATSYLINE", 184 | 219: "RPL_ENDOFSTATS", 185 | 221: "RPL_UMODEIS", 186 | 241: "RPL_STATSLLINE", 187 | 242: "RPL_STATSUPTIME", 188 | 243: "RPL_STATSOLINE", 189 | 244: "RPL_STATSHLINE", 190 | 251: "RPL_LUSERCLIENT", 191 | 252: "RPL_LUSEROP", 192 | 253: "RPL_LUSERUNKNOWN", 193 | 254: "RPL_LUSERCHANNELS", 194 | 255: "RPL_LUSERME", 195 | 256: "RPL_ADMINME", 196 | 257: "RPL_ADMINLOC1", 197 | 258: "RPL_ADMINLOC2", 198 | 259: "RPL_ADMINEMAIL", 199 | 261: "RPL_TRACELOG", 200 | 300: "RPL_NONE", 201 | 301: "RPL_AWAY", 202 | 302: "RPL_USERHOST", 203 | 303: "RPL_ISON", 204 | 305: "RPL_UNAWAY", 205 | 306: "RPL_NOAWAY", 206 | 311: "RPL_WHOISUSER", 207 | 312: "RPL_WHOISSERVER", 208 | 313: "RPL_WHOISOPERATOR", 209 | 314: "RPL_WHOWASUSER", 210 | 315: "RPL_ENDOFWHO", 211 | 317: "RPL_WHOISIDLE", 212 | 318: "RPL_ENDOFWHOIS", 213 | 319: "RPL_WHOISCHANNELS", 214 | 321: "RPL_LISTSTART", 215 | 322: "RPL_LIST", 216 | 323: "RPL_LISTEND", 217 | 324: "RPL_CHANNELMODEIS", 218 | 331: "RPL_NOTOPIC", 219 | 332: "RPL_TOPIC", 220 | 341: "RPL_INVITING", 221 | 342: "RPL_SUMMONING", 222 | 351: "RPL_VERSION", 223 | 352: "RPL_WHOREPLY", 224 | 353: "RPL_NAMREPLY", 225 | 364: "RPL_LINKS", 226 | 365: "RPL_ENDOFLINKS", 227 | 366: "RPL_ENDOFNAMES", 228 | 367: "RPL_BANLIST", 229 | 368: "RPL_ENDOFBANLIST", 230 | 369: "RPL_ENDOFWHOWAS", 231 | 371: "RPL_INFO", 232 | 372: "RPL_MOTD", 233 | 374: "RPL_ENDOFINFO", 234 | 375: "RPL_MOTDSTART", 235 | 376: "RPL_ENDOFMOTD", 236 | 381: "RPL_YOUREOPER", 237 | 382: "RPL_REHASHING", 238 | 391: "RPL_TIME", 239 | 392: "RPL_USERSSTART", 240 | 393: "RPL_USERS", 241 | 394: "RPL_ENDOFUSERS", 242 | 395: "RPL_NOUSERS", 243 | 244 | 401: "ERR_NOSUCHNICK", 245 | 402: "ERR_NOSUCHSERVER", 246 | 403: "ERR_NOSUCHCHANNEL", 247 | 404: "ERR_CANNOTSENDTOCHAN", 248 | 405: "ERR_TOOMANYCHANNELS", 249 | 406: "ERR_WASNOSUCHNICK", 250 | 407: "ERR_TOOMANYTARGETS", 251 | 409: "ERR_NOORIGIN", 252 | 411: "ERR_NORECIPIENT", 253 | 412: "ERR_NOTEXTTOSEND", 254 | 413: "ERR_NOTOPLEVEL", 255 | 414: "ERR_WILDTOPLEVEL", 256 | 421: "ERR_UNKNOWNCOMMAND", 257 | 422: "ERR_NOMOTD", 258 | 423: "ERR_NOADMININFO", 259 | 424: "ERR_FILEERROR", 260 | 431: "ERR_NONICKNAMEGIVEN", 261 | 432: "ERR_ERRONEUSENICKNAME", 262 | 433: "ERR_NICKNAMEINUSE", 263 | 436: "ERR_NICKCOLLISION", 264 | 441: "ERR_USERNOTINCHANNEL", 265 | 442: "ERR_NOTONCHANNEL", 266 | 443: "ERR_USERONCHANNEL", 267 | 444: "ERR_NOLOGIN", 268 | 445: "ERR_SUMMONDISABLED", 269 | 446: "ERR_USERSDISABLED", 270 | 451: "ERR_NOTREGISTERED", 271 | 461: "ERR_NEEDMOREPARAMS", 272 | 462: "ERR_ALREADYREGISTERED", 273 | 463: "ERR_NOPERFORMHOST", 274 | 464: "ERR_PASSWDMISMATCH", 275 | 465: "ERR_YOUREBANNEDCREEP", 276 | 467: "ERR_KEYSET", 277 | 471: "ERR_CHANNELISFULL", 278 | 472: "ERR_UNKNOWNMODE", 279 | 473: "ERR_INVITEONLYCHAN", 280 | 474: "ERR_BANNEDFROMCHAN", 281 | 475: "ERR_BADCHANNELKEY", 282 | 481: "ERR_NOPRIVILEGES", 283 | 482: "ERR_CHANOPRIVSNEEDED", 284 | 483: "ERR_CANTKILLSERVER", 285 | 491: "ERR_NOOPERHOST", 286 | 501: "ERR_UMODEUNKNOWNFLAG", 287 | 502: "ERR_USERSDONTMATCH", 288 | } 289 | -------------------------------------------------------------------------------- /examples/ircbot/cmdbot.py: -------------------------------------------------------------------------------- 1 | import basebot 2 | 3 | 4 | class CmdBot(basebot.Bot): 5 | class commander(object): 6 | def __init__(self, bot): 7 | self.bot = bot 8 | 9 | def ping(self, rest): 10 | return 'pong' 11 | 12 | def __init__(self, *args, **kwargs): 13 | super(CmdBot, self).__init__(*args, **kwargs) 14 | self.cmds = self.commander(self) 15 | 16 | def on_privmsg(self, cmd, args, prefix): 17 | parent = super(CmdBot, self) 18 | if hasattr(parent, "on_privmsg"): 19 | parent.on_privmsg(cmd, args, prefix) 20 | 21 | sender = prefix.split("!", 1)[0] 22 | to, msg = args 23 | 24 | if to in self.in_rooms: 25 | if not msg.startswith(self.nick): 26 | return 27 | rest = msg[len(self.nick):].lstrip(": ") 28 | else: 29 | rest = msg 30 | 31 | if " " in rest: 32 | cmd_name, rest = rest.split(" ", 1) 33 | else: 34 | cmd_name, rest = rest, "" 35 | 36 | handler = getattr(self.cmds, cmd_name, None) 37 | result = None 38 | if handler: 39 | result = handler(rest) 40 | 41 | if result: 42 | if to in self.in_rooms: 43 | self.message(to, "%s: %s" % (sender, result)) 44 | elif to == self.nick: 45 | self.message(sender, result) 46 | else: 47 | raise basebot.IRCError("unknown target: %s" % to) 48 | -------------------------------------------------------------------------------- /examples/ircbot/logbot.py: -------------------------------------------------------------------------------- 1 | import logging.handlers 2 | 3 | import basebot 4 | 5 | 6 | class LogBot(basebot.Bot): 7 | filename_format = "%s.log" 8 | message_format = "[%(asctime)s] %(message)s" 9 | 10 | def __init__(self, *args, **kwargs): 11 | super(LogBot, self).__init__(*args, **kwargs) 12 | self.logs = {} 13 | 14 | def join_room(self, room, password=None): 15 | super(LogBot, self).join_room(room, password) 16 | 17 | handler = logging.handlers.RotatingFileHandler( 18 | self.filename_format % room, 'a', 6291456, 5) 19 | handler.setFormatter(logging.Formatter(self.message_format)) 20 | logger = logging.Logger(room) 21 | logger.addHandler(handler) 22 | self.logs[room] = logger 23 | 24 | self.message(room, "(this conversation is being recorded)") 25 | 26 | def on_privmsg(self, cmd, args, prefix): 27 | parent = super(LogBot, self) 28 | if hasattr(parent, "on_privmsg"): 29 | parent.on_privmsg(cmd, args, prefix) 30 | 31 | sender = prefix.split("!", 1)[0] 32 | to, msg = args 33 | if to in self.logs: 34 | self.logs[to].info("<%s> %s" % (sender, msg)) 35 | 36 | def on_join(self, cmd, args, prefix): 37 | parent = super(LogBot, self) 38 | if hasattr(parent, "on_join"): 39 | parent.on_join(cmd, args, prefix) 40 | 41 | sender = prefix.split("!", 1)[0] 42 | room = args[0] 43 | 44 | if room in self.logs: 45 | self.logs[room].info("<%s> joined %s" % (sender, room)) 46 | 47 | def on_part(self, cmd, args, prefix): 48 | parent = super(LogBot, self) 49 | if hasattr(parent, "on_part"): 50 | parent.on_part(cmd, args, prefix) 51 | 52 | sender = prefix.split("!", 1)[0] 53 | room = args[0] 54 | 55 | if room in self.logs: 56 | self.logs[room].info("<%s> left %s" % (sender, room)) 57 | 58 | def cmd(self, cmd, *args): 59 | super(LogBot, self).cmd(cmd, *args) 60 | 61 | if cmd.lower() == "privmsg": 62 | target, message = args 63 | if target in self.logs: 64 | self.logs[target].info("<%s> %s" % (self.nick, message)) 65 | -------------------------------------------------------------------------------- /examples/ircbot/opbot.py: -------------------------------------------------------------------------------- 1 | import basebot 2 | 3 | 4 | class OpsHolderBot(basebot.Bot): 5 | '''A most inactive bot, it will just sit in a room and hold any chanops you 6 | give it, doling out chanops upon request (when you provide the password) 7 | 8 | to get it to op you, /msg it with the password, a space, and the room name 9 | 10 | to start it up, do something like this: 11 | >>> bot = OpsHolderBot( 12 | >>> ("example.irc.server.com", 6667), 13 | >>> "bot_nick", 14 | >>> password=irc_password_for_nick, 15 | >>> ops_password=password_for_giving_chanops, 16 | >>> rooms=["#list", "#of", "#channels", "#to", "#serve"]) 17 | >>> bot.start() 18 | >>> coro.event_loop() 19 | ''' 20 | def __init__(self, *args, **kwargs): 21 | self._ops_password = kwargs.pop("ops_password") 22 | super(OpsHolderBot, self).__init__(*args, **kwargs) 23 | self._chanops = {} 24 | 25 | def join_room(self, room, passwd=None): 26 | super(OpsHolderBot, self).join_room(room, passwd) 27 | self._chanops[room] = False 28 | 29 | def on_mode(self, cmd, args, prefix): 30 | parent = super(OpsHolderBot, self) 31 | if hasattr(parent, "on_mode"): 32 | parent.on_mode(cmd, args, prefix) 33 | 34 | # MODE is how another user would give us ops -- test for that case 35 | context, mode_change = args[:2] 36 | if context.startswith("#") and \ 37 | mode_change.startswith("+") and \ 38 | "o" in mode_change and\ 39 | len(args) > 2 and args[2] == self.nick: 40 | self._chanops[context] = True 41 | 42 | def on_privmsg(self, cmd, args, prefix): 43 | parent = super(OpsHolderBot, self) 44 | if hasattr(parent, "on_privmsg"): 45 | parent.on_privmsg(cmd, args, prefix) 46 | 47 | # ignore channels 48 | if args[0] != self.nick: 49 | return 50 | 51 | sender = prefix.split("!", 1)[0] 52 | 53 | if " " not in args[1]: 54 | return 55 | passwd, roomname = args[1].rsplit(" ", 1) 56 | 57 | if passwd == self._ops_password: 58 | if self._chanops.get(roomname): 59 | # grant chanops 60 | self.cmd("mode", roomname, "+o", sender) 61 | else: 62 | self.message(sender, "I don't have ops to give in that room") 63 | else: 64 | self.message(sender, "sorry, wrong password") 65 | 66 | def on_reply_353(self, code, args, prefix): 67 | parent = super(OpsHolderBot, self) 68 | if hasattr(parent, "on_reply_353"): 69 | parent.on_reply_353(cmd, args, prefix) 70 | 71 | # 353 is sent when joining a room -- this tests to see if we are given 72 | # chanops upon entrance (we are the first here, creating the room) 73 | names = args[-1] 74 | if "@" + self.nick in names.split(" "): 75 | self._chanops[args[-2]] = True 76 | -------------------------------------------------------------------------------- /gogreen/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 1999, 2000 by eGroups, Inc. 2 | # Copyright (c) 2005-2010 Slide, Inc. 3 | # All rights reserved. 4 | # 5 | # Redistribution and use in source and binary forms, with or without 6 | # modification, are permitted provided that the following conditions are 7 | # met: 8 | # 9 | # * Redistributions of source code must retain the above copyright 10 | # notice, this list of conditions and the following disclaimer. 11 | # * Redistributions in binary form must reproduce the above 12 | # copyright notice, this list of conditions and the following 13 | # disclaimer in the documentation and/or other materials provided 14 | # with the distribution. 15 | # * Neither the name of the author nor the names of other 16 | # contributors may be used to endorse or promote products derived 17 | # from this software without specific prior written permission. 18 | # 19 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | """gogreen 32 | 33 | Combination of the egroups coroutine utilities with the py.lib 34 | greenlet coroutine implementation. 35 | """ 36 | -------------------------------------------------------------------------------- /gogreen/backdoor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- Mode: Python; tab-width: 4 -*- 3 | 4 | # Copyright (c) 1999, 2000 by eGroups, Inc. 5 | # Copyright (c) 2005-2010 Slide, Inc. 6 | # All rights reserved. 7 | # 8 | # Redistribution and use in source and binary forms, with or without 9 | # modification, are permitted provided that the following conditions are 10 | # met: 11 | # 12 | # * Redistributions of source code must retain the above copyright 13 | # notice, this list of conditions and the following disclaimer. 14 | # * Redistributions in binary form must reproduce the above 15 | # copyright notice, this list of conditions and the following 16 | # disclaimer in the documentation and/or other materials provided 17 | # with the distribution. 18 | # * Neither the name of the author nor the names of other 19 | # contributors may be used to endorse or promote products derived 20 | # from this software without specific prior written permission. 21 | # 22 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 23 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 24 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 25 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 26 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 27 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 28 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 32 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | 34 | import coro 35 | import socket 36 | import string 37 | import StringIO 38 | import sys 39 | import traceback 40 | 41 | # Originally, this object implemented the file-output api, and set 42 | # sys.stdout and sys.stderr to 'self'. However, if any other 43 | # coroutine ran, it would see the captured definition of sys.stdout, 44 | # and would send its output here, instead of the expected place. Now 45 | # the code captures all output using StringIO. A little less 46 | # flexible, a little less efficient, but much less surprising! 47 | # [Note: this is exactly the same problem addressed by Scheme 48 | # dynamic-wind facility] 49 | 50 | class BackDoorClient(coro.Thread): 51 | def init(self): 52 | self.address = None 53 | self.socket = None 54 | self.buffer = '' 55 | self.lines = [] 56 | self.exit = False 57 | self.multilines = [] 58 | self.line_separator = '\r\n' 59 | # 60 | # allow the user to change the prompts: 61 | # 62 | if not sys.__dict__.has_key('ps1'): 63 | sys.ps1 = '>>> ' 64 | if not sys.__dict__.has_key('ps2'): 65 | sys.ps2 = '... ' 66 | 67 | def send (self, data): 68 | olb = lb = len(data) 69 | while lb: 70 | ns = self.socket.send (data) 71 | lb = lb - ns 72 | return olb 73 | 74 | def prompt (self): 75 | if self.multilines: 76 | self.send (sys.ps2) 77 | else: 78 | self.send (sys.ps1) 79 | 80 | def read_line (self): 81 | if self.lines: 82 | l = self.lines[0] 83 | self.lines = self.lines[1:] 84 | return l 85 | else: 86 | while not self.lines: 87 | block = self.socket.recv (8192) 88 | if not block: 89 | return None 90 | elif block == '\004': 91 | self.socket.close() 92 | return None 93 | else: 94 | self.buffer = self.buffer + block 95 | lines = string.split (self.buffer, self.line_separator) 96 | for l in lines[:-1]: 97 | self.lines.append (l) 98 | self.buffer = lines[-1] 99 | return self.read_line() 100 | 101 | def run(self, conn, addr): 102 | self.socket = conn 103 | self.address = addr 104 | 105 | # a call to socket.setdefaulttimeout will mean that this backdoor 106 | # has a timeout associated with it. to counteract this set the 107 | # socket timeout to None here. 108 | self.socket.settimeout(None) 109 | 110 | self.info('Incoming backdoor connection from %r' % (self.address,)) 111 | # 112 | # print header for user 113 | # 114 | self.send ('Python ' + sys.version + self.line_separator) 115 | self.send (sys.copyright + self.line_separator) 116 | # 117 | # this does the equivalent of 'from __main__ import *' 118 | # 119 | env = sys.modules['__main__'].__dict__.copy() 120 | # 121 | # wait for imput and process 122 | # 123 | while not self.exit: 124 | self.prompt() 125 | try: 126 | line = self.read_line() 127 | except coro.CoroutineSocketWake: 128 | continue 129 | 130 | if line is None: 131 | break 132 | elif self.multilines: 133 | self.multilines.append(line) 134 | if line == '': 135 | code = string.join(self.multilines, '\n') 136 | self.parse(code, env) 137 | # we do this after the parsing so parse() knows not to do 138 | # a second round of multiline input if it really is an 139 | # unexpected EOF 140 | self.multilines = [] 141 | else: 142 | self.parse(line, env) 143 | 144 | self.info('Backdoor connection closing') 145 | 146 | self.socket.close() 147 | self.socket = None 148 | return None 149 | 150 | def parse(self, line, env): 151 | save = sys.stdout, sys.stderr 152 | output = StringIO.StringIO() 153 | try: 154 | try: 155 | sys.stdout = sys.stderr = output 156 | co = compile (line, repr(self), 'eval') 157 | result = eval (co, env) 158 | if result is not None: 159 | print repr(result) 160 | env['_'] = result 161 | except SyntaxError: 162 | try: 163 | co = compile (line, repr(self), 'exec') 164 | exec co in env 165 | except SyntaxError, msg: 166 | # this is a hack, but it is a righteous hack: 167 | if not self.multilines and str(msg) == 'unexpected EOF while parsing': 168 | self.multilines.append(line) 169 | else: 170 | traceback.print_exc() 171 | except: 172 | traceback.print_exc() 173 | except: 174 | traceback.print_exc() 175 | finally: 176 | sys.stdout, sys.stderr = save 177 | self.send (output.getvalue()) 178 | del output 179 | 180 | def shutdown(self): 181 | if not self.exit: 182 | self.exit = True 183 | self.socket.wake() 184 | 185 | 186 | class BackDoorServer(coro.Thread): 187 | def init(self): 188 | self._exit = False 189 | self._s = None 190 | 191 | def run(self, port=8023, ip=''): 192 | self._s = coro.make_socket(socket.AF_INET, socket.SOCK_STREAM) 193 | self._s.set_reuse_addr() 194 | self._s.bind((ip, port)) 195 | self._s.listen(1024) 196 | 197 | port = self._s.getsockname()[1] 198 | self.info('Backdoor listening on port %d' % (port,)) 199 | 200 | while not self._exit: 201 | try: 202 | conn, addr = self._s.accept() 203 | except coro.CoroutineSocketWake: 204 | continue 205 | 206 | client = BackDoorClient(args = (conn, addr)) 207 | client.start() 208 | 209 | self.info('Backdoor exiting (children: %d)' % self.child_count()) 210 | 211 | self._s.close() 212 | self._s = None 213 | 214 | for child in self.child_list(): 215 | child.shutdown() 216 | 217 | self.child_wait() 218 | return None 219 | 220 | def shutdown(self): 221 | if self._exit: 222 | return None 223 | 224 | self._exit = True 225 | 226 | if self._s is not None: 227 | self._s.wake() 228 | # 229 | # extremely minimal test server 230 | # 231 | if __name__ == '__main__': 232 | server = BackDoorServer() 233 | server.start() 234 | coro.event_loop (30.0) 235 | # 236 | # end... 237 | -------------------------------------------------------------------------------- /gogreen/btree.py: -------------------------------------------------------------------------------- 1 | # -*- Mode: Python; tab-width: 4 -*- 2 | 3 | # Copyright (c) 2010 Slide, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are 8 | # met: 9 | # 10 | # * Redistributions of source code must retain the above copyright 11 | # notice, this list of conditions and the following disclaimer. 12 | # * Redistributions in binary form must reproduce the above 13 | # copyright notice, this list of conditions and the following 14 | # disclaimer in the documentation and/or other materials provided 15 | # with the distribution. 16 | # * Neither the name of the author nor the names of other 17 | # contributors may be used to endorse or promote products derived 18 | # from this software without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | import bisect 33 | import itertools 34 | import operator 35 | 36 | 37 | class BTreeNode(object): 38 | def shrink(self, path): 39 | parent = None 40 | if path: 41 | parent, parent_index = path.pop() 42 | 43 | # first, try to pass a value left 44 | if parent_index: 45 | left = parent.children[parent_index - 1] 46 | if len(left.values) < left.order: 47 | parent.neighbor_pass_left(self, parent_index) 48 | return 49 | 50 | # second, try to pass one right 51 | if parent_index + 1 < len(parent.children): 52 | right = parent.children[parent_index + 1] 53 | if len(right.values) < right.order: 54 | parent.neighbor_pass_right(self, parent_index) 55 | return 56 | 57 | # finally, split the current node, then shrink the parent if we must 58 | center = len(self.values) // 2 59 | median = self.values[center] 60 | 61 | # create a sibling node with the second half of our data 62 | args = [self.tree, self.values[center + 1:]] 63 | if self.BRANCH: 64 | args.append(self.children[center + 1:]) 65 | sibling = type(self)(*args) 66 | 67 | # cut our data down to the first half 68 | self.values = self.values[:center] 69 | if self.BRANCH: 70 | self.children = self.children[:center + 1] 71 | 72 | if not parent: 73 | parent = self.tree.BRANCH_NODE(self.tree, [], [self]) 74 | parent_index = 0 75 | self.tree._root = parent 76 | 77 | # pass the median element up to the parent 78 | parent.values.insert(parent_index, median) 79 | parent.children.insert(parent_index + 1, sibling) 80 | if len(parent.values) > parent.order: 81 | parent.shrink(path) 82 | 83 | def grow(self, path, count=1): 84 | parent, parent_index = path.pop() 85 | minimum = self.order // 2 86 | left, right = None, None 87 | 88 | # first try to borrow from the right sibling 89 | if parent_index + 1 < len(parent.children): 90 | right = parent.children[parent_index + 1] 91 | if len(right.values) - count >= minimum: 92 | parent.neighbor_pass_left(right, parent_index + 1, count) 93 | return 94 | 95 | # then try borrowing from the left sibling 96 | if parent_index: 97 | left = parent.children[parent_index - 1] 98 | if len(left.values) - count >= minimum: 99 | parent.neighbor_pass_right(left, parent_index - 1, count) 100 | return 101 | 102 | # see if we can borrow a few from both 103 | if count > 1 and left and right: 104 | lspares = len(left.values) - minimum 105 | rspares = len(right.values) - minimum 106 | if lspares + rspares >= count: 107 | # distribute the pulling evenly between the two neighbors 108 | even_remaining = lspares + rspares - count 109 | from_right = rspares - (even_remaining // 2) 110 | parent.neighbor_pass_right(left, parent_index - 1, from_left) 111 | parent.neighbor_pass_left(right, parent_index + 1, from_right) 112 | return 113 | 114 | # consolidate with a sibling -- try left first 115 | if left: 116 | left.values.append(parent.values.pop(parent_index - 1)) 117 | left.values.extend(self.values) 118 | if self.BRANCH: 119 | left.children.extend(self.children) 120 | parent.children.pop(parent_index) 121 | else: 122 | self.values.append(parent.values.pop(parent_index)) 123 | self.values.extend(right.values) 124 | if self.BRANCH: 125 | self.children.extend(right.children) 126 | parent.children.pop(parent_index + 1) 127 | 128 | if len(parent.values) < minimum: 129 | if path: 130 | # parent is not the root 131 | parent.grow(path) 132 | elif not parent.values: 133 | # parent is root and is now empty 134 | self.tree._root = left or self 135 | 136 | def __repr__(self): 137 | name = self.BRANCH and "BRANCH" or "LEAF" 138 | return "<%s %s>" % (name, ", ".join(map(str, self.values))) 139 | 140 | 141 | class BTreeBranchNode(BTreeNode): 142 | BRANCH = True 143 | __slots__ = ["tree", "order", "values", "children"] 144 | 145 | def __init__(self, tree, values, children): 146 | self.tree = tree 147 | self.order = tree.order 148 | self.values = values 149 | self.children = children 150 | 151 | def neighbor_pass_right(self, child, child_index, count=1): 152 | separator_index = child_index 153 | target = self.children[child_index + 1] 154 | index = len(child.values) - count 155 | 156 | target.values[0:0] = (child.values[index + 1:] + 157 | [self.values[separator_index]]) 158 | self.values[separator_index] = child.values[index] 159 | child.values[index:] = [] 160 | 161 | if child.BRANCH: 162 | target.children[0:0] = child.children[-count:] 163 | child.children[-count:] = [] 164 | 165 | def neighbor_pass_left(self, child, child_index, count=1): 166 | separator_index = child_index - 1 167 | target = self.children[child_index - 1] 168 | 169 | target.values.extend([self.values[separator_index]] + 170 | child.values[:count - 1]) 171 | self.values[separator_index] = child.values[count - 1] 172 | child.values[:count] = [] 173 | 174 | if child.BRANCH: 175 | index = len(child.values) + 1 176 | target.children.extend(child.children[:count]) 177 | child.children[:count] = [] 178 | 179 | def remove(self, index, path): 180 | minimum = self.order // 2 181 | 182 | # try replacing the to-be removed item from the right subtree first 183 | to_leaf = [(self, index + 1)] 184 | descendent = self.children[index + 1] 185 | while descendent.BRANCH: 186 | to_leaf.append((descendent, 0)) 187 | descendent = descendent.children[0] 188 | if len(descendent.values) > minimum: 189 | path.extend(to_leaf) 190 | self.values[index] = descendent.values[0] 191 | descendent.remove(0, path) 192 | return 193 | 194 | # fall back to promoting from the left subtree 195 | to_leaf = [(self, index)] 196 | descendent = self.children[index] 197 | while descendent.BRANCH: 198 | to_leaf.append((descendent, len(descendent.children) - 1)) 199 | descendent = descendent.children[-1] 200 | path.extend(to_leaf) 201 | self.values[index] = descendent.values[-1] 202 | descendent.remove(len(descendent.values) - 1, path) 203 | 204 | def split(self, value): 205 | index = bisect.bisect_right(self.values, value) 206 | child = self.children[index] 207 | 208 | left = type(self)(self.tree, self.values[:index], self.children[:index]) 209 | 210 | self.values = self.values[index:] 211 | self.children = self.children[index + 1:] 212 | 213 | # right here both left and self has the same number of children as 214 | # values -- but the relevant child hasn't been split yet, so we'll add 215 | # the two resultant children to the respective child list 216 | 217 | left_child, right_child = child.split(value) 218 | left.children.append(left_child) 219 | self.children.insert(0, right_child) 220 | 221 | return left, self 222 | 223 | 224 | class BTreeLeafNode(BTreeNode): 225 | BRANCH = False 226 | __slots__ = ["tree", "order", "values"] 227 | 228 | def __init__(self, tree, values): 229 | self.tree = tree 230 | self.order = tree.order 231 | self.values = values 232 | 233 | def remove(self, index, path): 234 | self.values.pop(index) 235 | if path and len(self.values) < self.order // 2: 236 | self.grow(path) 237 | 238 | def split(self, value): 239 | index = bisect.bisect_right(self.values, value) 240 | 241 | left = type(self)(self.tree, self.values[:index]) 242 | 243 | self.values = self.values[index:] 244 | 245 | self.tree._first = self 246 | 247 | return left, self 248 | 249 | 250 | class BTree(object): 251 | BRANCH_NODE = BTreeBranchNode 252 | LEAF_NODE = BTreeLeafNode 253 | 254 | def __init__(self, order): 255 | self.order = order 256 | self._root = self._first = self.LEAF_NODE(self, []) 257 | 258 | def __nonzero__(self): 259 | return bool(self._root.values) 260 | 261 | @property 262 | def first(self): 263 | if not self: 264 | return None 265 | return self._first.values[0] 266 | 267 | def insert(self, value, after=False): 268 | path = self.find_path_to_leaf(value, after) 269 | node, index = path.pop() 270 | 271 | node.values.insert(index, value) 272 | 273 | if len(node.values) > self.order: 274 | node.shrink(path) 275 | 276 | def remove(self, value, last=True): 277 | test = last and self._test_right or self._test_left 278 | path = self.find_path(value, last) 279 | node, index = path.pop() 280 | 281 | if test(node.values, index, value): 282 | if last: 283 | index -= 1 284 | node.remove(index, path) 285 | else: 286 | raise ValueError("%r not in %s" % (value, self.__class__.__name__)) 287 | 288 | def __repr__(self): 289 | def recurse(node, accum, depth): 290 | accum.append((" " * depth) + repr(node)) 291 | if node.BRANCH: 292 | for child in node.children: 293 | recurse(child, accum, depth + 1) 294 | 295 | accum = [] 296 | recurse(self._root, accum, 0) 297 | return "\n".join(accum) 298 | 299 | def _test_right(self, values, index, value): 300 | return index and values[index - 1] == value 301 | 302 | def _test_left(self, values, index, value): 303 | return index < len(values) and values[index] == value 304 | 305 | def find_path(self, value, after=False): 306 | cut = after and bisect.bisect_right or bisect.bisect_left 307 | test = after and self._test_right or self._test_left 308 | 309 | path, node = [], self._root 310 | index = cut(node.values, value) 311 | path.append((node, index)) 312 | 313 | while node.BRANCH and not test(node.values, index, value): 314 | node = node.children[index] 315 | index = cut(node.values, value) 316 | path.append((node, index)) 317 | 318 | return path 319 | 320 | def find_path_to_leaf(self, value, after=False): 321 | cut = after and bisect.bisect_right or bisect.bisect_left 322 | 323 | path = self.find_path(value, after) 324 | node, index = path[-1] 325 | 326 | while node.BRANCH: 327 | node = node.children[index] 328 | index = cut(node.values, value) 329 | path.append((node, index)) 330 | 331 | return path 332 | 333 | def _iter_recurse(self, node): 334 | if node.BRANCH: 335 | for child, value in itertools.izip(node.children, node.values): 336 | for ancestor_value in self._iter_recurse(child): 337 | yield ancestor_value 338 | 339 | yield value 340 | 341 | for value in self._iter_recurse(node.children[-1]): 342 | yield value 343 | else: 344 | for value in node.values: 345 | yield value 346 | 347 | def __iter__(self): 348 | return self._iter_recurse(self._root) 349 | 350 | def pull_prefix(self, value): 351 | ''' 352 | get and remove the prefix section of the btree up to and 353 | including all values for `value`, and return it as a list 354 | 355 | http://www.chiark.greenend.org.uk/~sgtatham/tweak/btree.html#S6.2 356 | ''' 357 | left, right = self._root.split(value) 358 | 359 | # first eliminate redundant roots 360 | while self._root.BRANCH and not self._root.values: 361 | self._root = self._root.children[0] 362 | 363 | # next traverse down, rebalancing as we go 364 | if self._root.BRANCH: 365 | path = [(self._root, 0)] 366 | node = self._root.children[0] 367 | 368 | while node.BRANCH: 369 | short_by = (node.order // 2) - len(node.values) 370 | if short_by > 0: 371 | node.grow(path[:], short_by + 1) 372 | path.append((node, 0)) 373 | node = node.children[0] 374 | 375 | short_by = (node.order // 2) - len(node.values) 376 | if short_by > 0: 377 | node.grow(path[:], short_by + 1) 378 | 379 | throwaway = object.__new__(type(self)) 380 | throwaway._root = left # just using you for your __iter__ 381 | return iter(throwaway) 382 | 383 | @classmethod 384 | def bulkload(cls, values, order): 385 | tree = object.__new__(cls) 386 | tree.order = order 387 | 388 | minimum = order // 2 389 | valuegroups, separators = [[]], [] 390 | 391 | for value in values: 392 | if len(valuegroups[-1]) < order: 393 | valuegroups[-1].append(value) 394 | else: 395 | separators.append(value) 396 | valuegroups.append([]) 397 | 398 | if len(valuegroups[-1]) < minimum and separators: 399 | sep_value = separators.pop() 400 | last_two_values = valuegroups[-2] + [sep_value] + valuegroups[-1] 401 | valuegroups[-2] = last_two_values[:minimum] 402 | valuegroups[-1] = last_two_values[minimum + 1:] 403 | separators.append(last_two_values[minimum]) 404 | 405 | last_generation = [] 406 | for values in valuegroups: 407 | last_generation.append(cls.LEAF_NODE(tree, values)) 408 | 409 | tree._first = last_generation[0] 410 | 411 | if not separators: 412 | tree._root = last_generation[0] 413 | return tree 414 | 415 | while len(separators) > order + 1: 416 | pairs, separators = separators, [] 417 | last_values, values = values, [[]] 418 | 419 | for value in pairs: 420 | if len(values[-1]) < order: 421 | values[-1].append(value) 422 | else: 423 | separators.append(value) 424 | values.append([]) 425 | 426 | if len(values[-1]) < minimum and separators: 427 | sep_value = separators.pop() 428 | last_two_values = values[-2] + [sep_value] + values[-1] 429 | values[-2] = last_two_values[:minimum] 430 | values[-1] = last_two_values[minimum + 1:] 431 | separators.append(last_two_values[minimum]) 432 | 433 | offset = 0 434 | for i, value_group in enumerate(values): 435 | children = last_generation[offset:offset + len(value_group) + 1] 436 | values[i] = cls.BRANCH_NODE(tree, value_group, children) 437 | offset += len(value_group) + 1 438 | 439 | last_generation = values 440 | 441 | root = cls.BRANCH_NODE(tree, separators, last_generation) 442 | 443 | tree._root = root 444 | return tree 445 | -------------------------------------------------------------------------------- /gogreen/cache.py: -------------------------------------------------------------------------------- 1 | # -*- Mode: Python; tab-width: 4 -*- 2 | 3 | # Copyright (c) 2005-2010 Slide, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are 8 | # met: 9 | # 10 | # * Redistributions of source code must retain the above copyright 11 | # notice, this list of conditions and the following disclaimer. 12 | # * Redistributions in binary form must reproduce the above 13 | # copyright notice, this list of conditions and the following 14 | # disclaimer in the documentation and/or other materials provided 15 | # with the distribution. 16 | # * Neither the name of the author nor the names of other 17 | # contributors may be used to endorse or promote products derived 18 | # from this software without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | '''cache 33 | 34 | various simple caches 35 | ''' 36 | 37 | from gogreen import dqueue 38 | 39 | DEFAULT_CACHE_SIZE = 1024 40 | 41 | class CacheObject(dqueue.QueueObject): 42 | __slots__ = ['id', 'value'] 43 | 44 | def __init__(self, *args, **kwargs): 45 | super(CacheObject, self).__init__(*args, **kwargs) 46 | 47 | self.id = args[0] 48 | self.value = kwargs.get('value', None) 49 | 50 | def __repr__(self): 51 | return '' % (self.id, self.value) 52 | 53 | 54 | class LRU(object): 55 | def __init__(self, *args, **kwargs): 56 | self._size = kwargs.get('size', DEFAULT_CACHE_SIZE) 57 | self._ordr = dqueue.ObjectQueue() 58 | self._objs = {} 59 | 60 | def __len__(self): 61 | return len(self._objs) 62 | 63 | def _balance(self): 64 | if len(self._objs) > self._size: 65 | obj = self._ordr.get_tail() 66 | del(self._objs[obj.id]) 67 | 68 | def _lookup(self, id): 69 | obj = self._objs.get(id, None) 70 | if obj is not None: 71 | self._ordr.remove(obj) 72 | self._ordr.put_head(obj) 73 | 74 | return obj 75 | 76 | def _insert(self, obj): 77 | self._objs[obj.id] = obj 78 | self._ordr.put_head(obj) 79 | 80 | self._balance() 81 | 82 | def lookup(self, id): 83 | obj = self._lookup(id) 84 | if obj is None: 85 | return None 86 | else: 87 | return obj.value 88 | 89 | def insert(self, id, value): 90 | obj = self._lookup(id) 91 | if obj is None: 92 | obj = CacheObject(id) 93 | self._insert(obj) 94 | 95 | obj.value = value 96 | 97 | def reset(self, size = None): 98 | if size is not None: 99 | self._size = size 100 | 101 | self._ordr.clear() 102 | self._objs = {} 103 | 104 | def head(self): 105 | return self._ordr.look_head() 106 | 107 | def tail(self): 108 | return self._ordr.look_tail() 109 | # 110 | # end... 111 | 112 | -------------------------------------------------------------------------------- /gogreen/corocurl.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- Mode: Python; tab-width: 4 -*- 3 | 4 | # Copyright (c) 2005-2010 Slide, Inc. 5 | # All rights reserved. 6 | # 7 | # Redistribution and use in source and binary forms, with or without 8 | # modification, are permitted provided that the following conditions are 9 | # met: 10 | # 11 | # * Redistributions of source code must retain the above copyright 12 | # notice, this list of conditions and the following disclaimer. 13 | # * Redistributions in binary form must reproduce the above 14 | # copyright notice, this list of conditions and the following 15 | # disclaimer in the documentation and/or other materials provided 16 | # with the distribution. 17 | # * Neither the name of the author nor the names of other 18 | # contributors may be used to endorse or promote products derived 19 | # from this software without specific prior written permission. 20 | # 21 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | 33 | """corocurl 34 | 35 | Emulation of PycURL objects which can be used inside the coro framework. 36 | 37 | Written by Libor Michalek. 38 | """ 39 | import os 40 | import getopt 41 | import logging 42 | 43 | import sys 44 | import exceptions 45 | import pycurl 46 | import coro 47 | import time 48 | import select 49 | 50 | __real_curl__ = pycurl.Curl 51 | __real_mult__ = pycurl.CurlMulti 52 | 53 | DEFAULT_YIELD_TIMEOUT = 2 54 | 55 | class CoroutineCurlError (exceptions.Exception): 56 | pass 57 | 58 | class coroutine_curl(object): 59 | '''coroutine_curl 60 | 61 | Emulation replacement for the standard pycurl.Curl() object. The 62 | object uses the non-blocking pycurl.CurlMulti interface to execute 63 | requests. 64 | ''' 65 | 66 | def __init__(self): 67 | self._curl = __real_curl__() 68 | self._mult = __real_mult__() 69 | 70 | self._waits = {None: 0} 71 | self._fdno = None 72 | 73 | def __getattr__(self, name): 74 | return getattr(pycurl, name) 75 | # 76 | # poll() registration API interface functions 77 | # 78 | def fileno(self): 79 | if self._fdno is None: 80 | raise CoroutineCurlError, 'coroutine curl no fd yet' 81 | 82 | return self._fdno 83 | 84 | def _wait_add(self, mask): 85 | self._waits[coro.current_thread()] = mask 86 | return reduce(lambda x, y: x|y, self._waits.values()) 87 | 88 | def _wait_del(self): 89 | del(self._waits[coro.current_thread()]) 90 | return reduce(lambda x, y: x|y, self._waits.values()) 91 | 92 | def _waiting(self): 93 | return filter(lambda y: y[0] is not None, self._waits.items()) 94 | # 95 | # internal 96 | # 97 | def _perform(self): 98 | current = coro.current_thread() 99 | if current is None: 100 | raise CoroutineCurlError, "coroutine curl in 'main'" 101 | 102 | while 1: 103 | ret = pycurl.E_CALL_MULTI_PERFORM 104 | 105 | while ret == pycurl.E_CALL_MULTI_PERFORM: 106 | ret, num = self._mult.perform() 107 | 108 | if not num: 109 | break 110 | # 111 | # build fdset and eventmask 112 | # 113 | read_set, send_set, excp_set = self._mult.fdset() 114 | if not read_set and not send_set: 115 | raise CoroutineCurlError, 'coroutine curl empty fdset' 116 | 117 | if read_set and send_set and read_set != send_set: 118 | raise CoroutineCurlError, 'coroutine curl bad fdset' 119 | 120 | self._fdno = (read_set + send_set)[0] 121 | eventmask = 0 122 | 123 | if read_set: 124 | eventmask |= select.POLLIN 125 | 126 | if send_set: 127 | eventmask |= select.POLLOUT 128 | # 129 | # get timeout 130 | # 131 | # starting with pycurl version 7.16.0 the multi object 132 | # supplies the max yield timeout, otherwise we just use 133 | # a reasonable floor value. 134 | # 135 | if hasattr(self._mult, 'timeout'): 136 | timeout = self._mult.timeout() 137 | else: 138 | timeout = DEFAULT_YIELD_TIMEOUT 139 | # 140 | # wait 141 | # 142 | coro.the_event_poll.register(self, eventmask) 143 | result = current.Yield(timeout, 0) 144 | coro.the_event_poll.unregister(self) 145 | # 146 | # process results (result of 0 is a timeout) 147 | # 148 | self._fdno = None 149 | 150 | if result is None: 151 | raise CoroutineSocketWake, 'socket has been awakened' 152 | 153 | if 0 < (result & coro.ERROR_MASK): 154 | raise pycurl.error, (socket.EBADF, 'Bad file descriptor') 155 | 156 | queued, success, failed = self._mult.info_read() 157 | if failed: 158 | raise pycurl.error, (failed[0][1], failed[0][2]) 159 | # 160 | # emulated API 161 | # 162 | def perform(self): 163 | self._mult.add_handle(self._curl) 164 | try: 165 | self._perform() 166 | finally: 167 | self._mult.remove_handle(self._curl) 168 | 169 | def close(self): 170 | self._mult.close() 171 | self._curl.close() 172 | 173 | def errstr(self): 174 | return self._curl.errstr() 175 | 176 | def getinfo(self, option): 177 | return self._curl.getinfo(option) 178 | 179 | def setopt(self, option, value): 180 | return self._curl.setopt(option, value) 181 | 182 | def unsetopt(self, option): 183 | return self._curl.setopt(option) 184 | 185 | class coroutine_multi(object): 186 | '''coroutine_multi 187 | 188 | coroutine replacement for the standard pycurl.CurlMulti() object. 189 | Since one should not need to deal with CurlMulti interface while 190 | using the coroutine framework, this remains unimplemented. 191 | ''' 192 | 193 | def __init__(self): 194 | raise CoroutineCurlError, 'Are you sure you know what you are doing?' 195 | 196 | def emulate(): 197 | "replace some pycurl objects with coroutine aware objects" 198 | pycurl.Curl = coroutine_curl 199 | pycurl.CurlMulti = coroutine_multi 200 | # sys.modules['pycurl'] = sys.modules[__name__] 201 | # 202 | # Standalone testing interface 203 | # 204 | TEST_CONNECT_TIMEOUT = 15 205 | TEST_DATA_TIMEOUT = 15 206 | 207 | class CurlEater: 208 | def __init__(self): 209 | self.contents = '' 210 | 211 | def body_callback(self, buf): 212 | self.contents = self.contents + buf 213 | 214 | class CurlFetch(coro.Thread): 215 | def run(self, url): 216 | self.info('starting fetch <%s>' % (url,)) 217 | ce = CurlEater() 218 | 219 | c = pycurl.Curl() 220 | c.setopt(pycurl.URL, url) 221 | c.setopt(pycurl.CONNECTTIMEOUT, TEST_CONNECT_TIMEOUT) 222 | c.setopt(pycurl.TIMEOUT, TEST_DATA_TIMEOUT) 223 | c.setopt(pycurl.WRITEFUNCTION, ce.body_callback) 224 | 225 | try: 226 | c.perform() 227 | except: 228 | self.traceback() 229 | 230 | self.info('fetched %d bytes' % (len(ce.contents),)) 231 | return None 232 | 233 | def run(url, log, loglevel): 234 | # 235 | # turn on curl emulation 236 | emulate() 237 | # 238 | # webserver and handler 239 | fetch = CurlFetch(log = log, args = (url,)) 240 | fetch.set_log_level(loglevel) 241 | fetch.start() 242 | # 243 | # primary event loop. 244 | coro.event_loop() 245 | # 246 | # never reached... 247 | return None 248 | 249 | LOG_FRMT = '[%(name)s|%(coro)s|%(asctime)s|%(levelname)s] %(message)s' 250 | LOGLEVELS = dict( 251 | CRITICAL=logging.CRITICAL, DEBUG=logging.DEBUG, ERROR=logging.ERROR, 252 | FATAL=logging.FATAL, INFO=logging.INFO, WARN=logging.WARN, 253 | WARNING=logging.WARNING) 254 | 255 | COMMAND_LINE_ARGS = ['help', 'logfile=', 'loglevel=', 'url='] 256 | 257 | def usage(name, error = None): 258 | if error: 259 | print 'Error:', error 260 | print " usage: %s [options]" % name 261 | 262 | def main(argv, environ): 263 | progname = sys.argv[0] 264 | 265 | url = None 266 | logfile = None 267 | loglevel = 'INFO' 268 | 269 | dirname = os.path.dirname(os.path.abspath(progname)) 270 | os.chdir(dirname) 271 | 272 | try: 273 | list, args = getopt.getopt(argv[1:], [], COMMAND_LINE_ARGS) 274 | except getopt.error, why: 275 | usage(progname, why) 276 | return None 277 | 278 | for (field, val) in list: 279 | if field == '--help': 280 | usage(progname) 281 | return None 282 | elif field == '--url': 283 | url = val 284 | elif field == '--logfile': 285 | logfile = val 286 | elif field == '--loglevel': 287 | loglevel = val 288 | # 289 | # setup logging 290 | # 291 | hndlr = logging.StreamHandler(sys.stdout) 292 | log = coro.coroutine_logger('corocurl') 293 | fmt = logging.Formatter(LOG_FRMT) 294 | 295 | log.setLevel(logging.DEBUG) 296 | 297 | sys.stdout = coro.coroutine_stdout(log) 298 | sys.stderr = coro.coroutine_stderr(log) 299 | 300 | hndlr.setFormatter(fmt) 301 | log.addHandler(hndlr) 302 | loglevel = LOGLEVELS.get(loglevel, None) 303 | if loglevel is None: 304 | log.warn('Unknown logging level, using INFO: %r' % (loglevel, )) 305 | loglevel = logging.INFO 306 | # 307 | # check param and execute 308 | # 309 | if url is not None: 310 | result = run(url, log, loglevel) 311 | else: 312 | log.error('no url provided!') 313 | result = None 314 | 315 | return result 316 | 317 | if __name__ == '__main__': 318 | main(sys.argv, os.environ) 319 | 320 | 321 | # 322 | # end... 323 | -------------------------------------------------------------------------------- /gogreen/corofile.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- Mode: Python; tab-width: 4 -*- 3 | 4 | # Copyright (c) 2005-2010 Slide, Inc. 5 | # All rights reserved. 6 | # 7 | # Redistribution and use in source and binary forms, with or without 8 | # modification, are permitted provided that the following conditions are 9 | # met: 10 | # 11 | # * Redistributions of source code must retain the above copyright 12 | # notice, this list of conditions and the following disclaimer. 13 | # * Redistributions in binary form must reproduce the above 14 | # copyright notice, this list of conditions and the following 15 | # disclaimer in the documentation and/or other materials provided 16 | # with the distribution. 17 | # * Neither the name of the author nor the names of other 18 | # contributors may be used to endorse or promote products derived 19 | # from this software without specific prior written permission. 20 | # 21 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | 33 | """corofile 34 | 35 | Emulation of file objects with nonblocking semantics. Useful 36 | for handling standard io. 37 | 38 | Written by Donovan Preston. 39 | """ 40 | 41 | import coro 42 | 43 | import os 44 | import sys 45 | import fcntl 46 | import errno 47 | import select 48 | 49 | ERROR_MASK = select.POLLERR|select.POLLHUP|select.POLLNVAL 50 | BUFFER_CHUNK_SIZE = 32*1024 51 | 52 | class NonblockingFile(object): 53 | def __init__(self, fp): 54 | self._fp = fp 55 | 56 | F = fcntl.fcntl(self._fp.fileno(), fcntl.F_GETFL) 57 | F = F | os.O_NONBLOCK 58 | fcntl.fcntl(self._fp.fileno(), fcntl.F_SETFL, F) 59 | 60 | self._chunk = BUFFER_CHUNK_SIZE 61 | self._data = '' 62 | self._waits = {None: 0} 63 | 64 | def fileno(self): 65 | return self._fp.fileno() 66 | 67 | def _wait_add(self, mask): 68 | self._waits[coro.current_thread()] = mask 69 | return reduce(lambda x, y: x|y, self._waits.values()) 70 | 71 | def _wait_del(self): 72 | del(self._waits[coro.current_thread()]) 73 | return reduce(lambda x, y: x|y, self._waits.values()) 74 | 75 | def _waiting(self): 76 | return filter(lambda y: y[0] is not None, self._waits.items()) 77 | 78 | def _wait_for_event(self, eventmask): 79 | me = coro.current_thread() 80 | if me is None: 81 | raise coro.CoroutineThreadError, "coroutine sockets in 'main'" 82 | 83 | coro.the_event_poll.register(self, eventmask) 84 | result = me.Yield() 85 | coro.the_event_poll.unregister(self) 86 | 87 | if result is None: 88 | raise coro.CoroutineSocketWake, 'file descriptor has been awakened' 89 | 90 | if result & eventmask: 91 | return None 92 | 93 | if result & ERROR_MASK: 94 | raise IOError(errno.EPIPE, 'Broken pipe') 95 | # 96 | # all cases should have been handled by this point 97 | return None 98 | 99 | def read(self, numbytes = -1): 100 | while numbytes < 0 or numbytes > len(self._data): 101 | self._wait_for_event(select.POLLIN|select.POLLHUP) 102 | 103 | read = os.read(self.fileno(), self._chunk) 104 | if not read: 105 | break 106 | 107 | self._data += read 108 | 109 | if numbytes < 0: 110 | result, self._data = self._data, '' 111 | else: 112 | result, self._data = self._data[:numbytes], self._data[numbytes:] 113 | 114 | return result 115 | 116 | def write(self, data): 117 | pos = 0 118 | 119 | while pos < len(data): 120 | self._wait_for_event(select.POLLOUT) 121 | 122 | size = os.write(self.fileno(), data[pos:pos + self._chunk]) 123 | pos += size 124 | 125 | def flush(self): 126 | if hasattr(self._fp, 'flush'): 127 | self._wait_for_event(select.POLLOUT) 128 | self._fp.flush() 129 | 130 | def close(self): 131 | self._fp.close() 132 | 133 | 134 | __old_popen2__ = os.popen2 135 | 136 | def popen2(*args, **kw): 137 | stdin, stdout = __old_popen2__(*args, **kw) 138 | return NonblockingFile(stdin), NonblockingFile(stdout) 139 | 140 | 141 | def emulate_popen2(): 142 | if os.popen2 is not popen2: 143 | os.popen2 = popen2 144 | 145 | __old_popen3__ = os.popen3 146 | 147 | def popen3(*args, **kwargs): 148 | stdin, stdout, stderr = __old_popen3__(*args, **kwargs) 149 | return NonblockingFile(stdin), NonblockingFile(stdout), NonblockingFile(stderr) 150 | 151 | def install_stdio(): 152 | sys.stdin = NonblockingFile(sys.stdin) 153 | sys.stdout = NonblockingFile(sys.stdout) 154 | return sys.stdin, sys.stdout 155 | 156 | def echostdin(): 157 | sin, sout = install_stdio() 158 | 159 | sout.write("HELLO WORLD!\n") 160 | echo = '' 161 | while True: 162 | char = sin.read(1) 163 | if not char: 164 | return 165 | if char == '\n': 166 | sout.write('%s\n' % (echo, )) 167 | echo = '' 168 | else: 169 | echo += char 170 | 171 | 172 | def readToEOF(): 173 | sin, sout = install_stdio() 174 | 175 | read = sin.read() 176 | sout.write(read) 177 | 178 | 179 | if __name__ == '__main__': 180 | coro.spawn(echostdin) 181 | # coro.spawn(readToEOF) 182 | coro.event_loop() 183 | 184 | 185 | -------------------------------------------------------------------------------- /gogreen/coromysqlerr.py: -------------------------------------------------------------------------------- 1 | # -*- Mode: Python; tab-width: 4 -*- 2 | 3 | # Copyright (c) 1999 eGroups, Inc. 4 | # Copyright (c) 2005-2010 Slide, Inc. 5 | # All rights reserved. 6 | # 7 | # Redistribution and use in source and binary forms, with or without 8 | # modification, are permitted provided that the following conditions are 9 | # met: 10 | # 11 | # * Redistributions of source code must retain the above copyright 12 | # notice, this list of conditions and the following disclaimer. 13 | # * Redistributions in binary form must reproduce the above 14 | # copyright notice, this list of conditions and the following 15 | # disclaimer in the documentation and/or other materials provided 16 | # with the distribution. 17 | # * Neither the name of the author nor the names of other 18 | # contributors may be used to endorse or promote products derived 19 | # from this software without specific prior written permission. 20 | # 21 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | 33 | # =========================================================================== 34 | # Errors 35 | # =========================================================================== 36 | ER_HASHCHK = 1000 37 | ER_NISAMCHK = 1001 38 | ER_NO = 1002 39 | ER_YES = 1003 40 | ER_CANT_CREATE_FILE = 1004 41 | ER_CANT_CREATE_TABLE = 1005 42 | ER_CANT_CREATE_DB = 1006 43 | ER_DB_CREATE_EXISTS = 1007 44 | ER_DB_DROP_EXISTS = 1008 45 | ER_DB_DROP_DELETE = 1009 46 | ER_DB_DROP_RMDIR = 1010 47 | ER_CANT_DELETE_FILE = 1011 48 | ER_CANT_FIND_SYSTEM_REC = 1012 49 | ER_CANT_GET_STAT = 1013 50 | ER_CANT_GET_WD = 1014 51 | ER_CANT_LOCK = 1015 52 | ER_CANT_OPEN_FILE = 1016 53 | ER_FILE_NOT_FOUND = 1017 54 | ER_CANT_READ_DIR = 1018 55 | ER_CANT_SET_WD = 1019 56 | ER_CHECKREAD = 1020 57 | ER_DISK_FULL = 1021 58 | ER_DUP_KEY = 1022 59 | ER_ERROR_ON_CLOSE = 1023 60 | ER_ERROR_ON_READ = 1024 61 | ER_ERROR_ON_RENAME = 1025 62 | ER_ERROR_ON_WRITE = 1026 63 | ER_FILE_USED = 1027 64 | ER_FILSORT_ABORT = 1028 65 | ER_FORM_NOT_FOUND = 1029 66 | ER_GET_ERRNO = 1030 67 | ER_ILLEGAL_HA = 1031 68 | ER_KEY_NOT_FOUND = 1032 69 | ER_NOT_FORM_FILE = 1033 70 | ER_NOT_KEYFILE = 1034 71 | ER_OLD_KEYFILE = 1035 72 | ER_OPEN_AS_READONLY = 1036 73 | ER_OUTOFMEMORY = 1037 74 | ER_OUT_OF_SORTMEMORY = 1038 75 | ER_UNEXPECTED_EOF = 1039 76 | ER_CON_COUNT_ERROR = 1040 77 | ER_OUT_OF_RESOURCES = 1041 78 | ER_BAD_HOST_ERROR = 1042 79 | ER_HANDSHAKE_ERROR = 1043 80 | ER_DBACCESS_DENIED_ERROR = 1044 81 | ER_ACCESS_DENIED_ERROR = 1045 82 | ER_NO_DB_ERROR = 1046 83 | ER_UNKNOWN_COM_ERROR = 1047 84 | ER_BAD_NULL_ERROR = 1048 85 | ER_BAD_DB_ERROR = 1049 86 | ER_TABLE_EXISTS_ERROR = 1050 87 | ER_BAD_TABLE_ERROR = 1051 88 | ER_NON_UNIQ_ERROR = 1052 89 | ER_SERVER_SHUTDOWN = 1053 90 | ER_BAD_FIELD_ERROR = 1054 91 | ER_WRONG_FIELD_WITH_GROUP = 1055 92 | ER_WRONG_GROUP_FIELD = 1056 93 | ER_WRONG_SUM_SELECT = 1057 94 | ER_WRONG_VALUE_COUNT = 1058 95 | ER_TOO_LONG_IDENT = 1059 96 | ER_DUP_FIELDNAME = 1060 97 | ER_DUP_KEYNAME = 1061 98 | ER_DUP_ENTRY = 1062 99 | ER_WRONG_FIELD_SPEC = 1063 100 | ER_PARSE_ERROR = 1064 101 | ER_EMPTY_QUERY = 1065 102 | ER_NONUNIQ_TABLE = 1066 103 | ER_INVALID_DEFAULT = 1067 104 | ER_MULTIPLE_PRI_KEY = 1068 105 | ER_TOO_MANY_KEYS = 1069 106 | ER_TOO_MANY_KEY_PARTS = 1070 107 | ER_TOO_LONG_KEY = 1071 108 | ER_KEY_COLUMN_DOES_NOT_EXITS = 1072 109 | ER_BLOB_USED_AS_KEY = 1073 110 | ER_TOO_BIG_FIELDLENGTH = 1074 111 | ER_WRONG_AUTO_KEY = 1075 112 | ER_READY = 1076 113 | ER_NORMAL_SHUTDOWN = 1077 114 | ER_GOT_SIGNAL = 1078 115 | ER_SHUTDOWN_COMPLETE = 1079 116 | ER_FORCING_CLOSE = 1080 117 | ER_IPSOCK_ERROR = 1081 118 | ER_NO_SUCH_INDEX = 1082 119 | ER_WRONG_FIELD_TERMINATORS = 1083 120 | ER_BLOBS_AND_NO_TERMINATED = 1084 121 | ER_TEXTFILE_NOT_READABLE = 1085 122 | ER_FILE_EXISTS_ERROR = 1086 123 | ER_LOAD_INFO = 1087 124 | ER_ALTER_INFO = 1088 125 | ER_WRONG_SUB_KEY = 1089 126 | ER_CANT_REMOVE_ALL_FIELDS = 1090 127 | ER_CANT_DROP_FIELD_OR_KEY = 1091 128 | ER_INSERT_INFO = 1092 129 | ER_UPDATE_TABLE_USED = 1093 130 | ER_NO_SUCH_THREAD = 1094 131 | ER_KILL_DENIED_ERROR = 1095 132 | ER_NO_TABLES_USED = 1096 133 | ER_TOO_BIG_SET = 1097 134 | ER_NO_UNIQUE_LOGFILE = 1098 135 | ER_TABLE_NOT_LOCKED_FOR_WRITE = 1099 136 | ER_TABLE_NOT_LOCKED = 1100 137 | ER_BLOB_CANT_HAVE_DEFAULT = 1101 138 | ER_WRONG_DB_NAME = 1102 139 | ER_WRONG_TABLE_NAME = 1103 140 | ER_TOO_BIG_SELECT = 1104 141 | ER_UNKNOWN_ERROR = 1105 142 | ER_UNKNOWN_PROCEDURE = 1106 143 | ER_WRONG_PARAMCOUNT_TO_PROCEDURE = 1107 144 | ER_WRONG_PARAMETERS_TO_PROCEDURE = 1108 145 | ER_UNKNOWN_TABLE = 1109 146 | ER_FIELD_SPECIFIED_TWICE = 1110 147 | ER_INVALID_GROUP_FUNC_USE = 1111 148 | ER_UNSUPPORTED_EXTENSION = 1112 149 | ER_TABLE_MUST_HAVE_COLUMNS = 1113 150 | ER_RECORD_FILE_FULL = 1114 151 | ER_UNKNOWN_CHARACTER_SET = 1115 152 | ER_TOO_MANY_TABLES = 1116 153 | ER_TOO_MANY_FIELDS = 1117 154 | ER_TOO_BIG_ROWSIZE = 1118 155 | ER_STACK_OVERRUN = 1119 156 | ER_WRONG_OUTER_JOIN = 1120 157 | ER_NULL_COLUMN_IN_INDEX = 1121 158 | ER_CANT_FIND_UDF = 1122 159 | ER_CANT_INITIALIZE_UDF = 1123 160 | ER_UDF_NO_PATHS = 1124 161 | ER_UDF_EXISTS = 1125 162 | ER_CANT_OPEN_LIBRARY = 1126 163 | ER_CANT_FIND_DL_ENTRY = 1127 164 | ER_FUNCTION_NOT_DEFINED = 1128 165 | ER_HOST_IS_BLOCKED = 1129 166 | ER_HOST_NOT_PRIVILEGED = 1130 167 | ER_PASSWORD_ANONYMOUS_USER = 1131 168 | ER_PASSWORD_NOT_ALLOWED = 1132 169 | ER_PASSWORD_NO_MATCH = 1133 170 | ER_UPDATE_INFO = 1134 171 | ER_CANT_CREATE_THREAD = 1135 172 | ER_WRONG_VALUE_COUNT_ON_ROW = 1136 173 | ER_CANT_REOPEN_TABLE = 1137 174 | ER_INVALID_USE_OF_NULL = 1138 175 | ER_REGEXP_ERROR = 1139 176 | ER_MIX_OF_GROUP_FUNC_AND_FIELDS = 1140 177 | ER_NONEXISTING_GRANT = 1141 178 | ER_TABLEACCESS_DENIED_ERROR = 1142 179 | ER_COLUMNACCESS_DENIED_ERROR = 1143 180 | ER_ILLEGAL_GRANT_FOR_TABLE = 1144 181 | ER_GRANT_WRONG_HOST_OR_USER = 1145 182 | ER_NO_SUCH_TABLE = 1146 183 | ER_NONEXISTING_TABLE_GRANT = 1147 184 | ER_NOT_ALLOWED_COMMAND = 1148 185 | ER_SYNTAX_ERROR = 1149 186 | ER_DELAYED_CANT_CHANGE_LOCK = 1150 187 | ER_TOO_MANY_DELAYED_THREADS = 1151 188 | ER_ABORTING_CONNECTION = 1152 189 | ER_NET_PACKET_TOO_LARGE = 1153 190 | ER_NET_READ_ERROR_FROM_PIPE = 1154 191 | ER_NET_FCNTL_ERROR = 1155 192 | ER_NET_PACKETS_OUT_OF_ORDER = 1156 193 | ER_NET_UNCOMPRESS_ERROR = 1157 194 | ER_NET_READ_ERROR = 1158 195 | ER_NET_READ_INTERRUPTED = 1159 196 | ER_NET_ERROR_ON_WRITE = 1160 197 | ER_NET_WRITE_INTERRUPTED = 1161 198 | ER_TOO_LONG_STRING = 1162 199 | ER_TABLE_CANT_HANDLE_BLOB = 1163 200 | ER_TABLE_CANT_HANDLE_AUTO_INCREMENT = 1164 201 | ER_DELAYED_INSERT_TABLE_LOCKED = 1165 202 | ER_WRONG_COLUMN_NAME = 1166 203 | ER_WRONG_KEY_COLUMN = 1167 204 | ER_WRONG_MRG_TABLE = 1168 205 | ER_DUP_UNIQUE = 1169 206 | ER_BLOB_KEY_WITHOUT_LENGTH = 1170 207 | ER_PRIMARY_CANT_HAVE_NULL = 1171 208 | ER_TOO_MANY_ROWS = 1172 209 | ER_REQUIRES_PRIMARY_KEY = 1173 210 | ER_NO_RAID_COMPILED = 1174 211 | ER_UPDATE_WITHOUT_KEY_IN_SAFE_MODE = 1175 212 | ER_KEY_DOES_NOT_EXITS = 1176 213 | ER_CHECK_NO_SUCH_TABLE = 1177 214 | ER_CHECK_NOT_IMPLEMENTED = 1178 215 | ER_CANT_DO_THIS_DURING_AN_TRANSACTION = 1179 216 | ER_ERROR_DURING_COMMIT = 1180 217 | ER_ERROR_DURING_ROLLBACK = 1181 218 | ER_ERROR_DURING_FLUSH_LOGS = 1182 219 | ER_ERROR_DURING_CHECKPOINT = 1183 220 | ER_NEW_ABORTING_CONNECTION = 1184 221 | ER_DUMP_NOT_IMPLEMENTED = 1185 222 | ER_FLUSH_MASTER_BINLOG_CLOSED = 1186 223 | ER_INDEX_REBUILD = 1187 224 | ER_MASTER = 1188 225 | ER_MASTER_NET_READ = 1189 226 | ER_MASTER_NET_WRITE = 1190 227 | ER_FT_MATCHING_KEY_NOT_FOUND = 1191 228 | ER_LOCK_OR_ACTIVE_TRANSACTION = 1192 229 | ER_UNKNOWN_SYSTEM_VARIABLE = 1193 230 | ER_CRASHED_ON_USAGE = 1194 231 | ER_CRASHED_ON_REPAIR = 1195 232 | ER_WARNING_NOT_COMPLETE_ROLLBACK = 1196 233 | ER_TRANS_CACHE_FULL = 1197 234 | ER_SLAVE_MUST_STOP = 1198 235 | ER_SLAVE_NOT_RUNNING = 1199 236 | ER_BAD_SLAVE = 1200 237 | ER_MASTER_INFO = 1201 238 | ER_SLAVE_THREAD = 1202 239 | ER_TOO_MANY_USER_CONNECTIONS = 1203 240 | ER_SET_CONSTANTS_ONLY = 1204 241 | ER_LOCK_WAIT_TIMEOUT = 1205 242 | ER_LOCK_TABLE_FULL = 1206 243 | ER_READ_ONLY_TRANSACTION = 1207 244 | ER_DROP_DB_WITH_READ_LOCK = 1208 245 | ER_CREATE_DB_WITH_READ_LOCK = 1209 246 | ER_WRONG_ARGUMENTS = 1210 247 | ER_NO_PERMISSION_TO_CREATE_USER = 1211 248 | ER_UNION_TABLES_IN_DIFFERENT_DIR = 1212 249 | ER_LOCK_DEADLOCK = 1213 250 | ER_TABLE_CANT_HANDLE_FT = 1214 251 | ER_CANNOT_ADD_FOREIGN = 1215 252 | ER_NO_REFERENCED_ROW = 1216 253 | ER_ROW_IS_REFERENCED = 1217 254 | ER_CONNECT_TO_MASTER = 1218 255 | ER_QUERY_ON_MASTER = 1219 256 | ER_ERROR_WHEN_EXECUTING_COMMAND = 1220 257 | ER_WRONG_USAGE = 1221 258 | ER_WRONG_NUMBER_OF_COLUMNS_IN_SELECT = 1222 259 | ER_CANT_UPDATE_WITH_READLOCK = 1223 260 | ER_MIXING_NOT_ALLOWED = 1224 261 | ER_DUP_ARGUMENT = 1225 262 | ER_USER_LIMIT_REACHED = 1226 263 | ER_SPECIFIC_ACCESS_DENIED_ERROR = 1227 264 | ER_LOCAL_VARIABLE = 1228 265 | ER_GLOBAL_VARIABLE = 1229 266 | ER_NO_DEFAULT = 1230 267 | ER_WRONG_VALUE_FOR_VAR = 1231 268 | ER_WRONG_TYPE_FOR_VAR = 1232 269 | ER_VAR_CANT_BE_READ = 1233 270 | ER_CANT_USE_OPTION_HERE = 1234 271 | ER_NOT_SUPPORTED_YET = 1235 272 | ER_MASTER_FATAL_ERROR_READING_BINLOG = 1236 273 | ER_SLAVE_IGNORED_TABLE = 1237 274 | ER_INCORRECT_GLOBAL_LOCAL_VAR = 1238 275 | ER_WRONG_FK_DEF = 1239 276 | ER_KEY_REF_DO_NOT_MATCH_TABLE_REF = 1240 277 | ER_OPERAND_COLUMNS = 1241 278 | ER_SUBQUERY_NO_1_ROW = 1242 279 | ER_UNKNOWN_STMT_HANDLER = 1243 280 | ER_CORRUPT_HELP_DB = 1244 281 | ER_CYCLIC_REFERENCE = 1245 282 | ER_AUTO_CONVERT = 1246 283 | ER_ILLEGAL_REFERENCE = 1247 284 | ER_DERIVED_MUST_HAVE_ALIAS = 1248 285 | ER_SELECT_REDUCED = 1249 286 | ER_TABLENAME_NOT_ALLOWED_HERE = 1250 287 | ER_NOT_SUPPORTED_AUTH_MODE = 1251 288 | ER_SPATIAL_CANT_HAVE_NULL = 1252 289 | ER_COLLATION_CHARSET_MISMATCH = 1253 290 | ER_SLAVE_WAS_RUNNING = 1254 291 | ER_SLAVE_WAS_NOT_RUNNING = 1255 292 | ER_TOO_BIG_FOR_UNCOMPRESS = 1256 293 | ER_ZLIB_Z_MEM_ERROR = 1257 294 | ER_ZLIB_Z_BUF_ERROR = 1258 295 | ER_ZLIB_Z_DATA_ERROR = 1259 296 | ER_CUT_VALUE_GROUP_CONCAT = 1260 297 | ER_WARN_TOO_FEW_RECORDS = 1261 298 | ER_WARN_TOO_MANY_RECORDS = 1262 299 | ER_WARN_NULL_TO_NOTNULL = 1263 300 | ER_WARN_DATA_OUT_OF_RANGE = 1264 301 | ER_WARN_DATA_TRUNCATED = 1265 302 | ER_WARN_USING_OTHER_HANDLER = 1266 303 | ER_CANT_AGGREGATE_2COLLATIONS = 1267 304 | ER_DROP_USER = 1268 305 | ER_REVOKE_GRANTS = 1269 306 | ER_CANT_AGGREGATE_3COLLATIONS = 1270 307 | ER_CANT_AGGREGATE_NCOLLATIONS = 1271 308 | ER_VARIABLE_IS_NOT_STRUCT = 1272 309 | ER_UNKNOWN_COLLATION = 1273 310 | ER_SLAVE_IGNORED_SSL_PARAMS = 1274 311 | ER_SERVER_IS_IN_SECURE_AUTH_MODE = 1275 312 | ER_WARN_FIELD_RESOLVED = 1276 313 | ER_BAD_SLAVE_UNTIL_COND = 1277 314 | ER_MISSING_SKIP_SLAVE = 1278 315 | ER_UNTIL_COND_IGNORED = 1279 316 | ER_WRONG_NAME_FOR_INDEX = 1280 317 | ER_WRONG_NAME_FOR_CATALOG = 1281 318 | ER_WARN_QC_RESIZE = 1282 319 | ER_BAD_FT_COLUMN = 1283 320 | ER_UNKNOWN_KEY_CACHE = 1284 321 | ER_WARN_HOSTNAME_WONT_WORK = 1285 322 | ER_UNKNOWN_STORAGE_ENGINE = 1286 323 | ER_WARN_DEPRECATED_SYNTAX = 1287 324 | ER_NON_UPDATABLE_TABLE = 1288 325 | ER_FEATURE_DISABLED = 1289 326 | ER_OPTION_PREVENTS_STATEMENT = 1290 327 | ER_DUPLICATED_VALUE_IN_TYPE = 1291 328 | ER_TRUNCATED_WRONG_VALUE = 1292 329 | ER_TOO_MUCH_AUTO_TIMESTAMP_COLS = 1293 330 | ER_INVALID_ON_UPDATE = 1294 331 | ER_UNSUPPORTED_PS = 1295 332 | ER_GET_ERRMSG = 1296 333 | ER_GET_TEMPORARY_ERRMSG = 1297 334 | ER_UNKNOWN_TIME_ZONE = 1298 335 | ER_WARN_INVALID_TIMESTAMP = 1299 336 | ER_INVALID_CHARACTER_STRING = 1300 337 | ER_WARN_ALLOWED_PACKET_OVERFLOWED = 1301 338 | ER_CONFLICTING_DECLARATI_ERROR_MESSAGES = 303 339 | 340 | CR_MIN_ERROR = 2000 341 | CR_MAX_ERROR = 2999 342 | 343 | CR_UNKNOWN_ERROR = 2000 344 | CR_SOCKET_CREATE_ERROR = 2001 345 | CR_CONNECTION_ERROR = 2002 346 | CR_CONN_HOST_ERROR = 2003 347 | CR_IPSOCK_ERROR = 2004 348 | CR_UNKNOWN_HOST = 2005 349 | CR_SERVER_GONE_ERROR = 2006 350 | CR_VERSION_ERROR = 2007 351 | CR_OUT_OF_MEMORY = 2008 352 | CR_WRONG_HOST_INFO = 2009 353 | CR_LOCALHOST_CONNECTION = 2010 354 | CR_TCP_CONNECTION = 2011 355 | CR_SERVER_HANDSHAKE_ERR = 2012 356 | CR_SERVER_LOST = 2013 357 | CR_COMMANDS_OUT_OF_SYNC = 2014 358 | CR_NAMEDPIPE_CONNECTION = 2015 359 | CR_NAMEDPIPEWAIT_ERROR = 2016 360 | CR_NAMEDPIPEOPEN_ERROR = 2017 361 | CR_NAMEDPIPESETSTATE_ERROR = 2018 362 | CR_CANT_READ_CHARSET = 2019 363 | CR_NET_PACKET_TOO_LARGE = 2020 364 | CR_EMBEDDED_CONNECTION = 2021 365 | CR_PROBE_SLAVE_STATUS = 2022 366 | CR_PROBE_SLAVE_HOSTS = 2023 367 | CR_PROBE_SLAVE_CONNECT = 2024 368 | CR_PROBE_MASTER_CONNECT = 2025 369 | CR_SSL_CONNECTION_ERROR = 2026 370 | CR_MALFORMED_PACKET = 2027 371 | CR_WRONG_LICENSE = 2028 372 | CR_NULL_POINTER = 2029 373 | CR_NO_PREPARE_STMT = 2030 374 | CR_PARAMS_NOT_BOUND = 2031 375 | CR_DATA_TRUNCATED = 2032 376 | CR_NO_PARAMETERS_EXISTS = 2033 377 | CR_INVALID_PARAMETER_NO = 2034 378 | CR_INVALID_BUFFER_USE = 2035 379 | CR_UNSUPPORTED_PARAM_TYPE = 2036 380 | 381 | CR_SHARED_MEMORY_CONNECTION = 2037 382 | CR_SHARED_MEMORY_CONNECT_REQUEST_ERROR = 2038 383 | CR_SHARED_MEMORY_CONNECT_ANSWER_ERROR = 2039 384 | CR_SHARED_MEMORY_CONNECT_FILE_MAP_ERROR = 2040 385 | CR_SHARED_MEMORY_CONNECT_MAP_ERROR = 2041 386 | CR_SHARED_MEMORY_FILE_MAP_ERROR = 2042 387 | CR_SHARED_MEMORY_MAP_ERROR = 2043 388 | CR_SHARED_MEMORY_EVENT_ERROR = 2044 389 | CR_SHARED_MEMORY_CONNECT_ABANDONED_ERROR = 2045 390 | CR_SHARED_MEMORY_CONNECT_SET_ERROR = 2046 391 | CR_CONN_UNKNOW_PROTOCOL = 2047 392 | CR_INVALID_CONN_HANDLE = 2048 393 | CR_SECURE_AUTH = 2049 394 | CR_FETCH_CANCELED = 2050 395 | CR_NO_DATA = 2051 396 | CR_NO_STMT_METADATA = 2052 397 | -------------------------------------------------------------------------------- /gogreen/coroqueue.py: -------------------------------------------------------------------------------- 1 | # -*- Mode: Python; tab-width: 4 -*- 2 | 3 | # Copyright (c) 2005-2010 Slide, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are 8 | # met: 9 | # 10 | # * Redistributions of source code must retain the above copyright 11 | # notice, this list of conditions and the following disclaimer. 12 | # * Redistributions in binary form must reproduce the above 13 | # copyright notice, this list of conditions and the following 14 | # disclaimer in the documentation and/or other materials provided 15 | # with the distribution. 16 | # * Neither the name of the author nor the names of other 17 | # contributors may be used to endorse or promote products derived 18 | # from this software without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | """coroqueue 33 | 34 | A corosafe Queue implementation. 35 | 36 | Written by Libor Michalek. 37 | """ 38 | 39 | import weakref 40 | import coro 41 | import time 42 | import bisect 43 | import exceptions 44 | import operator 45 | import copy 46 | import smtplib 47 | import socket 48 | 49 | import pyinfo 50 | 51 | DEFAULT_PRIORITY = 0x01 52 | 53 | 54 | def merge(dst, src): 55 | if dst is None: 56 | return copy.deepcopy(src) 57 | 58 | if not isinstance(dst, type(src)): 59 | raise TypeError( 60 | 'Cannot merge two different types %r:%r' % (type(dst), type(src))) 61 | 62 | if isinstance(dst, type(0)): 63 | return dst + src 64 | 65 | if isinstance(dst, type(())): 66 | return tuple(map(operator.add, dst, src)) 67 | 68 | if isinstance(dst, type({})): 69 | return filter( 70 | lambda i: dst.update({i[0]: merge(dst.get(i[0]), i[1])}), 71 | src.items()) or dst 72 | 73 | raise TypeError('Unhandled destination type: %r' % (type(dst),)) 74 | 75 | 76 | class Timeout(object): 77 | def __init__(self, timeout): 78 | if timeout is None: 79 | self.expire = None 80 | else: 81 | self.expire = time.time() + timeout 82 | 83 | def __repr__(self): 84 | if self.expire is None: 85 | return repr(None) 86 | else: 87 | return repr(self.expire - time.time()) 88 | 89 | def __nonzero__(self): 90 | if self.expire is None: 91 | return True 92 | else: 93 | return bool(self.expire > time.time()) 94 | 95 | def __call__(self): 96 | if self.expire is None: 97 | return None 98 | else: 99 | return self.expire - time.time() 100 | 101 | 102 | class ElementContainer(object): 103 | def __init__(self, *args, **kwargs): 104 | self._item_list = [] 105 | self._item_set = set() 106 | 107 | def __len__(self): 108 | return len(self._item_list) 109 | 110 | def __nonzero__(self): 111 | return bool(self._item_list) 112 | 113 | def __contains__(self, obj): 114 | return obj in self._item_set 115 | 116 | def add(self, obj): 117 | self._item_list.append((time.time(), obj)) 118 | self._item_set.add(obj) 119 | 120 | def pop(self): 121 | timestamp, obj = self._item_list.pop() 122 | self._item_set.remove(obj) 123 | return obj 124 | 125 | def rip(self, timestamp = None): 126 | if timestamp is None: 127 | index = len(self._item_list) 128 | else: 129 | index = bisect.bisect(self._item_list, (timestamp, None)) 130 | 131 | if not index: 132 | return [] 133 | 134 | data = map(lambda i: i[1], self._item_list[:index]) 135 | del(self._item_list[:index]) 136 | self._item_set.difference_update(data) 137 | 138 | return data 139 | 140 | class TimeoutError(exceptions.Exception): 141 | pass 142 | 143 | class QueueError(exceptions.Exception): 144 | pass 145 | 146 | class QueueDeadlock(exceptions.Exception): 147 | pass 148 | 149 | class Queue(object): 150 | ''' 151 | Simple generic queue. 152 | ''' 153 | def __init__(self, object_allocator, args, kwargs, **kw): 154 | # 155 | # 156 | # 157 | self._timeout = kw.get('timeout', None) 158 | self._patient = kw.get('patient', False) 159 | self._active = True 160 | self._item_out = 0 161 | self._item_list = ElementContainer() 162 | 163 | self._item_refs = {} 164 | self._item_save = {} 165 | self._item_time = {} 166 | 167 | self._out_total = 0L 168 | self._out_time = 0 169 | self._out_max = 0 170 | 171 | self._item_args = args 172 | self._item_kwargs = kwargs 173 | self._item_alloc = object_allocator 174 | # 175 | # priority list is presented lowest to highest 176 | # 177 | self._item_max = {} 178 | self._item_queues = {} 179 | self._item_prios = [] 180 | 181 | size = kw.get('size', 0) 182 | prios = kw.get('prios', [DEFAULT_PRIORITY]) 183 | 184 | for index in range(len(prios)): 185 | if isinstance(prios[index], type(tuple())): 186 | size += prios[index][1] 187 | prio = prios[index][0] 188 | else: 189 | prio = prios[index] 190 | 191 | self._item_queues[prio] = coro.stats_cond(timeout = self._timeout) 192 | self._item_max[prio] = size 193 | 194 | self._item_prios.insert(0, prio) 195 | 196 | def __repr__(self): 197 | state = self.state() 198 | 199 | entries = '(use: %d, max: %r, waiting: %r)' % \ 200 | (state['out'], state['max'], state['wait']) 201 | return 'Queue: status <%s> entries %s' % ( 202 | ((self._active and 'on') or 'off'), entries,) 203 | 204 | def __alloc(self): 205 | o = self._item_alloc( 206 | *self._item_args, **self._item_kwargs) 207 | 208 | def dropped(ref): 209 | self._stop_info(self._item_save.pop(self._item_refs[ref], {})) 210 | 211 | self._item_out = self._item_out - 1 212 | del(self._item_refs[ref]) 213 | self.wake_one() 214 | 215 | r = weakref.ref(o, dropped) 216 | self._item_refs[r] = id(o) 217 | 218 | return o 219 | 220 | def __dealloc(self, o): 221 | for r in weakref.getweakrefs(o): 222 | self._item_refs.pop(r, None) 223 | 224 | def _stop_info(self, data): 225 | '''_stop_info 226 | 227 | When an object is being returned to the queue, stop recording 228 | trace information in the thread that removed the object from 229 | the queue. 230 | ''' 231 | thrd = coro._get_thread(data.get('thread', 0)) 232 | if thrd is not None: 233 | thrd.trace(False) 234 | 235 | return data 236 | 237 | def _save_info(self, o): 238 | '''_save_info 239 | 240 | When an item is fetched from the queue, save information about 241 | the item request and the requester. 242 | ''' 243 | data = {'caller': pyinfo.rawstack(depth = 2), 'time': time.time()} 244 | current = coro.current_thread() 245 | 246 | if current is not None: 247 | data.update({'thread': current.thread_id()}) 248 | current.trace(True) 249 | 250 | self._item_save[id(o)] = data 251 | return o 252 | 253 | def _drop_info(self, o): 254 | '''_drop_info 255 | 256 | When an item is returned to the queue, release information about 257 | the original item request and the requester. 258 | ''' 259 | return self._stop_info(self._item_save.pop(id(o), {})) 260 | 261 | def _dealloc(self, o): 262 | self.__dealloc(o) 263 | 264 | def _drop_all(self): 265 | '''_drop_all 266 | 267 | All currently allocated and queued (e.g. not checked out) are 268 | released and deallocated. 269 | ''' 270 | while self._item_list: 271 | self._dealloc(self._item_list.pop()) 272 | 273 | def timeout(self, *args): 274 | if not args: 275 | return self._timeout 276 | 277 | self._timeout = args[0] 278 | 279 | for cond in self._item_queues.values(): 280 | cond.timeout = self._timeout 281 | 282 | def empty(self, prio): 283 | prio = (prio is None and self._item_prios[-1]) or prio 284 | return (not (self._item_out < self._item_max[prio])) 285 | 286 | def wake_all(self): 287 | for prio in self._item_prios: 288 | self._item_queues[prio].wake_all() 289 | 290 | def wake_one(self): 291 | for prio in self._item_prios: 292 | if len(self._item_queues[prio]): 293 | self._item_queues[prio].wake_one() 294 | break 295 | 296 | def wait_one(self, prio = None, timeout = None): 297 | prio = (prio is None and self._item_prios[-1]) or prio 298 | 299 | result = self._item_queues[prio].wait(timeout) 300 | if result is None: 301 | return (False, prio) # wait timed out 302 | elif result: 303 | return (True, result[0]) # wait had a priority adjustment 304 | else: 305 | return (True, prio) # wait completed successfully 306 | 307 | def bump(self, id, prio): 308 | for cond in self._item_queues.values(): 309 | try: 310 | cond.wake(id, prio) 311 | except coro.CoroutineCondError: 312 | pass 313 | else: 314 | break 315 | 316 | def get(self, prio = None, poll = False): 317 | '''get 318 | 319 | Return an object from the queue, wait/yield if one is not available 320 | until it becomes available. 321 | 322 | Note: when the queue has a timeout defined, the return value 323 | will be None when the timeout expires and no object has 324 | become available. 325 | ''' 326 | wait = not poll 327 | 328 | while wait and \ 329 | ((self.empty(prio) and self._active) or \ 330 | (self._patient and not self._active)): 331 | wait, prio = self.wait_one(prio) 332 | 333 | if self.empty(prio) or not self._active: 334 | return None 335 | 336 | if not self._item_list: 337 | self._item_list.add(self.__alloc()) 338 | 339 | self._item_out += 1 340 | self._out_total += 1 341 | 342 | return self._save_info(self._item_list.pop()) 343 | 344 | def put(self, o): 345 | if o in self._item_list: 346 | raise QueueError('cannnot put object already in queue', o) 347 | 348 | timestamp = self._drop_info(o).get('time', None) 349 | if timestamp is not None: 350 | out_time = time.time() - timestamp 351 | 352 | self._out_time += out_time 353 | self._out_max = max(out_time, self._out_max) 354 | 355 | self._item_out = self._item_out - 1 356 | 357 | if self._active: 358 | self._item_list.add(o) 359 | else: 360 | self._dealloc(o) 361 | 362 | self.wake_one() 363 | return None 364 | 365 | def active(self): 366 | return self._active 367 | 368 | def patient(self, v): 369 | self._patient = v 370 | 371 | def on(self, *args, **kwargs): 372 | self._active = True 373 | self.wake_all() 374 | 375 | def off(self, **kwargs): 376 | timeout = Timeout(kwargs.get('timeout', None)) 377 | wait = True 378 | # 379 | # It is possible to use the regular wait queue to wait for all 380 | # outstanding objects, since wake_all() clears out the entire 381 | # wait list synchronously and _active set to false prevents 382 | # anyone else from entering. 383 | # 384 | self._active = False 385 | self.wake_all() 386 | 387 | while wait and self._item_out and not self._active: 388 | wait, prio = self.wait_one(timeout = timeout()) 389 | 390 | if not self._active: 391 | self._drop_all() 392 | 393 | return wait 394 | 395 | def trim(self, age): 396 | '''trim 397 | 398 | Any item in the queue which is older then age seconds gets reaped. 399 | ''' 400 | count = 0 401 | 402 | for o in self._item_list.rip(time.time() - age): 403 | self._dealloc(o) 404 | count += 1 405 | 406 | return count 407 | 408 | def resize(self, size, prio = None): 409 | prio = (prio is None and self._item_prios[-1]) or prio 410 | self._item_max[prio] = size 411 | 412 | def size(self, prio = None): 413 | prio = (prio is None and self._item_prios[-1]) or prio 414 | return self._item_max[prio] 415 | 416 | def stats(self): 417 | qstats = {} 418 | for stat in map(lambda x: x.stats(), self._item_queues.values()): 419 | qstats['waiting'] = qstats.get('waiting', []) + stat['waiting'] 420 | qstats['timeouts'] = qstats.get('timeouts', 0) + stat['timeouts'] 421 | qstats['total'] = qstats.get('total', 0) + stat['total'] 422 | qstats['waits'] = qstats.get('waits', 0) + stat['waits'] 423 | 424 | if qstats['total']: 425 | qstats['average'] = qstats['total']/qstats['waits'] 426 | else: 427 | qstats['average'] = 0 428 | 429 | qstats.update({ 430 | 'item_max': self._item_max.values(), 431 | 'item_out': self._item_out, 432 | 'requests': self._out_total, 433 | 'out_max' : self._out_max, 434 | 'pending' : self._item_save}) 435 | qstats['pending'].sort() 436 | qstats['waiting'].sort() 437 | 438 | if self._out_time: 439 | average = self._out_time/(self._out_total-self._item_out) 440 | else: 441 | average = 0 442 | 443 | qstats.update({'avg_pend': average}) 444 | return qstats 445 | 446 | def state(self): 447 | item_max = [] 448 | item_wait = [] 449 | item_left = [] 450 | 451 | for index in range(len(self._item_prios)-1, -1, -1): 452 | prio = self._item_prios[index] 453 | 454 | item_max.append(self._item_max[prio]) 455 | item_left.append(self._item_max[prio] - self._item_out) 456 | item_wait.append(len(self._item_queues[prio])) 457 | 458 | return { 459 | 'out': self._item_out, 460 | 'max': item_max, 461 | 'in': item_left, 462 | 'wait': item_wait} 463 | 464 | def find(self): 465 | '''find 466 | 467 | Return information about each object that is currently checked out 468 | and therefore not in the queue. 469 | 470 | caller - execution stack trace at the time the object was 471 | removed from the queue 472 | thread - coroutine ID of the thread executing at the time the 473 | object was removed from the queue 474 | trace - current execution stack trace of the coroutine which 475 | removed the object from the queue 476 | time - unix timestamp when the object was removed from the 477 | queue. 478 | ''' 479 | saved = self._item_save.copy() 480 | for oid, data in saved.items(): 481 | thread = coro._get_thread(data['thread']) 482 | if thread is None: 483 | data.update({'trace': None}) 484 | else: 485 | data.update({'trace': thread.where()}) 486 | 487 | return saved 488 | 489 | 490 | class SortedQueue(object): 491 | def __init__( 492 | self, object_allocator, args, kwargs, size = 0, timeout = None): 493 | 494 | self._timeout = timeout 495 | self._item_args = args 496 | self._item_kwargs = kwargs 497 | self._item_alloc = object_allocator 498 | self._item_max = size 499 | self._item_out = 0 500 | 501 | self._out_total = 0L 502 | self._out_time = 0 503 | self._out_max = 0 504 | 505 | self._item_list = [] 506 | self._item_refs = {} 507 | self._item_time = {} 508 | self._item_cond = coro.stats_cond(timeout = self._timeout) 509 | 510 | def __repr__(self): 511 | state = self.state() 512 | entries = '(use: %d, max: %r, waiting: %r)' % \ 513 | (state['out'], state['max'], state['wait']) 514 | 515 | return 'SortedQueue: entries %s' % entries 516 | 517 | def __alloc(self): 518 | o = self._item_alloc( 519 | *self._item_args, **self._item_kwargs) 520 | 521 | def dropped(ref): 522 | self._item_out = self._item_out - 1 523 | del(self._item_refs[ref]) 524 | self.wake_one() 525 | 526 | r = weakref.ref(o, dropped) 527 | self._item_refs[r] = id(o) 528 | 529 | return o 530 | 531 | def empty(self): 532 | return not (self._item_out < self._item_max) 533 | 534 | 535 | def wake_all(self): 536 | self._item_cond.wake_all() 537 | 538 | def wake_one(self): 539 | self._item_cond.wake_one() 540 | 541 | def wait_one(self): 542 | result = self._item_cond.wait() 543 | return not (result is None) 544 | 545 | def head(self, poll = False): 546 | return self._get(poll) 547 | 548 | def tail(self, poll = False): 549 | return self._get(poll, True) 550 | 551 | def _get(self, poll = False, tail = False): 552 | 553 | wait = not poll 554 | 555 | # wait for something to come in if we're not polling 556 | while wait and self.empty(): 557 | wait = self.wait_one() 558 | 559 | if self.empty(): 560 | return None 561 | 562 | if not self._item_list: 563 | if tail: 564 | return None 565 | self._item_list.append(self.__alloc()) 566 | 567 | self._item_time[coro.current_id()] = time.time() 568 | self._item_out += 1 569 | self._out_total += 1 570 | 571 | if tail: 572 | o = self._item_list[0] 573 | del(self._item_list[0]) 574 | else: 575 | o = self._item_list.pop() 576 | 577 | return o 578 | 579 | def put(self, o): 580 | id = coro.current_id() 581 | if id in self._item_time: 582 | out_time = time.time() - self._item_time[id] 583 | self._out_time += out_time 584 | self._out_max = max(self._out_max, out_time) 585 | del(self._item_time[id]) 586 | 587 | self._item_out -= 1 588 | bisect.insort(self._item_list, o) 589 | 590 | self.wake_one() 591 | return None 592 | 593 | def resize(self, size): 594 | self._item_max = size 595 | 596 | def size(self): 597 | return self._item_max 598 | 599 | def stats(self): 600 | qstats = {} 601 | qstats.update(self._item_cond.stats()) 602 | 603 | qstats.update({ 604 | 'item_max' : self._item_max, 605 | 'item_out' : self._item_out, 606 | 'requests' : self._out_total, 607 | 'out_max' : self._out_max, 608 | 'pending' : self._item_time.values() }) 609 | qstats['pending'].sort() 610 | qstats['waiting'].sort() 611 | 612 | if self._out_time: 613 | average = self._out_time/(self._out_total-self._item_out) 614 | else: 615 | average = 0 616 | 617 | qstats.update({'avg_pend' : average }) 618 | return qstats 619 | 620 | def state(self): 621 | return { 'out' : self._item_out, 622 | 'max' : self._item_max, 623 | 'in' : self._item_max - self._item_out, 624 | 'wait' : len(self._item_cond) } 625 | 626 | class SafeQueue(Queue): 627 | '''SafeQueue 628 | 629 | Queue with protection against a single coro.thread requesting multiple 630 | objects and potentially deadlocking in the process. 631 | ''' 632 | def __init__(self, *args, **kwargs): 633 | super(SafeQueue, self).__init__(*args, **kwargs) 634 | 635 | self._item_thrd = {} 636 | 637 | def _stop_info(self, data): 638 | data = super(SafeQueue, self)._stop_info(data) 639 | self._item_thrd.pop(data.get('thread', -1), None) 640 | 641 | return data 642 | 643 | def _save_info(self, o): 644 | o = super(SafeQueue, self)._save_info(o) 645 | t = self._item_save[id(o)].get('thread', -1) 646 | 647 | self._item_thrd[t] = id(o) 648 | return o 649 | 650 | def get(self, *args, **kwargs): 651 | '''get 652 | 653 | Return an object from the queue, wait/yield if one is not available 654 | until it becomes available. 655 | 656 | Note: when the queue has a timeout defined, the return value 657 | will be None when the timeout expires and no object has 658 | become available. 659 | 660 | QueueDeadlock raised if an attempt is made to get an object and 661 | the current thread already has fetched an object from the queue 662 | which has yet to be returned. 663 | ''' 664 | if coro.current_id() in self._item_thrd: 665 | obj = self._item_thrd[coro.current_id()] 666 | raise QueueDeadlock(obj, self._item_save[obj]) 667 | 668 | return super(SafeQueue, self).get(*args, **kwargs) 669 | 670 | class ThreadQueue(Queue): 671 | def __init__(self, thread, args, size = 0,priorities = [DEFAULT_PRIORITY]): 672 | super(ThreadQueue, self).__init__( 673 | thread, (), {'args': args}, size = size, prios = priorities) 674 | 675 | def get(self, prio = None, poll = False): 676 | thrd = super(type(self), self).get(prio, poll) 677 | if thrd is not None: 678 | thrd.start() 679 | return thrd 680 | 681 | class SimpleObject(object): 682 | def __init__(self, name): 683 | self.name = name 684 | def __repr__(self): 685 | return '' % (self.name,) 686 | 687 | class SimpleQueue(Queue): 688 | def __init__(self, name, size = 0, priorities = [DEFAULT_PRIORITY]): 689 | super(SimpleQueue, self).__init__( 690 | SimpleObject, (name, ), {}, size = size, prios = priorities) 691 | 692 | class SMTPQueue(SafeQueue): 693 | 694 | def __init__(self, host_primary, host_secondary=None, priorities=None, 695 | size=0, optional={}, timeout=None, trace=False, **kw): 696 | priorities = priorities or [DEFAULT_PRIORITY] 697 | self.host_primary = host_primary 698 | self.host_secondary = host_secondary 699 | super(SMTPQueue, self).__init__( 700 | smtplib.SMTP, (), optional, size = size, 701 | prios = priorities, timeout = timeout, **kw) 702 | 703 | def get(self, *args, **kwargs): 704 | smtp = super(SMTPQueue, self).get(*args, **kwargs) 705 | if not getattr(smtp, 'sock', None): 706 | try: 707 | smtp.connect(self.host_primary) 708 | except socket.gaierror, gaie: 709 | if self.host_secondary: 710 | smtp.connect(self.host_secondary) 711 | raise 712 | return smtp 713 | 714 | def _dealloc(self, o): 715 | super(SMTPQueue, self)._dealloc(o) 716 | o.close() 717 | 718 | # 719 | # 720 | # end 721 | 722 | -------------------------------------------------------------------------------- /gogreen/corowork.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python 2 | # -*- Mode: Python; tab-width: 4 -*- 3 | 4 | # Copyright (c) 2005-2010 Slide, Inc. 5 | # All rights reserved. 6 | # 7 | # Redistribution and use in source and binary forms, with or without 8 | # modification, are permitted provided that the following conditions are 9 | # met: 10 | # 11 | # * Redistributions of source code must retain the above copyright 12 | # notice, this list of conditions and the following disclaimer. 13 | # * Redistributions in binary form must reproduce the above 14 | # copyright notice, this list of conditions and the following 15 | # disclaimer in the documentation and/or other materials provided 16 | # with the distribution. 17 | # * Neither the name of the author nor the names of other 18 | # contributors may be used to endorse or promote products derived 19 | # from this software without specific prior written permission. 20 | # 21 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | 33 | """corowork 34 | 35 | Queue with worker threads to service entries. 36 | """ 37 | 38 | import coro 39 | import coroqueue 40 | import statistics 41 | 42 | import exceptions 43 | import time 44 | import sys 45 | import os 46 | 47 | # same as btserv/cacches.py but not importable because of Logic circular import 48 | DB_USAGE_MARKER = 'cache-db-usage' 49 | 50 | class QueueError (exceptions.Exception): 51 | pass 52 | 53 | class Worker(coro.Thread): 54 | def __init__(self, *args, **kwargs): 55 | super(Worker, self).__init__(*args, **kwargs) 56 | 57 | self._requests = kwargs['requests'] 58 | self._filter = kwargs['filter'] 59 | self._prio = kwargs.get('prio', 0) 60 | 61 | self._exit = False 62 | self._drain = False 63 | 64 | def execute(self, *args, **kwargs): 65 | self.info('REQUEST args: %r kwargs: %r' % (args, kwargs)) 66 | 67 | def run(self): 68 | self.info('Starting QueueWorker [%s]: %s' % (self._prio, self.__class__.__name__)) 69 | 70 | while not self._exit: 71 | if self._drain: 72 | timeout = 0 73 | else: 74 | timeout = None 75 | 76 | try: 77 | args, kwargs, start = self._requests.pop(timeout = timeout) 78 | except coro.CoroutineCondWake: 79 | continue 80 | except coro.TimeoutError: 81 | break 82 | 83 | self.profile_clear() 84 | # 85 | # execution 86 | # 87 | self.execute(*args, **kwargs) 88 | # 89 | # record statistics 90 | # 91 | stamp = time.time() 92 | delta = stamp - start 93 | label = self._filter(args, kwargs) 94 | 95 | self.parent().record(stamp, delta, label) 96 | # 97 | # shutdown 98 | # 99 | self.info('Exiting QueueWorker') 100 | 101 | def shutdown(self): 102 | self._exit = True 103 | 104 | def drain(self): 105 | self._drain = True 106 | 107 | def requeue(self, *args, **kwargs): 108 | self._requests.push((args, kwargs, time.time())) 109 | 110 | class CommandWorker(Worker): 111 | '''A worker thread that that expects command from a 112 | notify/rpc_call call to be part of the execute args. Attempts to 113 | lookup the method on this thread object and execute it. 114 | ''' 115 | 116 | def execute(self, object, id, cmd, args, server = None, seq = None): 117 | handler = getattr(self, cmd, None) 118 | result = None 119 | if handler is not None and getattr(handler, 'command', 0): 120 | try: 121 | result = handler(args) 122 | except exceptions.Exception, e: 123 | self.traceback() 124 | if server is not None: 125 | server.rpc_response(seq, result) 126 | 127 | 128 | class Server(coro.Thread): 129 | name = 'WorkQueueServer' 130 | 131 | def __init__(self, *args, **kwargs): 132 | super(Server, self).__init__(*args, **kwargs) 133 | 134 | self.work = kwargs.get('worker', Worker) 135 | self.filter = kwargs.get('filter', lambda i,j: None) 136 | self.sizes = kwargs.get('sizes') or [kwargs['size']] 137 | self.kwargs = kwargs 138 | self.stop = False 139 | self.drain = False 140 | 141 | self.rpcs = {} 142 | self.cmdq = [] 143 | self.klist = [] 144 | self.workers = [] 145 | 146 | self.identity = kwargs.get('identity', {}) 147 | self.waiter = coro.coroutine_cond() 148 | self.requests = [] 149 | 150 | for size in self.sizes: 151 | self.requests.append(coro.coroutine_fifo()) 152 | self.workers.append([]) 153 | 154 | self.stats = statistics.Recorder() 155 | self.wqsize = statistics.WQSizeRecorder() 156 | self.dbuse = statistics.Recorder() 157 | 158 | self.wlimit = statistics.TopRecorder(threshold = 0.0) 159 | self.elimit = statistics.TopRecorder(threshold = 0.0) 160 | self.tlimit = statistics.TopRecorder(threshold = 0.0) 161 | self.rlimit = statistics.TopRecorder(threshold = 0) 162 | 163 | self.kwargs.update({ 164 | 'filter': self.filter}) 165 | 166 | def run(self): 167 | self.info('Starting %s.' % (self.name,)) 168 | 169 | while not self.stop: 170 | # 171 | # spawn workers 172 | # 173 | while self.klist: 174 | work_klass, prio = self.klist.pop(0) 175 | work_kwargs = self.kwargs.copy() 176 | work_kwargs.update( 177 | {'requests' : self.requests[prio], 178 | 'prio' : prio, }) 179 | worker = work_klass(**work_kwargs) 180 | worker.start() 181 | self.workers[prio].append(worker) 182 | # 183 | # execute management commands 184 | # 185 | while self.cmdq: 186 | try: 187 | self.command_dispatch(*self.cmdq.pop(0)) 188 | except: 189 | self.traceback() 190 | # 191 | # wait for something to do 192 | self.waiter.wait() 193 | 194 | self.info( 195 | 'Stopping %s. (children: %d)' % ( 196 | self.name, 197 | self.child_count())) 198 | 199 | self.command_clear() 200 | 201 | for child in self.child_list(): 202 | child.drain() 203 | 204 | for r in self.requests: 205 | r.wake() 206 | self.child_wait() 207 | 208 | def shutdown(self, timeout = None): 209 | if not self.stop: 210 | self.stop = True 211 | self.waiter.wake_all() 212 | 213 | return self.join(timeout) 214 | 215 | def resize(self, size, prio = 0): 216 | if not size: 217 | return None 218 | 219 | if size == self.sizes[prio]: 220 | return None 221 | 222 | kill = max(0, len(self.workers[prio]) - size) 223 | while kill: 224 | kill -= 1 225 | work = self.workers[prio].pop() 226 | work.drain() 227 | 228 | self.requests[prio].wake() 229 | self.sizes[prio] = size 230 | 231 | def spawn(self, prio = 0): 232 | 233 | if self.requests[prio].waiters(): 234 | return None 235 | 236 | size = self.sizes[prio] 237 | if size > self._wcount(prio): 238 | self.klist.append((self.work, prio)) 239 | self.waiter.wake_all() 240 | 241 | def request(self, *args, **kwargs): 242 | '''make a request for this work server. this method can take a prio 243 | kwarg: when it is present, it specifies the maximum priority slot 244 | this request is made in. defaults to the first slot, 0. 245 | 246 | requests are put into the lowest possible slot until that slot is 247 | "full," here full meaning # of worker threads < size for priority slot. 248 | ''' 249 | if self.stop: 250 | raise QueueError('Sevrer has been shutdown.') 251 | 252 | # find the priority to queue this request in to 253 | max_prio = min(kwargs.get('prio', 0), len(self.sizes)) 254 | for prio in xrange(max_prio+1): 255 | if self.requests[prio].waiters(): 256 | break 257 | self.workers[prio] = filter( 258 | lambda w: w.isAlive(), self.workers[prio]) 259 | 260 | # if we got here it means nothing is waiting for something 261 | # to do at this prio. check to see if we can spawn. 262 | room = max(self.sizes[prio] - len(self.workers[prio]), 0) 263 | if not room: 264 | continue 265 | room -= len(filter(lambda i: i[1] == prio, self.klist)) 266 | if room > 0: 267 | self.klist.append((self.work, prio)) 268 | self.waiter.wake_all() 269 | break 270 | 271 | self.requests[prio].push((args, kwargs, time.time())) 272 | 273 | def record(self, current, elapse, label): 274 | self.stats.request(elapse, name = label, current = current) 275 | if coro.get_local(DB_USAGE_MARKER, False): 276 | self.dbuse.request(elapse, name = label, current = current) 277 | self.wqsize.request(sum(map(len, self.requests))) 278 | 279 | data = ( 280 | label, current, elapse, 281 | coro.current_thread().total_time(), 282 | coro.current_thread().long_time(), 283 | coro.current_thread().resume_count()) 284 | 285 | self.wlimit.save(data[2], data) # longest wall clock + queue wait times 286 | self.elimit.save(data[3], data) # longest execution times 287 | self.tlimit.save(data[4], data) # longest non-yield times 288 | self.rlimit.save(data[5], data) # most request yields 289 | 290 | def least_prio(self): 291 | '''return the smallest request priority that is not full. a full 292 | request priority is one that has created all of its worker threads 293 | ''' 294 | for prio in xrange(len(self.sizes)): 295 | if self._wcount(prio) < self.sizes[prio]: 296 | break 297 | return prio 298 | 299 | def _wcount(self, prio): 300 | return len(filter(lambda t: t.isAlive(), self.workers[prio])) + \ 301 | len(filter(lambda k: k[1] == prio, self.klist)) 302 | # 303 | # dispatch management/stats commands 304 | # 305 | def command_list(self): 306 | result = [] 307 | 308 | for name in dir(self): 309 | handler = getattr(self, name, None) 310 | if not callable(handler): 311 | continue 312 | if getattr(handler, 'command', None) is None: 313 | continue 314 | name = name.split('_') 315 | if name[0] != 'object': 316 | continue 317 | name = '_'.join(name[1:]) 318 | if not name: 319 | continue 320 | 321 | result.append(name) 322 | 323 | return result 324 | 325 | def command_clear(self): 326 | for seq in self.rpcs.keys(): 327 | self.command_response(seq) 328 | 329 | def command_push(self, obj, id, cmd, args, seq, server): 330 | self.rpcs[seq] = server 331 | self.cmdq.append((obj, cmd, args, seq)) 332 | 333 | self.waiter.wake_all() 334 | 335 | def command_response(self, seq, response = None): 336 | server = self.rpcs.pop(seq, None) 337 | if server is not None: server.rpc_response(seq, response) 338 | 339 | def command_dispatch(self, obj, cmd, args, seq): 340 | name = 'object_%s' % (cmd,) 341 | handler = getattr(self, name, None) 342 | response = self.identity.copy() 343 | 344 | if getattr(handler, 'command', None) is None: 345 | response.update({'rc': 1, 'msg': 'no handler: <%s>' % name}) 346 | self.command_response(seq, response) 347 | return None 348 | 349 | try: 350 | result = handler(args) 351 | except exceptions.Exception, e: 352 | self.traceback() 353 | t,v,tb = coro.traceback_info() 354 | 355 | response.update({ 356 | 'rc': 1, 357 | 'msg': 'Exception: [%s|%s]' % (t,v), 358 | 'tb': tb}) 359 | else: 360 | response.update({'rc': 0, 'result': result}) 361 | 362 | self.command_response(seq, response) 363 | return None 364 | # 365 | # end.. 366 | -------------------------------------------------------------------------------- /gogreen/coutil.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # -*- Mode: Python; tab-width: 4 -*- 4 | 5 | # Copyright (c) 2005-2010 Slide, Inc. 6 | # All rights reserved. 7 | # 8 | # Redistribution and use in source and binary forms, with or without 9 | # modification, are permitted provided that the following conditions are 10 | # met: 11 | # 12 | # * Redistributions of source code must retain the above copyright 13 | # notice, this list of conditions and the following disclaimer. 14 | # * Redistributions in binary form must reproduce the above 15 | # copyright notice, this list of conditions and the following 16 | # disclaimer in the documentation and/or other materials provided 17 | # with the distribution. 18 | # * Neither the name of the author nor the names of other 19 | # contributors may be used to endorse or promote products derived 20 | # from this software without specific prior written permission. 21 | # 22 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 23 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 24 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 25 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 26 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 27 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 28 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 32 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | 34 | """coutil 35 | 36 | Various coroutine-safe utilities. 37 | 38 | Written by Libor Michalek. 39 | """ 40 | 41 | import os 42 | import sys 43 | import string 44 | import types 45 | import random as whrandom 46 | 47 | import coro 48 | 49 | class object_queue: 50 | 51 | def __init__ (self): 52 | # object queue 53 | self._queue = [] 54 | self._c = coro.coroutine_cond() 55 | 56 | def __len__ (self): 57 | return len(self._queue) 58 | 59 | def push (self, q): 60 | # place data in the queue, and wake up a consumer 61 | self._queue.append(q) 62 | self._c.wake_one() 63 | 64 | def pop (self): 65 | # if there is nothing in the queue, wait to be awoken 66 | while not len(self._queue): 67 | self._c.wait() 68 | 69 | item = self._queue[0] 70 | del self._queue[0] 71 | 72 | return item 73 | 74 | class critical_section: 75 | 76 | def __init__(self): 77 | 78 | self._lock = 0 79 | self._error = 0 80 | self._c = coro.coroutine_cond() 81 | 82 | return None 83 | 84 | def get_lock(self): 85 | 86 | while self._lock and not self._error: 87 | self._c.wait() 88 | 89 | if not self._error: 90 | self._lock = 1 91 | 92 | return self._error 93 | 94 | def release_lock(self): 95 | 96 | self._lock = 0 97 | self._c.wake_one() 98 | 99 | return None 100 | 101 | def error(self): 102 | 103 | self._lock = 0 104 | self._error = 1 105 | self._c.wake_all() 106 | 107 | return None 108 | 109 | class conditional_id: 110 | 111 | def __init__(self): 112 | 113 | self.__wait_map = {} 114 | self.__map_cond = coro.coroutine_cond() 115 | 116 | def wait(self, id): 117 | 118 | self.__wait_map[id] = coro.current_thread().thread_id() 119 | self.__map_cond.wait() 120 | 121 | return None 122 | 123 | def wake(self, id): 124 | 125 | if self.__wait_map.has_key(id): 126 | self.__map_cond.wake(self.__wait_map[id]) 127 | del self.__wait_map[id] 128 | 129 | return None 130 | 131 | def wake_one(self): 132 | 133 | if len(self.__wait_map): 134 | id = whrandom.choice(self.__wait_map.keys()) 135 | self.wake(id) 136 | 137 | return None 138 | 139 | def wake_all(self): 140 | 141 | self.__wait_map = {} 142 | self.__map_cond.wake_all() 143 | 144 | return None 145 | -------------------------------------------------------------------------------- /gogreen/dqueue.py: -------------------------------------------------------------------------------- 1 | # -*- Mode: Python; tab-width: 4 -*- 2 | 3 | # Copyright (c) 2005-2010 Slide, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are 8 | # met: 9 | # 10 | # * Redistributions of source code must retain the above copyright 11 | # notice, this list of conditions and the following disclaimer. 12 | # * Redistributions in binary form must reproduce the above 13 | # copyright notice, this list of conditions and the following 14 | # disclaimer in the documentation and/or other materials provided 15 | # with the distribution. 16 | # * Neither the name of the author nor the names of other 17 | # contributors may be used to endorse or promote products derived 18 | # from this software without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | '''dqueue 33 | 34 | Object and Queue class for implementing a doubly linked list queue. 35 | ''' 36 | 37 | import exceptions 38 | import operator 39 | 40 | class ObjectQueueError(exceptions.Exception): 41 | pass 42 | 43 | class QueueObject(object): 44 | __slots__ = ['__item_next__', '__item_prev__'] 45 | 46 | def __init__(self, *args, **kwargs): 47 | self.__item_next__ = None 48 | self.__item_prev__ = None 49 | 50 | def __queued__(self): 51 | return not (self.__item_next__ is None or self.__item_prev__ is None) 52 | 53 | def __clip__(self): 54 | '''__clip__ 55 | 56 | void queue connectivity, used when object has been cloned/copied 57 | and the references are no longer meaningful, since the queue 58 | neighbors correctly reference the original object. 59 | ''' 60 | self.__item_next__ = None 61 | self.__item_prev__ = None 62 | 63 | class QueueIterator(object): 64 | def __init__(self, queue, *args, **kwargs): 65 | self._queue = queue 66 | self._object = self._queue.look_head() 67 | 68 | def __iter__(self): 69 | return self 70 | 71 | def next(self): 72 | if self._object is None: 73 | raise StopIteration 74 | 75 | value, self._object = self._object, self._queue.next(self._object) 76 | if self._object is self._queue.look_head(): 77 | self._object = None 78 | 79 | return value 80 | 81 | class ObjectQueue(object): 82 | def __init__(self, *args, **kwargs): 83 | self._head = None 84 | self._size = 0 85 | 86 | def __len__(self): 87 | return self._size 88 | 89 | def __del__(self): 90 | return self.clear() 91 | 92 | def __nonzero__(self): 93 | return self._head is not None 94 | 95 | def __iter__(self): 96 | return QueueIterator(self) 97 | 98 | def _validate(self, obj): 99 | if not hasattr(obj, '__item_next__'): 100 | raise ObjectQueueError, 'not a queueable object' 101 | if not hasattr(obj, '__item_prev__'): 102 | raise ObjectQueueError, 'not a queueable object' 103 | 104 | def put(self, obj, fifo = False): 105 | self._validate(obj) 106 | 107 | if self._in_queue(obj): 108 | raise ObjectQueueError, 'object already queued' 109 | 110 | if self._head is None: 111 | obj.__item_next__ = obj 112 | obj.__item_prev__ = obj 113 | self._head = obj 114 | else: 115 | obj.__item_next__ = self._head 116 | obj.__item_prev__ = self._head.__item_prev__ 117 | 118 | obj.__item_next__.__item_prev__ = obj 119 | obj.__item_prev__.__item_next__ = obj 120 | 121 | if fifo: 122 | self._head = obj 123 | 124 | self._size += 1 125 | 126 | def get(self, fifo = False): 127 | if self._head is None: 128 | return None 129 | 130 | if fifo: 131 | obj = self._head 132 | else: 133 | obj = self._head.__item_prev__ 134 | 135 | if obj.__item_next__ is obj and obj.__item_prev__ is obj: 136 | self._head = None 137 | else: 138 | obj.__item_next__.__item_prev__ = obj.__item_prev__ 139 | obj.__item_prev__.__item_next__ = obj.__item_next__ 140 | 141 | self._head = obj.__item_next__ 142 | 143 | obj.__item_next__ = None 144 | obj.__item_prev__ = None 145 | 146 | self._size -= 1 147 | return obj 148 | 149 | def look(self, fifo = False): 150 | if self._head is None: 151 | return None 152 | 153 | if fifo: 154 | return self._head 155 | else: 156 | return self._head.__item_prev__ 157 | 158 | def remove(self, obj): 159 | self._validate(obj) 160 | 161 | if not self._in_queue(obj): 162 | raise ObjectQueueError, 'object not queued' 163 | 164 | if obj.__item_next__ is obj and obj.__item_prev__ is obj: 165 | self._head = None 166 | else: 167 | next = obj.__item_next__ 168 | prev = obj.__item_prev__ 169 | next.__item_prev__ = prev 170 | prev.__item_next__ = next 171 | 172 | if self._head == obj: 173 | self._head = next 174 | 175 | obj.__item_next__ = None 176 | obj.__item_prev__ = None 177 | 178 | self._size -= 1 179 | 180 | def next(self, obj): 181 | if self._head == obj.__item_next__: 182 | return None 183 | else: 184 | return obj.__item_next__ 185 | 186 | def prev(self, obj): 187 | if self._head.__item_prev__ == obj.__item_prev__: 188 | return None 189 | else: 190 | return obj.__item_prev__ 191 | 192 | def put_head(self, obj): 193 | return self.put(obj, fifo = True) 194 | 195 | def put_tail(self, obj): 196 | return self.put(obj, fifo = False) 197 | 198 | def get_head(self): 199 | return self.get(fifo = True) 200 | 201 | def get_tail(self): 202 | return self.get(fifo = False) 203 | 204 | def look_head(self): 205 | return self.look(fifo = True) 206 | 207 | def look_tail(self): 208 | return self.look(fifo = False) 209 | 210 | def clear(self): 211 | while self: 212 | self.get() 213 | 214 | def _in_queue(self, obj): 215 | return obj.__item_next__ is not None and obj.__item_prev__ is not None 216 | 217 | def iterqueue(queue, forward = True): 218 | '''emits elements out of the queue in a given order/direction.''' 219 | if forward: 220 | start = queue.look_head 221 | move = queue.next 222 | else: 223 | start = queue.look_tail 224 | move = queue.prev 225 | item = start() 226 | while item: 227 | yield item 228 | item = move(item) 229 | 230 | # 231 | # end... 232 | -------------------------------------------------------------------------------- /gogreen/emulate.py: -------------------------------------------------------------------------------- 1 | # -*- Mode: Python; tab-width: 4 -*- 2 | 3 | # Copyright (c) 2005-2010 Slide, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are 8 | # met: 9 | # 10 | # * Redistributions of source code must retain the above copyright 11 | # notice, this list of conditions and the following disclaimer. 12 | # * Redistributions in binary form must reproduce the above 13 | # copyright notice, this list of conditions and the following 14 | # disclaimer in the documentation and/or other materials provided 15 | # with the distribution. 16 | # * Neither the name of the author nor the names of other 17 | # contributors may be used to endorse or promote products derived 18 | # from this software without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | '''emulate 33 | 34 | Various emulation/monkey-patch functions for functionality which we want 35 | behaving differently in the coro environment. Generally that behaviour 36 | is a coroutine yield and reschedule on code which would have otherwise 37 | blocked the entire process. 38 | 39 | In the past each module maintained individual emulation code, this becomes 40 | unwieldy as the scope continues to expand and encourages functionality 41 | creep. 42 | ''' 43 | import time 44 | 45 | import coro 46 | import corocurl 47 | import coromysql 48 | import corofile 49 | # 50 | # save needed implementations. 51 | # 52 | original = { 53 | 'sleep': time.sleep, 54 | } 55 | 56 | def sleep(value): 57 | thrd = coro.current_thread() 58 | if thrd is None: 59 | original['sleep'](value) 60 | else: 61 | thrd.Yield(timeout = value) 62 | 63 | def emulate_sleep(): 64 | time.sleep = sleep 65 | 66 | 67 | def init(): 68 | '''all 69 | 70 | Enable emulation for all modules/code which can be emulated. 71 | 72 | NOTE: only code which works correctly in coro AND main can/should 73 | be automatically initialized, the below emulations do not 74 | fall into that category, they only work in a coroutine, which 75 | is why the are here. 76 | ''' 77 | coro.socket_emulate() 78 | corocurl.emulate() 79 | coromysql.emulate() 80 | corofile.emulate_popen2() 81 | # 82 | # auto-emulations 83 | # 84 | emulate_sleep() 85 | # 86 | # end.. 87 | -------------------------------------------------------------------------------- /gogreen/fileobject.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- Mode: Python; tab-width: 4 -*- 3 | 4 | # Copyright (c) 2005-2010 Slide, Inc. 5 | # All rights reserved. 6 | # 7 | # Redistribution and use in source and binary forms, with or without 8 | # modification, are permitted provided that the following conditions are 9 | # met: 10 | # 11 | # * Redistributions of source code must retain the above copyright 12 | # notice, this list of conditions and the following disclaimer. 13 | # * Redistributions in binary form must reproduce the above 14 | # copyright notice, this list of conditions and the following 15 | # disclaimer in the documentation and/or other materials provided 16 | # with the distribution. 17 | # * Neither the name of the author nor the names of other 18 | # contributors may be used to endorse or promote products derived 19 | # from this software without specific prior written permission. 20 | # 21 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | 33 | '''fileobject 34 | 35 | emulate a fileobject using a seperate process to handle IO for coroutine 36 | concurrency 37 | 38 | Written by Libor Michalek. 2006 39 | ''' 40 | 41 | import coro 42 | import corofile 43 | import coroqueue 44 | 45 | # corofile.emulate_popen2() 46 | 47 | import os 48 | import sys 49 | import struct 50 | import exceptions 51 | import pickle 52 | import signal 53 | import weakref 54 | import signal 55 | 56 | STAUS_OPENED = 1 57 | STAUS_CLOSED = 0 58 | 59 | COMMAND_SIZE = 8 # cmd is 8 bytes, four for identifier and four for length 60 | COMMAND_OPEN = 0 61 | COMMAND_CLOSE = 1 62 | COMMAND_READ = 2 63 | COMMAND_WRITE = 3 64 | COMMAND_FLUSH = 4 65 | COMMAND_SEEK = 5 66 | 67 | COMMAND_STAT = 6 68 | COMMAND_MKDIRS = 7 69 | COMMAND_CHMOD = 8 70 | COMMAND_UNLINK = 9 71 | COMMAND_UTIME = 10 72 | COMMAND_TELL = 11 73 | COMMAND_READLN = 12 74 | 75 | COMMAND_MAP = { 76 | COMMAND_OPEN: 'open', 77 | COMMAND_CLOSE: 'close', 78 | COMMAND_READ: 'read', 79 | COMMAND_WRITE: 'write', 80 | COMMAND_FLUSH: 'flush', 81 | COMMAND_SEEK: 'seek', 82 | COMMAND_STAT: 'stat', 83 | COMMAND_MKDIRS: 'makedirs', 84 | COMMAND_CHMOD: 'chmod', 85 | COMMAND_UNLINK: 'unlink', 86 | COMMAND_UTIME: 'utime', 87 | COMMAND_TELL: 'tell', 88 | COMMAND_READLN: 'readline', 89 | } 90 | 91 | RESPONSE_SUCCESS = 100 92 | RESPONSE_ERROR = 101 93 | 94 | BUFFER_READ_SIZE = 32*1024 95 | BUFFER_WRITE_SIZE = 16*1024*1024 96 | 97 | DEFAULT_QUEUE_SIZE = 16 98 | 99 | NOWAIT = False 100 | 101 | class CoroFileObjectError (exceptions.Exception): 102 | pass 103 | 104 | class CoroFileObject(object): 105 | def __init__(self): 106 | self._stdi, self._stdo = os.popen2([ 107 | os.path.realpath(__file__.replace('.pyc', '.py'))]) 108 | 109 | self._data = '' 110 | self._status = STAUS_CLOSED 111 | self.name = None 112 | 113 | def __del__(self): 114 | if self._status == STAUS_OPENED: 115 | self.close(return_to_queue = False) 116 | 117 | if self._stdi: self._stdi.close() 118 | if self._stdo: self._stdo.close() 119 | if not NOWAIT: os.wait() 120 | 121 | def _cmd_(self, cmd, data = ''): 122 | self._stdi.write(struct.pack('!ii', cmd, len(data))) 123 | self._stdi.write(data) 124 | self._stdi.flush() 125 | 126 | data = self._stdo.read(COMMAND_SIZE) 127 | if not data: 128 | raise CoroFileObjectError('No command from child') 129 | 130 | response, size = struct.unpack('!ii', data) 131 | if size: 132 | data = self._stdo.read(size) 133 | else: 134 | data = None 135 | 136 | if response == RESPONSE_ERROR: 137 | raise pickle.loads(data) 138 | else: 139 | return data 140 | 141 | def open(self, name, mode = 'r'): 142 | if self._status == STAUS_OPENED: 143 | self.close() 144 | 145 | result = self._cmd_(COMMAND_OPEN, '%s\n%s' % (name, mode)) 146 | self._status = STAUS_OPENED 147 | self.name = name 148 | return result 149 | 150 | def read(self, bytes = -1): 151 | while bytes < 0 or bytes > len(self._data): 152 | result = self._cmd_(COMMAND_READ) 153 | if result is None: 154 | break 155 | 156 | self._data += result 157 | # 158 | # either have all the data we want or file EOF 159 | # 160 | if bytes < 0: 161 | # Return all remaining data 162 | data = self._data 163 | self._data = '' 164 | else: 165 | data = self._data[:bytes] 166 | self._data = self._data[bytes:] 167 | 168 | return data 169 | 170 | def readline(self, bytes = -1): 171 | while bytes < 0 or bytes > len(self._data): 172 | result = self._cmd_(COMMAND_READLN) 173 | if result is None: 174 | break 175 | self._data += result 176 | if result[-1] == '\n': 177 | break 178 | 179 | # 180 | # either have all the data we want or file EOF 181 | # 182 | if bytes < 0: 183 | # Return all remaining data 184 | data = self._data 185 | self._data = '' 186 | else: 187 | data = self._data[:bytes] 188 | self._data = self._data[bytes:] 189 | 190 | return data 191 | 192 | 193 | def write(self, data): 194 | while data: 195 | self._cmd_(COMMAND_WRITE, data[:BUFFER_WRITE_SIZE]) 196 | data = data[BUFFER_WRITE_SIZE:] 197 | 198 | def seek(self, offset): 199 | self._data = '' 200 | return self._cmd_(COMMAND_SEEK, struct.pack('!i', offset)) 201 | 202 | def tell(self): 203 | return int(self._cmd_(COMMAND_TELL)) 204 | 205 | def flush(self): 206 | return self._cmd_(COMMAND_FLUSH) 207 | 208 | def close(self, return_to_queue = True): 209 | """close closes this fileobject 210 | NB: The return_to_queue parameter is ignored. It is required 211 | for interface compatability with the AutoCleanFileOjbect subclass. 212 | """ 213 | self._data = '' 214 | self._status = STAUS_CLOSED 215 | return self._cmd_(COMMAND_CLOSE) 216 | # 217 | # non-standard extensions 218 | # 219 | def stat(self, path): 220 | args = eval(self._cmd_(COMMAND_STAT, path)) 221 | return os._make_stat_result(*args) 222 | 223 | def makedirs(self, path): 224 | return eval(self._cmd_(COMMAND_MKDIRS, path)) 225 | 226 | def chmod(self, path, mode): 227 | return eval(self._cmd_(COMMAND_CHMOD, '%s\n%d' % (path, mode))) 228 | 229 | def unlink(self, path): 230 | return eval(self._cmd_(COMMAND_UNLINK, path)) 231 | 232 | def utime(self, path, value = None): 233 | return eval(self._cmd_(COMMAND_UTIME, '%s\n%s' % (path, str(value)))) 234 | 235 | 236 | class AutoCleanFileObject(CoroFileObject): 237 | """AutoCleanFileOjbect overrides close to optionally return itself 238 | to the filequeue after closing. 239 | """ 240 | def close(self, return_to_queue = True): 241 | """close closes this fileobject 242 | 243 | return_to_queue: return this object back to the filequeue if 244 | True, the default. 245 | """ 246 | res = super(AutoCleanFileObject, self).close() 247 | if return_to_queue: 248 | filequeue.put(self) 249 | return res 250 | 251 | class FileObjectHandler(object): 252 | def __init__(self, stdin, stdout): 253 | self._stdi = stdin 254 | self._stdo = stdout 255 | self._fd = None 256 | 257 | def open(self, data): 258 | name, mode = data.split('\n') 259 | 260 | self._fd = file(name, mode, BUFFER_READ_SIZE) 261 | 262 | def close(self, data): 263 | self._fd.close() 264 | 265 | def read(self, data): 266 | return self._fd.read(BUFFER_READ_SIZE) 267 | 268 | def readline(self, data): 269 | r = self._fd.readline(BUFFER_READ_SIZE) 270 | return r 271 | 272 | def write(self, data): 273 | return self._fd.write(data) 274 | 275 | def flush(self, data): 276 | return self._fd.flush() 277 | 278 | def seek(self, data): 279 | (offset,) = struct.unpack('!i', data) 280 | return self._fd.seek(offset) 281 | 282 | def tell(self, data): 283 | return str(self._fd.tell()) 284 | # 285 | # non-standard extensions 286 | # 287 | def stat(self, data): 288 | return str(os.stat(data).__reduce__()[1]) 289 | 290 | def makedirs(self, data): 291 | return str(os.makedirs(data)) 292 | 293 | def chmod(self, data): 294 | path, mode = data.split('\n') 295 | return str(os.chmod(path, int(mode))) 296 | 297 | def unlink(self, data): 298 | return str(os.unlink(data)) 299 | 300 | def utime(self, data): 301 | path, value = data.split('\n') 302 | return str(os.utime(path, eval(value))) 303 | 304 | def run(self): 305 | result = 0 306 | 307 | while True: 308 | try: 309 | data = self._stdi.read(COMMAND_SIZE) 310 | except KeyboardInterrupt: 311 | data = None 312 | 313 | if not data: 314 | result = 1 315 | break 316 | 317 | cmd, size = struct.unpack('!ii', data) 318 | if size: 319 | data = self._stdi.read(size) 320 | else: 321 | data = '' 322 | 323 | if size != len(data): 324 | result = 2 325 | break 326 | 327 | handler = getattr(self, COMMAND_MAP.get(cmd, 'none'), None) 328 | if handler is None: 329 | result = 3 330 | break 331 | 332 | try: 333 | result = handler(data) 334 | except exceptions.Exception, e: 335 | result = pickle.dumps(e) 336 | response = RESPONSE_ERROR 337 | else: 338 | response = RESPONSE_SUCCESS 339 | 340 | if result is None: 341 | result = '' 342 | 343 | try: 344 | self._stdo.write(struct.pack('!ii', response, len(result))) 345 | self._stdo.write(result) 346 | self._stdo.flush() 347 | except IOError: 348 | result = 4 349 | break 350 | 351 | return result 352 | 353 | class CoroFileQueue(coroqueue.Queue): 354 | def __init__(self, size, timeout = None): 355 | super(CoroFileQueue, self).__init__( 356 | AutoCleanFileObject, (), {}, size = size, timeout = timeout) 357 | 358 | self._fd_save = {} 359 | self._fd_refs = {} 360 | 361 | def _save_info(self, o): 362 | def dropped(ref): 363 | self._fd_save.pop(self._fd_refs.pop(id(ref), None), None) 364 | 365 | p = weakref.proxy(o, dropped) 366 | self._fd_save[id(o)] = p 367 | self._fd_refs[id(p)] = id(o) 368 | 369 | return super(CoroFileQueue, self)._save_info(o) 370 | 371 | def _drop_info(self, o): 372 | self._fd_refs.pop(id(self._fd_save.pop(id(o), None)), None) 373 | 374 | return super(CoroFileQueue, self)._drop_info(o) 375 | 376 | def outstanding(self): 377 | return map(lambda i: getattr(i, 'name', None), self._fd_save.values()) 378 | 379 | 380 | filequeue = CoroFileQueue(DEFAULT_QUEUE_SIZE) 381 | 382 | def resize(size): 383 | return filequeue.resize(size) 384 | 385 | def size(): 386 | return filequeue.size() 387 | 388 | def _fo_open(name, mode = 'r'): 389 | fd = filequeue.get() 390 | if fd is None: 391 | return None 392 | 393 | try: 394 | fd.open(name, mode) 395 | except: 396 | filequeue.put(fd) 397 | raise 398 | return fd 399 | 400 | # 401 | # TODO: Remove close method once all references to it have been purged. 402 | # 403 | def _fo_close(fd): 404 | fd.close(return_to_queue=False) 405 | return filequeue.put(fd) 406 | 407 | def __command__(name, *args, **kwargs): 408 | fd = filequeue.get() 409 | if fd is None: 410 | return None 411 | 412 | try: 413 | return getattr(fd, name)(*args, **kwargs) 414 | finally: 415 | filequeue.put(fd) 416 | 417 | def _fo_stat(path): 418 | return __command__('stat', path) 419 | 420 | def _fo_makedirs(path): 421 | return __command__('makedirs', path) 422 | 423 | def _fo_chmod(path, mode): 424 | return __command__('chmod', path, mode) 425 | 426 | def _fo_unlink(path): 427 | return __command__('unlink', path) 428 | 429 | def _fo_utime(path, value = None): 430 | return __command__('utime', path, value = value) 431 | 432 | # 433 | # os.* calls 434 | # 435 | stat = os.stat 436 | makedirs = os.makedirs 437 | chmod = os.chmod 438 | unlink = os.unlink 439 | utime = os.utime 440 | 441 | # 442 | # file/open call 443 | # 444 | open = file 445 | # 446 | # close call 447 | # 448 | def dummy_close(fd): 449 | return fd.close() 450 | 451 | close = dummy_close 452 | 453 | # a generator function for doing readlines in a fileobject friendly manner 454 | def iterlines(fd): 455 | while True: 456 | ln = fd.readline() 457 | if not ln: 458 | break 459 | yield ln 460 | 461 | def iterfiles(filenames): 462 | for fn in filenames: 463 | fd = open(fn) 464 | for ln in iterlines(fd): 465 | yield ln 466 | fd.close() 467 | 468 | def emulate(): 469 | fileobject = sys.modules['gogreen.fileobject'] 470 | # 471 | # os.* calls 472 | # 473 | fileobject.stat = _fo_stat 474 | fileobject.makedirs = _fo_makedirs 475 | fileobject.chmod = _fo_chmod 476 | fileobject.unlink = _fo_unlink 477 | fileobject.utime = _fo_utime 478 | 479 | # 480 | # file/open call 481 | # 482 | fileobject.open = _fo_open 483 | fileobject.close = _fo_close 484 | 485 | def nowait(): 486 | '''nowait 487 | 488 | NOTE: GLOBAL SIGNAL CHANGE! 489 | 490 | Do not wait for the terminated/exiting fileobject, since this can 491 | block. To prevent the processes from becoming unreaped zombies we 492 | disable the SIGCHILD signal. (see man wait(2)) 493 | ''' 494 | global NOWAIT 495 | NOWAIT = True 496 | 497 | signal.signal(signal.SIGCHLD, signal.SIG_IGN) 498 | 499 | 500 | if __name__ == '__main__': 501 | import prctl 502 | 503 | prctl.prctl(prctl.PDEATHSIG, signal.SIGTERM) 504 | 505 | handler = FileObjectHandler(sys.stdin, sys.stdout) 506 | value = handler.run() 507 | sys.exit(value) 508 | # 509 | # end.. 510 | -------------------------------------------------------------------------------- /gogreen/purepydns.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- Mode: Python; tab-width: 4 -*- 3 | 4 | # Copyright (c) 2005-2010 Slide, Inc. 5 | # All rights reserved. 6 | # 7 | # Redistribution and use in source and binary forms, with or without 8 | # modification, are permitted provided that the following conditions are 9 | # met: 10 | # 11 | # * Redistributions of source code must retain the above copyright 12 | # notice, this list of conditions and the following disclaimer. 13 | # * Redistributions in binary form must reproduce the above 14 | # copyright notice, this list of conditions and the following 15 | # disclaimer in the documentation and/or other materials provided 16 | # with the distribution. 17 | # * Neither the name of the author nor the names of other 18 | # contributors may be used to endorse or promote products derived 19 | # from this software without specific prior written permission. 20 | # 21 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | 33 | """purepydns.py 34 | 35 | Pure Python DNS resolution routines, and a wrapper method that swaps 36 | them with the baked-in DNS routines. 37 | 38 | Requires the dnspython library available at: 39 | http://www.dnspython.org/ 40 | 41 | Not to be confused with PyDNS, which is an entirely different Python 42 | module for performing DNS lookups. 43 | """ 44 | import os 45 | import sys 46 | import socket 47 | import time 48 | 49 | import dns.exception 50 | import dns.inet 51 | import dns.message 52 | import dns.name 53 | import dns.rdata 54 | import dns.rdataset 55 | import dns.rdatatype 56 | import dns.resolver 57 | import dns.reversename 58 | 59 | import coro 60 | import cache 61 | 62 | DNS_QUERY_TIMEOUT = 10.0 63 | DNS_CACHE_MAXSIZE = 512 64 | 65 | # 66 | # Resolver instance used to perfrom DNS lookups. 67 | # 68 | class FakeAnswer(list): 69 | expiration = 0 70 | class FakeRecord(object): 71 | pass 72 | 73 | class ResolverProxy(object): 74 | def __init__(self, *args, **kwargs): 75 | self._resolver = None 76 | self._filename = kwargs.get('filename', '/etc/resolv.conf') 77 | self._hosts = {} 78 | if kwargs.pop('etc_hosts', True): 79 | self._load_etc_hosts() 80 | 81 | def _load_etc_hosts(self): 82 | try: 83 | fd = open('/etc/hosts', 'r') 84 | contents = fd.read() 85 | fd.close() 86 | except EnvironmentError: 87 | return 88 | 89 | contents = [line for line in contents.split('\n') if line and not line[0] == '#'] 90 | 91 | for line in contents: 92 | line = line.replace('\t', ' ') 93 | parts = line.split(' ') 94 | parts = [p for p in parts if p] 95 | if not len(parts): 96 | continue 97 | ip = parts[0] 98 | for part in parts[1:]: 99 | self._hosts[part] = ip 100 | 101 | def clear(self): 102 | self._resolver = None 103 | 104 | def query(self, *args, **kwargs): 105 | if self._resolver is None: 106 | self._resolver = dns.resolver.Resolver(filename = self._filename) 107 | 108 | query = args[0] 109 | if query is None: 110 | args = list(args) 111 | query = args[0] = '0.0.0.0' 112 | if self._hosts and self._hosts.get(query): 113 | answer = FakeAnswer() 114 | record = FakeRecord() 115 | setattr(record, 'address', self._hosts[query]) 116 | answer.append(record) 117 | return answer 118 | 119 | return self._resolver.query(*args, **kwargs) 120 | # 121 | # cache 122 | # 123 | resolver = ResolverProxy() 124 | responses = cache.LRU(size = DNS_CACHE_MAXSIZE) 125 | 126 | def resolve(name): 127 | error = None 128 | rrset = responses.lookup(name) 129 | 130 | if rrset is None or time.time() > rrset.expiration: 131 | try: 132 | rrset = resolver.query(name) 133 | except dns.exception.Timeout, e: 134 | error = (socket.EAI_AGAIN, 'Lookup timed out') 135 | except dns.exception.DNSException, e: 136 | error = (socket.EAI_NODATA, 'No address associated with hostname') 137 | else: 138 | responses.insert(name, rrset) 139 | 140 | if error: 141 | if rrset is None: 142 | raise socket.gaierror(error) 143 | else: 144 | sys.stderr.write('DNS error: %r %r\n' % (name, error)) 145 | 146 | return rrset 147 | # 148 | # methods 149 | # 150 | def getaliases(host): 151 | """Checks for aliases of the given hostname (cname records) 152 | returns a list of alias targets 153 | will return an empty list if no aliases 154 | """ 155 | cnames = [] 156 | error = None 157 | 158 | try: 159 | answers = dns.resolver.query(host, 'cname') 160 | except dns.exception.Timeout, e: 161 | error = (socket.EAI_AGAIN, 'Lookup timed out') 162 | except dns.exception.DNSException, e: 163 | error = (socket.EAI_NODATA, 'No address associated with hostname') 164 | else: 165 | for record in answers: 166 | cnames.append(str(answers[0].target)) 167 | 168 | if error: 169 | sys.stderr.write('DNS error: %r %r\n' % (host, error)) 170 | 171 | return cnames 172 | 173 | 174 | def getaddrinfo(host, port, family=0, socktype=0, proto=0, flags=0): 175 | """Replacement for Python's socket.getaddrinfo. 176 | 177 | Currently only supports IPv4. At present, flags are not 178 | implemented. 179 | """ 180 | socktype = socktype or socket.SOCK_STREAM 181 | 182 | if is_ipv4_addr(host): 183 | return [(socket.AF_INET, socktype, proto, '', (host, port))] 184 | 185 | rrset = resolve(host) 186 | value = [] 187 | 188 | for rr in rrset: 189 | value.append((socket.AF_INET, socktype, proto, '', (rr.address, port))) 190 | 191 | return value 192 | 193 | def gethostbyname(hostname): 194 | """Replacement for Python's socket.gethostbyname. 195 | 196 | Currently only supports IPv4. 197 | """ 198 | if is_ipv4_addr(hostname): 199 | return hostname 200 | 201 | rrset = resolve(hostname) 202 | return rrset[0].address 203 | 204 | def gethostbyname_ex(hostname): 205 | """Replacement for Python's socket.gethostbyname_ex. 206 | 207 | Currently only supports IPv4. 208 | """ 209 | 210 | if is_ipv4_addr(hostname): 211 | return (hostname, [], [hostname]) 212 | 213 | rrset = resolve(hostname) 214 | addrs = [] 215 | 216 | for rr in rrset: 217 | addrs.append(rr.address) 218 | 219 | return (hostname, [], addrs) 220 | 221 | def getnameinfo(sockaddr, flags): 222 | """Replacement for Python's socket.getnameinfo. 223 | 224 | Currently only supports IPv4. 225 | """ 226 | try: 227 | host, port = sockaddr 228 | except (ValueError, TypeError): 229 | if not isinstance(sockaddr, tuple): 230 | # there's a stdlib test that's hyper-careful about refcounts 231 | del sockaddr 232 | raise TypeError('getnameinfo() argument 1 must be a tuple') 233 | else: 234 | # must be an ipv6 sockaddr, pretend we don't know how to resolve it 235 | raise socket.gaierror( 236 | socket.EAI_NONAME, 'Name or service not known') 237 | 238 | if (flags & socket.NI_NAMEREQD) and (flags & socket.NI_NUMERICHOST): 239 | # Conflicting flags. Punt. 240 | raise socket.gaierror( 241 | (socket.EAI_NONAME, 'Name or service not known')) 242 | 243 | if is_ipv4_addr(host): 244 | try: 245 | rrset = resolver.query( 246 | dns.reversename.from_address(host), dns.rdatatype.PTR) 247 | if len(rrset) > 1: 248 | raise socket.error('sockaddr resolved to multiple addresses') 249 | host = rrset[0].target.to_text(omit_final_dot=True) 250 | except dns.exception.Timeout, e: 251 | if flags & socket.NI_NAMEREQD: 252 | raise socket.gaierror((socket.EAI_AGAIN, 'Lookup timed out')) 253 | except dns.exception.DNSException, e: 254 | if flags & socket.NI_NAMEREQD: 255 | raise socket.gaierror( 256 | (socket.EAI_NONAME, 'Name or service not known')) 257 | else: 258 | try: 259 | rrset = resolver.query(host) 260 | if len(rrset) > 1: 261 | raise socket.error('sockaddr resolved to multiple addresses') 262 | if flags & socket.NI_NUMERICHOST: 263 | host = rrset[0].address 264 | except dns.exception.Timeout, e: 265 | raise socket.gaierror((socket.EAI_AGAIN, 'Lookup timed out')) 266 | except dns.exception.DNSException, e: 267 | raise socket.gaierror( 268 | (socket.EAI_NODATA, 'No address associated with hostname')) 269 | 270 | if not (flags & socket.NI_NUMERICSERV): 271 | proto = (flags & socket.NI_DGRAM) and 'udp' or 'tcp' 272 | port = socket.getservbyport(port, proto) 273 | 274 | return (host, port) 275 | 276 | def is_ipv4_addr(host): 277 | """is_ipv4_addr returns true if host is a valid IPv4 address in 278 | dotted quad notation. 279 | """ 280 | try: 281 | d1, d2, d3, d4 = map(int, host.split('.')) 282 | except (ValueError, AttributeError): 283 | return False 284 | 285 | if 0 <= d1 <= 255 and 0 <= d2 <= 255 and 0 <= d3 <= 255 and 0 <= d4 <= 255: 286 | return True 287 | 288 | return False 289 | 290 | def _net_read(sock, count, expiration): 291 | """coro friendly replacement for dns.query._net_write 292 | Read the specified number of bytes from sock. Keep trying until we 293 | either get the desired amount, or we hit EOF. 294 | A Timeout exception will be raised if the operation is not completed 295 | by the expiration time. 296 | """ 297 | s = '' 298 | while count > 0: 299 | try: 300 | n = sock.recv(count) 301 | except coro.TimeoutError: 302 | ## Q: Do we also need to catch coro.CoroutineSocketWake and pass? 303 | if expiration - time.time() <= 0.0: 304 | raise dns.exception.Timeout 305 | if n == '': 306 | raise EOFError 307 | count = count - len(n) 308 | s = s + n 309 | return s 310 | 311 | def _net_write(sock, data, expiration): 312 | """coro friendly replacement for dns.query._net_write 313 | Write the specified data to the socket. 314 | A Timeout exception will be raised if the operation is not completed 315 | by the expiration time. 316 | """ 317 | current = 0 318 | l = len(data) 319 | while current < l: 320 | try: 321 | current += sock.send(data[current:]) 322 | except coro.TimeoutError: 323 | ## Q: Do we also need to catch coro.CoroutineSocketWake and pass? 324 | if expiration - time.time() <= 0.0: 325 | raise dns.exception.Timeout 326 | 327 | def udp( 328 | q, where, timeout=DNS_QUERY_TIMEOUT, port=53, af=None, source=None, 329 | source_port=0, ignore_unexpected=False): 330 | """coro friendly replacement for dns.query.udp 331 | Return the response obtained after sending a query via UDP. 332 | 333 | @param q: the query 334 | @type q: dns.message.Message 335 | @param where: where to send the message 336 | @type where: string containing an IPv4 or IPv6 address 337 | @param timeout: The number of seconds to wait before the query times out. 338 | If None, the default, wait forever. 339 | @type timeout: float 340 | @param port: The port to which to send the message. The default is 53. 341 | @type port: int 342 | @param af: the address family to use. The default is None, which 343 | causes the address family to use to be inferred from the form of of where. 344 | If the inference attempt fails, AF_INET is used. 345 | @type af: int 346 | @rtype: dns.message.Message object 347 | @param source: source address. The default is the IPv4 wildcard address. 348 | @type source: string 349 | @param source_port: The port from which to send the message. 350 | The default is 0. 351 | @type source_port: int 352 | @param ignore_unexpected: If True, ignore responses from unexpected 353 | sources. The default is False. 354 | @type ignore_unexpected: bool""" 355 | 356 | wire = q.to_wire() 357 | if af is None: 358 | try: 359 | af = dns.inet.af_for_address(where) 360 | except: 361 | af = dns.inet.AF_INET 362 | if af == dns.inet.AF_INET: 363 | destination = (where, port) 364 | if source is not None: 365 | source = (source, source_port) 366 | elif af == dns.inet.AF_INET6: 367 | destination = (where, port, 0, 0) 368 | if source is not None: 369 | source = (source, source_port, 0, 0) 370 | 371 | s = socket.socket(af, socket.SOCK_DGRAM) 372 | s.settimeout(timeout) 373 | try: 374 | expiration = dns.query._compute_expiration(timeout) 375 | if source is not None: 376 | s.bind(source) 377 | try: 378 | s.sendto(wire, destination) 379 | except coro.TimeoutError: 380 | ## Q: Do we also need to catch coro.CoroutineSocketWake and pass? 381 | if expiration - time.time() <= 0.0: 382 | raise dns.exception.Timeout 383 | while 1: 384 | try: 385 | (wire, from_address) = s.recvfrom(65535) 386 | except coro.TimeoutError: 387 | ## Q: Do we also need to catch coro.CoroutineSocketWake and pass? 388 | if expiration - time.time() <= 0.0: 389 | raise dns.exception.Timeout 390 | if from_address == destination: 391 | break 392 | if not ignore_unexpected: 393 | raise dns.query.UnexpectedSource( 394 | 'got a response from %s instead of %s' 395 | % (from_address, destination)) 396 | finally: 397 | s.close() 398 | r = dns.message.from_wire(wire, keyring=q.keyring, request_mac=q.mac) 399 | if not q.is_response(r): 400 | raise dns.query.BadResponse() 401 | return r 402 | 403 | def tcp(q, where, timeout=DNS_QUERY_TIMEOUT, port=53, 404 | af=None, source=None, source_port=0): 405 | """coro friendly replacement for dns.query.tcp 406 | Return the response obtained after sending a query via TCP. 407 | 408 | @param q: the query 409 | @type q: dns.message.Message object 410 | @param where: where to send the message 411 | @type where: string containing an IPv4 or IPv6 address 412 | @param timeout: The number of seconds to wait before the query times out. 413 | If None, the default, wait forever. 414 | @type timeout: float 415 | @param port: The port to which to send the message. The default is 53. 416 | @type port: int 417 | @param af: the address family to use. The default is None, which 418 | causes the address family to use to be inferred from the form of of where. 419 | If the inference attempt fails, AF_INET is used. 420 | @type af: int 421 | @rtype: dns.message.Message object 422 | @param source: source address. The default is the IPv4 wildcard address. 423 | @type source: string 424 | @param source_port: The port from which to send the message. 425 | The default is 0. 426 | @type source_port: int""" 427 | 428 | wire = q.to_wire() 429 | if af is None: 430 | try: 431 | af = dns.inet.af_for_address(where) 432 | except: 433 | af = dns.inet.AF_INET 434 | if af == dns.inet.AF_INET: 435 | destination = (where, port) 436 | if source is not None: 437 | source = (source, source_port) 438 | elif af == dns.inet.AF_INET6: 439 | destination = (where, port, 0, 0) 440 | if source is not None: 441 | source = (source, source_port, 0, 0) 442 | s = socket.socket(af, socket.SOCK_STREAM) 443 | s.settimeout(timeout) 444 | try: 445 | expiration = dns.query._compute_expiration(timeout) 446 | if source is not None: 447 | s.bind(source) 448 | try: 449 | s.connect(destination) 450 | except coro.TimeoutError: 451 | ## Q: Do we also need to catch coro.CoroutineSocketWake and pass? 452 | if expiration - time.time() <= 0.0: 453 | raise dns.exception.Timeout 454 | 455 | l = len(wire) 456 | # copying the wire into tcpmsg is inefficient, but lets us 457 | # avoid writev() or doing a short write that would get pushed 458 | # onto the net 459 | tcpmsg = struct.pack("!H", l) + wire 460 | _net_write(s, tcpmsg, expiration) 461 | ldata = _net_read(s, 2, expiration) 462 | (l,) = struct.unpack("!H", ldata) 463 | wire = _net_read(s, l, expiration) 464 | finally: 465 | s.close() 466 | r = dns.message.from_wire(wire, keyring=q.keyring, request_mac=q.mac) 467 | if not q.is_response(r): 468 | raise dns.query.BadResponse() 469 | return r 470 | 471 | def emulate(): 472 | """Resolve DNS with this module instead of the DNS resolution 473 | functionality that's baked into Python's socket module. 474 | """ 475 | socket.getaddrinfo = getaddrinfo 476 | socket.gethostbyname = gethostbyname 477 | socket.gethostbyname_ex = gethostbyname_ex 478 | socket.getnameinfo = getnameinfo 479 | 480 | # Install our coro-friendly replacements for the tcp and udp 481 | # query methods. 482 | dns.query.tcp = tcp 483 | dns.query.udp = udp 484 | 485 | def reset(): 486 | resolver.clear() 487 | responses.reset(size = DNS_CACHE_MAXSIZE) 488 | 489 | def main(argv=None): 490 | if argv is None: argv = sys.argv 491 | 492 | print "getaddrinfo('www.google.com', 80) returns: %s" % ( 493 | getaddrinfo('www.google.com', 80), ) 494 | print "getaddrinfo('www.slide.com', 80) returns: %s" % ( 495 | getaddrinfo('www.slide.com', 80), ) 496 | print "getaddrinfo('208.76.68.33', 80) returns: %s" % ( 497 | getaddrinfo('208.76.68.33', 80), ) 498 | try: 499 | getaddrinfo('bogus.ghosthacked.net', 80) 500 | except socket.gaierror: 501 | print "getaddrinfo('bogus.ghosthacked.net', 80) failed as expected." 502 | print 503 | 504 | print "gethostbyname('www.google.com') returns: %s" % ( 505 | gethostbyname('www.google.com'), ) 506 | print "gethostbyname('www.slide.com') returns: %s" % ( 507 | gethostbyname('www.slide.com'), ) 508 | print "gethostbyname('208.76.68.33') returns: %s" % ( 509 | gethostbyname('208.76.68.33'), ) 510 | try: 511 | gethostbyname('bogus.ghosthacked.net') 512 | except socket.gaierror: 513 | print "gethostbyname('bogus.ghosthacked.net') failed as expected." 514 | print 515 | 516 | print "gethostbyname_ex('www.google.com') returns: %s" % ( 517 | gethostbyname_ex('www.google.com'), ) 518 | print "gethostbyname_ex('www.slide.com') returns: %s" % ( 519 | gethostbyname_ex('www.slide.com'), ) 520 | print "gethostbyname_ex('208.76.68.33') returns: %s" % ( 521 | gethostbyname_ex('208.76.68.33'), ) 522 | try: 523 | gethostbyname_ex('bogus.ghosthacked.net') 524 | except socket.gaierror: 525 | print "gethostbyname_ex('bogus.ghosthacked.net') failed as expected." 526 | print 527 | 528 | try: 529 | getnameinfo(('www.google.com', 80), 0) 530 | except socket.error: 531 | print "getnameinfo(('www.google.com'), 80, 0) failed as expected." 532 | print "getnameinfo(('www.slide.com', 80), 0) returns: %s" % ( 533 | getnameinfo(('www.slide.com', 80), 0), ) 534 | print "getnameinfo(('208.76.68.33', 80), 0) returns: %s" % ( 535 | getnameinfo(('208.76.68.33', 80), 0), ) 536 | try: 537 | getnameinfo(('bogus.ghosthacked.net', 80), 0) 538 | except socket.gaierror: 539 | print "getnameinfo('bogus.ghosthacked.net') failed as expected." 540 | 541 | return 0 542 | 543 | if __name__ == '__main__': 544 | sys.exit(main()) 545 | -------------------------------------------------------------------------------- /gogreen/pyinfo.py: -------------------------------------------------------------------------------- 1 | # -*- Mode: Python; tab-width: 4 -*- 2 | 3 | # Copyright (c) 2005-2010 Slide, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are 8 | # met: 9 | # 10 | # * Redistributions of source code must retain the above copyright 11 | # notice, this list of conditions and the following disclaimer. 12 | # * Redistributions in binary form must reproduce the above 13 | # copyright notice, this list of conditions and the following 14 | # disclaimer in the documentation and/or other materials provided 15 | # with the distribution. 16 | # * Neither the name of the author nor the names of other 17 | # contributors may be used to endorse or promote products derived 18 | # from this software without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | '''pyinfo 33 | 34 | Functions for gathering and presenting information about the running 35 | python program. 36 | ''' 37 | 38 | import sys 39 | 40 | def rawstack(depth = 1): 41 | data = [] 42 | 43 | while depth is not None: 44 | try: 45 | f = sys._getframe(depth) 46 | except ValueError: 47 | depth = None 48 | else: 49 | data.append(( 50 | '/'.join(f.f_code.co_filename.split('/')[-2:]), 51 | f.f_code.co_name, 52 | f.f_lineno)) 53 | depth += 1 54 | 55 | return data 56 | 57 | def callstack(outfunc = sys.stdout.write, depth = 1, compact = False): 58 | data = rawstack(depth) 59 | data = map(lambda i: '%s:%s:%d' % i, data) 60 | 61 | if compact: 62 | return outfunc('|'.join(data)) 63 | 64 | for value in data: 65 | outfunc(value) 66 | 67 | return None 68 | # 69 | # end... 70 | -------------------------------------------------------------------------------- /gogreen/sendfd.py: -------------------------------------------------------------------------------- 1 | # -*- Mode: Python; tab-width: 4 -*- 2 | 3 | # Copyright (c) 2005-2010 Slide, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are 8 | # met: 9 | # 10 | # * Redistributions of source code must retain the above copyright 11 | # notice, this list of conditions and the following disclaimer. 12 | # * Redistributions in binary form must reproduce the above 13 | # copyright notice, this list of conditions and the following 14 | # disclaimer in the documentation and/or other materials provided 15 | # with the distribution. 16 | # * Neither the name of the author nor the names of other 17 | # contributors may be used to endorse or promote products derived 18 | # from this software without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | '''sendfd.py 33 | 34 | Wrapper functions for sending/receiving a socket between two 35 | unix socket domain endpoints in a coro environment 36 | ''' 37 | 38 | import os 39 | import struct 40 | import socket 41 | 42 | import coro 43 | import sendmsg 44 | 45 | 46 | def sendsocket(control, identifier, sock): 47 | payload = struct.pack('i', sock.fileno()) 48 | 49 | control.wait_for_write() 50 | sendmsg.sendmsg( 51 | control.fileno(), 52 | identifier, 53 | 0, 54 | (socket.SOL_SOCKET, sendmsg.SCM_RIGHTS, payload)) 55 | 56 | 57 | def recvsocket(control): 58 | control.wait_for_read() 59 | result = sendmsg.recvmsg(control.fileno()) 60 | 61 | identifier, flags, [(level, type, data)] = result 62 | 63 | fd = struct.unpack('i', data)[0] 64 | try: 65 | sock = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM) 66 | sock = coro.coroutine_socket(sock) 67 | finally: 68 | os.close(fd) 69 | 70 | return sock 71 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /gogreen/start.py: -------------------------------------------------------------------------------- 1 | from gogreen import coro 2 | from gogreen import emulate 3 | from gogreen import purepydns 4 | 5 | import prctl 6 | 7 | import logging 8 | import sys 9 | import os 10 | import getopt 11 | import resource 12 | import curses 13 | import errno 14 | import stat 15 | import getpass 16 | import socket 17 | 18 | import logging 19 | import logging.handlers 20 | 21 | emulate.init() 22 | purepydns.emulate() 23 | 24 | LOG_SIZE_MAX = 16*1024*1024 25 | LOG_COUNT_MAX = 50 26 | 27 | SERVER_MAX_FD = 32768 28 | 29 | SRV_LOG_FRMT = '[%(name)s|%(coro)s|%(asctime)s|%(levelname)s] %(message)s' 30 | LOGLEVELS = dict( 31 | CRITICAL=logging.CRITICAL, DEBUG=logging.DEBUG, ERROR=logging.ERROR, 32 | FATAL=logging.FATAL, INFO=logging.INFO, WARN=logging.WARN, 33 | WARNING=logging.WARNING,INSANE=5) 34 | 35 | class GreenFormatter(object): 36 | def __init__(self, fmt = SRV_LOG_FRMT, width = 0): 37 | self.fmt = logging.Formatter(fmt) 38 | self.width = width 39 | def set_width(self, width): 40 | self.width = width 41 | def formatTime(self, record, datefmt=None): 42 | return self.fmt.formatTime(record, datefmt) 43 | def formatException(self, ei): 44 | return self.fmt.formatException(ei) 45 | def format(self, record): 46 | msg = self.fmt.format(record) 47 | if self.width: 48 | return msg[:self.width] 49 | else: 50 | return msg 51 | 52 | def get_local_servers(config_map, user = None): 53 | """get_local_servers returns a dictionary keyed by server ids of 54 | all configuration blocks from config_map for servers that run as 55 | the current user on the local machine. The dictionary is keyed 56 | list of configuration blocks all server configurations in 57 | config_map. 58 | """ 59 | local_srvs = {} 60 | user = user or getpass.getuser() 61 | name = socket.gethostname() 62 | for id, server in config_map.items(): 63 | if server.get('host', 'localhost') in [name, 'localhost'] and \ 64 | user == server.get('user', user): 65 | local_srvs[id] = server 66 | 67 | return local_srvs 68 | 69 | 70 | def _lock(server): 71 | """_lock attempt to bind and listen to server's lockport. 72 | Returns the listening socket object on success. 73 | Raises socket.error if the port is already locked. 74 | """ 75 | s = coro.make_socket(socket.AF_INET, socket.SOCK_STREAM) 76 | s.set_reuse_addr() 77 | s.bind((server.get('bind_ip', ''), server['lockport'])) 78 | s.listen(1024) 79 | 80 | return s 81 | 82 | def lock_node(config_map, id=None): 83 | if id is None: 84 | for id, server in get_local_servers(config_map).items(): 85 | try: 86 | s = _lock(server) 87 | except socket.error: 88 | pass 89 | else: 90 | server['lock'] = s 91 | server['total'] = len(config_map) 92 | server['id'] = id 93 | return server 94 | else: 95 | server = get_local_servers(config_map)[id] 96 | try: 97 | s = _lock(server) 98 | except socket.error: 99 | pass 100 | else: 101 | server['lock'] = s 102 | server['total'] = len(config_map) 103 | server['id'] = id 104 | return server 105 | 106 | return None # unused value 107 | 108 | 109 | BASE_COMMAND_LINE_ARGS = [ 110 | 'help', 'fork', 'nowrap', 'logfile=', 'loglevel=', 'pidfile=', 111 | ] 112 | 113 | def usage(name, error = None): 114 | if error: 115 | print 'Error:', error 116 | print " usage: %s [options]" % name 117 | 118 | def main( 119 | serv_dict, exec_func, 120 | name = 'none', base_dir = '.', arg_list = [], defaults = {}, 121 | prefork = None): 122 | 123 | log = coro.coroutine_logger(name) 124 | fmt = GreenFormatter() 125 | 126 | log.setLevel(logging.DEBUG) 127 | # 128 | # check for argument collisions 129 | # 130 | extra = set(map(lambda i: i.strip('='), arg_list)) 131 | base = set(map(lambda i: i.strip('='), BASE_COMMAND_LINE_ARGS)) 132 | both = tuple(extra & base) 133 | 134 | if both: 135 | raise AttributeError( 136 | 'Collision between standard and extended command line arguments', 137 | both) 138 | 139 | progname = sys.argv[0] 140 | # 141 | # internal parameter defaults 142 | # 143 | max_fd = defaults.get('max_fd', SERVER_MAX_FD) 144 | loglevel = 'INFO' 145 | logdir = None 146 | logfile = None 147 | pidfile = None 148 | dofork = False 149 | linewrap = True 150 | # 151 | # setup defaults for true/false parameters 152 | # 153 | parameters = {} 154 | 155 | for key in filter(lambda i: not i.endswith('='), arg_list): 156 | parameters[key] = False 157 | 158 | dirname = os.path.dirname(os.path.abspath(progname)) 159 | os.chdir(dirname) 160 | 161 | try: 162 | list, args = getopt.getopt( 163 | sys.argv[1:], 164 | [], 165 | BASE_COMMAND_LINE_ARGS + arg_list) 166 | except getopt.error, why: 167 | usage(progname, why) 168 | return None 169 | 170 | for (field, val) in list: 171 | field = field.strip('-') 172 | 173 | if field == 'help': 174 | usage(progname) 175 | return None 176 | elif field == 'nowrap': 177 | linewrap = False 178 | elif field == 'logfile': 179 | logfile = val 180 | elif field == 'loglevel': 181 | loglevel = val 182 | elif field == 'pidfile': 183 | pidfile = val 184 | elif field == 'fork': 185 | dofork = True 186 | elif field in extra: 187 | if field in arg_list: 188 | parameters[field] = True 189 | else: 190 | try: 191 | parameters[field] = int(val) 192 | except (TypeError, ValueError): 193 | parameters[field] = val 194 | 195 | # init 196 | here = lock_node(serv_dict) 197 | if here is None: 198 | return 128 199 | 200 | if 'logdir' in here: 201 | logdir = os.path.join(base_dir, here['logdir']) 202 | try: 203 | value = os.stat(logdir) 204 | except OSError, e: 205 | if errno.ENOENT == e[0]: 206 | os.makedirs(logdir) 207 | os.chmod(logdir, stat.S_IRWXU|stat.S_IRWXG|stat.S_IRWXO) 208 | else: 209 | print 'logdir lookup error: %r' % (e,) 210 | return 127 211 | 212 | if prefork is not None: 213 | parameters['prefork'] = prefork() 214 | 215 | if dofork: 216 | pid = os.fork() 217 | if pid: 218 | return 0 219 | 220 | try: 221 | resource.setrlimit(resource.RLIMIT_NOFILE, (max_fd, max_fd)) 222 | except ValueError, e: 223 | if not os.getuid(): 224 | print 'MAX FD error: %s, %d' % (e.args[0], max_fd) 225 | try: 226 | resource.setrlimit(resource.RLIMIT_CORE, (-1, -1)) 227 | except ValueError, e: 228 | print 'CORE size error:', e 229 | 230 | if not os.getuid(): 231 | os.setgid(1) 232 | os.setuid(1) 233 | 234 | try: 235 | prctl.prctl(prctl.DUMPABLE, 1) 236 | except (AttributeError, ValueError, prctl.PrctlError), e: 237 | print 'PRCTL DUMPABLE error:', e 238 | 239 | if logdir and pidfile: 240 | pidfile = logdir + '/' + pidfile 241 | try: 242 | fd = open(pidfile, 'w') 243 | except IOError, e: 244 | print 'IO error: %s' % (e.args[1]) 245 | return None 246 | else: 247 | fd.write('%d' % os.getpid()) 248 | fd.close() 249 | 250 | if logdir and logfile: 251 | logfile = logdir + '/' + logfile 252 | hndlr = logging.handlers.RotatingFileHandler( 253 | logfile, 'a', LOG_SIZE_MAX, LOG_COUNT_MAX) 254 | 255 | os.close(sys.stdin.fileno()) 256 | os.close(sys.stdout.fileno()) 257 | os.close(sys.stderr.fileno()) 258 | else: 259 | if not linewrap: 260 | win = curses.initscr() 261 | height, width = win.getmaxyx() 262 | win = None 263 | curses.reset_shell_mode() 264 | curses.endwin() 265 | 266 | fmt.set_width(width) 267 | 268 | hndlr = logging.StreamHandler(sys.stdout) 269 | 270 | sys.stdout = coro.coroutine_stdout(log) 271 | sys.stderr = coro.coroutine_stderr(log) 272 | 273 | hndlr.setFormatter(fmt) 274 | log.addHandler(hndlr) 275 | loglevel = LOGLEVELS.get(loglevel, None) 276 | if loglevel is None: 277 | log.warn('Unknown logging level, using INFO: %r' % (loglevel, )) 278 | loglevel = logging.INFO 279 | 280 | max_fd = resource.getrlimit(resource.RLIMIT_NOFILE)[0] 281 | log.info('uid: %d, gid: %d, max fd: %d' % (os.getuid(),os.getgid(),max_fd)) 282 | 283 | result = exec_func(here, log, loglevel, logdir, **parameters) 284 | 285 | if result is not None: 286 | log.critical('Server exiting: %r' % (result,)) 287 | else: 288 | log.critical('Server exit') 289 | 290 | return 0 291 | -------------------------------------------------------------------------------- /gogreen/statistics.py: -------------------------------------------------------------------------------- 1 | # -*- Mode: Python; tab-width: 4 -*- 2 | 3 | # Copyright (c) 2005-2010 Slide, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are 8 | # met: 9 | # 10 | # * Redistributions of source code must retain the above copyright 11 | # notice, this list of conditions and the following disclaimer. 12 | # * Redistributions in binary form must reproduce the above 13 | # copyright notice, this list of conditions and the following 14 | # disclaimer in the documentation and/or other materials provided 15 | # with the distribution. 16 | # * Neither the name of the author nor the names of other 17 | # contributors may be used to endorse or promote products derived 18 | # from this software without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | '''statistics 33 | 34 | Maintain second resolution statistics which can then be consolidated into 35 | statistics for multiple time periods. 36 | ''' 37 | 38 | import time 39 | import bisect 40 | 41 | QUEUE_PERIOD = [15, 60, 300, 600] 42 | QUEUE_DEPTH = QUEUE_PERIOD[-1] 43 | HEAP_LIMIT = 128 44 | 45 | class Recorder(object): 46 | def __init__(self, depth = QUEUE_DEPTH, period = QUEUE_PERIOD): 47 | self._global = [{'timestamp': 0, 'elapse': 0, 'count': 0}] 48 | self._local = {} 49 | self._depth = depth 50 | self._period = period 51 | 52 | def _local_request(self, elapse, name, current): 53 | # 54 | # track average request time per second for a given period 55 | # 56 | record = self._local.setdefault(name, []) 57 | 58 | if not record or record[-1]['timestamp'] != current: 59 | record.append({'timestamp': current, 'elapse': 0, 'count': 0}) 60 | 61 | record[-1]['count'] += 1 62 | record[-1]['elapse'] += int(elapse * 1000000) 63 | # 64 | # clear old entries 65 | # 66 | while len(record) > self._depth: 67 | del(record[0]) 68 | 69 | def _global_request(self, elapse, current): 70 | # 71 | # track average request time per second for a given period 72 | # 73 | if self._global[-1]['timestamp'] != current: 74 | self._global.append({'timestamp': current,'elapse': 0,'count': 0}) 75 | 76 | self._global[-1]['count'] += 1 77 | self._global[-1]['elapse'] += int(elapse * 1000000) 78 | # 79 | # clear old entries 80 | # 81 | while len(self._global) > self._depth: 82 | del(self._global[0]) 83 | # 84 | # info API 85 | # 86 | def last(self): 87 | return self._global[-1]['timestamp'] 88 | # 89 | # logging API 90 | # 91 | def request(self, elapse, name = 'none', current = None): 92 | if current is None: 93 | current = int(time.time()) 94 | else: 95 | current = int(current) 96 | 97 | self._local_request(elapse, name, current) 98 | self._global_request(elapse, current) 99 | # 100 | # query API 101 | # 102 | def rate(self, current = None): 103 | current = current or int(time.time()) 104 | results = [] 105 | index = 0 106 | 107 | timestamps, counts = zip(*map( 108 | lambda x: (x['timestamp'], x['count']), 109 | self._global)) 110 | 111 | for period in self._period[::-1]: 112 | index = bisect.bisect(timestamps, (current - period), index) 113 | results.append(sum(counts[index:])/period) 114 | 115 | return results[::-1] 116 | 117 | def details(self, current = None): 118 | current = current or int(time.time()) 119 | results = {} 120 | 121 | for name, record in self._local.items(): 122 | results[name] = [] 123 | index = 0 124 | 125 | timestamps, counts, elapse = zip(*map( 126 | lambda x: (x['timestamp'], x['count'], x['elapse']), 127 | record)) 128 | 129 | for period in self._period[::-1]: 130 | index = bisect.bisect(timestamps, (current - period), index) 131 | results[name].append({ 132 | 'count': sum(counts[index:]), 133 | 'elapse': sum(elapse[index:]), 134 | 'seconds': period}) 135 | 136 | results[name].reverse() 137 | return results 138 | 139 | def averages(self, current = None): 140 | current = current or int(time.time()) 141 | results = [] 142 | index = 0 143 | 144 | timestamps, counts, elapse = zip(*map( 145 | lambda x: (x['timestamp'], x['count'], x['elapse']), 146 | self._global)) 147 | 148 | for period in self._period[::-1]: 149 | index = bisect.bisect(timestamps, (current - period), index) 150 | reqs = sum(counts[index:]) 151 | 152 | results.append({ 153 | 'count': reqs/period, 154 | 'elapse': (reqs and sum(elapse[index:])/reqs) or 0, 155 | 'seconds': period}) 156 | 157 | return results[::-1] 158 | 159 | 160 | class RecorderHitRate(object): 161 | def __init__(self, depth = QUEUE_DEPTH, period = QUEUE_PERIOD): 162 | self._global = [(0, 0, 0)] 163 | self._depth = depth 164 | self._period = period 165 | # 166 | # logging API 167 | # 168 | def request(self, hit = True, current = None): 169 | if current is None: 170 | current = int(time.time()) 171 | else: 172 | current = int(current) 173 | 174 | pos = int(hit) 175 | neg = int(not hit) 176 | 177 | if self._global[-1][0] != current: 178 | self._global.append((current, pos, neg)) 179 | else: 180 | current, oldpos, oldneg = self._global[-1] 181 | self._global[-1] = (current, oldpos + pos, oldneg + neg) 182 | 183 | while len(self._global) > self._depth: 184 | del(self._global[0]) 185 | # 186 | # query API 187 | # 188 | def data(self): 189 | current = int(time.time()) 190 | results = [] 191 | index = 0 192 | 193 | timelist, poslist, neglist = zip(*self._global) 194 | 195 | for period in self._period[::-1]: 196 | index = bisect.bisect(timelist, (current - period), index) 197 | 198 | pos = sum(poslist[index:]) 199 | neg = sum(neglist[index:]) 200 | 201 | results.append((pos, neg)) 202 | 203 | return results[::-1] 204 | 205 | def rate(self): 206 | return map( 207 | lambda i: sum(i) and float(i[0])/sum(i) or None, 208 | self.data()) 209 | 210 | class WQSizeRecorder(object): 211 | 212 | def __init__(self, depth = QUEUE_DEPTH, period = QUEUE_PERIOD): 213 | self._global = [(0, 0, 0)] 214 | self._depth = depth 215 | self._period = period 216 | 217 | # logging API 218 | def request(self, size, current = None): 219 | if current is None: 220 | current = int(time.time()) 221 | else: 222 | current = int(current) 223 | 224 | size = int(size) 225 | 226 | if self._global[-1][0] != current: 227 | self._global.append((current, size, 1)) 228 | else: 229 | current, oldsize, oldcnt = self._global[-1] 230 | self._global[-1] = (current, oldsize + size, oldcnt + 1) 231 | 232 | while len(self._global) > self._depth: 233 | del(self._global[0]) 234 | 235 | # query api 236 | def sizes(self): 237 | current = int(time.time()) 238 | results = [] 239 | index = 0 240 | 241 | timelist, sizelist, cntlist = zip(*self._global) 242 | 243 | for period in self._period[::-1]: 244 | index = bisect.bisect(timelist, (current - period), index) 245 | 246 | sizes = sizelist[index:] 247 | cnts = cntlist[index:] 248 | results.append((sum(sizes), len(sizes)+sum(cnts))) 249 | 250 | return results[::-1] 251 | 252 | class TopRecorder(object): 253 | def __init__(self, depth = HEAP_LIMIT, threshold = None): 254 | self._heap = [(0, None)] 255 | self._depth = depth 256 | self._hold = threshold 257 | 258 | def fetch(self, limit = None): 259 | if limit: 260 | return self._heap[-limit:] 261 | else: 262 | return self._heap 263 | 264 | def save(self, value, data): 265 | if not value > self._hold: 266 | return None 267 | 268 | if value < self._heap[0][0]: 269 | return None 270 | 271 | bisect.insort(self._heap, (value, data)) 272 | 273 | while len(self._heap) > self._depth: 274 | del(self._heap[0]) 275 | # 276 | # end.. 277 | -------------------------------------------------------------------------------- /gogreen/wsgi.py: -------------------------------------------------------------------------------- 1 | ''' 2 | untested, exploratory module serving a WSGI app on the corohttpd framework 3 | ''' 4 | 5 | import coro 6 | import corohttpd 7 | import logging 8 | import sys 9 | 10 | 11 | class _WSGIInput(object): 12 | def __init__(self, request): 13 | self._request = request 14 | self._length = int(request.get_header('Content-Length', '0')) 15 | 16 | def read(self, size=-1): 17 | conn = self._request._connection 18 | gathered = len(conn.buffer) 19 | 20 | # reading Content-Length bytes should behave like EOF 21 | if size >= 0: 22 | size = min(size, self._length) 23 | 24 | if not size: 25 | return '' 26 | 27 | while 1: 28 | data = conn.connection.recv(corohttpd.READ_CHUNK_SIZE) 29 | gathered += len(data) 30 | conn.buffer += data 31 | if not data: 32 | data, conn.buffer = conn.buffer, '' 33 | self._length -= len(data) 34 | return data 35 | if size >= 0 and gathered >= size: 36 | break 37 | 38 | data, conn.buffer = conn.buffer[:size], conn.buffer[size:] 39 | self._length -= len(data) 40 | return data 41 | 42 | def readline(self): 43 | conn = self._request._connection 44 | while 1: 45 | index = conn.buffer.find("\r\n") 46 | if index >= 0 or len(conn.buffer) >= self._length: 47 | if index < 0: 48 | index = len(conn.buffer) 49 | result, conn.buffer = conn.buffer[:index], conn.buffer[index:] 50 | result = result[:self._length] 51 | self._length -= len(result) 52 | return result 53 | 54 | data = conn.connection.recv(corohttpd.READ_CHUNK_SIZE) 55 | if not data: 56 | break 57 | conn.buffer += data 58 | 59 | result, conn.buffer = conn.buffer, '' 60 | self._length -= len(result) 61 | return result 62 | 63 | def readlines(self, hint=None): 64 | return list(self._readlines()) 65 | 66 | def _readlines(self): 67 | line = self.readline() 68 | while line: 69 | yield line 70 | line = self.readline() 71 | 72 | 73 | class _WSGIErrors(object): 74 | def __init__(self, logger): 75 | self._log = logger 76 | 77 | def flush(self): 78 | pass 79 | 80 | def write(self, msg): 81 | self._log.log(logging.ERROR, msg) 82 | 83 | def writelines(self, lines): 84 | map(self.write, lines) 85 | 86 | 87 | class WSGIAppHandler(object): 88 | def __init__(self, app): 89 | self.app = app 90 | 91 | def match(self, request): 92 | return True 93 | 94 | def handle_request(self, request): 95 | address = request.server().socket.getsockname() 96 | if request._connection._log: 97 | errors = _WSGIErrors(request._connection._log) 98 | else: 99 | errors = sys.stderr 100 | 101 | environ = { 102 | 'wsgi.version': (1, 0), 103 | 'wsgi.url_scheme': 'http', 104 | 'wsgi.input': _WSGIInput(request), 105 | 'wsgi.errors': errors, 106 | 'wsgi.multithread': False, 107 | 'wsgi.multiprocess': False, 108 | 'wsgi.run_once': False, 109 | 'SCRIPT_NAME': '', 110 | 'PATH_INFO': request._path, 111 | 'SERVER_NAME': address[0], 112 | 'SERVER_PORT': address[1], 113 | 'REQUEST_METHOD': request.method().upper(), 114 | 'SERVER_PROTOCOL': request._connection.protocol_version, 115 | } 116 | 117 | if request._query: 118 | environ['QUERY_STRING'] = request._query 119 | 120 | clheader = request.get_header('Content-Length') 121 | if clheader: 122 | environ['CONTENT_LENGTH'] = clheader 123 | 124 | ctheader = request.get_header('Content-Type') 125 | if ctheader: 126 | environ['CONTENT_TYPE'] = ctheader 127 | 128 | for name, value in request.get_headers().items(): 129 | environ['HTTP_%s' % name.replace('-', '_').upper()] = value 130 | 131 | headers_sent = [False] 132 | 133 | def start_response(status, headers, exc_info=None): 134 | if exc_info and collector[1]: 135 | raise exc_info[0], exc_info[1], exc_info[2] 136 | else: 137 | exc_info = None 138 | 139 | # this is goofy -- get the largest status prefix that is an integer 140 | for index, character in enumerate(status): 141 | if index and not status[:index].isdigit(): 142 | break 143 | 144 | code = int(status[:index - 1]) 145 | request.response(code) 146 | 147 | for name, value in headers: 148 | request.set_header(name, value) 149 | 150 | headers_sent[0] = True 151 | 152 | return request.push 153 | 154 | body_iterable = self.app(environ, start_response) 155 | for chunk in body_iterable: 156 | request.push(chunk) 157 | 158 | 159 | def serve(address, wsgiapp, access_log='', error_log=None): 160 | kwargs = {} 161 | if error_log: 162 | handler = logging.handlers.RotatingFileHandler( 163 | filename or 'log', 164 | 'a', 165 | corohttpd.ACCESS_LOG_SIZE_MAX, 166 | corohttpd.ACCESS_LOG_COUNT_MAX) 167 | handler.setFormatter(logging.Formatter('%(message)s')) 168 | kwargs['log'] = logging.Logger('error') 169 | kwargs['log'].addHandler(handler) 170 | 171 | server = corohttpd.HttpServer(args=(address, access_log), **kwargs) 172 | server.push_handler(WSGIAppHandler(wsgiapp)) 173 | server.start() 174 | coro.event_loop() 175 | 176 | 177 | def main(): 178 | def hello_wsgi_world(environ, start_response): 179 | start_response('200 OK', [ 180 | ('Content-Type', 'text/plain'), 181 | ('Content-Length', '13')]) 182 | return ["Hello, World!"] 183 | 184 | serve(("", 8000), hello_wsgi_world, access_log='access.log') 185 | 186 | 187 | if __name__ == '__main__': 188 | main() 189 | -------------------------------------------------------------------------------- /pavement.py: -------------------------------------------------------------------------------- 1 | import errno 2 | import os 3 | from setuptools import Extension 4 | 5 | from paver.easy import * 6 | from paver.path import path 7 | from paver.setuputils import setup 8 | 9 | 10 | setup( 11 | name="gogreen", 12 | description="Coroutine utilities for non-blocking I/O with greenlet", 13 | version="1.0.1", 14 | license="bsd", 15 | author="Libor Michalek", 16 | author_email="libor@pobox.com", 17 | packages=["gogreen"], 18 | classifiers = [ 19 | "Development Status :: 5 - Production/Stable", 20 | "Intended Audience :: Developers", 21 | "License :: OSI Approved :: BSD License", 22 | "Natural Language :: English", 23 | "Operating System :: Unix", 24 | "Programming Language :: Python", 25 | "Topic :: System :: Networking", 26 | ] 27 | ) 28 | 29 | MANIFEST = ( 30 | "LICENSE", 31 | "setup.py", 32 | "paver-minilib.zip", 33 | ) 34 | 35 | @task 36 | def manifest(): 37 | path('MANIFEST.in').write_lines('include %s' % x for x in MANIFEST) 38 | 39 | @task 40 | @needs('generate_setup', 'minilib', 'manifest', 'setuptools.command.sdist') 41 | def sdist(): 42 | pass 43 | 44 | @task 45 | def clean(): 46 | for p in map(path, ('gogreen.egg-info', 'dist', 'build', 'MANIFEST.in')): 47 | if p.exists(): 48 | if p.isdir(): 49 | p.rmtree() 50 | else: 51 | p.remove() 52 | for p in path(__file__).abspath().parent.walkfiles(): 53 | if p.endswith(".pyc") or p.endswith(".pyo"): 54 | try: 55 | p.remove() 56 | except OSError, exc: 57 | if exc.args[0] == errno.EACCES: 58 | continue 59 | raise 60 | -------------------------------------------------------------------------------- /paver-minilib.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/slideinc/gogreen/2b505886f9bd0e23f48205a7964510006286108e/paver-minilib.zip -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | if os.path.exists("paver-minilib.zip"): 3 | import sys 4 | sys.path.insert(0, "paver-minilib.zip") 5 | 6 | import paver.tasks 7 | paver.tasks.main() 8 | --------------------------------------------------------------------------------