├── .gitignore ├── LICENSE ├── README.md ├── doozer ├── __init__.py ├── client.py └── msg_pb2.py ├── examples ├── client.py ├── doozerdata.py ├── watch.py └── watch_glob.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.egg-info 2 | *.swp 3 | *.pyc 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011 Jeff Lindsay 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PyDoozer 2 | 3 | A Python client for [Doozer](https://github.com/ha/doozerd) using gevent. 4 | 5 | ## Status 6 | 7 | Still quite early (but is running in production in at least one place :) ) 8 | 9 | ## Todo 10 | 11 | * Entity class to wrap Response objects about entities 12 | * Finish watch, access support 13 | * tests, docs 14 | * Make work with standard python as well as gevent 15 | 16 | ## Contributors 17 | 18 | * Jeff Lindsay 19 | * Neuman Vong 20 | * Allan Beaufour [beaufour](http://github.com/beaufour) / 21 | 22 | ## License 23 | 24 | MIT 25 | -------------------------------------------------------------------------------- /doozer/__init__.py: -------------------------------------------------------------------------------- 1 | from client import connect 2 | 3 | __version__ = '0.2.2' 4 | -------------------------------------------------------------------------------- /doozer/client.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import random 4 | import struct 5 | 6 | import gevent 7 | import gevent.event 8 | import gevent.socket 9 | 10 | from msg_pb2 import Response 11 | from msg_pb2 import Request 12 | 13 | REQUEST_TIMEOUT = 2.0 14 | 15 | DEFAULT_RETRY_WAIT = 2.0 16 | """Default connection retry waiting time (seconds)""" 17 | 18 | DEFAULT_URI = "doozer:?%s" % "&".join([ 19 | "ca=127.0.0.1:8046", 20 | "ca=127.0.0.1:8041", 21 | "ca=127.0.0.1:8042", 22 | "ca=127.0.0.1:8043", 23 | ]) 24 | 25 | _spawner = gevent.spawn 26 | 27 | 28 | class ConnectError(Exception): pass 29 | class ResponseError(Exception): 30 | def __init__(self, response, request): 31 | self.code = response.err_code 32 | self.detail = response.err_detail 33 | self.response = response 34 | self.request = request 35 | 36 | def __str__(self): 37 | return str(pb_dict(self.request)) 38 | 39 | class TagInUse(ResponseError): pass 40 | class UnknownVerb(ResponseError): pass 41 | class Readonly(ResponseError): pass 42 | class TooLate(ResponseError): pass 43 | class RevMismatch(ResponseError): pass 44 | class BadPath(ResponseError): pass 45 | class MissingArg(ResponseError): pass 46 | class Range(ResponseError): pass 47 | class NotDirectory(ResponseError): pass 48 | class IsDirectory(ResponseError): pass 49 | class NoEntity(ResponseError): pass 50 | 51 | 52 | def response_exception(response): 53 | """Takes a response, returns proper exception if it has an error code""" 54 | exceptions = { 55 | Response.TAG_IN_USE: TagInUse, Response.UNKNOWN_VERB: UnknownVerb, 56 | Response.READONLY: Readonly, Response.TOO_LATE: TooLate, 57 | Response.REV_MISMATCH: RevMismatch, Response.BAD_PATH: BadPath, 58 | Response.MISSING_ARG: MissingArg, Response.RANGE: Range, 59 | Response.NOTDIR: NotDirectory, Response.ISDIR: IsDirectory, 60 | Response.NOENT: NoEntity, } 61 | if 'err_code' in [field.name for field, value in response.ListFields()]: 62 | return exceptions[response.err_code] 63 | else: 64 | return None 65 | 66 | 67 | def pb_dict(message): 68 | """Create dict representation of a protobuf message""" 69 | return dict([(field.name, value) for field, value in message.ListFields()]) 70 | 71 | 72 | def parse_uri(uri): 73 | """Parse the doozerd URI scheme to get node addresses""" 74 | if uri.startswith("doozer:?"): 75 | before, params = uri.split("?", 1) 76 | addrs = [] 77 | for param in params.split("&"): 78 | key, value = param.split("=", 1) 79 | if key == "ca": 80 | addrs.append(value) 81 | return addrs 82 | else: 83 | raise ValueError("invalid doozerd uri") 84 | 85 | 86 | def connect(uri=None, timeout=None): 87 | """ 88 | Start a Doozer client connection 89 | 90 | @param uri: str|None, Doozer URI 91 | @param timeout: float|None, connection timeout in seconds (per address) 92 | """ 93 | 94 | uri = uri or os.environ.get("DOOZER_URI", DEFAULT_URI) 95 | addrs = parse_uri(uri) 96 | if not addrs: 97 | raise ValueError("there were no addrs supplied in the uri (%s)" % uri) 98 | return Client(addrs, timeout) 99 | 100 | 101 | class Connection(object): 102 | def __init__(self, addrs=None, timeout=None): 103 | """ 104 | @param timeout: float|None, connection timeout in seconds (per address) 105 | """ 106 | self._logger = logging.getLogger('pydoozer.Connection') 107 | self._logger.debug('__init__(%s)', addrs) 108 | 109 | if addrs is None: 110 | addrs = [] 111 | self.addrs = addrs 112 | self.addrs_index = 0 113 | """Next address to connect to in self.addrs""" 114 | 115 | self.pending = {} 116 | self.loop = None 117 | self.sock = None 118 | self.address = None 119 | self.timeout = timeout 120 | self.ready = gevent.event.Event() 121 | 122 | # Shuffle the addresses so all clients don't connect to the 123 | # same node in the cluster. 124 | random.shuffle(addrs) 125 | 126 | def connect(self): 127 | self.reconnect() 128 | 129 | def reconnect(self, kill_loop=True): 130 | """ 131 | Reconnect to the cluster. 132 | 133 | @param kill_loop: bool, kill the current receive loop 134 | """ 135 | 136 | self._logger.debug('reconnect()') 137 | 138 | self.disconnect(kill_loop) 139 | 140 | # Default to the socket timeout 141 | retry_wait = self.timeout or gevent.socket.getdefaulttimeout() or DEFAULT_RETRY_WAIT 142 | for retry in range(5): 143 | addrs_left = len(self.addrs) 144 | while addrs_left: 145 | try: 146 | parts = self.addrs[self.addrs_index].split(':') 147 | self.addrs_index = (self.addrs_index + 1) % len(self.addrs) 148 | host = parts[0] 149 | port = parts[1] if len(parts) > 1 else 8046 150 | self.address = "%s:%s" % (host, port) 151 | self._logger.debug('Connecting to %s...', self.address) 152 | self.sock = gevent.socket.create_connection((host, int(port)), 153 | timeout=self.timeout) 154 | self._logger.debug('Connection successful') 155 | 156 | # Reset the timeout on the connection so it 157 | # doesn't make .recv() and .send() timeout. 158 | self.sock.settimeout(None) 159 | self.ready.set() 160 | 161 | # Any commands that were in transit when the 162 | # connection was lost is obviously not getting a 163 | # reply. Retransmit them. 164 | self._retransmit_pending() 165 | self.loop = _spawner(self._recv_loop) 166 | return 167 | 168 | except IOError, e: 169 | self._logger.info('Failed to connect to %s (%s)', self.address, e) 170 | pass 171 | addrs_left -= 1 172 | 173 | self._logger.debug('Waiting %d seconds to reconnect', retry_wait) 174 | gevent.sleep(retry_wait) 175 | retry_wait *= 2 176 | 177 | self._logger.error('Could not connect to any of the defined addresses') 178 | raise ConnectError("Can't connect to any of the addresses: %s" % self.addrs) 179 | 180 | def disconnect(self, kill_loop=True): 181 | """ 182 | Disconnect current connection. 183 | 184 | @param kill_loop: bool, Kill the current receive loop 185 | """ 186 | self._logger.debug('disconnect()') 187 | 188 | if kill_loop and self.loop: 189 | self._logger.debug('killing loop') 190 | self.loop.kill() 191 | self.loop = None 192 | if self.sock: 193 | self._logger.debug('closing connection') 194 | self.sock.close() 195 | self.sock = None 196 | 197 | self._logger.debug('clearing ready signal') 198 | self.ready.clear() 199 | self.address = None 200 | 201 | def send(self, request, retry=True): 202 | request.tag = 0 203 | while request.tag in self.pending: 204 | request.tag += 1 205 | request.tag %= 2**31 206 | 207 | # Create and send request 208 | data = request.SerializeToString() 209 | data_len = len(data) 210 | head = struct.pack(">I", data_len) 211 | packet = ''.join([head, data]) 212 | entry = self.pending[request.tag] = { 213 | 'event': gevent.event.AsyncResult(), 214 | 'packet': packet, 215 | } 216 | self._logger.debug('Sending packet, tag: %d, len: %d', request.tag, data_len) 217 | try: 218 | self._send_pack(packet, retry) 219 | 220 | # Wait for response 221 | try: 222 | response = entry['event'].get(timeout=REQUEST_TIMEOUT) 223 | except gevent.timeout.Timeout: 224 | if retry: 225 | # If we get a timeout (which is conservatively high), 226 | # something is probably wrong with the 227 | # connection/instance so reconnect to the 228 | # cluster. This will trigger a retransmit of the 229 | # packages in transit. 230 | logging.debug('Got timeout on receive, triggering reconnect()') 231 | self.reconnect() 232 | response = entry['event'].get(timeout=REQUEST_TIMEOUT) 233 | 234 | except Exception: 235 | raise 236 | finally: 237 | # We want to ensure that we always clear the pending 238 | # request, since nothing is now waiting for the answer. 239 | del self.pending[request.tag] 240 | 241 | exception = response_exception(response) 242 | if exception: 243 | raise exception(response, request) 244 | return response 245 | 246 | def _send_pack(self, packet, retry=True): 247 | """ 248 | Send the given packet to the currently connected node. 249 | 250 | @param packet: struct, packet to send 251 | @param retry: bool, retry the sending once 252 | """ 253 | try: 254 | self.ready.wait(timeout=2) 255 | self.sock.send(packet) 256 | except IOError, e: 257 | self._logger.warning('Error sending packet (%s)', e) 258 | self.reconnect() 259 | if retry: 260 | self._logger.debug('Retrying sending packet') 261 | self.ready.wait() 262 | self.sock.send(packet) 263 | else: 264 | self._logger.warning('Failed retrying to send packet') 265 | raise e 266 | 267 | def _recv_loop(self): 268 | self._logger.debug('_recv_loop(%s)', self.address) 269 | 270 | while True: 271 | try: 272 | head = self.sock.recv(4) 273 | length = struct.unpack(">I", head)[0] 274 | data = self.sock.recv(length) 275 | response = Response() 276 | response.ParseFromString(data) 277 | self._logger.debug('Received packet, tag: %d, len: %d', response.tag, length) 278 | if response.tag in self.pending: 279 | self.pending[response.tag]['event'].set(response) 280 | except struct.error, e: 281 | self._logger.warning('Got invalid packet from server (%s)', e) 282 | # If some extra bytes are sent, just reconnect. 283 | # This is related to this bug: 284 | # https://github.com/ha/doozerd/issues/5 285 | break 286 | except IOError, e: 287 | self._logger.warning('Lost connection? (%s)', e) 288 | break 289 | 290 | # Note: .reconnect() will spawn a new loop 291 | self.reconnect(kill_loop=False) 292 | 293 | def _retransmit_pending(self): 294 | """ 295 | Retransmits all pending packets. 296 | """ 297 | 298 | for i in xrange(0, len(self.pending)): 299 | self._logger.debug('Retransmitting packet') 300 | try: 301 | self._send_pack(self.pending[i]['packet'], retry=False) 302 | except Exception: 303 | # If we can't even retransmit the package, we give 304 | # up. The consumer will timeout. 305 | logging.warning('Got exception retransmitting package') 306 | 307 | 308 | class Client(object): 309 | def __init__(self, addrs=None, timeout=None): 310 | """ 311 | @param timeout: float|None, connection timeout in seconds (per address) 312 | """ 313 | if addrs is None: 314 | addrs = [] 315 | self.connection = Connection(addrs, timeout) 316 | self.connect() 317 | 318 | def rev(self): 319 | request = Request(verb=Request.REV) 320 | return self.connection.send(request) 321 | 322 | def set(self, path, value, rev): 323 | request = Request(path=path, value=value, rev=rev, verb=Request.SET) 324 | return self.connection.send(request, retry=False) 325 | 326 | def get(self, path, rev=None): 327 | request = Request(path=path, verb=Request.GET) 328 | if rev: 329 | request.rev = rev 330 | return self.connection.send(request) 331 | 332 | def delete(self, path, rev): 333 | request = Request(path=path, rev=rev, verb=Request.DEL) 334 | return self.connection.send(request, retry=False) 335 | 336 | def wait(self, path, rev): 337 | request = Request(path=path, rev=rev, verb=Request.WAIT) 338 | return self.connection.send(request) 339 | 340 | def stat(self, path, rev): 341 | request = Request(path=path, rev=rev, verb=Request.STAT) 342 | return self.connection.send(request) 343 | 344 | def access(self, secret): 345 | request = Request(value=secret, verb=Request.ACCESS) 346 | return self.connection.send(request) 347 | 348 | def _getdir(self, path, offset=0, rev=None): 349 | request = Request(path=path, offset=offset, verb=Request.GETDIR) 350 | if rev: 351 | request.rev = rev 352 | return self.connection.send(request) 353 | 354 | def _walk(self, path, offset=0, rev=None): 355 | request = Request(path=path, offset=offset, verb=Request.WALK) 356 | if rev: 357 | request.rev = rev 358 | return self.connection.send(request) 359 | 360 | def watch(self, path, rev): 361 | raise NotImplementedError() 362 | 363 | def _list(self, method, path, offset=None, rev=None): 364 | offset = offset or 0 365 | entities = [] 366 | try: 367 | while True: 368 | response = getattr(self, method)(path, offset, rev) 369 | entities.append(response) 370 | offset += 1 371 | except ResponseError, e: 372 | if e.code == Response.RANGE: 373 | return entities 374 | else: 375 | raise e 376 | 377 | def walk(self, path, offset=None, rev=None): 378 | return self._list('_walk', path, offset, rev) 379 | 380 | def getdir(self, path, offset=None, rev=None): 381 | return self._list('_getdir', path, offset, rev) 382 | 383 | def disconnect(self): 384 | self.connection.disconnect() 385 | 386 | def connect(self): 387 | self.connection.connect() 388 | -------------------------------------------------------------------------------- /doozer/msg_pb2.py: -------------------------------------------------------------------------------- 1 | # Generated by the protocol buffer compiler. DO NOT EDIT! 2 | 3 | from google.protobuf import descriptor 4 | from google.protobuf import message 5 | from google.protobuf import reflection 6 | from google.protobuf import descriptor_pb2 7 | # @@protoc_insertion_point(imports) 8 | 9 | 10 | 11 | DESCRIPTOR = descriptor.FileDescriptor( 12 | name='msg.proto', 13 | package='server', 14 | serialized_pb='\n\tmsg.proto\x12\x06server\"\xf2\x01\n\x07Request\x12\x0b\n\x03tag\x18\x01 \x01(\x05\x12\"\n\x04verb\x18\x02 \x01(\x0e\x32\x14.server.Request.Verb\x12\x0c\n\x04path\x18\x04 \x01(\t\x12\r\n\x05value\x18\x05 \x01(\x0c\x12\x11\n\tother_tag\x18\x06 \x01(\x05\x12\x0e\n\x06offset\x18\x07 \x01(\x05\x12\x0b\n\x03rev\x18\t \x01(\x03\"i\n\x04Verb\x12\x07\n\x03GET\x10\x01\x12\x07\n\x03SET\x10\x02\x12\x07\n\x03\x44\x45L\x10\x03\x12\x07\n\x03REV\x10\x05\x12\x08\n\x04WAIT\x10\x06\x12\x07\n\x03NOP\x10\x07\x12\x08\n\x04WALK\x10\t\x12\n\n\x06GETDIR\x10\x0e\x12\x08\n\x04STAT\x10\x10\x12\n\n\x06\x41\x43\x43\x45SS\x10\x63\"\xc8\x02\n\x08Response\x12\x0b\n\x03tag\x18\x01 \x01(\x05\x12\r\n\x05\x66lags\x18\x02 \x01(\x05\x12\x0b\n\x03rev\x18\x03 \x01(\x03\x12\x0c\n\x04path\x18\x05 \x01(\t\x12\r\n\x05value\x18\x06 \x01(\x0c\x12\x0b\n\x03len\x18\x08 \x01(\x05\x12&\n\x08\x65rr_code\x18\x64 \x01(\x0e\x32\x14.server.Response.Err\x12\x12\n\nerr_detail\x18\x65 \x01(\t\"\xac\x01\n\x03\x45rr\x12\t\n\x05OTHER\x10\x7f\x12\x0e\n\nTAG_IN_USE\x10\x01\x12\x10\n\x0cUNKNOWN_VERB\x10\x02\x12\x0c\n\x08READONLY\x10\x03\x12\x0c\n\x08TOO_LATE\x10\x04\x12\x10\n\x0cREV_MISMATCH\x10\x05\x12\x0c\n\x08\x42\x41\x44_PATH\x10\x06\x12\x0f\n\x0bMISSING_ARG\x10\x07\x12\t\n\x05RANGE\x10\x08\x12\n\n\x06NOTDIR\x10\x14\x12\t\n\x05ISDIR\x10\x15\x12\t\n\x05NOENT\x10\x16') 15 | 16 | 17 | 18 | _REQUEST_VERB = descriptor.EnumDescriptor( 19 | name='Verb', 20 | full_name='server.Request.Verb', 21 | filename=None, 22 | file=DESCRIPTOR, 23 | values=[ 24 | descriptor.EnumValueDescriptor( 25 | name='GET', index=0, number=1, 26 | options=None, 27 | type=None), 28 | descriptor.EnumValueDescriptor( 29 | name='SET', index=1, number=2, 30 | options=None, 31 | type=None), 32 | descriptor.EnumValueDescriptor( 33 | name='DEL', index=2, number=3, 34 | options=None, 35 | type=None), 36 | descriptor.EnumValueDescriptor( 37 | name='REV', index=3, number=5, 38 | options=None, 39 | type=None), 40 | descriptor.EnumValueDescriptor( 41 | name='WAIT', index=4, number=6, 42 | options=None, 43 | type=None), 44 | descriptor.EnumValueDescriptor( 45 | name='NOP', index=5, number=7, 46 | options=None, 47 | type=None), 48 | descriptor.EnumValueDescriptor( 49 | name='WALK', index=6, number=9, 50 | options=None, 51 | type=None), 52 | descriptor.EnumValueDescriptor( 53 | name='GETDIR', index=7, number=14, 54 | options=None, 55 | type=None), 56 | descriptor.EnumValueDescriptor( 57 | name='STAT', index=8, number=16, 58 | options=None, 59 | type=None), 60 | descriptor.EnumValueDescriptor( 61 | name='ACCESS', index=9, number=99, 62 | options=None, 63 | type=None), 64 | ], 65 | containing_type=None, 66 | options=None, 67 | serialized_start=159, 68 | serialized_end=264, 69 | ) 70 | 71 | _RESPONSE_ERR = descriptor.EnumDescriptor( 72 | name='Err', 73 | full_name='server.Response.Err', 74 | filename=None, 75 | file=DESCRIPTOR, 76 | values=[ 77 | descriptor.EnumValueDescriptor( 78 | name='OTHER', index=0, number=127, 79 | options=None, 80 | type=None), 81 | descriptor.EnumValueDescriptor( 82 | name='TAG_IN_USE', index=1, number=1, 83 | options=None, 84 | type=None), 85 | descriptor.EnumValueDescriptor( 86 | name='UNKNOWN_VERB', index=2, number=2, 87 | options=None, 88 | type=None), 89 | descriptor.EnumValueDescriptor( 90 | name='READONLY', index=3, number=3, 91 | options=None, 92 | type=None), 93 | descriptor.EnumValueDescriptor( 94 | name='TOO_LATE', index=4, number=4, 95 | options=None, 96 | type=None), 97 | descriptor.EnumValueDescriptor( 98 | name='REV_MISMATCH', index=5, number=5, 99 | options=None, 100 | type=None), 101 | descriptor.EnumValueDescriptor( 102 | name='BAD_PATH', index=6, number=6, 103 | options=None, 104 | type=None), 105 | descriptor.EnumValueDescriptor( 106 | name='MISSING_ARG', index=7, number=7, 107 | options=None, 108 | type=None), 109 | descriptor.EnumValueDescriptor( 110 | name='RANGE', index=8, number=8, 111 | options=None, 112 | type=None), 113 | descriptor.EnumValueDescriptor( 114 | name='NOTDIR', index=9, number=20, 115 | options=None, 116 | type=None), 117 | descriptor.EnumValueDescriptor( 118 | name='ISDIR', index=10, number=21, 119 | options=None, 120 | type=None), 121 | descriptor.EnumValueDescriptor( 122 | name='NOENT', index=11, number=22, 123 | options=None, 124 | type=None), 125 | ], 126 | containing_type=None, 127 | options=None, 128 | serialized_start=423, 129 | serialized_end=595, 130 | ) 131 | 132 | 133 | _REQUEST = descriptor.Descriptor( 134 | name='Request', 135 | full_name='server.Request', 136 | filename=None, 137 | file=DESCRIPTOR, 138 | containing_type=None, 139 | fields=[ 140 | descriptor.FieldDescriptor( 141 | name='tag', full_name='server.Request.tag', index=0, 142 | number=1, type=5, cpp_type=1, label=1, 143 | has_default_value=False, default_value=0, 144 | message_type=None, enum_type=None, containing_type=None, 145 | is_extension=False, extension_scope=None, 146 | options=None), 147 | descriptor.FieldDescriptor( 148 | name='verb', full_name='server.Request.verb', index=1, 149 | number=2, type=14, cpp_type=8, label=1, 150 | has_default_value=False, default_value=1, 151 | message_type=None, enum_type=None, containing_type=None, 152 | is_extension=False, extension_scope=None, 153 | options=None), 154 | descriptor.FieldDescriptor( 155 | name='path', full_name='server.Request.path', index=2, 156 | number=4, type=9, cpp_type=9, label=1, 157 | has_default_value=False, default_value=unicode("", "utf-8"), 158 | message_type=None, enum_type=None, containing_type=None, 159 | is_extension=False, extension_scope=None, 160 | options=None), 161 | descriptor.FieldDescriptor( 162 | name='value', full_name='server.Request.value', index=3, 163 | number=5, type=12, cpp_type=9, label=1, 164 | has_default_value=False, default_value="", 165 | message_type=None, enum_type=None, containing_type=None, 166 | is_extension=False, extension_scope=None, 167 | options=None), 168 | descriptor.FieldDescriptor( 169 | name='other_tag', full_name='server.Request.other_tag', index=4, 170 | number=6, type=5, cpp_type=1, label=1, 171 | has_default_value=False, default_value=0, 172 | message_type=None, enum_type=None, containing_type=None, 173 | is_extension=False, extension_scope=None, 174 | options=None), 175 | descriptor.FieldDescriptor( 176 | name='offset', full_name='server.Request.offset', index=5, 177 | number=7, type=5, cpp_type=1, label=1, 178 | has_default_value=False, default_value=0, 179 | message_type=None, enum_type=None, containing_type=None, 180 | is_extension=False, extension_scope=None, 181 | options=None), 182 | descriptor.FieldDescriptor( 183 | name='rev', full_name='server.Request.rev', index=6, 184 | number=9, type=3, cpp_type=2, label=1, 185 | has_default_value=False, default_value=0, 186 | message_type=None, enum_type=None, containing_type=None, 187 | is_extension=False, extension_scope=None, 188 | options=None), 189 | ], 190 | extensions=[ 191 | ], 192 | nested_types=[], 193 | enum_types=[ 194 | _REQUEST_VERB, 195 | ], 196 | options=None, 197 | is_extendable=False, 198 | extension_ranges=[], 199 | serialized_start=22, 200 | serialized_end=264, 201 | ) 202 | 203 | 204 | _RESPONSE = descriptor.Descriptor( 205 | name='Response', 206 | full_name='server.Response', 207 | filename=None, 208 | file=DESCRIPTOR, 209 | containing_type=None, 210 | fields=[ 211 | descriptor.FieldDescriptor( 212 | name='tag', full_name='server.Response.tag', index=0, 213 | number=1, type=5, cpp_type=1, label=1, 214 | has_default_value=False, default_value=0, 215 | message_type=None, enum_type=None, containing_type=None, 216 | is_extension=False, extension_scope=None, 217 | options=None), 218 | descriptor.FieldDescriptor( 219 | name='flags', full_name='server.Response.flags', index=1, 220 | number=2, type=5, cpp_type=1, label=1, 221 | has_default_value=False, default_value=0, 222 | message_type=None, enum_type=None, containing_type=None, 223 | is_extension=False, extension_scope=None, 224 | options=None), 225 | descriptor.FieldDescriptor( 226 | name='rev', full_name='server.Response.rev', index=2, 227 | number=3, type=3, cpp_type=2, label=1, 228 | has_default_value=False, default_value=0, 229 | message_type=None, enum_type=None, containing_type=None, 230 | is_extension=False, extension_scope=None, 231 | options=None), 232 | descriptor.FieldDescriptor( 233 | name='path', full_name='server.Response.path', index=3, 234 | number=5, type=9, cpp_type=9, label=1, 235 | has_default_value=False, default_value=unicode("", "utf-8"), 236 | message_type=None, enum_type=None, containing_type=None, 237 | is_extension=False, extension_scope=None, 238 | options=None), 239 | descriptor.FieldDescriptor( 240 | name='value', full_name='server.Response.value', index=4, 241 | number=6, type=12, cpp_type=9, label=1, 242 | has_default_value=False, default_value="", 243 | message_type=None, enum_type=None, containing_type=None, 244 | is_extension=False, extension_scope=None, 245 | options=None), 246 | descriptor.FieldDescriptor( 247 | name='len', full_name='server.Response.len', index=5, 248 | number=8, type=5, cpp_type=1, label=1, 249 | has_default_value=False, default_value=0, 250 | message_type=None, enum_type=None, containing_type=None, 251 | is_extension=False, extension_scope=None, 252 | options=None), 253 | descriptor.FieldDescriptor( 254 | name='err_code', full_name='server.Response.err_code', index=6, 255 | number=100, type=14, cpp_type=8, label=1, 256 | has_default_value=False, default_value=127, 257 | message_type=None, enum_type=None, containing_type=None, 258 | is_extension=False, extension_scope=None, 259 | options=None), 260 | descriptor.FieldDescriptor( 261 | name='err_detail', full_name='server.Response.err_detail', index=7, 262 | number=101, type=9, cpp_type=9, label=1, 263 | has_default_value=False, default_value=unicode("", "utf-8"), 264 | message_type=None, enum_type=None, containing_type=None, 265 | is_extension=False, extension_scope=None, 266 | options=None), 267 | ], 268 | extensions=[ 269 | ], 270 | nested_types=[], 271 | enum_types=[ 272 | _RESPONSE_ERR, 273 | ], 274 | options=None, 275 | is_extendable=False, 276 | extension_ranges=[], 277 | serialized_start=267, 278 | serialized_end=595, 279 | ) 280 | 281 | _REQUEST.fields_by_name['verb'].enum_type = _REQUEST_VERB 282 | _REQUEST_VERB.containing_type = _REQUEST; 283 | _RESPONSE.fields_by_name['err_code'].enum_type = _RESPONSE_ERR 284 | _RESPONSE_ERR.containing_type = _RESPONSE; 285 | DESCRIPTOR.message_types_by_name['Request'] = _REQUEST 286 | DESCRIPTOR.message_types_by_name['Response'] = _RESPONSE 287 | 288 | class Request(message.Message): 289 | __metaclass__ = reflection.GeneratedProtocolMessageType 290 | DESCRIPTOR = _REQUEST 291 | 292 | # @@protoc_insertion_point(class_scope:server.Request) 293 | 294 | class Response(message.Message): 295 | __metaclass__ = reflection.GeneratedProtocolMessageType 296 | DESCRIPTOR = _RESPONSE 297 | 298 | # @@protoc_insertion_point(class_scope:server.Response) 299 | 300 | # @@protoc_insertion_point(module_scope) 301 | -------------------------------------------------------------------------------- /examples/client.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import os 3 | import sys 4 | sys.path.append(os.path.dirname(__file__) + "/..") 5 | 6 | import doozer 7 | 8 | client = doozer.connect() 9 | 10 | rev = client.set("/foo", "test", 0).rev 11 | print "Setting /foo to test with rev %s" % rev 12 | 13 | foo = client.get("/foo") 14 | print "Got /foo with %s" % foo.value 15 | 16 | root = client.getdir("/") 17 | print "Directly under / is %s" % ', '.join([file.path for file in root]) 18 | 19 | client.delete("/foo", rev) 20 | print "Deleted /foo" 21 | 22 | foo = client.get("/foo") 23 | print repr(foo) 24 | 25 | walk = client.walk("/**") 26 | for file in walk: 27 | print ' '.join([file.path, str(file.rev), file.value]) 28 | 29 | client.disconnect() 30 | -------------------------------------------------------------------------------- /examples/doozerdata.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | """ 3 | created by stephan preeker. 2011-10 4 | """ 5 | import doozer 6 | import gevent 7 | 8 | from doozer.client import RevMismatch, TooLate, NoEntity, BadPath 9 | from gevent import Timeout 10 | 11 | class DoozerData(): 12 | """ 13 | class which stores data to doozerd backend 14 | 15 | locally we store the path -> revision numbers dict 16 | 17 | -all values need to be strings. 18 | -we watch changes to values. 19 | in case of a update we call the provided callback method with value. 20 | 21 | on initialization a path/folder can be specified where all keys 22 | will be stored. 23 | """ 24 | 25 | def __init__(self, client, callback=None, path='/pydooz'): 26 | self.client = client 27 | self._folder = path 28 | self.revisions = {} 29 | #method called when external source changes value. 30 | self.callback = callback 31 | 32 | #load existing values. 33 | walk = client.walk('%s/**' % path) 34 | for file in walk: 35 | self.revisions[self.key_path(file.path)] = file.rev 36 | 37 | #start watching for changes. 38 | if self.callback: 39 | self.watch() 40 | 41 | def watch(self): 42 | """ 43 | watch the directory path for changes. 44 | call callback on change. 45 | do NOT call callback if it is a change of our own, 46 | thus when the revision is the same as the rev we have 47 | stored in out revisions. 48 | """ 49 | 50 | rev = self.client.rev().rev 51 | 52 | def watchjob(rev): 53 | change = None 54 | 55 | while True: 56 | try: 57 | change = self.client.wait("%s/**" % self._folder, rev) 58 | except Timeout: 59 | rev = self.client.rev().rev 60 | change = None 61 | 62 | if change: 63 | self._handle_change(change) 64 | rev = change.rev+1 65 | #print '.....', rev 66 | 67 | self.watchjob = gevent.spawn(watchjob, rev) 68 | 69 | def _handle_change(self, change): 70 | """ 71 | A change has been watched. deal with it. 72 | """ 73 | #get the last part of the path. 74 | key_path = self.key_path(change.path) 75 | 76 | print 'handle change', self.revisions.get(key_path, 0), change.rev 77 | 78 | if self._old_or_delete(key_path, change): 79 | return 80 | 81 | print change.path 82 | self.revisions[key_path] = change.rev 83 | #create or update route. 84 | if change.flags == 4: 85 | self.revisions[key_path] = change.rev 86 | self.callback(change.value) 87 | return 88 | 89 | print 'i could get here ...if i saw my own/old delete.' 90 | print change 91 | 92 | def _old_or_delete(self, key_path, change): 93 | """ 94 | If the change is done by ourselves or already seen 95 | we don't have to do a thing. 96 | if change is a delete not from outselves call the callback. 97 | """ 98 | if key_path in self.revisions: 99 | #check if we already have this change. 100 | if self.revisions[key_path] == change.rev: 101 | return True 102 | #check if it is an delete action. 103 | #if key_path is still in revisions it is not our 104 | #own or old delete action. 105 | if change.flags == 8: 106 | print 'got delete!!' 107 | self.revisions.pop(key_path) 108 | self.callback(change.value, path=key_path, destroy=True) 109 | return True 110 | 111 | return False 112 | 113 | def get(self, key_path): 114 | """ 115 | get the latest data for path. 116 | 117 | get the local revision number 118 | if revision revnumber does not match?? 119 | -SUCCEED and update rev number. 120 | return the doozer data 121 | """ 122 | rev = 0 123 | if key_path in self.revisions: 124 | rev = self.revisions[key_path] 125 | try: 126 | return self.client.get(self.folder(key_path), rev).value 127 | except RevMismatch: 128 | print 'revision mismach..' 129 | item = self.client.get(self.folder(key_path)) 130 | self.revisions[key_path] = item.rev 131 | return item.value 132 | 133 | def set(self, key_path, value): 134 | """ 135 | set a value, BUT check if you have the latest revision. 136 | """ 137 | if not isinstance(value, str): 138 | raise TypeError('Keywords for this object must be strings. You supplied %s' % type(value)) 139 | 140 | rev = 0 141 | if key_path in self.revisions: 142 | rev = self.revisions[key_path] 143 | self._set(key_path, value, rev) 144 | 145 | def _set(self, key_path, value, rev): 146 | try: 147 | newrev = self.client.set(self.folder(key_path), value, rev) 148 | self.revisions[key_path] = newrev.rev 149 | print self.revisions[key_path] 150 | print 'setting %s with rev %s oldrev %s' % (key_path, newrev.rev, rev) 151 | except RevMismatch: 152 | print 'ERROR failed to set %s %s %s' % (key_path, rev, self.revisions[key_path]) 153 | 154 | def key_path(self, path): 155 | return path.split('/')[-1] 156 | 157 | def folder(self, key_path): 158 | return "%s/%s" % (self._folder, key_path) 159 | 160 | def delete(self, key_path): 161 | """ 162 | delete path. only with correct latest revision. 163 | """ 164 | try: 165 | rev = self.revisions[key_path] 166 | self.revisions.pop(key_path) 167 | item = self.client.delete(self.folder(key_path), rev) 168 | except RevMismatch: 169 | print 'ERROR!! rev value changed meanwhile!!', item.path, item.value 170 | except BadPath: 171 | print 'ERROR!! path is bad.', self.folder(key_path) 172 | 173 | def delete_all(self): 174 | """ clear all data. 175 | """ 176 | for path, rev, value in self.items(): 177 | try: 178 | item = self.client.delete(self.folder(path), rev) 179 | except RevMismatch: 180 | item = self.client.delete(self.folder(path)) 181 | print 'value changed meanwhile!!', item.path, item.value 182 | except TooLate: 183 | print 'too late..' 184 | rev = self.client.rev().rev 185 | item = self.client.delete(self.folder(path), rev) 186 | 187 | def items(self): 188 | """ 189 | return all current items from doozer. 190 | update local rev numbers. 191 | """ 192 | try: 193 | folder = self.client.getdir(self._folder) 194 | except NoEntity: 195 | print 'we are empty' 196 | folder = [] 197 | 198 | for thing in folder: 199 | item = self.client.get(self.folder(thing.path)) 200 | yield (thing.path, item.rev, item.value) 201 | 202 | 203 | def print_change(change, path=None, destroy=True): 204 | print 'watched a change..' 205 | print change, destroy, path 206 | 207 | def change_value(d): 208 | 209 | gevent.sleep(1) 210 | d.set('test', '0') 211 | gevent.sleep(1) 212 | d.set('test2', '0') 213 | gevent.sleep(1) 214 | d.set('test', '1') 215 | 216 | #make sure you start doozerd(s). 217 | def test_doozerdata(): 218 | 219 | client = doozer.connect() 220 | d = DoozerData(client, callback=print_change) 221 | d.set('foo1', 'bar1') 222 | d.set('foo2', 'bar2') 223 | d.set('foo3', 'bar3') 224 | #create a second client 225 | 226 | client2 = doozer.connect() 227 | d2 = DoozerData(client2, callback=print_change) 228 | d2.set('foo4', 'bar4') 229 | #let the second client change values to 230 | #those should be printed. 231 | cv = gevent.spawn(change_value, d2) 232 | 233 | for path, rev, value in d.items(): 234 | print path,'->', value 235 | 236 | print d.get('foo1') 237 | print d.get('foo2') 238 | 239 | d.delete_all() 240 | 241 | #should be empty. 242 | for di in d.items(): 243 | print di 244 | 245 | #the change value function added content over time.. 246 | gevent.sleep(3) 247 | print 'data in d1' 248 | for di in d.items(): 249 | print di 250 | print 'data in d2' 251 | for dii in d2.items(): 252 | print dii 253 | # there is content. in both instances. 254 | # because the change_value job adds data later. 255 | cv.join(cv) 256 | #d.delete_all() 257 | 258 | if __name__ == '__main__': 259 | test_doozerdata() 260 | 261 | -------------------------------------------------------------------------------- /examples/watch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import os 3 | import sys 4 | sys.path.append(os.path.dirname(__file__) + "/..") 5 | 6 | import gevent 7 | import doozer 8 | 9 | from gevent import Timeout 10 | 11 | client = doozer.connect() 12 | rev = client.rev().rev 13 | 14 | def watch_test(rev): 15 | while True: 16 | try: 17 | change = client.wait("/watch", rev) 18 | print change.rev, change.value 19 | rev = change.rev+1 20 | except Timeout, t: 21 | print t 22 | rev = client.rev().rev 23 | change = None 24 | 25 | watch_job = gevent.spawn(watch_test, rev+1) 26 | 27 | for i in range(10): 28 | gevent.sleep(1) 29 | rev = client.set("/watch", "test4%d" % i, rev).rev 30 | print rev 31 | 32 | 33 | foo = client.get("/watch") 34 | print "Got /watch with %s" % foo.value 35 | 36 | gevent.sleep(2) 37 | client.delete("/watch", rev) 38 | print "Deleted /watch" 39 | 40 | foo = client.get("/watch") 41 | print foo 42 | 43 | client.disconnect() 44 | watch_job.kill() 45 | -------------------------------------------------------------------------------- /examples/watch_glob.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import os 3 | import sys 4 | sys.path.append(os.path.dirname(__file__) + "/..") 5 | 6 | import gevent 7 | import doozer 8 | import simplejson 9 | 10 | from gevent import Timeout 11 | 12 | client = doozer.connect() 13 | 14 | #clean out the foo dir. 15 | walk = client.walk("/foo/**") 16 | for node in walk: 17 | client.delete(node.path, node.rev) 18 | 19 | rev = client.set("/foo/bar", "test", 0).rev 20 | 21 | def watch_test(rev): 22 | while True: 23 | try: 24 | change = client.wait("/foo/**", rev ) 25 | print "saw change at %s with %s" % ( change.rev, change.value) 26 | rev = change.rev+1 27 | except Timeout, t: 28 | change = None 29 | print t 30 | rev = client.rev().rev 31 | #rev =+1 32 | 33 | #spawn the process that watches the foo dir for changes. 34 | watch_job = gevent.spawn(watch_test, rev+1) 35 | 36 | #add new data in foo 37 | for i in range(10): 38 | gevent.sleep(0.5) 39 | revk = client.set("/foo/bar%d" % i, simplejson.dumps({'data': i}), 0).rev 40 | 41 | foo = client.getdir("/foo") 42 | 43 | print "Directly under /foo is " 44 | for f in foo: 45 | print f.path, f.rev, 46 | print client.get("/foo/"+f.path).value 47 | 48 | dir(f) 49 | 50 | client.disconnect() 51 | watch_job.kill() 52 | 53 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from setuptools import setup 4 | 5 | setup( 6 | name='PyDoozer', 7 | version='0.2.2', 8 | author='Jeff Lindsay', 9 | author_email='jeff.lindsay@twilio.com', 10 | description='doozer client', 11 | packages=['doozer'], 12 | install_requires=['gevent', 'protobuf'], 13 | data_files=[], 14 | ) 15 | --------------------------------------------------------------------------------