├── .gitignore ├── AUTHORS ├── Changelog ├── FAQ ├── INSTALL ├── LICENSE ├── MANIFEST.in ├── README ├── README.rst ├── THANKS ├── TODO ├── carrot ├── __init__.py ├── backends │ ├── __init__.py │ ├── base.py │ ├── librabbitmq.py │ ├── pikachu.py │ ├── pyamqplib.py │ ├── pystomp.py │ └── queue.py ├── connection.py ├── messaging.py ├── serialization.py └── utils.py ├── contrib ├── doc2ghpages └── requirements │ ├── default.txt │ └── test.txt ├── docs ├── .static │ └── .keep ├── Makefile ├── _ext │ ├── applyxrefs.py │ └── literals_to_xrefs.py ├── _theme │ ├── agogo │ │ ├── layout.html │ │ ├── static │ │ │ ├── agogo.css_t │ │ │ ├── bgfooter.png │ │ │ └── bgtop.png │ │ └── theme.conf │ └── nature │ │ ├── static │ │ ├── nature.css_t │ │ └── pygments.css │ │ └── theme.conf ├── changelog.rst ├── conf.py ├── faq.rst ├── index.rst ├── introduction.rst └── reference │ ├── carrot.backends.base.rst │ ├── carrot.backends.librabbitmq.rst │ ├── carrot.backends.pikachu.rst │ ├── carrot.backends.pyamqplib.rst │ ├── carrot.backends.pystomp.rst │ ├── carrot.backends.queue.rst │ ├── carrot.backends.rst │ ├── carrot.connection.rst │ ├── carrot.messaging.rst │ ├── carrot.serialization.rst │ ├── carrot.utils.rst │ └── index.rst ├── setup.cfg ├── setup.py ├── tests ├── __init__.py ├── backend.py ├── test_django.py ├── test_examples.py ├── test_pyamqplib.py ├── test_pyqueue.py ├── test_serialization.py ├── test_utils.py ├── test_with_statement.py ├── utils.py └── xxxstmop.py └── tox.ini /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.py[co] 3 | *$py.class 4 | erl_crash.dump 5 | .ropeproject/ 6 | .project 7 | .pydevproject 8 | *~ 9 | *.sqlite 10 | *.sqlite-journal 11 | settings_local.py 12 | .build 13 | build 14 | .*.sw[po] 15 | dist/ 16 | *.egg-info 17 | doc/__build/* 18 | pip-log.txt 19 | devdatabase.db 20 | parts 21 | eggs 22 | bin 23 | developer-eggs 24 | downloads 25 | Documentation/* 26 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | ================================== 2 | AUTHORS (in chronological order) 3 | ================================== 4 | 5 | Ask Solem 6 | Travis Cline 7 | Rune Halvorsen 8 | Sean Creeley 9 | Jason Cater 10 | Ian Struble 11 | Patrick Schneider 12 | Travis Swicegood 13 | Stephen Day 14 | Andrew Watts 15 | Paul McLanahan 16 | Ralf Nyren 17 | Jeff Balogh 18 | Adam Wentz 19 | Vincent Driessen 20 | Matt Swanson 21 | Christian Legnitto 22 | -------------------------------------------------------------------------------- /Changelog: -------------------------------------------------------------------------------- 1 | ================ 2 | Change history 3 | ================ 4 | 5 | 0.10.7 [2010-10-07 03:20 P.M CEST] 6 | ---------------------------------- 7 | 8 | * ``ConsumerSet``: Now possible to add/cancel consumers at runtime 9 | 10 | * To add another consumer to an active set do:: 11 | 12 | >>> cset.add_consumer(C) 13 | >>> # or 14 | >>> cset.add_consumer_from_dict(**declaration) 15 | 16 | >>> # ... 17 | >>> # consume() will declare new consumers 18 | >>> cset.consume() 19 | 20 | * To cancel an active consumer by queue name do:: 21 | 22 | >>> cset.cancel_by_queue(queue_name) 23 | 24 | 0.10.6 [2010-09-03 10:16 A.M CEST] 25 | ---------------------------------- 26 | 27 | * ``Publisher.send``: Now supports an exchange argument used to override the 28 | exchange to send the message to (defaults to ``Publisher.exchange``). 29 | 30 | Note that this exchange must have been declared. 31 | 32 | * STOMP backend: Now supports username and password authentication. 33 | 34 | * pika backend called basic_get with incorrect arguments. 35 | 36 | 0.10.5 [2010-06-03 09:02 A.M CEST] 37 | ---------------------------------- 38 | 39 | * In-memory backend: discard_all() now works correctly. 40 | 41 | * Added msgpack serialization support 42 | 43 | See http://msgpack.sourceforge.net for more information. 44 | 45 | To enable set:: 46 | 47 | serializer="msgpack" 48 | 49 | * Added dependency specification for building RPMs. 50 | 51 | $ python setup.py bdist_rpm 52 | 53 | 0.10.4 [2010-05-14 10:26 A.M CEST] 54 | ---------------------------------- 55 | 56 | * Added ``BrokerConnection.drain_events()`` (only works for amqplib/pika) 57 | 58 | `drain_events` waits for events on all active channels. 59 | 60 | * amqplib: Added timeout support to ``drain_events``. 61 | 62 | Example usage: 63 | 64 | >>> c = Consumer() 65 | >>> it = c.iterconsume() 66 | >>> # wait for event on any channel 67 | >>> try: 68 | ... connection.drain_events(timeout=1) 69 | ... except socket.timeout: 70 | pass 71 | 72 | * Added Consumer.consume / ConsumerSet.consume 73 | 74 | We're slowly moving away from ``iterconsume`` as this flow doesn't 75 | make sense. You often want to consume from several channels at once, 76 | so ``iterconsume`` will probably be deprecated at some point. 77 | 78 | "new-style" consume example:: 79 | 80 | >>> connection = BrokerConnection(..) 81 | >>> consumer = Consumer(connection, ...) 82 | >>> consumer.register_callback(handle_message) 83 | >>> consumer.consume() # declares consumers 84 | >>> while True: 85 | ... connection.drain_events() 86 | >>> consumer.cancel() # Cancel consumer. 87 | 88 | More elaborate example consuming from two channels, 89 | where the first channel consumes from multiple queues:: 90 | 91 | >>> connection = BrokerConnection(..) 92 | 93 | # The first channel receives jobs from several 94 | # queues. 95 | >>> queues = {"video": {"exchange": "jobs", 96 | ... "queue": "video", 97 | ... "routing_key": "video"}, 98 | ... "image": {"exchange": "jobs", 99 | ... "queue": "image", 100 | ... "routing_key": "image"}} 101 | >>> job_consumer = ConsumerSet(connection, from_dict=queues) 102 | >>> job_consumer.register_callback(handle_job) 103 | >>> job_consumer.consume() 104 | 105 | # The second channel receives remote control commands. 106 | >>> remote_consumer = Consumer(connection, queue="control", 107 | ... exchange="control") 108 | >>> remote_consumer.register_callback(handle_remote_command) 109 | >>> remote_consumer.consume() 110 | 111 | # The eventloop. 112 | # Receives a single message a pass and calls the appropriate 113 | # callback. 114 | >>> try: 115 | ... while True: 116 | ... connection.drain_events() 117 | ... finally: 118 | ... job_consumer.close() 119 | ... remote_consumer.close() 120 | ... connection.close() 121 | 122 | * amqplib: now raises ``KeyError`` if hostname isn't set. 123 | 124 | 0.10.3 [2010-03-08 05:01 P.M CEST] 125 | ---------------------------------- 126 | 127 | * Consumer/Publisher: A kwarg option set to ``None`` should always 128 | mean "use the class default value". This was not always the case, 129 | but has been fixed in this release. 130 | 131 | * DjangoBrokerConnection: Now accepts a custom ``settings`` object. E.g.: 132 | 133 | >>> conn = DjangoBrokerConnection(settings=mysettings) 134 | 135 | * Consumer.iterconsume: Now raises :exc:`StopIteration` if the channel is 136 | closed. (http://github.com/ask/carrot/issues/issue/24) 137 | 138 | * Fixed syntax error in the DjangoBrokerConnection which could be triggered 139 | if some conditions were met. 140 | 141 | * setup.cfg: Disable --enable-coverage from nosetests section 142 | 143 | * Consumer.iterconsume now works properly when using the Python Queue 144 | module based backend (http://github.com/ask/carrot/issues/issue/23). 145 | 146 | 147 | 0.10.2 [2010-02-03 11:43 A.M CEST] 148 | ---------------------------------- 149 | * Resolved a typo in the experimental Pika backend. 150 | 151 | 0.10.1 [2010-01-16 10:17 P.M CEST] 152 | ---------------------------------- 153 | 154 | * Fixed syntax error typo in the Pika backend. 155 | 156 | 0.10.0 [2010-01-15 12:08 A.M CEST] 157 | ---------------------------------- 158 | 159 | * Added experimental Pika backend for async support. See 160 | http://github.com/tonyg/pika 161 | 162 | * Python 2.4 compatibility. 163 | 164 | * Added intent revealing names for use with the delivery_mode attribute 165 | 166 | * The amqplib internal connection now supports waiting for events on any 167 | channel, so as to not block the event loop on a single channel. Example: 168 | 169 | >>> from carrot.connection import BrokerConnection 170 | >>> from carrot.messaging import Consumer, Publisher 171 | >>> from functools import partial 172 | >>> connection = BrokerConnection(...) 173 | >>> consumer1 = Consumer(queue="foo", exchange="foo") 174 | >>> consumer2 = Consumer(queue="bar", exchange="bar") 175 | >>> def callback(channel, message_data, message): 176 | ... print(%s: %s" % (channel, message_data)) 177 | >>> consumer1.register_callback(partial(callback, "channel1")) 178 | >>> consumer2.register_callback(partial(callback, "channel2")) 179 | 180 | >>> pub1 = Publisher(exchange="foo") 181 | >>> pub2 = Publisher(exchange="bar") 182 | >>> [(i % 2 and pub1 or pub2).send({"hello": "world"}) 183 | ... for i in range(100)] 184 | 185 | >>> while True: 186 | ... connection.connection.drain_events() 187 | 188 | But please be sure to note that this is an internal feature only, 189 | hopefully we will have a public interface for this for v1.0. 190 | 191 | 0.8.0 [2009-11-16 05:11 P.M CEST] 192 | --------------------------------- 193 | 194 | **BACKWARD INCOMPATIBLE CHANGES** 195 | 196 | * Django: ``AMQP_SERVER`` has been deprecated and renamed to ``BROKER_HOST``. 197 | ``AMQP_SERVER`` is still supported but will be removed in version 1.0. 198 | 199 | * Django: All ``AMQP_*`` settings has been renamed to ``BROKER_*``, 200 | the old settings still work but gives a deprecation warning. 201 | ``AMQP_*`` is scheduled for removal in v1.0. 202 | 203 | * You now have to include the class name in the 204 | CARROT_BACKEND string. E.g. where you before had "my.module.backends", you now 205 | need to include the class name: ``"my.module.backends.BackendClass"``. 206 | The aliases works like before. 207 | 208 | *BUGFIXES* 209 | 210 | * Cancel should delete the affected _consumer 211 | 212 | * qos()/flow() now working properly. 213 | 214 | * Fixed the bug in UUID4 which makes it return the same id over and over. 215 | 216 | *OTHER* 217 | 218 | * Sphinx docs: Remove django dependency when building docs. Thanks jetfar! 219 | 220 | * You can now build the documentatin using ``python setup.py build_sphinx`` 221 | (thanks jetfar!) 222 | 223 | 224 | 225 | 0.6.1 [2009-09-30 12:29 P.M CET] 226 | ------------------------------------------------------------------ 227 | 228 | * Forgot to implement qos/flow in the pyamqplib backend (doh). 229 | Big thanks to stevvooe! See issue `#18`_ 230 | 231 | * Renamed ConsumerSet._open_channels -> _open_consumers 232 | 233 | * Channel is now a weak reference so it's collected on channel exception. 234 | See issue `#17`_. Thanks to ltucker. 235 | 236 | * Publisher.auto_declare/Consumer.auto_declare: Can now disable the default 237 | behaviour of automatically declaring the queue, exchange and queue binding. 238 | 239 | * Need to close the connection to receive mandatory/immediate errors 240 | (related to issue `#16`_). Thanks to ltucker. 241 | 242 | * pyamqplib backend close didn't work properly, typo channel -> _channel 243 | 244 | * Adds carrot.backends.pystomp to the reference documentation. 245 | 246 | .. _`#18`: http://github.com/ask/carrot/issues#issue/18 247 | .. _`#17`: http://github.com/ask/carrot/issues#issue/17 248 | .. _`#16`: http://github.com/ask/carrot/issues#issue/16 249 | 250 | 251 | 0.6.0 [2009-09-17 16:41 P.M CET] 252 | ------------------------------------------------------------------ 253 | 254 | **BACKWARD INCOMPATIBLE CHANGES** 255 | 256 | * AMQPConnection renamed to BrokerConnection with AMQPConnection remaining 257 | an alias for backwards compatability. Similarly DjangoAMQPConnection is 258 | renamed to DjangoBrokerConnection. 259 | 260 | * AMQPConnection renamed to BrokerConnection 261 | DjangoAMQPConnection renamed to DjangoBrokerConnection 262 | (The previous names are still available but will be deprecated in 1.0) 263 | 264 | * The connection is now lazy, requested only when it's needed. 265 | To explicitly connect you have to evaluate the BrokerConnections 266 | ``connection`` attribute. 267 | 268 | >>> connection = BrokerConnection(...) # Not connected yet. 269 | >>> connection.connection; # Now it's connected 270 | 271 | * A channel is now lazy, requested only when it's needed. 272 | 273 | * pyamqplib.Message.amqp_message is now a private attribute 274 | 275 | **NEW FEATURES** 276 | 277 | * Basic support for STOMP using the stompy library. 278 | (Available at http://bitbucket.org/benjaminws/python-stomp/overview/) 279 | 280 | * Implements :meth:`Consumer.qos` and :meth:`Consumer.flow` for setting 281 | quality of service and flow control. 282 | 283 | **NEWS** 284 | 285 | * The current Message class is now available as an attribute on the 286 | Backend. 287 | 288 | * Default port is now handled by the backend and all AMQP_* settings to the 289 | DjangoBrokerConnection is now optional 290 | 291 | * Backend is now initialized in the connection instead of Consumer/Publisher, 292 | so backend_cls now has to be sent to AMQPConnection if you want to 293 | explicitly set it. 294 | 295 | * Specifying utf-8 as the content type when forcing unicode into a string. 296 | This removes the reference to the unbound content_type variable. 297 | 298 | 299 | 0.5.1 [2009-07-19 06:19 P.M CET] 300 | ------------------------------------------------------------------ 301 | 302 | * Handle messages without content_encoding attribute set. 303 | 304 | * Make delivery_info available on the Message instance. 305 | 306 | * Use anyjson to detect best installed JSON module on the system. 307 | 308 | 0.5.0 [2009-06-25 08:16 P.M CET] 309 | ------------------------------------------------------------------ 310 | 311 | **BACKWARD-INCOMPATIBLE CHANGES** 312 | 313 | * Custom encoder/decoder support has been moved to a centralized 314 | registry in ``carrot.serialization``. This means the 315 | ``encoder``/``decoder`` optional arguments to ``Publisher.send`` 316 | `(and the similar attributes of ``Publisher`` and ``Consumer`` 317 | classes) no longer exist. See ``README`` for details of the new 318 | system. 319 | 320 | * Any ``Consumer`` and ``Publisher`` instances should be 321 | upgraded at the same time since carrot now uses the AMQP 322 | ``content-type`` field to know how to automatically de-serialize 323 | your data. If you use an old-style ``Publisher`` with a new-style 324 | ``Consumer``, you will get a raw string back as ``message_data`` 325 | instead of your de-serialized data. An old-style ``Consumer`` 326 | will work with a new-style ``Publisher`` as long as you're using 327 | the default ``JSON`` serialization methods. 328 | 329 | * Acknowledging/Rejecting/Requeuing a message twice now raises 330 | an exception. 331 | 332 | *ENHANCEMENTS* 333 | 334 | * ``ConsumerSet``: Receive messages from several consumers. 335 | 336 | 337 | 0.4.5 [2009-06-15 01:58 P.M CET] 338 | ------------------------------------------------------------------ 339 | 340 | **BACKWARD-INCOMPATIBLE CHANGES** 341 | 342 | * the exchange is now also declared in the ``Publisher``. This means 343 | the following attributes (if used) must be set on *both* 344 | the ``Publisher`` and the ``Consumer``: 345 | ``exchange_type``, ``durable`` and ``auto_delete``. 346 | 347 | **IMPORTANT BUGS** 348 | 349 | * No_ack was always set to ``True`` when using ``Consumer.iterconsume``. 350 | 351 | 352 | 0.4.4 [2009-06-15 01:58 P.M CET] 353 | ------------------------------------------------------------------ 354 | 355 | * __init__.pyc was included in the distribution by error. 356 | 357 | 0.4.3 [2009-06-13 09:26 P.M CET] 358 | ------------------------------------------------------------------ 359 | 360 | * Fix typo with long_description in ``setup.py``. 361 | 362 | 0.4.2 [2009-06-13 08:30 P.M CET] 363 | ------------------------------------------------------------------ 364 | 365 | * Make sure README exists before reading it for ``long_description``. 366 | Thanks to jcater. 367 | 368 | * ``discard_all``: Use ``AMQP.queue_purge`` if ``filterfunc`` is not 369 | specified 370 | 371 | 0.4.1 [2009-06-08 04:21 P.M CET] 372 | ------------------------------------------------------------------ 373 | 374 | * Consumer/Publisher now correctly sets the encoder/decoder if they 375 | have been overriden by setting the class attribute. 376 | 377 | 0.4.0 [2009-06-06 01:39 P.M CET] 378 | ------------------------------------------------------------------ 379 | 380 | **IMPORTANT** Please don't use ``Consumer.wait`` in production. Use either 381 | of ``Consumer.iterconsume`` or ``Consumer.iterqueue``. 382 | 383 | **IMPORTANT** The ``ack`` argument to ``Consumer.process_next`` has been 384 | removed, use the instance-wide ``auto_ack`` argument/attribute instead. 385 | 386 | **IMPORTANT** ``Consumer.message_to_python`` has been removed, use 387 | ``message.decode()`` on the returned message object instead. 388 | 389 | **IMPORTANT** Consumer/Publisher/Messaging now no longer takes a backend 390 | instance, but a backend class, so the ``backend`` argument is renamed to 391 | ``backend_cls``. 392 | 393 | *WARNING* ``Consumer.process_next`` has been deprecated in favor of 394 | ``Consumer.fetch(enable_callbacks=True)`` and emits a ``DeprecationWarning`` 395 | if used. 396 | 397 | * ``Consumer.iterconsume``: New sane way to use basic_consume instead of ``Consumer.wait``: 398 | (Consmer.wait now uses this behind the scenes, just wrapped around 399 | a highly unsafe infinite loop.) 400 | 401 | * Consumer: New options: ``auto_ack`` and ``no_ack``. Auto ack means the 402 | consumer will automatically acknowledge new messages, and No-Ack 403 | means it will disable acknowledgement on the server alltogether 404 | (probably not what you want) 405 | 406 | * ``Consumer.iterqueue``: Now supports infinite argument, which means the 407 | iterator doesn't raise ``StopIteration`` if there's no messages, 408 | but instead yields ``None`` (thus never ends) 409 | 410 | * message argument to consumer callbacks is now a 411 | ``carrot.backends.pyamqplib.Message`` instance. See `[GH #4]`_. 412 | Thanks gregoirecachet! 413 | 414 | .. _`[GH #4]`: http://github.com/ask/carrot/issues/closed/#issue/4 415 | 416 | * AMQPConnection, Consumer, Publisher and Messaging now supports 417 | the with statement. They automatically close when the with-statement 418 | block exists. 419 | 420 | * Consumer tags are now automatically generated for each class instance, 421 | so you should never get the "consumer tag already open" error anymore. 422 | 423 | * Loads of new unit tests. 424 | 425 | 0.4.0-pre7 [2009-06-03 05:08 P.M CET] 426 | ------------------------------------------------------------------ 427 | 428 | * Conform to pep8.py trying to raise our pypants score. 429 | 430 | * Documentation cleanup (thanks Rune Halvorsen) 431 | 432 | 0.4.0-pre6 [2009-06-03 04:55 P.M CET] 433 | ------------------------------------------------------------------ 434 | 435 | * exclusive implies auto_delete, not durable. Closes #2. 436 | Thanks gregoirecachet 437 | 438 | * Consumer tags are now automatically generated by the class module, 439 | name and a UUID. 440 | 441 | * New attribute ``Consumer.warn_if_exists:`` 442 | If True, emits a warning if the queue has already been declared. 443 | If a queue already exists, and you try to redeclare the queue 444 | with new settings, the new settings will be silently ignored, 445 | so this can be useful if you've recently changed the 446 | `routing_key` attribute or other settings. 447 | 448 | 0.4.0-pre3 [2009-05-29 02:27 P.M CET] 449 | ------------------------------------------------------------------ 450 | 451 | * Publisher.send: Now supports message priorities (a number between ``0`` 452 | and ``9``) 453 | 454 | * Publihser.send: Added ``routing_key`` keyword argument. Can override 455 | the routing key for a single message. 456 | 457 | * Publisher.send: Support for the ``immediate`` and ``mandatory`` flags. 458 | 459 | 0.4.0-pre2 [2009-05-29 02:27 P.M CET] 460 | ------------------------------------------------------------------ 461 | 462 | * AMQPConnection: Added ``connect_timeout`` timeout option, which is 463 | the timeout in seconds before we exhaust trying to establish a 464 | connection to the AMQP server. 465 | 466 | 0.4.0-pre1 [2009-05-27 04:27 P.M CET] 467 | ------------------------------------------------------------------ 468 | 469 | * This version introduces backends. The consumers and publishers 470 | all have an associated backend. Currently there are two backends 471 | available; ``pyamqlib`` and ``pyqueue``. The ``amqplib`` backend 472 | is for production use, while the ``Queue`` backend is for use while 473 | unit testing. 474 | 475 | * Consumer/Publisher operations no longer re-establishes the connection 476 | if the connection has been closed. 477 | 478 | * ``Consumer.next`` has been deprecated for a while, and has now been 479 | removed. 480 | 481 | * Message objects now have a ``decode`` method, to deserialize the 482 | message body. 483 | 484 | * You can now use the Consumer class standalone, without subclassing, 485 | by registering callbacks by using ``Consumer.register_callback``. 486 | 487 | * Ability to filter messages in ``Consumer.discard_all``. 488 | 489 | * carrot now includes a basic unit test suite, which hopefully will 490 | be more complete in later releases. 491 | 492 | * carrot now uses the Sphinx documentation system. 493 | 494 | 0.3.9 [2009-05-18 04:49 P.M CET] 495 | -------------------------------------------------------------- 496 | 497 | * Consumer.wait() now works properly again. Thanks Alexander Solovyov! 498 | 499 | 0.3.8 [2009-05-11 02:14 P.M CET] 500 | -------------------------------------------------------------- 501 | 502 | * Rearranged json module import order. 503 | New order is cjson > simplejson > json > django.util.somplejson 504 | 505 | * _Backwards incompatible change: 506 | Have to instantiate AMQPConnection object before passing 507 | it to consumers/publishers. e.g before when you did 508 | 509 | >>> consumer = Consumer(connection=DjangoAMQPConnection) 510 | 511 | you now have to do 512 | 513 | >>> consumer = Consumer(connection=DjangoAMQPConnection()) 514 | 515 | or sometimes you might even want to share the same connection with 516 | publisher/consumer. 517 | 518 | 519 | 0.2.1 [2009-03-24 05:48 P.M CET] 520 | -------------------------------------------------------------- 521 | 522 | * Fix typo "package" -> "packages" in setup.py 523 | 524 | 0.2.0 [2009-03-24 05:23 P.M ]` 525 | -------------------------------------------------------------- 526 | 527 | * hasty bugfix release, fixed syntax errors. 528 | 529 | 0.1.0 [2009-03-24 05:16 P.M ]` 530 | -------------------------------------------------------------- 531 | 532 | * Initial release 533 | -------------------------------------------------------------------------------- /FAQ: -------------------------------------------------------------------------------- 1 | ============================ 2 | Frequently Asked Questions 3 | ============================ 4 | 5 | Questions 6 | ========= 7 | 8 | Q: Message.reject doesn't work? 9 | -------------------------------------- 10 | **Answer**: RabbitMQ (as of v1.5.5) has not implemented reject yet. 11 | There was a brief discussion about it on their mailing list, and the reason 12 | why it's not implemented yet is revealed: 13 | 14 | http://lists.rabbitmq.com/pipermail/rabbitmq-discuss/2009-January/003183.html 15 | 16 | Q: Message.requeue doesn't work? 17 | -------------------------------------- 18 | 19 | **Answer**: See _`Message.reject doesn't work?` 20 | -------------------------------------------------------------------------------- /INSTALL: -------------------------------------------------------------------------------- 1 | Installation 2 | ============ 3 | 4 | You can install ``carrot`` either via the Python Package Index (PyPI) 5 | or from source. 6 | 7 | To install using ``pip``,:: 8 | 9 | $ pip install carrot 10 | 11 | 12 | To install using ``easy_install``,:: 13 | 14 | $ easy_install carrot 15 | 16 | 17 | If you have downloaded a source tarball you can install it 18 | by doing the following,:: 19 | 20 | $ python setup.py build 21 | # python setup.py install # as root 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2009, Ask Solem 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | 13 | Neither the name of Ask Solem nor the names of its contributors may be used 14 | to endorse or promote products derived from this software without specific 15 | prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 19 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 | PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS 21 | BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | POSSIBILITY OF SUCH DAMAGE. 28 | 29 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include AUTHORS 2 | include Changelog 3 | include FAQ 4 | include INSTALL 5 | include LICENSE 6 | include MANIFEST.in 7 | include README.rst 8 | include README 9 | include THANKS 10 | include TODO 11 | recursive-include docs * 12 | recursive-include carrot *.py 13 | recursive-include tests * 14 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | README.rst -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ############################################## 2 | carrot - AMQP Messaging Framework for Python 3 | ############################################## 4 | 5 | :Version: 0.10.7 6 | 7 | Status 8 | ------ 9 | 10 | Carrot is discontinued in favor of the new `Kombu`_ framework. 11 | 12 | * Kombu is ready, start to use it now! 13 | * Kombu comes with a Carrot compatible API, so it's easy to port your software. 14 | * Carrot will not be actively maintained, only critical bugs will be fixed. 15 | 16 | Kombu links: 17 | 18 | * Download: http://pypi.python.org/pypi/kombu 19 | * Documentation: http://packages.python.org/kombu 20 | * Development: http://github.com/ask/kombu 21 | 22 | .. _`Kombu`: http://pypi.python.org/pypi/kombu 23 | 24 | ** ORIGINAL CARROT README CONTINUES BELOW ** 25 | 26 | Introduction 27 | ------------ 28 | 29 | `carrot` is an `AMQP`_ messaging queue framework. AMQP is the Advanced Message 30 | Queuing Protocol, an open standard protocol for message orientation, queuing, 31 | routing, reliability and security. 32 | 33 | The aim of `carrot` is to make messaging in Python as easy as possible by 34 | providing a high-level interface for producing and consuming messages. At the 35 | same time it is a goal to re-use what is already available as much as possible. 36 | 37 | `carrot` has pluggable messaging back-ends, so it is possible to support 38 | several messaging systems. Currently, there is support for `AMQP`_ 39 | (`py-amqplib`_, `pika`_), `STOMP`_ (`python-stomp`_). There's also an 40 | in-memory backend for testing purposes, using the `Python queue module`_. 41 | 42 | Several AMQP message broker implementations exists, including `RabbitMQ`_, 43 | `ZeroMQ`_ and `Apache ActiveMQ`_. You'll need to have one of these installed, 44 | personally we've been using `RabbitMQ`_. 45 | 46 | Before you start playing with ``carrot``, you should probably read up on 47 | AMQP, and you could start with the excellent article about using RabbitMQ 48 | under Python, `Rabbits and warrens`_. For more detailed information, you can 49 | refer to the `Wikipedia article about AMQP`_. 50 | 51 | .. _`RabbitMQ`: http://www.rabbitmq.com/ 52 | .. _`ZeroMQ`: http://www.zeromq.org/ 53 | .. _`AMQP`: http://amqp.org 54 | .. _`STOMP`: http://stomp.codehaus.org 55 | .. _`python-stomp`: http://bitbucket.org/asksol/python-stomp 56 | .. _`Python Queue module`: http://docs.python.org/library/queue.html 57 | .. _`Apache ActiveMQ`: http://activemq.apache.org/ 58 | .. _`Django`: http://www.djangoproject.com/ 59 | .. _`Rabbits and warrens`: http://blogs.digitar.com/jjww/2009/01/rabbits-and-warrens/ 60 | .. _`py-amqplib`: http://barryp.org/software/py-amqplib/ 61 | .. _`pika`: http://github.com/tonyg/pika 62 | .. _`Wikipedia article about AMQP`: http://en.wikipedia.org/wiki/AMQP 63 | 64 | Documentation 65 | ------------- 66 | 67 | Carrot is using Sphinx, and the latest documentation is available at GitHub: 68 | 69 | http://github.com/ask/carrot/ 70 | 71 | Installation 72 | ============ 73 | 74 | You can install ``carrot`` either via the Python Package Index (PyPI) 75 | or from source. 76 | 77 | To install using ``pip``,:: 78 | 79 | $ pip install carrot 80 | 81 | 82 | To install using ``easy_install``,:: 83 | 84 | $ easy_install carrot 85 | 86 | 87 | If you have downloaded a source tarball you can install it 88 | by doing the following,:: 89 | 90 | $ python setup.py build 91 | # python setup.py install # as root 92 | 93 | 94 | Terminology 95 | =========== 96 | 97 | There are some concepts you should be familiar with before starting: 98 | 99 | * Publishers 100 | 101 | Publishers sends messages to an exchange. 102 | 103 | * Exchanges 104 | 105 | Messages are sent to exchanges. Exchanges are named and can be 106 | configured to use one of several routing algorithms. The exchange 107 | routes the messages to consumers by matching the routing key in the 108 | message with the routing key the consumer provides when binding to 109 | the exchange. 110 | 111 | * Consumers 112 | 113 | Consumers declares a queue, binds it to a exchange and receives 114 | messages from it. 115 | 116 | * Queues 117 | 118 | Queues receive messages sent to exchanges. The queues are declared 119 | by consumers. 120 | 121 | * Routing keys 122 | 123 | Every message has a routing key. The interpretation of the routing 124 | key depends on the exchange type. There are four default exchange 125 | types defined by the AMQP standard, and vendors can define custom 126 | types (so see your vendors manual for details). 127 | 128 | These are the default exchange types defined by AMQP/0.8: 129 | 130 | * Direct exchange 131 | 132 | Matches if the routing key property of the message and 133 | the ``routing_key`` attribute of the consumer are identical. 134 | 135 | * Fan-out exchange 136 | 137 | Always matches, even if the binding does not have a routing 138 | key. 139 | 140 | * Topic exchange 141 | 142 | Matches the routing key property of the message by a primitive 143 | pattern matching scheme. The message routing key then consists 144 | of words separated by dots (``"."``, like domain names), and 145 | two special characters are available; star (``"*"``) and hash 146 | (``"#"``). The star matches any word, and the hash matches 147 | zero or more words. For example ``"*.stock.#"`` matches the 148 | routing keys ``"usd.stock"`` and ``"eur.stock.db"`` but not 149 | ``"stock.nasdaq"``. 150 | 151 | 152 | Examples 153 | ======== 154 | 155 | Creating a connection 156 | --------------------- 157 | 158 | You can set up a connection by creating an instance of 159 | ``carrot.messaging.BrokerConnection``, with the appropriate options for 160 | your broker: 161 | 162 | >>> from carrot.connection import BrokerConnection 163 | >>> conn = BrokerConnection(hostname="localhost", port=5672, 164 | ... userid="guest", password="guest", 165 | ... virtual_host="/") 166 | 167 | 168 | If you're using Django you can use the 169 | ``carrot.connection.DjangoBrokerConnection`` class instead, which loads 170 | the connection settings from your ``settings.py``:: 171 | 172 | BROKER_HOST = "localhost" 173 | BROKER_PORT = 5672 174 | BROKER_USER = "guest" 175 | BROKER_PASSWORD = "guest" 176 | BROKER_VHOST = "/" 177 | 178 | Then create a connection by doing: 179 | 180 | >>> from carrot.connection import DjangoBrokerConnection 181 | >>> conn = DjangoBrokerConnection() 182 | 183 | 184 | 185 | Receiving messages using a Consumer 186 | ----------------------------------- 187 | 188 | First we open up a Python shell and start a message consumer. 189 | 190 | This consumer declares a queue named ``"feed"``, receiving messages with 191 | the routing key ``"importer"`` from the ``"feed"`` exchange. 192 | 193 | The example then uses the consumers ``wait()`` method to go into consume 194 | mode, where it continuously polls the queue for new messages, and when a 195 | message is received it passes the message to all registered callbacks. 196 | 197 | >>> from carrot.messaging import Consumer 198 | >>> consumer = Consumer(connection=conn, queue="feed", 199 | ... exchange="feed", routing_key="importer") 200 | >>> def import_feed_callback(message_data, message): 201 | ... feed_url = message_data["import_feed"] 202 | ... print("Got feed import message for: %s" % feed_url) 203 | ... # something importing this feed url 204 | ... # import_feed(feed_url) 205 | ... message.ack() 206 | >>> consumer.register_callback(import_feed_callback) 207 | >>> consumer.wait() # Go into the consumer loop. 208 | 209 | Sending messages using a Publisher 210 | ---------------------------------- 211 | 212 | Then we open up another Python shell to send some messages to the consumer 213 | defined in the last section. 214 | 215 | >>> from carrot.messaging import Publisher 216 | >>> publisher = Publisher(connection=conn, 217 | ... exchange="feed", routing_key="importer") 218 | >>> publisher.send({"import_feed": "http://cnn.com/rss/edition.rss"}) 219 | >>> publisher.close() 220 | 221 | 222 | Look in the first Python shell again (where ``consumer.wait()`` is running), 223 | where the following text has been printed to the screen:: 224 | 225 | Got feed import message for: http://cnn.com/rss/edition.rss 226 | 227 | 228 | Serialization of Data 229 | ----------------------- 230 | 231 | By default every message is encoded using `JSON`_, so sending 232 | Python data structures like dictionaries and lists works. 233 | `YAML`_, `msgpack`_ and Python's built-in ``pickle`` module is also supported, 234 | and if needed you can register any custom serialization scheme you 235 | want to use. 236 | 237 | .. _`JSON`: http://www.json.org/ 238 | .. _`YAML`: http://yaml.org/ 239 | .. _`msgpack`: http://msgpack.sourceforge.net/ 240 | 241 | Each option has its advantages and disadvantages. 242 | 243 | ``json`` -- JSON is supported in many programming languages, is now 244 | a standard part of Python (since 2.6), and is fairly fast to 245 | decode using the modern Python libraries such as ``cjson or 246 | ``simplejson``. 247 | 248 | The primary disadvantage to ``JSON`` is that it limits you to 249 | the following data types: strings, unicode, floats, boolean, 250 | dictionaries, and lists. Decimals and dates are notably missing. 251 | 252 | Also, binary data will be transferred using base64 encoding, which 253 | will cause the transferred data to be around 34% larger than an 254 | encoding which supports native binary types. 255 | 256 | However, if your data fits inside the above constraints and 257 | you need cross-language support, the default setting of ``JSON`` 258 | is probably your best choice. 259 | 260 | ``pickle`` -- If you have no desire to support any language other than 261 | Python, then using the ``pickle`` encoding will gain you 262 | the support of all built-in Python data types (except class instances), 263 | smaller messages when sending binary files, and a slight speedup 264 | over ``JSON`` processing. 265 | 266 | ``yaml`` -- YAML has many of the same characteristics as ``json``, 267 | except that it natively supports more data types (including dates, 268 | recursive references, etc.) 269 | 270 | However, the Python libraries for YAML are a good bit slower 271 | than the libraries for JSON. 272 | 273 | If you need a more expressive set of data types and need to maintain 274 | cross-language compatibility, then ``YAML`` may be a better fit 275 | than the above. 276 | 277 | To instruct carrot to use an alternate serialization method, 278 | use one of the following options. 279 | 280 | 1. Set the serialization option on a per-Publisher basis: 281 | 282 | >>> from carrot.messaging import Publisher 283 | >>> publisher = Publisher(connection=conn, 284 | ... exchange="feed", routing_key="importer", 285 | ... serializer="yaml") 286 | 287 | 2. Set the serialization option on a per-call basis 288 | 289 | >>> from carrot.messaging import Publisher 290 | >>> publisher = Publisher(connection=conn, 291 | ... exchange="feed", routing_key="importer") 292 | >>> publisher.send({"import_feed": "http://cnn.com/rss/edition.rss"}, 293 | ... serializer="pickle") 294 | >>> publisher.close() 295 | 296 | Note that ``Consumer``s do not need the serialization method specified in 297 | their code. They can auto-detect the serialization method since we supply 298 | the ``Content-type`` header as part of the AMQP message. 299 | 300 | 301 | Sending raw data without Serialization 302 | --------------------------------------- 303 | 304 | In some cases, you don't need your message data to be serialized. If you 305 | pass in a plain string or unicode object as your message, then carrot will 306 | not waste cycles serializing/deserializing the data. 307 | 308 | You can optionally specify a ``content_type`` and ``content_encoding`` 309 | for the raw data: 310 | 311 | >>> from carrot.messaging import Publisher 312 | >>> publisher = Publisher(connection=conn, 313 | ... exchange="feed", 314 | routing_key="import_pictures") 315 | >>> publisher.send(open('~/my_picture.jpg','rb').read(), 316 | content_type="image/jpeg", 317 | content_encoding="binary") 318 | >>> publisher.close() 319 | 320 | The ``message`` object returned by the ``Consumer`` class will have a 321 | ``content_type`` and ``content_encoding`` attribute. 322 | 323 | 324 | Receiving messages without a callback 325 | -------------------------------------- 326 | 327 | You can also poll the queue manually, by using the ``fetch`` method. 328 | This method returns a ``Message`` object, from where you can get the 329 | message body, de-serialize the body to get the data, acknowledge, reject or 330 | re-queue the message. 331 | 332 | >>> consumer = Consumer(connection=conn, queue="feed", 333 | ... exchange="feed", routing_key="importer") 334 | >>> message = consumer.fetch() 335 | >>> if message: 336 | ... message_data = message.payload 337 | ... message.ack() 338 | ... else: 339 | ... # No messages waiting on the queue. 340 | >>> consumer.close() 341 | 342 | Sub-classing the messaging classes 343 | ---------------------------------- 344 | 345 | The ``Consumer``, and ``Publisher`` classes can also be sub classed. Thus you 346 | can define the above publisher and consumer like so: 347 | 348 | >>> from carrot.messaging import Publisher, Consumer 349 | 350 | >>> class FeedPublisher(Publisher): 351 | ... exchange = "feed" 352 | ... routing_key = "importer" 353 | ... 354 | ... def import_feed(self, feed_url): 355 | ... return self.send({"action": "import_feed", 356 | ... "feed_url": feed_url}) 357 | 358 | >>> class FeedConsumer(Consumer): 359 | ... queue = "feed" 360 | ... exchange = "feed" 361 | ... routing_key = "importer" 362 | ... 363 | ... def receive(self, message_data, message): 364 | ... action = message_data["action"] 365 | ... if action == "import_feed": 366 | ... # something importing this feed 367 | ... # import_feed(message_data["feed_url"]) 368 | message.ack() 369 | ... else: 370 | ... raise Exception("Unknown action: %s" % action) 371 | 372 | >>> publisher = FeedPublisher(connection=conn) 373 | >>> publisher.import_feed("http://cnn.com/rss/edition.rss") 374 | >>> publisher.close() 375 | 376 | >>> consumer = FeedConsumer(connection=conn) 377 | >>> consumer.wait() # Go into the consumer loop. 378 | 379 | Getting Help 380 | ============ 381 | 382 | Mailing list 383 | ------------ 384 | 385 | Join the `carrot-users`_ mailing list. 386 | 387 | .. _`carrot-users`: http://groups.google.com/group/carrot-users/ 388 | 389 | Bug tracker 390 | =========== 391 | 392 | If you have any suggestions, bug reports or annoyances please report them 393 | to our issue tracker at http://github.com/ask/carrot/issues/ 394 | 395 | Contributing 396 | ============ 397 | 398 | Development of ``carrot`` happens at Github: http://github.com/ask/carrot 399 | 400 | You are highly encouraged to participate in the development. If you don't 401 | like Github (for some reason) you're welcome to send regular patches. 402 | 403 | License 404 | ======= 405 | 406 | This software is licensed under the ``New BSD License``. See the ``LICENSE`` 407 | file in the top distribution directory for the full license text. 408 | -------------------------------------------------------------------------------- /THANKS: -------------------------------------------------------------------------------- 1 | Thanks to Barry Pederson for the py-amqplib library. 2 | Thanks to Grégoire Cachet for bug reports. 3 | Thanks to Martin Mahner for the Sphinx theme. 4 | Thanks to jcater for bug reports. 5 | Thanks to sebest for bug reports. 6 | Thanks to greut for bug reports 7 | -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | Please see our Issue Tracker at GitHub: 2 | http://github.com/ask/carrot/issues 3 | -------------------------------------------------------------------------------- /carrot/__init__.py: -------------------------------------------------------------------------------- 1 | """AMQP Messaging Framework for Python""" 2 | VERSION = (0, 10, 7) 3 | __version__ = ".".join(map(str, VERSION)) 4 | __author__ = "Ask Solem" 5 | __contact__ = "ask@celeryproject.org" 6 | __homepage__ = "http://github.com/ask/carrot/" 7 | __docformat__ = "restructuredtext" 8 | -------------------------------------------------------------------------------- /carrot/backends/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | Working with Backends. 4 | 5 | """ 6 | import sys 7 | 8 | from carrot.utils import rpartition 9 | 10 | DEFAULT_BACKEND = "carrot.backends.pyamqplib.Backend" 11 | 12 | BACKEND_ALIASES = { 13 | "amqp": "carrot.backends.pyamqplib.Backend", 14 | "amqplib": "carrot.backends.pyamqplib.Backend", 15 | "stomp": "carrot.backends.pystomp.Backend", 16 | "stompy": "carrot.backends.pystomp.Backend", 17 | "memory": "carrot.backends.queue.Backend", 18 | "mem": "carrot.backends.queue.Backend", 19 | "pika": "carrot.backends.pikachu.AsyncoreBackend", 20 | "pikachu": "carrot.backends.pikachu.AsyncoreBackend", 21 | "syncpika": "carrot.backends.pikachu.SyncBackend", 22 | } 23 | 24 | _backend_cache = {} 25 | 26 | 27 | def resolve_backend(backend=None): 28 | backend = BACKEND_ALIASES.get(backend, backend) 29 | backend_module_name, _, backend_cls_name = rpartition(backend, ".") 30 | return backend_module_name, backend_cls_name 31 | 32 | 33 | def _get_backend_cls(backend=None): 34 | backend_module_name, backend_cls_name = resolve_backend(backend) 35 | __import__(backend_module_name) 36 | backend_module = sys.modules[backend_module_name] 37 | return getattr(backend_module, backend_cls_name) 38 | 39 | 40 | def get_backend_cls(backend=None): 41 | """Get backend class by name. 42 | 43 | The backend string is the full path to a backend class, e.g.:: 44 | 45 | "carrot.backends.pyamqplib.Backend" 46 | 47 | If the name does not include "``.``" (is not fully qualified), 48 | the alias table will be consulted. 49 | 50 | """ 51 | backend = backend or DEFAULT_BACKEND 52 | if backend not in _backend_cache: 53 | _backend_cache[backend] = _get_backend_cls(backend) 54 | return _backend_cache[backend] 55 | -------------------------------------------------------------------------------- /carrot/backends/base.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | Backend base classes. 4 | 5 | """ 6 | from carrot import serialization 7 | 8 | ACKNOWLEDGED_STATES = frozenset(["ACK", "REJECTED", "REQUEUED"]) 9 | 10 | 11 | class MessageStateError(Exception): 12 | """The message has already been acknowledged.""" 13 | 14 | 15 | class BaseMessage(object): 16 | """Base class for received messages.""" 17 | _state = None 18 | 19 | MessageStateError = MessageStateError 20 | 21 | def __init__(self, backend, **kwargs): 22 | self.backend = backend 23 | self.body = kwargs.get("body") 24 | self.delivery_tag = kwargs.get("delivery_tag") 25 | self.content_type = kwargs.get("content_type") 26 | self.content_encoding = kwargs.get("content_encoding") 27 | self.delivery_info = kwargs.get("delivery_info", {}) 28 | self._decoded_cache = None 29 | self._state = "RECEIVED" 30 | 31 | def decode(self): 32 | """Deserialize the message body, returning the original 33 | python structure sent by the publisher.""" 34 | return serialization.decode(self.body, self.content_type, 35 | self.content_encoding) 36 | 37 | @property 38 | def payload(self): 39 | """The decoded message.""" 40 | if not self._decoded_cache: 41 | self._decoded_cache = self.decode() 42 | return self._decoded_cache 43 | 44 | def ack(self): 45 | """Acknowledge this message as being processed., 46 | This will remove the message from the queue. 47 | 48 | :raises MessageStateError: If the message has already been 49 | acknowledged/requeued/rejected. 50 | 51 | """ 52 | if self.acknowledged: 53 | raise self.MessageStateError( 54 | "Message already acknowledged with state: %s" % self._state) 55 | self.backend.ack(self.delivery_tag) 56 | self._state = "ACK" 57 | 58 | def reject(self): 59 | """Reject this message. 60 | 61 | The message will be discarded by the server. 62 | 63 | :raises MessageStateError: If the message has already been 64 | acknowledged/requeued/rejected. 65 | 66 | """ 67 | if self.acknowledged: 68 | raise self.MessageStateError( 69 | "Message already acknowledged with state: %s" % self._state) 70 | self.backend.reject(self.delivery_tag) 71 | self._state = "REJECTED" 72 | 73 | def requeue(self): 74 | """Reject this message and put it back on the queue. 75 | 76 | You must not use this method as a means of selecting messages 77 | to process. 78 | 79 | :raises MessageStateError: If the message has already been 80 | acknowledged/requeued/rejected. 81 | 82 | """ 83 | if self.acknowledged: 84 | raise self.MessageStateError( 85 | "Message already acknowledged with state: %s" % self._state) 86 | self.backend.requeue(self.delivery_tag) 87 | self._state = "REQUEUED" 88 | 89 | @property 90 | def acknowledged(self): 91 | return self._state in ACKNOWLEDGED_STATES 92 | 93 | 94 | class BaseBackend(object): 95 | """Base class for backends.""" 96 | default_port = None 97 | extra_options = None 98 | 99 | connection_errors = () 100 | channel_errors = () 101 | 102 | def __init__(self, connection, **kwargs): 103 | self.connection = connection 104 | self.extra_options = kwargs.get("extra_options") 105 | 106 | def queue_declare(self, *args, **kwargs): 107 | """Declare a queue by name.""" 108 | pass 109 | 110 | def queue_delete(self, *args, **kwargs): 111 | """Delete a queue by name.""" 112 | pass 113 | 114 | def exchange_declare(self, *args, **kwargs): 115 | """Declare an exchange by name.""" 116 | pass 117 | 118 | def queue_bind(self, *args, **kwargs): 119 | """Bind a queue to an exchange.""" 120 | pass 121 | 122 | def get(self, *args, **kwargs): 123 | """Pop a message off the queue.""" 124 | pass 125 | 126 | def declare_consumer(self, *args, **kwargs): 127 | pass 128 | 129 | def consume(self, *args, **kwargs): 130 | """Iterate over the declared consumers.""" 131 | pass 132 | 133 | def cancel(self, *args, **kwargs): 134 | """Cancel the consumer.""" 135 | pass 136 | 137 | def ack(self, delivery_tag): 138 | """Acknowledge the message.""" 139 | pass 140 | 141 | def queue_purge(self, queue, **kwargs): 142 | """Discard all messages in the queue. This will delete the messages 143 | and results in an empty queue.""" 144 | return 0 145 | 146 | def reject(self, delivery_tag): 147 | """Reject the message.""" 148 | pass 149 | 150 | def requeue(self, delivery_tag): 151 | """Requeue the message.""" 152 | pass 153 | 154 | def purge(self, queue, **kwargs): 155 | """Discard all messages in the queue.""" 156 | pass 157 | 158 | def message_to_python(self, raw_message): 159 | """Convert received message body to a python datastructure.""" 160 | return raw_message 161 | 162 | def prepare_message(self, message_data, delivery_mode, **kwargs): 163 | """Prepare message for sending.""" 164 | return message_data 165 | 166 | def publish(self, message, exchange, routing_key, **kwargs): 167 | """Publish a message.""" 168 | pass 169 | 170 | def close(self): 171 | """Close the backend.""" 172 | pass 173 | 174 | def establish_connection(self): 175 | """Establish a connection to the backend.""" 176 | pass 177 | 178 | def close_connection(self, connection): 179 | """Close the connection.""" 180 | pass 181 | 182 | def flow(self, active): 183 | """Enable/disable flow from peer.""" 184 | pass 185 | 186 | def qos(self, prefetch_size, prefetch_count, apply_global=False): 187 | """Request specific Quality of Service.""" 188 | pass 189 | -------------------------------------------------------------------------------- /carrot/backends/librabbitmq.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | `amqplib`_ backend for carrot. 4 | 5 | .. _`amqplib`: http://barryp.org/software/py-amqplib/ 6 | 7 | """ 8 | import pylibrabbitmq as amqp 9 | from pylibrabbitmq import ChannelError, ConnectionError 10 | from carrot.backends.base import BaseMessage, BaseBackend 11 | from itertools import count 12 | import warnings 13 | import weakref 14 | 15 | DEFAULT_PORT = 5672 16 | 17 | 18 | class Message(BaseMessage): 19 | """A message received by the broker. 20 | 21 | Usually you don't insantiate message objects yourself, but receive 22 | them using a :class:`carrot.messaging.Consumer`. 23 | 24 | :param backend: see :attr:`backend`. 25 | :param amqp_message: see :attr:`_amqp_message`. 26 | 27 | 28 | .. attribute:: body 29 | 30 | The message body. 31 | 32 | .. attribute:: delivery_tag 33 | 34 | The message delivery tag, uniquely identifying this message. 35 | 36 | .. attribute:: backend 37 | 38 | The message backend used. 39 | A subclass of :class:`carrot.backends.base.BaseBackend`. 40 | 41 | .. attribute:: _amqp_message 42 | 43 | A :class:`amqplib.client_0_8.basic_message.Message` instance. 44 | This is a private attribute and should not be accessed by 45 | production code. 46 | 47 | """ 48 | 49 | def __init__(self, backend, amqp_message, **kwargs): 50 | self._amqp_message = amqp_message 51 | self.backend = backend 52 | kwargs["body"] = amqp_message.body 53 | properties = amqp_message.properties 54 | kwargs["content_type"] = properties["content_type"] 55 | kwargs["content_encoding"] = properties["content_encoding"] 56 | kwargs["delivery_info"] = amqp_message.delivery_info 57 | kwargs["delivery_tag"] = amqp_message.delivery_info["delivery_tag"] 58 | 59 | super(Message, self).__init__(backend, **kwargs) 60 | 61 | 62 | class Backend(BaseBackend): 63 | """amqplib backend 64 | 65 | :param connection: see :attr:`connection`. 66 | 67 | 68 | .. attribute:: connection 69 | 70 | A :class:`carrot.connection.BrokerConnection` instance. An established 71 | connection to the broker. 72 | 73 | """ 74 | default_port = DEFAULT_PORT 75 | 76 | Message = Message 77 | 78 | def __init__(self, connection, **kwargs): 79 | self.connection = connection 80 | self.default_port = kwargs.get("default_port", self.default_port) 81 | self._channel_ref = None 82 | 83 | @property 84 | def _channel(self): 85 | return callable(self._channel_ref) and self._channel_ref() 86 | 87 | @property 88 | def channel(self): 89 | """If no channel exists, a new one is requested.""" 90 | if not self._channel: 91 | self._channel_ref = weakref.ref(self.connection.get_channel()) 92 | return self._channel 93 | 94 | def establish_connection(self): 95 | """Establish connection to the AMQP broker.""" 96 | conninfo = self.connection 97 | if not conninfo.hostname: 98 | raise KeyError("Missing hostname for AMQP connection.") 99 | if conninfo.userid is None: 100 | raise KeyError("Missing user id for AMQP connection.") 101 | if conninfo.password is None: 102 | raise KeyError("Missing password for AMQP connection.") 103 | if not conninfo.port: 104 | conninfo.port = self.default_port 105 | conn = amqp.Connection(host=conninfo.hostname, 106 | port=conninfo.port, 107 | userid=conninfo.userid, 108 | password=conninfo.password, 109 | virtual_host=conninfo.virtual_host) 110 | return conn 111 | 112 | def close_connection(self, connection): 113 | """Close the AMQP broker connection.""" 114 | connection.close() 115 | 116 | def queue_exists(self, queue): 117 | return True 118 | 119 | def queue_delete(self, queue, if_unused=False, if_empty=False): 120 | """Delete queue by name.""" 121 | pass 122 | 123 | def queue_purge(self, queue, **kwargs): 124 | """Discard all messages in the queue. This will delete the messages 125 | and results in an empty queue.""" 126 | return self.channel.queue_purge(queue=queue) 127 | 128 | def queue_declare(self, queue, durable, exclusive, auto_delete, 129 | warn_if_exists=False): 130 | """Declare a named queue.""" 131 | return self.channel.queue_declare(queue=queue, 132 | durable=durable, 133 | exclusive=exclusive, 134 | auto_delete=auto_delete) 135 | 136 | def exchange_declare(self, exchange, type, durable, auto_delete): 137 | """Declare an named exchange.""" 138 | return self.channel.exchange_declare(exchange=exchange, 139 | type=type, 140 | durable=durable, 141 | auto_delete=auto_delete) 142 | 143 | def queue_bind(self, queue, exchange, routing_key, arguments=None): 144 | """Bind queue to an exchange using a routing key.""" 145 | return self.channel.queue_bind(queue=queue, 146 | exchange=exchange, 147 | routing_key=routing_key, 148 | arguments=arguments) 149 | 150 | def message_to_python(self, raw_message): 151 | """Convert encoded message body back to a Python value.""" 152 | return self.Message(backend=self, amqp_message=raw_message) 153 | 154 | def get(self, queue, no_ack=False): 155 | """Receive a message from a declared queue by name. 156 | 157 | :returns: A :class:`Message` object if a message was received, 158 | ``None`` otherwise. If ``None`` was returned, it probably means 159 | there was no messages waiting on the queue. 160 | 161 | """ 162 | raw_message = self.channel.basic_get(queue, no_ack=no_ack) 163 | if not raw_message: 164 | return None 165 | return self.message_to_python(raw_message) 166 | 167 | def declare_consumer(self, queue, no_ack, callback, consumer_tag, 168 | nowait=False): 169 | """Declare a consumer.""" 170 | return self.channel.basic_consume(queue=queue, 171 | no_ack=no_ack, 172 | callback=callback, 173 | consumer_tag=consumer_tag) 174 | 175 | def consume(self, limit=None): 176 | """Returns an iterator that waits for one message at a time.""" 177 | for total_message_count in count(): 178 | if limit and total_message_count >= limit: 179 | raise StopIteration 180 | 181 | if not self.channel.is_open: 182 | raise StopIteration 183 | 184 | self.channel.conn.drain_events() 185 | yield True 186 | 187 | def cancel(self, consumer_tag): 188 | """Cancel a channel by consumer tag.""" 189 | if not self.channel.conn: 190 | return 191 | self.channel.basic_cancel(consumer_tag) 192 | 193 | def close(self): 194 | """Close the channel if open.""" 195 | if self._channel and self._channel.is_open: 196 | self._channel.close() 197 | self._channel_ref = None 198 | 199 | def ack(self, delivery_tag): 200 | """Acknowledge a message by delivery tag.""" 201 | return self.channel.basic_ack(delivery_tag) 202 | 203 | def reject(self, delivery_tag): 204 | """Reject a message by deliver tag.""" 205 | return self.channel.basic_reject(delivery_tag, requeue=False) 206 | 207 | def requeue(self, delivery_tag): 208 | """Reject and requeue a message by delivery tag.""" 209 | return self.channel.basic_reject(delivery_tag, requeue=True) 210 | 211 | def prepare_message(self, message_data, delivery_mode, priority=None, 212 | content_type=None, content_encoding=None): 213 | """Encapsulate data into a AMQP message.""" 214 | return amqp.Message(message_data, properties={ 215 | "delivery_mode": delivery_mode, 216 | "priority": priority, 217 | "content_type": content_type, 218 | "content_encoding": content_encoding}) 219 | 220 | def publish(self, message, exchange, routing_key, mandatory=None, 221 | immediate=None, headers=None): 222 | """Publish a message to a named exchange.""" 223 | 224 | if headers: 225 | message.properties["headers"] = headers 226 | 227 | ret = self.channel.basic_publish(message, exchange=exchange, 228 | routing_key=routing_key, 229 | mandatory=mandatory, 230 | immediate=immediate) 231 | if mandatory or immediate: 232 | self.close() 233 | 234 | def qos(self, prefetch_size, prefetch_count, apply_global=False): 235 | """Request specific Quality of Service.""" 236 | pass 237 | #self.channel.basic_qos(prefetch_size, prefetch_count, 238 | # apply_global) 239 | 240 | def flow(self, active): 241 | """Enable/disable flow from peer.""" 242 | pass 243 | #self.channel.flow(active) 244 | -------------------------------------------------------------------------------- /carrot/backends/pikachu.py: -------------------------------------------------------------------------------- 1 | import asyncore 2 | import weakref 3 | import functools 4 | import itertools 5 | 6 | import pika 7 | 8 | from carrot.backends.base import BaseMessage, BaseBackend 9 | 10 | DEFAULT_PORT = 5672 11 | 12 | 13 | class Message(BaseMessage): 14 | 15 | def __init__(self, backend, amqp_message, **kwargs): 16 | channel, method, header, body = amqp_message 17 | self._channel = channel 18 | self._method = method 19 | self._header = header 20 | self.backend = backend 21 | 22 | kwargs.update({"body": body, 23 | "delivery_tag": method.delivery_tag, 24 | "content_type": header.content_type, 25 | "content_encoding": header.content_encoding, 26 | "delivery_info": dict( 27 | consumer_tag=method.consumer_tag, 28 | routing_key=method.routing_key, 29 | delivery_tag=method.delivery_tag, 30 | exchange=method.exchange)}) 31 | 32 | super(Message, self).__init__(backend, **kwargs) 33 | 34 | 35 | class SyncBackend(BaseBackend): 36 | default_port = DEFAULT_PORT 37 | _connection_cls = pika.BlockingConnection 38 | 39 | Message = Message 40 | 41 | def __init__(self, connection, **kwargs): 42 | self.connection = connection 43 | self.default_port = kwargs.get("default_port", self.default_port) 44 | self._channel_ref = None 45 | 46 | @property 47 | def _channel(self): 48 | return callable(self._channel_ref) and self._channel_ref() 49 | 50 | @property 51 | def channel(self): 52 | """If no channel exists, a new one is requested.""" 53 | if not self._channel: 54 | self._channel_ref = weakref.ref(self.connection.get_channel()) 55 | return self._channel 56 | 57 | def establish_connection(self): 58 | """Establish connection to the AMQP broker.""" 59 | conninfo = self.connection 60 | if not conninfo.port: 61 | conninfo.port = self.default_port 62 | credentials = pika.PlainCredentials(conninfo.userid, 63 | conninfo.password) 64 | return self._connection_cls(pika.ConnectionParameters( 65 | conninfo.hostname, 66 | port=conninfo.port, 67 | virtual_host=conninfo.virtual_host, 68 | credentials=credentials)) 69 | 70 | def close_connection(self, connection): 71 | """Close the AMQP broker connection.""" 72 | connection.close() 73 | 74 | def queue_exists(self, queue): 75 | return False # FIXME 76 | 77 | def queue_delete(self, queue, if_unused=False, if_empty=False): 78 | """Delete queue by name.""" 79 | return self.channel.queue_delete(queue=queue, if_unused=if_unused, 80 | if_empty=if_empty) 81 | 82 | def queue_purge(self, queue, **kwargs): 83 | """Discard all messages in the queue. This will delete the messages 84 | and results in an empty queue.""" 85 | return self.channel.queue_purge(queue=queue).message_count 86 | 87 | def queue_declare(self, queue, durable, exclusive, auto_delete, 88 | warn_if_exists=False, arguments=None): 89 | """Declare a named queue.""" 90 | 91 | return self.channel.queue_declare(queue=queue, 92 | durable=durable, 93 | exclusive=exclusive, 94 | auto_delete=auto_delete, 95 | arguments=arguments) 96 | 97 | def exchange_declare(self, exchange, type, durable, auto_delete): 98 | """Declare an named exchange.""" 99 | return self.channel.exchange_declare(exchange=exchange, 100 | type=type, 101 | durable=durable, 102 | auto_delete=auto_delete) 103 | 104 | def queue_bind(self, queue, exchange, routing_key, arguments={}): 105 | """Bind queue to an exchange using a routing key.""" 106 | if not arguments: 107 | arguments = {} 108 | return self.channel.queue_bind(queue=queue, 109 | exchange=exchange, 110 | routing_key=routing_key, 111 | arguments=arguments) 112 | 113 | def message_to_python(self, raw_message): 114 | """Convert encoded message body back to a Python value.""" 115 | return self.Message(backend=self, amqp_message=raw_message) 116 | 117 | def get(self, queue, no_ack=False): 118 | """Receive a message from a declared queue by name. 119 | 120 | :returns: A :class:`Message` object if a message was received, 121 | ``None`` otherwise. If ``None`` was returned, it probably means 122 | there was no messages waiting on the queue. 123 | 124 | """ 125 | raw_message = self.channel.basic_get(queue=queue, no_ack=no_ack) 126 | if not raw_message: 127 | return None 128 | return self.message_to_python(raw_message) 129 | 130 | def declare_consumer(self, queue, no_ack, callback, consumer_tag, 131 | nowait=False): 132 | """Declare a consumer.""" 133 | 134 | @functools.wraps(callback) 135 | def _callback_decode(channel, method, header, body): 136 | return callback((channel, method, header, body)) 137 | 138 | return self.channel.basic_consume(_callback_decode, 139 | queue=queue, 140 | no_ack=no_ack, 141 | consumer_tag=consumer_tag) 142 | 143 | def consume(self, limit=None): 144 | """Returns an iterator that waits for one message at a time.""" 145 | for total_message_count in itertools.count(): 146 | if limit and total_message_count >= limit: 147 | raise StopIteration 148 | self.connection.connection.drain_events() 149 | yield True 150 | 151 | def cancel(self, consumer_tag): 152 | """Cancel a channel by consumer tag.""" 153 | if not self._channel: 154 | return 155 | self.channel.basic_cancel(consumer_tag) 156 | 157 | def close(self): 158 | """Close the channel if open.""" 159 | if self._channel and not self._channel.handler.channel_close: 160 | self._channel.close() 161 | self._channel_ref = None 162 | 163 | def ack(self, delivery_tag): 164 | """Acknowledge a message by delivery tag.""" 165 | return self.channel.basic_ack(delivery_tag) 166 | 167 | def reject(self, delivery_tag): 168 | """Reject a message by deliver tag.""" 169 | return self.channel.basic_reject(delivery_tag, requeue=False) 170 | 171 | def requeue(self, delivery_tag): 172 | """Reject and requeue a message by delivery tag.""" 173 | return self.channel.basic_reject(delivery_tag, requeue=True) 174 | 175 | def prepare_message(self, message_data, delivery_mode, priority=None, 176 | content_type=None, content_encoding=None): 177 | """Encapsulate data into a AMQP message.""" 178 | properties = pika.BasicProperties(priority=priority, 179 | content_type=content_type, 180 | content_encoding=content_encoding, 181 | delivery_mode=delivery_mode) 182 | return message_data, properties 183 | 184 | def publish(self, message, exchange, routing_key, mandatory=None, 185 | immediate=None, headers=None): 186 | """Publish a message to a named exchange.""" 187 | body, properties = message 188 | 189 | if headers: 190 | properties.headers = headers 191 | 192 | ret = self.channel.basic_publish(body=body, 193 | properties=properties, 194 | exchange=exchange, 195 | routing_key=routing_key, 196 | mandatory=mandatory, 197 | immediate=immediate) 198 | if mandatory or immediate: 199 | self.close() 200 | 201 | def qos(self, prefetch_size, prefetch_count, apply_global=False): 202 | """Request specific Quality of Service.""" 203 | self.channel.basic_qos(prefetch_size, prefetch_count, 204 | apply_global) 205 | 206 | def flow(self, active): 207 | """Enable/disable flow from peer.""" 208 | self.channel.flow(active) 209 | 210 | 211 | class AsyncoreBackend(SyncBackend): 212 | _connection_cls = pika.AsyncoreConnection 213 | -------------------------------------------------------------------------------- /carrot/backends/pyamqplib.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | `amqplib`_ backend for carrot. 4 | 5 | .. _`amqplib`: http://barryp.org/software/py-amqplib/ 6 | 7 | """ 8 | from amqplib.client_0_8 import transport 9 | # amqplib's handshake mistakenly identifies as protocol version 1191, 10 | # this breaks in RabbitMQ tip, which no longer falls back to 11 | # 0-8 for unknown ids. 12 | transport.AMQP_PROTOCOL_HEADER = "AMQP\x01\x01\x08\x00" 13 | 14 | from amqplib import client_0_8 as amqp 15 | from amqplib.client_0_8.exceptions import AMQPConnectionException 16 | from amqplib.client_0_8.exceptions import AMQPChannelException 17 | from amqplib.client_0_8.serialization import AMQPReader, AMQPWriter 18 | from carrot.backends.base import BaseMessage, BaseBackend 19 | from itertools import count 20 | 21 | import socket 22 | import warnings 23 | import weakref 24 | 25 | DEFAULT_PORT = 5672 26 | 27 | 28 | class Connection(amqp.Connection): 29 | 30 | def drain_events(self, allowed_methods=None, timeout=None): 31 | """Wait for an event on any channel.""" 32 | return self.wait_multi(self.channels.values(), timeout=timeout) 33 | 34 | def wait_multi(self, channels, allowed_methods=None, timeout=None): 35 | """Wait for an event on a channel.""" 36 | chanmap = dict((chan.channel_id, chan) for chan in channels) 37 | chanid, method_sig, args, content = self._wait_multiple( 38 | chanmap.keys(), allowed_methods, timeout=timeout) 39 | 40 | channel = chanmap[chanid] 41 | 42 | if content \ 43 | and channel.auto_decode \ 44 | and hasattr(content, 'content_encoding'): 45 | try: 46 | content.body = content.body.decode(content.content_encoding) 47 | except Exception: 48 | pass 49 | 50 | amqp_method = channel._METHOD_MAP.get(method_sig, None) 51 | 52 | if amqp_method is None: 53 | raise Exception('Unknown AMQP method (%d, %d)' % method_sig) 54 | 55 | if content is None: 56 | return amqp_method(channel, args) 57 | else: 58 | return amqp_method(channel, args, content) 59 | 60 | def read_timeout(self, timeout=None): 61 | if timeout is None: 62 | return self.method_reader.read_method() 63 | sock = self.transport.sock 64 | prev = sock.gettimeout() 65 | sock.settimeout(timeout) 66 | try: 67 | return self.method_reader.read_method() 68 | finally: 69 | sock.settimeout(prev) 70 | 71 | def _wait_multiple(self, channel_ids, allowed_methods, timeout=None): 72 | for channel_id in channel_ids: 73 | method_queue = self.channels[channel_id].method_queue 74 | for queued_method in method_queue: 75 | method_sig = queued_method[0] 76 | if (allowed_methods is None) \ 77 | or (method_sig in allowed_methods) \ 78 | or (method_sig == (20, 40)): 79 | method_queue.remove(queued_method) 80 | method_sig, args, content = queued_method 81 | return channel_id, method_sig, args, content 82 | 83 | # Nothing queued, need to wait for a method from the peer 84 | while True: 85 | channel, method_sig, args, content = self.read_timeout(timeout) 86 | 87 | if (channel in channel_ids) \ 88 | and ((allowed_methods is None) \ 89 | or (method_sig in allowed_methods) \ 90 | or (method_sig == (20, 40))): 91 | return channel, method_sig, args, content 92 | 93 | # Not the channel and/or method we were looking for. Queue 94 | # this method for later 95 | self.channels[channel].method_queue.append((method_sig, 96 | args, 97 | content)) 98 | 99 | # 100 | # If we just queued up a method for channel 0 (the Connection 101 | # itself) it's probably a close method in reaction to some 102 | # error, so deal with it right away. 103 | # 104 | if channel == 0: 105 | self.wait() 106 | 107 | 108 | class QueueAlreadyExistsWarning(UserWarning): 109 | """A queue with that name already exists, so a recently changed 110 | ``routing_key`` or other settings might be ignored unless you 111 | rename the queue or restart the broker.""" 112 | 113 | 114 | class Message(BaseMessage): 115 | """A message received by the broker. 116 | 117 | Usually you don't insantiate message objects yourself, but receive 118 | them using a :class:`carrot.messaging.Consumer`. 119 | 120 | :param backend: see :attr:`backend`. 121 | :param amqp_message: see :attr:`_amqp_message`. 122 | 123 | 124 | .. attribute:: body 125 | 126 | The message body. 127 | 128 | .. attribute:: delivery_tag 129 | 130 | The message delivery tag, uniquely identifying this message. 131 | 132 | .. attribute:: backend 133 | 134 | The message backend used. 135 | A subclass of :class:`carrot.backends.base.BaseBackend`. 136 | 137 | .. attribute:: _amqp_message 138 | 139 | A :class:`amqplib.client_0_8.basic_message.Message` instance. 140 | This is a private attribute and should not be accessed by 141 | production code. 142 | 143 | """ 144 | 145 | def __init__(self, backend, amqp_message, **kwargs): 146 | self._amqp_message = amqp_message 147 | self.backend = backend 148 | 149 | for attr_name in ("body", 150 | "delivery_tag", 151 | "content_type", 152 | "content_encoding", 153 | "delivery_info"): 154 | kwargs[attr_name] = getattr(amqp_message, attr_name, None) 155 | 156 | super(Message, self).__init__(backend, **kwargs) 157 | 158 | 159 | class Backend(BaseBackend): 160 | """amqplib backend 161 | 162 | :param connection: see :attr:`connection`. 163 | 164 | 165 | .. attribute:: connection 166 | 167 | A :class:`carrot.connection.BrokerConnection` instance. An established 168 | connection to the broker. 169 | 170 | """ 171 | default_port = DEFAULT_PORT 172 | 173 | connection_errors = (AMQPConnectionException, 174 | socket.error, 175 | IOError, 176 | OSError) 177 | channel_errors = (AMQPChannelException, ) 178 | 179 | Message = Message 180 | 181 | def __init__(self, connection, **kwargs): 182 | self.connection = connection 183 | self.default_port = kwargs.get("default_port", self.default_port) 184 | self._channel_ref = None 185 | 186 | @property 187 | def _channel(self): 188 | return callable(self._channel_ref) and self._channel_ref() 189 | 190 | @property 191 | def channel(self): 192 | """If no channel exists, a new one is requested.""" 193 | if not self._channel: 194 | connection = self.connection.connection 195 | self._channel_ref = weakref.ref(connection.channel()) 196 | return self._channel 197 | 198 | def establish_connection(self): 199 | """Establish connection to the AMQP broker.""" 200 | conninfo = self.connection 201 | if not conninfo.hostname: 202 | raise KeyError("Missing hostname for AMQP connection.") 203 | if conninfo.userid is None: 204 | raise KeyError("Missing user id for AMQP connection.") 205 | if conninfo.password is None: 206 | raise KeyError("Missing password for AMQP connection.") 207 | if not conninfo.port: 208 | conninfo.port = self.default_port 209 | return Connection(host=conninfo.host, 210 | userid=conninfo.userid, 211 | password=conninfo.password, 212 | virtual_host=conninfo.virtual_host, 213 | insist=conninfo.insist, 214 | ssl=conninfo.ssl, 215 | connect_timeout=conninfo.connect_timeout) 216 | 217 | def close_connection(self, connection): 218 | """Close the AMQP broker connection.""" 219 | connection.close() 220 | 221 | def queue_exists(self, queue): 222 | """Check if a queue has been declared. 223 | 224 | :rtype bool: 225 | 226 | """ 227 | try: 228 | self.channel.queue_declare(queue=queue, passive=True) 229 | except AMQPChannelException, e: 230 | if e.amqp_reply_code == 404: 231 | return False 232 | raise e 233 | else: 234 | return True 235 | 236 | def queue_delete(self, queue, if_unused=False, if_empty=False): 237 | """Delete queue by name.""" 238 | return self.channel.queue_delete(queue, if_unused, if_empty) 239 | 240 | def queue_purge(self, queue, **kwargs): 241 | """Discard all messages in the queue. This will delete the messages 242 | and results in an empty queue.""" 243 | return self.channel.queue_purge(queue=queue) 244 | 245 | def queue_declare(self, queue, durable, exclusive, auto_delete, 246 | warn_if_exists=False, arguments=None): 247 | """Declare a named queue.""" 248 | if warn_if_exists and self.queue_exists(queue): 249 | warnings.warn(QueueAlreadyExistsWarning( 250 | QueueAlreadyExistsWarning.__doc__)) 251 | 252 | return self.channel.queue_declare(queue=queue, 253 | durable=durable, 254 | exclusive=exclusive, 255 | auto_delete=auto_delete, 256 | arguments=arguments) 257 | 258 | def exchange_declare(self, exchange, type, durable, auto_delete): 259 | """Declare an named exchange.""" 260 | return self.channel.exchange_declare(exchange=exchange, 261 | type=type, 262 | durable=durable, 263 | auto_delete=auto_delete) 264 | 265 | def queue_bind(self, queue, exchange, routing_key, arguments=None): 266 | """Bind queue to an exchange using a routing key.""" 267 | return self.channel.queue_bind(queue=queue, 268 | exchange=exchange, 269 | routing_key=routing_key, 270 | arguments=arguments) 271 | 272 | def message_to_python(self, raw_message): 273 | """Convert encoded message body back to a Python value.""" 274 | return self.Message(backend=self, amqp_message=raw_message) 275 | 276 | def get(self, queue, no_ack=False): 277 | """Receive a message from a declared queue by name. 278 | 279 | :returns: A :class:`Message` object if a message was received, 280 | ``None`` otherwise. If ``None`` was returned, it probably means 281 | there was no messages waiting on the queue. 282 | 283 | """ 284 | raw_message = self.channel.basic_get(queue, no_ack=no_ack) 285 | if not raw_message: 286 | return None 287 | return self.message_to_python(raw_message) 288 | 289 | def declare_consumer(self, queue, no_ack, callback, consumer_tag, 290 | nowait=False): 291 | """Declare a consumer.""" 292 | return self.channel.basic_consume(queue=queue, 293 | no_ack=no_ack, 294 | callback=callback, 295 | consumer_tag=consumer_tag, 296 | nowait=nowait) 297 | 298 | def consume(self, limit=None): 299 | """Returns an iterator that waits for one message at a time.""" 300 | for total_message_count in count(): 301 | if limit and total_message_count >= limit: 302 | raise StopIteration 303 | 304 | if not self.channel.is_open: 305 | raise StopIteration 306 | 307 | self.channel.wait() 308 | yield True 309 | 310 | def cancel(self, consumer_tag): 311 | """Cancel a channel by consumer tag.""" 312 | if not self.channel.connection: 313 | return 314 | self.channel.basic_cancel(consumer_tag) 315 | 316 | def close(self): 317 | """Close the channel if open.""" 318 | if self._channel and self._channel.is_open: 319 | self._channel.close() 320 | self._channel_ref = None 321 | 322 | def ack(self, delivery_tag): 323 | """Acknowledge a message by delivery tag.""" 324 | return self.channel.basic_ack(delivery_tag) 325 | 326 | def reject(self, delivery_tag): 327 | """Reject a message by deliver tag.""" 328 | return self.channel.basic_reject(delivery_tag, requeue=False) 329 | 330 | def requeue(self, delivery_tag): 331 | """Reject and requeue a message by delivery tag.""" 332 | return self.channel.basic_reject(delivery_tag, requeue=True) 333 | 334 | def prepare_message(self, message_data, delivery_mode, priority=None, 335 | content_type=None, content_encoding=None): 336 | """Encapsulate data into a AMQP message.""" 337 | message = amqp.Message(message_data, priority=priority, 338 | content_type=content_type, 339 | content_encoding=content_encoding) 340 | message.properties["delivery_mode"] = delivery_mode 341 | return message 342 | 343 | def publish(self, message, exchange, routing_key, mandatory=None, 344 | immediate=None, headers=None): 345 | """Publish a message to a named exchange.""" 346 | 347 | if headers: 348 | message.properties["headers"] = headers 349 | 350 | ret = self.channel.basic_publish(message, exchange=exchange, 351 | routing_key=routing_key, 352 | mandatory=mandatory, 353 | immediate=immediate) 354 | if mandatory or immediate: 355 | self.close() 356 | 357 | def qos(self, prefetch_size, prefetch_count, apply_global=False): 358 | """Request specific Quality of Service.""" 359 | self.channel.basic_qos(prefetch_size, prefetch_count, 360 | apply_global) 361 | 362 | def flow(self, active): 363 | """Enable/disable flow from peer.""" 364 | self.channel.flow(active) 365 | -------------------------------------------------------------------------------- /carrot/backends/pystomp.py: -------------------------------------------------------------------------------- 1 | import time 2 | import socket 3 | from itertools import count 4 | 5 | from stompy import Client 6 | from stompy import Empty as QueueEmpty 7 | 8 | from carrot.backends.base import BaseMessage, BaseBackend 9 | 10 | DEFAULT_PORT = 61613 11 | 12 | 13 | class Message(BaseMessage): 14 | """A message received by the STOMP broker. 15 | 16 | Usually you don't insantiate message objects yourself, but receive 17 | them using a :class:`carrot.messaging.Consumer`. 18 | 19 | :param backend: see :attr:`backend`. 20 | :param frame: see :attr:`_frame`. 21 | 22 | .. attribute:: body 23 | 24 | The message body. 25 | 26 | .. attribute:: delivery_tag 27 | 28 | The message delivery tag, uniquely identifying this message. 29 | 30 | .. attribute:: backend 31 | 32 | The message backend used. 33 | A subclass of :class:`carrot.backends.base.BaseBackend`. 34 | 35 | .. attribute:: _frame 36 | 37 | The frame received by the STOMP client. This is considered a private 38 | variable and should never be used in production code. 39 | 40 | """ 41 | 42 | def __init__(self, backend, frame, **kwargs): 43 | self._frame = frame 44 | self.backend = backend 45 | 46 | kwargs["body"] = frame.body 47 | kwargs["delivery_tag"] = frame.headers["message-id"] 48 | kwargs["content_type"] = frame.headers.get("content-type") 49 | kwargs["content_encoding"] = frame.headers.get("content-encoding") 50 | kwargs["priority"] = frame.headers.get("priority") 51 | 52 | super(Message, self).__init__(backend, **kwargs) 53 | 54 | def ack(self): 55 | """Acknowledge this message as being processed., 56 | This will remove the message from the queue. 57 | 58 | :raises MessageStateError: If the message has already been 59 | acknowledged/requeued/rejected. 60 | 61 | """ 62 | if self.acknowledged: 63 | raise self.MessageStateError( 64 | "Message already acknowledged with state: %s" % self._state) 65 | self.backend.ack(self._frame) 66 | self._state = "ACK" 67 | 68 | def reject(self): 69 | raise NotImplementedError( 70 | "The STOMP backend does not implement basic.reject") 71 | 72 | def requeue(self): 73 | raise NotImplementedError( 74 | "The STOMP backend does not implement requeue") 75 | 76 | 77 | class Backend(BaseBackend): 78 | Stomp = Client 79 | Message = Message 80 | default_port = DEFAULT_PORT 81 | 82 | def __init__(self, connection, **kwargs): 83 | self.connection = connection 84 | self.default_port = kwargs.get("default_port", self.default_port) 85 | self._channel = None 86 | self._consumers = {} # open consumers by consumer tag 87 | self._callbacks = {} 88 | 89 | def establish_connection(self): 90 | conninfo = self.connection 91 | if not conninfo.port: 92 | conninfo.port = self.default_port 93 | stomp = self.Stomp(conninfo.hostname, conninfo.port) 94 | stomp.connect(username=conninfo.userid, password=conninfo.password) 95 | stomp.drain_events = self.drain_events 96 | return stomp 97 | 98 | def close_connection(self, connection): 99 | try: 100 | connection.disconnect() 101 | except socket.error: 102 | pass 103 | 104 | def queue_exists(self, queue): 105 | return True 106 | 107 | def queue_purge(self, queue, **kwargs): 108 | for purge_count in count(0): 109 | try: 110 | frame = self.channel.get_nowait() 111 | except QueueEmpty: 112 | return purge_count 113 | else: 114 | self.channel.ack(frame) 115 | 116 | def declare_consumer(self, queue, no_ack, callback, consumer_tag, 117 | **kwargs): 118 | ack = no_ack and "auto" or "client" 119 | self.channel.subscribe(queue, ack=ack) 120 | self._consumers[consumer_tag] = queue 121 | self._callbacks[queue] = callback 122 | 123 | def drain_events(self, timeout=None): 124 | start_time = time.time() 125 | while True: 126 | frame = self.channel.get() 127 | if frame: 128 | break 129 | if time.time() - time_start > timeout: 130 | raise socket.timeout("the operation timed out.") 131 | queue = frame.headers.get("destination") 132 | if not queue or queue not in self._callbacks: 133 | return 134 | self._callbacks[queue](frame) 135 | 136 | def consume(self, limit=None): 137 | """Returns an iterator that waits for one message at a time.""" 138 | for total_message_count in count(): 139 | if limit and total_message_count >= limit: 140 | raise StopIteration 141 | self.drain_events() 142 | yield True 143 | 144 | def queue_declare(self, queue, *args, **kwargs): 145 | self.channel.subscribe(queue, ack="client") 146 | 147 | def get(self, queue, no_ack=False): 148 | try: 149 | frame = self.channel.get_nowait() 150 | except QueueEmpty: 151 | return None 152 | else: 153 | return self.message_to_python(frame) 154 | 155 | def ack(self, frame): 156 | self.channel.ack(frame) 157 | 158 | def message_to_python(self, raw_message): 159 | """Convert encoded message body back to a Python value.""" 160 | return self.Message(backend=self, frame=raw_message) 161 | 162 | def prepare_message(self, message_data, delivery_mode, priority=0, 163 | content_type=None, content_encoding=None): 164 | persistent = "false" 165 | if delivery_mode == 2: 166 | persistent = "true" 167 | priority = priority or 0 168 | return {"body": message_data, 169 | "persistent": persistent, 170 | "priority": priority, 171 | "content-encoding": content_encoding, 172 | "content-type": content_type} 173 | 174 | def publish(self, message, exchange, routing_key, **kwargs): 175 | message["destination"] = exchange 176 | self.channel.stomp.send(message) 177 | 178 | def cancel(self, consumer_tag): 179 | if not self._channel or consumer_tag not in self._consumers: 180 | return 181 | queue = self._consumers.pop(consumer_tag) 182 | self.channel.unsubscribe(queue) 183 | 184 | def close(self): 185 | for consumer_tag in self._consumers.keys(): 186 | self.cancel(consumer_tag) 187 | if self._channel: 188 | try: 189 | self._channel.disconnect() 190 | except socket.error: 191 | pass 192 | 193 | @property 194 | def channel(self): 195 | if not self._channel: 196 | # Sorry, but the python-stomp library needs one connection 197 | # for each channel. 198 | self._channel = self.establish_connection() 199 | return self._channel 200 | -------------------------------------------------------------------------------- /carrot/backends/queue.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | Backend for unit-tests, using the Python :mod:`Queue` module. 4 | 5 | """ 6 | from Queue import Queue 7 | from carrot.backends.base import BaseMessage, BaseBackend 8 | import time 9 | import itertools 10 | 11 | mqueue = Queue() 12 | 13 | 14 | class Message(BaseMessage): 15 | """Message received from the backend. 16 | 17 | See :class:`carrot.backends.base.BaseMessage`. 18 | 19 | """ 20 | 21 | 22 | class Backend(BaseBackend): 23 | """Backend using the Python :mod:`Queue` library. Usually only 24 | used while executing unit tests. 25 | 26 | Please not that this backend does not support queues, exchanges 27 | or routing keys, so *all messages will be sent to all consumers*. 28 | 29 | """ 30 | 31 | Message = Message 32 | 33 | def get(self, *args, **kwargs): 34 | """Get the next waiting message from the queue. 35 | 36 | :returns: A :class:`Message` instance, or ``None`` if there is 37 | no messages waiting. 38 | 39 | """ 40 | if not mqueue.qsize(): 41 | return None 42 | message_data, content_type, content_encoding = mqueue.get() 43 | return self.Message(backend=self, body=message_data, 44 | content_type=content_type, 45 | content_encoding=content_encoding) 46 | 47 | def establish_connection(self): 48 | # for drain_events 49 | return self 50 | 51 | def drain_events(self, timeout=None): 52 | message = self.get() 53 | if message: 54 | self.callback(message) 55 | else: 56 | time.sleep(0.1) 57 | 58 | def consume(self, limit=None): 59 | """Go into consume mode.""" 60 | for total_message_count in itertools.count(): 61 | if limit and total_message_count >= limit: 62 | raise StopIteration 63 | self.drain_events() 64 | yield True 65 | 66 | def declare_consumer(self, queue, no_ack, callback, consumer_tag, 67 | nowait=False): 68 | self.queue = queue 69 | self.no_ack = no_ack 70 | self.callback = callback 71 | self.consumer_tag = consumer_tag 72 | self.nowait = nowait 73 | 74 | def queue_purge(self, queue, **kwargs): 75 | """Discard all messages in the queue.""" 76 | qsize = mqueue.qsize() 77 | mqueue.queue.clear() 78 | return qsize 79 | 80 | def prepare_message(self, message_data, delivery_mode, 81 | content_type, content_encoding, **kwargs): 82 | """Prepare message for sending.""" 83 | return (message_data, content_type, content_encoding) 84 | 85 | def publish(self, message, exchange, routing_key, **kwargs): 86 | """Publish a message to the queue.""" 87 | mqueue.put(message) 88 | -------------------------------------------------------------------------------- /carrot/connection.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | Getting a connection to the AMQP server. 4 | 5 | """ 6 | import socket 7 | import warnings 8 | 9 | from collections import deque 10 | from copy import copy 11 | from Queue import Queue, Empty as QueueEmpty 12 | 13 | from amqplib.client_0_8.connection import AMQPConnectionException 14 | from carrot.backends import get_backend_cls 15 | from carrot.utils import retry_over_time 16 | 17 | DEFAULT_CONNECT_TIMEOUT = 5 # seconds 18 | SETTING_PREFIX = "BROKER" 19 | COMPAT_SETTING_PREFIX = "AMQP" 20 | ARG_TO_DJANGO_SETTING = { 21 | "hostname": "HOST", 22 | "userid": "USER", 23 | "password": "PASSWORD", 24 | "virtual_host": "VHOST", 25 | "port": "PORT", 26 | } 27 | SETTING_DEPRECATED_FMT = "Setting %s has been renamed to %s and is " \ 28 | "scheduled for removal in version 1.0." 29 | 30 | 31 | class BrokerConnection(object): 32 | """A network/socket connection to an AMQP message broker. 33 | 34 | :param hostname: see :attr:`hostname`. 35 | :param userid: see :attr:`userid`. 36 | :param password: see :attr:`password`. 37 | 38 | :keyword virtual_host: see :attr:`virtual_host`. 39 | :keyword port: see :attr:`port`. 40 | :keyword insist: see :attr:`insist`. 41 | :keyword connect_timeout: see :attr:`connect_timeout`. 42 | :keyword ssl: see :attr:`ssl`. 43 | 44 | .. attribute:: hostname 45 | 46 | The hostname to the AMQP server 47 | 48 | .. attribute:: userid 49 | 50 | A valid username used to authenticate to the server. 51 | 52 | .. attribute:: password 53 | 54 | The password used to authenticate to the server. 55 | 56 | .. attribute:: virtual_host 57 | 58 | The name of the virtual host to work with. This virtual host must 59 | exist on the server, and the user must have access to it. Consult 60 | your brokers manual for help with creating, and mapping 61 | users to virtual hosts. 62 | Default is ``"/"``. 63 | 64 | .. attribute:: port 65 | 66 | The port of the AMQP server. Default is ``5672`` (amqp). 67 | 68 | .. attribute:: insist 69 | 70 | Insist on connecting to a server. In a configuration with multiple 71 | load-sharing servers, the insist option tells the server that the 72 | client is insisting on a connection to the specified server. 73 | Default is ``False``. 74 | 75 | .. attribute:: connect_timeout 76 | 77 | The timeout in seconds before we give up connecting to the server. 78 | The default is no timeout. 79 | 80 | .. attribute:: ssl 81 | 82 | Use SSL to connect to the server. 83 | The default is ``False``. 84 | 85 | .. attribute:: backend_cls 86 | 87 | The messaging backend class used. Defaults to the ``pyamqplib`` 88 | backend. 89 | 90 | """ 91 | virtual_host = "/" 92 | port = None 93 | insist = False 94 | connect_timeout = DEFAULT_CONNECT_TIMEOUT 95 | ssl = False 96 | _closed = True 97 | backend_cls = None 98 | 99 | ConnectionException = AMQPConnectionException 100 | 101 | @property 102 | def host(self): 103 | """The host as a hostname/port pair separated by colon.""" 104 | return ":".join([self.hostname, str(self.port)]) 105 | 106 | def __init__(self, hostname=None, userid=None, password=None, 107 | virtual_host=None, port=None, pool=None, **kwargs): 108 | self.hostname = hostname 109 | self.userid = userid 110 | self.password = password 111 | self.virtual_host = virtual_host or self.virtual_host 112 | self.port = port or self.port 113 | self.insist = kwargs.get("insist", self.insist) 114 | self.pool = pool 115 | self.connect_timeout = kwargs.get("connect_timeout", 116 | self.connect_timeout) 117 | self.ssl = kwargs.get("ssl", self.ssl) 118 | self.backend_cls = (kwargs.get("backend_cls") or 119 | kwargs.get("transport")) 120 | self._closed = None 121 | self._connection = None 122 | 123 | def __copy__(self): 124 | return self.__class__(self.hostname, self.userid, self.password, 125 | self.virtual_host, self.port, 126 | insist=self.insist, 127 | connect_timeout=self.connect_timeout, 128 | ssl=self.ssl, 129 | backend_cls=self.backend_cls, 130 | pool=self.pool) 131 | 132 | @property 133 | def connection(self): 134 | if self._closed == True: 135 | return 136 | if not self._connection: 137 | self._connection = self._establish_connection() 138 | self._closed = False 139 | return self._connection 140 | 141 | def __enter__(self): 142 | return self 143 | 144 | def __exit__(self, e_type, e_value, e_trace): 145 | if e_type: 146 | raise e_type(e_value) 147 | self.close() 148 | 149 | def _establish_connection(self): 150 | return self.create_backend().establish_connection() 151 | 152 | def get_backend_cls(self): 153 | """Get the currently used backend class.""" 154 | backend_cls = self.backend_cls 155 | if not backend_cls or isinstance(backend_cls, basestring): 156 | backend_cls = get_backend_cls(backend_cls) 157 | return backend_cls 158 | 159 | def create_backend(self): 160 | """Create a new instance of the current backend in 161 | :attr:`backend_cls`.""" 162 | backend_cls = self.get_backend_cls() 163 | return backend_cls(connection=self) 164 | 165 | def channel(self): 166 | """For Kombu compatibility.""" 167 | return self.create_backend() 168 | 169 | def get_channel(self): 170 | """Request a new AMQP channel.""" 171 | return self.connection.channel() 172 | 173 | def connect(self): 174 | """Establish a connection to the AMQP server.""" 175 | self._closed = False 176 | return self.connection 177 | 178 | def drain_events(self, **kwargs): 179 | return self.connection.drain_events(**kwargs) 180 | 181 | def ensure_connection(self, errback=None, max_retries=None, 182 | interval_start=2, interval_step=2, interval_max=30): 183 | """Ensure we have a connection to the server. 184 | 185 | If not retry establishing the connection with the settings 186 | specified. 187 | 188 | :keyword errback: Optional callback called each time the connection 189 | can't be established. Arguments provided are the exception 190 | raised and the interval that will be slept ``(exc, interval)``. 191 | 192 | :keyword max_retries: Maximum number of times to retry. 193 | If this limit is exceeded the connection error will be re-raised. 194 | 195 | :keyword interval_start: The number of seconds we start sleeping for. 196 | :keyword interval_step: How many seconds added to the interval 197 | for each retry. 198 | :keyword interval_max: Maximum number of seconds to sleep between 199 | each retry. 200 | 201 | """ 202 | retry_over_time(self.connect, self.connection_errors, (), {}, 203 | errback, max_retries, 204 | interval_start, interval_step, interval_max) 205 | return self 206 | 207 | def close(self): 208 | """Close the currently open connection.""" 209 | try: 210 | if self._connection: 211 | backend = self.create_backend() 212 | backend.close_connection(self._connection) 213 | except socket.error: 214 | pass 215 | self._closed = True 216 | 217 | def release(self): 218 | if not self.pool: 219 | raise NotImplementedError( 220 | "Trying to release connection not part of a pool") 221 | self.pool.release(self) 222 | 223 | def info(self): 224 | """Get connection info.""" 225 | backend_cls = self.backend_cls or "amqplib" 226 | port = self.port or self.create_backend().default_port 227 | return {"hostname": self.hostname, 228 | "userid": self.userid, 229 | "password": self.password, 230 | "virtual_host": self.virtual_host, 231 | "port": port, 232 | "insist": self.insist, 233 | "ssl": self.ssl, 234 | "transport_cls": backend_cls, 235 | "backend_cls": backend_cls, 236 | "connect_timeout": self.connect_timeout} 237 | 238 | @property 239 | def connection_errors(self): 240 | """List of exceptions that may be raised by the connection.""" 241 | return self.create_backend().connection_errors 242 | 243 | @property 244 | def channel_errors(self): 245 | """List of exceptions that may be raised by the channel.""" 246 | return self.create_backend().channel_errors 247 | 248 | # For backwards compatability. 249 | AMQPConnection = BrokerConnection 250 | 251 | 252 | class ConnectionLimitExceeded(Exception): 253 | """The maximum number of pool connections has been exceeded.""" 254 | 255 | 256 | class ConnectionPool(object): 257 | 258 | def __init__(self, source_connection, min=2, max=None, preload=True): 259 | self.source_connection = source_connection 260 | self.min = min 261 | self.max = max 262 | self.preload = preload 263 | self.source_connection.pool = self 264 | 265 | self._connections = Queue() 266 | self._dirty = deque() 267 | 268 | self._connections.put(self.source_connection) 269 | for i in range(min - 1): 270 | self._connections.put_nowait(self._new_connection()) 271 | 272 | def acquire(self, block=False, timeout=None, connect_timeout=None): 273 | try: 274 | conn = self._connections.get(block=block, timeout=timeout) 275 | except QueueEmpty: 276 | conn = self._new_connection() 277 | self._dirty.append(conn) 278 | if connect_timeout is not None: 279 | conn.connect_timeout = connect_timeout 280 | return conn 281 | 282 | def release(self, connection): 283 | self._dirty.remove(connection) 284 | self._connections.put_nowait(connection) 285 | 286 | def _new_connection(self): 287 | if len(self._dirty) >= self.max: 288 | raise ConnectionLimitExceeded(self.max) 289 | return copy(self.source_connection) 290 | 291 | 292 | 293 | 294 | 295 | def get_django_conninfo(settings=None): 296 | # FIXME can't wait to remove this mess in 1.0 [askh] 297 | ci = {} 298 | if settings is None: 299 | from django.conf import settings 300 | 301 | ci["backend_cls"] = getattr(settings, "CARROT_BACKEND", None) 302 | 303 | for arg_name, setting_name in ARG_TO_DJANGO_SETTING.items(): 304 | setting = "%s_%s" % (SETTING_PREFIX, setting_name) 305 | compat_setting = "%s_%s" % (COMPAT_SETTING_PREFIX, setting_name) 306 | if hasattr(settings, setting): 307 | ci[arg_name] = getattr(settings, setting, None) 308 | elif hasattr(settings, compat_setting): 309 | ci[arg_name] = getattr(settings, compat_setting, None) 310 | warnings.warn(DeprecationWarning(SETTING_DEPRECATED_FMT % ( 311 | compat_setting, setting))) 312 | 313 | if "hostname" not in ci: 314 | if hasattr(settings, "AMQP_SERVER"): 315 | ci["hostname"] = settings.AMQP_SERVER 316 | warnings.warn(DeprecationWarning( 317 | "AMQP_SERVER has been renamed to BROKER_HOST and is" 318 | "scheduled for removal in version 1.0.")) 319 | 320 | return ci 321 | 322 | 323 | class DjangoBrokerConnection(BrokerConnection): 324 | """A version of :class:`BrokerConnection` that takes configuration 325 | from the Django ``settings.py`` module. 326 | 327 | :keyword hostname: The hostname of the AMQP server to connect to, 328 | if not provided this is taken from ``settings.BROKER_HOST``. 329 | 330 | :keyword userid: The username of the user to authenticate to the server 331 | as. If not provided this is taken from ``settings.BROKER_USER``. 332 | 333 | :keyword password: The users password. If not provided this is taken 334 | from ``settings.BROKER_PASSWORD``. 335 | 336 | :keyword virtual_host: The name of the virtual host to work with. 337 | This virtual host must exist on the server, and the user must 338 | have access to it. Consult your brokers manual for help with 339 | creating, and mapping users to virtual hosts. If not provided 340 | this is taken from ``settings.BROKER_VHOST``. 341 | 342 | :keyword port: The port the AMQP server is running on. If not provided 343 | this is taken from ``settings.BROKER_PORT``, or if that is not set, 344 | the default is ``5672`` (amqp). 345 | 346 | """ 347 | def __init__(self, *args, **kwargs): 348 | settings = kwargs.pop("settings", None) 349 | kwargs = dict(get_django_conninfo(settings), **kwargs) 350 | super(DjangoBrokerConnection, self).__init__(*args, **kwargs) 351 | 352 | # For backwards compatability. 353 | DjangoAMQPConnection = DjangoBrokerConnection 354 | -------------------------------------------------------------------------------- /carrot/serialization.py: -------------------------------------------------------------------------------- 1 | """ 2 | Centralized support for encoding/decoding of data structures. 3 | Requires a json library (`cjson`_, `simplejson`_, or `Python 2.6+`_). 4 | 5 | Pickle support is built-in. 6 | 7 | Optionally installs support for ``YAML`` if the `PyYAML`_ package 8 | is installed. 9 | 10 | Optionally installs support for `msgpack`_ if the `msgpack-python`_ 11 | package is installed. 12 | 13 | .. _`cjson`: http://pypi.python.org/pypi/python-cjson/ 14 | .. _`simplejson`: http://code.google.com/p/simplejson/ 15 | .. _`Python 2.6+`: http://docs.python.org/library/json.html 16 | .. _`PyYAML`: http://pyyaml.org/ 17 | .. _`msgpack`: http://msgpack.sourceforge.net/ 18 | .. _`msgpack-python`: http://pypi.python.org/pypi/msgpack-python/ 19 | 20 | """ 21 | 22 | import codecs 23 | 24 | __all__ = ['SerializerNotInstalled', 'registry'] 25 | 26 | 27 | class SerializerNotInstalled(StandardError): 28 | """Support for the requested serialization type is not installed""" 29 | 30 | 31 | class SerializerRegistry(object): 32 | """The registry keeps track of serialization methods.""" 33 | 34 | def __init__(self): 35 | self._encoders = {} 36 | self._decoders = {} 37 | self._default_encode = None 38 | self._default_content_type = None 39 | self._default_content_encoding = None 40 | 41 | def register(self, name, encoder, decoder, content_type, 42 | content_encoding='utf-8'): 43 | """Register a new encoder/decoder. 44 | 45 | :param name: A convenience name for the serialization method. 46 | 47 | :param encoder: A method that will be passed a python data structure 48 | and should return a string representing the serialized data. 49 | If ``None``, then only a decoder will be registered. Encoding 50 | will not be possible. 51 | 52 | :param decoder: A method that will be passed a string representing 53 | serialized data and should return a python data structure. 54 | If ``None``, then only an encoder will be registered. 55 | Decoding will not be possible. 56 | 57 | :param content_type: The mime-type describing the serialized 58 | structure. 59 | 60 | :param content_encoding: The content encoding (character set) that 61 | the :param:`decoder` method will be returning. Will usually be 62 | ``utf-8``, ``us-ascii``, or ``binary``. 63 | 64 | """ 65 | if encoder: 66 | self._encoders[name] = (content_type, content_encoding, encoder) 67 | if decoder: 68 | self._decoders[content_type] = decoder 69 | 70 | def _set_default_serializer(self, name): 71 | """ 72 | Set the default serialization method used by this library. 73 | 74 | :param name: The name of the registered serialization method. 75 | For example, ``json`` (default), ``pickle``, ``yaml``, 76 | or any custom methods registered using :meth:`register`. 77 | 78 | :raises SerializerNotInstalled: If the serialization method 79 | requested is not available. 80 | """ 81 | try: 82 | (self._default_content_type, self._default_content_encoding, 83 | self._default_encode) = self._encoders[name] 84 | except KeyError: 85 | raise SerializerNotInstalled( 86 | "No encoder installed for %s" % name) 87 | 88 | def encode(self, data, serializer=None): 89 | """ 90 | Serialize a data structure into a string suitable for sending 91 | as an AMQP message body. 92 | 93 | :param data: The message data to send. Can be a list, 94 | dictionary or a string. 95 | 96 | :keyword serializer: An optional string representing 97 | the serialization method you want the data marshalled 98 | into. (For example, ``json``, ``raw``, or ``pickle``). 99 | 100 | If ``None`` (default), then `JSON`_ will be used, unless 101 | ``data`` is a ``str`` or ``unicode`` object. In this 102 | latter case, no serialization occurs as it would be 103 | unnecessary. 104 | 105 | Note that if ``serializer`` is specified, then that 106 | serialization method will be used even if a ``str`` 107 | or ``unicode`` object is passed in. 108 | 109 | :returns: A three-item tuple containing the content type 110 | (e.g., ``application/json``), content encoding, (e.g., 111 | ``utf-8``) and a string containing the serialized 112 | data. 113 | 114 | :raises SerializerNotInstalled: If the serialization method 115 | requested is not available. 116 | """ 117 | if serializer == "raw": 118 | return raw_encode(data) 119 | if serializer and not self._encoders.get(serializer): 120 | raise SerializerNotInstalled( 121 | "No encoder installed for %s" % serializer) 122 | 123 | # If a raw string was sent, assume binary encoding 124 | # (it's likely either ASCII or a raw binary file, but 'binary' 125 | # charset will encompass both, even if not ideal. 126 | if not serializer and isinstance(data, str): 127 | # In Python 3+, this would be "bytes"; allow binary data to be 128 | # sent as a message without getting encoder errors 129 | return "application/data", "binary", data 130 | 131 | # For unicode objects, force it into a string 132 | if not serializer and isinstance(data, unicode): 133 | payload = data.encode("utf-8") 134 | return "text/plain", "utf-8", payload 135 | 136 | if serializer: 137 | content_type, content_encoding, encoder = \ 138 | self._encoders[serializer] 139 | else: 140 | encoder = self._default_encode 141 | content_type = self._default_content_type 142 | content_encoding = self._default_content_encoding 143 | 144 | payload = encoder(data) 145 | return content_type, content_encoding, payload 146 | 147 | def decode(self, data, content_type, content_encoding): 148 | """Deserialize a data stream as serialized using ``encode`` 149 | based on :param:`content_type`. 150 | 151 | :param data: The message data to deserialize. 152 | 153 | :param content_type: The content-type of the data. 154 | (e.g., ``application/json``). 155 | 156 | :param content_encoding: The content-encoding of the data. 157 | (e.g., ``utf-8``, ``binary``, or ``us-ascii``). 158 | 159 | :returns: The unserialized data. 160 | """ 161 | content_type = content_type or 'application/data' 162 | content_encoding = (content_encoding or 'utf-8').lower() 163 | 164 | # Don't decode 8-bit strings or unicode objects 165 | if content_encoding not in ('binary', 'ascii-8bit') and \ 166 | not isinstance(data, unicode): 167 | data = codecs.decode(data, content_encoding) 168 | 169 | try: 170 | decoder = self._decoders[content_type] 171 | except KeyError: 172 | return data 173 | 174 | return decoder(data) 175 | 176 | 177 | """ 178 | .. data:: registry 179 | 180 | Global registry of serializers/deserializers. 181 | 182 | """ 183 | registry = SerializerRegistry() 184 | 185 | """ 186 | .. function:: encode(data, serializer=default_serializer) 187 | 188 | Encode data using the registry's default encoder. 189 | 190 | """ 191 | encode = registry.encode 192 | 193 | """ 194 | .. function:: decode(data, content_type, content_encoding): 195 | 196 | Decode data using the registry's default decoder. 197 | 198 | """ 199 | decode = registry.decode 200 | 201 | 202 | def raw_encode(data): 203 | """Special case serializer.""" 204 | content_type = 'application/data' 205 | payload = data 206 | if isinstance(payload, unicode): 207 | content_encoding = 'utf-8' 208 | payload = payload.encode(content_encoding) 209 | else: 210 | content_encoding = 'binary' 211 | return content_type, content_encoding, payload 212 | 213 | 214 | def register_json(): 215 | """Register a encoder/decoder for JSON serialization.""" 216 | from anyjson import serialize as json_serialize 217 | from anyjson import deserialize as json_deserialize 218 | 219 | registry.register('json', json_serialize, json_deserialize, 220 | content_type='application/json', 221 | content_encoding='utf-8') 222 | 223 | 224 | def register_yaml(): 225 | """Register a encoder/decoder for YAML serialization. 226 | 227 | It is slower than JSON, but allows for more data types 228 | to be serialized. Useful if you need to send data such as dates""" 229 | try: 230 | import yaml 231 | registry.register('yaml', yaml.safe_dump, yaml.safe_load, 232 | content_type='application/x-yaml', 233 | content_encoding='utf-8') 234 | except ImportError: 235 | 236 | def not_available(*args, **kwargs): 237 | """In case a client receives a yaml message, but yaml 238 | isn't installed.""" 239 | raise SerializerNotInstalled( 240 | "No decoder installed for YAML. Install the PyYAML library") 241 | registry.register('yaml', None, not_available, 'application/x-yaml') 242 | 243 | 244 | def register_pickle(): 245 | """The fastest serialization method, but restricts 246 | you to python clients.""" 247 | import cPickle 248 | registry.register('pickle', cPickle.dumps, cPickle.loads, 249 | content_type='application/x-python-serialize', 250 | content_encoding='binary') 251 | 252 | 253 | def register_msgpack(): 254 | """See http://msgpack.sourceforge.net/""" 255 | try: 256 | import msgpack 257 | registry.register('msgpack', msgpack.packs, msgpack.unpacks, 258 | content_type='application/x-msgpack', 259 | content_encoding='binary') 260 | except ImportError: 261 | 262 | def not_available(*args, **kwargs): 263 | """In case a client receives a msgpack message, but yaml 264 | isn't installed.""" 265 | raise SerializerNotInstalled( 266 | "No decoder installed for msgpack. " 267 | "Install the msgpack library") 268 | registry.register('msgpack', None, not_available, 269 | 'application/x-msgpack') 270 | 271 | # Register the base serialization methods. 272 | register_json() 273 | register_pickle() 274 | register_yaml() 275 | register_msgpack() 276 | 277 | # JSON is assumed to always be available, so is the default. 278 | # (this matches the historical use of carrot.) 279 | registry._set_default_serializer('json') 280 | -------------------------------------------------------------------------------- /carrot/utils.py: -------------------------------------------------------------------------------- 1 | from uuid import UUID, uuid4, _uuid_generate_random 2 | try: 3 | import ctypes 4 | except ImportError: 5 | ctypes = None 6 | 7 | 8 | def gen_unique_id(): 9 | """Generate a unique id, having - hopefully - a very small chance of 10 | collission. 11 | 12 | For now this is provided by :func:`uuid.uuid4`. 13 | """ 14 | # Workaround for http://bugs.python.org/issue4607 15 | if ctypes and _uuid_generate_random: 16 | buffer = ctypes.create_string_buffer(16) 17 | _uuid_generate_random(buffer) 18 | return str(UUID(bytes=buffer.raw)) 19 | return str(uuid4()) 20 | 21 | 22 | def _compat_rl_partition(S, sep, direction=None): 23 | if direction is None: 24 | direction = S.split 25 | items = direction(sep, 1) 26 | if len(items) == 1: 27 | return items[0], sep, '' 28 | return items[0], sep, items[1] 29 | 30 | 31 | def _compat_partition(S, sep): 32 | """``partition(S, sep) -> (head, sep, tail)`` 33 | 34 | Search for the separator ``sep`` in ``S``, and return the part before 35 | it, the separator itself, and the part after it. If the separator is not 36 | found, return ``S`` and two empty strings. 37 | 38 | """ 39 | return _compat_rl_partition(S, sep, direction=S.split) 40 | 41 | 42 | def _compat_rpartition(S, sep): 43 | """``rpartition(S, sep) -> (tail, sep, head)`` 44 | 45 | Search for the separator ``sep`` in ``S``, starting at the end of ``S``, 46 | and return the part before it, the separator itself, and the part 47 | after it. If the separator is not found, return two empty 48 | strings and ``S``. 49 | 50 | """ 51 | return _compat_rl_partition(S, sep, direction=S.rsplit) 52 | 53 | 54 | 55 | def partition(S, sep): 56 | if hasattr(S, 'partition'): 57 | return S.partition(sep) 58 | else: # Python <= 2.4: 59 | return _compat_partition(S, sep) 60 | 61 | 62 | def rpartition(S, sep): 63 | if hasattr(S, 'rpartition'): 64 | return S.rpartition(sep) 65 | else: # Python <= 2.4: 66 | return _compat_rpartition(S, sep) 67 | 68 | 69 | def repeatlast(it): 70 | """Iterate over all elements in the iterator, and when its exhausted 71 | yield the last value infinitely.""" 72 | for item in it: 73 | yield item 74 | while 1: # pragma: no cover 75 | yield item 76 | 77 | 78 | def retry_over_time(fun, catch, args=[], kwargs={}, errback=None, 79 | max_retries=None, interval_start=2, interval_step=2, interval_max=30): 80 | """Retry the function over and over until max retries is exceeded. 81 | 82 | For each retry we sleep a for a while before we try again, this interval 83 | is increased for every retry until the max seconds is reached. 84 | 85 | :param fun: The function to try 86 | :param catch: Exceptions to catch, can be either tuple or a single 87 | exception class. 88 | :keyword args: Positional arguments passed on to the function. 89 | :keyword kwargs: Keyword arguments passed on to the function. 90 | :keyword errback: Callback for when an exception in ``catch`` is raised. 91 | The callback must take two arguments: ``exc`` and ``interval``, where 92 | ``exc`` is the exception instance, and ``interval`` is the time in 93 | seconds to sleep next.. 94 | :keyword max_retries: Maximum number of retries before we give up. 95 | If this is not set, we will retry forever. 96 | :keyword interval_start: How long (in seconds) we start sleeping between 97 | retries. 98 | :keyword interval_step: By how much the interval is increased for each 99 | retry. 100 | :keyword interval_max: Maximum number of seconds to sleep between retries. 101 | 102 | """ 103 | retries = 0 104 | interval_range = xrange(interval_start, 105 | interval_max + interval_start, 106 | interval_step) 107 | 108 | for retries, interval in enumerate(repeatlast(interval_range)): 109 | try: 110 | retval = fun(*args, **kwargs) 111 | except catch, exc: 112 | if max_retries and retries > max_retries: 113 | raise 114 | if errback: 115 | errback(exc, interval) 116 | sleep(interval) 117 | else: 118 | return retval 119 | -------------------------------------------------------------------------------- /contrib/doc2ghpages: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | git checkout master 4 | (cd docs; 5 | rm -rf .build; 6 | make html; 7 | (cd .build/html; 8 | sphinx-to-github;)) 9 | git checkout gh-pages 10 | cp -r docs/.build/html/* . 11 | git commit . -m "Autogenerated documentation for github." 12 | git push origin gh-pages 13 | git checkout master 14 | -------------------------------------------------------------------------------- /contrib/requirements/default.txt: -------------------------------------------------------------------------------- 1 | anyjson 2 | amqplib>=0.6 3 | -------------------------------------------------------------------------------- /contrib/requirements/test.txt: -------------------------------------------------------------------------------- 1 | nose 2 | nose-cover3 3 | coverage>=3.0 4 | simplejson 5 | PyYAML 6 | msgpack-python 7 | 8 | -------------------------------------------------------------------------------- /docs/.static/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ask/carrot/5889a25cd2e274642071c9bba39772f4b3e3d9da/docs/.static/.keep -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | 9 | # Internal variables. 10 | PAPEROPT_a4 = -D latex_paper_size=a4 11 | PAPEROPT_letter = -D latex_paper_size=letter 12 | ALLSPHINXOPTS = -d .build/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 13 | 14 | .PHONY: help clean html web pickle htmlhelp latex changes linkcheck 15 | 16 | help: 17 | @echo "Please use \`make ' where is one of" 18 | @echo " html to make standalone HTML files" 19 | @echo " pickle to make pickle files" 20 | @echo " json to make JSON files" 21 | @echo " htmlhelp to make HTML files and a HTML help project" 22 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 23 | @echo " changes to make an overview over all changed/added/deprecated items" 24 | @echo " linkcheck to check all external links for integrity" 25 | 26 | clean: 27 | -rm -rf .build/* 28 | 29 | html: 30 | mkdir -p .build/html .build/doctrees 31 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) .build/html 32 | @echo 33 | @echo "Build finished. The HTML pages are in .build/html." 34 | 35 | pickle: 36 | mkdir -p .build/pickle .build/doctrees 37 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) .build/pickle 38 | @echo 39 | @echo "Build finished; now you can process the pickle files." 40 | 41 | web: pickle 42 | 43 | json: 44 | mkdir -p .build/json .build/doctrees 45 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) .build/json 46 | @echo 47 | @echo "Build finished; now you can process the JSON files." 48 | 49 | htmlhelp: 50 | mkdir -p .build/htmlhelp .build/doctrees 51 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) .build/htmlhelp 52 | @echo 53 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 54 | ".hhp project file in .build/htmlhelp." 55 | 56 | latex: 57 | mkdir -p .build/latex .build/doctrees 58 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) .build/latex 59 | @echo 60 | @echo "Build finished; the LaTeX files are in .build/latex." 61 | @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ 62 | "run these through (pdf)latex." 63 | 64 | changes: 65 | mkdir -p .build/changes .build/doctrees 66 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) .build/changes 67 | @echo 68 | @echo "The overview file is in .build/changes." 69 | 70 | linkcheck: 71 | mkdir -p .build/linkcheck .build/doctrees 72 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) .build/linkcheck 73 | @echo 74 | @echo "Link check complete; look for any errors in the above output " \ 75 | "or in .build/linkcheck/output.txt." 76 | -------------------------------------------------------------------------------- /docs/_ext/applyxrefs.py: -------------------------------------------------------------------------------- 1 | """Adds xref targets to the top of files.""" 2 | 3 | import sys 4 | import os 5 | 6 | testing = False 7 | 8 | DONT_TOUCH = ( 9 | './index.txt', 10 | ) 11 | 12 | 13 | def target_name(fn): 14 | if fn.endswith('.txt'): 15 | fn = fn[:-4] 16 | return '_' + fn.lstrip('./').replace('/', '-') 17 | 18 | 19 | def process_file(fn, lines): 20 | lines.insert(0, '\n') 21 | lines.insert(0, '.. %s:\n' % target_name(fn)) 22 | try: 23 | f = open(fn, 'w') 24 | except IOError: 25 | print("Can't open %s for writing. Not touching it." % fn) 26 | return 27 | try: 28 | f.writelines(lines) 29 | except IOError: 30 | print("Can't write to %s. Not touching it." % fn) 31 | finally: 32 | f.close() 33 | 34 | 35 | def has_target(fn): 36 | try: 37 | f = open(fn, 'r') 38 | except IOError: 39 | print("Can't open %s. Not touching it." % fn) 40 | return (True, None) 41 | readok = True 42 | try: 43 | lines = f.readlines() 44 | except IOError: 45 | print("Can't read %s. Not touching it." % fn) 46 | readok = False 47 | finally: 48 | f.close() 49 | if not readok: 50 | return (True, None) 51 | 52 | #print fn, len(lines) 53 | if len(lines) < 1: 54 | print("Not touching empty file %s." % fn) 55 | return (True, None) 56 | if lines[0].startswith('.. _'): 57 | return (True, None) 58 | return (False, lines) 59 | 60 | 61 | def main(argv=None): 62 | if argv is None: 63 | argv = sys.argv 64 | 65 | if len(argv) == 1: 66 | argv.extend('.') 67 | 68 | files = [] 69 | for root in argv[1:]: 70 | for (dirpath, dirnames, filenames) in os.walk(root): 71 | files.extend([(dirpath, f) for f in filenames]) 72 | files.sort() 73 | files = [os.path.join(p, fn) for p, fn in files if fn.endswith('.txt')] 74 | #print files 75 | 76 | for fn in files: 77 | if fn in DONT_TOUCH: 78 | print("Skipping blacklisted file %s." % fn) 79 | continue 80 | 81 | target_found, lines = has_target(fn) 82 | if not target_found: 83 | if testing: 84 | print '%s: %s' % (fn, lines[0]), 85 | else: 86 | print "Adding xref to %s" % fn 87 | process_file(fn, lines) 88 | else: 89 | print "Skipping %s: already has a xref" % fn 90 | 91 | if __name__ == '__main__': 92 | sys.exit(main()) 93 | -------------------------------------------------------------------------------- /docs/_ext/literals_to_xrefs.py: -------------------------------------------------------------------------------- 1 | """ 2 | Runs through a reST file looking for old-style literals, and helps replace them 3 | with new-style references. 4 | """ 5 | 6 | import re 7 | import sys 8 | import shelve 9 | 10 | refre = re.compile(r'``([^`\s]+?)``') 11 | 12 | ROLES = ( 13 | 'attr', 14 | 'class', 15 | "djadmin", 16 | 'data', 17 | 'exc', 18 | 'file', 19 | 'func', 20 | 'lookup', 21 | 'meth', 22 | 'mod', 23 | "djadminopt", 24 | "ref", 25 | "setting", 26 | "term", 27 | "tfilter", 28 | "ttag", 29 | 30 | # special 31 | "skip", 32 | ) 33 | 34 | ALWAYS_SKIP = [ 35 | "NULL", 36 | "True", 37 | "False", 38 | ] 39 | 40 | 41 | def fixliterals(fname): 42 | data = open(fname).read() 43 | 44 | last = 0 45 | new = [] 46 | storage = shelve.open("/tmp/literals_to_xref.shelve") 47 | lastvalues = storage.get("lastvalues", {}) 48 | 49 | for m in refre.finditer(data): 50 | 51 | new.append(data[last:m.start()]) 52 | last = m.end() 53 | 54 | line_start = data.rfind("\n", 0, m.start()) 55 | line_end = data.find("\n", m.end()) 56 | prev_start = data.rfind("\n", 0, line_start) 57 | next_end = data.find("\n", line_end + 1) 58 | 59 | # Skip always-skip stuff 60 | if m.group(1) in ALWAYS_SKIP: 61 | new.append(m.group(0)) 62 | continue 63 | 64 | # skip when the next line is a title 65 | next_line = data[m.end():next_end].strip() 66 | if next_line[0] in "!-/:-@[-`{-~" and \ 67 | all(c == next_line[0] for c in next_line): 68 | new.append(m.group(0)) 69 | continue 70 | 71 | sys.stdout.write("\n"+"-"*80+"\n") 72 | sys.stdout.write(data[prev_start+1:m.start()]) 73 | sys.stdout.write(colorize(m.group(0), fg="red")) 74 | sys.stdout.write(data[m.end():next_end]) 75 | sys.stdout.write("\n\n") 76 | 77 | replace_type = None 78 | while replace_type is None: 79 | replace_type = raw_input( 80 | colorize("Replace role: ", fg="yellow")).strip().lower() 81 | if replace_type and replace_type not in ROLES: 82 | replace_type = None 83 | 84 | if replace_type == "": 85 | new.append(m.group(0)) 86 | continue 87 | 88 | if replace_type == "skip": 89 | new.append(m.group(0)) 90 | ALWAYS_SKIP.append(m.group(1)) 91 | continue 92 | 93 | default = lastvalues.get(m.group(1), m.group(1)) 94 | if default.endswith("()") and \ 95 | replace_type in ("class", "func", "meth"): 96 | default = default[:-2] 97 | replace_value = raw_input( 98 | colorize("Text [", fg="yellow") + default + \ 99 | colorize("]: ", fg="yellow")).strip() 100 | if not replace_value: 101 | replace_value = default 102 | new.append(":%s:`%s`" % (replace_type, replace_value)) 103 | lastvalues[m.group(1)] = replace_value 104 | 105 | new.append(data[last:]) 106 | open(fname, "w").write("".join(new)) 107 | 108 | storage["lastvalues"] = lastvalues 109 | storage.close() 110 | 111 | 112 | def colorize(text='', opts=(), **kwargs): 113 | """ 114 | Returns your text, enclosed in ANSI graphics codes. 115 | 116 | Depends on the keyword arguments 'fg' and 'bg', and the contents of 117 | the opts tuple/list. 118 | 119 | Returns the RESET code if no parameters are given. 120 | 121 | Valid colors: 122 | 'black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white' 123 | 124 | Valid options: 125 | 'bold' 126 | 'underscore' 127 | 'blink' 128 | 'reverse' 129 | 'conceal' 130 | 'noreset' - string will not be auto-terminated with the RESET code 131 | 132 | Examples: 133 | colorize('hello', fg='red', bg='blue', opts=('blink',)) 134 | colorize() 135 | colorize('goodbye', opts=('underscore',)) 136 | print colorize('first line', fg='red', opts=('noreset',)) 137 | print 'this should be red too' 138 | print colorize('and so should this') 139 | print 'this should not be red' 140 | """ 141 | color_names = ('black', 'red', 'green', 'yellow', 142 | 'blue', 'magenta', 'cyan', 'white') 143 | foreground = dict([(color_names[x], '3%s' % x) for x in range(8)]) 144 | background = dict([(color_names[x], '4%s' % x) for x in range(8)]) 145 | 146 | RESET = '0' 147 | opt_dict = {'bold': '1', 148 | 'underscore': '4', 149 | 'blink': '5', 150 | 'reverse': '7', 151 | 'conceal': '8'} 152 | 153 | text = str(text) 154 | code_list = [] 155 | if text == '' and len(opts) == 1 and opts[0] == 'reset': 156 | return '\x1b[%sm' % RESET 157 | for k, v in kwargs.iteritems(): 158 | if k == 'fg': 159 | code_list.append(foreground[v]) 160 | elif k == 'bg': 161 | code_list.append(background[v]) 162 | for o in opts: 163 | if o in opt_dict: 164 | code_list.append(opt_dict[o]) 165 | if 'noreset' not in opts: 166 | text = text + '\x1b[%sm' % RESET 167 | return ('\x1b[%sm' % ';'.join(code_list)) + text 168 | 169 | if __name__ == '__main__': 170 | try: 171 | fixliterals(sys.argv[1]) 172 | except (KeyboardInterrupt, SystemExit): 173 | print 174 | -------------------------------------------------------------------------------- /docs/_theme/agogo/layout.html: -------------------------------------------------------------------------------- 1 | {%- block doctype -%} 2 | 4 | {%- endblock %} 5 | {%- set reldelim1 = reldelim1 is not defined and ' »' or reldelim1 %} 6 | {%- set reldelim2 = reldelim2 is not defined and ' |' or reldelim2 %} 7 | 8 | {%- macro relbar() %} 9 |
10 |

{{ _('Navigation') }}

11 |
    12 | {%- for rellink in rellinks %} 13 |
  • 14 | {{ rellink[3] }} 16 | {%- if not loop.first %}{{ reldelim2 }}{% endif %}
    • 17 | {%- endfor %} 18 | {%- block rootrellink %} 19 |
  • {{ shorttitle|e }}{{ reldelim1 }}
    • 20 | {%- endblock %} 21 | {%- for parent in parents %} 22 |
  • {{ parent.title }}{{ reldelim1 }}
    • 23 | {%- endfor %} 24 | {%- block relbaritems %} {% endblock %} 25 |
26 |
27 | {%- endmacro %} 28 | 29 | {%- macro sidebar() %} 30 | {%- if not embedded %}{% if not theme_nosidebar|tobool %} 31 |
32 |
33 | {%- block sidebarlogo %} 34 | {%- if logo %} 35 | 38 | {%- endif %} 39 | {%- endblock %} 40 | {%- block sidebartoc %} 41 | {%- if display_toc %} 42 |

{{ _('Table Of Contents') }}

43 | {{ toc }} 44 | {%- endif %} 45 | {%- endblock %} 46 | {%- block sidebarrel %} 47 | {%- if prev %} 48 |

{{ _('Previous topic') }}

49 |

{{ prev.title }}

51 | {%- endif %} 52 | {%- if next %} 53 |

{{ _('Next topic') }}

54 |

{{ next.title }}

56 | {%- endif %} 57 | {%- endblock %} 58 | {%- block sidebarsourcelink %} 59 | {%- if show_source and has_source and sourcename %} 60 |

{{ _('This Page') }}

61 | 65 | {%- endif %} 66 | {%- endblock %} 67 | {%- if customsidebar %} 68 | {% include customsidebar %} 69 | {%- endif %} 70 | {%- block sidebarsearch %} 71 | {%- if pagename != "search" %} 72 | 84 | 85 | {%- endif %} 86 | {%- endblock %} 87 |
88 |
89 | {%- endif %}{% endif %} 90 | {%- endmacro %} 91 | 92 | 93 | 94 | 95 | {{ metatags }} 96 | {%- if not embedded %} 97 | {%- set titlesuffix = " — "|safe + docstitle|e %} 98 | {%- else %} 99 | {%- set titlesuffix = "" %} 100 | {%- endif %} 101 | {{ title|striptags }}{{ titlesuffix }} 102 | 103 | 104 | {%- if not embedded %} 105 | 114 | {%- for scriptfile in script_files %} 115 | 116 | {%- endfor %} 117 | {%- if use_opensearch %} 118 | 121 | {%- endif %} 122 | {%- if favicon %} 123 | 124 | {%- endif %} 125 | {%- endif %} 126 | {%- block linktags %} 127 | {%- if hasdoc('about') %} 128 | 129 | {%- endif %} 130 | {%- if hasdoc('genindex') %} 131 | 132 | {%- endif %} 133 | {%- if hasdoc('search') %} 134 | 135 | {%- endif %} 136 | {%- if hasdoc('copyright') %} 137 | 138 | {%- endif %} 139 | 140 | {%- if parents %} 141 | 142 | {%- endif %} 143 | {%- if next %} 144 | 145 | {%- endif %} 146 | {%- if prev %} 147 | 148 | {%- endif %} 149 | {%- endblock %} 150 | {%- block extrahead %} {% endblock %} 151 | 152 | 153 | 154 |
155 |
156 |

{{ shorttitle|e }}

157 |
158 | {%- for rellink in rellinks %} 159 | {{ rellink[3] }} 161 | {%- if not loop.last %}{{ reldelim2 }}{% endif %} 162 | {%- endfor %} 163 |
164 |
165 |
166 | 167 |
168 |
169 |
170 | {%- block document %} 171 |
172 | {%- if not embedded %}{% if not theme_nosidebar|tobool %} 173 |
174 | {%- endif %}{% endif %} 175 |
176 | {% block body %} {% endblock %} 177 |
178 | {%- if not embedded %}{% if not theme_nosidebar|tobool %} 179 |
180 | {%- endif %}{% endif %} 181 |
182 | {%- endblock %} 183 |
184 |
185 |

{{ _('Contents') }}

186 | {{ toctree() }} 187 |

{{ _('Search') }}

188 |
189 | 190 | 191 | 192 | 193 |
194 |

195 | {{ _('Enter search terms or a module, class or function name.') }} 196 |

197 |
198 |
199 |
200 |
201 | 202 |
203 | {%- block footer %} 204 |
205 |
206 | {%- for rellink in rellinks %} 207 | {{ rellink[3] }} 209 | {%- if not loop.last %}{{ reldelim2 }}{% endif %} 210 | {%- endfor %} 211 | {%- if show_source and has_source and sourcename %} 212 |
213 | {{ _('Show Source') }} 215 | {%- endif %} 216 |
217 | 218 |
219 | {%- if hasdoc('copyright') %} 220 | {% trans path=pathto('copyright'), copyright=copyright|e %}© Copyright {{ copyright }}.{% endtrans %} 221 | {%- else %} 222 | {% trans copyright=copyright|e %}© Copyright {{ copyright }}.{% endtrans %} 223 | {%- endif %} 224 | {%- if last_updated %} 225 | {% trans last_updated=last_updated|e %}Last updated on {{ last_updated }}.{% endtrans %} 226 | {%- endif %} 227 | {%- if show_sphinx %}
228 | {% trans sphinx_version=sphinx_version|e %}Created using Sphinx {{ sphinx_version }}.{% endtrans %} 229 | {%- endif %} 230 |
231 |
232 |
233 | {%- endblock %} 234 |
235 | 236 | 237 | 238 | -------------------------------------------------------------------------------- /docs/_theme/agogo/static/agogo.css_t: -------------------------------------------------------------------------------- 1 | * { 2 | margin: 0px; 3 | padding: 0px; 4 | } 5 | 6 | body { 7 | font-family: {{ theme_bodyfont }}; 8 | line-height: 1.4em; 9 | font-size: 14px; 10 | color: black; 11 | background-color: {{ theme_bgcolor }}; 12 | } 13 | 14 | 15 | /* Page layout */ 16 | 17 | div.header, div.content, div.footer { 18 | width: {{ theme_pagewidth }}; 19 | margin-left: auto; 20 | margin-right: auto; 21 | } 22 | 23 | div.header-wrapper { 24 | background: {{ theme_headerbg }}; 25 | border-bottom: 3px solid #2e3436; 26 | } 27 | 28 | 29 | /* Default body styles */ 30 | a { 31 | text-decoration: none; 32 | color: {{ theme_linkcolor }}; 33 | } 34 | 35 | .clearer { 36 | clear: both; 37 | } 38 | 39 | .left { 40 | float: left; 41 | } 42 | 43 | .right { 44 | float: right; 45 | } 46 | 47 | h1, h2, h3, h4 { 48 | font-family: {{ theme_headerfont }}; 49 | font-weight: normal; 50 | color: {{ theme_headercolor2 }}; 51 | margin-bottom: .8em; 52 | } 53 | 54 | h1 { 55 | color: {{ theme_headercolor1 }}; 56 | } 57 | 58 | h2 { 59 | padding-bottom: .5em; 60 | border-bottom: 1px solid {{ theme_headercolor2 }}; 61 | } 62 | 63 | a.headerlink { 64 | visibility: hidden; 65 | color: #dddddd; 66 | padding-left: .3em; 67 | } 68 | 69 | h1:hover > a.headerlink, 70 | h2:hover > a.headerlink, 71 | h3:hover > a.headerlink, 72 | h4:hover > a.headerlink, 73 | h5:hover > a.headerlink, 74 | h6:hover > a.headerlink, 75 | dt:hover > a.headerlink { 76 | visibility: visible; 77 | } 78 | 79 | 80 | 81 | /* Header */ 82 | 83 | div.header { 84 | padding-top: 10px; 85 | padding-bottom: 10px; 86 | } 87 | 88 | div.header h1 { 89 | font-family: {{ theme_headerfont }}; 90 | font-weight: normal; 91 | font-size: 160%; 92 | letter-spacing: .08em; 93 | } 94 | 95 | div.header h1 a { 96 | color: white; 97 | } 98 | 99 | div.header div.rel { 100 | margin-top: 1em; 101 | } 102 | 103 | div.header div.rel a { 104 | color: {{ theme_headerlinkcolor }}; 105 | letter-spacing: .1em; 106 | text-transform: uppercase; 107 | } 108 | 109 | 110 | /* Content */ 111 | div.content-wrapper { 112 | background-color: white; 113 | padding-top: 20px; 114 | padding-bottom: 20px; 115 | } 116 | 117 | div.document { 118 | width: {{ theme_documentwidth }}; 119 | float: left; 120 | } 121 | 122 | div.body { 123 | padding-right: 2em; 124 | text-align: justify; 125 | } 126 | 127 | div.document ul { 128 | margin-left: 1.2em; 129 | list-style-type: square; 130 | } 131 | 132 | div.document dd { 133 | margin-left: 1.2em; 134 | margin-top: .4em; 135 | margin-bottom: 1em; 136 | } 137 | 138 | div.document .section { 139 | margin-top: 1.7em; 140 | } 141 | div.document .section:first-child { 142 | margin-top: 0px; 143 | } 144 | 145 | div.document div.highlight { 146 | padding: 3px; 147 | background-color: #eeeeec; 148 | border-top: 2px solid #dddddd; 149 | border-bottom: 2px solid #dddddd; 150 | margin-top: .8em; 151 | margin-bottom: .8em; 152 | } 153 | 154 | div.document h2 { 155 | margin-top: .7em; 156 | } 157 | 158 | div.document p { 159 | margin-bottom: .5em; 160 | } 161 | 162 | div.document li.toctree-l1 { 163 | margin-bottom: 1em; 164 | } 165 | 166 | div.document .descname { 167 | font-weight: bold; 168 | } 169 | 170 | div.document .docutils.literal { 171 | background-color: #eeeeec; 172 | padding: 1px; 173 | } 174 | 175 | div.document .docutils.xref.literal { 176 | background-color: transparent; 177 | padding: 0px; 178 | } 179 | 180 | 181 | /* Sidebar */ 182 | 183 | div.sidebar { 184 | width: {{ theme_sidebarwidth }}; 185 | float: right; 186 | font-size: .9em; 187 | } 188 | 189 | div.sidebar h3 { 190 | color: #2e3436; 191 | text-transform: uppercase; 192 | font-size: 130%; 193 | letter-spacing: .1em; 194 | } 195 | 196 | div.sidebar ul { 197 | list-style-type: none; 198 | } 199 | 200 | div.sidebar li.toctree-l1 a { 201 | display: block; 202 | padding: 1px; 203 | border: 1px solid #dddddd; 204 | background-color: #eeeeec; 205 | margin-bottom: .4em; 206 | padding-left: 3px; 207 | color: #2e3436; 208 | } 209 | 210 | div.sidebar li.toctree-l2 a { 211 | background-color: transparent; 212 | border: none; 213 | border-bottom: 1px solid #dddddd; 214 | } 215 | 216 | div.sidebar li.toctree-l2:last-child a { 217 | border-bottom: none; 218 | } 219 | 220 | div.sidebar li.toctree-l1.current a { 221 | border-right: 5px solid {{ theme_headerlinkcolor }}; 222 | } 223 | 224 | div.sidebar li.toctree-l1.current li.toctree-l2 a { 225 | border-right: none; 226 | } 227 | 228 | 229 | /* Footer */ 230 | 231 | div.footer-wrapper { 232 | background: {{ theme_footerbg }}; 233 | border-top: 4px solid #babdb6; 234 | padding-top: 10px; 235 | padding-bottom: 10px; 236 | min-height: 80px; 237 | } 238 | 239 | div.footer, div.footer a { 240 | color: #888a85; 241 | } 242 | 243 | div.footer .right { 244 | text-align: right; 245 | } 246 | 247 | div.footer .left { 248 | text-transform: uppercase; 249 | } 250 | 251 | 252 | /* Styles copied form basic theme */ 253 | 254 | /* -- search page ----------------------------------------------------------- */ 255 | 256 | ul.search { 257 | margin: 10px 0 0 20px; 258 | padding: 0; 259 | } 260 | 261 | ul.search li { 262 | padding: 5px 0 5px 20px; 263 | background-image: url(file.png); 264 | background-repeat: no-repeat; 265 | background-position: 0 7px; 266 | } 267 | 268 | ul.search li a { 269 | font-weight: bold; 270 | } 271 | 272 | ul.search li div.context { 273 | color: #888; 274 | margin: 2px 0 0 30px; 275 | text-align: left; 276 | } 277 | 278 | ul.keywordmatches li.goodmatch a { 279 | font-weight: bold; 280 | } 281 | 282 | /* -- index page ------------------------------------------------------------ */ 283 | 284 | table.contentstable { 285 | width: 90%; 286 | } 287 | 288 | table.contentstable p.biglink { 289 | line-height: 150%; 290 | } 291 | 292 | a.biglink { 293 | font-size: 1.3em; 294 | } 295 | 296 | span.linkdescr { 297 | font-style: italic; 298 | padding-top: 5px; 299 | font-size: 90%; 300 | } 301 | 302 | /* -- general index --------------------------------------------------------- */ 303 | 304 | table.indextable td { 305 | text-align: left; 306 | vertical-align: top; 307 | } 308 | 309 | table.indextable dl, table.indextable dd { 310 | margin-top: 0; 311 | margin-bottom: 0; 312 | } 313 | 314 | table.indextable tr.pcap { 315 | height: 10px; 316 | } 317 | 318 | table.indextable tr.cap { 319 | margin-top: 10px; 320 | background-color: #f2f2f2; 321 | } 322 | 323 | img.toggler { 324 | margin-right: 3px; 325 | margin-top: 3px; 326 | cursor: pointer; 327 | } 328 | 329 | 330 | -------------------------------------------------------------------------------- /docs/_theme/agogo/static/bgfooter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ask/carrot/5889a25cd2e274642071c9bba39772f4b3e3d9da/docs/_theme/agogo/static/bgfooter.png -------------------------------------------------------------------------------- /docs/_theme/agogo/static/bgtop.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ask/carrot/5889a25cd2e274642071c9bba39772f4b3e3d9da/docs/_theme/agogo/static/bgtop.png -------------------------------------------------------------------------------- /docs/_theme/agogo/theme.conf: -------------------------------------------------------------------------------- 1 | [theme] 2 | inherit = basic 3 | stylesheet = agogo.css 4 | pygments_style = tango 5 | 6 | [options] 7 | bodyfont = "Verdana", Arial, sans-serif 8 | headerfont = "Georgia", "Times New Roman", serif 9 | pagewidth = 70em 10 | documentwidth = 50em 11 | sidebarwidth = 20em 12 | bgcolor = #eeeeec 13 | headerbg = url(bgtop.png) top left repeat-x 14 | footerbg = url(bgfooter.png) top left repeat-x 15 | linkcolor = #ce5c00 16 | headercolor1 = #204a87 17 | headercolor2 = #3465a4 18 | headerlinkcolor = #fcaf3e 19 | -------------------------------------------------------------------------------- /docs/_theme/nature/static/nature.css_t: -------------------------------------------------------------------------------- 1 | /** 2 | * Sphinx stylesheet -- default theme 3 | * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 4 | */ 5 | 6 | @import url("basic.css"); 7 | 8 | /* -- page layout ----------------------------------------------------------- */ 9 | 10 | body { 11 | font-family: Arial, sans-serif; 12 | font-size: 100%; 13 | background-color: #111; 14 | color: #555; 15 | margin: 0; 16 | padding: 0; 17 | } 18 | 19 | hr{ 20 | border: 1px solid #B1B4B6; 21 | } 22 | 23 | div.document { 24 | background-color: #eee; 25 | } 26 | 27 | div.body { 28 | background-color: #ffffff; 29 | color: #3E4349; 30 | padding: 0 30px 30px 30px; 31 | font-size: 0.8em; 32 | } 33 | 34 | div.footer { 35 | color: #555; 36 | width: 100%; 37 | padding: 13px 0; 38 | text-align: center; 39 | font-size: 75%; 40 | } 41 | 42 | div.footer a { 43 | color: #444; 44 | text-decoration: underline; 45 | } 46 | 47 | div.related { 48 | background-color: #6BA81E; 49 | line-height: 32px; 50 | color: #fff; 51 | text-shadow: 0px 1px 0 #444; 52 | font-size: 0.80em; 53 | } 54 | 55 | div.related a { 56 | color: #E2F3CC; 57 | } 58 | 59 | div.sphinxsidebar { 60 | font-size: 0.75em; 61 | line-height: 1.5em; 62 | } 63 | 64 | div.sphinxsidebarwrapper{ 65 | padding: 20px 0; 66 | } 67 | 68 | div.sphinxsidebar h3, 69 | div.sphinxsidebar h4 { 70 | font-family: Arial, sans-serif; 71 | color: #222; 72 | font-size: 1.2em; 73 | font-weight: normal; 74 | margin: 0; 75 | padding: 5px 10px; 76 | background-color: #ddd; 77 | text-shadow: 1px 1px 0 white 78 | } 79 | 80 | div.sphinxsidebar h4{ 81 | font-size: 1.1em; 82 | } 83 | 84 | div.sphinxsidebar h3 a { 85 | color: #444; 86 | } 87 | 88 | 89 | div.sphinxsidebar p { 90 | color: #888; 91 | padding: 5px 20px; 92 | } 93 | 94 | div.sphinxsidebar p.topless { 95 | } 96 | 97 | div.sphinxsidebar ul { 98 | margin: 10px 20px; 99 | padding: 0; 100 | color: #000; 101 | } 102 | 103 | div.sphinxsidebar a { 104 | color: #444; 105 | } 106 | 107 | div.sphinxsidebar input { 108 | border: 1px solid #ccc; 109 | font-family: sans-serif; 110 | font-size: 1em; 111 | } 112 | 113 | div.sphinxsidebar input[type=text]{ 114 | margin-left: 20px; 115 | } 116 | 117 | /* -- body styles ----------------------------------------------------------- */ 118 | 119 | a { 120 | color: #005B81; 121 | text-decoration: none; 122 | } 123 | 124 | a:hover { 125 | color: #E32E00; 126 | text-decoration: underline; 127 | } 128 | 129 | div.body h1, 130 | div.body h2, 131 | div.body h3, 132 | div.body h4, 133 | div.body h5, 134 | div.body h6 { 135 | font-family: Arial, sans-serif; 136 | background-color: #BED4EB; 137 | font-weight: normal; 138 | color: #212224; 139 | margin: 30px 0px 10px 0px; 140 | padding: 5px 0 5px 10px; 141 | text-shadow: 0px 1px 0 white 142 | } 143 | 144 | div.body h1 { border-top: 20px solid white; margin-top: 0; font-size: 200%; } 145 | div.body h2 { font-size: 150%; background-color: #C8D5E3; } 146 | div.body h3 { font-size: 120%; background-color: #D8DEE3; } 147 | div.body h4 { font-size: 110%; background-color: #D8DEE3; } 148 | div.body h5 { font-size: 100%; background-color: #D8DEE3; } 149 | div.body h6 { font-size: 100%; background-color: #D8DEE3; } 150 | 151 | a.headerlink { 152 | color: #c60f0f; 153 | font-size: 0.8em; 154 | padding: 0 4px 0 4px; 155 | text-decoration: none; 156 | } 157 | 158 | a.headerlink:hover { 159 | background-color: #c60f0f; 160 | color: white; 161 | } 162 | 163 | div.body p, div.body dd, div.body li { 164 | text-align: justify; 165 | line-height: 1.5em; 166 | } 167 | 168 | div.admonition p.admonition-title + p { 169 | display: inline; 170 | } 171 | 172 | div.highlight{ 173 | background-color: white; 174 | } 175 | 176 | div.note { 177 | background-color: #eee; 178 | border: 1px solid #ccc; 179 | } 180 | 181 | div.seealso { 182 | background-color: #ffc; 183 | border: 1px solid #ff6; 184 | } 185 | 186 | div.topic { 187 | background-color: #eee; 188 | } 189 | 190 | div.warning { 191 | background-color: #ffe4e4; 192 | border: 1px solid #f66; 193 | } 194 | 195 | p.admonition-title { 196 | display: inline; 197 | } 198 | 199 | p.admonition-title:after { 200 | content: ":"; 201 | } 202 | 203 | pre { 204 | padding: 10px; 205 | background-color: White; 206 | color: #222; 207 | line-height: 1.2em; 208 | border: 1px solid #C6C9CB; 209 | font-size: 1.2em; 210 | margin: 1.5em 0 1.5em 0; 211 | -webkit-box-shadow: 1px 1px 1px #d8d8d8; 212 | -moz-box-shadow: 1px 1px 1px #d8d8d8; 213 | } 214 | 215 | tt { 216 | background-color: #ecf0f3; 217 | color: #222; 218 | padding: 1px 2px; 219 | font-size: 1.2em; 220 | font-family: monospace; 221 | } -------------------------------------------------------------------------------- /docs/_theme/nature/static/pygments.css: -------------------------------------------------------------------------------- 1 | .c { color: #999988; font-style: italic } /* Comment */ 2 | .k { font-weight: bold } /* Keyword */ 3 | .o { font-weight: bold } /* Operator */ 4 | .cm { color: #999988; font-style: italic } /* Comment.Multiline */ 5 | .cp { color: #999999; font-weight: bold } /* Comment.preproc */ 6 | .c1 { color: #999988; font-style: italic } /* Comment.Single */ 7 | .gd { color: #000000; background-color: #ffdddd } /* Generic.Deleted */ 8 | .ge { font-style: italic } /* Generic.Emph */ 9 | .gr { color: #aa0000 } /* Generic.Error */ 10 | .gh { color: #999999 } /* Generic.Heading */ 11 | .gi { color: #000000; background-color: #ddffdd } /* Generic.Inserted */ 12 | .go { color: #111 } /* Generic.Output */ 13 | .gp { color: #555555 } /* Generic.Prompt */ 14 | .gs { font-weight: bold } /* Generic.Strong */ 15 | .gu { color: #aaaaaa } /* Generic.Subheading */ 16 | .gt { color: #aa0000 } /* Generic.Traceback */ 17 | .kc { font-weight: bold } /* Keyword.Constant */ 18 | .kd { font-weight: bold } /* Keyword.Declaration */ 19 | .kp { font-weight: bold } /* Keyword.Pseudo */ 20 | .kr { font-weight: bold } /* Keyword.Reserved */ 21 | .kt { color: #445588; font-weight: bold } /* Keyword.Type */ 22 | .m { color: #009999 } /* Literal.Number */ 23 | .s { color: #bb8844 } /* Literal.String */ 24 | .na { color: #008080 } /* Name.Attribute */ 25 | .nb { color: #999999 } /* Name.Builtin */ 26 | .nc { color: #445588; font-weight: bold } /* Name.Class */ 27 | .no { color: #ff99ff } /* Name.Constant */ 28 | .ni { color: #800080 } /* Name.Entity */ 29 | .ne { color: #990000; font-weight: bold } /* Name.Exception */ 30 | .nf { color: #990000; font-weight: bold } /* Name.Function */ 31 | .nn { color: #555555 } /* Name.Namespace */ 32 | .nt { color: #000080 } /* Name.Tag */ 33 | .nv { color: purple } /* Name.Variable */ 34 | .ow { font-weight: bold } /* Operator.Word */ 35 | .mf { color: #009999 } /* Literal.Number.Float */ 36 | .mh { color: #009999 } /* Literal.Number.Hex */ 37 | .mi { color: #009999 } /* Literal.Number.Integer */ 38 | .mo { color: #009999 } /* Literal.Number.Oct */ 39 | .sb { color: #bb8844 } /* Literal.String.Backtick */ 40 | .sc { color: #bb8844 } /* Literal.String.Char */ 41 | .sd { color: #bb8844 } /* Literal.String.Doc */ 42 | .s2 { color: #bb8844 } /* Literal.String.Double */ 43 | .se { color: #bb8844 } /* Literal.String.Escape */ 44 | .sh { color: #bb8844 } /* Literal.String.Heredoc */ 45 | .si { color: #bb8844 } /* Literal.String.Interpol */ 46 | .sx { color: #bb8844 } /* Literal.String.Other */ 47 | .sr { color: #808000 } /* Literal.String.Regex */ 48 | .s1 { color: #bb8844 } /* Literal.String.Single */ 49 | .ss { color: #bb8844 } /* Literal.String.Symbol */ 50 | .bp { color: #999999 } /* Name.Builtin.Pseudo */ 51 | .vc { color: #ff99ff } /* Name.Variable.Class */ 52 | .vg { color: #ff99ff } /* Name.Variable.Global */ 53 | .vi { color: #ff99ff } /* Name.Variable.Instance */ 54 | .il { color: #009999 } /* Literal.Number.Integer.Long */ -------------------------------------------------------------------------------- /docs/_theme/nature/theme.conf: -------------------------------------------------------------------------------- 1 | [theme] 2 | inherit = basic 3 | stylesheet = nature.css 4 | pygments_style = tango 5 | 6 | [options] 7 | -------------------------------------------------------------------------------- /docs/changelog.rst: -------------------------------------------------------------------------------- 1 | ../Changelog -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Carrot documentation build configuration file, created by 4 | # sphinx-quickstart on Mon May 18 21:37:44 2009. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | #containing dir. 8 | # 9 | # The contents of this file are pickled, so don't put values in the namespace 10 | # that aren't pickleable (module imports are okay, they're removed 11 | #automatically). 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | import sys 17 | import os 18 | 19 | # If your extensions are in another directory, add it here. If the directory 20 | # is relative to the documentation root, use os.path.abspath to make it 21 | # absolute, like shown here. 22 | sys.path.insert(0, "../") 23 | import carrot 24 | 25 | 26 | # General configuration 27 | # --------------------- 28 | 29 | # Add any Sphinx extension module names here, as strings. 30 | # They can be extensions 31 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 32 | extensions = ['sphinx.ext.autodoc'] 33 | 34 | # Add any paths that contain templates here, relative to this directory. 35 | templates_path = ['.templates'] 36 | 37 | # The suffix of source filenames. 38 | source_suffix = '.rst' 39 | 40 | # The encoding of source files. 41 | #source_encoding = 'utf-8' 42 | 43 | # The master toctree document. 44 | master_doc = 'index' 45 | 46 | # General information about the project. 47 | project = u'Carrot' 48 | copyright = u'2009, Ask Solem' 49 | 50 | # The version info for the project you're documenting, acts as replacement for 51 | # |version| and |release|, also used in various other places throughout the 52 | # built documents. 53 | # 54 | # The short X.Y version. 55 | version = ".".join(map(str, carrot.VERSION[0:2])) 56 | # The full version, including alpha/beta/rc tags. 57 | release = carrot.__version__ 58 | 59 | # The language for content autogenerated by Sphinx. Refer to documentation 60 | # for a list of supported languages. 61 | #language = None 62 | 63 | # There are two options for replacing |today|: either, you set today to some 64 | # non-false value, then it is used: 65 | #today = '' 66 | # Else, today_fmt is used as the format for a strftime call. 67 | #today_fmt = '%B %d, %Y' 68 | 69 | # List of documents that shouldn't be included in the build. 70 | #unused_docs = [] 71 | 72 | # List of directories, relative to source directory, that shouldn't be searched 73 | # for source files. 74 | exclude_trees = ['.build'] 75 | 76 | # The reST default role (used for this markup: `text`) to use for all 77 | # documents. 78 | #default_role = None 79 | 80 | # If true, '()' will be appended to :func: etc. cross-reference text. 81 | add_function_parentheses = True 82 | 83 | # If true, the current module name will be prepended to all description 84 | # unit titles (such as .. function::). 85 | #add_module_names = True 86 | 87 | # If true, sectionauthor and moduleauthor directives will be shown in the 88 | # output. They are ignored by default. 89 | #show_authors = False 90 | 91 | # The name of the Pygments (syntax highlighting) style to use. 92 | pygments_style = 'trac' 93 | 94 | #html_translator_class = "djangodocs.DjangoHTMLTranslator" 95 | 96 | 97 | # Options for HTML output 98 | # ----------------------- 99 | 100 | # The style sheet to use for HTML and HTML Help pages. A file of that name 101 | # must exist either in Sphinx' static/ path, or in one of the custom paths 102 | # given in html_static_path. 103 | #html_style = 'agogo.css' 104 | 105 | # The name for this set of Sphinx documents. If None, it defaults to 106 | # " v documentation". 107 | #html_title = None 108 | 109 | # A shorter title for the navigation bar. Default is the same as html_title. 110 | #html_short_title = None 111 | 112 | # The name of an image file (relative to this directory) to place at the top 113 | # of the sidebar. 114 | #html_logo = None 115 | 116 | # The name of an image file (within the static path) to use as favicon of the 117 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 118 | # pixels large. 119 | #html_favicon = None 120 | 121 | # Add any paths that contain custom static files (such as style sheets) here, 122 | # relative to this directory. They are copied after the builtin static files, 123 | # so a file named "default.css" will overwrite the builtin "default.css". 124 | html_static_path = ['.static'] 125 | 126 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 127 | # using the given strftime format. 128 | #html_last_updated_fmt = '%b %d, %Y' 129 | 130 | # If true, SmartyPants will be used to convert quotes and dashes to 131 | # typographically correct entities. 132 | html_use_smartypants = True 133 | 134 | # Custom sidebar templates, maps document names to template names. 135 | #html_sidebars = {} 136 | 137 | # Additional templates that should be rendered to pages, maps page names to 138 | # template names. 139 | #html_additional_pages = {} 140 | 141 | # If false, no module index is generated. 142 | html_use_modindex = True 143 | 144 | # If false, no index is generated. 145 | html_use_index = True 146 | 147 | # If true, the index is split into individual pages for each letter. 148 | #html_split_index = False 149 | 150 | # If true, the reST sources are included in the HTML build as _sources/. 151 | #html_copy_source = True 152 | 153 | # If true, an OpenSearch description file will be output, and all pages will 154 | # contain a tag referring to it. The value of this option must be the 155 | # base URL from which the finished HTML is served. 156 | #html_use_opensearch = '' 157 | 158 | # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). 159 | #html_file_suffix = '' 160 | 161 | # Output file base name for HTML help builder. 162 | htmlhelp_basename = 'Carrotdoc' 163 | 164 | 165 | # Options for LaTeX output 166 | # ------------------------ 167 | 168 | # The paper size ('letter' or 'a4'). 169 | #latex_paper_size = 'letter' 170 | 171 | # The font size ('10pt', '11pt' or '12pt'). 172 | #latex_font_size = '10pt' 173 | 174 | # Grouping the document tree into LaTeX files. List of tuples 175 | # (source start file, target name, title, author, document class 176 | # [howto/manual]). 177 | latex_documents = [ 178 | ('index', 'Carrot.tex', ur'Carrot Documentation', 179 | ur'Ask Solem', 'manual'), 180 | ] 181 | 182 | # The name of an image file (relative to this directory) to place at the top of 183 | # the title page. 184 | #latex_logo = None 185 | 186 | # For "manual" documents, if this is true, then toplevel headings are parts, 187 | # not chapters. 188 | #latex_use_parts = False 189 | 190 | # Additional stuff for the LaTeX preamble. 191 | #latex_preamble = '' 192 | 193 | # Documents to append as an appendix to all manuals. 194 | #latex_appendices = [] 195 | 196 | # If false, no module index is generated. 197 | #latex_use_modindex = True 198 | 199 | html_theme = "nature" 200 | html_theme_path = ["_theme"] 201 | -------------------------------------------------------------------------------- /docs/faq.rst: -------------------------------------------------------------------------------- 1 | ../FAQ -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. Carrot documentation master file, created by sphinx-quickstart on Mon May 18 21:37:44 2009. 2 | You can adapt this file completely to your liking, but it should at least 3 | contain the root `toctree` directive. 4 | 5 | Carrot Documentation 6 | ================================== 7 | 8 | Contents: 9 | 10 | .. toctree:: 11 | :maxdepth: 3 12 | 13 | introduction 14 | faq 15 | reference/index 16 | changelog 17 | 18 | 19 | Indices and tables 20 | ================== 21 | 22 | * :ref:`genindex` 23 | * :ref:`modindex` 24 | * :ref:`search` 25 | 26 | -------------------------------------------------------------------------------- /docs/introduction.rst: -------------------------------------------------------------------------------- 1 | ../README.rst -------------------------------------------------------------------------------- /docs/reference/carrot.backends.base.rst: -------------------------------------------------------------------------------- 1 | ===================================== 2 | carrot.backends.base 3 | ===================================== 4 | 5 | .. currentmodule:: carrot.backends.base 6 | 7 | .. automodule:: carrot.backends.base 8 | :members: 9 | :undoc-members: 10 | 11 | 12 | -------------------------------------------------------------------------------- /docs/reference/carrot.backends.librabbitmq.rst: -------------------------------------------------------------------------------- 1 | ===================================== 2 | carrot.backends.librabbitmq 3 | ===================================== 4 | 5 | .. currentmodule:: carrot.backends.librabbitmq 6 | 7 | .. automodule:: carrot.backends.librabbitmq 8 | :members: 9 | :undoc-members: 10 | 11 | 12 | -------------------------------------------------------------------------------- /docs/reference/carrot.backends.pikachu.rst: -------------------------------------------------------------------------------- 1 | ===================================== 2 | carrot.backends.pikachu 3 | ===================================== 4 | 5 | .. currentmodule:: carrot.backends.pikachu 6 | 7 | .. automodule:: carrot.backends.pikachu 8 | :members: 9 | :undoc-members: 10 | 11 | 12 | -------------------------------------------------------------------------------- /docs/reference/carrot.backends.pyamqplib.rst: -------------------------------------------------------------------------------- 1 | ===================================== 2 | carrot.backends.pyamqplib 3 | ===================================== 4 | 5 | .. currentmodule:: carrot.backends.pyamqplib 6 | 7 | .. automodule:: carrot.backends.pyamqplib 8 | :members: 9 | :undoc-members: 10 | 11 | 12 | -------------------------------------------------------------------------------- /docs/reference/carrot.backends.pystomp.rst: -------------------------------------------------------------------------------- 1 | ===================================== 2 | carrot.backends.pystomp 3 | ===================================== 4 | 5 | .. currentmodule:: carrot.backends.pystomp 6 | 7 | .. automodule:: carrot.backends.pystomp 8 | :members: 9 | :undoc-members: 10 | 11 | 12 | -------------------------------------------------------------------------------- /docs/reference/carrot.backends.queue.rst: -------------------------------------------------------------------------------- 1 | ===================================== 2 | carrot.backends.queue 3 | ===================================== 4 | 5 | .. currentmodule:: carrot.backends.queue 6 | 7 | .. automodule:: carrot.backends.queue 8 | :members: 9 | :undoc-members: 10 | 11 | 12 | -------------------------------------------------------------------------------- /docs/reference/carrot.backends.rst: -------------------------------------------------------------------------------- 1 | ===================================== 2 | carrot.backends 3 | ===================================== 4 | 5 | .. currentmodule:: carrot.backends 6 | 7 | .. automodule:: carrot.backends 8 | :members: 9 | :undoc-members: 10 | 11 | 12 | -------------------------------------------------------------------------------- /docs/reference/carrot.connection.rst: -------------------------------------------------------------------------------- 1 | ===================================== 2 | carrot.connection 3 | ===================================== 4 | 5 | .. currentmodule:: carrot.connection 6 | 7 | .. automodule:: carrot.connection 8 | :members: 9 | :undoc-members: 10 | 11 | 12 | -------------------------------------------------------------------------------- /docs/reference/carrot.messaging.rst: -------------------------------------------------------------------------------- 1 | ===================================== 2 | carrot.messaging 3 | ===================================== 4 | 5 | .. currentmodule:: carrot.messaging 6 | 7 | .. automodule:: carrot.messaging 8 | :members: 9 | :undoc-members: 10 | 11 | 12 | -------------------------------------------------------------------------------- /docs/reference/carrot.serialization.rst: -------------------------------------------------------------------------------- 1 | ===================================== 2 | carrot.serialization 3 | ===================================== 4 | 5 | .. currentmodule:: carrot.serialization 6 | 7 | .. automodule:: carrot.serialization 8 | :members: 9 | :undoc-members: 10 | 11 | 12 | -------------------------------------------------------------------------------- /docs/reference/carrot.utils.rst: -------------------------------------------------------------------------------- 1 | ===================================== 2 | carrot.utils 3 | ===================================== 4 | 5 | .. currentmodule:: carrot.utils 6 | 7 | .. automodule:: carrot.utils 8 | :members: 9 | :undoc-members: 10 | 11 | 12 | -------------------------------------------------------------------------------- /docs/reference/index.rst: -------------------------------------------------------------------------------- 1 | =========================== 2 | API Reference 3 | =========================== 4 | 5 | :Release: |version| 6 | :Date: |today| 7 | 8 | .. toctree:: 9 | :maxdepth: 2 10 | 11 | carrot.connection 12 | carrot.messaging 13 | carrot.backends 14 | carrot.backends.base 15 | carrot.backends.pyamqplib 16 | carrot.backends.pikachu 17 | carrot.backends.librabbitmq 18 | carrot.backends.pystomp 19 | carrot.backends.queue 20 | carrot.serialization 21 | carrot.utils 22 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [nosetests] 2 | verbosity = 1 3 | detailed-errors = 1 4 | 5 | [build_sphinx] 6 | source-dir = docs/ 7 | build-dir = docs/.build 8 | all_files = 1 9 | 10 | [upload_sphinx] 11 | upload-dir = docs/.build/html 12 | 13 | 14 | [bdist_rpm] 15 | requires = anyjson 16 | amqplib >= 0.6 17 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | import os 4 | import codecs 5 | 6 | try: 7 | from setuptools import setup, find_packages 8 | except ImportError: 9 | from ez_setup import use_setuptools 10 | use_setuptools() 11 | from setuptools import setup, find_packages 12 | 13 | from distutils.command.install_data import install_data 14 | from distutils.command.install import INSTALL_SCHEMES 15 | import sys 16 | 17 | import carrot 18 | 19 | packages, data_files = [], [] 20 | root_dir = os.path.dirname(__file__) 21 | if root_dir != '': 22 | os.chdir(root_dir) 23 | src_dir = "carrot" 24 | 25 | 26 | def osx_install_data(install_data): 27 | 28 | def finalize_options(self): 29 | self.set_undefined_options("install", ("install_lib", "install_dir")) 30 | install_data.finalize_options(self) 31 | 32 | 33 | def fullsplit(path, result=None): 34 | if result is None: 35 | result = [] 36 | head, tail = os.path.split(path) 37 | if head == '': 38 | return [tail] + result 39 | if head == path: 40 | return result 41 | return fullsplit(head, [tail] + result) 42 | 43 | 44 | for scheme in INSTALL_SCHEMES.values(): 45 | scheme['data'] = scheme['purelib'] 46 | 47 | for dirpath, dirnames, filenames in os.walk(src_dir): 48 | # Ignore dirnames that start with '.' 49 | for i, dirname in enumerate(dirnames): 50 | if dirname.startswith("."): 51 | del dirnames[i] 52 | for filename in filenames: 53 | if filename.endswith(".py"): 54 | packages.append('.'.join(fullsplit(dirpath))) 55 | else: 56 | data_files.append([dirpath, [os.path.join(dirpath, f) for f in 57 | filenames]]) 58 | 59 | if os.path.exists("README.rst"): 60 | long_description = codecs.open('README.rst', "r", "utf-8").read() 61 | else: 62 | long_description = "See http://pypi.python.org/pypi/carrot" 63 | 64 | setup( 65 | name='carrot', 66 | version=carrot.__version__, 67 | description=carrot.__doc__, 68 | author=carrot.__author__, 69 | author_email=carrot.__contact__, 70 | url=carrot.__homepage__, 71 | platforms=["any"], 72 | packages=packages, 73 | data_files=data_files, 74 | zip_safe=False, 75 | test_suite="nose.collector", 76 | install_requires=[ 77 | 'anyjson', 78 | 'amqplib>=0.6', 79 | ], 80 | classifiers=[ 81 | "Development Status :: 4 - Beta", 82 | "Framework :: Django", 83 | "Operating System :: OS Independent", 84 | "Programming Language :: Python", 85 | "License :: OSI Approved :: BSD License", 86 | "Intended Audience :: Developers", 87 | "Topic :: Communications", 88 | "Topic :: System :: Distributed Computing", 89 | "Topic :: Software Development :: Libraries :: Python Modules", 90 | ], 91 | long_description=long_description, 92 | ) 93 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ask/carrot/5889a25cd2e274642071c9bba39772f4b3e3d9da/tests/__init__.py -------------------------------------------------------------------------------- /tests/backend.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import unittest 4 | import pickle 5 | import time 6 | 7 | from itertools import count 8 | 9 | sys.path.insert(0, os.pardir) 10 | sys.path.append(os.getcwd()) 11 | 12 | from carrot.messaging import Consumer, Publisher, ConsumerSet 13 | from carrot import serialization 14 | from tests.utils import establish_test_connection 15 | 16 | 17 | class AdvancedDataType(object): 18 | 19 | def __init__(self, something): 20 | self.data = something 21 | 22 | 23 | def fetch_next_message(consumer): 24 | while True: 25 | message = consumer.fetch() 26 | if message: 27 | return message 28 | 29 | 30 | class BackendMessagingCase(unittest.TestCase): 31 | nextq = count(1).next 32 | 33 | def setUp(self): 34 | self.conn = establish_test_connection() 35 | self.queue = TEST_QUEUE 36 | self.exchange = TEST_EXCHANGE 37 | self.routing_key = TEST_ROUTING_KEY 38 | 39 | def create_consumer(self, **options): 40 | queue = "%s%s" % (self.queue, self.nextq()) 41 | return Consumer(connection=self.conn, 42 | queue=queue, exchange=self.exchange, 43 | routing_key=self.routing_key, **options) 44 | 45 | def create_consumerset(self, queues={}, consumers=[], **options): 46 | return ConsumerSet(connection=self.conn, 47 | from_dict=queues, consumers=consumers, **options) 48 | 49 | def create_publisher(self, exchange=None, routing_key=None, **options): 50 | exchange = exchange or self.exchange 51 | routing_key = routing_key or self.routing_key 52 | return Publisher(connection=self.conn, 53 | exchange=exchange, routing_key=routing_key, 54 | **options) 55 | 56 | def test_regression_implied_auto_delete(self): 57 | consumer = self.create_consumer(exclusive=True, auto_declare=False) 58 | self.assertTrue(consumer.auto_delete, "exclusive implies auto_delete") 59 | consumer.close() 60 | 61 | consumer = self.create_consumer(durable=True, auto_delete=False, 62 | auto_declare=False) 63 | self.assertFalse(consumer.auto_delete, 64 | """durable does *not* imply auto_delete. 65 | regression: http://github.com/ask/carrot/issues/closed#issue/2""") 66 | consumer.close() 67 | 68 | def test_consumer_options(self): 69 | opposite_defaults = { 70 | "queue": "xyxyxyxy", 71 | "exchange": "xyxyxyxy", 72 | "routing_key": "xyxyxyxy", 73 | "durable": False, 74 | "exclusive": True, 75 | "auto_delete": True, 76 | "exchange_type": "topic", 77 | } 78 | consumer = Consumer(connection=self.conn, **opposite_defaults) 79 | for opt_name, opt_value in opposite_defaults.items(): 80 | self.assertEquals(getattr(consumer, opt_name), opt_value) 81 | consumer.close() 82 | 83 | def test_consumer_backend(self): 84 | consumer = self.create_consumer() 85 | self.assertTrue(consumer.backend.connection is self.conn) 86 | consumer.close() 87 | 88 | def test_consumer_queue_declared(self): 89 | consumer = self.create_consumer() 90 | self.assertTrue(consumer.backend.queue_exists(consumer.queue)) 91 | consumer.close() 92 | 93 | def test_consumer_callbacks(self): 94 | consumer = self.create_consumer() 95 | publisher = self.create_publisher() 96 | 97 | # raises on no callbacks 98 | self.assertRaises(NotImplementedError, consumer.receive, {}, {}) 99 | 100 | callback1_scratchpad = {} 101 | 102 | def callback1(message_data, message): 103 | callback1_scratchpad["message_data"] = message_data 104 | 105 | callback2_scratchpad = {} 106 | 107 | def callback2(message_data, message): 108 | callback2_scratchpad.update({"delivery_tag": message.delivery_tag, 109 | "message_body": message.body}) 110 | 111 | self.assertFalse(consumer.callbacks, "no default callbacks") 112 | consumer.register_callback(callback1) 113 | consumer.register_callback(callback2) 114 | self.assertEquals(len(consumer.callbacks), 2, "callbacks registered") 115 | 116 | self.assertTrue(consumer.callbacks[0] is callback1, 117 | "callbacks are ordered") 118 | self.assertTrue(consumer.callbacks[1] is callback2, 119 | "callbacks are ordered") 120 | 121 | body = {"foo": "bar"} 122 | 123 | message = self.create_raw_message(publisher, body, "Elaine was here") 124 | consumer._receive_callback(message) 125 | 126 | self.assertEquals(callback1_scratchpad.get("message_data"), body, 127 | "callback1 was called") 128 | self.assertEquals(callback2_scratchpad.get("delivery_tag"), 129 | "Elaine was here") 130 | 131 | consumer.close() 132 | publisher.close() 133 | 134 | def create_raw_message(self, publisher, body, delivery_tag): 135 | raw_message = publisher.create_message(body) 136 | raw_message.delivery_tag = delivery_tag 137 | return raw_message 138 | 139 | def test_empty_queue_returns_None(self): 140 | consumer = self.create_consumer() 141 | consumer.discard_all() 142 | self.assertFalse(consumer.fetch()) 143 | consumer.close() 144 | 145 | def test_custom_serialization_scheme(self): 146 | serialization.registry.register('custom_test', 147 | pickle.dumps, pickle.loads, 148 | content_type='application/x-custom-test', 149 | content_encoding='binary') 150 | 151 | consumer = self.create_consumer() 152 | publisher = self.create_publisher() 153 | consumer.discard_all() 154 | 155 | data = {"string": "The quick brown fox jumps over the lazy dog", 156 | "int": 10, 157 | "float": 3.14159265, 158 | "unicode": u"The quick brown fox jumps over the lazy dog", 159 | "advanced": AdvancedDataType("something"), 160 | "set": set(["george", "jerry", "elaine", "cosmo"]), 161 | "exception": Exception("There was an error"), 162 | } 163 | 164 | publisher.send(data, serializer='custom_test') 165 | message = fetch_next_message(consumer) 166 | backend = self.conn.create_backend() 167 | self.assertTrue(isinstance(message, backend.Message)) 168 | self.assertEquals(message.payload.get("int"), 10) 169 | self.assertEquals(message.content_type, 'application/x-custom-test') 170 | self.assertEquals(message.content_encoding, 'binary') 171 | 172 | decoded_data = message.decode() 173 | 174 | self.assertEquals(decoded_data.get("string"), 175 | "The quick brown fox jumps over the lazy dog") 176 | self.assertEquals(decoded_data.get("int"), 10) 177 | self.assertEquals(decoded_data.get("float"), 3.14159265) 178 | self.assertEquals(decoded_data.get("unicode"), 179 | u"The quick brown fox jumps over the lazy dog") 180 | self.assertEquals(decoded_data.get("set"), 181 | set(["george", "jerry", "elaine", "cosmo"])) 182 | self.assertTrue(isinstance(decoded_data.get("exception"), Exception)) 183 | self.assertEquals(decoded_data.get("exception").args[0], 184 | "There was an error") 185 | self.assertTrue(isinstance(decoded_data.get("advanced"), 186 | AdvancedDataType)) 187 | self.assertEquals(decoded_data["advanced"].data, "something") 188 | 189 | consumer.close() 190 | publisher.close() 191 | 192 | def test_consumer_fetch(self): 193 | consumer = self.create_consumer() 194 | publisher = self.create_publisher() 195 | consumer.discard_all() 196 | 197 | data = {"string": "The quick brown fox jumps over the lazy dog", 198 | "int": 10, 199 | "float": 3.14159265, 200 | "unicode": u"The quick brown fox jumps over the lazy dog", 201 | } 202 | 203 | publisher.send(data) 204 | message = fetch_next_message(consumer) 205 | backend = self.conn.create_backend() 206 | self.assertTrue(isinstance(message, backend.Message)) 207 | 208 | self.assertEquals(message.decode(), data) 209 | 210 | consumer.close() 211 | publisher.close() 212 | 213 | def test_consumer_process_next(self): 214 | consumer = self.create_consumer() 215 | publisher = self.create_publisher() 216 | consumer.discard_all() 217 | 218 | scratchpad = {} 219 | 220 | def callback(message_data, message): 221 | scratchpad["delivery_tag"] = message.delivery_tag 222 | consumer.register_callback(callback) 223 | 224 | publisher.send({"name_discovered": { 225 | "first_name": "Cosmo", 226 | "last_name": "Kramer"}}) 227 | 228 | while True: 229 | message = consumer.fetch(enable_callbacks=True) 230 | if message: 231 | break 232 | 233 | self.assertEquals(scratchpad.get("delivery_tag"), 234 | message.delivery_tag) 235 | 236 | consumer.close() 237 | publisher.close() 238 | 239 | def test_consumer_discard_all(self): 240 | consumer = self.create_consumer() 241 | publisher = self.create_publisher() 242 | consumer.discard_all() 243 | 244 | for i in xrange(100): 245 | publisher.send({"foo": "bar"}) 246 | time.sleep(0.5) 247 | 248 | self.assertEquals(consumer.discard_all(), 100) 249 | 250 | consumer.close() 251 | publisher.close() 252 | 253 | def test_iterqueue(self): 254 | consumer = self.create_consumer() 255 | publisher = self.create_publisher() 256 | num = consumer.discard_all() 257 | 258 | it = consumer.iterqueue(limit=100) 259 | consumer.register_callback(lambda *args: args) 260 | 261 | for i in xrange(100): 262 | publisher.send({"foo%d" % i: "bar%d" % i}) 263 | time.sleep(0.5) 264 | 265 | for i in xrange(100): 266 | try: 267 | message = it.next() 268 | data = message.decode() 269 | self.assertTrue("foo%d" % i in data, "foo%d not in data" % i) 270 | self.assertEquals(data.get("foo%d" % i), "bar%d" % i) 271 | except StopIteration: 272 | self.assertTrue(False, "iterqueue fails StopIteration") 273 | 274 | self.assertRaises(StopIteration, it.next) 275 | 276 | # no messages on queue raises StopIteration if infinite=False 277 | it = consumer.iterqueue() 278 | self.assertRaises(StopIteration, it.next) 279 | 280 | it = consumer.iterqueue(infinite=True) 281 | self.assertTrue(it.next() is None, 282 | "returns None if no messages and inifite=True") 283 | 284 | consumer.close() 285 | publisher.close() 286 | 287 | def test_publisher_message_priority(self): 288 | consumer = self.create_consumer() 289 | publisher = self.create_publisher() 290 | consumer.discard_all() 291 | 292 | m = publisher.create_message("foo", priority=9) 293 | 294 | publisher.send({"foo": "bar"}, routing_key="nowhere", priority=9, 295 | mandatory=False, immediate=False) 296 | 297 | consumer.discard_all() 298 | 299 | consumer.close() 300 | publisher.close() 301 | 302 | def test_backend_survives_channel_close_regr17(self): 303 | """ 304 | test that a backend instance is still functional after 305 | a method that results in a channel closure. 306 | """ 307 | backend = self.create_publisher().backend 308 | assert not backend.queue_exists('notaqueue') 309 | # after calling this once, the channel seems to close, but the 310 | # backend may be holding a reference to it... 311 | assert not backend.queue_exists('notaqueue') 312 | 313 | def disabled_publisher_mandatory_flag_regr16(self): 314 | """ 315 | Test that the publisher "mandatory" flag 316 | raises exceptions at appropriate times. 317 | """ 318 | routing_key = 'black_hole' 319 | 320 | assert self.conn.connection is not None 321 | 322 | message = {'foo': 'mandatory'} 323 | 324 | # sanity check cleanup from last test 325 | assert not self.create_consumer().backend.queue_exists(routing_key) 326 | 327 | publisher = self.create_publisher() 328 | 329 | # this should just get discarded silently, it's not mandatory 330 | publisher.send(message, routing_key=routing_key, mandatory=False) 331 | 332 | # This raises an unspecified exception because there is no queue to 333 | # deliver to 334 | self.assertRaises(Exception, publisher.send, message, 335 | routing_key=routing_key, mandatory=True) 336 | 337 | # now bind a queue to it 338 | consumer = Consumer(connection=self.conn, 339 | queue=routing_key, exchange=self.exchange, 340 | routing_key=routing_key, durable=False, 341 | exclusive=True) 342 | 343 | # check that it exists 344 | assert self.create_consumer().backend.queue_exists(routing_key) 345 | 346 | # this should now get routed to our consumer with no exception 347 | publisher.send(message, routing_key=routing_key, mandatory=True) 348 | 349 | def test_consumer_auto_ack(self): 350 | consumer = self.create_consumer(auto_ack=True) 351 | publisher = self.create_publisher() 352 | consumer.discard_all() 353 | 354 | publisher.send({"foo": "Baz"}) 355 | message = fetch_next_message(consumer) 356 | self.assertEquals(message._state, "ACK") 357 | consumer.close() 358 | publisher.close() 359 | 360 | publisher = self.create_publisher() 361 | consumer = self.create_consumer(auto_ack=False) 362 | publisher.send({"foo": "Baz"}) 363 | message = fetch_next_message(consumer) 364 | self.assertEquals(message._state, "RECEIVED") 365 | 366 | consumer.close() 367 | publisher.close() 368 | 369 | def test_consumer_consume(self): 370 | consumer = self.create_consumer(auto_ack=True) 371 | publisher = self.create_publisher() 372 | consumer.discard_all() 373 | 374 | data = {"foo": "Baz"} 375 | publisher.send(data) 376 | try: 377 | data2 = {"company": "Vandelay Industries"} 378 | publisher.send(data2) 379 | scratchpad = {} 380 | 381 | def callback(message_data, message): 382 | scratchpad["data"] = message_data 383 | consumer.register_callback(callback) 384 | 385 | it = consumer.iterconsume() 386 | it.next() 387 | self.assertEquals(scratchpad.get("data"), data) 388 | it.next() 389 | self.assertEquals(scratchpad.get("data"), data2) 390 | 391 | # Cancel consumer/close and restart. 392 | consumer.close() 393 | consumer = self.create_consumer(auto_ack=True) 394 | consumer.register_callback(callback) 395 | consumer.discard_all() 396 | scratchpad = {} 397 | 398 | # Test limits 399 | it = consumer.iterconsume(limit=4) 400 | publisher.send(data) 401 | publisher.send(data2) 402 | publisher.send(data) 403 | publisher.send(data2) 404 | publisher.send(data) 405 | 406 | it.next() 407 | self.assertEquals(scratchpad.get("data"), data) 408 | it.next() 409 | self.assertEquals(scratchpad.get("data"), data2) 410 | it.next() 411 | self.assertEquals(scratchpad.get("data"), data) 412 | it.next() 413 | self.assertEquals(scratchpad.get("data"), data2) 414 | self.assertRaises(StopIteration, it.next) 415 | 416 | 417 | finally: 418 | consumer.close() 419 | publisher.close() 420 | 421 | def test_consumerset_iterconsume(self): 422 | consumerset = self.create_consumerset(queues={ 423 | "bar": { 424 | "exchange": "foo", 425 | "exchange_type": "direct", 426 | "routing_key": "foo.bar", 427 | }, 428 | "baz": { 429 | "exchange": "foo", 430 | "exchange_type": "direct", 431 | "routing_key": "foo.baz", 432 | }, 433 | "bam": { 434 | "exchange": "foo", 435 | "exchange_type": "direct", 436 | "routing_key": "foo.bam", 437 | }, 438 | "xuzzy": { 439 | "exchange": "foo", 440 | "exchange_type": "direct", 441 | "routing_key": "foo.xuzzy", 442 | }}) 443 | publisher = self.create_publisher(exchange="foo") 444 | consumerset.discard_all() 445 | 446 | scratchpad = {} 447 | 448 | def callback(message_data, message): 449 | scratchpad["data"] = message_data 450 | 451 | def assertDataIs(what): 452 | self.assertEquals(scratchpad.get("data"), what) 453 | 454 | try: 455 | consumerset.register_callback(callback) 456 | it = consumerset.iterconsume() 457 | publisher.send({"rkey": "foo.xuzzy"}, routing_key="foo.xuzzy") 458 | it.next() 459 | assertDataIs({"rkey": "foo.xuzzy"}) 460 | 461 | publisher.send({"rkey": "foo.xuzzy"}, routing_key="foo.xuzzy") 462 | publisher.send({"rkey": "foo.bar"}, routing_key="foo.bar") 463 | publisher.send({"rkey": "foo.baz"}, routing_key="foo.baz") 464 | publisher.send({"rkey": "foo.bam"}, routing_key="foo.bam") 465 | 466 | it.next() 467 | assertDataIs({"rkey": "foo.xuzzy"}) 468 | it.next() 469 | assertDataIs({"rkey": "foo.bar"}) 470 | it.next() 471 | assertDataIs({"rkey": "foo.baz"}) 472 | it.next() 473 | assertDataIs({"rkey": "foo.bam"}) 474 | 475 | finally: 476 | consumerset.close() 477 | publisher.close() 478 | -------------------------------------------------------------------------------- /tests/test_django.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import unittest 4 | import pickle 5 | import time 6 | sys.path.insert(0, os.pardir) 7 | sys.path.append(os.getcwd()) 8 | 9 | from tests.utils import BROKER_HOST, BROKER_PORT, BROKER_VHOST, \ 10 | BROKER_USER, BROKER_PASSWORD 11 | from carrot.connection import DjangoBrokerConnection, BrokerConnection 12 | from UserDict import UserDict 13 | 14 | CARROT_BACKEND = "amqp" 15 | 16 | 17 | class DictWrapper(UserDict): 18 | 19 | def __init__(self, data): 20 | self.data = data 21 | 22 | def __getattr__(self, key): 23 | try: 24 | return self.data[key] 25 | except KeyError: 26 | raise AttributeError("'%s' object has no attribute '%s'" % ( 27 | self.__class__.__name__, key)) 28 | 29 | 30 | def configured_or_configure(settings, **conf): 31 | if settings.configured: 32 | for conf_name, conf_value in conf.items(): 33 | setattr(settings, conf_name, conf_value) 34 | else: 35 | settings.configure(default_settings=DictWrapper(conf)) 36 | 37 | 38 | class TestDjangoSpecific(unittest.TestCase): 39 | 40 | def test_DjangoBrokerConnection(self): 41 | try: 42 | from django.conf import settings 43 | except ImportError: 44 | sys.stderr.write( 45 | "Django is not installed. \ 46 | Not testing django specific features.\n") 47 | return 48 | configured_or_configure(settings, 49 | CARROT_BACKEND=CARROT_BACKEND, 50 | BROKER_HOST=BROKER_HOST, 51 | BROKER_PORT=BROKER_PORT, 52 | BROKER_VHOST=BROKER_VHOST, 53 | BROKER_USER=BROKER_USER, 54 | BROKER_PASSWORD=BROKER_PASSWORD) 55 | 56 | expected_values = { 57 | "backend_cls": CARROT_BACKEND, 58 | "hostname": BROKER_HOST, 59 | "port": BROKER_PORT, 60 | "virtual_host": BROKER_VHOST, 61 | "userid": BROKER_USER, 62 | "password": BROKER_PASSWORD} 63 | 64 | conn = DjangoBrokerConnection() 65 | self.assertTrue(isinstance(conn, BrokerConnection)) 66 | 67 | for val_name, val_value in expected_values.items(): 68 | self.assertEquals(getattr(conn, val_name, None), val_value) 69 | 70 | 71 | if __name__ == '__main__': 72 | unittest.main() 73 | -------------------------------------------------------------------------------- /tests/test_examples.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import unittest 4 | sys.path.insert(0, os.pardir) 5 | sys.path.append(os.getcwd()) 6 | 7 | from tests.utils import establish_test_connection 8 | from carrot.connection import BrokerConnection 9 | from carrot.backends.pyamqplib import Message 10 | 11 | README_QUEUE = "feed" 12 | README_EXCHANGE = "feed" 13 | README_ROUTING_KEY = "feed" 14 | 15 | 16 | class TimeoutError(Exception): 17 | """The operation timed out.""" 18 | 19 | 20 | def receive_a_message(consumer): 21 | while True: 22 | message = consumer.fetch() 23 | if message: 24 | return message 25 | 26 | 27 | def emulate_wait(consumer): 28 | message = receive_a_message(consumer) 29 | consumer._receive_callback(message) 30 | 31 | 32 | class CallbacksTestable(object): 33 | last_feed = None 34 | last_status = None 35 | last_body = None 36 | last_delivery_tag = None 37 | 38 | def import_feed(self, message_data, message): 39 | feed_url = message_data.get("import_feed") 40 | self.last_feed = feed_url 41 | if not feed_url: 42 | self.last_status = "REJECT" 43 | message.reject() 44 | else: 45 | self.last_status = "ACK" 46 | message.ack() 47 | 48 | def dump_message(self, message_data, message): 49 | self.last_body = message.body 50 | self.last_delivery_tag = message.delivery_tag 51 | 52 | 53 | def create_README_consumer(amqpconn): 54 | from carrot.messaging import Consumer 55 | consumer = Consumer(connection=amqpconn, 56 | queue=README_QUEUE, exchange=README_EXCHANGE, 57 | routing_key=README_ROUTING_KEY) 58 | tcallbacks = CallbacksTestable() 59 | consumer.register_callback(tcallbacks.import_feed) 60 | consumer.register_callback(tcallbacks.dump_message) 61 | return consumer, tcallbacks 62 | 63 | 64 | def create_README_publisher(amqpconn): 65 | from carrot.messaging import Publisher 66 | publisher = Publisher(connection=amqpconn, exchange=README_EXCHANGE, 67 | routing_key=README_ROUTING_KEY) 68 | return publisher 69 | 70 | 71 | class TestExamples(unittest.TestCase): 72 | 73 | def setUp(self): 74 | self.conn = establish_test_connection() 75 | self.consumer, self.tcallbacks = create_README_consumer(self.conn) 76 | self.consumer.discard_all() 77 | 78 | def test_connection(self): 79 | self.assertTrue(self.conn) 80 | self.assertTrue(self.conn.connection.channel()) 81 | 82 | def test_README_consumer(self): 83 | consumer = self.consumer 84 | tcallbacks = self.tcallbacks 85 | self.assertTrue(consumer.connection) 86 | self.assertTrue(isinstance(consumer.connection, BrokerConnection)) 87 | self.assertEquals(consumer.queue, README_QUEUE) 88 | self.assertEquals(consumer.exchange, README_EXCHANGE) 89 | self.assertEquals(consumer.routing_key, README_ROUTING_KEY) 90 | self.assertTrue(len(consumer.callbacks), 2) 91 | 92 | def test_README_publisher(self): 93 | publisher = create_README_publisher(self.conn) 94 | self.assertTrue(publisher.connection) 95 | self.assertTrue(isinstance(publisher.connection, BrokerConnection)) 96 | self.assertEquals(publisher.exchange, README_EXCHANGE) 97 | self.assertEquals(publisher.routing_key, README_ROUTING_KEY) 98 | 99 | def test_README_together(self): 100 | consumer = self.consumer 101 | tcallbacks = self.tcallbacks 102 | 103 | publisher = create_README_publisher(self.conn) 104 | feed_url = "http://cnn.com/rss/edition.rss" 105 | body = {"import_feed": feed_url} 106 | publisher.send(body) 107 | publisher.close() 108 | emulate_wait(consumer) 109 | 110 | self.assertEquals(tcallbacks.last_feed, feed_url) 111 | self.assertTrue(tcallbacks.last_delivery_tag) 112 | self.assertEquals(tcallbacks.last_status, "ACK") 113 | 114 | publisher = create_README_publisher(self.conn) 115 | body = {"foo": "FOO"} 116 | publisher.send(body) 117 | publisher.close() 118 | emulate_wait(consumer) 119 | 120 | self.assertFalse(tcallbacks.last_feed) 121 | self.assertTrue(tcallbacks.last_delivery_tag) 122 | self.assertEquals(tcallbacks.last_status, "REJECT") 123 | 124 | def test_subclassing(self): 125 | from carrot.messaging import Consumer, Publisher 126 | feed_url = "http://cnn.com/rss/edition.rss" 127 | testself = self 128 | 129 | class TConsumer(Consumer): 130 | queue = README_QUEUE 131 | exchange = README_EXCHANGE 132 | routing_key = README_ROUTING_KEY 133 | 134 | def receive(self, message_data, message): 135 | testself.assertTrue(isinstance(message, Message)) 136 | testself.assertTrue("import_feed" in message_data) 137 | testself.assertEquals(message_data.get("import_feed"), 138 | feed_url) 139 | 140 | class TPublisher(Publisher): 141 | exchange = README_EXCHANGE 142 | routing_key = README_ROUTING_KEY 143 | 144 | consumer = TConsumer(connection=self.conn) 145 | publisher = TPublisher(connection=self.conn) 146 | 147 | consumer.discard_all() 148 | publisher.send({"import_feed": feed_url}) 149 | publisher.close() 150 | emulate_wait(consumer) 151 | 152 | consumer.close() 153 | 154 | 155 | if __name__ == '__main__': 156 | unittest.main() 157 | -------------------------------------------------------------------------------- /tests/test_pyamqplib.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import unittest 4 | import pickle 5 | import time 6 | sys.path.insert(0, os.pardir) 7 | sys.path.append(os.getcwd()) 8 | 9 | from tests.utils import establish_test_connection 10 | from carrot.connection import BrokerConnection 11 | from carrot.messaging import Consumer, Publisher, ConsumerSet 12 | from carrot.backends.pyamqplib import Backend as AMQPLibBackend 13 | from carrot.backends.pyamqplib import Message as AMQPLibMessage 14 | from carrot import serialization 15 | from tests.backend import BackendMessagingCase 16 | 17 | TEST_QUEUE = "carrot.unittest" 18 | TEST_EXCHANGE = "carrot.unittest" 19 | TEST_ROUTING_KEY = "carrot.unittest" 20 | 21 | TEST_QUEUE_TWO = "carrot.unittest.two" 22 | TEST_EXCHANGE_TWO = "carrot.unittest.two" 23 | TEST_ROUTING_KEY_TWO = "carrot.unittest.two" 24 | 25 | TEST_CELERY_QUEUE = { 26 | TEST_QUEUE: { 27 | "exchange": TEST_EXCHANGE, 28 | "exchange_type": "direct", 29 | "routing_key": TEST_ROUTING_KEY, 30 | }, 31 | TEST_QUEUE_TWO: { 32 | "exchange": TEST_EXCHANGE_TWO, 33 | "exchange_type": "direct", 34 | "routing_key": TEST_ROUTING_KEY_TWO, 35 | }, 36 | } 37 | 38 | 39 | class TestAMQPlibMessaging(BackendMessagingCase): 40 | 41 | def setUp(self): 42 | self.conn = establish_test_connection() 43 | self.queue = TEST_QUEUE 44 | self.exchange = TEST_EXCHANGE 45 | self.routing_key = TEST_ROUTING_KEY 46 | BackendMessagingCase = None 47 | 48 | if __name__ == '__main__': 49 | unittest.main() 50 | -------------------------------------------------------------------------------- /tests/test_pyqueue.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import unittest 4 | import uuid 5 | sys.path.insert(0, os.pardir) 6 | sys.path.append(os.getcwd()) 7 | 8 | from carrot.backends.queue import Message as PyQueueMessage 9 | from carrot.backends.queue import Backend as PyQueueBackend 10 | from carrot.connection import BrokerConnection 11 | from carrot.messaging import Messaging, Consumer, Publisher 12 | 13 | 14 | def create_backend(): 15 | return PyQueueBackend(connection=BrokerConnection()) 16 | 17 | 18 | class TestPyQueueMessage(unittest.TestCase): 19 | 20 | def test_message(self): 21 | b = create_backend() 22 | self.assertTrue(b) 23 | 24 | message_body = "George Constanza" 25 | delivery_tag = str(uuid.uuid4()) 26 | 27 | m1 = PyQueueMessage(backend=b, 28 | body=message_body, 29 | delivery_tag=delivery_tag) 30 | m2 = PyQueueMessage(backend=b, 31 | body=message_body, 32 | delivery_tag=delivery_tag) 33 | m3 = PyQueueMessage(backend=b, 34 | body=message_body, 35 | delivery_tag=delivery_tag) 36 | self.assertEquals(m1.body, message_body) 37 | self.assertEquals(m1.delivery_tag, delivery_tag) 38 | 39 | m1.ack() 40 | m2.reject() 41 | m3.requeue() 42 | 43 | 44 | class TestPyQueueBackend(unittest.TestCase): 45 | 46 | def test_backend(self): 47 | b = create_backend() 48 | message_body = "Vandelay Industries" 49 | b.publish(b.prepare_message(message_body, "direct", 50 | content_type='text/plain', 51 | content_encoding="ascii"), 52 | exchange="test", 53 | routing_key="test") 54 | m_in_q = b.get() 55 | self.assertTrue(isinstance(m_in_q, PyQueueMessage)) 56 | self.assertEquals(m_in_q.body, message_body) 57 | 58 | def test_consumer_interface(self): 59 | to_send = ['No', 'soup', 'for', 'you!'] 60 | messages = [] 61 | def cb(message_data, message): 62 | messages.append(message_data) 63 | conn = BrokerConnection(backend_cls='memory') 64 | consumer = Consumer(connection=conn, queue="test", 65 | exchange="test", routing_key="test") 66 | consumer.register_callback(cb) 67 | publisher = Publisher(connection=conn, exchange="test", 68 | routing_key="test") 69 | for i in to_send: 70 | publisher.send(i) 71 | it = consumer.iterconsume() 72 | for i in range(len(to_send)): 73 | it.next() 74 | self.assertEqual(messages, to_send) 75 | 76 | 77 | class TMessaging(Messaging): 78 | exchange = "test" 79 | routing_key = "test" 80 | queue = "test" 81 | 82 | 83 | class TestMessaging(unittest.TestCase): 84 | 85 | def test_messaging(self): 86 | m = TMessaging(connection=BrokerConnection(backend_cls=PyQueueBackend)) 87 | self.assertTrue(m) 88 | 89 | self.assertEquals(m.fetch(), None) 90 | mdata = {"name": "Cosmo Kramer"} 91 | m.send(mdata) 92 | next_msg = m.fetch() 93 | next_msg_data = next_msg.decode() 94 | self.assertEquals(next_msg_data, mdata) 95 | self.assertEquals(m.fetch(), None) 96 | 97 | 98 | if __name__ == '__main__': 99 | unittest.main() 100 | -------------------------------------------------------------------------------- /tests/test_serialization.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | import cPickle 5 | import sys 6 | import os 7 | import unittest 8 | import uuid 9 | sys.path.insert(0, os.pardir) 10 | sys.path.append(os.getcwd()) 11 | 12 | 13 | from carrot.serialization import registry 14 | 15 | # For content_encoding tests 16 | unicode_string = u'abcdé\u8463' 17 | unicode_string_as_utf8 = unicode_string.encode('utf-8') 18 | latin_string = u'abcdé' 19 | latin_string_as_latin1 = latin_string.encode('latin-1') 20 | latin_string_as_utf8 = latin_string.encode('utf-8') 21 | 22 | 23 | # For serialization tests 24 | py_data = {"string": "The quick brown fox jumps over the lazy dog", 25 | "int": 10, 26 | "float": 3.14159265, 27 | "unicode": u"Thé quick brown fox jumps over thé lazy dog", 28 | "list": ["george", "jerry", "elaine", "cosmo"], 29 | } 30 | 31 | # JSON serialization tests 32 | json_data = ('{"int": 10, "float": 3.1415926500000002, ' 33 | '"list": ["george", "jerry", "elaine", "cosmo"], ' 34 | '"string": "The quick brown fox jumps over the lazy ' 35 | 'dog", "unicode": "Th\\u00e9 quick brown fox jumps over ' 36 | 'th\\u00e9 lazy dog"}') 37 | 38 | # Pickle serialization tests 39 | pickle_data = cPickle.dumps(py_data) 40 | 41 | # YAML serialization tests 42 | yaml_data = ('float: 3.1415926500000002\nint: 10\n' 43 | 'list: [george, jerry, elaine, cosmo]\n' 44 | 'string: The quick brown fox jumps over the lazy dog\n' 45 | 'unicode: "Th\\xE9 quick brown fox ' 46 | 'jumps over th\\xE9 lazy dog"\n') 47 | 48 | 49 | msgpack_py_data = dict(py_data) 50 | # msgpack only supports tuples 51 | msgpack_py_data["list"] = tuple(msgpack_py_data["list"]) 52 | # Unicode chars are lost in transmit :( 53 | msgpack_py_data["unicode"] = 'Th quick brown fox jumps over th lazy dog' 54 | msgpack_data = ('\x85\xa3int\n\xa5float\xcb@\t!\xfbS\xc8\xd4\xf1\xa4list' 55 | '\x94\xa6george\xa5jerry\xa6elaine\xa5cosmo\xa6string\xda' 56 | '\x00+The quick brown fox jumps over the lazy dog\xa7unicode' 57 | '\xda\x00)Th quick brown fox jumps over th lazy dog') 58 | 59 | 60 | def say(m): 61 | sys.stderr.write("%s\n" % (m, )) 62 | 63 | 64 | class TestSerialization(unittest.TestCase): 65 | 66 | def test_content_type_decoding(self): 67 | content_type = 'plain/text' 68 | 69 | self.assertEquals(unicode_string, 70 | registry.decode( 71 | unicode_string_as_utf8, 72 | content_type='plain/text', 73 | content_encoding='utf-8')) 74 | self.assertEquals(latin_string, 75 | registry.decode( 76 | latin_string_as_latin1, 77 | content_type='application/data', 78 | content_encoding='latin-1')) 79 | 80 | def test_content_type_binary(self): 81 | content_type = 'plain/text' 82 | 83 | self.assertNotEquals(unicode_string, 84 | registry.decode( 85 | unicode_string_as_utf8, 86 | content_type='application/data', 87 | content_encoding='binary')) 88 | 89 | self.assertEquals(unicode_string_as_utf8, 90 | registry.decode( 91 | unicode_string_as_utf8, 92 | content_type='application/data', 93 | content_encoding='binary')) 94 | 95 | def test_content_type_encoding(self): 96 | # Using the "raw" serializer 97 | self.assertEquals(unicode_string_as_utf8, 98 | registry.encode( 99 | unicode_string, serializer="raw")[-1]) 100 | self.assertEquals(latin_string_as_utf8, 101 | registry.encode( 102 | latin_string, serializer="raw")[-1]) 103 | # And again w/o a specific serializer to check the 104 | # code where we force unicode objects into a string. 105 | self.assertEquals(unicode_string_as_utf8, 106 | registry.encode(unicode_string)[-1]) 107 | self.assertEquals(latin_string_as_utf8, 108 | registry.encode(latin_string)[-1]) 109 | 110 | def test_json_decode(self): 111 | self.assertEquals(py_data, 112 | registry.decode( 113 | json_data, 114 | content_type='application/json', 115 | content_encoding='utf-8')) 116 | 117 | def test_json_encode(self): 118 | self.assertEquals(registry.decode( 119 | registry.encode(py_data, serializer="json")[-1], 120 | content_type='application/json', 121 | content_encoding='utf-8'), 122 | registry.decode( 123 | json_data, 124 | content_type='application/json', 125 | content_encoding='utf-8')) 126 | 127 | def test_msgpack_decode(self): 128 | try: 129 | import msgpack 130 | except ImportError: 131 | return say("* msgpack-python not installed, will not execute " 132 | "related tests.") 133 | self.assertEquals(msgpack_py_data, 134 | registry.decode( 135 | msgpack_data, 136 | content_type='application/x-msgpack', 137 | content_encoding='binary')) 138 | 139 | def test_msgpack_encode(self): 140 | try: 141 | import msgpack 142 | except ImportError: 143 | return say("* msgpack-python not installed, will not execute " 144 | "related tests.") 145 | self.assertEquals(registry.decode( 146 | registry.encode(msgpack_py_data, serializer="msgpack")[-1], 147 | content_type='application/x-msgpack', 148 | content_encoding='binary'), 149 | registry.decode( 150 | msgpack_data, 151 | content_type='application/x-msgpack', 152 | content_encoding='binary')) 153 | 154 | 155 | def test_yaml_decode(self): 156 | try: 157 | import yaml 158 | except ImportError: 159 | return say("* PyYAML not installed, will not execute " 160 | "related tests.") 161 | self.assertEquals(py_data, 162 | registry.decode( 163 | yaml_data, 164 | content_type='application/x-yaml', 165 | content_encoding='utf-8')) 166 | 167 | def test_yaml_encode(self): 168 | try: 169 | import yaml 170 | except ImportError: 171 | return say("* PyYAML not installed, will not execute " 172 | "related tests.") 173 | self.assertEquals(registry.decode( 174 | registry.encode(py_data, serializer="yaml")[-1], 175 | content_type='application/x-yaml', 176 | content_encoding='utf-8'), 177 | registry.decode( 178 | yaml_data, 179 | content_type='application/x-yaml', 180 | content_encoding='utf-8')) 181 | 182 | def test_pickle_decode(self): 183 | self.assertEquals(py_data, 184 | registry.decode( 185 | pickle_data, 186 | content_type='application/x-python-serialize', 187 | content_encoding='binary')) 188 | 189 | def test_pickle_encode(self): 190 | self.assertEquals(pickle_data, 191 | registry.encode(py_data, 192 | serializer="pickle")[-1]) 193 | 194 | 195 | if __name__ == '__main__': 196 | unittest.main() 197 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from carrot import utils 4 | 5 | 6 | class TestUtils(unittest.TestCase): 7 | 8 | def test_partition_unicode(self): 9 | s = u'hi mom' 10 | self.assertEqual(utils.partition(s, ' '), (u'hi', u' ', u'mom')) 11 | 12 | def test_rpartition_unicode(self): 13 | s = u'hi mom !' 14 | self.assertEqual(utils.rpartition(s, ' '), (u'hi mom', u' ', u'!')) 15 | -------------------------------------------------------------------------------- /tests/test_with_statement.py: -------------------------------------------------------------------------------- 1 | from __future__ import with_statement 2 | import os 3 | import sys 4 | import unittest 5 | sys.path.insert(0, os.pardir) 6 | sys.path.append(os.getcwd()) 7 | 8 | from tests.utils import test_connection_args 9 | from carrot.connection import BrokerConnection 10 | from carrot.messaging import Consumer, Publisher 11 | 12 | 13 | class TestTransactioned(unittest.TestCase): 14 | 15 | def test_with_statement(self): 16 | 17 | with BrokerConnection(**test_connection_args()) as conn: 18 | self.assertFalse(conn._closed) 19 | with Publisher(connection=conn, exchange="F", routing_key="G") \ 20 | as publisher: 21 | self.assertFalse(publisher._closed) 22 | self.assertTrue(conn._closed) 23 | self.assertTrue(publisher._closed) 24 | 25 | with BrokerConnection(**test_connection_args()) as conn: 26 | self.assertFalse(conn._closed) 27 | with Consumer(connection=conn, queue="E", exchange="F", 28 | routing_key="G") as consumer: 29 | self.assertFalse(consumer._closed) 30 | self.assertTrue(conn._closed) 31 | self.assertTrue(consumer._closed) 32 | 33 | 34 | if __name__ == '__main__': 35 | unittest.main() 36 | -------------------------------------------------------------------------------- /tests/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from carrot.connection import BrokerConnection 4 | 5 | 6 | BROKER_HOST = os.environ.get('BROKER_HOST', "localhost") 7 | BROKER_PORT = os.environ.get('BROKER_PORT', 5672) 8 | BROKER_VHOST = os.environ.get('BROKER_VHOST', "/") 9 | BROKER_USER = os.environ.get('BROKER_USER', "guest") 10 | BROKER_PASSWORD = os.environ.get('BROKER_PASSWORD', "guest") 11 | 12 | STOMP_HOST = os.environ.get('STOMP_HOST', 'localhost') 13 | STOMP_PORT = os.environ.get('STOMP_PORT', 61613) 14 | STOMP_QUEUE = os.environ.get('STOMP_QUEUE', '/queue/testcarrot') 15 | 16 | 17 | def test_connection_args(): 18 | return {"hostname": BROKER_HOST, "port": BROKER_PORT, 19 | "virtual_host": BROKER_VHOST, "userid": BROKER_USER, 20 | "password": BROKER_PASSWORD} 21 | 22 | 23 | def test_stomp_connection_args(): 24 | return {"hostname": STOMP_HOST, "port": STOMP_PORT} 25 | 26 | 27 | def establish_test_connection(): 28 | return BrokerConnection(**test_connection_args()) 29 | -------------------------------------------------------------------------------- /tests/xxxstmop.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import unittest 4 | import uuid 5 | sys.path.insert(0, os.pardir) 6 | sys.path.append(os.getcwd()) 7 | 8 | try: 9 | import stompy 10 | except ImportError: 11 | stompy = None 12 | Frame = StompMessage = StompBackend = object 13 | 14 | else: 15 | from carrot.backends.pystomp import Message as StompMessage 16 | from carrot.backends.pystomp import Backend as StompBackend 17 | from stompy.frame import Frame 18 | 19 | from carrot.connection import BrokerConnection 20 | from carrot.messaging import Publisher, Consumer 21 | from tests.utils import test_stomp_connection_args, STOMP_QUEUE 22 | from carrot.serialization import encode 23 | 24 | _no_stompy_msg = "* stompy (python-stomp) not installed. " \ 25 | + "Will not execute related tests." 26 | _no_stompy_msg_emitted = False 27 | 28 | 29 | def stompy_or_None(): 30 | 31 | def emit_no_stompy_msg(): 32 | global _no_stompy_msg_emitted 33 | if not _no_stompy_msg_emitted: 34 | sys.stderr.write("\n" + _no_stompy_msg + "\n") 35 | _no_stompy_msg_emitted = True 36 | 37 | if stompy is None: 38 | emit_no_stompy_msg() 39 | return None 40 | return stompy 41 | 42 | 43 | def create_connection(): 44 | return BrokerConnection(backend_cls=StompBackend, 45 | **test_stomp_connection_args()) 46 | 47 | 48 | def create_backend(): 49 | return create_connection().create_backend() 50 | 51 | 52 | class MockFrame(Frame): 53 | 54 | def mock(self, command=None, headers=None, body=None): 55 | self.command = command 56 | self.headers = headers 57 | self.body = body 58 | return self 59 | 60 | 61 | class TestStompMessage(unittest.TestCase): 62 | 63 | def test_message(self): 64 | if not stompy_or_None(): 65 | return 66 | b = create_backend() 67 | 68 | self.assertTrue(b) 69 | 70 | message_body = "George Constanza" 71 | delivery_tag = str(uuid.uuid4()) 72 | 73 | frame = MockFrame().mock(body=message_body, headers={ 74 | "message-id": delivery_tag, 75 | "content_type": "text/plain", 76 | "content_encoding": "utf-8", 77 | }) 78 | 79 | m1 = StompMessage(backend=b, frame=frame) 80 | m2 = StompMessage(backend=b, frame=frame) 81 | m3 = StompMessage(backend=b, frame=frame) 82 | self.assertEquals(m1.body, message_body) 83 | self.assertEquals(m1.delivery_tag, delivery_tag) 84 | 85 | #m1.ack() 86 | self.assertRaises(NotImplementedError, m2.reject) 87 | self.assertRaises(NotImplementedError, m3.requeue) 88 | 89 | 90 | class TestPyStompMessaging(unittest.TestCase): 91 | 92 | def setUp(self): 93 | if stompy_or_None(): 94 | self.conn = create_connection() 95 | self.queue = STOMP_QUEUE 96 | self.exchange = STOMP_QUEUE 97 | self.routing_key = STOMP_QUEUE 98 | 99 | def create_consumer(self, **options): 100 | return Consumer(connection=self.conn, 101 | queue=self.queue, exchange=self.exchange, 102 | routing_key=self.routing_key, **options) 103 | 104 | def create_publisher(self, **options): 105 | return Publisher(connection=self.conn, 106 | exchange=self.exchange, 107 | routing_key=self.routing_key, **options) 108 | 109 | def test_backend(self): 110 | if not stompy_or_None(): 111 | return 112 | publisher = self.create_publisher() 113 | consumer = self.create_consumer() 114 | for i in range(100): 115 | publisher.send({"foo%d" % i: "bar%d" % i}) 116 | publisher.close() 117 | 118 | discarded = consumer.discard_all() 119 | self.assertEquals(discarded, 100) 120 | publisher.close() 121 | consumer.close() 122 | 123 | publisher = self.create_publisher() 124 | for i in range(100): 125 | publisher.send({"foo%d" % i: "bar%d" % i}) 126 | 127 | consumer = self.create_consumer() 128 | for i in range(100): 129 | while True: 130 | message = consumer.fetch() 131 | if message: 132 | break 133 | self.assertTrue("foo%d" % i in message.payload) 134 | message.ack() 135 | 136 | publisher.close() 137 | consumer.close() 138 | 139 | 140 | consumer = self.create_consumer() 141 | discarded = consumer.discard_all() 142 | self.assertEquals(discarded, 0) 143 | 144 | def create_raw_message(self, publisher, body, delivery_tag): 145 | content_type, content_encoding, payload = encode(body) 146 | frame = MockFrame().mock(body=payload, headers={ 147 | "message-id": delivery_tag, 148 | "content-type": content_type, 149 | "content-encoding": content_encoding, 150 | }) 151 | return frame 152 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py24,py25,py26,py27 3 | 4 | [testenv] 5 | distribute = True 6 | sitepackages = False 7 | commands = nosetests 8 | 9 | [testenv:py24] 10 | basepython = python2.4 11 | commands = pip -E {envdir} install -r contrib/requirements/default.txt 12 | pip -E {envdir} install -r contrib/requirements/test.txt 13 | nosetests --with-xunit --xunit-file=nosetests.xml \ 14 | --with-coverage3 --cover3-xml \ 15 | --cover3-xml-file=coverage.xml 16 | 17 | 18 | [testenv:py25] 19 | basepython = python2.5 20 | commands = pip -E {envdir} install -r contrib/requirements/default.txt 21 | pip -E {envdir} install -r contrib/requirements/test.txt 22 | nosetests --with-xunit --xunit-file=nosetests.xml \ 23 | --with-coverage3 --cover3-xml \ 24 | --cover3-xml-file=coverage.xml 25 | 26 | [testenv:py26] 27 | basepython = python2.6 28 | commands = pip -E {envdir} install -r contrib/requirements/default.txt 29 | pip -E {envdir} install -r contrib/requirements/test.txt 30 | nosetests --with-xunit --xunit-file=nosetests.xml \ 31 | --with-coverage3 --cover3-xml \ 32 | --cover3-xml-file=coverage.xml 33 | 34 | [testenv:py27] 35 | basepython = python2.7 36 | commands = pip -E {envdir} install -r contrib/requirements/default.txt 37 | pip -E {envdir} install -r contrib/requirements/test.txt 38 | nosetests --with-xunit --xunit-file=nosetests.xml \ 39 | --with-coverage3 --cover3-xml \ 40 | --cover3-xml-file=coverage.xml 41 | --------------------------------------------------------------------------------