├── REQUIREMENTS ├── MANIFEST.in ├── docs ├── _themes │ └── warm │ │ ├── theme.conf │ │ └── static │ │ ├── pygments.css │ │ └── warm.css_t ├── changelog.rst ├── apireference.rst ├── index.rst ├── Makefile ├── tutorial.rst └── conf.py ├── .gitignore ├── LICENSE ├── README.rst ├── setup.py ├── tests.py └── hotqueue.py /REQUIREMENTS: -------------------------------------------------------------------------------- 1 | redis>=2.0.0 -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include docs 2 | include LICENSE 3 | include README.rst 4 | include REQUIREMENTS 5 | include tests.py -------------------------------------------------------------------------------- /docs/_themes/warm/theme.conf: -------------------------------------------------------------------------------- 1 | [theme] 2 | inherit = basic 3 | stylesheet = warm.css 4 | pygments_style = tango 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.egg-info 2 | *.pyc 3 | *.pyo 4 | .DS_Store 5 | build 6 | dist 7 | docs/.build 8 | docs/_build 9 | MANIFEST -------------------------------------------------------------------------------- /docs/changelog.rst: -------------------------------------------------------------------------------- 1 | ========= 2 | Changelog 3 | ========= 4 | 5 | Changes in v0.2.0 6 | ================= 7 | - Renamed ``dequeue()`` method on ``HotQueue`` to ``get()`` 8 | - Renamed ``enqueue()`` method on ``HotQueue`` to ``put()`` 9 | - Added ``HotQueue.worker()`` function decorator 10 | - ``HotQueue.get()`` method now supports block and timeout arguments 11 | - Added test suite 12 | 13 | Changes in v0.1 14 | =============== 15 | - Initial release -------------------------------------------------------------------------------- /docs/apireference.rst: -------------------------------------------------------------------------------- 1 | ============= 2 | API Reference 3 | ============= 4 | 5 | HotQueue 6 | ======== 7 | 8 | .. autoclass:: hotqueue.HotQueue 9 | 10 | .. autoattribute:: hotqueue.HotQueue.key 11 | 12 | .. automethod:: hotqueue.HotQueue.clear 13 | 14 | .. automethod:: hotqueue.HotQueue.consume 15 | 16 | .. automethod:: hotqueue.HotQueue.get 17 | 18 | .. automethod:: hotqueue.HotQueue.put 19 | 20 | .. automethod:: hotqueue.HotQueue.worker -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2010 Richard Henry 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | =========================== 2 | HotQueue User Documentation 3 | =========================== 4 | 5 | HotQueue is a Python library that allows you to use `Redis `_ as a message queue within your Python programs. 6 | 7 | The main advantage of this model is that there is no queue server to run, other than Redis. This is particularly ideal if you're already using Redis as a datastore elsewhere. To install it, run: 8 | 9 | .. code-block:: console 10 | 11 | # easy_install -U hotqueue 12 | 13 | The source code is available on `GitHub `_. 14 | 15 | To get help with HotQueue, use the `HotQueue Users mailing list 16 | `_. 17 | 18 | Explore Documentation 19 | ===================== 20 | 21 | .. toctree:: 22 | :maxdepth: 2 23 | 24 | tutorial 25 | apireference 26 | changelog 27 | 28 | Requirements 29 | ============ 30 | 31 | HotQueue requires, at a minimum: 32 | 33 | - Python version 2.6+ (not Python 3 compatible) 34 | - `Redis `_ version 1.3.1+ 35 | - `redis-py `_ version 2.0.0+ -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ======== 2 | HotQueue 3 | ======== 4 | :Info: HotQueue is a Python library that allows you to use Redis as a message queue within your Python programs. 5 | :Author: Richard Henry (http://github.com/richardhenry) 6 | 7 | About HotQueue 8 | ============== 9 | 10 | HotQueue is a Python library that allows you to use `Redis `_ as a message queue within your Python programs. 11 | 12 | The main advantage of this model is that there is no queue server to run, other than Redis. This is particularly ideal if you're already using Redis as a datastore elsewhere. To install it, run: 13 | 14 | easy_install -U hotqueue 15 | 16 | The best place to get started is `the documentation `_. 17 | 18 | The source code is available on `GitHub `_. 19 | 20 | To get help with HotQueue, use the `HotQueue Users mailing list 21 | `_. 22 | 23 | Contributing 24 | ============ 25 | The source is available on `GitHub `_. To contribute to the project, fork it on GitHub and send a pull request, all contributions and suggestions are welcome. 26 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import os 5 | import re 6 | 7 | from distutils.core import setup 8 | 9 | 10 | rel_file = lambda *args: os.path.join(os.path.dirname(os.path.abspath(__file__)), *args) 11 | 12 | def read_from(filename): 13 | fp = open(filename) 14 | try: 15 | return fp.read() 16 | finally: 17 | fp.close() 18 | 19 | def get_long_description(): 20 | return read_from(rel_file('README.rst')) 21 | 22 | def get_requirements(): 23 | data = read_from(rel_file('REQUIREMENTS')) 24 | lines = map(lambda s: s.strip(), data.splitlines()) 25 | return filter(None, lines) 26 | 27 | def get_version(): 28 | data = read_from(rel_file('hotqueue.py')) 29 | return re.search(r"__version__ = '([^']+)'", data).group(1) 30 | 31 | 32 | setup( 33 | name = 'hotqueue', 34 | author = 'Richard Henry', 35 | author_email = 'richardhenry@me.com', 36 | description = 'HotQueue is a Python library that allows you to use Redis as a message queue within your Python programs.', 37 | license = 'MIT', 38 | long_description = get_long_description(), 39 | install_requires = get_requirements(), 40 | py_modules = ['hotqueue'], 41 | url = 'http://github.com/richardhenry/hotqueue', 42 | version = get_version(), 43 | classifiers = [ 44 | 'Programming Language :: Python', 45 | 'Development Status :: 4 - Beta', 46 | 'Intended Audience :: Developers', 47 | 'License :: Public Domain', 48 | 'Operating System :: OS Independent', 49 | 'Topic :: Software Development :: Libraries :: Python Modules', 50 | ], 51 | ) 52 | 53 | -------------------------------------------------------------------------------- /docs/_themes/warm/static/pygments.css: -------------------------------------------------------------------------------- 1 | .c { color: #999988; font-style: italic } /* Comment */ 2 | .k { font-weight: bold } /* Keyword */ 3 | .o { font-weight: bold } /* Operator */ 4 | .cm { color: #999988; font-style: italic } /* Comment.Multiline */ 5 | .cp { color: #999999; font-weight: bold } /* Comment.preproc */ 6 | .c1 { color: #999988; font-style: italic } /* Comment.Single */ 7 | .gd { color: #000000; background-color: #ffdddd } /* Generic.Deleted */ 8 | .ge { font-style: italic } /* Generic.Emph */ 9 | .gr { color: #aa0000 } /* Generic.Error */ 10 | .gh { color: #999999 } /* Generic.Heading */ 11 | .gi { color: #000000; background-color: #ddffdd } /* Generic.Inserted */ 12 | .go { color: #111 } /* Generic.Output */ 13 | .gp { color: #555555 } /* Generic.Prompt */ 14 | .gs { font-weight: bold } /* Generic.Strong */ 15 | .gu { color: #aaaaaa } /* Generic.Subheading */ 16 | .gt { color: #aa0000 } /* Generic.Traceback */ 17 | .kc { font-weight: bold } /* Keyword.Constant */ 18 | .kd { font-weight: bold } /* Keyword.Declaration */ 19 | .kp { font-weight: bold } /* Keyword.Pseudo */ 20 | .kr { font-weight: bold } /* Keyword.Reserved */ 21 | .kt { color: #445588; font-weight: bold } /* Keyword.Type */ 22 | .m { color: #009999 } /* Literal.Number */ 23 | .s { color: #bb8844 } /* Literal.String */ 24 | .na { color: #008080 } /* Name.Attribute */ 25 | .nb { color: #999999 } /* Name.Builtin */ 26 | .nc { color: #445588; font-weight: bold } /* Name.Class */ 27 | .no { color: #ff99ff } /* Name.Constant */ 28 | .ni { color: #800080 } /* Name.Entity */ 29 | .ne { color: #990000; font-weight: bold } /* Name.Exception */ 30 | .nf { color: #990000; font-weight: bold } /* Name.Function */ 31 | .nn { color: #555555 } /* Name.Namespace */ 32 | .nt { color: #000080 } /* Name.Tag */ 33 | .nv { color: purple } /* Name.Variable */ 34 | .ow { font-weight: bold } /* Operator.Word */ 35 | .mf { color: #009999 } /* Literal.Number.Float */ 36 | .mh { color: #009999 } /* Literal.Number.Hex */ 37 | .mi { color: #009999 } /* Literal.Number.Integer */ 38 | .mo { color: #009999 } /* Literal.Number.Oct */ 39 | .sb { color: #bb8844 } /* Literal.String.Backtick */ 40 | .sc { color: #bb8844 } /* Literal.String.Char */ 41 | .sd { color: #bb8844 } /* Literal.String.Doc */ 42 | .s2 { color: #bb8844 } /* Literal.String.Double */ 43 | .se { color: #bb8844 } /* Literal.String.Escape */ 44 | .sh { color: #bb8844 } /* Literal.String.Heredoc */ 45 | .si { color: #bb8844 } /* Literal.String.Interpol */ 46 | .sx { color: #bb8844 } /* Literal.String.Other */ 47 | .sr { color: #808000 } /* Literal.String.Regex */ 48 | .s1 { color: #bb8844 } /* Literal.String.Single */ 49 | .ss { color: #bb8844 } /* Literal.String.Symbol */ 50 | .bp { color: #999999 } /* Name.Builtin.Pseudo */ 51 | .vc { color: #ff99ff } /* Name.Variable.Class */ 52 | .vg { color: #ff99ff } /* Name.Variable.Global */ 53 | .vi { color: #ff99ff } /* Name.Variable.Instance */ 54 | .il { color: #009999 } /* Literal.Number.Integer.Long */ -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | 15 | .PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest 16 | 17 | help: 18 | @echo "Please use \`make ' where is one of" 19 | @echo " html to make standalone HTML files" 20 | @echo " dirhtml to make HTML files named index.html in directories" 21 | @echo " pickle to make pickle files" 22 | @echo " json to make JSON files" 23 | @echo " htmlhelp to make HTML files and a HTML help project" 24 | @echo " qthelp to make HTML files and a qthelp project" 25 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 26 | @echo " changes to make an overview of all changed/added/deprecated items" 27 | @echo " linkcheck to check all external links for integrity" 28 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 29 | 30 | clean: 31 | -rm -rf $(BUILDDIR)/* 32 | 33 | html: 34 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 35 | @echo 36 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 37 | 38 | dirhtml: 39 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 40 | @echo 41 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 42 | 43 | pickle: 44 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 45 | @echo 46 | @echo "Build finished; now you can process the pickle files." 47 | 48 | json: 49 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 50 | @echo 51 | @echo "Build finished; now you can process the JSON files." 52 | 53 | htmlhelp: 54 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 55 | @echo 56 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 57 | ".hhp project file in $(BUILDDIR)/htmlhelp." 58 | 59 | qthelp: 60 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 61 | @echo 62 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 63 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 64 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/MongoEngine.qhcp" 65 | @echo "To view the help file:" 66 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/MongoEngine.qhc" 67 | 68 | latex: 69 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 70 | @echo 71 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 72 | @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ 73 | "run these through (pdf)latex." 74 | 75 | changes: 76 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 77 | @echo 78 | @echo "The overview file is in $(BUILDDIR)/changes." 79 | 80 | linkcheck: 81 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 82 | @echo 83 | @echo "Link check complete; look for any errors in the above output " \ 84 | "or in $(BUILDDIR)/linkcheck/output.txt." 85 | 86 | doctest: 87 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 88 | @echo "Testing of doctests in the sources finished, look at the " \ 89 | "results in $(BUILDDIR)/doctest/output.txt." 90 | -------------------------------------------------------------------------------- /docs/tutorial.rst: -------------------------------------------------------------------------------- 1 | ======== 2 | Tutorial 3 | ======== 4 | 5 | A HotQueue is a simple FIFO queue that maps to a list key in Redis. The following is a brief introduction explaining how you can use HotQueue in practice with a simple example. 6 | 7 | Connecting to Redis 8 | =================== 9 | 10 | Creating a queue is as simple as creating a :class:`~hotqueue.HotQueue` instance: 11 | 12 | >>> from hotqueue import HotQueue 13 | >>> queue = HotQueue('myqueue', host='localhost', port=6379, db=0) 14 | 15 | The queue will be stored a Redis list key named ``hotqueue:myqueue``, on the Redis server running at ``localhost:6379``, in database ``0``. The :attr:`host`, :attr:`port` and :attr:`db` arguments are optional. 16 | 17 | Putting Items Onto the Queue 18 | ============================ 19 | 20 | Then you may have one (or many) Python programs pushing to the queue using :meth:`hotqueue.HotQueue.put`: 21 | 22 | >>> queue.put(4) 23 | >>> queue.put(5) 24 | 25 | You can push more than one item onto the queue at once: 26 | 27 | >>> queue.put(6, 'my message', 7) 28 | 29 | You can safely push **any Python object** that can be `pickled `_. Let's use Python's built-in ``Decimal`` as an example: 30 | 31 | >>> from decimal import Decimal 32 | >>> queue.put(Decimal('1.4')) 33 | 34 | Getting Items Off the Queue 35 | =========================== 36 | 37 | You can then pull items off the queue using :meth:`hotqueue.HotQueue.get`. You would usually do this in another Python program, but you can do it wherever you like. 38 | 39 | >>> queue.get() 40 | 4 41 | >>> queue.get() 42 | 5 43 | >>> queue.get() 44 | 6 45 | >>> queue.get() 46 | 'my message' 47 | >>> queue.get() 48 | 7 49 | >>> dec = queue.get() 50 | >>> dec 51 | Decimal('1.4') 52 | >>> dec + Decimal('0.3') 53 | Decimal('1.7') 54 | 55 | Consuming the Queue 56 | =================== 57 | 58 | A better way to pull items off the queue is to use :meth:`hotqueue.HotQueue.consume`, which returns a generator that yields whenever an item is on the queue and blocks otherwise. Here's an example: 59 | 60 | >>> for item in queue.consume(): 61 | ... print item 62 | 63 | If you push to the queue using :meth:`hotqueue.HotQueue.put` in another Python program, you will see this program print the message then wait indefinitely for another. Replace the ``print`` statement with something more interesting, like saving a record to a database, and you've created an asynchronous task. 64 | 65 | Writing a Queue Worker 66 | ====================== 67 | 68 | An `even better` way to pull items off the queue is to use the :meth:`hotqueue.HotQueue.worker` decorator. Using this decorator is like wrapping the decorated function in a :meth:`hotqueue.HotQueue.consume` loop. Here's an example:: 69 | 70 | from hotqueue import HotQueue 71 | 72 | queue = HotQueue('myqueue', host='localhost', port=6379, db=0) 73 | 74 | @queue.worker 75 | def square(num): 76 | print num * num 77 | 78 | Then run the function: 79 | 80 | >>> square() 81 | 82 | It will wait indefinitely and print the square of any integers it pulls off the queue. Try pushing some integers to the queue in another Python program: 83 | 84 | >>> queue.put(2, 3, 4) 85 | 86 | To distribute the work, run a second instance of ``square()``. You now have two queue workers. You can run as many workers as you like, and no two workers will ever receive the same message. 87 | 88 | To run and manage your worker processes, you could use something like `Supervisord `_. 89 | -------------------------------------------------------------------------------- /tests.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """Test suite for the HotQueue library. To run this test suite, execute this 4 | Python program (``python tests.py``). Redis must be running on localhost:6379, 5 | and a list key named 'hotqueue:testqueue' will be created and deleted in db 0 6 | several times while the tests are running. 7 | """ 8 | 9 | from time import sleep 10 | import threading 11 | import unittest 12 | 13 | from hotqueue import HotQueue 14 | 15 | 16 | class HotQueueTestCase(unittest.TestCase): 17 | 18 | def setUp(self): 19 | """Create the queue instance before the test.""" 20 | self.queue = HotQueue('testqueue') 21 | 22 | def tearDown(self): 23 | """Clear the queue after the test.""" 24 | self.queue.clear() 25 | 26 | def test_consume(self): 27 | """Test the consume generator method.""" 28 | nums = [1, 2, 3, 4, 5, 6, 7, 8] 29 | # Test blocking with timeout: 30 | self.queue.put(*nums) 31 | msgs = [] 32 | for msg in self.queue.consume(timeout=1): 33 | msgs.append(msg) 34 | self.assertEquals(msgs, nums) 35 | # Test non-blocking: 36 | self.queue.put(*nums) 37 | msgs = [] 38 | for msg in self.queue.consume(block=False): 39 | msgs.append(msg) 40 | self.assertEquals(msgs, nums) 41 | 42 | def test_cleared(self): 43 | """Test for correct behaviour if the Redis list does not exist.""" 44 | self.assertEquals(len(self.queue), 0) 45 | self.assertEquals(self.queue.get(), None) 46 | 47 | def test_get_order(self): 48 | """Test that messages are get in the same order they are put.""" 49 | alphabet = ['abc', 'def', 'ghi', 'jkl', 'mno'] 50 | self.queue.put(alphabet[0], alphabet[1], alphabet[2]) 51 | self.queue.put(alphabet[3]) 52 | self.queue.put(alphabet[4]) 53 | msgs = [] 54 | msgs.append(self.queue.get()) 55 | msgs.append(self.queue.get()) 56 | msgs.append(self.queue.get()) 57 | msgs.append(self.queue.get()) 58 | msgs.append(self.queue.get()) 59 | self.assertEquals(msgs, alphabet) 60 | 61 | def test_length(self): 62 | """Test that the length of a queue is returned correctly.""" 63 | self.queue.put('a message') 64 | self.queue.put('another message') 65 | self.assertEquals(len(self.queue), 2) 66 | 67 | def test_worker(self): 68 | """Test the worker decorator.""" 69 | colors = ['blue', 'green', 'red', 'pink', 'black'] 70 | # Test blocking with timeout: 71 | self.queue.put(*colors) 72 | msgs = [] 73 | @self.queue.worker(timeout=1) 74 | def appender(msg): 75 | msgs.append(msg) 76 | appender() 77 | self.assertEqual(msgs, colors) 78 | # Test non-blocking: 79 | self.queue.put(*colors) 80 | msgs = [] 81 | @self.queue.worker(block=False) 82 | def appender(msg): 83 | msgs.append(msg) 84 | appender() 85 | self.assertEqual(msgs, colors) 86 | 87 | def test_threaded(self): 88 | """Threaded test of put and consume methods.""" 89 | msgs = [] 90 | def put(): 91 | for num in range(3): 92 | self.queue.put('message %d' % num) 93 | sleep(0.1) 94 | def consume(): 95 | for msg in self.queue.consume(timeout=1): 96 | msgs.append(msg) 97 | putter = threading.Thread(target=put) 98 | consumer = threading.Thread(target=consume) 99 | putter.start() 100 | consumer.start() 101 | for thread in [putter, consumer]: 102 | thread.join() 103 | self.assertEqual(msgs, ['message 0', 'message 1', 'message 2']) 104 | 105 | 106 | if __name__ == "__main__": 107 | unittest.main() 108 | 109 | -------------------------------------------------------------------------------- /hotqueue.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """HotQueue is a Python library that allows you to use Redis as a message queue 4 | within your Python programs. 5 | """ 6 | 7 | from functools import wraps 8 | import cPickle 9 | 10 | from redis import Redis 11 | 12 | 13 | __all__ = ['HotQueue'] 14 | 15 | __version__ = '0.2.0' 16 | 17 | 18 | class HotQueue(object): 19 | 20 | """Simple FIFO message queue stored in a Redis list. Example: 21 | 22 | >>> from hotqueue import HotQueue 23 | >>> queue = HotQueue('myqueue', host='localhost', port=6379, db=0) 24 | 25 | :param name: name of the queue 26 | :param kwargs: additional kwargs to pass to :class:`Redis`, most commonly 27 | :attr:`host`, :attr:`port`, :attr:`db` 28 | """ 29 | 30 | def __init__(self, name, **kwargs): 31 | self.name = name 32 | self.__redis = Redis(**kwargs) 33 | 34 | def __len__(self): 35 | return self.__redis.llen(self.key) 36 | 37 | def __repr__(self): 38 | return ('' % 39 | (self.name, self.__redis.host, self.__redis.port, self.__redis.db)) 40 | 41 | @property 42 | def key(self): 43 | """Return the key name used to store this queue in Redis, which is 44 | a concatenation of "hotqueue:" and :attr:`name`. 45 | """ 46 | return 'hotqueue:%s' % self.name 47 | 48 | def clear(self): 49 | """Clear the queue of all messages, deleting the Redis key.""" 50 | self.__redis.delete(self.key) 51 | 52 | def consume(self, **kwargs): 53 | """Return a generator that yields whenever a message is waiting in the 54 | queue. Will block otherwise. Example: 55 | 56 | >>> for msg in queue.consume(timeout=1): 57 | ... print msg 58 | my message 59 | another message 60 | 61 | :param kwargs: any arguments that :meth:`~hotqueue.HotQueue.get` can 62 | accept (:attr:`block` will default to ``True`` if not given) 63 | """ 64 | kwargs.setdefault('block', True) 65 | try: 66 | while True: 67 | msg = self.get(**kwargs) 68 | if msg is None: 69 | break 70 | yield msg 71 | except KeyboardInterrupt: 72 | print; return 73 | 74 | def get(self, block=False, timeout=None): 75 | """Return a message from the queue. Example: 76 | 77 | >>> queue.get() 78 | 'my message' 79 | >>> queue.get() 80 | 'another message' 81 | 82 | :param block: whether or not to wait until a msg is available in 83 | the queue before returning; ``False`` by default 84 | :param timeout: when using :attr:`block`, if no msg is available 85 | for :attr:`timeout` in seconds, give up and return ``None`` 86 | """ 87 | if block: 88 | if timeout is None: 89 | timeout = 0 90 | msg = self.__redis.blpop(self.key, timeout=timeout) 91 | if msg is not None: 92 | msg = msg[1] 93 | else: 94 | msg = self.__redis.lpop(self.key) 95 | if msg is not None: 96 | msg = cPickle.loads(msg) 97 | return msg 98 | 99 | def put(self, *msgs): 100 | """Put one or more messages onto the queue. Example: 101 | 102 | >>> queue.put('my message') 103 | >>> queue.put('another message') 104 | """ 105 | for msg in msgs: 106 | msg = cPickle.dumps(msg) 107 | self.__redis.rpush(self.key, msg) 108 | 109 | def worker(self, *args, **kwargs): 110 | """Decorator for using a function as a queue worker. Example: 111 | 112 | >>> @queue.worker(timeout=1) 113 | ... def printer(msg): 114 | ... print msg 115 | >>> printer() 116 | my message 117 | another message 118 | 119 | You can also use it without passing any keyword arguments: 120 | 121 | >>> @queue.worker 122 | ... def printer(msg): 123 | ... print msg 124 | >>> printer() 125 | my message 126 | another message 127 | 128 | :param kwargs: any arguments that :meth:`~hotqueue.HotQueue.get` can 129 | accept (:attr:`block` will default to ``True`` if not given) 130 | """ 131 | def decorator(worker): 132 | @wraps(worker) 133 | def wrapper(): 134 | for msg in self.consume(**kwargs): 135 | worker(msg) 136 | return wrapper 137 | if args: 138 | return decorator(*args) 139 | return decorator 140 | 141 | -------------------------------------------------------------------------------- /docs/_themes/warm/static/warm.css_t: -------------------------------------------------------------------------------- 1 | /** 2 | * Sphinx stylesheet -- warm theme 3 | * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 4 | */ 5 | 6 | @import url("basic.css"); 7 | 8 | /* -- page layout ----------------------------------------------------------- */ 9 | 10 | body { 11 | font-family: Arial, sans-serif; 12 | font-size: 100%; 13 | background-color: #111; 14 | color: #555; 15 | margin: 0; 16 | padding: 0; 17 | } 18 | 19 | div.documentwrapper { 20 | float: left; 21 | width: 100%; 22 | } 23 | 24 | div.bodywrapper { 25 | margin: 0 0 0 230px; 26 | } 27 | 28 | hr{ 29 | border: 1px solid #B1B4B6; 30 | } 31 | 32 | div.document { 33 | background-color: #eee; 34 | } 35 | 36 | div.body { 37 | background-color: #ffffff; 38 | color: #3E4349; 39 | padding: 0 30px 30px 30px; 40 | font-size: 0.8em; 41 | } 42 | 43 | div.footer { 44 | color: #555; 45 | width: 100%; 46 | padding: 13px 0; 47 | text-align: center; 48 | font-size: 75%; 49 | } 50 | 51 | div.footer a { 52 | color: #444; 53 | text-decoration: underline; 54 | } 55 | 56 | div.related { 57 | background-color: #6a5236; 58 | line-height: 32px; 59 | color: #fff; 60 | text-shadow: 0px 1px 0 #444; 61 | font-size: 0.80em; 62 | } 63 | 64 | div.related a { 65 | color: #f0e9e2; 66 | } 67 | 68 | div.sphinxsidebar { 69 | font-size: 0.75em; 70 | line-height: 1.5em; 71 | } 72 | 73 | div.sphinxsidebarwrapper{ 74 | padding: 20px 0; 75 | } 76 | 77 | div.sphinxsidebar h3, 78 | div.sphinxsidebar h4 { 79 | font-family: Arial, sans-serif; 80 | color: #222; 81 | font-size: 1.2em; 82 | font-weight: normal; 83 | margin: 0; 84 | padding: 5px 10px; 85 | background-color: #ddd; 86 | text-shadow: 1px 1px 0 white 87 | } 88 | 89 | div.sphinxsidebar h4{ 90 | font-size: 1.1em; 91 | } 92 | 93 | div.sphinxsidebar h3 a { 94 | color: #444; 95 | } 96 | 97 | 98 | div.sphinxsidebar p { 99 | color: #888; 100 | padding: 5px 20px; 101 | } 102 | 103 | div.sphinxsidebar p.topless { 104 | } 105 | 106 | div.sphinxsidebar ul { 107 | margin: 10px 20px; 108 | padding: 0; 109 | color: #000; 110 | } 111 | 112 | div.sphinxsidebar a { 113 | color: #444; 114 | } 115 | 116 | div.sphinxsidebar input { 117 | border: 1px solid #ccc; 118 | font-family: sans-serif; 119 | font-size: 1em; 120 | } 121 | 122 | div.sphinxsidebar input[type=text]{ 123 | margin-left: 20px; 124 | } 125 | 126 | /* -- body styles ----------------------------------------------------------- */ 127 | 128 | a { 129 | color: #005B81; 130 | text-decoration: none; 131 | } 132 | 133 | a:hover { 134 | color: #E32E00; 135 | text-decoration: underline; 136 | } 137 | 138 | div.body h1, 139 | div.body h2, 140 | div.body h3, 141 | div.body h4, 142 | div.body h5, 143 | div.body h6 { 144 | font-family: Arial, sans-serif; 145 | background-color: #BED4EB; 146 | font-weight: normal; 147 | color: #212224; 148 | margin: 30px 0px 10px 0px; 149 | padding: 5px 0 5px 10px; 150 | text-shadow: 0px 1px 0 white 151 | } 152 | 153 | div.body h1 { border-top: 20px solid white; margin-top: 0; font-size: 200%; } 154 | div.body h2 { font-size: 150%; background-color: #C8D5E3; } 155 | div.body h3 { font-size: 120%; background-color: #D8DEE3; } 156 | div.body h4 { font-size: 110%; background-color: #D8DEE3; } 157 | div.body h5 { font-size: 100%; background-color: #D8DEE3; } 158 | div.body h6 { font-size: 100%; background-color: #D8DEE3; } 159 | 160 | a.headerlink { 161 | color: #c60f0f; 162 | font-size: 0.8em; 163 | padding: 0 4px 0 4px; 164 | text-decoration: none; 165 | } 166 | 167 | a.headerlink:hover { 168 | background-color: #c60f0f; 169 | color: white; 170 | } 171 | 172 | div.body p, div.body dd, div.body li { 173 | line-height: 1.5em; 174 | } 175 | 176 | div.admonition p.admonition-title + p { 177 | display: inline; 178 | } 179 | 180 | div.highlight{ 181 | background-color: white; 182 | } 183 | 184 | div.note { 185 | background-color: #eee; 186 | border: 1px solid #ccc; 187 | } 188 | 189 | div.seealso { 190 | background-color: #ffc; 191 | border: 1px solid #ff6; 192 | } 193 | 194 | div.topic { 195 | background-color: #eee; 196 | } 197 | 198 | div.warning { 199 | background-color: #ffe4e4; 200 | border: 1px solid #f66; 201 | } 202 | 203 | p.admonition-title { 204 | display: inline; 205 | } 206 | 207 | p.admonition-title:after { 208 | content: ":"; 209 | } 210 | 211 | pre { 212 | padding: 10px; 213 | background-color: White; 214 | color: #222; 215 | line-height: 1.2em; 216 | border: 1px solid #C6C9CB; 217 | font-size: 1.2em; 218 | margin: 1.5em 0 1.5em 0; 219 | -webkit-box-shadow: 1px 1px 1px #d8d8d8; 220 | -moz-box-shadow: 1px 1px 1px #d8d8d8; 221 | } 222 | 223 | tt { 224 | background-color: #ecf0f3; 225 | color: #222; 226 | padding: 1px 2px; 227 | font-size: 1.2em; 228 | font-family: monospace; 229 | } 230 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # HotQueue documentation build configuration file, created by 4 | # sphinx-quickstart on Sun Nov 22 18:14:13 2009. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | import sys, os 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | sys.path.append(os.path.abspath('..')) 20 | 21 | # -- General configuration ----------------------------------------------------- 22 | 23 | # Add any Sphinx extension module names here, as strings. They can be extensions 24 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 25 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinxtogithub'] 26 | 27 | # Add any paths that contain templates here, relative to this directory. 28 | templates_path = ['_templates'] 29 | 30 | # The suffix of source filenames. 31 | source_suffix = '.rst' 32 | 33 | # The encoding of source files. 34 | #source_encoding = 'utf-8' 35 | 36 | # The master toctree document. 37 | master_doc = 'index' 38 | 39 | # General information about the project. 40 | project = u'HotQueue' 41 | copyright = u'2010, Richard Henry' 42 | 43 | # The version info for the project you're documenting, acts as replacement for 44 | # |version| and |release|, also used in various other places throughout the 45 | # built documents. 46 | # 47 | # The short X.Y version. 48 | version = '0.2.0' 49 | # The full version, including alpha/beta/rc tags. 50 | release = '0.2.0' 51 | 52 | # The language for content autogenerated by Sphinx. Refer to documentation 53 | # for a list of supported languages. 54 | #language = None 55 | 56 | # There are two options for replacing |today|: either, you set today to some 57 | # non-false value, then it is used: 58 | #today = '' 59 | # Else, today_fmt is used as the format for a strftime call. 60 | #today_fmt = '%B %d, %Y' 61 | 62 | # List of documents that shouldn't be included in the build. 63 | #unused_docs = [] 64 | 65 | # List of directories, relative to source directory, that shouldn't be searched 66 | # for source files. 67 | exclude_trees = ['_build'] 68 | 69 | # The reST default role (used for this markup: `text`) to use for all documents. 70 | #default_role = None 71 | 72 | # If true, '()' will be appended to :func: etc. cross-reference text. 73 | #add_function_parentheses = True 74 | 75 | # If true, the current module name will be prepended to all description 76 | # unit titles (such as .. function::). 77 | #add_module_names = True 78 | 79 | # If true, sectionauthor and moduleauthor directives will be shown in the 80 | # output. They are ignored by default. 81 | #show_authors = False 82 | 83 | # The name of the Pygments (syntax highlighting) style to use. 84 | pygments_style = 'sphinx' 85 | 86 | # A list of ignored prefixes for module index sorting. 87 | #modindex_common_prefix = [] 88 | 89 | 90 | # -- Options for HTML output --------------------------------------------------- 91 | 92 | # The theme to use for HTML and HTML Help pages. Major themes that come with 93 | # Sphinx are currently 'default' and 'sphinxdoc'. 94 | html_theme = 'warm' 95 | 96 | # Theme options are theme-specific and customize the look and feel of a theme 97 | # further. For a list of options available for each theme, see the 98 | # documentation. 99 | #html_theme_options = {} 100 | 101 | # Add any paths that contain custom themes here, relative to this directory. 102 | html_theme_path = ['_themes'] 103 | 104 | # The name for this set of Sphinx documents. If None, it defaults to 105 | # " v documentation". 106 | #html_title = None 107 | 108 | # A shorter title for the navigation bar. Default is the same as html_title. 109 | #html_short_title = None 110 | 111 | # The name of an image file (relative to this directory) to place at the top 112 | # of the sidebar. 113 | #html_logo = None 114 | 115 | # The name of an image file (within the static path) to use as favicon of the 116 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 117 | # pixels large. 118 | #html_favicon = None 119 | 120 | # Add any paths that contain custom static files (such as style sheets) here, 121 | # relative to this directory. They are copied after the builtin static files, 122 | # so a file named "default.css" will overwrite the builtin "default.css". 123 | html_static_path = ['_static'] 124 | 125 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 126 | # using the given strftime format. 127 | #html_last_updated_fmt = '%b %d, %Y' 128 | 129 | # If true, SmartyPants will be used to convert quotes and dashes to 130 | # typographically correct entities. 131 | html_use_smartypants = True 132 | 133 | # Custom sidebar templates, maps document names to template names. 134 | #html_sidebars = {} 135 | 136 | # Additional templates that should be rendered to pages, maps page names to 137 | # template names. 138 | #html_additional_pages = {} 139 | 140 | # If false, no module index is generated. 141 | #html_use_modindex = True 142 | 143 | # If false, no index is generated. 144 | #html_use_index = True 145 | 146 | # If true, the index is split into individual pages for each letter. 147 | #html_split_index = False 148 | 149 | # If true, links to the reST sources are added to the pages. 150 | #html_show_sourcelink = True 151 | 152 | # If true, an OpenSearch description file will be output, and all pages will 153 | # contain a tag referring to it. The value of this option must be the 154 | # base URL from which the finished HTML is served. 155 | #html_use_opensearch = '' 156 | 157 | # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). 158 | #html_file_suffix = '' 159 | 160 | # Output file base name for HTML help builder. 161 | htmlhelp_basename = 'HotQueuedoc' 162 | 163 | 164 | # -- Options for LaTeX output -------------------------------------------------- 165 | 166 | # The paper size ('letter' or 'a4'). 167 | latex_paper_size = 'a4' 168 | 169 | # The font size ('10pt', '11pt' or '12pt'). 170 | #latex_font_size = '10pt' 171 | 172 | # Grouping the document tree into LaTeX files. List of tuples 173 | # (source start file, target name, title, author, documentclass [howto/manual]). 174 | latex_documents = [ 175 | ('index', 'HotQueue.tex', u'HotQueue Documentation', 176 | u'Richard Henry', 'manual'), 177 | ] 178 | 179 | # The name of an image file (relative to this directory) to place at the top of 180 | # the title page. 181 | #latex_logo = None 182 | 183 | # For "manual" documents, if this is true, then toplevel headings are parts, 184 | # not chapters. 185 | #latex_use_parts = False 186 | 187 | # Additional stuff for the LaTeX preamble. 188 | #latex_preamble = '' 189 | 190 | # Documents to append as an appendix to all manuals. 191 | #latex_appendices = [] 192 | 193 | # If false, no module index is generated. 194 | #latex_use_modindex = True 195 | --------------------------------------------------------------------------------