├── .gitignore ├── Distributed_Computation_with_Python_and_Celery.pdf ├── LICENSE ├── README ├── celeryconfig.py ├── celeryconfig.py-db ├── demo.py └── tasks.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | misc/ 3 | result.db 4 | backend.db 5 | 6 | -------------------------------------------------------------------------------- /Distributed_Computation_with_Python_and_Celery.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/larsbutler/celery-examples/52682891d245e7ada9c6c0584267489aac55a9e7/Distributed_Computation_with_Python_and_Celery.pdf -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012, Lars Butler & contributors. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | * Redistributions of source code must retain the above copyright 7 | notice, this list of conditions and the following disclaimer. 8 | * Redistributions in binary form must reproduce the above copyright 9 | notice, this list of conditions and the following disclaimer in the 10 | documentation and/or other materials provided with the distribution. 11 | * Neither the name of Lars Butler nor the 12 | names of its contributors may be used to endorse or promote products 13 | derived from this software without specific prior written permission. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 17 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 18 | PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL Lars Butler OR CONTRIBUTORS 19 | BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 | POSSIBILITY OF SUCH DAMAGE. 26 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | This is a small and extremely lightweight demo of using Celery for distributed computation. 2 | 3 | 4 | Required packages: 5 | 6 | celery (http://celeryproject.org/) 7 | This example was tested with version 2.3.1 8 | 9 | Other requirements: 10 | 11 | RabbitMQ 12 | This example was tested with version 2.6.1 13 | 14 | Several RabbitMQ server installation packages are also available here: 15 | http://www.rabbitmq.com/server.html 16 | 17 | You will need to run the following commands to configure RabbitMQ: 18 | $ sudo rabbitmqctl add_user celeryuser celery 19 | $ sudo rabbitmqctl add_vhost celeryvhost 20 | $ sudo rabbitmqctl set_permissions -p celeryvhost celeryuser ".*" ".*" ".*" 21 | (Note: the user/password configuration here must match celeryconfig.py.) 22 | 23 | Starting RabbitMQ: 24 | $ sudo rabbitmq-server 25 | 26 | To run the demo: 27 | 28 | - start rabbitmq-server 29 | - cd into the directory containing celeryconfig.py 30 | - launch celery daemon: $ celeryd 31 | - in a separate terminal (in the same dir): $ python demo.py 32 | -------------------------------------------------------------------------------- /celeryconfig.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | sys.path.append('.') 4 | 5 | BROKER_HOST = "localhost" 6 | BROKER_PORT = 5672 7 | BROKER_USER = "celeryuser" 8 | BROKER_PASSWORD = "celery" 9 | BROKER_VHOST = "celeryvhost" 10 | 11 | CELERY_RESULT_BACKEND = "amqp" 12 | 13 | CELERY_IMPORTS = ("tasks",) 14 | -------------------------------------------------------------------------------- /celeryconfig.py-db: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | sys.path.append('.') 4 | 5 | BROKER_BACKEND = "sqlalchemy" 6 | BROKER_HOST = "sqlite:///backend.db" 7 | 8 | # optional: "database" is the default result backend 9 | # CELERY_RESULT_BACKEND = "database" 10 | CELERY_RESULT_DBURI = "sqlite:///result.db" 11 | 12 | # optional: enable verbose logging from SQLAlchemy. 13 | # CELERY_RESULT_ENGINE_OPTIONS = {"echo": True} 14 | 15 | CELERY_IMPORTS = ("tasks",) 16 | -------------------------------------------------------------------------------- /demo.py: -------------------------------------------------------------------------------- 1 | from celery.task.sets import TaskSet 2 | 3 | import tasks 4 | 5 | NUM_CALCS = [10**5, 10**6, 10**7] 6 | 7 | def make_pi_tasks(): 8 | 9 | taskset = TaskSet(tasks.make_pi.subtask((x, )) for x in NUM_CALCS) 10 | print "Dispatching tasks" 11 | taskset_result = taskset.apply_async() 12 | 13 | print "Waiting for results" 14 | results = taskset_result.join_native() 15 | print "Results:" 16 | for i in results: 17 | print i 18 | 19 | 20 | if __name__ == '__main__': 21 | print "Making pi" 22 | make_pi_tasks() 23 | -------------------------------------------------------------------------------- /tasks.py: -------------------------------------------------------------------------------- 1 | from celery.decorators import task 2 | 3 | @task 4 | def make_pi(num_calcs): 5 | """ 6 | Simple pi approximation based on the Leibniz formula for pi. 7 | http://en.wikipedia.org/wiki/Leibniz_formula_for_pi 8 | 9 | :param num_calcs: defines the length of the sequence 10 | :type num_calcs: positive int 11 | 12 | :returns: an approximation of pi 13 | """ 14 | print "Approximating pi with %s iterations" % num_calcs 15 | pi = 0.0 16 | for k in xrange(num_calcs): 17 | pi += 4 * ((-1)**k / ((2.0 * k) + 1)) 18 | return pi 19 | --------------------------------------------------------------------------------