├── docs ├── .static │ └── .keep ├── changelog.rst ├── images │ ├── monitor.png │ ├── celery_128.png │ ├── celery_512.png │ ├── dashboard.png │ ├── result_graph.png │ ├── celeryevshotsm.jpg │ └── worker_graph_full.png ├── _theme │ └── celery │ │ └── theme.conf ├── reference │ ├── celery.states.rst │ ├── celery.contrib.methods.rst │ ├── celery.contrib.rdb.rst │ ├── celery.events.rst │ ├── celery.result.rst │ ├── celery.security.rst │ ├── celery.app.log.rst │ ├── celery.beat.rst │ ├── celery.bin.base.rst │ ├── celery.app.task.rst │ ├── celery.app.utils.rst │ ├── celery.exceptions.rst │ ├── celery.worker.rst │ ├── celery.app.registry.rst │ ├── celery.loaders.app.rst │ ├── celery.utils.mail.rst │ ├── celery.contrib.migrate.rst │ ├── celery.loaders.rst │ ├── celery.task.http.rst │ ├── celery.worker.job.rst │ ├── celery.apps.worker.rst │ ├── celery.bootsteps.rst │ ├── celery.worker.state.rst │ ├── celery.bin.celery.rst │ ├── celery.bin.worker.rst │ ├── celery.bin.multi.rst │ ├── celery.worker.strategy.rst │ ├── celery.apps.beat.rst │ ├── celery.bin.beat.rst │ ├── celery.contrib.batches.rst │ ├── celery.loaders.base.rst │ ├── celery.signals.rst │ ├── celery.bin.graph.rst │ ├── celery.loaders.default.rst │ ├── celery.schedules.rst │ ├── celery.bin.events.rst │ ├── celery.app.builtins.rst │ ├── celery.app.control.rst │ ├── celery.bin.amqp.rst │ ├── celery.worker.consumer.rst │ ├── celery.app.defaults.rst │ ├── celery.events.state.rst │ ├── celery.contrib.abortable.rst │ ├── celery.app.rst │ ├── celery.utils.debug.rst │ ├── celery.app.amqp.rst │ └── index.rst ├── getting-started │ ├── resources.rst │ ├── index.rst │ └── brokers │ │ ├── django.rst │ │ ├── mongodb.rst │ │ ├── beanstalk.rst │ │ ├── couchdb.rst │ │ ├── ironmq.rst │ │ └── sqlalchemy.rst ├── tutorials │ ├── index.rst │ └── task-cookbook.rst ├── userguide │ ├── concurrency │ │ └── index.rst │ └── index.rst ├── django │ ├── index.rst │ └── unit-testing.rst ├── internals │ ├── reference │ │ ├── celery.utils.rst │ │ ├── celery.backends.rst │ │ ├── celery._state.rst │ │ ├── celery.app.routes.rst │ │ ├── celery.concurrency.rst │ │ ├── celery.platforms.rst │ │ ├── celery.utils.timer2.rst │ │ ├── celery.app.trace.rst │ │ ├── celery.worker.loops.rst │ │ ├── celery.backends.rpc.rst │ │ ├── celery.worker.pidbox.rst │ │ ├── celery.backends.amqp.rst │ │ ├── celery.security.key.rst │ │ ├── celery.events.dumper.rst │ │ ├── celery.utils.compat.rst │ │ ├── celery.utils.threads.rst │ │ ├── celery.backends.base.rst │ │ ├── celery.backends.cache.rst │ │ ├── celery.backends.redis.rst │ │ ├── celery.security.utils.rst │ │ ├── celery.utils.dispatch.rst │ │ ├── celery.utils.log.rst │ │ ├── celery.worker.autoreload.rst │ │ ├── celery.app.annotations.rst │ │ ├── celery.events.snapshot.rst │ │ ├── celery.utils.term.rst │ │ ├── celery.utils.text.rst │ │ ├── celery.worker.autoscale.rst │ │ ├── celery.worker.control.rst │ │ ├── celery.backends.mongodb.rst │ │ ├── celery.events.cursesmon.rst │ │ ├── celery.utils.iso8601.rst │ │ ├── celery.utils.objects.rst │ │ ├── celery.utils.sysinfo.rst │ │ ├── celery.worker.components.rst │ │ ├── celery.worker.heartbeat.rst │ │ ├── celery.backends.couchbase.rst │ │ ├── celery.concurrency.base.rst │ │ ├── celery.utils.imports.rst │ │ ├── celery.utils.timeutils.rst │ │ ├── celery.security.certificate.rst │ │ ├── celery.utils.serialization.rst │ │ ├── celery.backends.cassandra.rst │ │ ├── celery.backends.database.models.rst │ │ ├── celery.security.serialization.rst │ │ ├── celery.utils.functional.rst │ │ ├── celery.backends.database.rst │ │ ├── celery.backends.database.session.rst │ │ ├── celery.utils.dispatch.signal.rst │ │ ├── celery.concurrency.eventlet.rst │ │ ├── celery.concurrency.prefork.rst │ │ ├── celery.concurrency.solo.rst │ │ ├── celery.utils.dispatch.saferef.rst │ │ ├── celery.concurrency.gevent.rst │ │ ├── celery.concurrency.threads.rst │ │ ├── celery.datastructures.rst │ │ └── index.rst │ ├── index.rst │ └── worker.rst ├── templates │ └── readme.txt ├── THANKS ├── history │ └── index.rst ├── .templates │ ├── page.html │ ├── sidebarlogo.html │ └── sidebarintro.html ├── copyright.rst ├── community.rst ├── index.rst └── includes │ └── resources.txt ├── celery ├── apps │ └── __init__.py ├── contrib │ └── __init__.py ├── fixups │ └── __init__.py ├── tests │ ├── app │ │ ├── __init__.py │ │ ├── test_celery.py │ │ ├── test_utils.py │ │ ├── test_exceptions.py │ │ ├── test_annotations.py │ │ └── test_defaults.py │ ├── bin │ │ ├── __init__.py │ │ └── proj │ │ │ ├── app.py │ │ │ └── __init__.py │ ├── backends │ │ ├── __init__.py │ │ └── test_backends.py │ ├── contrib │ │ ├── __init__.py │ │ ├── test_methods.py │ │ └── test_abortable.py │ ├── events │ │ └── __init__.py │ ├── fixups │ │ └── __init__.py │ ├── slow │ │ └── __init__.py │ ├── tasks │ │ ├── __init__.py │ │ └── test_states.py │ ├── utils │ │ ├── __init__.py │ │ ├── test_encoding.py │ │ ├── test_sysinfo.py │ │ ├── test_serialization.py │ │ ├── test_imports.py │ │ ├── test_pickle.py │ │ └── test_mail.py │ ├── worker │ │ ├── __init__.py │ │ ├── test_revoke.py │ │ ├── test_components.py │ │ └── test_heartbeat.py │ ├── concurrency │ │ ├── __init__.py │ │ ├── test_solo.py │ │ └── test_threads.py │ ├── functional │ │ ├── __init__.py │ │ └── tasks.py │ ├── compat_modules │ │ ├── __init__.py │ │ ├── test_messaging.py │ │ ├── test_decorators.py │ │ └── test_compat_utils.py │ └── security │ │ ├── case.py │ │ └── test_key.py ├── utils │ ├── compat.py │ ├── dispatch │ │ ├── __init__.py │ │ └── license.txt │ ├── encoding.py │ ├── sysinfo.py │ ├── objects.py │ └── text.py ├── bin │ └── __init__.py ├── loaders │ ├── app.py │ ├── __init__.py │ └── default.py ├── task │ ├── trace.py │ └── __init__.py ├── security │ ├── key.py │ ├── utils.py │ └── __init__.py ├── concurrency │ ├── solo.py │ ├── __init__.py │ └── threads.py ├── __main__.py ├── app │ ├── annotations.py │ └── registry.py ├── backends │ ├── database │ │ └── session.py │ ├── rpc.py │ └── __init__.py └── worker │ └── heartbeat.py ├── examples ├── django │ ├── proj │ │ ├── __init__.py │ │ ├── celery.py │ │ ├── urls.py │ │ └── wsgi.py │ ├── demoapp │ │ ├── __init__.py │ │ ├── views.py │ │ ├── models.py │ │ ├── tasks.py │ │ └── tests.py │ ├── manage.py │ └── README.rst ├── httpexample │ ├── __init__.py │ ├── views.py │ ├── urls.py │ ├── manage.py │ └── README.rst ├── next-steps │ ├── proj │ │ ├── __init__.py │ │ ├── tasks.py │ │ └── celery.py │ └── setup.py ├── celery_http_gateway │ ├── __init__.py │ ├── tasks.py │ ├── manage.py │ ├── urls.py │ └── README.rst ├── tutorial │ └── tasks.py ├── eventlet │ ├── tasks.py │ ├── celeryconfig.py │ ├── README.rst │ ├── bulk_task_producer.py │ └── webcrawler.py ├── README.rst ├── gevent │ ├── celeryconfig.py │ └── tasks.py └── app │ └── myapp.py ├── requirements ├── extras │ ├── pyro.txt │ ├── auth.txt │ ├── couchdb.txt │ ├── gevent.txt │ ├── beanstalk.rst │ ├── beanstalk.txt │ ├── cassandra.txt │ ├── couchbase.txt │ ├── eventlet.txt │ ├── memcache.txt │ ├── redis.txt │ ├── sqs.txt │ ├── threads.txt │ ├── yaml.txt │ ├── zeromq.txt │ ├── mongodb.txt │ ├── sqlalchemy.txt │ ├── zookeeper.txt │ ├── librabbitmq.txt │ ├── msgpack.txt │ └── slmq.txt ├── jython.txt ├── security.txt ├── test3.txt ├── test.txt ├── docs.txt ├── default.txt ├── test-ci.txt ├── pkgutils.txt ├── dev.txt └── README.rst ├── TODO ├── extra ├── systemd │ ├── celery.conf │ └── celery.service ├── release │ ├── removepyc.sh │ ├── gen-cert.sh │ ├── attribution.py │ ├── verify-reference-index.sh │ ├── verify_config_reference.py │ ├── doc4allmods │ └── sphinx-to-rst.py ├── centos │ ├── celeryd.sysconfig │ └── test_celeryd.sh ├── osx │ ├── org.celeryq.beat.plist │ └── org.celeryq.worker.plist └── supervisord │ ├── celerybeat.conf │ ├── supervisord.conf │ └── celeryd.conf ├── funtests ├── setup.cfg ├── stress │ ├── stress │ │ ├── __init__.py │ │ ├── __main__.py │ │ ├── data.py │ │ └── fbi.py │ └── rabbit-restart-loop.sh ├── suite │ ├── __init__.py │ └── config.py └── setup.py ├── .travis.yml ├── .gitignore ├── Changelog ├── setup.cfg └── MANIFEST.in /docs/.static/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celery/apps/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celery/contrib/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celery/fixups/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celery/tests/app/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celery/tests/bin/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celery/tests/backends/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celery/tests/contrib/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celery/tests/events/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celery/tests/fixups/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celery/tests/slow/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celery/tests/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celery/tests/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celery/tests/worker/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/changelog.rst: -------------------------------------------------------------------------------- 1 | ../Changelog -------------------------------------------------------------------------------- /examples/django/proj/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/httpexample/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celery/tests/concurrency/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celery/tests/functional/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/django/demoapp/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/next-steps/proj/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements/extras/pyro.txt: -------------------------------------------------------------------------------- 1 | pyro4 2 | -------------------------------------------------------------------------------- /requirements/jython.txt: -------------------------------------------------------------------------------- 1 | threadpool 2 | -------------------------------------------------------------------------------- /celery/tests/compat_modules/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/celery_http_gateway/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements/extras/auth.txt: -------------------------------------------------------------------------------- 1 | pyOpenSSL 2 | -------------------------------------------------------------------------------- /requirements/extras/couchdb.txt: -------------------------------------------------------------------------------- 1 | couchdb 2 | -------------------------------------------------------------------------------- /requirements/extras/gevent.txt: -------------------------------------------------------------------------------- 1 | gevent 2 | -------------------------------------------------------------------------------- /requirements/security.txt: -------------------------------------------------------------------------------- 1 | PyOpenSSL 2 | -------------------------------------------------------------------------------- /requirements/extras/beanstalk.rst: -------------------------------------------------------------------------------- 1 | beanstalkc 2 | -------------------------------------------------------------------------------- /requirements/extras/beanstalk.txt: -------------------------------------------------------------------------------- 1 | beanstalkc 2 | -------------------------------------------------------------------------------- /requirements/extras/cassandra.txt: -------------------------------------------------------------------------------- 1 | pycassa 2 | -------------------------------------------------------------------------------- /requirements/extras/couchbase.txt: -------------------------------------------------------------------------------- 1 | couchbase 2 | -------------------------------------------------------------------------------- /requirements/extras/eventlet.txt: -------------------------------------------------------------------------------- 1 | eventlet 2 | -------------------------------------------------------------------------------- /requirements/extras/memcache.txt: -------------------------------------------------------------------------------- 1 | pylibmc 2 | -------------------------------------------------------------------------------- /requirements/extras/redis.txt: -------------------------------------------------------------------------------- 1 | redis>=2.8.0 2 | -------------------------------------------------------------------------------- /requirements/extras/sqs.txt: -------------------------------------------------------------------------------- 1 | boto>=2.13.3 2 | -------------------------------------------------------------------------------- /requirements/extras/threads.txt: -------------------------------------------------------------------------------- 1 | threadpool 2 | -------------------------------------------------------------------------------- /requirements/extras/yaml.txt: -------------------------------------------------------------------------------- 1 | PyYAML>=3.10 2 | -------------------------------------------------------------------------------- /requirements/extras/zeromq.txt: -------------------------------------------------------------------------------- 1 | pyzmq>=13.1.0 2 | -------------------------------------------------------------------------------- /requirements/extras/mongodb.txt: -------------------------------------------------------------------------------- 1 | pymongo>=2.6.2 2 | -------------------------------------------------------------------------------- /requirements/extras/sqlalchemy.txt: -------------------------------------------------------------------------------- 1 | sqlalchemy 2 | -------------------------------------------------------------------------------- /requirements/extras/zookeeper.txt: -------------------------------------------------------------------------------- 1 | kazoo>=1.3.1 2 | -------------------------------------------------------------------------------- /requirements/extras/librabbitmq.txt: -------------------------------------------------------------------------------- 1 | librabbitmq>=1.0.2 2 | -------------------------------------------------------------------------------- /requirements/extras/msgpack.txt: -------------------------------------------------------------------------------- 1 | msgpack-python>=0.3.0 2 | -------------------------------------------------------------------------------- /requirements/test3.txt: -------------------------------------------------------------------------------- 1 | nose 2 | nose-cover3 3 | mock 4 | -------------------------------------------------------------------------------- /celery/utils/compat.py: -------------------------------------------------------------------------------- 1 | from celery.five import * # noqa 2 | -------------------------------------------------------------------------------- /examples/django/demoapp/views.py: -------------------------------------------------------------------------------- 1 | # Create your views here. 2 | -------------------------------------------------------------------------------- /requirements/extras/slmq.txt: -------------------------------------------------------------------------------- 1 | softlayer_messaging>=1.0.3 2 | -------------------------------------------------------------------------------- /requirements/test.txt: -------------------------------------------------------------------------------- 1 | unittest2>=0.4.0 2 | nose 3 | nose-cover3 4 | mock 5 | -------------------------------------------------------------------------------- /requirements/docs.txt: -------------------------------------------------------------------------------- 1 | Sphinx 2 | sphinxcontrib-issuetracker>=0.9 3 | SQLAlchemy 4 | -------------------------------------------------------------------------------- /requirements/default.txt: -------------------------------------------------------------------------------- 1 | pytz>dev 2 | billiard>=3.3.0.3,<3.4 3 | kombu>=3.0.4,<4.0 4 | -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | Please see our Issue Tracker at GitHub: 2 | http://github.com/celery/celery/issues 3 | -------------------------------------------------------------------------------- /docs/images/monitor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dz0ny/celery/master/docs/images/monitor.png -------------------------------------------------------------------------------- /extra/systemd/celery.conf: -------------------------------------------------------------------------------- 1 | d /run/celery 0755 user users - 2 | d /var/log/celery 0755 user users - -------------------------------------------------------------------------------- /funtests/setup.cfg: -------------------------------------------------------------------------------- 1 | [nosetests] 2 | verbosity = 1 3 | detailed-errors = 1 4 | where = suite 5 | -------------------------------------------------------------------------------- /requirements/test-ci.txt: -------------------------------------------------------------------------------- 1 | coverage>=3.0 2 | redis 3 | #pymongo 4 | #SQLAlchemy 5 | #PyOpenSSL 6 | -------------------------------------------------------------------------------- /docs/images/celery_128.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dz0ny/celery/master/docs/images/celery_128.png -------------------------------------------------------------------------------- /docs/images/celery_512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dz0ny/celery/master/docs/images/celery_512.png -------------------------------------------------------------------------------- /docs/images/dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dz0ny/celery/master/docs/images/dashboard.png -------------------------------------------------------------------------------- /docs/_theme/celery/theme.conf: -------------------------------------------------------------------------------- 1 | [theme] 2 | inherit = basic 3 | stylesheet = celery.css 4 | 5 | [options] 6 | -------------------------------------------------------------------------------- /docs/images/result_graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dz0ny/celery/master/docs/images/result_graph.png -------------------------------------------------------------------------------- /docs/images/celeryevshotsm.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dz0ny/celery/master/docs/images/celeryevshotsm.jpg -------------------------------------------------------------------------------- /examples/django/demoapp/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models # noqa 2 | 3 | # Create your models here. 4 | -------------------------------------------------------------------------------- /requirements/pkgutils.txt: -------------------------------------------------------------------------------- 1 | paver 2 | flake8 3 | flakeplus 4 | tox 5 | Sphinx-PyPI-upload 6 | bundle>=1.1.0 7 | -------------------------------------------------------------------------------- /docs/images/worker_graph_full.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dz0ny/celery/master/docs/images/worker_graph_full.png -------------------------------------------------------------------------------- /celery/bin/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from .base import Option 4 | 5 | __all__ = ['Option'] 6 | -------------------------------------------------------------------------------- /extra/release/removepyc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | (cd "${1:-.}"; 3 | find . -name "*.pyc" | xargs rm -- 2>/dev/null) || echo "ok" 4 | -------------------------------------------------------------------------------- /funtests/stress/stress/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import absolute_import 3 | 4 | from .app import app # noqa 5 | -------------------------------------------------------------------------------- /celery/tests/bin/proj/app.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from celery import Celery 4 | 5 | app = Celery(set_as_current=False) 6 | -------------------------------------------------------------------------------- /celery/tests/bin/proj/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from celery import Celery 4 | 5 | hello = Celery(set_as_current=False) 6 | -------------------------------------------------------------------------------- /examples/celery_http_gateway/tasks.py: -------------------------------------------------------------------------------- 1 | from celery import task 2 | 3 | 4 | @task() 5 | def hello_world(to='world'): 6 | return 'Hello {0}'.format(to) 7 | -------------------------------------------------------------------------------- /celery/utils/dispatch/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import absolute_import 3 | 4 | from .signal import Signal 5 | 6 | __all__ = ['Signal'] 7 | -------------------------------------------------------------------------------- /docs/reference/celery.states.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: celery.states 2 | 3 | .. contents:: 4 | :local: 5 | 6 | .. automodule:: celery.states 7 | :members: 8 | 9 | -------------------------------------------------------------------------------- /docs/reference/celery.contrib.methods.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: celery.contrib.methods 2 | 3 | .. automodule:: celery.contrib.methods 4 | :members: 5 | :undoc-members: 6 | -------------------------------------------------------------------------------- /requirements/dev.txt: -------------------------------------------------------------------------------- 1 | https://github.com/celery/py-amqp/zipball/master 2 | https://github.com/celery/billiard/zipball/master 3 | https://github.com/celery/kombu/zipball/master 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - 2.6 4 | - 2.7 5 | - 3.3 6 | install: 7 | - pip install --use-mirrors tox 8 | script: TOXENV=py$(echo $TRAVIS_PYTHON_VERSION | tr -d .) tox -v 9 | -------------------------------------------------------------------------------- /docs/getting-started/resources.rst: -------------------------------------------------------------------------------- 1 | .. _resources: 2 | 3 | =========== 4 | Resources 5 | =========== 6 | 7 | .. contents:: 8 | :local: 9 | :depth: 2 10 | 11 | .. include:: ../includes/resources.txt 12 | -------------------------------------------------------------------------------- /docs/reference/celery.contrib.rdb.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: celery.contrib.rdb 2 | 3 | .. automodule:: celery.contrib.rdb 4 | 5 | .. autofunction:: set_trace 6 | .. autofunction:: debugger 7 | .. autoclass:: Rdb 8 | -------------------------------------------------------------------------------- /docs/tutorials/index.rst: -------------------------------------------------------------------------------- 1 | =========== 2 | Tutorials 3 | =========== 4 | 5 | :Release: |version| 6 | :Date: |today| 7 | 8 | .. toctree:: 9 | :maxdepth: 2 10 | 11 | daemonizing 12 | debugging 13 | task-cookbook 14 | -------------------------------------------------------------------------------- /docs/userguide/concurrency/index.rst: -------------------------------------------------------------------------------- 1 | .. _concurrency: 2 | 3 | ============= 4 | Concurrency 5 | ============= 6 | 7 | :Release: |version| 8 | :Date: |today| 9 | 10 | .. toctree:: 11 | :maxdepth: 2 12 | 13 | eventlet 14 | -------------------------------------------------------------------------------- /docs/django/index.rst: -------------------------------------------------------------------------------- 1 | .. _django: 2 | 3 | ========= 4 | Django 5 | ========= 6 | 7 | :Release: |version| 8 | :Date: |today| 9 | 10 | .. toctree:: 11 | :maxdepth: 2 12 | 13 | first-steps-with-django 14 | unit-testing 15 | 16 | -------------------------------------------------------------------------------- /docs/reference/celery.events.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | celery.events 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.events 8 | 9 | .. automodule:: celery.events 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.rst: -------------------------------------------------------------------------------- 1 | ========================== 2 | celery.utils 3 | ========================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils 8 | 9 | .. automodule:: celery.utils 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.result.rst: -------------------------------------------------------------------------------- 1 | ============================= 2 | celery.result 3 | ============================= 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.result 8 | 9 | .. automodule:: celery.result 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.security.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | celery.security 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.security 8 | 9 | .. automodule:: celery.security 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /examples/tutorial/tasks.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from celery import Celery 4 | 5 | app = Celery('tasks', broker='amqp://') 6 | 7 | 8 | @app.task() 9 | def add(x, y): 10 | return x + y 11 | 12 | if __name__ == '__main__': 13 | app.start() 14 | -------------------------------------------------------------------------------- /docs/reference/celery.app.log.rst: -------------------------------------------------------------------------------- 1 | ================================ 2 | celery.app.log 3 | ================================ 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.app.log 8 | 9 | .. automodule:: celery.app.log 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /examples/django/demoapp/tasks.py: -------------------------------------------------------------------------------- 1 | from celery import shared_task 2 | 3 | 4 | @shared_task 5 | def add(x, y): 6 | return x + y 7 | 8 | 9 | @shared_task 10 | def mul(x, y): 11 | return x * y 12 | 13 | 14 | @shared_task 15 | def xsum(numbers): 16 | return sum(numbers) 17 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.backends.rst: -------------------------------------------------------------------------------- 1 | =========================== 2 | celery.backends 3 | =========================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.backends 8 | 9 | .. automodule:: celery.backends 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.beat.rst: -------------------------------------------------------------------------------- 1 | ======================================== 2 | celery.beat 3 | ======================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.beat 8 | 9 | .. automodule:: celery.beat 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.bin.base.rst: -------------------------------------------------------------------------------- 1 | ================================ 2 | celery.bin.base 3 | ================================ 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.bin.base 8 | 9 | .. automodule:: celery.bin.base 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.app.task.rst: -------------------------------------------------------------------------------- 1 | =================================== 2 | celery.app.task 3 | =================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.app.task 8 | 9 | .. automodule:: celery.app.task 10 | :members: Task, Context, TaskType 11 | -------------------------------------------------------------------------------- /docs/reference/celery.app.utils.rst: -------------------------------------------------------------------------------- 1 | ================================ 2 | celery.app.utils 3 | ================================ 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.app.utils 8 | 9 | .. automodule:: celery.app.utils 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.exceptions.rst: -------------------------------------------------------------------------------- 1 | ================================ 2 | celery.exceptions 3 | ================================ 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.exceptions 8 | 9 | .. automodule:: celery.exceptions 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.worker.rst: -------------------------------------------------------------------------------- 1 | ======================================== 2 | celery.worker 3 | ======================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.worker 8 | 9 | .. automodule:: celery.worker 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.app.registry.rst: -------------------------------------------------------------------------------- 1 | ================================ 2 | celery.app.registry 3 | ================================ 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.app.registry 8 | 9 | .. automodule:: celery.app.registry 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.loaders.app.rst: -------------------------------------------------------------------------------- 1 | ================================= 2 | celery.loaders.app 3 | ================================= 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.loaders.app 8 | 9 | .. automodule:: celery.loaders.app 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.utils.mail.rst: -------------------------------------------------------------------------------- 1 | ==================================== 2 | celery.utils.mail 3 | ==================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.mail 8 | 9 | .. automodule:: celery.utils.mail 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/getting-started/index.rst: -------------------------------------------------------------------------------- 1 | ================= 2 | Getting Started 3 | ================= 4 | 5 | :Release: |version| 6 | :Date: |today| 7 | 8 | .. toctree:: 9 | :maxdepth: 2 10 | 11 | introduction 12 | brokers/index 13 | first-steps-with-celery 14 | next-steps 15 | resources 16 | -------------------------------------------------------------------------------- /docs/internals/reference/celery._state.rst: -------------------------------------------------------------------------------- 1 | ======================================== 2 | celery._state 3 | ======================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery._state 8 | 9 | .. automodule:: celery._state 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.app.routes.rst: -------------------------------------------------------------------------------- 1 | ================================= 2 | celery.app.routes 3 | ================================= 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.app.routes 8 | 9 | .. automodule:: celery.app.routes 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.contrib.migrate.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | celery.contrib.migrate 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | 8 | .. currentmodule:: celery.contrib.migrate 9 | 10 | .. automodule:: celery.contrib.migrate 11 | :members: 12 | :undoc-members: 13 | -------------------------------------------------------------------------------- /docs/reference/celery.loaders.rst: -------------------------------------------------------------------------------- 1 | ============================================ 2 | celery.loaders 3 | ============================================ 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.loaders 8 | 9 | .. automodule:: celery.loaders 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.task.http.rst: -------------------------------------------------------------------------------- 1 | ======================================== 2 | celery.task.http 3 | ======================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.task.http 8 | 9 | .. automodule:: celery.task.http 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.worker.job.rst: -------------------------------------------------------------------------------- 1 | ===================================== 2 | celery.worker.job 3 | ===================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.worker.job 8 | 9 | .. automodule:: celery.worker.job 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /examples/django/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "proj.settings") 7 | 8 | from django.core.management import execute_from_command_line 9 | 10 | execute_from_command_line(sys.argv) 11 | -------------------------------------------------------------------------------- /funtests/stress/rabbit-restart-loop.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | secs=${1:-30} 4 | secs=$((secs - 1)) 5 | 6 | while true; do 7 | sudo rabbitmqctl start_app 8 | echo "sleep for ${secs}s" 9 | sleep $secs 10 | sudo rabbitmqctl stop_app 11 | echo "sleep for 1s" 12 | sleep 1 13 | done 14 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.concurrency.rst: -------------------------------------------------------------------------------- 1 | ================================== 2 | celery.concurrency 3 | ================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.concurrency 8 | 9 | .. automodule:: celery.concurrency 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.platforms.rst: -------------------------------------------------------------------------------- 1 | ====================================== 2 | celery.platforms 3 | ====================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.platforms 8 | 9 | .. automodule:: celery.platforms 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.timer2.rst: -------------------------------------------------------------------------------- 1 | ============================== 2 | celery.utils.timer2 3 | ============================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.timer2 8 | 9 | .. automodule:: celery.utils.timer2 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.apps.worker.rst: -------------------------------------------------------------------------------- 1 | ======================================= 2 | celery.apps.worker 3 | ======================================= 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.apps.worker 8 | 9 | .. automodule:: celery.apps.worker 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.bootsteps.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | celery.bootsteps 3 | ========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.bootsteps 8 | 9 | .. automodule:: celery.bootsteps 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.worker.state.rst: -------------------------------------------------------------------------------- 1 | ==================================== 2 | celery.worker.state 3 | ==================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.worker.state 8 | 9 | .. automodule:: celery.worker.state 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.bin.celery.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | celery.bin.celery 3 | ========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.bin.celery 8 | 9 | .. automodule:: celery.bin.celery 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.bin.worker.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | celery.bin.worker 3 | ========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.bin.worker 8 | 9 | .. automodule:: celery.bin.worker 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/index.rst: -------------------------------------------------------------------------------- 1 | .. _internals: 2 | 3 | =========== 4 | Internals 5 | =========== 6 | 7 | :Release: |version| 8 | :Date: |today| 9 | 10 | .. toctree:: 11 | :maxdepth: 2 12 | 13 | guide 14 | deprecation 15 | worker 16 | protocol 17 | protov2 18 | app-overview 19 | reference/index 20 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.app.trace.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | celery.app.trace 3 | ========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.app.trace 8 | 9 | .. automodule:: celery.app.trace 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.worker.loops.rst: -------------------------------------------------------------------------------- 1 | ==================================== 2 | celery.worker.loops 3 | ==================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.worker.loops 8 | 9 | .. automodule:: celery.worker.loops 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.bin.multi.rst: -------------------------------------------------------------------------------- 1 | =============================================== 2 | celery.bin.multi 3 | =============================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.bin.multi 8 | 9 | .. automodule:: celery.bin.multi 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.worker.strategy.rst: -------------------------------------------------------------------------------- 1 | ==================================== 2 | celery.worker.strategy 3 | ==================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.worker.strategy 8 | 9 | .. automodule:: celery.worker.strategy 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /examples/next-steps/proj/tasks.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from proj.celery import app 4 | 5 | 6 | @app.task 7 | def add(x, y): 8 | return x + y 9 | 10 | 11 | @app.task 12 | def mul(x, y): 13 | return x * y 14 | 15 | 16 | @app.task 17 | def xsum(numbers): 18 | return sum(numbers) 19 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.backends.rpc.rst: -------------------------------------------------------------------------------- 1 | ======================================= 2 | celery.backends.rpc 3 | ======================================= 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.backends.rpc 8 | 9 | .. automodule:: celery.backends.rpc 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.worker.pidbox.rst: -------------------------------------------------------------------------------- 1 | ==================================== 2 | celery.worker.pidbox 3 | ==================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.worker.pidbox 8 | 9 | .. automodule:: celery.worker.pidbox 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.apps.beat.rst: -------------------------------------------------------------------------------- 1 | ================================================= 2 | celery.apps.beat 3 | ================================================= 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.apps.beat 8 | 9 | .. automodule:: celery.apps.beat 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.bin.beat.rst: -------------------------------------------------------------------------------- 1 | =================================================== 2 | celery.bin.beat 3 | =================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.bin.beat 8 | 9 | .. automodule:: celery.bin.beat 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.contrib.batches.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: celery.contrib.batches 2 | 3 | .. automodule:: celery.contrib.batches 4 | 5 | **API** 6 | 7 | .. autoclass:: Batches 8 | :members: 9 | :undoc-members: 10 | .. autoclass:: SimpleRequest 11 | :members: 12 | :undoc-members: 13 | -------------------------------------------------------------------------------- /docs/reference/celery.loaders.base.rst: -------------------------------------------------------------------------------- 1 | =========================================== 2 | celery.loaders.base 3 | =========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.loaders.base 8 | 9 | .. automodule:: celery.loaders.base 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.signals.rst: -------------------------------------------------------------------------------- 1 | ====================================================== 2 | celery.signals 3 | ====================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.signals 8 | 9 | .. automodule:: celery.signals 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.backends.amqp.rst: -------------------------------------------------------------------------------- 1 | ======================================= 2 | celery.backends.amqp 3 | ======================================= 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.backends.amqp 8 | 9 | .. automodule:: celery.backends.amqp 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.security.key.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | celery.security.key 3 | ========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.security.key 8 | 9 | .. automodule:: celery.security.key 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.bin.graph.rst: -------------------------------------------------------------------------------- 1 | ===================================================== 2 | celery.bin.graph 3 | ===================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.bin.graph 8 | 9 | .. automodule:: celery.bin.graph 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.loaders.default.rst: -------------------------------------------------------------------------------- 1 | ========================================= 2 | celery.loaders.default 3 | ========================================= 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.loaders.default 8 | 9 | .. automodule:: celery.loaders.default 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.schedules.rst: -------------------------------------------------------------------------------- 1 | ===================================================== 2 | celery.schedules 3 | ===================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.schedules 8 | 9 | .. automodule:: celery.schedules 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.events.dumper.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | celery.events.dumper 3 | ========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.events.dumper 8 | 9 | .. automodule:: celery.events.dumper 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.compat.rst: -------------------------------------------------------------------------------- 1 | ============================================ 2 | celery.utils.compat 3 | ============================================ 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.compat 8 | 9 | .. automodule:: celery.utils.compat 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.threads.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | celery.utils.threads 3 | ========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.threads 8 | 9 | .. automodule:: celery.utils.threads 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.bin.events.rst: -------------------------------------------------------------------------------- 1 | ===================================================== 2 | celery.bin.events 3 | ===================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.bin.events 8 | 9 | .. automodule:: celery.bin.events 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /funtests/suite/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | sys.path.insert(0, os.path.join(os.getcwd(), os.pardir)) 5 | 6 | config = os.environ.setdefault('CELERY_FUNTEST_CONFIG_MODULE', 7 | 'suite.config') 8 | 9 | os.environ['CELERY_CONFIG_MODULE'] = config 10 | os.environ['CELERY_LOADER'] = 'default' 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.pyc 3 | *$py.class 4 | *~ 5 | .*.sw[pon] 6 | dist/ 7 | *.egg-info 8 | *.egg 9 | *.egg/ 10 | doc/__build/* 11 | build/ 12 | .build/ 13 | pip-log.txt 14 | .directory 15 | erl_crash.dump 16 | *.db 17 | Documentation/ 18 | .tox/ 19 | .ropeproject/ 20 | .project 21 | .pydevproject 22 | .idea/ 23 | celery/tests/cover/ 24 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.backends.base.rst: -------------------------------------------------------------------------------- 1 | ===================================== 2 | celery.backends.base 3 | ===================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.backends.base 8 | 9 | .. automodule:: celery.backends.base 10 | :members: 11 | :undoc-members: 12 | 13 | 14 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.backends.cache.rst: -------------------------------------------------------------------------------- 1 | =========================================== 2 | celery.backends.cache 3 | =========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.backends.cache 8 | 9 | .. automodule:: celery.backends.cache 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.backends.redis.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | celery.backends.redis 3 | ========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.backends.redis 8 | 9 | .. automodule:: celery.backends.redis 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.security.utils.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | celery.security.utils 3 | ========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.security.utils 8 | 9 | .. automodule:: celery.security.utils 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.dispatch.rst: -------------------------------------------------------------------------------- 1 | ========================================= 2 | celery.utils.dispatch 3 | ========================================= 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.dispatch 8 | 9 | .. automodule:: celery.utils.dispatch 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.log.rst: -------------------------------------------------------------------------------- 1 | ===================================================== 2 | celery.utils.log 3 | ===================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.log 8 | 9 | .. automodule:: celery.utils.log 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.worker.autoreload.rst: -------------------------------------------------------------------------------- 1 | ==================================== 2 | celery.worker.autoreload 3 | ==================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.worker.autoreload 8 | 9 | .. automodule:: celery.worker.autoreload 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.app.builtins.rst: -------------------------------------------------------------------------------- 1 | ==================================================== 2 | celery.app.builtins 3 | ==================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.app.builtins 8 | 9 | .. automodule:: celery.app.builtins 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.app.control.rst: -------------------------------------------------------------------------------- 1 | ==================================================== 2 | celery.app.control 3 | ==================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.app.control 8 | 9 | .. automodule:: celery.app.control 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.bin.amqp.rst: -------------------------------------------------------------------------------- 1 | =========================================================== 2 | celery.bin.amqp 3 | =========================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.bin.amqp 8 | 9 | .. automodule:: celery.bin.amqp 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.app.annotations.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | celery.app.annotations 3 | ========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.app.annotations 8 | 9 | .. automodule:: celery.app.annotations 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.events.snapshot.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | celery.events.snapshot 3 | ========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.events.snapshot 8 | 9 | .. automodule:: celery.events.snapshot 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.term.rst: -------------------------------------------------------------------------------- 1 | ===================================================== 2 | celery.utils.term 3 | ===================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.term 8 | 9 | .. automodule:: celery.utils.term 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.text.rst: -------------------------------------------------------------------------------- 1 | ===================================================== 2 | celery.utils.text 3 | ===================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.text 8 | 9 | .. automodule:: celery.utils.text 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.worker.autoscale.rst: -------------------------------------------------------------------------------- 1 | ======================================== 2 | celery.worker.autoscale 3 | ======================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.worker.autoscale 8 | 9 | .. automodule:: celery.worker.autoscale 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.worker.control.rst: -------------------------------------------------------------------------------- 1 | ============================================= 2 | celery.worker.control 3 | ============================================= 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.worker.control 8 | 9 | .. automodule:: celery.worker.control 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.backends.mongodb.rst: -------------------------------------------------------------------------------- 1 | ============================================ 2 | celery.backends.mongodb 3 | ============================================ 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.backends.mongodb 8 | 9 | .. automodule:: celery.backends.mongodb 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.events.cursesmon.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | celery.events.cursesmon 3 | ========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.events.cursesmon 8 | 9 | .. automodule:: celery.events.cursesmon 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.iso8601.rst: -------------------------------------------------------------------------------- 1 | ================================================== 2 | celery.utils.iso8601 3 | ================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.iso8601 8 | 9 | .. automodule:: celery.utils.iso8601 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.objects.rst: -------------------------------------------------------------------------------- 1 | ================================================== 2 | celery.utils.objects 3 | ================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.objects 8 | 9 | .. automodule:: celery.utils.objects 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.sysinfo.rst: -------------------------------------------------------------------------------- 1 | ================================================== 2 | celery.utils.sysinfo 3 | ================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.sysinfo 8 | 9 | .. automodule:: celery.utils.sysinfo 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.worker.components.rst: -------------------------------------------------------------------------------- 1 | ======================================== 2 | celery.worker.components 3 | ======================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.worker.components 8 | 9 | .. automodule:: celery.worker.components 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.worker.consumer.rst: -------------------------------------------------------------------------------- 1 | ================================================== 2 | celery.worker.consumer 3 | ================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.worker.consumer 8 | 9 | .. automodule:: celery.worker.consumer 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /examples/eventlet/tasks.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | from celery import task 4 | 5 | 6 | @task() 7 | def urlopen(url): 8 | print('Opening: {0}'.format(url)) 9 | try: 10 | response = requests.get(url) 11 | except Exception as exc: 12 | print('URL {0} gave error: {1!r}'.format(url, exc)) 13 | return len(response.text) 14 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.worker.heartbeat.rst: -------------------------------------------------------------------------------- 1 | ============================================= 2 | celery.worker.heartbeat 3 | ============================================= 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.worker.heartbeat 8 | 9 | .. automodule:: celery.worker.heartbeat 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /celery/loaders/app.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | celery.loaders.app 4 | ~~~~~~~~~~~~~~~~~~ 5 | 6 | The default loader used with custom app instances. 7 | 8 | """ 9 | from __future__ import absolute_import 10 | 11 | from .base import BaseLoader 12 | 13 | __all__ = ['AppLoader'] 14 | 15 | 16 | class AppLoader(BaseLoader): 17 | pass 18 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.backends.couchbase.rst: -------------------------------------------------------------------------------- 1 | ============================================ 2 | celery.backends.couchbase 3 | ============================================ 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.backends.couchbase 8 | 9 | .. automodule:: celery.backends.couchbase 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.concurrency.base.rst: -------------------------------------------------------------------------------- 1 | =============================================== 2 | celery.concurrency.base 3 | =============================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.concurrency.base 8 | 9 | .. automodule:: celery.concurrency.base 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.imports.rst: -------------------------------------------------------------------------------- 1 | ===================================================== 2 | celery.utils.imports 3 | ===================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.imports 8 | 9 | .. automodule:: celery.utils.imports 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.timeutils.rst: -------------------------------------------------------------------------------- 1 | ================================================== 2 | celery.utils.timeutils 3 | ================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.timeutils 8 | 9 | .. automodule:: celery.utils.timeutils 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.security.certificate.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | celery.security.certificate 3 | ========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.security.certificate 8 | 9 | .. automodule:: celery.security.certificate 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.serialization.rst: -------------------------------------------------------------------------------- 1 | ============================================ 2 | celery.utils.serialization 3 | ============================================ 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.serialization 8 | 9 | .. automodule:: celery.utils.serialization 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.app.defaults.rst: -------------------------------------------------------------------------------- 1 | =============================================================== 2 | celery.app.defaults 3 | =============================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.app.defaults 8 | 9 | .. automodule:: celery.app.defaults 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/templates/readme.txt: -------------------------------------------------------------------------------- 1 | ================================= 2 | celery - Distributed Task Queue 3 | ================================= 4 | 5 | .. image:: http://cloud.github.com/downloads/celery/celery/celery_128.png 6 | 7 | .. include:: ../includes/introduction.txt 8 | 9 | .. include:: ../includes/installation.txt 10 | 11 | .. include:: ../includes/resources.txt 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.backends.cassandra.rst: -------------------------------------------------------------------------------- 1 | ================================================ 2 | celery.backends.cassandra 3 | ================================================ 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.backends.cassandra 8 | 9 | .. automodule:: celery.backends.cassandra 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.backends.database.models.rst: -------------------------------------------------------------------------------- 1 | ====================================== 2 | celery.backends.database.models 3 | ====================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.backends.database.models 8 | 9 | .. automodule:: celery.backends.database.models 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.security.serialization.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | celery.security.serialization 3 | ========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.security.serialization 8 | 9 | .. automodule:: celery.security.serialization 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.functional.rst: -------------------------------------------------------------------------------- 1 | ===================================================== 2 | celery.utils.functional 3 | ===================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.functional 8 | 9 | .. automodule:: celery.utils.functional 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.events.state.rst: -------------------------------------------------------------------------------- 1 | ================================================================= 2 | celery.events.state 3 | ================================================================= 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.events.state 8 | 9 | .. automodule:: celery.events.state 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /examples/httpexample/views.py: -------------------------------------------------------------------------------- 1 | from django.http import HttpResponse 2 | 3 | from anyjson import dumps 4 | 5 | 6 | def multiply(request): 7 | x = int(request.GET['x']) 8 | y = int(request.GET['y']) 9 | 10 | retval = x * y 11 | response = {'status': 'success', 'retval': retval} 12 | return HttpResponse(dumps(response), mimetype='application/json') 13 | -------------------------------------------------------------------------------- /docs/reference/celery.contrib.abortable.rst: -------------------------------------------------------------------------------- 1 | ======================================================= 2 | celery.contrib.abortable 3 | ======================================================= 4 | 5 | .. contents:: 6 | :local: 7 | 8 | .. currentmodule:: celery.contrib.abortable 9 | 10 | .. automodule:: celery.contrib.abortable 11 | :members: 12 | :undoc-members: 13 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.backends.database.rst: -------------------------------------------------------------------------------- 1 | ========================================================= 2 | celery.backends.database 3 | ========================================================= 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.backends.database 8 | 9 | .. automodule:: celery.backends.database 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.backends.database.session.rst: -------------------------------------------------------------------------------- 1 | ======================================== 2 | celery.backends.database.session 3 | ======================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.backends.database.session 8 | 9 | .. automodule:: celery.backends.database.session 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /celery/task/trace.py: -------------------------------------------------------------------------------- 1 | """This module has moved to celery.app.trace.""" 2 | from __future__ import absolute_import 3 | 4 | import sys 5 | 6 | from celery.utils import warn_deprecated 7 | 8 | warn_deprecated('celery.task.trace', removal='3.2', 9 | alternative='Please use celery.app.trace instead.') 10 | 11 | from celery.app import trace 12 | sys.modules[__name__] = trace 13 | -------------------------------------------------------------------------------- /celery/tests/security/case.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from nose import SkipTest 4 | 5 | from celery.tests.case import AppCase 6 | 7 | 8 | class SecurityCase(AppCase): 9 | 10 | def setup(self): 11 | try: 12 | from OpenSSL import crypto # noqa 13 | except ImportError: 14 | raise SkipTest('OpenSSL.crypto not installed') 15 | -------------------------------------------------------------------------------- /docs/THANKS: -------------------------------------------------------------------------------- 1 | Thanks to Rune Halvorsen for the name. 2 | Thanks to Anton Tsigularov for the previous name (crunchy) 3 | which we had to abandon because of an existing project with that name. 4 | Thanks to Armin Ronacher for the Sphinx theme. 5 | Thanks to Brian K. Jones for bunny.py (http://github.com/bkjones/bunny), the 6 | tool that inspired 'celery amqp'. 7 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.dispatch.signal.rst: -------------------------------------------------------------------------------- 1 | ==================================================== 2 | celery.utils.dispatch.signal 3 | ==================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.dispatch.signal 8 | 9 | .. automodule:: celery.utils.dispatch.signal 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /examples/README.rst: -------------------------------------------------------------------------------- 1 | ================= 2 | Celery Examples 3 | ================= 4 | 5 | 6 | * pythonproject 7 | 8 | Example Python project using celery. 9 | 10 | * httpexample 11 | 12 | Example project using remote tasks (webhook tasks) 13 | 14 | * celery_http_gateway 15 | 16 | Example HTTP service exposing the ability to apply tasks and query the 17 | resulting status/return value. 18 | 19 | -------------------------------------------------------------------------------- /examples/gevent/celeryconfig.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | sys.path.insert(0, os.getcwd()) 4 | 5 | ### Note: Start worker with -P gevent, 6 | # do not use the CELERYD_POOL option. 7 | 8 | BROKER_URL = 'amqp://guest:guest@localhost:5672//' 9 | CELERY_DISABLE_RATE_LIMITS = True 10 | CELERY_RESULT_BACKEND = 'amqp' 11 | CELERY_TASK_RESULT_EXPIRES = 30 * 60 12 | 13 | CELERY_IMPORTS = ('tasks', ) 14 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.concurrency.eventlet.rst: -------------------------------------------------------------------------------- 1 | ============================================================= 2 | celery.concurrency.eventlet 3 | ============================================================= 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.concurrency.eventlet 8 | 9 | .. automodule:: celery.concurrency.eventlet 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.concurrency.prefork.rst: -------------------------------------------------------------------------------- 1 | ============================================================= 2 | celery.concurrency.prefork 3 | ============================================================= 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.concurrency.prefork 8 | 9 | .. automodule:: celery.concurrency.prefork 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.concurrency.solo.rst: -------------------------------------------------------------------------------- 1 | =================================================================== 2 | celery.concurrency.solo 3 | =================================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.concurrency.solo 8 | 9 | .. automodule:: celery.concurrency.solo 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.dispatch.saferef.rst: -------------------------------------------------------------------------------- 1 | ========================================================== 2 | celery.utils.dispatch.saferef 3 | ========================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.dispatch.saferef 8 | 9 | .. automodule:: celery.utils.dispatch.saferef 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /examples/httpexample/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls.defaults import ( # noqa 2 | url, patterns, include, handler500, handler404, 3 | ) 4 | from . import views 5 | 6 | # Uncomment the next two lines to enable the admin: 7 | # from django.contrib import admin 8 | # admin.autodiscover() 9 | 10 | urlpatterns = patterns( 11 | '', 12 | url(r'^multiply/', views.multiply, name='multiply'), 13 | ) 14 | -------------------------------------------------------------------------------- /Changelog: -------------------------------------------------------------------------------- 1 | .. _changelog: 2 | 3 | ================ 4 | Change history 5 | ================ 6 | 7 | This document contains change notes for bugfix releases in the 3.1.x series 8 | (Cipater), please see :ref:`whatsnew-3.1` for an overview of what's 9 | new in Celery 3.1. 10 | 11 | .. _version-3.1.0: 12 | 13 | 3.1.0 14 | ======= 15 | :release-date: 2013-11-09 11:00 P.M UTC 16 | 17 | See :ref:`whatsnew-3.1`. 18 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.concurrency.gevent.rst: -------------------------------------------------------------------------------- 1 | ============================================================= 2 | celery.concurrency.gevent† (*experimental*) 3 | ============================================================= 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.concurrency.gevent 8 | 9 | .. automodule:: celery.concurrency.gevent 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /examples/gevent/tasks.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | from celery import task 4 | 5 | 6 | @task(ignore_result=True) 7 | def urlopen(url): 8 | print('Opening: {0}'.format(url)) 9 | try: 10 | requests.get(url) 11 | except Exception as exc: 12 | print('Exception for {0}: {1!r}'.format(url, exc)) 13 | return url, 0 14 | print('Done with: {0}'.format(url)) 15 | return url, 1 16 | -------------------------------------------------------------------------------- /celery/tests/worker/test_revoke.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from celery.worker import state 4 | from celery.tests.case import AppCase 5 | 6 | 7 | class test_revoked(AppCase): 8 | 9 | def test_is_working(self): 10 | state.revoked.add('foo') 11 | self.assertIn('foo', state.revoked) 12 | state.revoked.pop_value('foo') 13 | self.assertNotIn('foo', state.revoked) 14 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.concurrency.threads.rst: -------------------------------------------------------------------------------- 1 | =================================================================== 2 | celery.concurrency.threads‡ (**minefield**) 3 | =================================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.concurrency.threads 8 | 9 | .. automodule:: celery.concurrency.threads 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /celery/utils/encoding.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | celery.utils.encoding 4 | ~~~~~~~~~~~~~~~~~~~~~ 5 | 6 | This module has moved to :mod:`kombu.utils.encoding`. 7 | 8 | """ 9 | from __future__ import absolute_import 10 | 11 | from kombu.utils.encoding import ( # noqa 12 | default_encode, default_encoding, bytes_t, bytes_to_str, str_t, 13 | str_to_bytes, ensure_bytes, from_utf8, safe_str, safe_repr, 14 | ) 15 | -------------------------------------------------------------------------------- /celery/tests/compat_modules/test_messaging.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from celery import messaging 4 | from celery.tests.case import AppCase, depends_on_current_app 5 | 6 | 7 | @depends_on_current_app 8 | class test_compat_messaging_module(AppCase): 9 | 10 | def test_get_consume_set(self): 11 | conn = messaging.establish_connection() 12 | messaging.get_consumer_set(conn).close() 13 | conn.close() 14 | -------------------------------------------------------------------------------- /examples/eventlet/celeryconfig.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | sys.path.insert(0, os.getcwd()) 4 | 5 | ## Start worker with -P eventlet 6 | # Never use the CELERYD_POOL setting as that will patch 7 | # the worker too late. 8 | 9 | BROKER_URL = 'amqp://guest:guest@localhost:5672//' 10 | CELERY_DISABLE_RATE_LIMITS = True 11 | CELERY_RESULT_BACKEND = 'amqp' 12 | CELERY_TASK_RESULT_EXPIRES = 30 * 60 13 | 14 | CELERY_IMPORTS = ('tasks', 'webcrawler') 15 | -------------------------------------------------------------------------------- /docs/userguide/index.rst: -------------------------------------------------------------------------------- 1 | .. _guide: 2 | 3 | ============ 4 | User Guide 5 | ============ 6 | 7 | :Release: |version| 8 | :Date: |today| 9 | 10 | .. toctree:: 11 | :maxdepth: 1 12 | 13 | application 14 | tasks 15 | calling 16 | canvas 17 | workers 18 | periodic-tasks 19 | remote-tasks 20 | routing 21 | monitoring 22 | security 23 | optimizing 24 | concurrency/index 25 | signals 26 | extending 27 | -------------------------------------------------------------------------------- /examples/next-steps/proj/celery.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from celery import Celery 4 | 5 | app = Celery('proj.celery', 6 | broker='amqp://', 7 | backend='amqp://', 8 | include=['proj.tasks']) 9 | 10 | # Optional configuration, see the application user guide. 11 | app.conf.update( 12 | CELERY_TASK_RESULT_EXPIRES=3600, 13 | ) 14 | 15 | if __name__ == '__main__': 16 | app.start() 17 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [nosetests] 2 | where = celery/tests 3 | cover3-branch = 1 4 | cover3-html = 1 5 | cover3-package = celery 6 | cover3-exclude = celery.utils.debug,celery.tests.*,celery.bin.graph 7 | 8 | [build_sphinx] 9 | source-dir = docs/ 10 | build-dir = docs/.build 11 | all_files = 1 12 | 13 | [upload_sphinx] 14 | upload-dir = docs/.build/html 15 | 16 | [bdist_rpm] 17 | requires = pytz >= 2011b 18 | billiard >= 3.3.0.3 19 | kombu >= 3.0.4 20 | -------------------------------------------------------------------------------- /examples/httpexample/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from django.core.management import execute_manager 3 | try: 4 | from . import settings # Assumed to be in the same directory. 5 | except ImportError: 6 | import sys 7 | sys.stderr.write( 8 | "Error: Can't find the file 'settings.py' in the directory " 9 | "containing {0!r}.".format(__file__)) 10 | sys.exit(1) 11 | 12 | if __name__ == '__main__': 13 | execute_manager(settings) 14 | -------------------------------------------------------------------------------- /celery/tests/functional/tasks.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import time 4 | 5 | from celery import task, signature 6 | 7 | 8 | @task() 9 | def add(x, y): 10 | return x + y 11 | 12 | 13 | @task() 14 | def add_cb(x, y, callback=None): 15 | result = x + y 16 | if callback: 17 | return signature(callback).apply_async(result) 18 | return result 19 | 20 | 21 | @task() 22 | def sleeptask(i): 23 | time.sleep(i) 24 | return i 25 | -------------------------------------------------------------------------------- /examples/django/demoapp/tests.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file demonstrates writing tests using the unittest module. These will pass 3 | when you run "manage.py test". 4 | 5 | Replace this with more appropriate tests for your application. 6 | """ 7 | 8 | from django.test import TestCase 9 | 10 | 11 | class SimpleTest(TestCase): 12 | def test_basic_addition(self): 13 | """ 14 | Tests that 1 + 1 always equals 2. 15 | """ 16 | self.assertEqual(1 + 1, 2) 17 | -------------------------------------------------------------------------------- /docs/history/index.rst: -------------------------------------------------------------------------------- 1 | .. _history: 2 | 3 | ========= 4 | History 5 | ========= 6 | 7 | This section contains historical change histories, for the latest 8 | version please visit :ref:`changelog`. 9 | 10 | :Release: |version| 11 | :Date: |today| 12 | 13 | .. toctree:: 14 | :maxdepth: 2 15 | 16 | changelog-3.0 17 | changelog-2.5 18 | changelog-2.4 19 | changelog-2.3 20 | changelog-2.2 21 | changelog-2.1 22 | changelog-2.0 23 | changelog-1.0 24 | -------------------------------------------------------------------------------- /docs/reference/celery.app.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: celery.app 2 | 3 | .. automodule:: celery.app 4 | 5 | .. contents:: 6 | :local: 7 | 8 | Proxies 9 | ------- 10 | 11 | .. autodata:: default_app 12 | 13 | 14 | Functions 15 | --------- 16 | 17 | .. autofunction:: app_or_default 18 | .. autofunction:: enable_trace 19 | .. autofunction:: disable_trace 20 | 21 | 22 | Data 23 | ---- 24 | 25 | .. autodata:: default_loader 26 | 27 | -------------------------------------------------------------------------------- /examples/celery_http_gateway/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from django.core.management import execute_manager 3 | try: 4 | import settings # Assumed to be in the same directory. 5 | except ImportError: 6 | import sys 7 | sys.stderr.write( 8 | "Error: Can't find the file 'settings.py' in the directory " 9 | "containing {0!r}.".format(__file__)) 10 | sys.exit(1) 11 | 12 | if __name__ == '__main__': 13 | execute_manager(settings) 14 | -------------------------------------------------------------------------------- /examples/next-steps/setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | Example setup file for a project using Celery. 3 | 4 | This can be used to distribute your tasks and worker 5 | as a Python package, on PyPI or on your own private package index. 6 | 7 | """ 8 | from setuptools import setup, find_packages 9 | 10 | setup( 11 | name='example-tasks', 12 | version='1.0', 13 | description='Tasks for my project', 14 | packages=find_packages(exclude=['ez_setup', 'tests', 'tests.*']), 15 | zip_safe=False, 16 | install_requires=[ 17 | 'celery>=3.0', 18 | #'requests', 19 | ], 20 | ) 21 | -------------------------------------------------------------------------------- /extra/release/gen-cert.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Script for generating key-cert pairs 4 | # 5 | 6 | if [[ $# -ne 2 ]]; then 7 | echo "Usage: ${0##*/} KEY CERT" 8 | exit 9 | fi 10 | 11 | KEY=$1 12 | CERT=$2 13 | PSWD=test 14 | 15 | read -s -p "Enter Password: " $PSWD 16 | 17 | openssl genrsa -des3 -passout pass:$PSWD -out $KEY.key 1024 18 | openssl req -new -key $KEY.key -out $KEY.csr -passin pass:$PSWD 19 | cp $KEY.key $KEY.key.org 20 | openssl rsa -in $KEY.key.org -out $KEY.key -passin pass:$PSWD 21 | openssl x509 -req -days 365 -in $KEY.csr -signkey $KEY.key -out $CERT.crt 22 | rm $KEY.key.org $KEY.csr 23 | -------------------------------------------------------------------------------- /celery/tests/app/test_celery.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from celery.tests.case import AppCase 3 | 4 | import celery 5 | 6 | 7 | class test_celery_package(AppCase): 8 | 9 | def test_version(self): 10 | self.assertTrue(celery.VERSION) 11 | self.assertGreaterEqual(len(celery.VERSION), 3) 12 | celery.VERSION = (0, 3, 0) 13 | self.assertGreaterEqual(celery.__version__.count('.'), 2) 14 | 15 | def test_meta(self): 16 | for m in ('__author__', '__contact__', '__homepage__', 17 | '__docformat__'): 18 | self.assertTrue(getattr(celery, m, None)) 19 | -------------------------------------------------------------------------------- /celery/tests/utils/test_encoding.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from celery.utils import encoding 4 | from celery.tests.case import Case 5 | 6 | 7 | class test_encoding(Case): 8 | 9 | def test_safe_str(self): 10 | self.assertTrue(encoding.safe_str(object())) 11 | self.assertTrue(encoding.safe_str('foo')) 12 | 13 | def test_safe_repr(self): 14 | self.assertTrue(encoding.safe_repr(object())) 15 | 16 | class foo(object): 17 | def __repr__(self): 18 | raise ValueError('foo') 19 | 20 | self.assertTrue(encoding.safe_repr(foo())) 21 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include CONTRIBUTORS.txt 2 | include Changelog 3 | include LICENSE 4 | include README.rst 5 | include MANIFEST.in 6 | include TODO 7 | include setup.cfg 8 | include setup.py 9 | recursive-include celery *.py 10 | recursive-include docs * 11 | recursive-include extra/bash-completion * 12 | recursive-include extra/centos * 13 | recursive-include extra/generic-init.d * 14 | recursive-include extra/osx * 15 | recursive-include extra/supervisord * 16 | recursive-include extra/systemd * 17 | recursive-include extra/zsh-completion * 18 | recursive-include examples * 19 | recursive-include requirements *.txt 20 | prune *.pyc 21 | prune *.sw* 22 | -------------------------------------------------------------------------------- /funtests/suite/config.py: -------------------------------------------------------------------------------- 1 | import atexit 2 | import os 3 | 4 | BROKER_URL = os.environ.get('BROKER_URL') or 'amqp://' 5 | CELERY_RESULT_BACKEND = 'amqp://' 6 | CELERY_SEND_TASK_ERROR_EMAILS = False 7 | 8 | CELERY_DEFAULT_QUEUE = 'testcelery' 9 | CELERY_DEFAULT_EXCHANGE = 'testcelery' 10 | CELERY_DEFAULT_ROUTING_KEY = 'testcelery' 11 | CELERY_QUEUES = {'testcelery': {'routing_key': 'testcelery'}} 12 | 13 | CELERYD_LOG_COLOR = False 14 | 15 | CELERY_IMPORTS = ('celery.tests.functional.tasks', ) 16 | 17 | 18 | @atexit.register 19 | def teardown_testdb(): 20 | import os 21 | if os.path.exists('test.db'): 22 | os.remove('test.db') 23 | -------------------------------------------------------------------------------- /examples/django/proj/celery.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import os 4 | 5 | from celery import Celery 6 | 7 | from django.conf import settings 8 | 9 | # set the default Django settings module for the 'celery' program. 10 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'proj.settings') 11 | 12 | app = Celery('proj.celery') 13 | 14 | # Using a string here means the worker will not have to 15 | # pickle the object when using Windows. 16 | app.config_from_object('django.conf:settings') 17 | app.autodiscover_tasks(settings.INSTALLED_APPS) 18 | 19 | 20 | @app.task(bind=True) 21 | def debug_task(self): 22 | print(repr(self.request)) 23 | -------------------------------------------------------------------------------- /celery/tests/concurrency/test_solo.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import operator 4 | 5 | from celery.concurrency import solo 6 | from celery.utils.functional import noop 7 | from celery.tests.case import AppCase 8 | 9 | 10 | class test_solo_TaskPool(AppCase): 11 | 12 | def test_on_start(self): 13 | x = solo.TaskPool() 14 | x.on_start() 15 | 16 | def test_on_apply(self): 17 | x = solo.TaskPool() 18 | x.on_start() 19 | x.on_apply(operator.add, (2, 2), {}, noop, noop) 20 | 21 | def test_info(self): 22 | x = solo.TaskPool() 23 | x.on_start() 24 | self.assertTrue(x.info) 25 | -------------------------------------------------------------------------------- /examples/django/proj/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import ( # noqa 2 | patterns, include, url, handler404, handler500, 3 | ) 4 | 5 | # Uncomment the next two lines to enable the admin: 6 | # from django.contrib import admin 7 | # admin.autodiscover() 8 | 9 | urlpatterns = patterns( 10 | '', 11 | # Examples: 12 | # url(r'^$', 'proj.views.home', name='home'), 13 | # url(r'^proj/', include('proj.foo.urls')), 14 | 15 | # Uncomment the admin/doc line below to enable admin documentation: 16 | # url(r'^admin/doc/', include('django.contrib.admindocs.urls')), 17 | 18 | # Uncomment the next line to enable the admin: 19 | # url(r'^admin/', include(admin.site.urls)), 20 | ) 21 | -------------------------------------------------------------------------------- /docs/.templates/page.html: -------------------------------------------------------------------------------- 1 | {% extends "layout.html" %} 2 | {% block body %} 3 |
4 | 5 | {% if version == "3.2" or version == "4.0" %} 6 |

7 | This document is for Celery's development version, which can be 8 | significantly different from previous releases. Get old docs here: 9 | 10 | 3.0. 11 |

12 | {% else %} 13 |

14 | This document describes Celery {{ version }}. For development docs, 15 | go here. 16 |

17 | {% endif %} 18 | 19 |
20 | {{ body }} 21 | {% endblock %} 22 | -------------------------------------------------------------------------------- /examples/celery_http_gateway/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls.defaults import ( # noqa 2 | url, patterns, include, handler404, handler500, 3 | ) 4 | 5 | from djcelery import views as celery_views 6 | 7 | from celery_http_gateway.tasks import hello_world 8 | 9 | # Uncomment the next two lines to enable the admin: 10 | # from django.contrib import admin 11 | # admin.autodiscover() 12 | 13 | urlpatterns = patterns( 14 | '', 15 | url(r'^apply/(?P.+?)/', celery_views.apply), 16 | url(r'^hello/', celery_views.task_view(hello_world)), 17 | url(r'^(?P[\w\d\-]+)/done/?$', celery_views.is_task_successful, 18 | name='celery-is_task_successful'), 19 | url(r'^(?P[\w\d\-]+)/status/?$', celery_views.task_status, 20 | name='celery-task_status'), 21 | ) 22 | -------------------------------------------------------------------------------- /extra/systemd/celery.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Celery Nodes Daemon 3 | After=network.target 4 | 5 | [Service] 6 | Type=forking 7 | User=user 8 | Group=users 9 | #Environment=DJANGO_SETTINGS_MODULE=MyProject.settings 10 | EnvironmentFile=-/etc/conf.d/celery 11 | WorkingDirectory=/opt/Myproject/ 12 | ExecStart=/usr/bin/python2 ${CELERY_BIN} $CELERYD_MULTI start $CELERYD_NODES --pidfile=${CELERYD_PID_FILE} --logfile=${CELERYD_LOG_FILE} --loglevel="INFO" $CELERYD_OPTS 13 | ExecStop=/usr/bin/python2 ${CELERY_BIN} $CELERYD_MULTI stopwait $CELERYD_NODES --pidfile=${CELERYD_PID_FILE} 14 | ExecReload=/usr/bin/python2 ${CELERY_BIN} $CELERYD_MULTI restart $CELERYD_NODES --pidfile=${CELERYD_PID_FILE} --logfile=${CELERYD_LOG_FILE} --loglevel="INFO" $CELERYD_OPTS 15 | 16 | [Install] 17 | WantedBy=multi-user.target 18 | -------------------------------------------------------------------------------- /celery/security/key.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | celery.security.key 4 | ~~~~~~~~~~~~~~~~~~~ 5 | 6 | Private key for the security serializer. 7 | 8 | """ 9 | from __future__ import absolute_import 10 | 11 | from kombu.utils.encoding import ensure_bytes 12 | 13 | from .utils import crypto, reraise_errors 14 | 15 | __all__ = ['PrivateKey'] 16 | 17 | 18 | class PrivateKey(object): 19 | 20 | def __init__(self, key): 21 | with reraise_errors('Invalid private key: {0!r}'): 22 | self._key = crypto.load_privatekey(crypto.FILETYPE_PEM, key) 23 | 24 | def sign(self, data, digest): 25 | """sign string containing data.""" 26 | with reraise_errors('Unable to sign data: {0!r}'): 27 | return crypto.sign(self._key, ensure_bytes(data), digest) 28 | -------------------------------------------------------------------------------- /docs/.templates/sidebarlogo.html: -------------------------------------------------------------------------------- 1 | 4 | 16 | -------------------------------------------------------------------------------- /docs/.templates/sidebarintro.html: -------------------------------------------------------------------------------- 1 | 4 | 16 | -------------------------------------------------------------------------------- /celery/concurrency/solo.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | celery.concurrency.solo 4 | ~~~~~~~~~~~~~~~~~~~~~~~ 5 | 6 | Single-threaded pool implementation. 7 | 8 | """ 9 | from __future__ import absolute_import 10 | 11 | import os 12 | 13 | from .base import BasePool, apply_target 14 | 15 | __all__ = ['TaskPool'] 16 | 17 | 18 | class TaskPool(BasePool): 19 | """Solo task pool (blocking, inline, fast).""" 20 | 21 | def __init__(self, *args, **kwargs): 22 | super(TaskPool, self).__init__(*args, **kwargs) 23 | self.on_apply = apply_target 24 | 25 | def _get_info(self): 26 | return {'max-concurrency': 1, 27 | 'processes': [os.getpid()], 28 | 'max-tasks-per-child': None, 29 | 'put-guarded-by-semaphore': True, 30 | 'timeouts': ()} 31 | -------------------------------------------------------------------------------- /examples/app/myapp.py: -------------------------------------------------------------------------------- 1 | """myapp.py 2 | 3 | Usage: 4 | 5 | (window1)$ python myapp.py worker -l info 6 | 7 | (window2)$ python 8 | >>> from myapp import add 9 | >>> add.delay(16, 16).get() 10 | 32 11 | 12 | 13 | You can also specify the app to use with the `celery` command, 14 | using the `-A` / `--app` option:: 15 | 16 | $ celery -A myapp worker -l info 17 | 18 | With the `-A myproj` argument the program will search for an app 19 | instance in the module ``myproj``. You can also specify an explicit 20 | name using the fully qualified form:: 21 | 22 | $ celery -A myapp:app worker -l info 23 | 24 | """ 25 | from celery import Celery 26 | 27 | app = Celery('myapp', broker='amqp://guest@localhost//') 28 | 29 | 30 | @app.task() 31 | def add(x, y): 32 | return x + y 33 | 34 | if __name__ == '__main__': 35 | app.start() 36 | -------------------------------------------------------------------------------- /extra/centos/celeryd.sysconfig: -------------------------------------------------------------------------------- 1 | # In CentOS, contents should be placed in the file /etc/sysconfig/celeryd 2 | 3 | # Names of nodes to start (space-separated) 4 | #CELERYD_NODES="my_application-node_1" 5 | 6 | # Where to chdir at start. This could be the root of a virtualenv. 7 | #CELERYD_CHDIR="/path/to/my_application" 8 | 9 | # How to call celeryd-multi 10 | #CELERYD_MULTI="$CELERYD_CHDIR/bin/celeryd-multi" 11 | 12 | # Extra arguments 13 | #CELERYD_OPTS="--app=my_application.path.to.worker --time-limit=300 --concurrency=8 --loglevel=DEBUG" 14 | 15 | # Create log/pid dirs, if they don't already exist 16 | #CELERY_CREATE_DIRS=1 17 | 18 | # %n will be replaced with the nodename 19 | #CELERYD_LOG_FILE="/path/to/my_application/log/%n.log" 20 | #CELERYD_PID_FILE="/var/run/celery/%n.pid" 21 | 22 | # Workers run as an unprivileged user 23 | #CELERYD_USER=celery 24 | #CELERYD_GROUP=celery 25 | -------------------------------------------------------------------------------- /extra/osx/org.celeryq.beat.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Disabled 6 | 7 | GroupName 8 | celery-beat 9 | KeepAlive 10 | 11 | Label 12 | org.celeryq.beat 13 | Program 14 | celery 15 | ProgramArguments 16 | 17 | beat 18 | --loglevel=WARNING 19 | 20 | RunAtLoad 21 | 22 | Umask 23 | 7 24 | UserName 25 | nobody 26 | WorkingDirectory 27 | / 28 | 29 | 30 | -------------------------------------------------------------------------------- /extra/osx/org.celeryq.worker.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Disabled 6 | 7 | GroupName 8 | celery-worker 9 | KeepAlive 10 | 11 | Label 12 | org.celeryq.worker 13 | Program 14 | celery 15 | ProgramArguments 16 | 17 | worker 18 | --loglevel=WARNING 19 | 20 | RunAtLoad 21 | 22 | Umask 23 | 7 24 | UserName 25 | nobody 26 | WorkingDirectory 27 | / 28 | 29 | 30 | -------------------------------------------------------------------------------- /celery/security/utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | celery.security.utils 4 | ~~~~~~~~~~~~~~~~~~~~~ 5 | 6 | Utilities used by the message signing serializer. 7 | 8 | """ 9 | from __future__ import absolute_import 10 | 11 | import sys 12 | 13 | from contextlib import contextmanager 14 | 15 | from celery.exceptions import SecurityError 16 | from celery.five import reraise 17 | 18 | try: 19 | from OpenSSL import crypto 20 | except ImportError: # pragma: no cover 21 | crypto = None # noqa 22 | 23 | __all__ = ['reraise_errors'] 24 | 25 | 26 | @contextmanager 27 | def reraise_errors(msg='{0!r}', errors=None): 28 | assert crypto is not None 29 | errors = (crypto.Error, ) if errors is None else errors 30 | try: 31 | yield 32 | except errors as exc: 33 | reraise(SecurityError, 34 | SecurityError(msg.format(exc)), 35 | sys.exc_info()[2]) 36 | -------------------------------------------------------------------------------- /celery/tests/app/test_utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from collections import Mapping, MutableMapping 4 | 5 | from celery.app.utils import Settings, bugreport 6 | 7 | from celery.tests.case import AppCase, Mock 8 | 9 | 10 | class TestSettings(AppCase): 11 | """ 12 | Tests of celery.app.utils.Settings 13 | """ 14 | def test_is_mapping(self): 15 | """Settings should be a collections.Mapping""" 16 | self.assertTrue(issubclass(Settings, Mapping)) 17 | 18 | def test_is_mutable_mapping(self): 19 | """Settings should be a collections.MutableMapping""" 20 | self.assertTrue(issubclass(Settings, MutableMapping)) 21 | 22 | 23 | class test_bugreport(AppCase): 24 | 25 | def test_no_conn_driver_info(self): 26 | self.app.connection = Mock() 27 | conn = self.app.connection.return_value = Mock() 28 | conn.transport = None 29 | 30 | bugreport(self.app) 31 | -------------------------------------------------------------------------------- /examples/httpexample/README.rst: -------------------------------------------------------------------------------- 1 | ====================== 2 | Webhook Task Example 3 | ====================== 4 | 5 | This example is a simple Django HTTP service exposing a single task 6 | multiplying two numbers: 7 | 8 | The multiply http callback task is in `views.py`, mapped to a URL using 9 | `urls.py`. 10 | 11 | There are no models, so to start it do:: 12 | 13 | $ python manage.py runserver 14 | 15 | To execute the task you could use curl:: 16 | 17 | $ curl http://localhost:8000/multiply?x=10&y=10 18 | 19 | which then gives the expected JSON response:: 20 | 21 | {"status": "success": "retval": 100} 22 | 23 | 24 | To execute this http callback task asynchronously you could fire up 25 | a python shell with a properly configured celery and do: 26 | 27 | >>> from celery.task.http import URL 28 | >>> res = URL("http://localhost:8000/multiply").get_async(x=10, y=10) 29 | >>> res.wait() 30 | 100 31 | 32 | 33 | That's all! 34 | -------------------------------------------------------------------------------- /celery/tests/security/test_key.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from celery.exceptions import SecurityError 4 | from celery.security.key import PrivateKey 5 | 6 | from . import CERT1, KEY1, KEY2 7 | from .case import SecurityCase 8 | 9 | 10 | class test_PrivateKey(SecurityCase): 11 | 12 | def test_valid_private_key(self): 13 | PrivateKey(KEY1) 14 | PrivateKey(KEY2) 15 | 16 | def test_invalid_private_key(self): 17 | self.assertRaises(TypeError, PrivateKey, None) 18 | self.assertRaises(SecurityError, PrivateKey, '') 19 | self.assertRaises(SecurityError, PrivateKey, 'foo') 20 | self.assertRaises(SecurityError, PrivateKey, KEY1[:20] + KEY1[21:]) 21 | self.assertRaises(SecurityError, PrivateKey, CERT1) 22 | 23 | def test_sign(self): 24 | pkey = PrivateKey(KEY1) 25 | pkey.sign('test', 'sha1') 26 | self.assertRaises(ValueError, pkey.sign, 'test', 'unknown') 27 | -------------------------------------------------------------------------------- /celery/concurrency/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | celery.concurrency 4 | ~~~~~~~~~~~~~~~~~~ 5 | 6 | Pool implementation abstract factory, and alias definitions. 7 | 8 | """ 9 | from __future__ import absolute_import 10 | 11 | # Import from kombu directly as it's used 12 | # early in the import stage, where celery.utils loads 13 | # too much (e.g. for eventlet patching) 14 | from kombu.utils import symbol_by_name 15 | 16 | __all__ = ['get_implementation'] 17 | 18 | ALIASES = { 19 | 'prefork': 'celery.concurrency.prefork:TaskPool', 20 | 'eventlet': 'celery.concurrency.eventlet:TaskPool', 21 | 'gevent': 'celery.concurrency.gevent:TaskPool', 22 | 'threads': 'celery.concurrency.threads:TaskPool', 23 | 'solo': 'celery.concurrency.solo:TaskPool', 24 | 'processes': 'celery.concurrency.prefork:TaskPool', # XXX compat alias 25 | } 26 | 27 | 28 | def get_implementation(cls): 29 | return symbol_by_name(cls, ALIASES) 30 | -------------------------------------------------------------------------------- /extra/supervisord/celerybeat.conf: -------------------------------------------------------------------------------- 1 | ; ================================ 2 | ; celery beat supervisor example 3 | ; ================================ 4 | 5 | ; NOTE: If you're using Django, you shouldn't use this file. 6 | ; Use 7 | ; http://github.com/celery/django-celery/tree/master/extra/supervisord/celerybeat.conf 8 | ; instead! 9 | 10 | [program:celerybeat] 11 | command=celery beat -A myapp --schedule /var/lib/celery/celerybeat-schedule --loglevel=INFO 12 | 13 | ; remove the -A myapp argument if you are not using an app instance 14 | 15 | ; Set PYTHONPATH to the directory containing app/celeryconfig.py 16 | environment=PYTHONPATH=/path/to/project 17 | 18 | directory=/path/to/project 19 | user=nobody 20 | numprocs=1 21 | stdout_logfile=/var/log/celerybeat.log 22 | stderr_logfile=/var/log/celerybeat.log 23 | autostart=true 24 | autorestart=true 25 | startsecs=10 26 | 27 | ; if rabbitmq is supervised, set its priority higher 28 | ; so it starts first 29 | priority=999 30 | -------------------------------------------------------------------------------- /celery/tests/contrib/test_methods.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from celery.contrib.methods import task_method, task 4 | 5 | from celery.tests.case import AppCase, patch 6 | 7 | 8 | class test_task_method(AppCase): 9 | 10 | def test_task_method(self): 11 | 12 | class X(object): 13 | 14 | def __init__(self): 15 | self.state = 0 16 | 17 | @self.app.task(shared=False, filter=task_method) 18 | def add(self, x): 19 | self.state += x 20 | 21 | x = X() 22 | x.add(2) 23 | self.assertEqual(x.state, 2) 24 | x.add(4) 25 | self.assertEqual(x.state, 6) 26 | 27 | self.assertTrue(X.add) 28 | self.assertIs(x.add.__self__, x) 29 | 30 | def test_task(self): 31 | with patch('celery.contrib.methods.current_app') as curapp: 32 | fun = object() 33 | task(fun, x=1) 34 | curapp.task.assert_called_with(fun, x=1, filter=task_method) 35 | -------------------------------------------------------------------------------- /celery/tests/app/test_exceptions.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import pickle 4 | 5 | from datetime import datetime 6 | 7 | from celery.exceptions import Reject, Retry 8 | 9 | from celery.tests.case import AppCase 10 | 11 | 12 | class test_Retry(AppCase): 13 | 14 | def test_when_datetime(self): 15 | x = Retry('foo', KeyError(), when=datetime.utcnow()) 16 | self.assertTrue(x.humanize()) 17 | 18 | def test_pickleable(self): 19 | x = Retry('foo', KeyError(), when=datetime.utcnow()) 20 | self.assertTrue(pickle.loads(pickle.dumps(x))) 21 | 22 | 23 | class test_Reject(AppCase): 24 | 25 | def test_attrs(self): 26 | x = Reject('foo', requeue=True) 27 | self.assertEqual(x.reason, 'foo') 28 | self.assertTrue(x.requeue) 29 | 30 | def test_repr(self): 31 | self.assertTrue(repr(Reject('foo', True))) 32 | 33 | def test_pickleable(self): 34 | x = Retry('foo', True) 35 | self.assertTrue(pickle.loads(pickle.dumps(x))) 36 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.datastructures.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: celery.datastructures 2 | 3 | .. automodule:: celery.datastructures 4 | 5 | .. contents:: 6 | :local: 7 | 8 | AttributeDict 9 | ------------- 10 | 11 | .. autoclass:: AttributeDict 12 | :members: 13 | 14 | .. autoclass:: AttributeDictMixin 15 | :members: 16 | 17 | DictAttribute 18 | ------------- 19 | 20 | .. autoclass:: DictAttribute 21 | :members: 22 | :undoc-members: 23 | 24 | ConfigurationView 25 | ----------------- 26 | 27 | .. autoclass:: ConfigurationView 28 | :members: 29 | :undoc-members: 30 | 31 | ExceptionInfo 32 | ------------- 33 | 34 | .. autoclass:: ExceptionInfo 35 | :members: 36 | 37 | LimitedSet 38 | ---------- 39 | 40 | .. autoclass:: LimitedSet 41 | :members: 42 | :undoc-members: 43 | 44 | LRUCache 45 | -------- 46 | 47 | .. autoclass:: LRUCache 48 | :members: 49 | :undoc-members: 50 | -------------------------------------------------------------------------------- /docs/copyright.rst: -------------------------------------------------------------------------------- 1 | Copyright 2 | ========= 3 | 4 | *Celery User Manual* 5 | 6 | by Ask Solem 7 | 8 | .. |copy| unicode:: U+000A9 .. COPYRIGHT SIGN 9 | 10 | Copyright |copy| 2009-2013, Ask Solem. 11 | 12 | All rights reserved. This material may be copied or distributed only 13 | subject to the terms and conditions set forth in the `Creative Commons 14 | Attribution-Noncommercial-Share Alike 3.0 United States License 15 | `_. You must 16 | give the original author credit. You may not use this work for 17 | commercial purposes. If you alter, transform, or build upon this 18 | work, you may distribute the resulting work only under the same or 19 | similar license to this one. 20 | 21 | .. note:: 22 | 23 | While the *Celery* documentation is offered under the 24 | Creative Commons *attribution-nonconmmercial-share alike 3.0 united 25 | states* license, the Celery *software* is offered under the 26 | less restrictive 27 | `BSD License (3 Clause) `_ 28 | -------------------------------------------------------------------------------- /extra/release/attribution.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import absolute_import 3 | 4 | import fileinput 5 | 6 | from pprint import pprint 7 | 8 | 9 | def author(line): 10 | try: 11 | A, E = line.strip().rsplit(None, 1) 12 | E.replace(">", "").replace("<", "") 13 | except ValueError: 14 | A, E = line.strip(), None 15 | return A.lower() if A else A, E.lower() if E else E 16 | 17 | 18 | def proper_name(name): 19 | return name and " " in name 20 | 21 | 22 | def find_missing_authors(seen): 23 | with open("AUTHORS") as authors: 24 | known = [author(line) for line in authors.readlines()] 25 | 26 | seen_authors = set(filter(proper_name, (t[0] for t in seen))) 27 | known_authors = set(t[0] for t in known) 28 | # maybe later?: 29 | # seen_emails = set(t[1] for t in seen) 30 | # known_emails = set(t[1] for t in known) 31 | 32 | pprint(seen_authors - known_authors) 33 | 34 | 35 | if __name__ == "__main__": 36 | find_missing_authors([author(line) for line in fileinput.input()]) 37 | -------------------------------------------------------------------------------- /docs/reference/celery.utils.debug.rst: -------------------------------------------------------------------------------- 1 | ==================================== 2 | celery.utils.debug 3 | ==================================== 4 | 5 | .. contents:: 6 | :local: 7 | 8 | Sampling Memory Usage 9 | ===================== 10 | 11 | This module can be used to diagnose and sample the memory usage 12 | used by parts of your application. 13 | 14 | E.g to sample the memory usage of calling tasks you can do this: 15 | 16 | .. code-block:: python 17 | 18 | 19 | from celery.utils.debug import sample_mem, memdump 20 | 21 | from tasks import add 22 | 23 | 24 | try: 25 | for i in range(100): 26 | for j in range(100): 27 | add.delay(i, j) 28 | sample_mem() 29 | finally: 30 | memdump() 31 | 32 | 33 | API Reference 34 | ============= 35 | 36 | .. currentmodule:: celery.utils.debug 37 | 38 | .. automodule:: celery.utils.debug 39 | 40 | .. autofunction:: sample_mem 41 | 42 | .. autofunction:: memdump 43 | 44 | .. autofunction:: sample 45 | 46 | .. autofunction:: mem_rss 47 | 48 | .. autofunction:: ps 49 | -------------------------------------------------------------------------------- /celery/tests/utils/test_sysinfo.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import os 4 | 5 | from mock import patch 6 | from nose import SkipTest 7 | 8 | from celery.utils.sysinfo import load_average, df 9 | 10 | from celery.tests.case import Case 11 | 12 | 13 | class test_load_average(Case): 14 | 15 | def test_avg(self): 16 | if not hasattr(os, 'getloadavg'): 17 | raise SkipTest('getloadavg not available') 18 | with patch('os.getloadavg') as getloadavg: 19 | getloadavg.return_value = 0.54736328125, 0.6357421875, 0.69921875 20 | l = load_average() 21 | self.assertTrue(l) 22 | self.assertEqual(l, (0.55, 0.64, 0.7)) 23 | 24 | 25 | class test_df(Case): 26 | 27 | def test_df(self): 28 | try: 29 | from posix import statvfs_result # noqa 30 | except ImportError: 31 | raise SkipTest('statvfs not available') 32 | x = df('/') 33 | self.assertTrue(x.total_blocks) 34 | self.assertTrue(x.available) 35 | self.assertTrue(x.capacity) 36 | self.assertTrue(x.stat) 37 | -------------------------------------------------------------------------------- /celery/tests/worker/test_components.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | # some of these are tested in test_worker, so I've only written tests 4 | # here to complete coverage. Should move everyting to this module at some 5 | # point [-ask] 6 | 7 | from mock import Mock 8 | 9 | from celery.worker.components import ( 10 | Queues, 11 | Pool, 12 | ) 13 | 14 | from celery.tests.case import AppCase 15 | 16 | 17 | class test_Queues(AppCase): 18 | 19 | def test_create_when_eventloop(self): 20 | w = Mock() 21 | w.use_eventloop = w.pool_putlocks = w.pool_cls.uses_semaphore = True 22 | q = Queues(w) 23 | q.create(w) 24 | self.assertIs(w.process_task, w._process_task_sem) 25 | 26 | 27 | class test_Pool(AppCase): 28 | 29 | def test_close_terminate(self): 30 | w = Mock() 31 | comp = Pool(w) 32 | pool = w.pool = Mock() 33 | comp.close(w) 34 | pool.close.assert_called_with() 35 | comp.terminate(w) 36 | pool.terminate.assert_called_with() 37 | 38 | w.pool = None 39 | comp.close(w) 40 | comp.terminate(w) 41 | -------------------------------------------------------------------------------- /examples/django/README.rst: -------------------------------------------------------------------------------- 1 | ============================================================== 2 | Example Django project using Celery 3 | ============================================================== 4 | 5 | Contents 6 | ======== 7 | 8 | ``proj/`` 9 | --------- 10 | 11 | This is the project iself, created using 12 | ``django-admin.py startproject proj``, and then the settings module 13 | (``proj/settings.py``) was modified to add ``demoapp`` to 14 | ``INSTALLED_APPS`` 15 | 16 | ``proj/celery.py`` 17 | ---------- 18 | 19 | This module contains the Celery application instance for this project, 20 | we take configuration from Django settings and use ``autodiscover_tasks`` to 21 | find task modules inside all packages listed in ``INSTALLED_APPS``. 22 | 23 | ``demoapp/`` 24 | ------------ 25 | 26 | Example generic app. This is decoupled from the rest of the project by using 27 | the ``@shared_task`` decorator. This decorator returns a proxy that always 28 | points to the currently active Celery instance. 29 | 30 | 31 | Starting the worker 32 | =================== 33 | 34 | .. code-block:: bash 35 | 36 | $ celery -A proj worker -l info 37 | -------------------------------------------------------------------------------- /celery/loaders/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | celery.loaders 4 | ~~~~~~~~~~~~~~ 5 | 6 | Loaders define how configuration is read, what happens 7 | when workers start, when tasks are executed and so on. 8 | 9 | """ 10 | from __future__ import absolute_import 11 | 12 | from celery._state import current_app 13 | from celery.utils import deprecated 14 | from celery.utils.imports import symbol_by_name, import_from_cwd 15 | 16 | __all__ = ['get_loader_cls'] 17 | 18 | LOADER_ALIASES = {'app': 'celery.loaders.app:AppLoader', 19 | 'default': 'celery.loaders.default:Loader', 20 | 'django': 'djcelery.loaders:DjangoLoader'} 21 | 22 | 23 | def get_loader_cls(loader): 24 | """Get loader class by name/alias""" 25 | return symbol_by_name(loader, LOADER_ALIASES, imp=import_from_cwd) 26 | 27 | 28 | @deprecated(deprecation='2.5', removal='4.0', 29 | alternative='celery.current_app.loader') 30 | def current_loader(): 31 | return current_app.loader 32 | 33 | 34 | @deprecated(deprecation='2.5', removal='4.0', 35 | alternative='celery.current_app.conf') 36 | def load_settings(): 37 | return current_app.conf 38 | -------------------------------------------------------------------------------- /celery/utils/sysinfo.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import absolute_import 3 | 4 | import os 5 | 6 | from math import ceil 7 | 8 | from kombu.utils import cached_property 9 | 10 | __all__ = ['load_average', 'df'] 11 | 12 | 13 | if hasattr(os, 'getloadavg'): 14 | 15 | def load_average(): 16 | return tuple(ceil(l * 1e2) / 1e2 for l in os.getloadavg()) 17 | 18 | else: # pragma: no cover 19 | # Windows doesn't have getloadavg 20 | def load_average(): # noqa 21 | return (0.0, 0.0, 0.0) 22 | 23 | 24 | class df(object): 25 | 26 | def __init__(self, path): 27 | self.path = path 28 | 29 | @property 30 | def total_blocks(self): 31 | return self.stat.f_blocks * self.stat.f_frsize / 1024 32 | 33 | @property 34 | def available(self): 35 | return self.stat.f_bavail * self.stat.f_frsize / 1024 36 | 37 | @property 38 | def capacity(self): 39 | avail = self.stat.f_bavail 40 | used = self.stat.f_blocks - self.stat.f_bfree 41 | return int(ceil(used * 100.0 / (used + avail) + 0.5)) 42 | 43 | @cached_property 44 | def stat(self): 45 | return os.statvfs(os.path.abspath(self.path)) 46 | -------------------------------------------------------------------------------- /docs/community.rst: -------------------------------------------------------------------------------- 1 | .. _community: 2 | 3 | ======================= 4 | Community Resources 5 | ======================= 6 | 7 | This is a list of external blog posts, tutorials and slides related 8 | to Celery. If you have a link that's missing from this list, please 9 | contact the mailing-list or submit a patch. 10 | 11 | .. contents:: 12 | :local: 13 | 14 | .. _community-resources: 15 | 16 | Resources 17 | ========= 18 | 19 | .. _res-using-celery: 20 | 21 | Who's using Celery 22 | ------------------ 23 | 24 | http://wiki.github.com/celery/celery/using 25 | 26 | .. _res-wiki: 27 | 28 | Wiki 29 | ---- 30 | 31 | http://wiki.github.com/celery/celery/ 32 | 33 | .. _res-stackoverflow: 34 | 35 | Celery questions on Stack Overflow 36 | ---------------------------------- 37 | 38 | http://stackoverflow.com/search?q=celery&tab=newest 39 | 40 | .. _res-mailing-list-archive: 41 | 42 | Mailing-list Archive: celery-users 43 | ---------------------------------- 44 | 45 | http://blog.gmane.org/gmane.comp.python.amqp.celery.user 46 | 47 | .. _res-irc-logs: 48 | 49 | .. _community-news: 50 | 51 | News 52 | ==== 53 | 54 | This section has moved to the Celery homepage: 55 | http://celeryproject.org/community/ 56 | -------------------------------------------------------------------------------- /celery/utils/objects.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | celery.utils.objects 4 | ~~~~~~~~~~~~~~~~~~~~ 5 | 6 | Object related utilities including introspection, etc. 7 | 8 | """ 9 | from __future__ import absolute_import 10 | 11 | __all__ = ['mro_lookup'] 12 | 13 | 14 | def mro_lookup(cls, attr, stop=(), monkey_patched=[]): 15 | """Return the first node by MRO order that defines an attribute. 16 | 17 | :keyword stop: A list of types that if reached will stop the search. 18 | :keyword monkey_patched: Use one of the stop classes if the attr's 19 | module origin is not in this list, this to detect monkey patched 20 | attributes. 21 | 22 | :returns None: if the attribute was not found. 23 | 24 | """ 25 | for node in cls.mro(): 26 | if node in stop: 27 | try: 28 | attr = node.__dict__[attr] 29 | module_origin = attr.__module__ 30 | except (AttributeError, KeyError): 31 | pass 32 | else: 33 | if module_origin not in monkey_patched: 34 | return node 35 | return 36 | if attr in node.__dict__: 37 | return node 38 | -------------------------------------------------------------------------------- /extra/supervisord/supervisord.conf: -------------------------------------------------------------------------------- 1 | [unix_http_server] 2 | file=/tmp/supervisor.sock ; path to your socket file 3 | 4 | [supervisord] 5 | logfile=/var/log/supervisord/supervisord.log ; supervisord log file 6 | logfile_maxbytes=50MB ; maximum size of logfile before rotation 7 | logfile_backups=10 ; number of backed up logfiles 8 | loglevel=info ; info, debug, warn, trace 9 | pidfile=/var/run/supervisord.pid ; pidfile location 10 | nodaemon=false ; run supervisord as a daemon 11 | minfds=1024 ; number of startup file descriptors 12 | minprocs=200 ; number of process descriptors 13 | user=root ; default user 14 | childlogdir=/var/log/supervisord/ ; where child log files will live 15 | 16 | 17 | [rpcinterface:supervisor] 18 | supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface 19 | 20 | [supervisorctl] 21 | serverurl=unix:///tmp/supervisor.sock ; use unix:// schem for a unix sockets. 22 | 23 | 24 | [include] 25 | 26 | # Uncomment this line for celeryd for Python 27 | ;files=celeryd.conf 28 | 29 | # Uncomment this line for celeryd for Django. 30 | ;files=django/celeryd.conf 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /celery/__main__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import sys 4 | 5 | from os.path import basename 6 | 7 | from . import maybe_patch_concurrency 8 | 9 | __all__ = ['main'] 10 | 11 | DEPRECATED_FMT = """ 12 | The {old!r} command is deprecated, please use {new!r} instead: 13 | 14 | $ {new_argv} 15 | 16 | """ 17 | 18 | 19 | def _warn_deprecated(new): 20 | print(DEPRECATED_FMT.format( 21 | old=basename(sys.argv[0]), new=new, 22 | new_argv=' '.join([new] + sys.argv[1:])), 23 | ) 24 | 25 | 26 | def main(): 27 | maybe_patch_concurrency() 28 | from celery.bin.celery import main 29 | main() 30 | 31 | 32 | def _compat_worker(): 33 | maybe_patch_concurrency() 34 | _warn_deprecated('celery worker') 35 | from celery.bin.worker import main 36 | main() 37 | 38 | 39 | def _compat_multi(): 40 | maybe_patch_concurrency() 41 | _warn_deprecated('celery multi') 42 | from celery.bin.multi import main 43 | main() 44 | 45 | 46 | def _compat_beat(): 47 | maybe_patch_concurrency() 48 | _warn_deprecated('celery beat') 49 | from celery.bin.beat import main 50 | main() 51 | 52 | 53 | if __name__ == '__main__': # pragma: no cover 54 | main() 55 | -------------------------------------------------------------------------------- /examples/django/proj/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for proj project. 3 | 4 | This module contains the WSGI application used by Django's development server 5 | and any production WSGI deployments. It should expose a module-level variable 6 | named ``application``. Django's ``runserver`` and ``runfcgi`` commands discover 7 | this application via the ``WSGI_APPLICATION`` setting. 8 | 9 | Usually you will have the standard Django WSGI application here, but it also 10 | might make sense to replace the whole Django WSGI application with a custom one 11 | that later delegates to the Django one. For example, you could introduce WSGI 12 | middleware here, or combine a Django application with an application of another 13 | framework. 14 | 15 | """ 16 | import os 17 | 18 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "proj.settings") 19 | 20 | # This application object is used by any WSGI server configured to use this 21 | # file. This includes Django's development server, if the WSGI_APPLICATION 22 | # setting points here. 23 | from django.core.wsgi import get_wsgi_application 24 | application = get_wsgi_application() 25 | 26 | # Apply WSGI middleware here. 27 | # from helloworld.wsgi import HelloWorldApplication 28 | # application = HelloWorldApplication(application) 29 | -------------------------------------------------------------------------------- /celery/tests/compat_modules/test_decorators.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import warnings 4 | 5 | from celery.task import base 6 | 7 | from celery.tests.case import AppCase, depends_on_current_app 8 | 9 | 10 | def add(x, y): 11 | return x + y 12 | 13 | 14 | @depends_on_current_app 15 | class test_decorators(AppCase): 16 | 17 | def test_task_alias(self): 18 | from celery import task 19 | self.assertTrue(task.__file__) 20 | self.assertTrue(task(add)) 21 | 22 | def setup(self): 23 | with warnings.catch_warnings(record=True): 24 | from celery import decorators 25 | self.decorators = decorators 26 | 27 | def assertCompatDecorator(self, decorator, type, **opts): 28 | task = decorator(**opts)(add) 29 | self.assertEqual(task(8, 8), 16) 30 | self.assertTrue(task.accept_magic_kwargs) 31 | self.assertIsInstance(task, type) 32 | 33 | def test_task(self): 34 | self.assertCompatDecorator(self.decorators.task, base.BaseTask) 35 | 36 | def test_periodic_task(self): 37 | self.assertCompatDecorator(self.decorators.periodic_task, 38 | base.BaseTask, 39 | run_every=1) 40 | -------------------------------------------------------------------------------- /extra/supervisord/celeryd.conf: -------------------------------------------------------------------------------- 1 | ; ================================== 2 | ; celery worker supervisor example 3 | ; ================================== 4 | 5 | ; NOTE: If you're using Django, you shouldn't use this file. 6 | ; Use 7 | ; http://github.com/celery/django-celery/tree/master/extra/supervisord/celeryd.conf 8 | ; instead! 9 | 10 | [program:celery] 11 | command=celery worker -A proj.tasks:app --loglevel=INFO 12 | 13 | ; remove -A argument if you don't use a celery app instance. 14 | 15 | ; Set PYTHONPATH to the directory containing app/celeryconfig.py 16 | environment=PYTHONPATH=/path/to/project 17 | 18 | directory=/path/to/project 19 | user=nobody 20 | numprocs=1 21 | stdout_logfile=/var/log/celeryd.log 22 | stderr_logfile=/var/log/celeryd.log 23 | autostart=true 24 | autorestart=true 25 | startsecs=10 26 | 27 | ; Need to wait for currently executing tasks to finish at shutdown. 28 | ; Increase this if you have very long running tasks. 29 | stopwaitsecs = 600 30 | 31 | ; When resorting to send SIGKILL to the program to terminate it 32 | ; send SIGKILL to its whole process group instead, 33 | ; taking care of its children as well. 34 | killasgroup=true 35 | 36 | ; if rabbitmq is supervised, set its priority higher 37 | ; so it starts first 38 | priority=998 39 | -------------------------------------------------------------------------------- /docs/reference/celery.app.amqp.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: celery.app.amqp 2 | 3 | .. automodule:: celery.app.amqp 4 | 5 | .. contents:: 6 | :local: 7 | 8 | AMQP 9 | ---- 10 | 11 | .. autoclass:: AMQP 12 | 13 | .. attribute:: Connection 14 | 15 | Broker connection class used. Default is 16 | :class:`kombu.Connection`. 17 | 18 | .. attribute:: Consumer 19 | 20 | Base Consumer class used. Default is :class:`kombu.compat.Consumer`. 21 | 22 | .. attribute:: queues 23 | 24 | All currently defined task queues. (A :class:`Queues` instance). 25 | 26 | .. automethod:: Queues 27 | .. automethod:: Router 28 | .. autoattribute:: TaskConsumer 29 | .. autoattribute:: TaskProducer 30 | .. automethod:: flush_routes 31 | 32 | .. autoattribute:: default_queue 33 | .. autoattribute:: default_exchange 34 | .. autoattribute:: publisher_pool 35 | .. autoattribute:: router 36 | .. autoattribute:: routes 37 | 38 | Queues 39 | ------ 40 | 41 | .. autoclass:: Queues 42 | :members: 43 | :undoc-members: 44 | 45 | TaskPublisher 46 | ------------- 47 | 48 | .. autoclass:: TaskPublisher 49 | :members: 50 | :undoc-members: 51 | -------------------------------------------------------------------------------- /extra/release/verify-reference-index.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | RETVAL=0 4 | 5 | verify_index() { 6 | retval=0 7 | for refdir in $*; do 8 | verify_modules_in_index "$refdir/index.rst" 9 | verify_files "$refdir" 10 | done 11 | return $RETVAL 12 | } 13 | 14 | verify_files() { 15 | for path in $1/*.rst; do 16 | rst=${path##*/} 17 | modname=${rst%*.rst} 18 | if [ $modname != "index" ]; then 19 | modpath=$(echo $modname | tr . /) 20 | pkg="$modpath/__init__.py" 21 | mod="$modpath.py" 22 | if [ ! -f "$pkg" ]; then 23 | if [ ! -f "$mod" ]; then 24 | echo "*** NO MODULE $modname for reference '$path'" 25 | RETVAL=1 26 | fi 27 | fi 28 | fi 29 | done 30 | } 31 | 32 | verify_modules_in_index() { 33 | modules=$(grep "celery." "$1" | \ 34 | perl -ple's/^\s*|\s*$//g;s{\.}{/}g;') 35 | for module in $modules; do 36 | if [ ! -f "$module.py" ]; then 37 | if [ ! -f "$module/__init__.py" ]; then 38 | echo "*** IN INDEX BUT NO MODULE: $module" 39 | RETVAL=1 40 | fi 41 | fi 42 | done 43 | } 44 | 45 | verify_index docs/reference docs/internals/reference 46 | -------------------------------------------------------------------------------- /celery/tests/utils/test_serialization.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import sys 4 | 5 | from celery.utils.serialization import ( 6 | UnpickleableExceptionWrapper, 7 | get_pickleable_etype, 8 | ) 9 | 10 | from celery.tests.case import Case, mask_modules 11 | 12 | 13 | class test_AAPickle(Case): 14 | 15 | def test_no_cpickle(self): 16 | prev = sys.modules.pop('celery.utils.serialization', None) 17 | try: 18 | with mask_modules('cPickle'): 19 | from celery.utils.serialization import pickle 20 | import pickle as orig_pickle 21 | self.assertIs(pickle.dumps, orig_pickle.dumps) 22 | finally: 23 | sys.modules['celery.utils.serialization'] = prev 24 | 25 | 26 | class test_UnpickleExceptionWrapper(Case): 27 | 28 | def test_init(self): 29 | x = UnpickleableExceptionWrapper('foo', 'Bar', [10, lambda x: x]) 30 | self.assertTrue(x.exc_args) 31 | self.assertEqual(len(x.exc_args), 2) 32 | 33 | 34 | class test_get_pickleable_etype(Case): 35 | 36 | def test_get_pickleable_etype(self): 37 | 38 | class Unpickleable(Exception): 39 | def __reduce__(self): 40 | raise ValueError('foo') 41 | 42 | self.assertIs(get_pickleable_etype(Unpickleable), Exception) 43 | -------------------------------------------------------------------------------- /docs/reference/index.rst: -------------------------------------------------------------------------------- 1 | .. _apiref: 2 | 3 | =============== 4 | API Reference 5 | =============== 6 | 7 | :Release: |version| 8 | :Date: |today| 9 | 10 | .. toctree:: 11 | :maxdepth: 1 12 | 13 | celery 14 | celery.app 15 | celery.app.task 16 | celery.app.amqp 17 | celery.app.defaults 18 | celery.app.control 19 | celery.app.registry 20 | celery.app.builtins 21 | celery.app.log 22 | celery.app.utils 23 | celery.bootsteps 24 | celery.result 25 | celery.task.http 26 | celery.schedules 27 | celery.signals 28 | celery.security 29 | celery.utils.debug 30 | celery.utils.mail 31 | celery.exceptions 32 | celery.loaders 33 | celery.loaders.app 34 | celery.loaders.default 35 | celery.loaders.base 36 | celery.states 37 | celery.contrib.abortable 38 | celery.contrib.batches 39 | celery.contrib.migrate 40 | celery.contrib.rdb 41 | celery.contrib.methods 42 | celery.events 43 | celery.events.state 44 | celery.beat 45 | celery.apps.worker 46 | celery.apps.beat 47 | celery.worker 48 | celery.worker.consumer 49 | celery.worker.job 50 | celery.worker.state 51 | celery.worker.strategy 52 | celery.bin.base 53 | celery.bin.celery 54 | celery.bin.worker 55 | celery.bin.beat 56 | celery.bin.events 57 | celery.bin.amqp 58 | celery.bin.multi 59 | celery.bin.graph 60 | -------------------------------------------------------------------------------- /requirements/README.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | pip requirements files 3 | ======================== 4 | 5 | 6 | Index 7 | ===== 8 | 9 | * :file:`requirements/default.txt` 10 | 11 | Default requirements for Python 2.7+. 12 | 13 | * :file:`requirements/jython.txt` 14 | 15 | Extra requirements needed to run on Jython 2.5 16 | 17 | * :file:`requirements/security.txt` 18 | 19 | Extra requirements needed to use the message signing serializer, 20 | see the Security Guide. 21 | 22 | * :file:`requirements/test.txt` 23 | 24 | Requirements needed to run the full unittest suite. 25 | 26 | * :file:`requirements/test-ci.txt` 27 | 28 | Extra test requirements required by the CI suite (Tox). 29 | 30 | * :file:`requirements/doc.txt` 31 | 32 | Extra requirements required to build the Sphinx documentation. 33 | 34 | * :file:`requirements/pkgutils.txt` 35 | 36 | Extra requirements required to perform package distribution maintenance. 37 | 38 | * :file:`requirements/dev.txt` 39 | 40 | Requirement file installing the current master branch of Celery and deps. 41 | 42 | Examples 43 | ======== 44 | 45 | Installing requirements 46 | ----------------------- 47 | 48 | :: 49 | 50 | $ pip install -U -r requirements/default.txt 51 | 52 | 53 | Running the tests 54 | ----------------- 55 | 56 | :: 57 | 58 | $ pip install -U -r requirements/default.txt 59 | $ pip install -U -r requirements/test.txt 60 | -------------------------------------------------------------------------------- /celery/tests/tasks/test_states.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from celery.states import state 4 | from celery import states 5 | from celery.tests.case import Case 6 | 7 | 8 | class test_state_precedence(Case): 9 | 10 | def test_gt(self): 11 | self.assertGreater(state(states.SUCCESS), 12 | state(states.PENDING)) 13 | self.assertGreater(state(states.FAILURE), 14 | state(states.RECEIVED)) 15 | self.assertGreater(state(states.REVOKED), 16 | state(states.STARTED)) 17 | self.assertGreater(state(states.SUCCESS), 18 | state('CRASHED')) 19 | self.assertGreater(state(states.FAILURE), 20 | state('CRASHED')) 21 | self.assertFalse(state(states.REVOKED) > state('CRASHED')) 22 | 23 | def test_lt(self): 24 | self.assertLess(state(states.PENDING), state(states.SUCCESS)) 25 | self.assertLess(state(states.RECEIVED), state(states.FAILURE)) 26 | self.assertLess(state(states.STARTED), state(states.REVOKED)) 27 | self.assertLess(state('CRASHED'), state(states.SUCCESS)) 28 | self.assertLess(state('CRASHED'), state(states.FAILURE)) 29 | self.assertTrue(state(states.REVOKED) < state('CRASHED')) 30 | self.assertTrue(state(states.REVOKED) <= state('CRASHED')) 31 | self.assertTrue(state('CRASHED') >= state(states.REVOKED)) 32 | -------------------------------------------------------------------------------- /docs/getting-started/brokers/django.rst: -------------------------------------------------------------------------------- 1 | .. _broker-django: 2 | 3 | =========================== 4 | Using the Django Database 5 | =========================== 6 | 7 | .. admonition:: Experimental Status 8 | 9 | The Django database transport is in need of improvements in many areas 10 | and there are several open bugs. Sadly we don't have the resources or funds 11 | required to improve the situation, so we're looking for contributors 12 | and partners willing to help. 13 | 14 | .. _broker-django-installation: 15 | 16 | Installation 17 | ============ 18 | 19 | .. _broker-django-configuration: 20 | 21 | Configuration 22 | ============= 23 | 24 | The database transport uses the Django `DATABASE_*` settings for database 25 | configuration values. 26 | 27 | #. Set your broker transport:: 28 | 29 | BROKER_URL = 'django://' 30 | 31 | #. Add :mod:`kombu.transport.django` to `INSTALLED_APPS`:: 32 | 33 | INSTALLED_APPS = ('kombu.transport.django', ) 34 | 35 | #. Sync your database schema: 36 | 37 | .. code-block:: bash 38 | 39 | $ python manage.py syncdb 40 | 41 | .. _broker-django-limitations: 42 | 43 | Limitations 44 | =========== 45 | 46 | The Django database transport does not currently support: 47 | 48 | * Remote control commands (:program:`celery events` command, broadcast) 49 | * Events, including the Django Admin monitor. 50 | * Using more than a few workers (can lead to messages being executed 51 | multiple times). 52 | -------------------------------------------------------------------------------- /extra/release/verify_config_reference.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | from fileinput import input as _input 4 | from sys import exit, stderr 5 | 6 | from celery.app.defaults import NAMESPACES, flatten 7 | 8 | ignore = frozenset([ 9 | 'CELERYD_AGENT', 10 | 'CELERYD_POOL_PUTLOCKS', 11 | 'BROKER_HOST', 12 | 'BROKER_USER', 13 | 'BROKER_PASSWORD', 14 | 'BROKER_VHOST', 15 | 'BROKER_PORT', 16 | 'CELERY_REDIS_HOST', 17 | 'CELERY_REDIS_PORT', 18 | 'CELERY_REDIS_DB', 19 | 'CELERY_REDIS_PASSWORD', 20 | ]) 21 | 22 | 23 | def is_ignored(setting, option): 24 | return setting in ignore or option.deprecate_by 25 | 26 | 27 | def find_undocumented_settings(directive='.. setting:: '): 28 | settings = dict(flatten(NAMESPACES)) 29 | all = set(settings) 30 | documented = set( 31 | line.strip()[len(directive):].strip() for line in _input() 32 | if line.strip().startswith(directive) 33 | ) 34 | return [setting for setting in all ^ documented 35 | if not is_ignored(setting, settings[setting])] 36 | 37 | 38 | if __name__ == '__main__': 39 | sep = '\n * ' 40 | missing = find_undocumented_settings() 41 | if missing: 42 | print( 43 | 'Error: found undocumented settings:{0}{1}'.format( 44 | sep, sep.join(sorted(missing))), 45 | file=stderr, 46 | ) 47 | exit(1) 48 | print('OK: Configuration reference complete :-)') 49 | exit(0) 50 | -------------------------------------------------------------------------------- /extra/centos/test_celeryd.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # If you make changes to the celeryd init script, 4 | # you can use this test script to verify you didn't break the universe 5 | 6 | SERVICE="celeryd" 7 | SERVICE_CMD="sudo /sbin/service $SERVICE" 8 | 9 | run_test() { 10 | local msg="$1" 11 | local cmd="$2" 12 | local expected_retval="${3:-0}" 13 | local n=${#msg} 14 | 15 | echo 16 | echo `printf "%$((${n}+4))s" | tr " " "#"` 17 | echo "# $msg #" 18 | echo `printf "%$((${n}+4))s" | tr " " "#"` 19 | 20 | $cmd 21 | local retval=$? 22 | if [[ "$retval" == "$expected_retval" ]]; then 23 | echo "[PASSED]" 24 | else 25 | echo "[FAILED]" 26 | echo "Exit status: $retval, but expected: $expected_retval" 27 | exit $retval 28 | fi 29 | } 30 | 31 | run_test "stop should succeed" "$SERVICE_CMD stop" 0 32 | run_test "status on a stopped service should return 1" "$SERVICE_CMD status" 1 33 | run_test "stopping a stopped celery should not fail" "$SERVICE_CMD stop" 0 34 | run_test "start should succeed" "$SERVICE_CMD start" 0 35 | run_test "status on a running service should return 0" "$SERVICE_CMD status" 0 36 | run_test "starting a running service should fail" "$SERVICE_CMD start" 1 37 | run_test "restarting a running service should succeed" "$SERVICE_CMD restart" 0 38 | run_test "status on a restarted service should return 0" "$SERVICE_CMD status" 0 39 | run_test "stop should succeed" "$SERVICE_CMD stop" 0 40 | 41 | echo "All tests passed!" 42 | -------------------------------------------------------------------------------- /celery/tests/utils/test_imports.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from mock import Mock, patch 4 | 5 | from celery.utils.imports import ( 6 | qualname, 7 | reload_from_cwd, 8 | module_file, 9 | find_module, 10 | NotAPackage, 11 | ) 12 | 13 | from celery.tests.case import Case 14 | 15 | 16 | class test_import_utils(Case): 17 | 18 | def test_find_module(self): 19 | self.assertTrue(find_module('celery')) 20 | imp = Mock() 21 | imp.return_value = None 22 | with self.assertRaises(NotAPackage): 23 | find_module('foo.bar.baz', imp=imp) 24 | 25 | def test_qualname(self): 26 | Class = type('Fox', (object, ), {'__module__': 'quick.brown'}) 27 | self.assertEqual(qualname(Class), 'quick.brown.Fox') 28 | self.assertEqual(qualname(Class()), 'quick.brown.Fox') 29 | 30 | @patch('celery.utils.imports.reload') 31 | def test_reload_from_cwd(self, reload): 32 | reload_from_cwd('foo') 33 | self.assertTrue(reload.called) 34 | 35 | def test_reload_from_cwd_custom_reloader(self): 36 | reload = Mock() 37 | reload_from_cwd('foo', reload) 38 | self.assertTrue(reload.called) 39 | 40 | def test_module_file(self): 41 | m1 = Mock() 42 | m1.__file__ = '/opt/foo/xyz.pyc' 43 | self.assertEqual(module_file(m1), '/opt/foo/xyz.py') 44 | m2 = Mock() 45 | m2.__file__ = '/opt/foo/xyz.py' 46 | self.assertEqual(module_file(m1), '/opt/foo/xyz.py') 47 | -------------------------------------------------------------------------------- /extra/release/doc4allmods: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | PACKAGE="$1" 4 | SKIP_PACKAGES="$PACKAGE tests management urls" 5 | SKIP_FILES="celery.five.rst 6 | celery.__main__.rst 7 | celery.task.rst 8 | celery.task.base.rst 9 | celery.task.sets.rst 10 | celery.task.trace.rst 11 | celery.bin.rst 12 | celery.bin.celeryd_detach.rst 13 | celery.contrib.rst 14 | celery.fixups.rst 15 | celery.fixups.django.rst 16 | celery.local.rst 17 | celery.app.base.rst 18 | celery.apps.rst 19 | celery.canvas.rst 20 | celery.concurrency.asynpool.rst 21 | celery.utils.encoding.rst" 22 | 23 | modules=$(find "$PACKAGE" -name "*.py") 24 | 25 | failed=0 26 | for module in $modules; do 27 | dotted=$(echo $module | sed 's/\//\./g') 28 | name=${dotted%.__init__.py} 29 | name=${name%.py} 30 | rst=$name.rst 31 | skip=0 32 | for skip_package in $SKIP_PACKAGES; do 33 | [ $(echo "$name" | cut -d. -f 2) == "$skip_package" ] && skip=1 34 | done 35 | for skip_file in $SKIP_FILES; do 36 | [ "$skip_file" == "$rst" ] && skip=1 37 | done 38 | 39 | if [ $skip -eq 0 ]; then 40 | if [ ! -f "docs/reference/$rst" ]; then 41 | if [ ! -f "docs/internals/reference/$rst" ]; then 42 | echo $rst :: FAIL 43 | failed=1 44 | fi 45 | fi 46 | fi 47 | done 48 | 49 | exit $failed 50 | -------------------------------------------------------------------------------- /examples/celery_http_gateway/README.rst: -------------------------------------------------------------------------------- 1 | ============================== 2 | Example Celery->HTTP Gateway 3 | ============================== 4 | 5 | This is an example service exposing the ability to apply tasks and query 6 | statuses/results over HTTP. 7 | 8 | Some familiarity with Django is recommended. 9 | 10 | `settings.py` contains the celery settings, you probably want to configure 11 | at least the broker related settings. 12 | 13 | To run the service you have to run the following commands:: 14 | 15 | $ python manage.py syncdb # (if running the database backend) 16 | 17 | $ python manage.py runserver 18 | 19 | 20 | The service is now running at http://localhost:8000 21 | 22 | 23 | You can apply tasks, with the `/apply/` URL:: 24 | 25 | $ curl http://localhost:8000/apply/celery.ping/ 26 | {"ok": "true", "task_id": "e3a95109-afcd-4e54-a341-16c18fddf64b"} 27 | 28 | Then you can use the resulting task-id to get the return value:: 29 | 30 | $ curl http://localhost:8000/e3a95109-afcd-4e54-a341-16c18fddf64b/status/ 31 | {"task": {"status": "SUCCESS", "result": "pong", "id": "e3a95109-afcd-4e54-a341-16c18fddf64b"}} 32 | 33 | 34 | If you don't want to expose all tasks there are a few possible 35 | approaches. For instance you can extend the `apply` view to only 36 | accept a whitelist. Another possibility is to just make views for every task you want to 37 | expose. We made on such view for ping in `views.ping`:: 38 | 39 | $ curl http://localhost:8000/ping/ 40 | {"ok": "true", "task_id": "383c902c-ba07-436b-b0f3-ea09cc22107c"} 41 | -------------------------------------------------------------------------------- /docs/getting-started/brokers/mongodb.rst: -------------------------------------------------------------------------------- 1 | .. _broker-mongodb: 2 | 3 | =============== 4 | Using MongoDB 5 | =============== 6 | 7 | .. admonition:: Experimental Status 8 | 9 | The SQS transport is in need of improvements in many areas and there 10 | are several open bugs. Sadly we don't have the resources or funds 11 | required to improve the situation, so we're looking for contributors 12 | and partners willing to help. 13 | 14 | .. _broker-mongodb-installation: 15 | 16 | Installation 17 | ============ 18 | 19 | For the MongoDB support you have to install additional dependencies. 20 | You can install both Celery and these dependencies in one go using 21 | the ``celery[mongodb]`` :ref:`bundle `: 22 | 23 | .. code-block:: bash 24 | 25 | $ pip install -U celery[mongodb] 26 | 27 | .. _broker-mongodb-configuration: 28 | 29 | Configuration 30 | ============= 31 | 32 | Configuration is easy, set the transport, and configure the location of 33 | your MongoDB database:: 34 | 35 | BROKER_URL = 'mongodb://localhost:27017/database_name' 36 | 37 | Where the URL is in the format of:: 38 | 39 | mongodb://userid:password@hostname:port/database_name 40 | 41 | The host name will default to ``localhost`` and the port to 27017, 42 | and so they are optional. userid and password are also optional, 43 | but needed if your MongoDB server requires authentication. 44 | 45 | .. _mongodb-results-configuration: 46 | 47 | Results 48 | ------- 49 | 50 | If you also want to store the state and return values of tasks in MongoDB, 51 | you should see :ref:`conf-mongodb-result-backend`. 52 | -------------------------------------------------------------------------------- /docs/internals/worker.rst: -------------------------------------------------------------------------------- 1 | .. _internals-worker: 2 | 3 | ======================= 4 | Internals: The worker 5 | ======================= 6 | 7 | .. contents:: 8 | :local: 9 | 10 | Introduction 11 | ============ 12 | 13 | The worker consists of 4 main components: the consumer, the scheduler, 14 | the mediator and the task pool. All these components runs in parallel working 15 | with two data structures: the ready queue and the ETA schedule. 16 | 17 | Data structures 18 | =============== 19 | 20 | timer 21 | ----- 22 | 23 | The timer uses :mod:`heapq` to schedule internal functions. 24 | It's very efficient and can handle hundred of thousands of entries. 25 | 26 | 27 | Components 28 | ========== 29 | 30 | Consumer 31 | -------- 32 | 33 | Receives messages from the broker using `Kombu`_. 34 | 35 | .. _`Kombu`: http://pypi.python.org/pypi/kombu 36 | 37 | When a message is received it's converted into a 38 | :class:`celery.worker.job.TaskRequest` object. 39 | 40 | Tasks with an ETA, or rate-limit are entered into the `timer`, 41 | messages that can be immediately processed are sent to the execution pool. 42 | 43 | Timer 44 | ----- 45 | 46 | The timer schedules internal functions, like cleanup and internal monitoring, 47 | but also it schedules ETA tasks and rate limited tasks. 48 | If the scheduled tasks eta has passed it is moved to the execution pool. 49 | 50 | TaskPool 51 | -------- 52 | 53 | This is a slightly modified :class:`multiprocessing.Pool`. 54 | It mostly works the same way, except it makes sure all of the workers 55 | are running at all times. If a worker is missing, it replaces 56 | it with a new one. 57 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | ================================= 2 | Celery - Distributed Task Queue 3 | ================================= 4 | 5 | Celery is a simple, flexible and reliable distributed system to 6 | process vast amounts of messages, while providing operations with 7 | the tools required to maintain such a system. 8 | 9 | It's a task queue with focus on real-time processing, while also 10 | supporting task scheduling. 11 | 12 | Celery has a large and diverse community of users and contributors, 13 | you should come join us :ref:`on IRC ` 14 | or :ref:`our mailing-list `. 15 | 16 | Celery is Open Source and licensed under the `BSD License`_. 17 | 18 | Getting Started 19 | =============== 20 | 21 | - If you are new to Celery you can get started by following 22 | the :ref:`first-steps` tutorial. 23 | 24 | - You can also check out the :ref:`FAQ `. 25 | 26 | .. _`BSD License`: http://www.opensource.org/licenses/BSD-3-Clause 27 | 28 | Contents 29 | ======== 30 | 31 | .. toctree:: 32 | :maxdepth: 1 33 | 34 | copyright 35 | 36 | .. toctree:: 37 | :maxdepth: 2 38 | 39 | getting-started/index 40 | userguide/index 41 | 42 | .. toctree:: 43 | :maxdepth: 1 44 | 45 | configuration 46 | django/index 47 | contributing 48 | community 49 | tutorials/index 50 | faq 51 | changelog 52 | whatsnew-3.1 53 | whatsnew-3.0 54 | whatsnew-2.5 55 | reference/index 56 | internals/index 57 | history/index 58 | glossary 59 | 60 | 61 | Indices and tables 62 | ================== 63 | 64 | * :ref:`genindex` 65 | * :ref:`modindex` 66 | * :ref:`search` 67 | 68 | -------------------------------------------------------------------------------- /docs/getting-started/brokers/beanstalk.rst: -------------------------------------------------------------------------------- 1 | .. _broker-beanstalk: 2 | 3 | ================= 4 | Using Beanstalk 5 | ================= 6 | 7 | .. _broker-beanstalk-installation: 8 | 9 | .. admonition:: Out of order 10 | 11 | The Beanstalk transport is currently not working well. 12 | 13 | We are interested in contributions and donations that can go towards 14 | improving this situation. 15 | 16 | 17 | 18 | Installation 19 | ============ 20 | 21 | For the Beanstalk support you have to install additional dependencies. 22 | You can install both Celery and these dependencies in one go using 23 | the ``celery[beanstalk]`` :ref:`bundle `: 24 | 25 | .. code-block:: bash 26 | 27 | $ pip install -U celery[beanstalk] 28 | 29 | .. _broker-beanstalk-configuration: 30 | 31 | Configuration 32 | ============= 33 | 34 | Configuration is easy, set the transport, and configure the location of 35 | your Beanstalk database:: 36 | 37 | BROKER_URL = 'beanstalk://localhost:11300' 38 | 39 | Where the URL is in the format of:: 40 | 41 | beanstalk://hostname:port 42 | 43 | The host name will default to ``localhost`` and the port to 11300, 44 | and so they are optional. 45 | 46 | .. _beanstalk-results-configuration: 47 | 48 | Results 49 | ------- 50 | 51 | Using Beanstalk to store task state and results is currently **not supported**. 52 | 53 | .. _broker-beanstalk-limitations: 54 | 55 | Limitations 56 | =========== 57 | 58 | The Beanstalk message transport does not currently support: 59 | 60 | * Remote control commands (:program:`celery control`, 61 | :program:`celery inspect`, broadcast) 62 | * Authentication 63 | 64 | -------------------------------------------------------------------------------- /docs/includes/resources.txt: -------------------------------------------------------------------------------- 1 | .. _getting-help: 2 | 3 | Getting Help 4 | ============ 5 | 6 | .. _mailing-list: 7 | 8 | Mailing list 9 | ------------ 10 | 11 | For discussions about the usage, development, and future of celery, 12 | please join the `celery-users`_ mailing list. 13 | 14 | .. _`celery-users`: http://groups.google.com/group/celery-users/ 15 | 16 | .. _irc-channel: 17 | 18 | IRC 19 | --- 20 | 21 | Come chat with us on IRC. The **#celery** channel is located at the `Freenode`_ 22 | network. 23 | 24 | .. _`Freenode`: http://freenode.net 25 | 26 | .. _bug-tracker: 27 | 28 | Bug tracker 29 | =========== 30 | 31 | If you have any suggestions, bug reports or annoyances please report them 32 | to our issue tracker at http://github.com/celery/celery/issues/ 33 | 34 | .. _wiki: 35 | 36 | Wiki 37 | ==== 38 | 39 | http://wiki.github.com/celery/celery/ 40 | 41 | .. _contributing-short: 42 | 43 | Contributing 44 | ============ 45 | 46 | Development of `celery` happens at Github: http://github.com/celery/celery 47 | 48 | You are highly encouraged to participate in the development 49 | of `celery`. If you don't like Github (for some reason) you're welcome 50 | to send regular patches. 51 | 52 | Be sure to also read the `Contributing to Celery`_ section in the 53 | documentation. 54 | 55 | .. _`Contributing to Celery`: 56 | http://docs.celeryproject.org/en/master/contributing.html 57 | 58 | .. _license: 59 | 60 | License 61 | ======= 62 | 63 | This software is licensed under the `New BSD License`. See the :file:`LICENSE` 64 | file in the top distribution directory for the full license text. 65 | 66 | .. # vim: syntax=rst expandtab tabstop=4 shiftwidth=4 shiftround 67 | -------------------------------------------------------------------------------- /celery/tests/compat_modules/test_compat_utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import celery 4 | 5 | from celery.app.task import Task as ModernTask 6 | from celery.task.base import Task as CompatTask 7 | 8 | from celery.tests.case import AppCase, depends_on_current_app 9 | 10 | 11 | @depends_on_current_app 12 | class test_MagicModule(AppCase): 13 | 14 | def test_class_property_set_without_type(self): 15 | self.assertTrue(ModernTask.__dict__['app'].__get__(CompatTask())) 16 | 17 | def test_class_property_set_on_class(self): 18 | self.assertIs(ModernTask.__dict__['app'].__set__(None, None), 19 | ModernTask.__dict__['app']) 20 | 21 | def test_class_property_set(self): 22 | 23 | class X(CompatTask): 24 | pass 25 | ModernTask.__dict__['app'].__set__(X(), self.app) 26 | self.assertIs(X.app, self.app) 27 | 28 | def test_dir(self): 29 | self.assertTrue(dir(celery.messaging)) 30 | 31 | def test_direct(self): 32 | self.assertTrue(celery.task) 33 | 34 | def test_app_attrs(self): 35 | self.assertEqual(celery.task.control.broadcast, 36 | celery.current_app.control.broadcast) 37 | 38 | def test_decorators_task(self): 39 | @celery.decorators.task 40 | def _test_decorators_task(): 41 | pass 42 | 43 | self.assertTrue(_test_decorators_task.accept_magic_kwargs) 44 | 45 | def test_decorators_periodic_task(self): 46 | @celery.decorators.periodic_task(run_every=3600) 47 | def _test_decorators_ptask(): 48 | pass 49 | 50 | self.assertTrue(_test_decorators_ptask.accept_magic_kwargs) 51 | -------------------------------------------------------------------------------- /examples/eventlet/README.rst: -------------------------------------------------------------------------------- 1 | ================================== 2 | Example using the Eventlet Pool 3 | ================================== 4 | 5 | Introduction 6 | ============ 7 | 8 | This is a Celery application containing two example tasks. 9 | 10 | First you need to install Eventlet, and also recommended is the `dnspython` 11 | module (when this is installed all name lookups will be asynchronous):: 12 | 13 | $ pip install eventlet 14 | $ pip install dnspython 15 | $ pip install requests 16 | 17 | Before you run any of the example tasks you need to start 18 | the worker:: 19 | 20 | $ cd examples/eventlet 21 | $ celery worker -l info --concurrency=500 --pool=eventlet 22 | 23 | As usual you need to have RabbitMQ running, see the Celery getting started 24 | guide if you haven't installed it yet. 25 | 26 | Tasks 27 | ===== 28 | 29 | * `tasks.urlopen` 30 | 31 | This task simply makes a request opening the URL and returns the size 32 | of the response body:: 33 | 34 | $ cd examples/eventlet 35 | $ python 36 | >>> from tasks import urlopen 37 | >>> urlopen.delay("http://www.google.com/").get() 38 | 9980 39 | 40 | To open several URLs at once you can do:: 41 | 42 | $ cd examples/eventlet 43 | $ python 44 | >>> from tasks import urlopen 45 | >>> from celery import group 46 | >>> result = group(urlopen.s(url) 47 | ... for url in LIST_OF_URLS).apply_async() 48 | >>> for incoming_result in result.iter_native(): 49 | ... print(incoming_result, ) 50 | 51 | * `webcrawler.crawl` 52 | 53 | This is a simple recursive web crawler. It will only crawl 54 | URLs for the current host name. Please see comments in the 55 | `webcrawler.py` file. 56 | -------------------------------------------------------------------------------- /celery/tests/contrib/test_abortable.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from celery.contrib.abortable import AbortableTask, AbortableAsyncResult 4 | from celery.tests.case import AppCase 5 | 6 | 7 | class test_AbortableTask(AppCase): 8 | 9 | def setup(self): 10 | 11 | @self.app.task(base=AbortableTask, shared=False) 12 | def abortable(): 13 | return True 14 | self.abortable = abortable 15 | 16 | def test_async_result_is_abortable(self): 17 | result = self.abortable.apply_async() 18 | tid = result.id 19 | self.assertIsInstance( 20 | self.abortable.AsyncResult(tid), AbortableAsyncResult, 21 | ) 22 | 23 | def test_is_not_aborted(self): 24 | self.abortable.push_request() 25 | try: 26 | result = self.abortable.apply_async() 27 | tid = result.id 28 | self.assertFalse(self.abortable.is_aborted(task_id=tid)) 29 | finally: 30 | self.abortable.pop_request() 31 | 32 | def test_is_aborted_not_abort_result(self): 33 | self.abortable.AsyncResult = self.app.AsyncResult 34 | self.abortable.push_request() 35 | try: 36 | self.abortable.request.id = 'foo' 37 | self.assertFalse(self.abortable.is_aborted()) 38 | finally: 39 | self.abortable.pop_request() 40 | 41 | def test_abort_yields_aborted(self): 42 | self.abortable.push_request() 43 | try: 44 | result = self.abortable.apply_async() 45 | result.abort() 46 | tid = result.id 47 | self.assertTrue(self.abortable.is_aborted(task_id=tid)) 48 | finally: 49 | self.abortable.pop_request() 50 | -------------------------------------------------------------------------------- /celery/app/annotations.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | celery.app.annotations 4 | ~~~~~~~~~~~~~~~~~~~~~~ 5 | 6 | Annotations is a nice term for moneky patching 7 | task classes in the configuration. 8 | 9 | This prepares and performs the annotations in the 10 | :setting:`CELERY_ANNOTATIONS` setting. 11 | 12 | """ 13 | from __future__ import absolute_import 14 | 15 | from celery.five import string_t 16 | from celery.utils.functional import firstmethod, mlazy 17 | from celery.utils.imports import instantiate 18 | 19 | _first_match = firstmethod('annotate') 20 | _first_match_any = firstmethod('annotate_any') 21 | 22 | __all__ = ['MapAnnotation', 'prepare', 'resolve_all'] 23 | 24 | 25 | class MapAnnotation(dict): 26 | 27 | def annotate_any(self): 28 | try: 29 | return dict(self['*']) 30 | except KeyError: 31 | pass 32 | 33 | def annotate(self, task): 34 | try: 35 | return dict(self[task.name]) 36 | except KeyError: 37 | pass 38 | 39 | 40 | def prepare(annotations): 41 | """Expands the :setting:`CELERY_ANNOTATIONS` setting.""" 42 | 43 | def expand_annotation(annotation): 44 | if isinstance(annotation, dict): 45 | return MapAnnotation(annotation) 46 | elif isinstance(annotation, string_t): 47 | return mlazy(instantiate, annotation) 48 | return annotation 49 | 50 | if annotations is None: 51 | return () 52 | elif not isinstance(annotations, (list, tuple)): 53 | annotations = (annotations, ) 54 | return [expand_annotation(anno) for anno in annotations] 55 | 56 | 57 | def resolve_all(anno, task): 58 | return (x for x in (_first_match(anno, task), _first_match_any(anno)) if x) 59 | -------------------------------------------------------------------------------- /funtests/stress/stress/__main__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, print_function 2 | 3 | from celery.bin.base import Command, Option 4 | 5 | from .app import app 6 | from .suite import Suite 7 | 8 | 9 | class Stress(Command): 10 | 11 | def run(self, *names, **options): 12 | try: 13 | return Suite( 14 | self.app, 15 | block_timeout=options.get('block_timeout'), 16 | ).run(names, **options) 17 | except KeyboardInterrupt: 18 | pass 19 | 20 | def get_options(self): 21 | return ( 22 | Option('-i', '--iterations', type='int', default=50, 23 | help='Number of iterations for each test'), 24 | Option('-n', '--numtests', type='int', default=None, 25 | help='Number of tests to execute'), 26 | Option('-o', '--offset', type='int', default=0, 27 | help='Start at custom offset'), 28 | Option('--block-timeout', type='int', default=30 * 60), 29 | Option('-l', '--list', action='store_true', dest='list_all', 30 | help='List all tests'), 31 | Option('-r', '--repeat', type='float', default=0, 32 | help='Number of times to repeat the test suite'), 33 | Option('-g', '--group', default='all', 34 | help='Specify test group (all|green)'), 35 | Option('--diag', default=False, action='store_true', 36 | help='Enable diagnostics (slow)'), 37 | Option('-J', '--no-join', default=False, action='store_true', 38 | help='Do not wait for task results'), 39 | ) 40 | 41 | 42 | if __name__ == '__main__': 43 | Stress(app=app).execute_from_commandline() 44 | -------------------------------------------------------------------------------- /celery/tests/app/test_annotations.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from celery.app.annotations import MapAnnotation, prepare 4 | from celery.utils.imports import qualname 5 | 6 | from celery.tests.case import AppCase 7 | 8 | 9 | class MyAnnotation(object): 10 | foo = 65 11 | 12 | 13 | class AnnotationCase(AppCase): 14 | 15 | def setup(self): 16 | @self.app.task(shared=False) 17 | def add(x, y): 18 | return x + y 19 | self.add = add 20 | 21 | @self.app.task(shared=False) 22 | def mul(x, y): 23 | return x * y 24 | self.mul = mul 25 | 26 | 27 | class test_MapAnnotation(AnnotationCase): 28 | 29 | def test_annotate(self): 30 | x = MapAnnotation({self.add.name: {'foo': 1}}) 31 | self.assertDictEqual(x.annotate(self.add), {'foo': 1}) 32 | self.assertIsNone(x.annotate(self.mul)) 33 | 34 | def test_annotate_any(self): 35 | x = MapAnnotation({'*': {'foo': 2}}) 36 | self.assertDictEqual(x.annotate_any(), {'foo': 2}) 37 | 38 | x = MapAnnotation() 39 | self.assertIsNone(x.annotate_any()) 40 | 41 | 42 | class test_prepare(AnnotationCase): 43 | 44 | def test_dict_to_MapAnnotation(self): 45 | x = prepare({self.add.name: {'foo': 3}}) 46 | self.assertIsInstance(x[0], MapAnnotation) 47 | 48 | def test_returns_list(self): 49 | self.assertListEqual(prepare(1), [1]) 50 | self.assertListEqual(prepare([1]), [1]) 51 | self.assertListEqual(prepare((1, )), [1]) 52 | self.assertEqual(prepare(None), ()) 53 | 54 | def test_evalutes_qualnames(self): 55 | self.assertEqual(prepare(qualname(MyAnnotation))[0]().foo, 65) 56 | self.assertEqual(prepare([qualname(MyAnnotation)])[0]().foo, 65) 57 | -------------------------------------------------------------------------------- /celery/tests/utils/test_pickle.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from celery.utils.serialization import pickle 4 | from celery.tests.case import Case 5 | 6 | 7 | class RegularException(Exception): 8 | pass 9 | 10 | 11 | class ArgOverrideException(Exception): 12 | 13 | def __init__(self, message, status_code=10): 14 | self.status_code = status_code 15 | Exception.__init__(self, message, status_code) 16 | 17 | 18 | class test_Pickle(Case): 19 | 20 | def test_pickle_regular_exception(self): 21 | exc = None 22 | try: 23 | raise RegularException('RegularException raised') 24 | except RegularException as exc_: 25 | exc = exc_ 26 | 27 | pickled = pickle.dumps({'exception': exc}) 28 | unpickled = pickle.loads(pickled) 29 | exception = unpickled.get('exception') 30 | self.assertTrue(exception) 31 | self.assertIsInstance(exception, RegularException) 32 | self.assertTupleEqual(exception.args, ('RegularException raised', )) 33 | 34 | def test_pickle_arg_override_exception(self): 35 | 36 | exc = None 37 | try: 38 | raise ArgOverrideException( 39 | 'ArgOverrideException raised', status_code=100, 40 | ) 41 | except ArgOverrideException as exc_: 42 | exc = exc_ 43 | 44 | pickled = pickle.dumps({'exception': exc}) 45 | unpickled = pickle.loads(pickled) 46 | exception = unpickled.get('exception') 47 | self.assertTrue(exception) 48 | self.assertIsInstance(exception, ArgOverrideException) 49 | self.assertTupleEqual(exception.args, ( 50 | 'ArgOverrideException raised', 100)) 51 | self.assertEqual(exception.status_code, 100) 52 | -------------------------------------------------------------------------------- /docs/getting-started/brokers/couchdb.rst: -------------------------------------------------------------------------------- 1 | .. _broker-couchdb: 2 | 3 | =============== 4 | Using CouchDB 5 | =============== 6 | 7 | .. admonition:: Experimental Status 8 | 9 | The CouchDB transport is in need of improvements in many areas and there 10 | are several open bugs. Sadly we don't have the resources or funds 11 | required to improve the situation, so we're looking for contributors 12 | and partners willing to help. 13 | 14 | .. _broker-couchdb-installation: 15 | 16 | Installation 17 | ============ 18 | 19 | For the CouchDB support you have to install additional dependencies. 20 | You can install both Celery and these dependencies in one go using 21 | the ``celery[couchdb]`` :ref:`bundle `: 22 | 23 | .. code-block:: bash 24 | 25 | $ pip install -U celery[couchdb] 26 | 27 | .. _broker-couchdb-configuration: 28 | 29 | Configuration 30 | ============= 31 | 32 | Configuration is easy, set the transport, and configure the location of 33 | your CouchDB database:: 34 | 35 | BROKER_URL = 'couchdb://localhost:5984/database_name' 36 | 37 | Where the URL is in the format of:: 38 | 39 | couchdb://userid:password@hostname:port/database_name 40 | 41 | The host name will default to ``localhost`` and the port to 5984, 42 | and so they are optional. userid and password are also optional, 43 | but needed if your CouchDB server requires authentication. 44 | 45 | .. _couchdb-results-configuration: 46 | 47 | Results 48 | ------- 49 | 50 | Storing task state and results in CouchDB is currently **not supported**. 51 | 52 | .. _broker-couchdb-limitations: 53 | 54 | Limitations 55 | =========== 56 | 57 | The CouchDB message transport does not currently support: 58 | 59 | * Remote control commands (:program:`celery inspect`, 60 | :program:`celery control`, broadcast) 61 | -------------------------------------------------------------------------------- /celery/tests/utils/test_mail.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from mock import Mock, patch 4 | 5 | from celery.utils.mail import Message, Mailer, SSLError 6 | 7 | from celery.tests.case import Case 8 | 9 | 10 | msg = Message(to='george@vandelay.com', sender='elaine@pendant.com', 11 | subject="What's up with Jerry?", body='???!') 12 | 13 | 14 | class test_Message(Case): 15 | 16 | def test_repr(self): 17 | self.assertTrue(repr(msg)) 18 | 19 | def test_str(self): 20 | self.assertTrue(str(msg)) 21 | 22 | 23 | class test_Mailer(Case): 24 | 25 | def test_send_wrapper(self): 26 | mailer = Mailer() 27 | mailer._send = Mock() 28 | mailer.send(msg) 29 | mailer._send.assert_called_with(msg) 30 | 31 | @patch('smtplib.SMTP_SSL', create=True) 32 | def test_send_ssl_tls(self, SMTP_SSL): 33 | mailer = Mailer(use_ssl=True, use_tls=True) 34 | client = SMTP_SSL.return_value = Mock() 35 | mailer._send(msg) 36 | self.assertTrue(client.starttls.called) 37 | self.assertEqual(client.ehlo.call_count, 2) 38 | client.quit.assert_called_with() 39 | client.sendmail.assert_called_with(msg.sender, msg.to, str(msg)) 40 | mailer = Mailer(use_ssl=True, use_tls=True, user='foo', 41 | password='bar') 42 | mailer._send(msg) 43 | client.login.assert_called_with('foo', 'bar') 44 | 45 | @patch('smtplib.SMTP') 46 | def test_send(self, SMTP): 47 | client = SMTP.return_value = Mock() 48 | mailer = Mailer(use_ssl=False, use_tls=False) 49 | mailer._send(msg) 50 | 51 | client.sendmail.assert_called_With(msg.sender, msg.to, str(msg)) 52 | 53 | client.quit.side_effect = SSLError() 54 | mailer._send(msg) 55 | client.close.assert_called_with() 56 | -------------------------------------------------------------------------------- /funtests/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | try: 5 | from setuptools import setup 6 | from setuptools.command.install import install 7 | except ImportError: 8 | from ez_setup import use_setuptools 9 | use_setuptools() 10 | from setuptools import setup # noqa 11 | from setuptools.command.install import install # noqa 12 | 13 | import os 14 | import sys 15 | 16 | sys.path.insert(0, os.getcwd()) 17 | sys.path.insert(0, os.path.join(os.getcwd(), os.pardir)) 18 | import suite # noqa 19 | 20 | 21 | class no_install(install): 22 | 23 | def run(self, *args, **kwargs): 24 | import sys 25 | sys.stderr.write(""" 26 | ------------------------------------------------------ 27 | The Celery functional test suite cannot be installed. 28 | ------------------------------------------------------ 29 | 30 | 31 | But you can execute the tests by running the command: 32 | 33 | $ python setup.py test 34 | 35 | 36 | """) 37 | 38 | 39 | setup( 40 | name='celery-funtests', 41 | version='DEV', 42 | description='Functional test suite for Celery', 43 | author='Ask Solem', 44 | author_email='ask@celeryproject.org', 45 | url='http://github.com/celery/celery', 46 | platforms=['any'], 47 | packages=[], 48 | data_files=[], 49 | zip_safe=False, 50 | cmdclass={'install': no_install}, 51 | test_suite='nose.collector', 52 | tests_require=[ 53 | 'unittest2>=0.4.0', 54 | 'simplejson', 55 | 'nose', 56 | 'redis', 57 | 'pymongo', 58 | ], 59 | classifiers=[ 60 | 'Operating System :: OS Independent', 61 | 'Programming Language :: Python', 62 | 'License :: OSI Approved :: BSD License', 63 | 'Intended Audience :: Developers', 64 | ], 65 | long_description='Do not install this package', 66 | ) 67 | -------------------------------------------------------------------------------- /celery/utils/dispatch/license.txt: -------------------------------------------------------------------------------- 1 | django.dispatch was originally forked from PyDispatcher. 2 | 3 | PyDispatcher License: 4 | 5 | Copyright (c) 2001-2003, Patrick K. O'Brien and Contributors 6 | All rights reserved. 7 | 8 | Redistribution and use in source and binary forms, with or without 9 | modification, are permitted provided that the following conditions 10 | are met: 11 | 12 | Redistributions of source code must retain the above copyright 13 | notice, this list of conditions and the following disclaimer. 14 | 15 | Redistributions in binary form must reproduce the above 16 | copyright notice, this list of conditions and the following 17 | disclaimer in the documentation and/or other materials 18 | provided with the distribution. 19 | 20 | The name of Patrick K. O'Brien, or the name of any Contributor, 21 | may not be used to endorse or promote products derived from this 22 | software without specific prior written permission. 23 | 24 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 25 | ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 26 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 27 | FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 28 | COPYRIGHT HOLDERS AND CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 29 | INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 30 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 31 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 32 | HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 33 | STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 35 | OF THE POSSIBILITY OF SUCH DAMAGE. 36 | -------------------------------------------------------------------------------- /examples/eventlet/bulk_task_producer.py: -------------------------------------------------------------------------------- 1 | 2 | from eventlet import spawn_n, monkey_patch, Timeout 3 | from eventlet.queue import LightQueue 4 | from eventlet.event import Event 5 | 6 | from celery import current_app 7 | 8 | monkey_patch() 9 | 10 | 11 | class Receipt(object): 12 | result = None 13 | 14 | def __init__(self, callback=None): 15 | self.callback = None 16 | self.ready = Event() 17 | 18 | def finished(self, result): 19 | self.result = result 20 | if self.callback: 21 | self.callback(result) 22 | self.ready.send() 23 | 24 | def wait(self, timeout=None): 25 | with Timeout(timeout): 26 | return self.ready.wait() 27 | 28 | 29 | class ProducerPool(object): 30 | Receipt = Receipt 31 | 32 | def __init__(self, size=20): 33 | self.size = size 34 | self.inqueue = LightQueue() 35 | self._running = None 36 | self._producers = None 37 | 38 | def apply_async(self, task, args, kwargs, callback=None, **options): 39 | if self._running is None: 40 | self._running = spawn_n(self._run) 41 | receipt = self.Receipt(callback) 42 | self.inqueue.put((task, args, kwargs, options, receipt)) 43 | return receipt 44 | 45 | def _run(self): 46 | self._producers = [ 47 | spawn_n(self._producer) for _ in range(self.size) 48 | ] 49 | 50 | def _producer(self): 51 | connection = current_app.connection() 52 | publisher = current_app.amqp.TaskProducer(connection) 53 | inqueue = self.inqueue 54 | 55 | while 1: 56 | task, args, kwargs, options, receipt = inqueue.get() 57 | result = task.apply_async(args, kwargs, 58 | publisher=publisher, 59 | **options) 60 | receipt.finished(result) 61 | -------------------------------------------------------------------------------- /celery/loaders/default.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | celery.loaders.default 4 | ~~~~~~~~~~~~~~~~~~~~~~ 5 | 6 | The default loader used when no custom app has been initialized. 7 | 8 | """ 9 | from __future__ import absolute_import 10 | 11 | import os 12 | import warnings 13 | 14 | from celery.datastructures import DictAttribute 15 | from celery.exceptions import NotConfigured 16 | from celery.utils import strtobool 17 | 18 | from .base import BaseLoader 19 | 20 | __all__ = ['Loader', 'DEFAULT_CONFIG_MODULE'] 21 | 22 | DEFAULT_CONFIG_MODULE = 'celeryconfig' 23 | 24 | #: Warns if configuration file is missing if :envvar:`C_WNOCONF` is set. 25 | C_WNOCONF = strtobool(os.environ.get('C_WNOCONF', False)) 26 | 27 | 28 | class Loader(BaseLoader): 29 | """The loader used by the default app.""" 30 | 31 | def setup_settings(self, settingsdict): 32 | return DictAttribute(settingsdict) 33 | 34 | def read_configuration(self, fail_silently=True): 35 | """Read configuration from :file:`celeryconfig.py` and configure 36 | celery and Django so it can be used by regular Python.""" 37 | configname = os.environ.get('CELERY_CONFIG_MODULE', 38 | DEFAULT_CONFIG_MODULE) 39 | try: 40 | usercfg = self._import_config_module(configname) 41 | except ImportError: 42 | if not fail_silently: 43 | raise 44 | # billiard sets this if forked using execv 45 | if C_WNOCONF and not os.environ.get('FORKED_BY_MULTIPROCESSING'): 46 | warnings.warn(NotConfigured( 47 | 'No {module} module found! Please make sure it exists and ' 48 | 'is available to Python.'.format(module=configname))) 49 | return self.setup_settings({}) 50 | else: 51 | self.configured = True 52 | return self.setup_settings(usercfg) 53 | -------------------------------------------------------------------------------- /funtests/stress/stress/data.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import absolute_import 3 | 4 | import json 5 | 6 | from celery.utils.debug import humanbytes 7 | from celery.utils.imports import qualname 8 | 9 | type_registry = {} 10 | 11 | 12 | def json_reduce(obj, attrs): 13 | return {'py/obj': {'type': qualname(obj), 'attrs': attrs}} 14 | 15 | 16 | def jsonable(cls): 17 | type_registry[qualname(cls)] = cls.__from_json__ 18 | return cls 19 | 20 | 21 | @jsonable 22 | class Data(object): 23 | 24 | def __init__(self, label, data): 25 | self.label = label 26 | self.data = data 27 | 28 | def __str__(self): 29 | return ''.format( 30 | self.label, humanbytes(len(self.data)), 31 | ) 32 | 33 | def __to_json__(self): 34 | return json_reduce(self, {'label': self.label, 'data': self.data}) 35 | 36 | @classmethod 37 | def __from_json__(cls, label=None, data=None, **kwargs): 38 | return cls(label, data) 39 | 40 | def __reduce__(self): 41 | return Data, (self.label, self.data) 42 | __unicode__ = __repr__ = __str__ 43 | 44 | BIG = Data('BIG', 'x' * 2 ** 20 * 8) 45 | SMALL = Data('SMALL', 'e' * 1024) 46 | 47 | 48 | class JSONEncoder(json.JSONEncoder): 49 | 50 | def default(self, obj): 51 | try: 52 | return super(JSONEncoder, self).default(obj) 53 | except TypeError: 54 | reducer = getattr(obj, '__to_json__', None) 55 | if reducer: 56 | return reducer() 57 | raise 58 | 59 | 60 | def decode_hook(d): 61 | try: 62 | d = d['py/obj'] 63 | except KeyError: 64 | return d 65 | type_registry[d['type']](**d['attrs']) 66 | 67 | 68 | def install_json(): 69 | json._default_encoder = JSONEncoder() 70 | json._default_decoder.object_hook = decode_hook 71 | install_json() # ugh, ugly but it's a test suite after all 72 | -------------------------------------------------------------------------------- /docs/internals/reference/index.rst: -------------------------------------------------------------------------------- 1 | =========================== 2 | Internal Module Reference 3 | =========================== 4 | 5 | :Release: |version| 6 | :Date: |today| 7 | 8 | .. toctree:: 9 | :maxdepth: 1 10 | 11 | celery.worker.components 12 | celery.worker.loops 13 | celery.worker.heartbeat 14 | celery.worker.control 15 | celery.worker.pidbox 16 | celery.worker.autoreload 17 | celery.worker.autoscale 18 | celery.concurrency 19 | celery.concurrency.solo 20 | celery.concurrency.prefork 21 | celery.concurrency.eventlet 22 | celery.concurrency.gevent 23 | celery.concurrency.base 24 | celery.concurrency.threads 25 | celery.backends 26 | celery.backends.base 27 | celery.backends.rpc 28 | celery.backends.database 29 | celery.backends.cache 30 | celery.backends.amqp 31 | celery.backends.mongodb 32 | celery.backends.redis 33 | celery.backends.cassandra 34 | celery.backends.couchbase 35 | celery.app.trace 36 | celery.app.annotations 37 | celery.app.routes 38 | celery.datastructures 39 | celery.security.certificate 40 | celery.security.key 41 | celery.security.serialization 42 | celery.security.utils 43 | celery.events.snapshot 44 | celery.events.cursesmon 45 | celery.events.dumper 46 | celery.backends.database.models 47 | celery.backends.database.session 48 | celery.utils 49 | celery.utils.functional 50 | celery.utils.objects 51 | celery.utils.term 52 | celery.utils.timeutils 53 | celery.utils.iso8601 54 | celery.utils.compat 55 | celery.utils.serialization 56 | celery.utils.sysinfo 57 | celery.utils.threads 58 | celery.utils.timer2 59 | celery.utils.imports 60 | celery.utils.log 61 | celery.utils.text 62 | celery.utils.dispatch 63 | celery.utils.dispatch.signal 64 | celery.utils.dispatch.saferef 65 | celery.platforms 66 | celery._state 67 | -------------------------------------------------------------------------------- /celery/backends/database/session.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | celery.backends.database.session 4 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 5 | 6 | SQLAlchemy sessions. 7 | 8 | """ 9 | from __future__ import absolute_import 10 | 11 | from collections import defaultdict 12 | from multiprocessing.util import register_after_fork 13 | 14 | from sqlalchemy import create_engine 15 | from sqlalchemy.orm import sessionmaker 16 | from sqlalchemy.ext.declarative import declarative_base 17 | 18 | ResultModelBase = declarative_base() 19 | 20 | _SETUP = defaultdict(lambda: False) 21 | _ENGINES = {} 22 | _SESSIONS = {} 23 | 24 | __all__ = ['ResultSession', 'get_engine', 'create_session'] 25 | 26 | 27 | class _after_fork(object): 28 | registered = False 29 | 30 | def __call__(self): 31 | self.registered = False # child must reregister 32 | for engine in list(_ENGINES.values()): 33 | engine.dispose() 34 | _ENGINES.clear() 35 | _SESSIONS.clear() 36 | after_fork = _after_fork() 37 | 38 | 39 | def get_engine(dburi, **kwargs): 40 | try: 41 | return _ENGINES[dburi] 42 | except KeyError: 43 | engine = _ENGINES[dburi] = create_engine(dburi, **kwargs) 44 | after_fork.registered = True 45 | register_after_fork(after_fork, after_fork) 46 | return engine 47 | 48 | 49 | def create_session(dburi, short_lived_sessions=False, **kwargs): 50 | engine = get_engine(dburi, **kwargs) 51 | if short_lived_sessions or dburi not in _SESSIONS: 52 | _SESSIONS[dburi] = sessionmaker(bind=engine) 53 | return engine, _SESSIONS[dburi] 54 | 55 | 56 | def setup_results(engine): 57 | if not _SETUP['results']: 58 | ResultModelBase.metadata.create_all(engine) 59 | _SETUP['results'] = True 60 | 61 | 62 | def ResultSession(dburi, **kwargs): 63 | engine, session = create_session(dburi, **kwargs) 64 | setup_results(engine) 65 | return session() 66 | -------------------------------------------------------------------------------- /celery/backends/rpc.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | celery.backends.rpc 4 | ~~~~~~~~~~~~~~~~~~~ 5 | 6 | RPC-style result backend, using reply-to and one queue per client. 7 | 8 | """ 9 | from __future__ import absolute_import 10 | 11 | from kombu import Consumer, Exchange 12 | from kombu.common import maybe_declare 13 | from kombu.utils import cached_property 14 | 15 | from celery import current_task 16 | from celery.backends import amqp 17 | 18 | __all__ = ['RPCBackend'] 19 | 20 | 21 | class RPCBackend(amqp.AMQPBackend): 22 | persistent = False 23 | 24 | class Consumer(Consumer): 25 | auto_declare = False 26 | 27 | def _create_exchange(self, name, type='direct', delivery_mode=2): 28 | # uses direct to queue routing (anon exchange). 29 | return Exchange(None) 30 | 31 | def on_task_call(self, producer, task_id): 32 | maybe_declare(self.binding(producer.channel), retry=True) 33 | 34 | def _create_binding(self, task_id): 35 | return self.binding 36 | 37 | def _many_bindings(self, ids): 38 | return [self.binding] 39 | 40 | def rkey(self, task_id): 41 | return task_id 42 | 43 | def destination_for(self, task_id, request): 44 | # Request is a new argument for backends, so must still support 45 | # old code that rely on current_task 46 | try: 47 | request = request or current_task.request 48 | except AttributeError: 49 | raise RuntimeError( 50 | 'RPC backend missing task request for {0!r}'.format(task_id), 51 | ) 52 | return request.reply_to, request.correlation_id or task_id 53 | 54 | def on_reply_declare(self, task_id): 55 | pass 56 | 57 | @property 58 | def binding(self): 59 | return self.Queue(self.oid, self.exchange, self.oid, 60 | durable=False, auto_delete=False) 61 | 62 | @cached_property 63 | def oid(self): 64 | return self.app.oid 65 | -------------------------------------------------------------------------------- /celery/task/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | celery.task 4 | ~~~~~~~~~~~ 5 | 6 | This is the old task module, it should not be used anymore, 7 | import from the main 'celery' module instead. 8 | If you're looking for the decorator implementation then that's in 9 | ``celery.app.base.Celery.task``. 10 | 11 | """ 12 | from __future__ import absolute_import 13 | 14 | from celery._state import current_app, current_task as current 15 | from celery.five import MagicModule, recreate_module 16 | from celery.local import Proxy 17 | 18 | __all__ = [ 19 | 'BaseTask', 'Task', 'PeriodicTask', 'task', 'periodic_task', 20 | 'group', 'chord', 'subtask', 'TaskSet', 21 | ] 22 | 23 | 24 | STATICA_HACK = True 25 | globals()['kcah_acitats'[::-1].upper()] = False 26 | if STATICA_HACK: # pragma: no cover 27 | # This is never executed, but tricks static analyzers (PyDev, PyCharm, 28 | # pylint, etc.) into knowing the types of these symbols, and what 29 | # they contain. 30 | from celery.canvas import group, chord, subtask 31 | from .base import BaseTask, Task, PeriodicTask, task, periodic_task 32 | from .sets import TaskSet 33 | 34 | 35 | class module(MagicModule): 36 | 37 | def __call__(self, *args, **kwargs): 38 | return self.task(*args, **kwargs) 39 | 40 | 41 | old_module, new_module = recreate_module( # pragma: no cover 42 | __name__, 43 | by_module={ 44 | 'celery.task.base': ['BaseTask', 'Task', 'PeriodicTask', 45 | 'task', 'periodic_task'], 46 | 'celery.canvas': ['group', 'chord', 'subtask'], 47 | 'celery.task.sets': ['TaskSet'], 48 | }, 49 | base=module, 50 | __package__='celery.task', 51 | __file__=__file__, 52 | __path__=__path__, 53 | __doc__=__doc__, 54 | current=current, 55 | discard_all=Proxy(lambda: current_app.control.purge), 56 | backend_cleanup=Proxy( 57 | lambda: current_app.tasks['celery.backend_cleanup'] 58 | ), 59 | ) 60 | -------------------------------------------------------------------------------- /celery/concurrency/threads.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | celery.concurrency.threads 4 | ~~~~~~~~~~~~~~~~~~~~~~~~~~ 5 | 6 | Pool implementation using threads. 7 | 8 | """ 9 | from __future__ import absolute_import 10 | 11 | from celery.five import UserDict 12 | 13 | from .base import apply_target, BasePool 14 | 15 | __all__ = ['TaskPool'] 16 | 17 | 18 | class NullDict(UserDict): 19 | 20 | def __setitem__(self, key, value): 21 | pass 22 | 23 | 24 | class TaskPool(BasePool): 25 | 26 | def __init__(self, *args, **kwargs): 27 | try: 28 | import threadpool 29 | except ImportError: 30 | raise ImportError( 31 | 'The threaded pool requires the threadpool module.') 32 | self.WorkRequest = threadpool.WorkRequest 33 | self.ThreadPool = threadpool.ThreadPool 34 | super(TaskPool, self).__init__(*args, **kwargs) 35 | 36 | def on_start(self): 37 | self._pool = self.ThreadPool(self.limit) 38 | # threadpool stores all work requests until they are processed 39 | # we don't need this dict, and it occupies way too much memory. 40 | self._pool.workRequests = NullDict() 41 | self._quick_put = self._pool.putRequest 42 | self._quick_clear = self._pool._results_queue.queue.clear 43 | 44 | def on_stop(self): 45 | self._pool.dismissWorkers(self.limit, do_join=True) 46 | 47 | def on_apply(self, target, args=None, kwargs=None, callback=None, 48 | accept_callback=None, **_): 49 | req = self.WorkRequest(apply_target, (target, args, kwargs, callback, 50 | accept_callback)) 51 | self._quick_put(req) 52 | # threadpool also has callback support, 53 | # but for some reason the callback is not triggered 54 | # before you've collected the results. 55 | # Clear the results (if any), so it doesn't grow too large. 56 | self._quick_clear() 57 | return req 58 | -------------------------------------------------------------------------------- /celery/tests/worker/test_heartbeat.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from celery.worker.heartbeat import Heart 4 | from celery.tests.case import AppCase 5 | 6 | 7 | class MockDispatcher(object): 8 | heart = None 9 | next_iter = 0 10 | 11 | def __init__(self): 12 | self.sent = [] 13 | self.on_enabled = set() 14 | self.on_disabled = set() 15 | self.enabled = True 16 | 17 | def send(self, msg, **_fields): 18 | self.sent.append(msg) 19 | if self.heart: 20 | if self.next_iter > 10: 21 | self.heart._shutdown.set() 22 | self.next_iter += 1 23 | 24 | 25 | class MockDispatcherRaising(object): 26 | 27 | def send(self, msg): 28 | if msg == 'worker-offline': 29 | raise Exception('foo') 30 | 31 | 32 | class MockTimer(object): 33 | 34 | def call_repeatedly(self, secs, fun, args=(), kwargs={}): 35 | 36 | class entry(tuple): 37 | cancelled = False 38 | 39 | def cancel(self): 40 | self.cancelled = True 41 | 42 | return entry((secs, fun, args, kwargs)) 43 | 44 | def cancel(self, entry): 45 | entry.cancel() 46 | 47 | 48 | class test_Heart(AppCase): 49 | 50 | def test_start_stop(self): 51 | timer = MockTimer() 52 | eventer = MockDispatcher() 53 | h = Heart(timer, eventer, interval=1) 54 | h.start() 55 | self.assertTrue(h.tref) 56 | h.stop() 57 | self.assertIsNone(h.tref) 58 | h.stop() 59 | 60 | def test_start_when_disabled(self): 61 | timer = MockTimer() 62 | eventer = MockDispatcher() 63 | eventer.enabled = False 64 | h = Heart(timer, eventer) 65 | h.start() 66 | self.assertFalse(h.tref) 67 | 68 | def test_stop_when_disabled(self): 69 | timer = MockTimer() 70 | eventer = MockDispatcher() 71 | eventer.enabled = False 72 | h = Heart(timer, eventer) 73 | h.stop() 74 | -------------------------------------------------------------------------------- /celery/worker/heartbeat.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | celery.worker.heartbeat 4 | ~~~~~~~~~~~~~~~~~~~~~~~ 5 | 6 | This is the internal thread that sends heartbeat events 7 | at regular intervals. 8 | 9 | """ 10 | from __future__ import absolute_import 11 | 12 | from celery.five import values 13 | from celery.utils.sysinfo import load_average 14 | 15 | from .state import SOFTWARE_INFO, active_requests, total_count 16 | 17 | __all__ = ['Heart'] 18 | 19 | 20 | class Heart(object): 21 | """Timer sending heartbeats at regular intervals. 22 | 23 | :param timer: Timer instance. 24 | :param eventer: Event dispatcher used to send the event. 25 | :keyword interval: Time in seconds between heartbeats. 26 | Default is 30 seconds. 27 | 28 | """ 29 | 30 | def __init__(self, timer, eventer, interval=None): 31 | self.timer = timer 32 | self.eventer = eventer 33 | self.interval = float(interval or 2.0) 34 | self.tref = None 35 | 36 | # Make event dispatcher start/stop us when enabled/disabled. 37 | self.eventer.on_enabled.add(self.start) 38 | self.eventer.on_disabled.add(self.stop) 39 | 40 | def _send(self, event): 41 | return self.eventer.send(event, freq=self.interval, 42 | active=len(active_requests), 43 | processed=sum(values(total_count)), 44 | loadavg=load_average(), 45 | **SOFTWARE_INFO) 46 | 47 | def start(self): 48 | if self.eventer.enabled: 49 | self._send('worker-online') 50 | self.tref = self.timer.call_repeatedly( 51 | self.interval, self._send, ('worker-heartbeat', ), 52 | ) 53 | 54 | def stop(self): 55 | if self.tref is not None: 56 | self.timer.cancel(self.tref) 57 | self.tref = None 58 | if self.eventer.enabled: 59 | self._send('worker-offline') 60 | -------------------------------------------------------------------------------- /celery/app/registry.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | celery.app.registry 4 | ~~~~~~~~~~~~~~~~~~~ 5 | 6 | Registry of available tasks. 7 | 8 | """ 9 | from __future__ import absolute_import 10 | 11 | import inspect 12 | 13 | from importlib import import_module 14 | 15 | from celery._state import get_current_app 16 | from celery.exceptions import NotRegistered 17 | from celery.five import items 18 | 19 | __all__ = ['TaskRegistry'] 20 | 21 | 22 | class TaskRegistry(dict): 23 | NotRegistered = NotRegistered 24 | 25 | def __missing__(self, key): 26 | raise self.NotRegistered(key) 27 | 28 | def register(self, task): 29 | """Register a task in the task registry. 30 | 31 | The task will be automatically instantiated if not already an 32 | instance. 33 | 34 | """ 35 | self[task.name] = inspect.isclass(task) and task() or task 36 | 37 | def unregister(self, name): 38 | """Unregister task by name. 39 | 40 | :param name: name of the task to unregister, or a 41 | :class:`celery.task.base.Task` with a valid `name` attribute. 42 | 43 | :raises celery.exceptions.NotRegistered: if the task has not 44 | been registered. 45 | 46 | """ 47 | try: 48 | self.pop(getattr(name, 'name', name)) 49 | except KeyError: 50 | raise self.NotRegistered(name) 51 | 52 | # -- these methods are irrelevant now and will be removed in 4.0 53 | def regular(self): 54 | return self.filter_types('regular') 55 | 56 | def periodic(self): 57 | return self.filter_types('periodic') 58 | 59 | def filter_types(self, type): 60 | return dict((name, task) for name, task in items(self) 61 | if getattr(task, 'type', 'regular') == type) 62 | 63 | 64 | def _unpickle_task(name): 65 | return get_current_app().tasks[name] 66 | 67 | 68 | def _unpickle_task_v2(name, module=None): 69 | if module: 70 | import_module(module) 71 | return get_current_app().tasks[name] 72 | -------------------------------------------------------------------------------- /celery/tests/app/test_defaults.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import sys 4 | 5 | from importlib import import_module 6 | from mock import Mock, patch 7 | 8 | from celery.app.defaults import NAMESPACES 9 | 10 | from celery.tests.case import AppCase, pypy_version, sys_platform 11 | 12 | 13 | class test_defaults(AppCase): 14 | 15 | def setup(self): 16 | self._prev = sys.modules.pop('celery.app.defaults', None) 17 | 18 | def teardown(self): 19 | if self._prev: 20 | sys.modules['celery.app.defaults'] = self._prev 21 | 22 | def test_option_repr(self): 23 | self.assertTrue(repr(NAMESPACES['BROKER']['URL'])) 24 | 25 | def test_any(self): 26 | val = object() 27 | self.assertIs(self.defaults.Option.typemap['any'](val), val) 28 | 29 | def test_default_pool_pypy_14(self): 30 | with sys_platform('darwin'): 31 | with pypy_version((1, 4, 0)): 32 | self.assertEqual(self.defaults.DEFAULT_POOL, 'solo') 33 | 34 | def test_default_pool_pypy_15(self): 35 | with sys_platform('darwin'): 36 | with pypy_version((1, 5, 0)): 37 | self.assertEqual(self.defaults.DEFAULT_POOL, 'prefork') 38 | 39 | def test_deprecated(self): 40 | source = Mock() 41 | source.CELERYD_LOG_LEVEL = 2 42 | with patch('celery.utils.warn_deprecated') as warn: 43 | self.defaults.find_deprecated_settings(source) 44 | self.assertTrue(warn.called) 45 | 46 | def test_default_pool_jython(self): 47 | with sys_platform('java 1.6.51'): 48 | self.assertEqual(self.defaults.DEFAULT_POOL, 'threads') 49 | 50 | def test_find(self): 51 | find = self.defaults.find 52 | 53 | self.assertEqual(find('server_email')[2].default, 'celery@localhost') 54 | self.assertEqual(find('default_queue')[2].default, 'celery') 55 | self.assertEqual(find('celery_default_exchange')[2], 'celery') 56 | 57 | @property 58 | def defaults(self): 59 | return import_module('celery.app.defaults') 60 | -------------------------------------------------------------------------------- /celery/tests/concurrency/test_threads.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from mock import Mock 4 | 5 | from celery.concurrency.threads import NullDict, TaskPool, apply_target 6 | 7 | from celery.tests.case import AppCase, Case, mask_modules, mock_module 8 | 9 | 10 | class test_NullDict(Case): 11 | 12 | def test_setitem(self): 13 | x = NullDict() 14 | x['foo'] = 1 15 | with self.assertRaises(KeyError): 16 | x['foo'] 17 | 18 | 19 | class test_TaskPool(AppCase): 20 | 21 | def test_without_threadpool(self): 22 | 23 | with mask_modules('threadpool'): 24 | with self.assertRaises(ImportError): 25 | TaskPool() 26 | 27 | def test_with_threadpool(self): 28 | with mock_module('threadpool'): 29 | x = TaskPool() 30 | self.assertTrue(x.ThreadPool) 31 | self.assertTrue(x.WorkRequest) 32 | 33 | def test_on_start(self): 34 | with mock_module('threadpool'): 35 | x = TaskPool() 36 | x.on_start() 37 | self.assertTrue(x._pool) 38 | self.assertIsInstance(x._pool.workRequests, NullDict) 39 | 40 | def test_on_stop(self): 41 | with mock_module('threadpool'): 42 | x = TaskPool() 43 | x.on_start() 44 | x.on_stop() 45 | x._pool.dismissWorkers.assert_called_with(x.limit, do_join=True) 46 | 47 | def test_on_apply(self): 48 | with mock_module('threadpool'): 49 | x = TaskPool() 50 | x.on_start() 51 | callback = Mock() 52 | accept_callback = Mock() 53 | target = Mock() 54 | req = x.on_apply(target, args=(1, 2), kwargs={'a': 10}, 55 | callback=callback, 56 | accept_callback=accept_callback) 57 | x.WorkRequest.assert_called_with( 58 | apply_target, 59 | (target, (1, 2), {'a': 10}, callback, accept_callback), 60 | ) 61 | x._pool.putRequest.assert_called_with(req) 62 | x._pool._results_queue.queue.clear.assert_called_with() 63 | -------------------------------------------------------------------------------- /docs/getting-started/brokers/ironmq.rst: -------------------------------------------------------------------------------- 1 | .. _broker-ironmq: 2 | 3 | ================== 4 | Using IronMQ 5 | ================== 6 | 7 | .. _broker-ironmq-installation: 8 | 9 | Installation 10 | ============ 11 | 12 | For IronMQ support, you'll need the [iron_celery](http://github.com/iron-io/iron_celery) library: 13 | 14 | .. code-block:: bash 15 | 16 | $ pip install iron_celery 17 | 18 | As well as an [Iron.io account](http://www.iron.io). Sign up for free at [iron.io](http://www.iron.io). 19 | 20 | .. _broker-ironmq-configuration: 21 | 22 | Configuration 23 | ============= 24 | 25 | First, you'll need to import the iron_celery library right after you import Celery, for example:: 26 | 27 | from celery import Celery 28 | import iron_celery 29 | 30 | app = Celery('mytasks', broker='ironmq://', backend='ironcache://') 31 | 32 | You have to specify IronMQ in the broker URL:: 33 | 34 | BROKER_URL = 'ironmq://ABCDEFGHIJKLMNOPQRST:ZYXK7NiynGlTogH8Nj+P9nlE73sq3@' 35 | 36 | where the URL format is:: 37 | 38 | ironmq://project_id:token@ 39 | 40 | you must *remember to include the "@" at the end*. 41 | 42 | The login credentials can also be set using the environment variables 43 | :envvar:`IRON_TOKEN` and :envvar:`IRON_PROJECT_ID`, which are set automatically if you use the IronMQ Heroku add-on. 44 | And in this case the broker url may only be:: 45 | 46 | ironmq:// 47 | 48 | Clouds 49 | ------ 50 | 51 | The default cloud/region is ``AWS us-east-1``. You can choose the IronMQ Rackspace (ORD) cloud by changing the URL to:: 52 | 53 | ironmq://project_id:token@mq-rackspace-ord.iron.io 54 | 55 | Results 56 | ======= 57 | 58 | You can store results in IronCache with the same Iron.io credentials, just set the results URL with the same syntax 59 | as the broker URL, but changing the start to ``ironcache``:: 60 | 61 | ironcache:://project_id:token@ 62 | 63 | This will default to a cache named "Celery", if you want to change that:: 64 | 65 | ironcache:://project_id:token@/awesomecache 66 | 67 | More Information 68 | ================ 69 | 70 | You can find more information in the [iron_celery README](http://github.com/iron-io/iron_celery). 71 | -------------------------------------------------------------------------------- /celery/tests/backends/test_backends.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from mock import patch 4 | 5 | from celery import backends 6 | from celery.backends.amqp import AMQPBackend 7 | from celery.backends.cache import CacheBackend 8 | from celery.tests.case import AppCase, depends_on_current_app 9 | 10 | 11 | class test_backends(AppCase): 12 | 13 | def test_get_backend_aliases(self): 14 | expects = [('amqp://', AMQPBackend), 15 | ('cache+memory://', CacheBackend)] 16 | 17 | for url, expect_cls in expects: 18 | backend, url = backends.get_backend_by_url(url, self.app.loader) 19 | self.assertIsInstance( 20 | backend(app=self.app, url=url), 21 | expect_cls, 22 | ) 23 | 24 | def test_get_backend_cache(self): 25 | backends.get_backend_cls.clear() 26 | hits = backends.get_backend_cls.hits 27 | misses = backends.get_backend_cls.misses 28 | self.assertTrue(backends.get_backend_cls('amqp', self.app.loader)) 29 | self.assertEqual(backends.get_backend_cls.misses, misses + 1) 30 | self.assertTrue(backends.get_backend_cls('amqp', self.app.loader)) 31 | self.assertEqual(backends.get_backend_cls.hits, hits + 1) 32 | 33 | def test_unknown_backend(self): 34 | with self.assertRaises(ImportError): 35 | backends.get_backend_cls('fasodaopjeqijwqe', self.app.loader) 36 | 37 | @depends_on_current_app 38 | def test_default_backend(self): 39 | self.assertEqual(backends.default_backend, self.app.backend) 40 | 41 | def test_backend_by_url(self, url='redis://localhost/1'): 42 | from celery.backends.redis import RedisBackend 43 | backend, url_ = backends.get_backend_by_url(url, self.app.loader) 44 | self.assertIs(backend, RedisBackend) 45 | self.assertEqual(url_, url) 46 | 47 | def test_sym_raises_ValuError(self): 48 | with patch('celery.backends.symbol_by_name') as sbn: 49 | sbn.side_effect = ValueError() 50 | with self.assertRaises(ValueError): 51 | backends.get_backend_cls('xxx.xxx:foo', self.app.loader) 52 | -------------------------------------------------------------------------------- /funtests/stress/stress/fbi.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, print_function 2 | 3 | import socket 4 | import sys 5 | 6 | from contextlib import contextmanager 7 | 8 | from celery import states 9 | 10 | 11 | class FBI(object): 12 | 13 | def __init__(self, app): 14 | self.app = app 15 | self.receiver = None 16 | self.state = self.app.events.State() 17 | self.connection = None 18 | self.enabled = False 19 | 20 | def enable(self, enabled): 21 | self.enabled = enabled 22 | 23 | @contextmanager 24 | def investigation(self): 25 | if self.enabled: 26 | with self.app.connection() as conn: 27 | receiver = self.app.events.Receiver( 28 | conn, handlers={'*': self.state.event}, 29 | ) 30 | with receiver.consumer_context() as (conn, _, _): 31 | self.connection = conn 32 | try: 33 | yield self 34 | finally: 35 | self.ffwd() 36 | else: 37 | yield 38 | 39 | def ffwd(self): 40 | while 1: 41 | try: 42 | self.connection.drain_events(timeout=1) 43 | except socket.error: 44 | break 45 | 46 | def state_of(self, tid): 47 | try: 48 | task = self.state.tasks[tid] 49 | except KeyError: 50 | return 'No events for {0}'.format(tid) 51 | 52 | if task.state in states.READY_STATES: 53 | return 'Task {0.uuid} completed with {0.state}'.format(task) 54 | elif task.state in states.UNREADY_STATES: 55 | return 'Task {0.uuid} waiting in {0.state} state'.format(task) 56 | else: 57 | return 'Task {0.uuid} in other state {0.state}'.format(task) 58 | 59 | def query(self, ids): 60 | return self.app.control.inspect().query_task(id) 61 | 62 | def diag(self, ids, file=sys.stderr): 63 | if self.enabled: 64 | self.ffwd() 65 | for tid in ids: 66 | print(self.state_of(tid), file=file) 67 | #print(self.query(ids), file=file) 68 | -------------------------------------------------------------------------------- /celery/security/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | celery.security 4 | ~~~~~~~~~~~~~~~ 5 | 6 | Module implementing the signing message serializer. 7 | 8 | """ 9 | from __future__ import absolute_import 10 | 11 | from kombu.serialization import ( 12 | registry, disable_insecure_serializers as _disable_insecure_serializers, 13 | ) 14 | 15 | from celery.exceptions import ImproperlyConfigured 16 | 17 | from .serialization import register_auth 18 | 19 | SSL_NOT_INSTALLED = """\ 20 | You need to install the pyOpenSSL library to use the auth serializer. 21 | Please install by: 22 | 23 | $ pip install pyOpenSSL 24 | """ 25 | 26 | SETTING_MISSING = """\ 27 | Sorry, but you have to configure the 28 | * CELERY_SECURITY_KEY 29 | * CELERY_SECURITY_CERTIFICATE, and the 30 | * CELERY_SECURITY_CERT_STORE 31 | configuration settings to use the auth serializer. 32 | 33 | Please see the configuration reference for more information. 34 | """ 35 | 36 | __all__ = ['setup_security'] 37 | 38 | 39 | def setup_security(allowed_serializers=None, key=None, cert=None, store=None, 40 | digest='sha1', serializer='json', app=None): 41 | """See :meth:`@Celery.setup_security`.""" 42 | if app is None: 43 | from celery import current_app 44 | app = current_app._get_current_object() 45 | 46 | _disable_insecure_serializers(allowed_serializers) 47 | 48 | conf = app.conf 49 | if conf.CELERY_TASK_SERIALIZER != 'auth': 50 | return 51 | 52 | try: 53 | from OpenSSL import crypto # noqa 54 | except ImportError: 55 | raise ImproperlyConfigured(SSL_NOT_INSTALLED) 56 | 57 | key = key or conf.CELERY_SECURITY_KEY 58 | cert = cert or conf.CELERY_SECURITY_CERTIFICATE 59 | store = store or conf.CELERY_SECURITY_CERT_STORE 60 | 61 | if not (key and cert and store): 62 | raise ImproperlyConfigured(SETTING_MISSING) 63 | 64 | with open(key) as kf: 65 | with open(cert) as cf: 66 | register_auth(kf.read(), cf.read(), store, digest, serializer) 67 | registry._set_default_serializer('auth') 68 | 69 | 70 | def disable_untrusted_serializers(whitelist=None): 71 | _disable_insecure_serializers(allowed=whitelist) 72 | -------------------------------------------------------------------------------- /docs/django/unit-testing.rst: -------------------------------------------------------------------------------- 1 | ================ 2 | Unit Testing 3 | ================ 4 | 5 | Testing with Django 6 | ------------------- 7 | 8 | The first problem you'll run in to when trying to write a test that runs a 9 | task is that Django's test runner doesn't use the same database as your celery 10 | daemon is using. If you're using the database backend, this means that your 11 | tombstones won't show up in your test database and you won't be able to 12 | get the return value or check the status of your tasks. 13 | 14 | There are two ways to get around this. You can either take advantage of 15 | ``CELERY_ALWAYS_EAGER = True`` to skip the daemon, or you can avoid testing 16 | anything that needs to check the status or result of a task. 17 | 18 | Using a custom test runner to test with celery 19 | ---------------------------------------------- 20 | 21 | If you're going the ``CELERY_ALWAYS_EAGER`` route, which is probably better than 22 | just never testing some parts of your app, a custom Django test runner does the 23 | trick. Celery provides a simple test runner, but it's easy enough to roll your 24 | own if you have other things that need to be done. 25 | http://docs.djangoproject.com/en/dev/topics/testing/#defining-a-test-runner 26 | 27 | For this example, we'll use the ``djcelery.contrib.test_runner`` to test the 28 | ``add`` task from the :ref:`guide-tasks` examples in the Celery 29 | documentation. 30 | 31 | To enable the test runner, set the following settings: 32 | 33 | .. code-block:: python 34 | 35 | TEST_RUNNER = 'djcelery.contrib.test_runner.CeleryTestSuiteRunner' 36 | 37 | Then we can put the tests in a ``tests.py`` somewhere: 38 | 39 | .. code-block:: python 40 | 41 | from django.test import TestCase 42 | from myapp.tasks import add 43 | 44 | class AddTestCase(TestCase): 45 | 46 | def testNoError(self): 47 | """Test that the ``add`` task runs with no errors, 48 | and returns the correct result.""" 49 | result = add.delay(8, 8) 50 | 51 | self.assertEqual(result.get(), 16) 52 | self.assertTrue(result.successful()) 53 | 54 | 55 | This test assumes that you put your example ``add`` task in ``maypp.tasks`` 56 | so adjust the import for wherever you put the class. 57 | -------------------------------------------------------------------------------- /extra/release/sphinx-to-rst.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import re 4 | import sys 5 | 6 | from collections import Callable 7 | 8 | dirname = "" 9 | 10 | RE_CODE_BLOCK = re.compile(r'.. code-block:: (.+?)\s*$') 11 | RE_INCLUDE = re.compile(r'.. include:: (.+?)\s*$') 12 | RE_REFERENCE = re.compile(r':(.+?):`(.+?)`') 13 | 14 | 15 | def include_file(lines, pos, match): 16 | global dirname 17 | orig_filename = match.groups()[0] 18 | filename = os.path.join(dirname, orig_filename) 19 | fh = open(filename) 20 | try: 21 | old_dirname = dirname 22 | dirname = os.path.dirname(orig_filename) 23 | try: 24 | lines[pos] = sphinx_to_rst(fh) 25 | finally: 26 | dirname = old_dirname 27 | finally: 28 | fh.close() 29 | 30 | 31 | def replace_code_block(lines, pos, match): 32 | lines[pos] = "" 33 | curpos = pos - 1 34 | # Find the first previous line with text to append "::" to it. 35 | while True: 36 | prev_line = lines[curpos] 37 | if not prev_line.isspace(): 38 | prev_line_with_text = curpos 39 | break 40 | curpos -= 1 41 | 42 | if lines[prev_line_with_text].endswith(":"): 43 | lines[prev_line_with_text] += ":" 44 | else: 45 | lines[prev_line_with_text] += "::" 46 | 47 | TO_RST_MAP = {RE_CODE_BLOCK: replace_code_block, 48 | RE_REFERENCE: r'``\2``', 49 | RE_INCLUDE: include_file} 50 | 51 | 52 | def _process(lines): 53 | lines = list(lines) # non-destructive 54 | for i, line in enumerate(lines): 55 | for regex, alt in TO_RST_MAP.items(): 56 | if isinstance(alt, Callable): 57 | match = regex.match(line) 58 | if match: 59 | alt(lines, i, match) 60 | line = lines[i] 61 | else: 62 | lines[i] = regex.sub(alt, line) 63 | return lines 64 | 65 | 66 | def sphinx_to_rst(fh): 67 | return "".join(_process(fh)) 68 | 69 | 70 | if __name__ == "__main__": 71 | global dirname 72 | dirname = os.path.dirname(sys.argv[1]) 73 | fh = open(sys.argv[1]) 74 | try: 75 | print(sphinx_to_rst(fh)) 76 | finally: 77 | fh.close() 78 | -------------------------------------------------------------------------------- /examples/eventlet/webcrawler.py: -------------------------------------------------------------------------------- 1 | """Recursive webcrawler example. 2 | 3 | For asynchronous DNS lookups install the `dnspython` package: 4 | 5 | $ pip install dnspython 6 | 7 | Requires the `pybloom` module for the bloom filter which is used 8 | to ensure a lower chance of recrawling an URL previously seen. 9 | 10 | Since the bloom filter is not shared, but only passed as an argument 11 | to each subtask, it would be much better to have this as a centralized 12 | service. Redis sets could also be a practical solution. 13 | 14 | A BloomFilter with a capacity of 100_000 members and an error rate 15 | of 0.001 is 2.8MB pickled, but if compressed with zlib it only takes 16 | up 2.9kB(!). 17 | 18 | We don't have to do compression manually, just set the tasks compression 19 | to "zlib", and the serializer to "pickle". 20 | 21 | 22 | """ 23 | 24 | 25 | import re 26 | 27 | try: 28 | from urllib.parse import urlsplit 29 | except ImportError: 30 | from urlparse import urlsplit # noqa 31 | 32 | import requests 33 | 34 | from celery import task, group 35 | from eventlet import Timeout 36 | 37 | from pybloom import BloomFilter 38 | 39 | # http://daringfireball.net/2009/11/liberal_regex_for_matching_urls 40 | url_regex = re.compile( 41 | r'\b(([\w-]+://?|www[.])[^\s()<>]+(?:\([\w\d]+\)|([^[:punct:]\s]|/)))') 42 | 43 | 44 | def domain(url): 45 | """Return the domain part of an URL.""" 46 | return urlsplit(url)[1].split(':')[0] 47 | 48 | 49 | @task(ignore_result=True, serializer='pickle', compression='zlib') 50 | def crawl(url, seen=None): 51 | print('crawling: {0}'.format(url)) 52 | if not seen: 53 | seen = BloomFilter(capacity=50000, error_rate=0.0001) 54 | 55 | with Timeout(5, False): 56 | try: 57 | response = requests.get(url) 58 | except Exception: 59 | return 60 | 61 | location = domain(url) 62 | wanted_urls = [] 63 | for url_match in url_regex.finditer(response.text): 64 | url = url_match.group(0) 65 | # To not destroy the internet, we only fetch URLs on the same domain. 66 | if url not in seen and location in domain(url): 67 | wanted_urls.append(url) 68 | seen.add(url) 69 | 70 | subtasks = group(crawl.s(url, seen) for url in wanted_urls) 71 | subtasks() 72 | -------------------------------------------------------------------------------- /celery/backends/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | celery.backends 4 | ~~~~~~~~~~~~~~~ 5 | 6 | Backend abstract factory (...did I just say that?) and alias definitions. 7 | 8 | """ 9 | from __future__ import absolute_import 10 | 11 | import sys 12 | 13 | from kombu.utils.url import _parse_url 14 | 15 | from celery.local import Proxy 16 | from celery._state import current_app 17 | from celery.five import reraise 18 | from celery.utils.imports import symbol_by_name 19 | from celery.utils.functional import memoize 20 | 21 | __all__ = ['get_backend_cls', 'get_backend_by_url'] 22 | 23 | UNKNOWN_BACKEND = """\ 24 | Unknown result backend: {0!r}. Did you spell that correctly? ({1!r})\ 25 | """ 26 | 27 | BACKEND_ALIASES = { 28 | 'amqp': 'celery.backends.amqp:AMQPBackend', 29 | 'rpc': 'celery.backends.rpc.RPCBackend', 30 | 'cache': 'celery.backends.cache:CacheBackend', 31 | 'redis': 'celery.backends.redis:RedisBackend', 32 | 'mongodb': 'celery.backends.mongodb:MongoBackend', 33 | 'db': 'celery.backends.database:DatabaseBackend', 34 | 'database': 'celery.backends.database:DatabaseBackend', 35 | 'cassandra': 'celery.backends.cassandra:CassandraBackend', 36 | 'couchbase': 'celery.backends.couchbase:CouchBaseBackend', 37 | 'disabled': 'celery.backends.base:DisabledBackend', 38 | } 39 | 40 | #: deprecated alias to ``current_app.backend``. 41 | default_backend = Proxy(lambda: current_app.backend) 42 | 43 | 44 | @memoize(100) 45 | def get_backend_cls(backend=None, loader=None): 46 | """Get backend class by name/alias""" 47 | backend = backend or 'disabled' 48 | loader = loader or current_app.loader 49 | aliases = dict(BACKEND_ALIASES, **loader.override_backends) 50 | try: 51 | return symbol_by_name(backend, aliases) 52 | except ValueError as exc: 53 | reraise(ValueError, ValueError(UNKNOWN_BACKEND.format( 54 | backend, exc)), sys.exc_info()[2]) 55 | 56 | 57 | def get_backend_by_url(backend=None, loader=None): 58 | url = None 59 | if backend and '://' in backend: 60 | url = backend 61 | if '+' in url[:url.index('://')]: 62 | backend, url = url.split('+', 1) 63 | else: 64 | backend, _, _, _, _, _, _ = _parse_url(url) 65 | return get_backend_cls(backend, loader), url 66 | -------------------------------------------------------------------------------- /docs/tutorials/task-cookbook.rst: -------------------------------------------------------------------------------- 1 | .. _cookbook-tasks: 2 | 3 | ================ 4 | Task Cookbook 5 | ================ 6 | 7 | .. contents:: 8 | :local: 9 | 10 | .. _cookbook-task-serial: 11 | 12 | Ensuring a task is only executed one at a time 13 | ============================================== 14 | 15 | You can accomplish this by using a lock. 16 | 17 | In this example we'll be using the cache framework to set a lock that is 18 | accessible for all workers. 19 | 20 | It's part of an imaginary RSS feed importer called `djangofeeds`. 21 | The task takes a feed URL as a single argument, and imports that feed into 22 | a Django model called `Feed`. We ensure that it's not possible for two or 23 | more workers to import the same feed at the same time by setting a cache key 24 | consisting of the MD5 checksum of the feed URL. 25 | 26 | The cache key expires after some time in case something unexpected happens 27 | (you never know, right?) 28 | 29 | .. code-block:: python 30 | 31 | from celery import task 32 | from celery.utils.log import get_task_logger 33 | from django.core.cache import cache 34 | from django.utils.hashcompat import md5_constructor as md5 35 | from djangofeeds.models import Feed 36 | 37 | logger = get_task_logger(__name__) 38 | 39 | LOCK_EXPIRE = 60 * 5 # Lock expires in 5 minutes 40 | 41 | @task 42 | def import_feed(feed_url): 43 | # The cache key consists of the task name and the MD5 digest 44 | # of the feed URL. 45 | feed_url_digest = md5(feed_url).hexdigest() 46 | lock_id = '{0}-lock-{1}'.format(self.name, feed_url_hexdigest) 47 | 48 | # cache.add fails if if the key already exists 49 | acquire_lock = lambda: cache.add(lock_id, 'true', LOCK_EXPIRE) 50 | # memcache delete is very slow, but we have to use it to take 51 | # advantage of using add() for atomic locking 52 | release_lock = lambda: cache.delete(lock_id) 53 | 54 | logger.debug('Importing feed: %s', feed_url) 55 | if acquire_lock(): 56 | try: 57 | feed = Feed.objects.import_feed(feed_url) 58 | finally: 59 | release_lock() 60 | return feed.url 61 | 62 | logger.debug( 63 | 'Feed %s is already being imported by another worker', feed_url) 64 | -------------------------------------------------------------------------------- /docs/getting-started/brokers/sqlalchemy.rst: -------------------------------------------------------------------------------- 1 | .. _broker-sqlalchemy: 2 | 3 | ================== 4 | Using SQLAlchemy 5 | ================== 6 | 7 | .. admonition:: Experimental Status 8 | 9 | The SQS transport is unstable in many areas and there are 10 | several issues open. Sadly we don't have the resources or funds 11 | required to improve the situation, so we're looking for contributors 12 | and partners willing to help. 13 | 14 | .. _broker-sqlalchemy-installation: 15 | 16 | Installation 17 | ============ 18 | 19 | .. _broker-sqlalchemy-configuration: 20 | 21 | Configuration 22 | ============= 23 | 24 | Celery needs to know the location of your database, which should be the usual 25 | SQLAlchemy connection string, but with 'sqla+' prepended to it:: 26 | 27 | BROKER_URL = 'sqla+sqlite:///celerydb.sqlite' 28 | 29 | This transport uses only the :setting:`BROKER_URL` setting, which have to be 30 | an SQLAlchemy database URI. 31 | 32 | 33 | Please see `SQLAlchemy: Supported Databases`_ for a table of supported databases. 34 | 35 | Here's a list of examples using a selection of other `SQLAlchemy Connection String`_'s: 36 | 37 | .. code-block:: python 38 | 39 | # sqlite (filename) 40 | BROKER_URL = 'sqla+sqlite:///celerydb.sqlite' 41 | 42 | # mysql 43 | BROKER_URL = 'sqla+mysql://scott:tiger@localhost/foo' 44 | 45 | # postgresql 46 | BROKER_URL = 'sqla+postgresql://scott:tiger@localhost/mydatabase' 47 | 48 | # oracle 49 | BROKER_URL = 'sqla+oracle://scott:tiger@127.0.0.1:1521/sidname' 50 | 51 | .. _`SQLAlchemy: Supported Databases`: 52 | http://www.sqlalchemy.org/docs/core/engines.html#supported-databases 53 | 54 | .. _`SQLAlchemy Connection String`: 55 | http://www.sqlalchemy.org/docs/core/engines.html#database-urls 56 | 57 | .. _sqlalchemy-results-configuration: 58 | 59 | Results 60 | ------- 61 | 62 | To store results in the database as well, you should configure the result 63 | backend. See :ref:`conf-database-result-backend`. 64 | 65 | .. _broker-sqlalchemy-limitations: 66 | 67 | Limitations 68 | =========== 69 | 70 | The SQLAlchemy database transport does not currently support: 71 | 72 | * Remote control commands (:program:`celery events` command, broadcast) 73 | * Events, including the Django Admin monitor. 74 | * Using more than a few workers (can lead to messages being executed 75 | multiple times). 76 | -------------------------------------------------------------------------------- /celery/utils/text.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | celery.utils.text 4 | ~~~~~~~~~~~~~~~~~ 5 | 6 | Text formatting utilities 7 | 8 | """ 9 | from __future__ import absolute_import 10 | 11 | from textwrap import fill 12 | 13 | from pprint import pformat 14 | 15 | __all__ = ['dedent_initial', 'dedent', 'fill_paragraphs', 'join', 16 | 'ensure_2lines', 'abbr', 'abbrtask', 'indent', 'truncate', 17 | 'pluralize', 'pretty'] 18 | 19 | 20 | def dedent_initial(s, n=4): 21 | return s[n:] if s[:n] == ' ' * n else s 22 | 23 | 24 | def dedent(s, n=4, sep='\n'): 25 | return sep.join(dedent_initial(l) for l in s.splitlines()) 26 | 27 | 28 | def fill_paragraphs(s, width, sep='\n'): 29 | return sep.join(fill(p, width) for p in s.split(sep)) 30 | 31 | 32 | def join(l, sep='\n'): 33 | return sep.join(v for v in l if v) 34 | 35 | 36 | def ensure_2lines(s, sep='\n'): 37 | if len(s.splitlines()) <= 2: 38 | return s + sep 39 | return s 40 | 41 | 42 | def abbr(S, max, ellipsis='...'): 43 | if S is None: 44 | return '???' 45 | if len(S) > max: 46 | return ellipsis and (S[:max - len(ellipsis)] + ellipsis) or S[:max] 47 | return S 48 | 49 | 50 | def abbrtask(S, max): 51 | if S is None: 52 | return '???' 53 | if len(S) > max: 54 | module, _, cls = S.rpartition('.') 55 | module = abbr(module, max - len(cls) - 3, False) 56 | return module + '[.]' + cls 57 | return S 58 | 59 | 60 | def indent(t, indent=0, sep='\n'): 61 | """Indent text.""" 62 | return sep.join(' ' * indent + p for p in t.split(sep)) 63 | 64 | 65 | def truncate(text, maxlen=128, suffix='...'): 66 | """Truncates text to a maximum number of characters.""" 67 | if len(text) >= maxlen: 68 | return text[:maxlen].rsplit(' ', 1)[0] + suffix 69 | return text 70 | 71 | 72 | def pluralize(n, text, suffix='s'): 73 | if n > 1: 74 | return text + suffix 75 | return text 76 | 77 | 78 | def pretty(value, width=80, nl_width=80, sep='\n', **kw): 79 | if isinstance(value, dict): 80 | return '{{{0} {1}'.format(sep, pformat(value, 4, nl_width)[1:]) 81 | elif isinstance(value, tuple): 82 | return '{0}{1}{2}'.format( 83 | sep, ' ' * 4, pformat(value, width=nl_width, **kw), 84 | ) 85 | else: 86 | return pformat(value, width=width, **kw) 87 | --------------------------------------------------------------------------------