├── docs ├── _static │ └── .keep ├── changelog.rst ├── images │ ├── dashboard.png │ ├── favicon.ico │ ├── monitor.png │ ├── celery_128.png │ ├── celery_512.png │ ├── celeryevshotsm.jpg │ ├── result_graph.png │ └── worker_graph_full.png ├── reference │ ├── celery.contrib.sphinx.rst │ ├── celery.states.rst │ ├── celery.contrib.rdb.rst │ ├── celery.events.rst │ ├── celery.result.rst │ ├── celery.security.rst │ ├── celery.app.log.rst │ ├── celery.beat.rst │ ├── celery.bin.base.rst │ ├── celery.app.task.rst │ ├── celery.app.utils.rst │ ├── celery.exceptions.rst │ ├── celery.worker.rst │ ├── celery.app.registry.rst │ ├── celery.loaders.app.rst │ ├── celery.loaders.rst │ ├── celery.apps.worker.rst │ ├── celery.bootsteps.rst │ ├── celery.contrib.migrate.rst │ ├── celery.worker.state.rst │ ├── celery.bin.celery.rst │ ├── celery.bin.worker.rst │ ├── celery.bin.multi.rst │ ├── celery.worker.request.rst │ ├── celery.worker.strategy.rst │ ├── celery.apps.beat.rst │ ├── celery.bin.beat.rst │ ├── celery.loaders.base.rst │ ├── celery.signals.rst │ ├── celery.bin.graph.rst │ ├── celery.loaders.default.rst │ ├── celery.schedules.rst │ ├── celery.app.control.rst │ ├── celery.bin.events.rst │ ├── celery.app.builtins.rst │ ├── celery.bin.amqp.rst │ ├── celery.bin.logtool.rst │ ├── celery.worker.consumer.rst │ ├── celery.app.defaults.rst │ ├── celery.events.state.rst │ ├── celery.contrib.abortable.rst │ ├── celery.worker.consumer.agent.rst │ ├── celery.worker.consumer.events.rst │ ├── celery.worker.consumer.gossip.rst │ ├── celery.worker.consumer.heart.rst │ ├── celery.worker.consumer.mingle.rst │ ├── celery.worker.consumer.tasks.rst │ ├── celery.worker.consumer.control.rst │ ├── celery.worker.consumer.consumer.rst │ ├── celery.app.rst │ ├── celery.worker.consumer.connection.rst │ ├── celery.utils.debug.rst │ ├── celery.app.amqp.rst │ └── index.rst ├── getting-started │ ├── resources.rst │ ├── index.rst │ └── brokers │ │ └── index.rst ├── django │ └── index.rst ├── internals │ ├── reference │ │ ├── celery.utils.collections.rst │ │ ├── celery.utils.rst │ │ ├── celery.backends.rst │ │ ├── celery._state.rst │ │ ├── celery.app.routes.rst │ │ ├── celery.utils.timer2.rst │ │ ├── celery.concurrency.rst │ │ ├── celery.platforms.rst │ │ ├── celery.app.trace.rst │ │ ├── celery.worker.loops.rst │ │ ├── celery.backends.rpc.rst │ │ ├── celery.utils.graph.rst │ │ ├── celery.worker.pidbox.rst │ │ ├── celery.backends.amqp.rst │ │ ├── celery.security.key.rst │ │ ├── celery.backends.base.rst │ │ ├── celery.backends.riak.rst │ │ ├── celery.events.dumper.rst │ │ ├── celery.utils.dispatch.rst │ │ ├── celery.utils.threads.rst │ │ ├── celery.backends.async.rst │ │ ├── celery.backends.cache.rst │ │ ├── celery.backends.consul.rst │ │ ├── celery.backends.redis.rst │ │ ├── celery.security.utils.rst │ │ ├── celery.utils.abstract.rst │ │ ├── celery.utils.log.rst │ │ ├── celery.utils.saferepr.rst │ │ ├── celery.app.annotations.rst │ │ ├── celery.events.cursesmon.rst │ │ ├── celery.events.snapshot.rst │ │ ├── celery.utils.deprecated.rst │ │ ├── celery.utils.nodenames.rst │ │ ├── celery.utils.term.rst │ │ ├── celery.utils.text.rst │ │ ├── celery.worker.components.rst │ │ ├── celery.worker.control.rst │ │ ├── celery.backends.couchdb.rst │ │ ├── celery.backends.mongodb.rst │ │ ├── celery.utils.iso8601.rst │ │ ├── celery.utils.objects.rst │ │ ├── celery.utils.sysinfo.rst │ │ ├── celery.concurrency.base.rst │ │ ├── celery.utils.imports.rst │ │ ├── celery.worker.heartbeat.rst │ │ ├── celery.backends.couchbase.rst │ │ ├── celery.backends.filesystem.rst │ │ ├── celery.utils.timeutils.rst │ │ ├── celery.backends.cassandra.rst │ │ ├── celery.security.certificate.rst │ │ ├── celery.utils.serialization.rst │ │ ├── celery.backends.database.models.rst │ │ ├── celery.security.serialization.rst │ │ ├── celery.utils.functional.rst │ │ ├── celery.backends.elasticsearch.rst │ │ ├── celery.backends.database.rst │ │ ├── celery.backends.database.session.rst │ │ ├── celery.utils.dispatch.signal.rst │ │ ├── celery.concurrency.gevent.rst │ │ ├── celery.concurrency.prefork.rst │ │ ├── celery.concurrency.solo.rst │ │ ├── celery.concurrency.eventlet.rst │ │ ├── celery.utils.dispatch.saferef.rst │ │ └── index.rst │ ├── index.rst │ └── worker.rst ├── tutorials │ └── index.rst ├── userguide │ ├── concurrency │ │ └── index.rst │ └── index.rst ├── THANKS ├── history │ └── index.rst ├── templates │ └── readme.txt ├── copyright.rst ├── community.rst ├── index.rst ├── includes │ └── resources.txt └── conf.py ├── celery ├── apps │ └── __init__.py ├── contrib │ ├── __init__.py │ └── sphinx.py ├── fixups │ └── __init__.py ├── tests │ ├── app │ │ ├── __init__.py │ │ ├── test_celery.py │ │ ├── test_exceptions.py │ │ ├── test_annotations.py │ │ ├── test_utils.py │ │ └── test_defaults.py │ ├── bin │ │ ├── __init__.py │ │ ├── celery.py │ │ └── proj │ │ │ ├── app.py │ │ │ └── __init__.py │ ├── backends │ │ ├── __init__.py │ │ ├── test_consul.py │ │ └── test_backends.py │ ├── contrib │ │ ├── __init__.py │ │ └── test_abortable.py │ ├── events │ │ └── __init__.py │ ├── fixups │ │ └── __init__.py │ ├── tasks │ │ ├── __init__.py │ │ └── test_states.py │ ├── utils │ │ ├── __init__.py │ │ ├── test_objects.py │ │ ├── test_nodenames.py │ │ ├── test_encoding.py │ │ ├── test_sysinfo.py │ │ ├── test_utils.py │ │ ├── test_pickle.py │ │ ├── test_imports.py │ │ ├── test_graph.py │ │ ├── test_deprecated.py │ │ └── test_saferef.py │ ├── worker │ │ ├── __init__.py │ │ ├── test_revoke.py │ │ └── test_heartbeat.py │ ├── concurrency │ │ ├── __init__.py │ │ └── test_solo.py │ ├── compat_modules │ │ ├── __init__.py │ │ ├── test_messaging.py │ │ ├── test_decorators.py │ │ ├── test_compat_utils.py │ │ └── test_compat.py │ └── security │ │ ├── case.py │ │ └── test_key.py ├── bin │ └── __init__.py ├── utils │ ├── dispatch │ │ ├── __init__.py │ │ └── license.txt │ ├── encoding.py │ ├── __init__.py │ └── sysinfo.py ├── loaders │ ├── app.py │ ├── __init__.py │ └── default.py ├── __main__.py ├── worker │ ├── consumer │ │ ├── __init__.py │ │ ├── agent.py │ │ ├── heart.py │ │ ├── control.py │ │ ├── connection.py │ │ ├── events.py │ │ ├── tasks.py │ │ └── mingle.py │ └── heartbeat.py ├── security │ ├── key.py │ ├── utils.py │ └── __init__.py ├── concurrency │ ├── __init__.py │ └── solo.py ├── app │ ├── annotations.py │ └── registry.py ├── task │ └── __init__.py └── backends │ ├── database │ └── session.py │ └── amqp.py ├── examples ├── django │ ├── demoapp │ │ ├── __init__.py │ │ ├── views.py │ │ ├── models.py │ │ └── tasks.py │ ├── requirements.txt │ ├── proj │ │ ├── __init__.py │ │ ├── celery.py │ │ ├── urls.py │ │ └── wsgi.py │ ├── manage.py │ └── README.rst ├── next-steps │ ├── proj │ │ ├── __init__.py │ │ ├── tasks.py │ │ └── celery.py │ └── setup.py ├── celery_http_gateway │ ├── __init__.py │ ├── tasks.py │ ├── manage.py │ ├── urls.py │ └── README.rst ├── tutorial │ └── tasks.py ├── README.rst ├── gevent │ ├── celeryconfig.py │ └── tasks.py ├── eventlet │ ├── tasks.py │ ├── celeryconfig.py │ ├── README.rst │ ├── bulk_task_producer.py │ └── webcrawler.py └── app │ └── myapp.py ├── requirements ├── extras │ ├── pyro.txt │ ├── auth.txt │ ├── gevent.txt │ ├── riak.txt │ ├── consul.txt │ ├── couchbase.txt │ ├── couchdb.txt │ ├── eventlet.txt │ ├── memcache.txt │ ├── redis.txt │ ├── sqs.txt │ ├── tblib.txt │ ├── yaml.txt │ ├── zeromq.txt │ ├── cassandra.txt │ ├── sqlalchemy.txt │ ├── zookeeper.txt │ ├── elasticsearch.txt │ ├── librabbitmq.txt │ ├── msgpack.txt │ ├── pymemcache.txt │ └── slmq.txt ├── test.txt ├── deps │ ├── mock.txt │ └── nose.txt ├── jython.txt ├── security.txt ├── test-pypy3.txt ├── docs.txt ├── default.txt ├── test-ci-base.txt ├── pkgutils.txt ├── test-ci-default.txt ├── dev.txt └── README.rst ├── TODO ├── funtests ├── setup.cfg ├── stress │ └── README.rst ├── suite │ ├── __init__.py │ └── config.py └── setup.py ├── extra ├── systemd │ ├── celery.tmpfiles │ ├── celery.conf │ └── celery.service ├── supervisord │ ├── celery.sh │ ├── celerybeat.conf │ ├── supervisord.conf │ └── celeryd.conf ├── release │ ├── sphinx2rst_config.py │ ├── gen-cert.sh │ └── attribution.py ├── macOS │ ├── org.celeryq.beat.plist │ └── org.celeryq.worker.plist └── appveyor │ └── run_with_compiler.cmd ├── .editorconfig ├── setup.cfg ├── .gitignore ├── .bumpversion.cfg ├── Changelog ├── .coveragerc ├── MANIFEST.in ├── .travis.yml ├── appveyor.yml └── tox.ini /docs/_static/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celery/apps/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celery/contrib/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celery/fixups/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celery/tests/app/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celery/tests/bin/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celery/tests/backends/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celery/tests/contrib/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celery/tests/events/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celery/tests/fixups/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celery/tests/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celery/tests/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celery/tests/worker/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celery/tests/concurrency/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/django/demoapp/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/next-steps/proj/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements/extras/pyro.txt: -------------------------------------------------------------------------------- 1 | pyro4 2 | -------------------------------------------------------------------------------- /requirements/test.txt: -------------------------------------------------------------------------------- 1 | case>=1.2.2 2 | -------------------------------------------------------------------------------- /celery/tests/compat_modules/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/celery_http_gateway/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements/deps/mock.txt: -------------------------------------------------------------------------------- 1 | mock>=1.3 2 | -------------------------------------------------------------------------------- /requirements/deps/nose.txt: -------------------------------------------------------------------------------- 1 | nose>=1.3.7 2 | -------------------------------------------------------------------------------- /requirements/extras/auth.txt: -------------------------------------------------------------------------------- 1 | pyOpenSSL 2 | -------------------------------------------------------------------------------- /requirements/extras/gevent.txt: -------------------------------------------------------------------------------- 1 | gevent 2 | -------------------------------------------------------------------------------- /requirements/extras/riak.txt: -------------------------------------------------------------------------------- 1 | riak >=2.0 2 | -------------------------------------------------------------------------------- /requirements/jython.txt: -------------------------------------------------------------------------------- 1 | multiprocessing 2 | -------------------------------------------------------------------------------- /docs/changelog.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../Changelog 2 | -------------------------------------------------------------------------------- /requirements/extras/consul.txt: -------------------------------------------------------------------------------- 1 | python-consul 2 | -------------------------------------------------------------------------------- /requirements/extras/couchbase.txt: -------------------------------------------------------------------------------- 1 | couchbase 2 | -------------------------------------------------------------------------------- /requirements/extras/couchdb.txt: -------------------------------------------------------------------------------- 1 | pycouchdb 2 | -------------------------------------------------------------------------------- /requirements/extras/eventlet.txt: -------------------------------------------------------------------------------- 1 | eventlet 2 | -------------------------------------------------------------------------------- /requirements/extras/memcache.txt: -------------------------------------------------------------------------------- 1 | pylibmc 2 | -------------------------------------------------------------------------------- /requirements/extras/redis.txt: -------------------------------------------------------------------------------- 1 | redis>=2.8.0 2 | -------------------------------------------------------------------------------- /requirements/extras/sqs.txt: -------------------------------------------------------------------------------- 1 | boto>=2.13.3 2 | -------------------------------------------------------------------------------- /requirements/extras/tblib.txt: -------------------------------------------------------------------------------- 1 | tblib>=1.3.0 2 | -------------------------------------------------------------------------------- /requirements/extras/yaml.txt: -------------------------------------------------------------------------------- 1 | PyYAML>=3.10 2 | -------------------------------------------------------------------------------- /requirements/extras/zeromq.txt: -------------------------------------------------------------------------------- 1 | pyzmq>=13.1.0 2 | -------------------------------------------------------------------------------- /requirements/security.txt: -------------------------------------------------------------------------------- 1 | -r extras/auth.txt 2 | -------------------------------------------------------------------------------- /requirements/test-pypy3.txt: -------------------------------------------------------------------------------- 1 | -r deps/mock.txt 2 | -------------------------------------------------------------------------------- /requirements/extras/cassandra.txt: -------------------------------------------------------------------------------- 1 | cassandra-driver -------------------------------------------------------------------------------- /requirements/extras/sqlalchemy.txt: -------------------------------------------------------------------------------- 1 | sqlalchemy 2 | -------------------------------------------------------------------------------- /requirements/extras/zookeeper.txt: -------------------------------------------------------------------------------- 1 | kazoo>=1.3.1 2 | -------------------------------------------------------------------------------- /requirements/extras/elasticsearch.txt: -------------------------------------------------------------------------------- 1 | elasticsearch 2 | -------------------------------------------------------------------------------- /requirements/extras/librabbitmq.txt: -------------------------------------------------------------------------------- 1 | librabbitmq>=1.5.0 2 | -------------------------------------------------------------------------------- /requirements/extras/msgpack.txt: -------------------------------------------------------------------------------- 1 | msgpack-python>=0.3.0 2 | -------------------------------------------------------------------------------- /requirements/extras/pymemcache.txt: -------------------------------------------------------------------------------- 1 | python-memcached 2 | -------------------------------------------------------------------------------- /examples/django/demoapp/views.py: -------------------------------------------------------------------------------- 1 | # Create your views here. 2 | -------------------------------------------------------------------------------- /requirements/extras/slmq.txt: -------------------------------------------------------------------------------- 1 | softlayer_messaging>=1.0.3 2 | -------------------------------------------------------------------------------- /examples/django/requirements.txt: -------------------------------------------------------------------------------- 1 | django==1.9.5 2 | sqlalchemy==1.0.9 3 | celery>=4.0 4 | -------------------------------------------------------------------------------- /requirements/docs.txt: -------------------------------------------------------------------------------- 1 | sphinx_celery>=1.3 2 | -r extras/sqlalchemy.txt 3 | -r dev.txt 4 | -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | Please see our Issue Tracker at GitHub: 2 | https://github.com/celery/celery/issues 3 | -------------------------------------------------------------------------------- /funtests/setup.cfg: -------------------------------------------------------------------------------- 1 | [nosetests] 2 | verbosity = 1 3 | detailed-errors = 1 4 | where = suite 5 | -------------------------------------------------------------------------------- /celery/tests/bin/celery.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | # here for a test 3 | -------------------------------------------------------------------------------- /docs/images/dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Unity-Technologies/celery/master/docs/images/dashboard.png -------------------------------------------------------------------------------- /docs/images/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Unity-Technologies/celery/master/docs/images/favicon.ico -------------------------------------------------------------------------------- /docs/images/monitor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Unity-Technologies/celery/master/docs/images/monitor.png -------------------------------------------------------------------------------- /examples/django/demoapp/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models # noqa 2 | 3 | # Create your models here. 4 | -------------------------------------------------------------------------------- /docs/images/celery_128.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Unity-Technologies/celery/master/docs/images/celery_128.png -------------------------------------------------------------------------------- /docs/images/celery_512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Unity-Technologies/celery/master/docs/images/celery_512.png -------------------------------------------------------------------------------- /extra/systemd/celery.tmpfiles: -------------------------------------------------------------------------------- 1 | d /var/run/celery 0755 celery celery - 2 | d /var/log/celery 0755 celery celery - 3 | -------------------------------------------------------------------------------- /requirements/default.txt: -------------------------------------------------------------------------------- 1 | pytz>dev 2 | billiard>dev 3 | kombu>dev 4 | 5 | # remove before release 6 | amqp>dev 7 | -------------------------------------------------------------------------------- /docs/images/celeryevshotsm.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Unity-Technologies/celery/master/docs/images/celeryevshotsm.jpg -------------------------------------------------------------------------------- /docs/images/result_graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Unity-Technologies/celery/master/docs/images/result_graph.png -------------------------------------------------------------------------------- /docs/images/worker_graph_full.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Unity-Technologies/celery/master/docs/images/worker_graph_full.png -------------------------------------------------------------------------------- /celery/bin/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from .base import Option 4 | 5 | __all__ = ['Option'] 6 | -------------------------------------------------------------------------------- /docs/reference/celery.contrib.sphinx.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: celery.contrib.sphinx 2 | 3 | .. automodule:: celery.contrib.sphinx 4 | :members: 5 | -------------------------------------------------------------------------------- /requirements/test-ci-base.txt: -------------------------------------------------------------------------------- 1 | coverage>=3.0 2 | codecov 3 | -r extras/redis.txt 4 | -r extras/sqlalchemy.txt 5 | -r extras/pymemcache.txt 6 | -r dev.txt 7 | -------------------------------------------------------------------------------- /celery/tests/bin/proj/app.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from celery import Celery 4 | 5 | app = Celery(set_as_current=False) 6 | -------------------------------------------------------------------------------- /docs/reference/celery.states.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: celery.states 2 | 3 | .. contents:: 4 | :local: 5 | 6 | .. automodule:: celery.states 7 | :members: 8 | 9 | -------------------------------------------------------------------------------- /extra/supervisord/celery.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | source {{ additional variables }} 3 | exec celery --app={{ application_name }}.celery:app worker --loglevel=INFO -n worker.%%h -------------------------------------------------------------------------------- /requirements/pkgutils.txt: -------------------------------------------------------------------------------- 1 | setuptools>=20.6.7 2 | wheel>=0.29.0 3 | flake8>=2.5.4 4 | flakeplus>=1.1 5 | tox>=2.3.1 6 | sphinx2rst>=1.0 7 | cyanide>=1.0.1 8 | bumpversion 9 | -------------------------------------------------------------------------------- /celery/tests/bin/proj/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from celery import Celery 4 | 5 | hello = Celery(set_as_current=False) 6 | -------------------------------------------------------------------------------- /requirements/test-ci-default.txt: -------------------------------------------------------------------------------- 1 | -r test-ci-base.txt 2 | #: Disabled for Cryptography crashing on 2.7 after interpreter shutdown. 3 | #-r extras/auth.txt 4 | -r extras/riak.txt 5 | -------------------------------------------------------------------------------- /celery/utils/dispatch/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import absolute_import, unicode_literals 3 | 4 | from .signal import Signal 5 | 6 | __all__ = ['Signal'] 7 | -------------------------------------------------------------------------------- /docs/getting-started/resources.rst: -------------------------------------------------------------------------------- 1 | .. _resources: 2 | 3 | =========== 4 | Resources 5 | =========== 6 | 7 | .. contents:: 8 | :local: 9 | :depth: 2 10 | 11 | .. include:: ../includes/resources.txt 12 | -------------------------------------------------------------------------------- /docs/django/index.rst: -------------------------------------------------------------------------------- 1 | .. _django: 2 | 3 | ========= 4 | Django 5 | ========= 6 | 7 | :Release: |version| 8 | :Date: |today| 9 | 10 | .. toctree:: 11 | :maxdepth: 2 12 | 13 | first-steps-with-django 14 | -------------------------------------------------------------------------------- /docs/reference/celery.contrib.rdb.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: celery.contrib.rdb 2 | 3 | .. automodule:: celery.contrib.rdb 4 | 5 | .. autofunction:: set_trace 6 | .. autofunction:: debugger 7 | .. autoclass:: Rdb 8 | -------------------------------------------------------------------------------- /examples/celery_http_gateway/tasks.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from celery import task 4 | 5 | 6 | @task() 7 | def hello_world(to='world'): 8 | return 'Hello {0}'.format(to) 9 | -------------------------------------------------------------------------------- /requirements/dev.txt: -------------------------------------------------------------------------------- 1 | https://github.com/celery/py-amqp/zipball/master 2 | https://github.com/celery/billiard/zipball/master 3 | https://github.com/celery/kombu/zipball/master 4 | https://github.com/celery/vine/zipball/master 5 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.collections.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: celery.utils.collections 2 | 3 | .. contents:: 4 | :local: 5 | 6 | .. automodule:: celery.utils.collections 7 | :members: 8 | :undoc-members: 9 | -------------------------------------------------------------------------------- /docs/tutorials/index.rst: -------------------------------------------------------------------------------- 1 | =========== 2 | Tutorials 3 | =========== 4 | 5 | :Release: |version| 6 | :Date: |today| 7 | 8 | .. toctree:: 9 | :maxdepth: 2 10 | 11 | daemonizing 12 | debugging 13 | task-cookbook 14 | -------------------------------------------------------------------------------- /docs/userguide/concurrency/index.rst: -------------------------------------------------------------------------------- 1 | .. _concurrency: 2 | 3 | ============= 4 | Concurrency 5 | ============= 6 | 7 | :Release: |version| 8 | :Date: |today| 9 | 10 | .. toctree:: 11 | :maxdepth: 2 12 | 13 | eventlet 14 | -------------------------------------------------------------------------------- /funtests/stress/README.rst: -------------------------------------------------------------------------------- 1 | ========================= 2 | Celery Stresstest Suite 3 | ========================= 4 | 5 | The stress test suite project has been moved to a dedicated 6 | repository here: 7 | 8 | https://github.com/celery/cyanide/ 9 | -------------------------------------------------------------------------------- /celery/tests/security/case.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from celery.tests.case import AppCase, skip 4 | 5 | 6 | @skip.unless_module('OpenSSL.crypto', name='pyOpenSSL') 7 | class SecurityCase(AppCase): 8 | pass 9 | -------------------------------------------------------------------------------- /docs/reference/celery.events.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | ``celery.events`` 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.events 8 | 9 | .. automodule:: celery.events 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | indent_style = space 7 | indent_size = 4 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | charset = utf-8 11 | end_of_line = lf 12 | 13 | [Makefile] 14 | indent_style = tab 15 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.rst: -------------------------------------------------------------------------------- 1 | ========================== 2 | ``celery.utils`` 3 | ========================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils 8 | 9 | .. automodule:: celery.utils 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.result.rst: -------------------------------------------------------------------------------- 1 | ============================= 2 | ``celery.result`` 3 | ============================= 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.result 8 | 9 | .. automodule:: celery.result 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.security.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | ``celery.security`` 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.security 8 | 9 | .. automodule:: celery.security 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /examples/django/proj/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | # This will make sure the app is always imported when 4 | # Django starts so that shared_task will use this app. 5 | from .celery import app as celery_app 6 | 7 | __all__ = ['celery_app'] 8 | -------------------------------------------------------------------------------- /docs/reference/celery.app.log.rst: -------------------------------------------------------------------------------- 1 | ================================ 2 | ``celery.app.log`` 3 | ================================ 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.app.log 8 | 9 | .. automodule:: celery.app.log 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.backends.rst: -------------------------------------------------------------------------------- 1 | =========================== 2 | ``celery.backends`` 3 | =========================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.backends 8 | 9 | .. automodule:: celery.backends 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.beat.rst: -------------------------------------------------------------------------------- 1 | ======================================== 2 | ``celery.beat`` 3 | ======================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.beat 8 | 9 | .. automodule:: celery.beat 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.bin.base.rst: -------------------------------------------------------------------------------- 1 | ================================ 2 | ``celery.bin.base`` 3 | ================================ 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.bin.base 8 | 9 | .. automodule:: celery.bin.base 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /examples/tutorial/tasks.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from celery import Celery 4 | 5 | app = Celery('tasks', broker='amqp://') 6 | 7 | 8 | @app.task() 9 | def add(x, y): 10 | return x + y 11 | 12 | if __name__ == '__main__': 13 | app.start() 14 | -------------------------------------------------------------------------------- /docs/reference/celery.app.task.rst: -------------------------------------------------------------------------------- 1 | =================================== 2 | ``celery.app.task`` 3 | =================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.app.task 8 | 9 | .. automodule:: celery.app.task 10 | :members: Task, Context, TaskType 11 | -------------------------------------------------------------------------------- /docs/reference/celery.app.utils.rst: -------------------------------------------------------------------------------- 1 | ================================ 2 | ``celery.app.utils`` 3 | ================================ 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.app.utils 8 | 9 | .. automodule:: celery.app.utils 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [nosetests] 2 | where = celery/tests 3 | 4 | [build_sphinx] 5 | source-dir = docs/ 6 | build-dir = docs/_build 7 | all_files = 1 8 | 9 | [bdist_rpm] 10 | requires = pytz >= 2011b 11 | billiard >= 3.3.0.17 12 | kombu >= 3.0.15 13 | 14 | [wheel] 15 | universal = 1 16 | -------------------------------------------------------------------------------- /docs/reference/celery.exceptions.rst: -------------------------------------------------------------------------------- 1 | ================================ 2 | ``celery.exceptions`` 3 | ================================ 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.exceptions 8 | 9 | .. automodule:: celery.exceptions 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.worker.rst: -------------------------------------------------------------------------------- 1 | ======================================== 2 | ``celery.worker`` 3 | ======================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.worker 8 | 9 | .. automodule:: celery.worker 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/getting-started/index.rst: -------------------------------------------------------------------------------- 1 | ================= 2 | Getting Started 3 | ================= 4 | 5 | :Release: |version| 6 | :Date: |today| 7 | 8 | .. toctree:: 9 | :maxdepth: 2 10 | 11 | introduction 12 | brokers/index 13 | first-steps-with-celery 14 | next-steps 15 | resources 16 | -------------------------------------------------------------------------------- /docs/internals/index.rst: -------------------------------------------------------------------------------- 1 | .. _internals: 2 | 3 | =========== 4 | Internals 5 | =========== 6 | 7 | :Release: |version| 8 | :Date: |today| 9 | 10 | .. toctree:: 11 | :maxdepth: 2 12 | 13 | guide 14 | deprecation 15 | worker 16 | protocol 17 | app-overview 18 | reference/index 19 | -------------------------------------------------------------------------------- /docs/reference/celery.app.registry.rst: -------------------------------------------------------------------------------- 1 | ================================ 2 | ``celery.app.registry`` 3 | ================================ 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.app.registry 8 | 9 | .. automodule:: celery.app.registry 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.loaders.app.rst: -------------------------------------------------------------------------------- 1 | ================================= 2 | ``celery.loaders.app`` 3 | ================================= 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.loaders.app 8 | 9 | .. automodule:: celery.loaders.app 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery._state.rst: -------------------------------------------------------------------------------- 1 | ======================================== 2 | ``celery._state`` 3 | ======================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery._state 8 | 9 | .. automodule:: celery._state 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.app.routes.rst: -------------------------------------------------------------------------------- 1 | ================================= 2 | ``celery.app.routes`` 3 | ================================= 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.app.routes 8 | 9 | .. automodule:: celery.app.routes 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.timer2.rst: -------------------------------------------------------------------------------- 1 | ============================== 2 | ``celery.utils.timer2`` 3 | ============================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.timer2 8 | 9 | .. automodule:: celery.utils.timer2 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.loaders.rst: -------------------------------------------------------------------------------- 1 | ============================================ 2 | ``celery.loaders`` 3 | ============================================ 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.loaders 8 | 9 | .. automodule:: celery.loaders 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.concurrency.rst: -------------------------------------------------------------------------------- 1 | ================================== 2 | ``celery.concurrency`` 3 | ================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.concurrency 8 | 9 | .. automodule:: celery.concurrency 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.platforms.rst: -------------------------------------------------------------------------------- 1 | ====================================== 2 | ``celery.platforms`` 3 | ====================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.platforms 8 | 9 | .. automodule:: celery.platforms 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.apps.worker.rst: -------------------------------------------------------------------------------- 1 | ======================================= 2 | ``celery.apps.worker`` 3 | ======================================= 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.apps.worker 8 | 9 | .. automodule:: celery.apps.worker 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.bootsteps.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | ``celery.bootsteps`` 3 | ========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.bootsteps 8 | 9 | .. automodule:: celery.bootsteps 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.contrib.migrate.rst: -------------------------------------------------------------------------------- 1 | ============================ 2 | ``celery.contrib.migrate`` 3 | ============================ 4 | 5 | .. contents:: 6 | :local: 7 | 8 | .. currentmodule:: celery.contrib.migrate 9 | 10 | .. automodule:: celery.contrib.migrate 11 | :members: 12 | :undoc-members: 13 | -------------------------------------------------------------------------------- /docs/reference/celery.worker.state.rst: -------------------------------------------------------------------------------- 1 | ==================================== 2 | ``celery.worker.state`` 3 | ==================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.worker.state 8 | 9 | .. automodule:: celery.worker.state 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.bin.celery.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | ``celery.bin.celery`` 3 | ========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.bin.celery 8 | 9 | .. automodule:: celery.bin.celery 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.bin.worker.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | ``celery.bin.worker`` 3 | ========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.bin.worker 8 | 9 | .. automodule:: celery.bin.worker 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.app.trace.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | ``celery.app.trace`` 3 | ========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.app.trace 8 | 9 | .. automodule:: celery.app.trace 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.worker.loops.rst: -------------------------------------------------------------------------------- 1 | ==================================== 2 | ``celery.worker.loops`` 3 | ==================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.worker.loops 8 | 9 | .. automodule:: celery.worker.loops 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.bin.multi.rst: -------------------------------------------------------------------------------- 1 | =============================================== 2 | ``celery.bin.multi`` 3 | =============================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.bin.multi 8 | 9 | .. automodule:: celery.bin.multi 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.worker.request.rst: -------------------------------------------------------------------------------- 1 | ===================================== 2 | ``celery.worker.request`` 3 | ===================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.worker.request 8 | 9 | .. automodule:: celery.worker.request 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.worker.strategy.rst: -------------------------------------------------------------------------------- 1 | ==================================== 2 | ``celery.worker.strategy`` 3 | ==================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.worker.strategy 8 | 9 | .. automodule:: celery.worker.strategy 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.backends.rpc.rst: -------------------------------------------------------------------------------- 1 | ======================================= 2 | ``celery.backends.rpc`` 3 | ======================================= 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.backends.rpc 8 | 9 | .. automodule:: celery.backends.rpc 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.graph.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | ``celery.utils.graph`` 3 | ========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.graph 8 | 9 | .. automodule:: celery.utils.graph 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.worker.pidbox.rst: -------------------------------------------------------------------------------- 1 | ==================================== 2 | ``celery.worker.pidbox`` 3 | ==================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.worker.pidbox 8 | 9 | .. automodule:: celery.worker.pidbox 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.apps.beat.rst: -------------------------------------------------------------------------------- 1 | ================================================= 2 | ``celery.apps.beat`` 3 | ================================================= 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.apps.beat 8 | 9 | .. automodule:: celery.apps.beat 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.bin.beat.rst: -------------------------------------------------------------------------------- 1 | =================================================== 2 | ``celery.bin.beat`` 3 | =================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.bin.beat 8 | 9 | .. automodule:: celery.bin.beat 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.loaders.base.rst: -------------------------------------------------------------------------------- 1 | =========================================== 2 | ``celery.loaders.base`` 3 | =========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.loaders.base 8 | 9 | .. automodule:: celery.loaders.base 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.signals.rst: -------------------------------------------------------------------------------- 1 | ====================================================== 2 | ``celery.signals`` 3 | ====================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.signals 8 | 9 | .. automodule:: celery.signals 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /examples/django/demoapp/tasks.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from celery import shared_task 4 | 5 | 6 | @shared_task 7 | def add(x, y): 8 | return x + y 9 | 10 | 11 | @shared_task 12 | def mul(x, y): 13 | return x * y 14 | 15 | 16 | @shared_task 17 | def xsum(numbers): 18 | return sum(numbers) 19 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.backends.amqp.rst: -------------------------------------------------------------------------------- 1 | ======================================= 2 | ``celery.backends.amqp`` 3 | ======================================= 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.backends.amqp 8 | 9 | .. automodule:: celery.backends.amqp 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.security.key.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | ``celery.security.key`` 3 | ========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.security.key 8 | 9 | .. automodule:: celery.security.key 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.bin.graph.rst: -------------------------------------------------------------------------------- 1 | ===================================================== 2 | ``celery.bin.graph`` 3 | ===================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.bin.graph 8 | 9 | .. automodule:: celery.bin.graph 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.loaders.default.rst: -------------------------------------------------------------------------------- 1 | ========================================= 2 | ``celery.loaders.default`` 3 | ========================================= 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.loaders.default 8 | 9 | .. automodule:: celery.loaders.default 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.schedules.rst: -------------------------------------------------------------------------------- 1 | ===================================================== 2 | ``celery.schedules`` 3 | ===================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.schedules 8 | 9 | .. automodule:: celery.schedules 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /examples/next-steps/proj/tasks.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from proj.celery import app 4 | 5 | 6 | @app.task 7 | def add(x, y): 8 | return x + y 9 | 10 | 11 | @app.task 12 | def mul(x, y): 13 | return x * y 14 | 15 | 16 | @app.task 17 | def xsum(numbers): 18 | return sum(numbers) 19 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.backends.base.rst: -------------------------------------------------------------------------------- 1 | ===================================== 2 | ``celery.backends.base`` 3 | ===================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.backends.base 8 | 9 | .. automodule:: celery.backends.base 10 | :members: 11 | :undoc-members: 12 | 13 | 14 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.backends.riak.rst: -------------------------------------------------------------------------------- 1 | =========================================== 2 | ``celery.backends.riak`` 3 | =========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.backends.riak 8 | 9 | .. automodule:: celery.backends.riak 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.events.dumper.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | ``celery.events.dumper`` 3 | ========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.events.dumper 8 | 9 | .. automodule:: celery.events.dumper 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.dispatch.rst: -------------------------------------------------------------------------------- 1 | ========================================= 2 | ``celery.utils.dispatch`` 3 | ========================================= 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.dispatch 8 | 9 | .. automodule:: celery.utils.dispatch 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.threads.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | ``celery.utils.threads`` 3 | ========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.threads 8 | 9 | .. automodule:: celery.utils.threads 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.app.control.rst: -------------------------------------------------------------------------------- 1 | ==================================================== 2 | ``celery.app.control`` 3 | ==================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.app.control 8 | 9 | .. automodule:: celery.app.control 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.bin.events.rst: -------------------------------------------------------------------------------- 1 | ===================================================== 2 | ``celery.bin.events`` 3 | ===================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.bin.events 8 | 9 | .. automodule:: celery.bin.events 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.backends.async.rst: -------------------------------------------------------------------------------- 1 | ===================================== 2 | ``celery.backends.async`` 3 | ===================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.backends.async 8 | 9 | .. automodule:: celery.backends.async 10 | :members: 11 | :undoc-members: 12 | 13 | 14 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.backends.cache.rst: -------------------------------------------------------------------------------- 1 | =========================================== 2 | ``celery.backends.cache`` 3 | =========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.backends.cache 8 | 9 | .. automodule:: celery.backends.cache 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.backends.consul.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | celery.backends.consul 3 | ========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.backends.consul 8 | 9 | .. automodule:: celery.backends.consul 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.backends.redis.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | ``celery.backends.redis`` 3 | ========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.backends.redis 8 | 9 | .. automodule:: celery.backends.redis 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.security.utils.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | ``celery.security.utils`` 3 | ========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.security.utils 8 | 9 | .. automodule:: celery.security.utils 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.abstract.rst: -------------------------------------------------------------------------------- 1 | =========================================== 2 | ``celery.utils.abstract`` 3 | =========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.abstract 8 | 9 | .. automodule:: celery.utils.abstract 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.log.rst: -------------------------------------------------------------------------------- 1 | ===================================================== 2 | ``celery.utils.log`` 3 | ===================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.log 8 | 9 | .. automodule:: celery.utils.log 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.saferepr.rst: -------------------------------------------------------------------------------- 1 | =========================================== 2 | ``celery.utils.saferepr`` 3 | =========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.saferepr 8 | 9 | .. automodule:: celery.utils.saferepr 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.app.builtins.rst: -------------------------------------------------------------------------------- 1 | ==================================================== 2 | ``celery.app.builtins`` 3 | ==================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.app.builtins 8 | 9 | .. automodule:: celery.app.builtins 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.bin.amqp.rst: -------------------------------------------------------------------------------- 1 | =========================================================== 2 | ``celery.bin.amqp`` 3 | =========================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.bin.amqp 8 | 9 | .. automodule:: celery.bin.amqp 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.bin.logtool.rst: -------------------------------------------------------------------------------- 1 | ===================================================== 2 | ``celery.bin.logtool`` 3 | ===================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.bin.logtool 8 | 9 | .. automodule:: celery.bin.logtool 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.app.annotations.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | ``celery.app.annotations`` 3 | ========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.app.annotations 8 | 9 | .. automodule:: celery.app.annotations 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.events.cursesmon.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | ``celery.events.cursesmon`` 3 | ========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.events.cursesmon 8 | 9 | .. automodule:: celery.events.cursesmon 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.events.snapshot.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | ``celery.events.snapshot`` 3 | ========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.events.snapshot 8 | 9 | .. automodule:: celery.events.snapshot 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.deprecated.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | ``celery.utils.deprecated`` 3 | ========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.deprecated 8 | 9 | .. automodule:: celery.utils.deprecated 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.nodenames.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | ``celery.utils.nodenames`` 3 | ========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.nodenames 8 | 9 | .. automodule:: celery.utils.nodenames 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.term.rst: -------------------------------------------------------------------------------- 1 | ===================================================== 2 | ``celery.utils.term`` 3 | ===================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.term 8 | 9 | .. automodule:: celery.utils.term 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.text.rst: -------------------------------------------------------------------------------- 1 | ===================================================== 2 | ``celery.utils.text`` 3 | ===================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.text 8 | 9 | .. automodule:: celery.utils.text 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.worker.components.rst: -------------------------------------------------------------------------------- 1 | ======================================== 2 | ``celery.worker.components`` 3 | ======================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.worker.components 8 | 9 | .. automodule:: celery.worker.components 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.worker.control.rst: -------------------------------------------------------------------------------- 1 | ============================================= 2 | ``celery.worker.control`` 3 | ============================================= 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.worker.control 8 | 9 | .. automodule:: celery.worker.control 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /extra/systemd/celery.conf: -------------------------------------------------------------------------------- 1 | # See 2 | # http://docs.celeryproject.org/en/latest/tutorials/daemonizing.html#available-options 3 | 4 | CELERY_APP="proj" 5 | CELERYD_NODES="worker" 6 | CELERYD_OPTS="" 7 | CELERY_BIN="/usr/bin/celery" 8 | CELERYD_PID_FILE="/var/run/celery/%n.pid" 9 | CELERYD_LOG_FILE="/var/log/celery/%n%I.log" 10 | CELERYD_LOG_LEVEL="INFO" 11 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.backends.couchdb.rst: -------------------------------------------------------------------------------- 1 | =========================================== 2 | ``celery.backends.couchdb`` 3 | =========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.backends.couchdb 8 | 9 | .. automodule:: celery.backends.couchdb 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.backends.mongodb.rst: -------------------------------------------------------------------------------- 1 | ============================================ 2 | ``celery.backends.mongodb`` 3 | ============================================ 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.backends.mongodb 8 | 9 | .. automodule:: celery.backends.mongodb 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.iso8601.rst: -------------------------------------------------------------------------------- 1 | ================================================== 2 | ``celery.utils.iso8601`` 3 | ================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.iso8601 8 | 9 | .. automodule:: celery.utils.iso8601 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.objects.rst: -------------------------------------------------------------------------------- 1 | ================================================== 2 | ``celery.utils.objects`` 3 | ================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.objects 8 | 9 | .. automodule:: celery.utils.objects 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.sysinfo.rst: -------------------------------------------------------------------------------- 1 | ================================================== 2 | ``celery.utils.sysinfo`` 3 | ================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.sysinfo 8 | 9 | .. automodule:: celery.utils.sysinfo 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.worker.consumer.rst: -------------------------------------------------------------------------------- 1 | ================================================== 2 | ``celery.worker.consumer`` 3 | ================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.worker.consumer 8 | 9 | .. automodule:: celery.worker.consumer 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.concurrency.base.rst: -------------------------------------------------------------------------------- 1 | =============================================== 2 | ``celery.concurrency.base`` 3 | =============================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.concurrency.base 8 | 9 | .. automodule:: celery.concurrency.base 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.imports.rst: -------------------------------------------------------------------------------- 1 | ===================================================== 2 | ``celery.utils.imports`` 3 | ===================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.imports 8 | 9 | .. automodule:: celery.utils.imports 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.worker.heartbeat.rst: -------------------------------------------------------------------------------- 1 | ============================================= 2 | ``celery.worker.heartbeat`` 3 | ============================================= 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.worker.heartbeat 8 | 9 | .. automodule:: celery.worker.heartbeat 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.backends.couchbase.rst: -------------------------------------------------------------------------------- 1 | ============================================ 2 | ``celery.backends.couchbase`` 3 | ============================================ 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.backends.couchbase 8 | 9 | .. automodule:: celery.backends.couchbase 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.backends.filesystem.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | ``celery.backends.filesystem`` 3 | ========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.backends.filesystem 8 | 9 | .. automodule:: celery.backends.filesystem 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.timeutils.rst: -------------------------------------------------------------------------------- 1 | ================================================== 2 | ``celery.utils.timeutils`` 3 | ================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.timeutils 8 | 9 | .. automodule:: celery.utils.timeutils 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.backends.cassandra.rst: -------------------------------------------------------------------------------- 1 | ================================================ 2 | ``celery.backends.cassandra`` 3 | ================================================ 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.backends.cassandra 8 | 9 | .. automodule:: celery.backends.cassandra 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.security.certificate.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | ``celery.security.certificate`` 3 | ========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.security.certificate 8 | 9 | .. automodule:: celery.security.certificate 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.serialization.rst: -------------------------------------------------------------------------------- 1 | ============================================ 2 | ``celery.utils.serialization`` 3 | ============================================ 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.serialization 8 | 9 | .. automodule:: celery.utils.serialization 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.app.defaults.rst: -------------------------------------------------------------------------------- 1 | =============================================================== 2 | ``celery.app.defaults`` 3 | =============================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.app.defaults 8 | 9 | .. automodule:: celery.app.defaults 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.events.state.rst: -------------------------------------------------------------------------------- 1 | ================================================================= 2 | ``celery.events.state`` 3 | ================================================================= 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.events.state 8 | 9 | .. automodule:: celery.events.state 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /celery/tests/utils/test_objects.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from celery.utils.objects import Bunch 4 | 5 | from celery.tests.case import Case 6 | 7 | 8 | class test_Bunch(Case): 9 | 10 | def test(self): 11 | x = Bunch(foo='foo', bar=2) 12 | self.assertEqual(x.foo, 'foo') 13 | self.assertEqual(x.bar, 2) 14 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.backends.database.models.rst: -------------------------------------------------------------------------------- 1 | ====================================== 2 | ``celery.backends.database.models`` 3 | ====================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.backends.database.models 8 | 9 | .. automodule:: celery.backends.database.models 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.security.serialization.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | ``celery.security.serialization`` 3 | ========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.security.serialization 8 | 9 | .. automodule:: celery.security.serialization 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.functional.rst: -------------------------------------------------------------------------------- 1 | ===================================================== 2 | ``celery.utils.functional`` 3 | ===================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.functional 8 | 9 | .. automodule:: celery.utils.functional 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /examples/django/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import absolute_import, unicode_literals 3 | 4 | import os 5 | import sys 6 | 7 | if __name__ == '__main__': 8 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'proj.settings') 9 | 10 | from django.core.management import execute_from_command_line 11 | 12 | execute_from_command_line(sys.argv) 13 | -------------------------------------------------------------------------------- /celery/loaders/app.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | celery.loaders.app 4 | ~~~~~~~~~~~~~~~~~~ 5 | 6 | The default loader used with custom app instances. 7 | 8 | """ 9 | from __future__ import absolute_import, unicode_literals 10 | 11 | from .base import BaseLoader 12 | 13 | __all__ = ['AppLoader'] 14 | 15 | 16 | class AppLoader(BaseLoader): 17 | pass 18 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.backends.elasticsearch.rst: -------------------------------------------------------------------------------- 1 | =========================================== 2 | ``celery.backends.elasticsearch`` 3 | =========================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.backends.elasticsearch 8 | 9 | .. automodule:: celery.backends.elasticsearch 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.contrib.abortable.rst: -------------------------------------------------------------------------------- 1 | ======================================================= 2 | ``celery.contrib.abortable`` 3 | ======================================================= 4 | 5 | .. contents:: 6 | :local: 7 | 8 | .. currentmodule:: celery.contrib.abortable 9 | 10 | .. automodule:: celery.contrib.abortable 11 | :members: 12 | :undoc-members: 13 | -------------------------------------------------------------------------------- /docs/reference/celery.worker.consumer.agent.rst: -------------------------------------------------------------------------------- 1 | ================================================== 2 | ``celery.worker.consumer.agent`` 3 | ================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.worker.consumer.agent 8 | 9 | .. automodule:: celery.worker.consumer.agent 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.worker.consumer.events.rst: -------------------------------------------------------------------------------- 1 | ================================================== 2 | ``celery.worker.consumer.events`` 3 | ================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.worker.consumer.events 8 | 9 | .. automodule:: celery.worker.consumer.events 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.worker.consumer.gossip.rst: -------------------------------------------------------------------------------- 1 | ================================================== 2 | ``celery.worker.consumer.gossip`` 3 | ================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.worker.consumer.gossip 8 | 9 | .. automodule:: celery.worker.consumer.gossip 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.worker.consumer.heart.rst: -------------------------------------------------------------------------------- 1 | ================================================== 2 | ``celery.worker.consumer.heart`` 3 | ================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.worker.consumer.heart 8 | 9 | .. automodule:: celery.worker.consumer.heart 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.worker.consumer.mingle.rst: -------------------------------------------------------------------------------- 1 | ================================================== 2 | ``celery.worker.consumer.mingle`` 3 | ================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.worker.consumer.mingle 8 | 9 | .. automodule:: celery.worker.consumer.mingle 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.worker.consumer.tasks.rst: -------------------------------------------------------------------------------- 1 | ================================================== 2 | ``celery.worker.consumer.tasks`` 3 | ================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.worker.consumer.tasks 8 | 9 | .. automodule:: celery.worker.consumer.tasks 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.pyc 3 | *$py.class 4 | *~ 5 | .*.sw[pon] 6 | dist/ 7 | *.egg-info 8 | *.egg 9 | *.egg/ 10 | build/ 11 | .build/ 12 | _build/ 13 | pip-log.txt 14 | .directory 15 | erl_crash.dump 16 | *.db 17 | Documentation/ 18 | .tox/ 19 | .ropeproject/ 20 | .project 21 | .pydevproject 22 | .idea/ 23 | .coverage 24 | celery/tests/cover/ 25 | .ve* 26 | cover/ 27 | .vagrant/ 28 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.backends.database.rst: -------------------------------------------------------------------------------- 1 | ========================================================= 2 | ``celery.backends.database`` 3 | ========================================================= 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.backends.database 8 | 9 | .. automodule:: celery.backends.database 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.backends.database.session.rst: -------------------------------------------------------------------------------- 1 | ======================================== 2 | ``celery.backends.database.session`` 3 | ======================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.backends.database.session 8 | 9 | .. automodule:: celery.backends.database.session 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.worker.consumer.control.rst: -------------------------------------------------------------------------------- 1 | ================================================== 2 | ``celery.worker.consumer.control`` 3 | ================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.worker.consumer.control 8 | 9 | .. automodule:: celery.worker.consumer.control 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.dispatch.signal.rst: -------------------------------------------------------------------------------- 1 | ==================================================== 2 | ``celery.utils.dispatch.signal`` 3 | ==================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.dispatch.signal 8 | 9 | .. automodule:: celery.utils.dispatch.signal 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.worker.consumer.consumer.rst: -------------------------------------------------------------------------------- 1 | ================================================== 2 | ``celery.worker.consumer.consumer`` 3 | ================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.worker.consumer.consumer 8 | 9 | .. automodule:: celery.worker.consumer.consumer 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.app.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: celery.app 2 | 3 | .. automodule:: celery.app 4 | 5 | .. contents:: 6 | :local: 7 | 8 | Proxies 9 | ------- 10 | 11 | .. autodata:: default_app 12 | 13 | 14 | Functions 15 | --------- 16 | 17 | .. autofunction:: app_or_default 18 | .. autofunction:: enable_trace 19 | .. autofunction:: disable_trace 20 | -------------------------------------------------------------------------------- /examples/README.rst: -------------------------------------------------------------------------------- 1 | ================= 2 | Celery Examples 3 | ================= 4 | 5 | 6 | * pythonproject 7 | 8 | Example Python project using celery. 9 | 10 | * httpexample 11 | 12 | Example project using remote tasks (webhook tasks) 13 | 14 | * celery_http_gateway 15 | 16 | Example HTTP service exposing the ability to apply tasks and query the 17 | resulting status/return value. 18 | 19 | -------------------------------------------------------------------------------- /celery/tests/utils/test_nodenames.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from kombu import Queue 4 | 5 | from celery.utils.nodenames import worker_direct 6 | 7 | from celery.tests.case import Case 8 | 9 | 10 | class test_worker_direct(Case): 11 | 12 | def test_returns_if_queue(self): 13 | q = Queue('foo') 14 | self.assertIs(worker_direct(q), q) 15 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.concurrency.gevent.rst: -------------------------------------------------------------------------------- 1 | ============================================================= 2 | ``celery.concurrency.gevent`` 3 | ============================================================= 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.concurrency.gevent 8 | 9 | .. automodule:: celery.concurrency.gevent 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.concurrency.prefork.rst: -------------------------------------------------------------------------------- 1 | ============================================================= 2 | ``celery.concurrency.prefork`` 3 | ============================================================= 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.concurrency.prefork 8 | 9 | .. automodule:: celery.concurrency.prefork 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.concurrency.solo.rst: -------------------------------------------------------------------------------- 1 | =================================================================== 2 | ``celery.concurrency.solo`` 3 | =================================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.concurrency.solo 8 | 9 | .. automodule:: celery.concurrency.solo 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/celery.worker.consumer.connection.rst: -------------------------------------------------------------------------------- 1 | ================================================== 2 | ``celery.worker.consumer.connection`` 3 | ================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.worker.consumer.connection 8 | 9 | .. automodule:: celery.worker.consumer.connection 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /examples/gevent/celeryconfig.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | import os 4 | import sys 5 | sys.path.insert(0, os.getcwd()) 6 | 7 | # ## Note: Start worker with -P gevent, 8 | # do not use the worker_pool option. 9 | 10 | broker_url = 'amqp://guest:guest@localhost:5672//' 11 | result_backend = 'amqp' 12 | result_expires = 30 * 60 13 | 14 | imports = ('tasks',) 15 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.concurrency.eventlet.rst: -------------------------------------------------------------------------------- 1 | ============================================================= 2 | ``celery.concurrency.eventlet`` 3 | ============================================================= 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.concurrency.eventlet 8 | 9 | .. automodule:: celery.concurrency.eventlet 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/internals/reference/celery.utils.dispatch.saferef.rst: -------------------------------------------------------------------------------- 1 | ========================================================== 2 | ``celery.utils.dispatch.saferef`` 3 | ========================================================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: celery.utils.dispatch.saferef 8 | 9 | .. automodule:: celery.utils.dispatch.saferef 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /.bumpversion.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 4.0.0rc2 3 | commit = True 4 | tag = True 5 | parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P[a-z]+)? 6 | serialize = 7 | {major}.{minor}.{patch}{releaselevel} 8 | {major}.{minor}.{patch} 9 | 10 | [bumpversion:file:celery/__init__.py] 11 | 12 | [bumpversion:file:docs/includes/introduction.txt] 13 | 14 | [bumpversion:file:README.rst] 15 | -------------------------------------------------------------------------------- /celery/__main__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, print_function, unicode_literals 2 | 3 | import sys 4 | 5 | from . import maybe_patch_concurrency 6 | 7 | __all__ = ['main'] 8 | 9 | 10 | def main(): 11 | if 'multi' not in sys.argv: 12 | maybe_patch_concurrency() 13 | from celery.bin.celery import main 14 | main() 15 | 16 | 17 | if __name__ == '__main__': # pragma: no cover 18 | main() 19 | -------------------------------------------------------------------------------- /examples/eventlet/tasks.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals, print_function 2 | 3 | import requests 4 | 5 | from celery import task 6 | 7 | 8 | @task() 9 | def urlopen(url): 10 | print('-open: {0}'.format(url)) 11 | try: 12 | response = requests.get(url) 13 | except Exception as exc: 14 | print('-url {0} gave error: {1!r}'.format(url, exc)) 15 | return len(response.text) 16 | -------------------------------------------------------------------------------- /docs/userguide/index.rst: -------------------------------------------------------------------------------- 1 | .. _guide: 2 | 3 | ============ 4 | User Guide 5 | ============ 6 | 7 | :Release: |version| 8 | :Date: |today| 9 | 10 | .. toctree:: 11 | :maxdepth: 1 12 | 13 | application 14 | tasks 15 | calling 16 | canvas 17 | workers 18 | periodic-tasks 19 | routing 20 | monitoring 21 | security 22 | optimizing 23 | concurrency/index 24 | signals 25 | extending 26 | -------------------------------------------------------------------------------- /celery/tests/worker/test_revoke.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from celery.worker import state 4 | from celery.tests.case import AppCase 5 | 6 | 7 | class test_revoked(AppCase): 8 | 9 | def test_is_working(self): 10 | state.revoked.add('foo') 11 | self.assertIn('foo', state.revoked) 12 | state.revoked.pop_value('foo') 13 | self.assertNotIn('foo', state.revoked) 14 | -------------------------------------------------------------------------------- /funtests/suite/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | import os 4 | import sys 5 | 6 | sys.path.insert(0, os.getcwd()) 7 | sys.path.insert(0, os.path.join(os.getcwd(), os.pardir)) 8 | 9 | config = os.environ.setdefault('CELERY_FUNTEST_CONFIG_MODULE', 10 | 'suite.config') 11 | 12 | os.environ['CELERY_CONFIG_MODULE'] = config 13 | os.environ['CELERY_LOADER'] = 'default' 14 | -------------------------------------------------------------------------------- /Changelog: -------------------------------------------------------------------------------- 1 | .. _changelog: 2 | 3 | ================ 4 | Change history 5 | ================ 6 | 7 | This document contains change notes for bugfix releases in 8 | the 4.0.x series (0today8), please see :ref:`whatsnew-4.0` for 9 | an overview of what's new in Celery 4.0. 10 | 11 | .. _version-4.0.0: 12 | 13 | 4.0.0 14 | ===== 15 | :release-date: TBA 16 | :status: *FROZEN* 17 | :branch: master 18 | :release-by: 19 | 20 | See :ref:`whatsnew-4.0`. 21 | -------------------------------------------------------------------------------- /examples/next-steps/proj/celery.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from celery import Celery 4 | 5 | app = Celery('proj', 6 | broker='amqp://', 7 | backend='amqp://', 8 | include=['proj.tasks']) 9 | 10 | # Optional configuration, see the application user guide. 11 | app.conf.update( 12 | result_expires=3600, 13 | ) 14 | 15 | if __name__ == '__main__': 16 | app.start() 17 | -------------------------------------------------------------------------------- /celery/utils/encoding.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | celery.utils.encoding 4 | ~~~~~~~~~~~~~~~~~~~~~ 5 | 6 | This module has moved to :mod:`kombu.utils.encoding`. 7 | 8 | """ 9 | from __future__ import absolute_import, unicode_literals 10 | 11 | from kombu.utils.encoding import ( # noqa 12 | default_encode, default_encoding, bytes_t, bytes_to_str, str_t, 13 | str_to_bytes, ensure_bytes, from_utf8, safe_str, safe_repr, 14 | ) 15 | -------------------------------------------------------------------------------- /docs/THANKS: -------------------------------------------------------------------------------- 1 | Thanks to Rune Halvorsen for the name. 2 | Thanks to Anton Tsigularov for the previous name (crunchy) 3 | which we had to abandon because of an existing project with that name. 4 | Thanks to Armin Ronacher for the Sphinx theme. 5 | Thanks to Brian K. Jones for bunny.py (https://github.com/bkjones/bunny), the 6 | tool that inspired 'celery amqp'. 7 | Thanks to Barry Pederson for amqplib (the project py-amqp forked). 8 | -------------------------------------------------------------------------------- /celery/tests/compat_modules/test_messaging.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from celery import messaging 4 | from celery.tests.case import AppCase, depends_on_current_app 5 | 6 | 7 | @depends_on_current_app 8 | class test_compat_messaging_module(AppCase): 9 | 10 | def test_get_consume_set(self): 11 | conn = messaging.establish_connection() 12 | messaging.get_consumer_set(conn).close() 13 | conn.close() 14 | -------------------------------------------------------------------------------- /examples/gevent/tasks.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | import requests 4 | 5 | from celery import task 6 | 7 | 8 | @task(ignore_result=True) 9 | def urlopen(url): 10 | print('Opening: {0}'.format(url)) 11 | try: 12 | requests.get(url) 13 | except Exception as exc: 14 | print('Exception for {0}: {1!r}'.format(url, exc)) 15 | return url, 0 16 | print('Done with: {0}'.format(url)) 17 | return url, 1 18 | -------------------------------------------------------------------------------- /examples/eventlet/celeryconfig.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | import os 4 | import sys 5 | sys.path.insert(0, os.getcwd()) 6 | 7 | # ## Start worker with -P eventlet 8 | # Never use the worker_pool setting as that will patch 9 | # the worker too late. 10 | 11 | broker_url = 'amqp://guest:guest@localhost:5672//' 12 | worker_disable_rate_limits = True 13 | result_backend = 'amqp' 14 | result_expires = 30 * 60 15 | 16 | imports = ('tasks', 'webcrawler') 17 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = 1 3 | cover_pylib = 0 4 | include=*celery/* 5 | omit = celery.tests.* 6 | 7 | [report] 8 | omit = 9 | */python?.?/* 10 | */site-packages/* 11 | */pypy/* 12 | */celery/bin/graph.py 13 | *celery/bin/logtool.py 14 | *celery/task/base.py 15 | *celery/five.py 16 | *celery/contrib/sphinx.py 17 | *celery/backends/couchdb.py 18 | *celery/backends/couchbase.py 19 | *celery/backends/riak.py 20 | *celery/concurrency/asynpool.py 21 | *celery/utils/debug.py 22 | -------------------------------------------------------------------------------- /celery/worker/consumer/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from .consumer import Consumer 4 | 5 | from .agent import Agent 6 | from .connection import Connection 7 | from .control import Control 8 | from .events import Events 9 | from .gossip import Gossip 10 | from .heart import Heart 11 | from .mingle import Mingle 12 | from .tasks import Tasks 13 | 14 | __all__ = [ 15 | 'Consumer', 'Agent', 'Connection', 'Control', 16 | 'Events', 'Gossip', 'Heart', 'Mingle', 'Tasks', 17 | ] 18 | -------------------------------------------------------------------------------- /docs/history/index.rst: -------------------------------------------------------------------------------- 1 | .. _history: 2 | 3 | ========= 4 | History 5 | ========= 6 | 7 | This section contains historical change histories, for the latest 8 | version please visit :ref:`changelog`. 9 | 10 | :Release: |version| 11 | :Date: |today| 12 | 13 | .. toctree:: 14 | :maxdepth: 2 15 | 16 | changelog-3.1 17 | whatsnew-3.0 18 | changelog-3.0 19 | whatsnew-2.5 20 | changelog-2.5 21 | changelog-2.4 22 | changelog-2.3 23 | changelog-2.2 24 | changelog-2.1 25 | changelog-2.0 26 | changelog-1.0 27 | -------------------------------------------------------------------------------- /examples/celery_http_gateway/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import absolute_import, unicode_literals 3 | 4 | from django.core.management import execute_manager 5 | try: 6 | import settings # Assumed to be in the same directory. 7 | except ImportError: 8 | import sys 9 | sys.stderr.write( 10 | "Error: Can't find the file 'settings.py' in the directory " 11 | "containing {0!r}.".format(__file__)) 12 | sys.exit(1) 13 | 14 | if __name__ == '__main__': 15 | execute_manager(settings) 16 | -------------------------------------------------------------------------------- /extra/release/sphinx2rst_config.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | REFBASE = 'http://docs.celeryproject.org/en/latest' 4 | REFS = { 5 | 'mailing-list': 6 | 'http://groups.google.com/group/celery-users', 7 | 'irc-channel': 'getting-started/resources.html#irc', 8 | 'breakpoint-signal': 'tutorials/debugging.html', 9 | 'internals-guide': 'internals/guide.html', 10 | 'bundles': 'getting-started/introduction.html#bundles', 11 | 'reporting-bugs': 'contributing.html#reporting-bugs', 12 | } 13 | -------------------------------------------------------------------------------- /funtests/suite/config.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | import atexit 4 | import os 5 | 6 | broker_url = os.environ.get('BROKER_URL') or 'amqp://' 7 | result_backend = 'amqp://' 8 | 9 | default_queue = 'testcelery' 10 | default_exchange = 'testcelery' 11 | default_routing_key = 'testcelery' 12 | queues = {'testcelery': {'routing_key': 'testcelery'}} 13 | 14 | log_color = False 15 | 16 | 17 | @atexit.register 18 | def teardown_testdb(): 19 | import os 20 | if os.path.exists('test.db'): 21 | os.remove('test.db') 22 | -------------------------------------------------------------------------------- /celery/worker/consumer/agent.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from celery import bootsteps 4 | 5 | from .connection import Connection 6 | 7 | __all__ = ['Agent'] 8 | 9 | 10 | class Agent(bootsteps.StartStopStep): 11 | 12 | conditional = True 13 | requires = (Connection,) 14 | 15 | def __init__(self, c, **kwargs): 16 | self.agent_cls = self.enabled = c.app.conf.worker_agent 17 | 18 | def create(self, c): 19 | agent = c.agent = self.instantiate(self.agent_cls, c.connection) 20 | return agent 21 | -------------------------------------------------------------------------------- /extra/release/gen-cert.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Script for generating key-cert pairs 4 | # 5 | 6 | if [[ $# -ne 2 ]]; then 7 | echo "Usage: ${0##*/} KEY CERT" 8 | exit 9 | fi 10 | 11 | KEY=$1 12 | CERT=$2 13 | PSWD=test 14 | 15 | read -s -p "Enter Password: " $PSWD 16 | 17 | openssl genrsa -des3 -passout pass:$PSWD -out $KEY.key 1024 18 | openssl req -new -key $KEY.key -out $KEY.csr -passin pass:$PSWD 19 | cp $KEY.key $KEY.key.org 20 | openssl rsa -in $KEY.key.org -out $KEY.key -passin pass:$PSWD 21 | openssl x509 -req -days 365 -in $KEY.csr -signkey $KEY.key -out $CERT.crt 22 | rm $KEY.key.org $KEY.csr 23 | -------------------------------------------------------------------------------- /celery/tests/utils/test_encoding.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from celery.utils import encoding 4 | from celery.tests.case import Case 5 | 6 | 7 | class test_encoding(Case): 8 | 9 | def test_safe_str(self): 10 | self.assertTrue(encoding.safe_str(object())) 11 | self.assertTrue(encoding.safe_str('foo')) 12 | 13 | def test_safe_repr(self): 14 | self.assertTrue(encoding.safe_repr(object())) 15 | 16 | class foo(object): 17 | def __repr__(self): 18 | raise ValueError('foo') 19 | 20 | self.assertTrue(encoding.safe_repr(foo())) 21 | -------------------------------------------------------------------------------- /celery/tests/app/test_celery.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from celery.tests.case import AppCase 4 | 5 | import celery 6 | 7 | 8 | class test_celery_package(AppCase): 9 | 10 | def test_version(self): 11 | self.assertTrue(celery.VERSION) 12 | self.assertGreaterEqual(len(celery.VERSION), 3) 13 | celery.VERSION = (0, 3, 0) 14 | self.assertGreaterEqual(celery.__version__.count('.'), 2) 15 | 16 | def test_meta(self): 17 | for m in ('__author__', '__contact__', '__homepage__', 18 | '__docformat__'): 19 | self.assertTrue(getattr(celery, m, None)) 20 | -------------------------------------------------------------------------------- /examples/next-steps/setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | Example setup file for a project using Celery. 3 | 4 | This can be used to distribute your tasks and worker 5 | as a Python package, on PyPI or on your own private package index. 6 | 7 | """ 8 | from __future__ import absolute_import, unicode_literals 9 | 10 | from setuptools import setup, find_packages 11 | 12 | setup( 13 | name='example-tasks', 14 | version='1.0', 15 | description='Tasks for my project', 16 | packages=find_packages(exclude=['ez_setup', 'tests', 'tests.*']), 17 | zip_safe=False, 18 | install_requires=[ 19 | 'celery>=4.0', 20 | # 'requests', 21 | ], 22 | ) 23 | -------------------------------------------------------------------------------- /celery/tests/concurrency/test_solo.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | import operator 4 | 5 | from celery.concurrency import solo 6 | from celery.utils.functional import noop 7 | from celery.tests.case import AppCase 8 | 9 | 10 | class test_solo_TaskPool(AppCase): 11 | 12 | def test_on_start(self): 13 | x = solo.TaskPool() 14 | x.on_start() 15 | 16 | def test_on_apply(self): 17 | x = solo.TaskPool() 18 | x.on_start() 19 | x.on_apply(operator.add, (2, 2), {}, noop, noop) 20 | 21 | def test_info(self): 22 | x = solo.TaskPool() 23 | x.on_start() 24 | self.assertTrue(x.info) 25 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include CONTRIBUTORS.txt 2 | include Changelog 3 | include LICENSE 4 | include README.rst 5 | include MANIFEST.in 6 | include TODO 7 | include setup.cfg 8 | include setup.py 9 | 10 | recursive-include docs * 11 | recursive-include extra/bash-completion * 12 | recursive-include extra/centos * 13 | recursive-include extra/generic-init.d * 14 | recursive-include extra/macOS * 15 | recursive-include extra/supervisord * 16 | recursive-include extra/systemd * 17 | recursive-include extra/zsh-completion * 18 | recursive-include examples * 19 | recursive-include requirements *.txt *.rst 20 | 21 | recursive-exclude * __pycache__ 22 | recursive-exclude * *.py[co] 23 | recursive-exclude * .*.sw[a-z] 24 | -------------------------------------------------------------------------------- /extra/supervisord/celerybeat.conf: -------------------------------------------------------------------------------- 1 | ; ================================ 2 | ; celery beat supervisor example 3 | ; ================================ 4 | 5 | [program:celerybeat] 6 | ; Set full path to celery program if using virtualenv 7 | command=celery beat -A myapp --schedule /var/lib/celery/beat.db --loglevel=INFO 8 | 9 | ; remove the -A myapp argument if you are not using an app instance 10 | 11 | directory=/path/to/project 12 | user=nobody 13 | numprocs=1 14 | stdout_logfile=/var/log/celery/beat.log 15 | stderr_logfile=/var/log/celery/beat.log 16 | autostart=true 17 | autorestart=true 18 | startsecs=10 19 | 20 | ; if rabbitmq is supervised, set its priority higher 21 | ; so it starts first 22 | priority=999 23 | -------------------------------------------------------------------------------- /examples/django/proj/celery.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | import os 4 | 5 | from celery import Celery 6 | 7 | # set the default Django settings module for the 'celery' program. 8 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'proj.settings') 9 | 10 | from django.conf import settings # noqa 11 | 12 | app = Celery('proj') 13 | 14 | # Using a string here means the worker does not have to serialize 15 | # the configuration object. 16 | app.config_from_object('django.conf:settings', namespace='CELERY') 17 | 18 | # load task modules from all registered Django app configs. 19 | app.autodiscover_tasks() 20 | 21 | 22 | @app.task(bind=True) 23 | def debug_task(self): 24 | print('Request: {0!r}'.format(self.request)) 25 | -------------------------------------------------------------------------------- /examples/django/proj/urls.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from django.conf.urls import ( # noqa 4 | patterns, include, url, handler404, handler500, 5 | ) 6 | 7 | # Uncomment the next two lines to enable the admin: 8 | # from django.contrib import admin 9 | # admin.autodiscover() 10 | 11 | urlpatterns = patterns( 12 | '', 13 | # Examples: 14 | # url(r'^$', 'proj.views.home', name='home'), 15 | # url(r'^proj/', include('proj.foo.urls')), 16 | 17 | # Uncomment the admin/doc line below to enable admin documentation: 18 | # url(r'^admin/doc/', include('django.contrib.admindocs.urls')), 19 | 20 | # Uncomment the next line to enable the admin: 21 | # url(r'^admin/', include(admin.site.urls)), 22 | ) 23 | -------------------------------------------------------------------------------- /extra/systemd/celery.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Celery Service 3 | After=network.target 4 | 5 | [Service] 6 | Type=forking 7 | User=celery 8 | Group=celery 9 | EnvironmentFile=-/etc/conf.d/celery 10 | WorkingDirectory=/opt/celery 11 | ExecStart=/bin/sh -c '${CELERY_BIN} multi start $CELERYD_NODES \ 12 | -A $CELERY_APP --logfile=${CELERYD_LOG_FILE} \ 13 | --pidfile=${CELERYD_PID_FILE} $CELERYD_OPTS' 14 | ExecStop=/bin/sh -c '${CELERY_BIN} multi stopwait $CELERYD_NODES \ 15 | --pidfile=${CELERYD_PID_FILE}' 16 | ExecReload=/bin/sh -c '${CELERY_BIN} multi restart $CELERYD_NODES \ 17 | -A $CELERY_APP --pidfile=${CELERYD_PID_FILE} --logfile=${CELERYD_LOG_FILE} \ 18 | --loglevel="${CELERYD_LOG_LEVEL}" $CELERYD_OPTS' 19 | 20 | [Install] 21 | WantedBy=multi-user.target 22 | -------------------------------------------------------------------------------- /celery/loaders/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | celery.loaders 4 | ~~~~~~~~~~~~~~ 5 | 6 | Loaders define how configuration is read, what happens 7 | when workers start, when tasks are executed and so on. 8 | 9 | """ 10 | from __future__ import absolute_import, unicode_literals 11 | 12 | from celery.utils.imports import symbol_by_name, import_from_cwd 13 | 14 | __all__ = ['get_loader_cls'] 15 | 16 | LOADER_ALIASES = {'app': 'celery.loaders.app:AppLoader', 17 | 'default': 'celery.loaders.default:Loader', 18 | 'django': 'djcelery.loaders:DjangoLoader'} 19 | 20 | 21 | def get_loader_cls(loader): 22 | """Get loader class by name/alias""" 23 | return symbol_by_name(loader, LOADER_ALIASES, imp=import_from_cwd) 24 | -------------------------------------------------------------------------------- /celery/security/key.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | celery.security.key 4 | ~~~~~~~~~~~~~~~~~~~ 5 | 6 | Private key for the security serializer. 7 | 8 | """ 9 | from __future__ import absolute_import, unicode_literals 10 | 11 | from kombu.utils.encoding import ensure_bytes 12 | 13 | from .utils import crypto, reraise_errors 14 | 15 | __all__ = ['PrivateKey'] 16 | 17 | 18 | class PrivateKey(object): 19 | 20 | def __init__(self, key): 21 | with reraise_errors('Invalid private key: {0!r}'): 22 | self._key = crypto.load_privatekey(crypto.FILETYPE_PEM, key) 23 | 24 | def sign(self, data, digest): 25 | """sign string containing data.""" 26 | with reraise_errors('Unable to sign data: {0!r}'): 27 | return crypto.sign(self._key, ensure_bytes(data), digest) 28 | -------------------------------------------------------------------------------- /celery/worker/consumer/heart.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from celery import bootsteps 4 | 5 | from celery.worker import heartbeat 6 | 7 | from .events import Events 8 | 9 | __all__ = ['Heart'] 10 | 11 | 12 | class Heart(bootsteps.StartStopStep): 13 | 14 | requires = (Events,) 15 | 16 | def __init__(self, c, 17 | without_heartbeat=False, heartbeat_interval=None, **kwargs): 18 | self.enabled = not without_heartbeat 19 | self.heartbeat_interval = heartbeat_interval 20 | c.heart = None 21 | 22 | def start(self, c): 23 | c.heart = heartbeat.Heart( 24 | c.timer, c.event_dispatcher, self.heartbeat_interval, 25 | ) 26 | c.heart.start() 27 | 28 | def stop(self, c): 29 | c.heart = c.heart and c.heart.stop() 30 | shutdown = stop 31 | -------------------------------------------------------------------------------- /examples/celery_http_gateway/urls.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from django.conf.urls.defaults import ( # noqa 4 | url, patterns, include, handler404, handler500, 5 | ) 6 | 7 | from djcelery import views as celery_views 8 | 9 | from celery_http_gateway.tasks import hello_world 10 | 11 | # Uncomment the next two lines to enable the admin: 12 | # from django.contrib import admin 13 | # admin.autodiscover() 14 | 15 | urlpatterns = patterns( 16 | '', 17 | url(r'^apply/(?P.+?)/', celery_views.apply), 18 | url(r'^hello/', celery_views.task_view(hello_world)), 19 | url(r'^(?P[\w\d\-]+)/done/?$', celery_views.is_task_successful, 20 | name='celery-is_task_successful'), 21 | url(r'^(?P[\w\d\-]+)/status/?$', celery_views.task_status, 22 | name='celery-task_status'), 23 | ) 24 | -------------------------------------------------------------------------------- /celery/worker/consumer/control.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from celery import bootsteps 4 | from celery.utils.log import get_logger 5 | 6 | from celery.worker import pidbox 7 | 8 | from .tasks import Tasks 9 | 10 | __all__ = ['Control'] 11 | logger = get_logger(__name__) 12 | 13 | 14 | class Control(bootsteps.StartStopStep): 15 | 16 | requires = (Tasks,) 17 | 18 | def __init__(self, c, **kwargs): 19 | self.is_green = c.pool is not None and c.pool.is_green 20 | self.box = (pidbox.gPidbox if self.is_green else pidbox.Pidbox)(c) 21 | self.start = self.box.start 22 | self.stop = self.box.stop 23 | self.shutdown = self.box.shutdown 24 | 25 | def include_if(self, c): 26 | return (c.app.conf.worker_enable_remote_control and 27 | c.conninfo.supports_exchange_type('fanout')) 28 | -------------------------------------------------------------------------------- /celery/tests/utils/test_sysinfo.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from celery.utils.sysinfo import load_average, df 4 | 5 | from celery.tests.case import Case, patch, skip 6 | 7 | 8 | @skip.unless_symbol('os.getloadavg') 9 | class test_load_average(Case): 10 | 11 | def test_avg(self): 12 | with patch('os.getloadavg') as getloadavg: 13 | getloadavg.return_value = 0.54736328125, 0.6357421875, 0.69921875 14 | l = load_average() 15 | self.assertTrue(l) 16 | self.assertEqual(l, (0.55, 0.64, 0.7)) 17 | 18 | 19 | @skip.unless_symbol('posix.statvfs_result') 20 | class test_df(Case): 21 | 22 | def test_df(self): 23 | x = df('/') 24 | self.assertTrue(x.total_blocks) 25 | self.assertTrue(x.available) 26 | self.assertTrue(x.capacity) 27 | self.assertTrue(x.stat) 28 | -------------------------------------------------------------------------------- /extra/macOS/org.celeryq.beat.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Disabled 6 | 7 | GroupName 8 | celery-beat 9 | KeepAlive 10 | 11 | Label 12 | org.celeryq.beat 13 | Program 14 | celery 15 | ProgramArguments 16 | 17 | beat 18 | --loglevel=WARNING 19 | 20 | RunAtLoad 21 | 22 | Umask 23 | 7 24 | UserName 25 | nobody 26 | WorkingDirectory 27 | / 28 | 29 | 30 | -------------------------------------------------------------------------------- /extra/macOS/org.celeryq.worker.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Disabled 6 | 7 | GroupName 8 | celery-worker 9 | KeepAlive 10 | 11 | Label 12 | org.celeryq.worker 13 | Program 14 | celery 15 | ProgramArguments 16 | 17 | worker 18 | --loglevel=WARNING 19 | 20 | RunAtLoad 21 | 22 | Umask 23 | 7 24 | UserName 25 | nobody 26 | WorkingDirectory 27 | / 28 | 29 | 30 | -------------------------------------------------------------------------------- /docs/templates/readme.txt: -------------------------------------------------------------------------------- 1 | ================================= 2 | celery - Distributed Task Queue 3 | ================================= 4 | 5 | .. image:: http://cloud.github.com/downloads/celery/celery/celery_128.png 6 | 7 | |build-status| |coverage| |bitdeli| 8 | 9 | .. include:: ../includes/introduction.txt 10 | 11 | .. include:: ../includes/installation.txt 12 | 13 | .. include:: ../includes/resources.txt 14 | 15 | .. |build-status| image:: https://secure.travis-ci.org/celery/celery.png?branch=master 16 | :alt: Build status 17 | :target: https://travis-ci.org/celery/celery 18 | 19 | .. |coverage| image:: https://codecov.io/github/celery/celery/coverage.svg?branch=master 20 | :target: https://codecov.io/github/celery/celery?branch=master 21 | 22 | .. |bitdeli| image:: https://d2weczhvl823v0.cloudfront.net/celery/celery/trend.png 23 | :alt: Bitdeli badge 24 | :target: https://bitdeli.com/free 25 | -------------------------------------------------------------------------------- /celery/concurrency/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | celery.concurrency 4 | ~~~~~~~~~~~~~~~~~~ 5 | 6 | Pool implementation abstract factory, and alias definitions. 7 | 8 | """ 9 | from __future__ import absolute_import, unicode_literals 10 | 11 | # Import from kombu directly as it's used 12 | # early in the import stage, where celery.utils loads 13 | # too much (e.g. for eventlet patching) 14 | from kombu.utils import symbol_by_name 15 | 16 | __all__ = ['get_implementation'] 17 | 18 | ALIASES = { 19 | 'prefork': 'celery.concurrency.prefork:TaskPool', 20 | 'eventlet': 'celery.concurrency.eventlet:TaskPool', 21 | 'gevent': 'celery.concurrency.gevent:TaskPool', 22 | 'solo': 'celery.concurrency.solo:TaskPool', 23 | 'processes': 'celery.concurrency.prefork:TaskPool', # XXX compat alias 24 | } 25 | 26 | 27 | def get_implementation(cls): 28 | return symbol_by_name(cls, ALIASES) 29 | -------------------------------------------------------------------------------- /celery/concurrency/solo.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | celery.concurrency.solo 4 | ~~~~~~~~~~~~~~~~~~~~~~~ 5 | 6 | Single-threaded pool implementation. 7 | 8 | """ 9 | from __future__ import absolute_import, unicode_literals 10 | 11 | import os 12 | 13 | from .base import BasePool, apply_target 14 | 15 | __all__ = ['TaskPool'] 16 | 17 | 18 | class TaskPool(BasePool): 19 | """Solo task pool (blocking, inline, fast).""" 20 | body_can_be_buffer = True 21 | 22 | def __init__(self, *args, **kwargs): 23 | super(TaskPool, self).__init__(*args, **kwargs) 24 | self.on_apply = apply_target 25 | self.limit = 1 26 | 27 | def _get_info(self): 28 | return {'max-concurrency': 1, 29 | 'processes': [os.getpid()], 30 | 'max-tasks-per-child': None, 31 | 'put-guarded-by-semaphore': True, 32 | 'timeouts': ()} 33 | -------------------------------------------------------------------------------- /celery/security/utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | celery.security.utils 4 | ~~~~~~~~~~~~~~~~~~~~~ 5 | 6 | Utilities used by the message signing serializer. 7 | 8 | """ 9 | from __future__ import absolute_import, unicode_literals 10 | 11 | import sys 12 | 13 | from contextlib import contextmanager 14 | 15 | from celery.exceptions import SecurityError 16 | from celery.five import reraise 17 | 18 | try: 19 | from OpenSSL import crypto 20 | except ImportError: # pragma: no cover 21 | crypto = None # noqa 22 | 23 | __all__ = ['reraise_errors'] 24 | 25 | 26 | @contextmanager 27 | def reraise_errors(msg='{0!r}', errors=None): 28 | assert crypto is not None 29 | errors = (crypto.Error,) if errors is None else errors 30 | try: 31 | yield 32 | except errors as exc: 33 | reraise(SecurityError, 34 | SecurityError(msg.format(exc)), 35 | sys.exc_info()[2]) 36 | -------------------------------------------------------------------------------- /celery/tests/backends/test_consul.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from celery.tests.case import AppCase, Mock, skip 4 | from celery.backends.consul import ConsulBackend 5 | 6 | 7 | @skip.unless_module('consul') 8 | class test_ConsulBackend(AppCase): 9 | 10 | def setup(self): 11 | self.backend = ConsulBackend( 12 | app=self.app, url='consul://localhost:800') 13 | 14 | def test_supports_autoexpire(self): 15 | self.assertTrue(self.backend.supports_autoexpire) 16 | 17 | def test_consul_consistency(self): 18 | self.assertEqual('consistent', self.backend.consistency) 19 | 20 | def test_get(self): 21 | index = 100 22 | data = {'Key': 'test-consul-1', 'Value': 'mypayload'} 23 | self.backend.client = Mock(name='c.client') 24 | self.backend.client.kv.get.return_value = (index, data) 25 | self.assertEqual(self.backend.get(data['Key']), 'mypayload') 26 | -------------------------------------------------------------------------------- /examples/app/myapp.py: -------------------------------------------------------------------------------- 1 | """myapp.py 2 | 3 | Usage:: 4 | 5 | (window1)$ python myapp.py worker -l info 6 | 7 | (window2)$ python 8 | >>> from myapp import add 9 | >>> add.delay(16, 16).get() 10 | 32 11 | 12 | 13 | You can also specify the app to use with the `celery` command, 14 | using the `-A` / `--app` option:: 15 | 16 | $ celery -A myapp worker -l info 17 | 18 | With the `-A myproj` argument the program will search for an app 19 | instance in the module ``myproj``. You can also specify an explicit 20 | name using the fully qualified form:: 21 | 22 | $ celery -A myapp:app worker -l info 23 | 24 | """ 25 | from __future__ import absolute_import, unicode_literals 26 | 27 | from celery import Celery 28 | 29 | app = Celery( 30 | 'myapp', 31 | broker='amqp://guest@localhost//', 32 | # ## add result backend here if needed. 33 | # backend='rpc' 34 | ) 35 | 36 | 37 | @app.task 38 | def add(x, y): 39 | return x + y 40 | 41 | if __name__ == '__main__': 42 | app.start() 43 | -------------------------------------------------------------------------------- /celery/tests/app/test_exceptions.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | import pickle 4 | 5 | from datetime import datetime 6 | 7 | from celery.exceptions import Reject, Retry 8 | 9 | from celery.tests.case import AppCase 10 | 11 | 12 | class test_Retry(AppCase): 13 | 14 | def test_when_datetime(self): 15 | x = Retry('foo', KeyError(), when=datetime.utcnow()) 16 | self.assertTrue(x.humanize()) 17 | 18 | def test_pickleable(self): 19 | x = Retry('foo', KeyError(), when=datetime.utcnow()) 20 | self.assertTrue(pickle.loads(pickle.dumps(x))) 21 | 22 | 23 | class test_Reject(AppCase): 24 | 25 | def test_attrs(self): 26 | x = Reject('foo', requeue=True) 27 | self.assertEqual(x.reason, 'foo') 28 | self.assertTrue(x.requeue) 29 | 30 | def test_repr(self): 31 | self.assertTrue(repr(Reject('foo', True))) 32 | 33 | def test_pickleable(self): 34 | x = Retry('foo', True) 35 | self.assertTrue(pickle.loads(pickle.dumps(x))) 36 | -------------------------------------------------------------------------------- /docs/copyright.rst: -------------------------------------------------------------------------------- 1 | Copyright 2 | ========= 3 | 4 | *Celery User Manual* 5 | 6 | by Ask Solem 7 | 8 | .. |copy| unicode:: U+000A9 .. COPYRIGHT SIGN 9 | 10 | Copyright |copy| 2009-2016, Ask Solem. 11 | 12 | All rights reserved. This material may be copied or distributed only 13 | subject to the terms and conditions set forth in the `Creative Commons 14 | Attribution-ShareAlike 4.0 International` 15 | `_ license. 16 | 17 | You may share and adapt the material, even for commercial purposes, but 18 | you must give the original author credit. 19 | If you alter, transform, or build upon this 20 | work, you may distribute the resulting work only under the same license or 21 | a license compatible to this one. 22 | 23 | .. note:: 24 | 25 | While the *Celery* documentation is offered under the 26 | Creative Commons *Attribution-ShareAlike 4.0 International* license 27 | the Celery *software* is offered under the 28 | `BSD License (3 Clause) `_ 29 | -------------------------------------------------------------------------------- /extra/release/attribution.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import absolute_import, unicode_literals 3 | 4 | import fileinput 5 | 6 | from pprint import pprint 7 | 8 | 9 | def author(line): 10 | try: 11 | A, E = line.strip().rsplit(None, 1) 12 | E.replace('>', '').replace('<', '') 13 | except ValueError: 14 | A, E = line.strip(), None 15 | return A.lower() if A else A, E.lower() if E else E 16 | 17 | 18 | def proper_name(name): 19 | return name and ' ' in name 20 | 21 | 22 | def find_missing_authors(seen): 23 | with open('AUTHORS') as authors: 24 | known = [author(line) for line in authors.readlines()] 25 | 26 | seen_authors = {t[0] for t in seen if proper_name(t[0])} 27 | known_authors = {t[0] for t in known} 28 | # maybe later?: 29 | # seen_emails = {t[1] for t in seen} 30 | # known_emails = {t[1] for t in known} 31 | 32 | pprint(seen_authors - known_authors) 33 | 34 | 35 | if __name__ == '__main__': 36 | find_missing_authors([author(line) for line in fileinput.input()]) 37 | -------------------------------------------------------------------------------- /celery/worker/consumer/connection.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from kombu.common import ignore_errors 4 | 5 | from celery import bootsteps 6 | from celery.utils.log import get_logger 7 | 8 | __all__ = ['Connection'] 9 | logger = get_logger(__name__) 10 | info = logger.info 11 | 12 | 13 | class Connection(bootsteps.StartStopStep): 14 | 15 | def __init__(self, c, **kwargs): 16 | c.connection = None 17 | 18 | def start(self, c): 19 | c.connection = c.connect() 20 | info('Connected to %s', c.connection.as_uri()) 21 | 22 | def shutdown(self, c): 23 | # We must set self.connection to None here, so 24 | # that the green pidbox thread exits. 25 | connection, c.connection = c.connection, None 26 | if connection: 27 | ignore_errors(connection, connection.close) 28 | 29 | def info(self, c, params='N/A'): 30 | if c.connection: 31 | params = c.connection.info() 32 | params.pop('password', None) # don't send password. 33 | return {'broker': params} 34 | -------------------------------------------------------------------------------- /docs/reference/celery.utils.debug.rst: -------------------------------------------------------------------------------- 1 | ==================================== 2 | ``celery.utils.debug`` 3 | ==================================== 4 | 5 | .. contents:: 6 | :local: 7 | 8 | Sampling Memory Usage 9 | ===================== 10 | 11 | This module can be used to diagnose and sample the memory usage 12 | used by parts of your application. 13 | 14 | E.g to sample the memory usage of calling tasks you can do this: 15 | 16 | .. code-block:: python 17 | 18 | 19 | from celery.utils.debug import sample_mem, memdump 20 | 21 | from tasks import add 22 | 23 | 24 | try: 25 | for i in range(100): 26 | for j in range(100): 27 | add.delay(i, j) 28 | sample_mem() 29 | finally: 30 | memdump() 31 | 32 | 33 | API Reference 34 | ============= 35 | 36 | .. currentmodule:: celery.utils.debug 37 | 38 | .. automodule:: celery.utils.debug 39 | 40 | .. autofunction:: sample_mem 41 | 42 | .. autofunction:: memdump 43 | 44 | .. autofunction:: sample 45 | 46 | .. autofunction:: mem_rss 47 | 48 | .. autofunction:: ps 49 | -------------------------------------------------------------------------------- /celery/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | celery.utils 4 | ~~~~~~~~~~~~ 5 | 6 | Utility functions. 7 | 8 | Do not import from here directly anymore, these are only 9 | here for backwards compatibility. 10 | 11 | """ 12 | from __future__ import absolute_import, print_function, unicode_literals 13 | 14 | import sys 15 | 16 | from .functional import memoize # noqa 17 | 18 | from .nodenames import worker_direct, nodename, nodesplit 19 | 20 | __all__ = ['worker_direct', 'gen_task_name', 'nodename', 'nodesplit', 21 | 'cached_property', 'uuid'] 22 | 23 | PY3 = sys.version_info[0] == 3 24 | 25 | 26 | # ------------------------------------------------------------------------ # 27 | # > XXX Compat 28 | from .log import LOG_LEVELS # noqa 29 | from .imports import ( # noqa 30 | qualname as get_full_cls_name, symbol_by_name as get_cls_by_name, 31 | instantiate, import_from_cwd, gen_task_name, 32 | ) 33 | from .functional import chunks, noop # noqa 34 | from kombu.utils import cached_property, uuid # noqa 35 | gen_unique_id = uuid 36 | -------------------------------------------------------------------------------- /extra/supervisord/supervisord.conf: -------------------------------------------------------------------------------- 1 | [unix_http_server] 2 | file=/tmp/supervisor.sock ; path to your socket file 3 | 4 | [supervisord] 5 | logfile=/var/log/supervisord/supervisord.log ; supervisord log file 6 | logfile_maxbytes=50MB ; maximum size of logfile before rotation 7 | logfile_backups=10 ; number of backed up logfiles 8 | loglevel=info ; info, debug, warn, trace 9 | pidfile=/var/run/supervisord.pid ; pidfile location 10 | nodaemon=false ; run supervisord as a daemon 11 | minfds=1024 ; number of startup file descriptors 12 | minprocs=200 ; number of process descriptors 13 | user=root ; default user 14 | childlogdir=/var/log/supervisord/ ; where child log files will live 15 | 16 | 17 | [rpcinterface:supervisor] 18 | supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface 19 | 20 | [supervisorctl] 21 | serverurl=unix:///tmp/supervisor.sock ; use unix:// schem for a unix sockets. 22 | 23 | 24 | [include] 25 | 26 | # Uncomment this line for celeryd for Python 27 | ;files=celeryd.conf 28 | 29 | -------------------------------------------------------------------------------- /celery/tests/security/test_key.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from celery.exceptions import SecurityError 4 | from celery.security.key import PrivateKey 5 | 6 | from . import CERT1, KEY1, KEY2 7 | from .case import SecurityCase 8 | 9 | 10 | class test_PrivateKey(SecurityCase): 11 | 12 | def test_valid_private_key(self): 13 | PrivateKey(KEY1) 14 | PrivateKey(KEY2) 15 | 16 | def test_invalid_private_key(self): 17 | with self.assertRaises((SecurityError, TypeError)): 18 | PrivateKey(None) 19 | with self.assertRaises(SecurityError): 20 | PrivateKey('') 21 | with self.assertRaises(SecurityError): 22 | PrivateKey('foo') 23 | with self.assertRaises(SecurityError): 24 | PrivateKey(KEY1[:20] + KEY1[21:]) 25 | with self.assertRaises(SecurityError): 26 | PrivateKey(CERT1) 27 | 28 | def test_sign(self): 29 | pkey = PrivateKey(KEY1) 30 | pkey.sign('test', b'sha1') 31 | with self.assertRaises(ValueError): 32 | pkey.sign('test', b'unknown') 33 | -------------------------------------------------------------------------------- /extra/supervisord/celeryd.conf: -------------------------------------------------------------------------------- 1 | ; ================================== 2 | ; celery worker supervisor example 3 | ; ================================== 4 | 5 | [program:celery] 6 | ; Set full path to celery program if using virtualenv 7 | command=celery worker -A proj --loglevel=INFO 8 | 9 | ; Alternatively, 10 | ;command=celery --app=your_app.celery:app worker --loglevel=INFO -n worker.%%h 11 | ; Or run a script 12 | ;command=celery.sh 13 | 14 | directory=/path/to/project 15 | user=nobody 16 | numprocs=1 17 | stdout_logfile=/var/log/celery/worker.log 18 | stderr_logfile=/var/log/celery/worker.log 19 | autostart=true 20 | autorestart=true 21 | startsecs=10 22 | 23 | ; Need to wait for currently executing tasks to finish at shutdown. 24 | ; Increase this if you have very long running tasks. 25 | stopwaitsecs = 600 26 | 27 | ; When resorting to send SIGKILL to the program to terminate it 28 | ; send SIGKILL to its whole process group instead, 29 | ; taking care of its children as well. 30 | killasgroup=true 31 | 32 | ; Set Celery priority higher than default (999) 33 | ; so, if rabbitmq is supervised, it will start first. 34 | priority=1000 35 | -------------------------------------------------------------------------------- /docs/community.rst: -------------------------------------------------------------------------------- 1 | .. _community: 2 | 3 | ======================= 4 | Community Resources 5 | ======================= 6 | 7 | This is a list of external blog posts, tutorials and slides related 8 | to Celery. If you have a link that's missing from this list, please 9 | contact the mailing-list or submit a patch. 10 | 11 | .. contents:: 12 | :local: 13 | 14 | .. _community-resources: 15 | 16 | Resources 17 | ========= 18 | 19 | .. _res-using-celery: 20 | 21 | Who's using Celery 22 | ------------------ 23 | 24 | http://wiki.github.com/celery/celery/using 25 | 26 | .. _res-wiki: 27 | 28 | Wiki 29 | ---- 30 | 31 | http://wiki.github.com/celery/celery/ 32 | 33 | .. _res-stackoverflow: 34 | 35 | Celery questions on Stack Overflow 36 | ---------------------------------- 37 | 38 | http://stackoverflow.com/search?q=celery&tab=newest 39 | 40 | .. _res-mailing-list-archive: 41 | 42 | Mailing-list Archive: celery-users 43 | ---------------------------------- 44 | 45 | http://blog.gmane.org/gmane.comp.python.amqp.celery.user 46 | 47 | .. _res-irc-logs: 48 | 49 | .. _community-news: 50 | 51 | News 52 | ==== 53 | 54 | This section has moved to the Celery homepage: 55 | http://celeryproject.org/community/ 56 | -------------------------------------------------------------------------------- /celery/utils/sysinfo.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import absolute_import, unicode_literals 3 | 4 | import os 5 | 6 | from math import ceil 7 | 8 | from kombu.utils import cached_property 9 | 10 | __all__ = ['load_average', 'df'] 11 | 12 | 13 | if hasattr(os, 'getloadavg'): 14 | 15 | def load_average(): 16 | return tuple(ceil(l * 1e2) / 1e2 for l in os.getloadavg()) 17 | 18 | else: # pragma: no cover 19 | # Windows doesn't have getloadavg 20 | def load_average(): # noqa 21 | return (0.0, 0.0, 0.0) 22 | 23 | 24 | class df(object): 25 | 26 | def __init__(self, path): 27 | self.path = path 28 | 29 | @property 30 | def total_blocks(self): 31 | return self.stat.f_blocks * self.stat.f_frsize / 1024 32 | 33 | @property 34 | def available(self): 35 | return self.stat.f_bavail * self.stat.f_frsize / 1024 36 | 37 | @property 38 | def capacity(self): 39 | avail = self.stat.f_bavail 40 | used = self.stat.f_blocks - self.stat.f_bfree 41 | return int(ceil(used * 100.0 / (used + avail) + 0.5)) 42 | 43 | @cached_property 44 | def stat(self): 45 | return os.statvfs(os.path.abspath(self.path)) 46 | -------------------------------------------------------------------------------- /celery/tests/compat_modules/test_decorators.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | import warnings 4 | 5 | from celery.task import base 6 | 7 | from celery.tests.case import AppCase, depends_on_current_app 8 | 9 | 10 | def add(x, y): 11 | return x + y 12 | 13 | 14 | @depends_on_current_app 15 | class test_decorators(AppCase): 16 | 17 | def test_task_alias(self): 18 | from celery import task 19 | self.assertTrue(task.__file__) 20 | self.assertTrue(task(add)) 21 | 22 | def setup(self): 23 | with warnings.catch_warnings(record=True): 24 | from celery import decorators 25 | self.decorators = decorators 26 | 27 | def assertCompatDecorator(self, decorator, type, **opts): 28 | task = decorator(**opts)(add) 29 | self.assertEqual(task(8, 8), 16) 30 | self.assertIsInstance(task, type) 31 | 32 | def test_task(self): 33 | self.assertCompatDecorator(self.decorators.task, base.BaseTask) 34 | 35 | def test_periodic_task(self): 36 | self.assertCompatDecorator(self.decorators.periodic_task, 37 | base.BaseTask, 38 | run_every=1) 39 | -------------------------------------------------------------------------------- /celery/tests/utils/test_utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from celery.utils import chunks, cached_property 4 | 5 | from celery.tests.case import Case 6 | 7 | 8 | class test_chunks(Case): 9 | 10 | def test_chunks(self): 11 | 12 | # n == 2 13 | x = chunks(iter([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), 2) 14 | self.assertListEqual( 15 | list(x), 16 | [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10]], 17 | ) 18 | 19 | # n == 3 20 | x = chunks(iter([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), 3) 21 | self.assertListEqual( 22 | list(x), 23 | [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10]], 24 | ) 25 | 26 | # n == 2 (exact) 27 | x = chunks(iter([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]), 2) 28 | self.assertListEqual( 29 | list(x), 30 | [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9]], 31 | ) 32 | 33 | 34 | class test_utils(Case): 35 | 36 | def test_cached_property(self): 37 | 38 | def fun(obj): 39 | return fun.value 40 | 41 | x = cached_property(fun) 42 | self.assertIs(x.__get__(None), x) 43 | self.assertIs(x.__set__(None, None), x) 44 | self.assertIs(x.__delete__(None), x) 45 | -------------------------------------------------------------------------------- /docs/reference/celery.app.amqp.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: celery.app.amqp 2 | 3 | .. automodule:: celery.app.amqp 4 | 5 | .. contents:: 6 | :local: 7 | 8 | AMQP 9 | ---- 10 | 11 | .. autoclass:: AMQP 12 | 13 | .. attribute:: Connection 14 | 15 | Broker connection class used. Default is 16 | :class:`kombu.Connection`. 17 | 18 | .. attribute:: Consumer 19 | 20 | Base Consumer class used. Default is :class:`kombu.Consumer`. 21 | 22 | .. attribute:: Producer 23 | 24 | Base Producer class used. Default is :class:`kombu.Producer`. 25 | 26 | .. attribute:: queues 27 | 28 | All currently defined task queues. (A :class:`Queues` instance). 29 | 30 | .. automethod:: Queues 31 | .. automethod:: Router 32 | .. automethod:: flush_routes 33 | 34 | .. autoattribute:: create_task_message 35 | .. autoattribute:: send_task_message 36 | .. autoattribute:: default_queue 37 | .. autoattribute:: default_exchange 38 | .. autoattribute:: producer_pool 39 | .. autoattribute:: router 40 | .. autoattribute:: routes 41 | 42 | Queues 43 | ------ 44 | 45 | .. autoclass:: Queues 46 | :members: 47 | :undoc-members: 48 | -------------------------------------------------------------------------------- /examples/django/proj/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for proj project. 3 | 4 | This module contains the WSGI application used by Django's development server 5 | and any production WSGI deployments. It should expose a module-level variable 6 | named ``application``. Django's ``runserver`` and ``runfcgi`` commands discover 7 | this application via the ``WSGI_APPLICATION`` setting. 8 | 9 | Usually you will have the standard Django WSGI application here, but it also 10 | might make sense to replace the whole Django WSGI application with a custom one 11 | that later delegates to the Django one. For example, you could introduce WSGI 12 | middleware here, or combine a Django application with an application of another 13 | framework. 14 | 15 | """ 16 | from __future__ import absolute_import, unicode_literals 17 | 18 | import os 19 | 20 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'proj.settings') 21 | 22 | # This application object is used by any WSGI server configured to use this 23 | # file. This includes Django's development server, if the WSGI_APPLICATION 24 | # setting points here. 25 | from django.core.wsgi import get_wsgi_application # noqa 26 | application = get_wsgi_application() 27 | 28 | # Apply WSGI middleware here. 29 | # from helloworld.wsgi import HelloWorldApplication 30 | # application = HelloWorldApplication(application) 31 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | sudo: false 3 | cache: false 4 | python: 5 | - '3.5' 6 | os: 7 | - linux 8 | - osx 9 | env: 10 | global: 11 | PYTHONUNBUFFERED=yes 12 | matrix: 13 | - TOXENV=2.7 14 | - TOXENV=3.4 15 | - TOXENV=3.5 16 | - TOXENV=pypy PYPY_VERSION="5.3" 17 | - TOXENV=pypy3 18 | - TOXENV=flake8 19 | - TOXENV=flakeplus 20 | - TOXENV=apicheck 21 | - TOXENV=configcheck 22 | before_install: 23 | - | 24 | if [ "$TOXENV" = "pypy" ]; then 25 | export PYENV_ROOT="$HOME/.pyenv" 26 | if [ -f "$PYENV_ROOT/bin/pyenv" ]; then 27 | cd "$PYENV_ROOT" && git pull 28 | else 29 | rm -rf "$PYENV_ROOT" && git clone --depth 1 https://github.com/yyuu/pyenv.git "$PYENV_ROOT" 30 | fi 31 | "$PYENV_ROOT/bin/pyenv" install "pypy-$PYPY_VERSION" 32 | virtualenv --python="$PYENV_ROOT/versions/pypy-$PYPY_VERSION/bin/python" "$HOME/virtualenvs/pypy-$PYPY_VERSION" 33 | source "$HOME/virtualenvs/pypy-$PYPY_VERSION/bin/activate" 34 | fi 35 | install: travis_retry pip install -U tox 36 | script: tox -v -- -v 37 | after_success: 38 | - .tox/$TRAVIS_PYTHON_VERSION/bin/coverage xml 39 | - .tox/$TRAVIS_PYTHON_VERSION/bin/codecov -e TOXENV 40 | notifications: 41 | irc: 42 | channels: 43 | - "chat.freenode.net#celery" 44 | on_success: change 45 | on_failure: change 46 | -------------------------------------------------------------------------------- /celery/tests/compat_modules/test_compat_utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | import celery 4 | 5 | from celery.app.task import Task as ModernTask 6 | from celery.task.base import Task as CompatTask 7 | 8 | from celery.tests.case import AppCase, depends_on_current_app 9 | 10 | 11 | @depends_on_current_app 12 | class test_MagicModule(AppCase): 13 | 14 | def test_class_property_set_without_type(self): 15 | self.assertTrue(ModernTask.__dict__['app'].__get__(CompatTask())) 16 | 17 | def test_class_property_set_on_class(self): 18 | self.assertIs(ModernTask.__dict__['app'].__set__(None, None), 19 | ModernTask.__dict__['app']) 20 | 21 | def test_class_property_set(self): 22 | 23 | class X(CompatTask): 24 | pass 25 | ModernTask.__dict__['app'].__set__(X(), self.app) 26 | self.assertIs(X.app, self.app) 27 | 28 | def test_dir(self): 29 | self.assertTrue(dir(celery.messaging)) 30 | 31 | def test_direct(self): 32 | self.assertTrue(celery.task) 33 | 34 | def test_app_attrs(self): 35 | self.assertEqual(celery.task.control.broadcast, 36 | celery.current_app.control.broadcast) 37 | 38 | def test_decorators_task(self): 39 | @celery.decorators.task 40 | def _test_decorators_task(): 41 | pass 42 | 43 | def test_decorators_periodic_task(self): 44 | @celery.decorators.periodic_task(run_every=3600) 45 | def _test_decorators_ptask(): 46 | pass 47 | -------------------------------------------------------------------------------- /examples/celery_http_gateway/README.rst: -------------------------------------------------------------------------------- 1 | ============================== 2 | Example Celery->HTTP Gateway 3 | ============================== 4 | 5 | This is an example service exposing the ability to apply tasks and query 6 | statuses/results over HTTP. 7 | 8 | Some familiarity with Django is recommended. 9 | 10 | `settings.py` contains the celery settings, you probably want to configure 11 | at least the broker related settings. 12 | 13 | To run the service you have to run the following commands:: 14 | 15 | $ python manage.py syncdb # (if running the database backend) 16 | 17 | $ python manage.py runserver 18 | 19 | 20 | The service is now running at http://localhost:8000 21 | 22 | 23 | You can apply tasks, with the `/apply/` URL:: 24 | 25 | $ curl http://localhost:8000/apply/celery.ping/ 26 | {"ok": "true", "task_id": "e3a95109-afcd-4e54-a341-16c18fddf64b"} 27 | 28 | Then you can use the resulting task-id to get the return value:: 29 | 30 | $ curl http://localhost:8000/e3a95109-afcd-4e54-a341-16c18fddf64b/status/ 31 | {"task": {"status": "SUCCESS", "result": "pong", "id": "e3a95109-afcd-4e54-a341-16c18fddf64b"}} 32 | 33 | 34 | If you don't want to expose all tasks there are a few possible 35 | approaches. For instance you can extend the `apply` view to only 36 | accept a white-list. Another possibility is to just make views for every task you want to 37 | expose. We made on such view for ping in `views.ping`:: 38 | 39 | $ curl http://localhost:8000/ping/ 40 | {"ok": "true", "task_id": "383c902c-ba07-436b-b0f3-ea09cc22107c"} 41 | -------------------------------------------------------------------------------- /requirements/README.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | pip requirements files 3 | ======================== 4 | 5 | 6 | Index 7 | ===== 8 | 9 | * :file:`requirements/default.txt` 10 | 11 | Default requirements for Python 2.7+. 12 | 13 | * :file:`requirements/jython.txt` 14 | 15 | Extra requirements needed to run on Jython 2.5 16 | 17 | * :file:`requirements/security.txt` 18 | 19 | Extra requirements needed to use the message signing serializer, 20 | see the Security Guide. 21 | 22 | * :file:`requirements/test.txt` 23 | 24 | Requirements needed to run the full unittest suite. 25 | 26 | * :file:`requirements/test-ci-base.txt` 27 | 28 | Extra test requirements required by the CI suite (Tox). 29 | 30 | * :file:`requirements/test-ci-default.txt` 31 | 32 | Extra test requirements required for Python 2.7 by the CI suite (Tox). 33 | 34 | * :file:`requirements/doc.txt` 35 | 36 | Extra requirements required to build the Sphinx documentation. 37 | 38 | * :file:`requirements/pkgutils.txt` 39 | 40 | Extra requirements required to perform package distribution maintenance. 41 | 42 | * :file:`requirements/dev.txt` 43 | 44 | Requirement file installing the current master branch of Celery and deps. 45 | 46 | Examples 47 | ======== 48 | 49 | Installing requirements 50 | ----------------------- 51 | 52 | :: 53 | 54 | $ pip install -U -r requirements/default.txt 55 | 56 | 57 | Running the tests 58 | ----------------- 59 | 60 | :: 61 | 62 | $ pip install -U -r requirements/default.txt 63 | $ pip install -U -r requirements/test.txt 64 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | environment: 2 | 3 | global: 4 | # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the 5 | # /E:ON and /V:ON options are not enabled in the batch script intepreter 6 | # See: http://stackoverflow.com/a/13751649/163740 7 | WITH_COMPILER: "cmd /E:ON /V:ON /C .\\extra\\appveyor\\run_with_compiler.cmd" 8 | 9 | matrix: 10 | 11 | # Pre-installed Python versions, which Appveyor may upgrade to 12 | # a later point release. 13 | # See: http://www.appveyor.com/docs/installed-software#python 14 | 15 | - PYTHON: "C:\\Python27" 16 | PYTHON_VERSION: "2.7.x" 17 | PYTHON_ARCH: "32" 18 | 19 | - PYTHON: "C:\\Python34" 20 | PYTHON_VERSION: "3.4.x" 21 | PYTHON_ARCH: "32" 22 | 23 | - PYTHON: "C:\\Python27-x64" 24 | PYTHON_VERSION: "2.7.x" 25 | PYTHON_ARCH: "64" 26 | WINDOWS_SDK_VERSION: "v7.0" 27 | 28 | - PYTHON: "C:\\Python34-x64" 29 | PYTHON_VERSION: "3.4.x" 30 | PYTHON_ARCH: "64" 31 | WINDOWS_SDK_VERSION: "v7.1" 32 | 33 | 34 | init: 35 | - "ECHO %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH%" 36 | 37 | install: 38 | - "powershell extra\\appveyor\\install.ps1" 39 | - "%PYTHON%/Scripts/pip.exe install -U setuptools" 40 | - "%PYTHON%/Scripts/pip.exe install -r requirements/dev.txt" 41 | 42 | build: off 43 | 44 | test_script: 45 | - "%WITH_COMPILER% %PYTHON%/python setup.py test" 46 | 47 | after_test: 48 | - "%WITH_COMPILER% %PYTHON%/python setup.py bdist_wheel" 49 | 50 | artifacts: 51 | - path: dist\* 52 | 53 | #on_success: 54 | # - TODO: upload the content of dist/*.whl to a public wheelhouse 55 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | ================================= 2 | Celery - Distributed Task Queue 3 | ================================= 4 | 5 | Celery is a simple, flexible and reliable distributed system to 6 | process vast amounts of messages, while providing operations with 7 | the tools required to maintain such a system. 8 | 9 | It's a task queue with focus on real-time processing, while also 10 | supporting task scheduling. 11 | 12 | Celery has a large and diverse community of users and contributors, 13 | you should come join us :ref:`on IRC ` 14 | or :ref:`our mailing-list `. 15 | 16 | Celery is Open Source and licensed under the `BSD License`_. 17 | 18 | Getting Started 19 | =============== 20 | 21 | - If you are new to Celery you can get started by following 22 | the :ref:`first-steps` tutorial. 23 | 24 | - You can also check out the :ref:`FAQ `. 25 | 26 | .. _`BSD License`: http://www.opensource.org/licenses/BSD-3-Clause 27 | 28 | Contents 29 | ======== 30 | 31 | .. toctree:: 32 | :maxdepth: 1 33 | 34 | copyright 35 | 36 | .. toctree:: 37 | :maxdepth: 2 38 | 39 | getting-started/index 40 | userguide/index 41 | 42 | .. toctree:: 43 | :maxdepth: 1 44 | 45 | configuration 46 | django/index 47 | contributing 48 | community 49 | tutorials/index 50 | faq 51 | changelog 52 | whatsnew-4.0 53 | whatsnew-3.1 54 | reference/index 55 | internals/index 56 | history/index 57 | glossary 58 | 59 | 60 | Indices and tables 61 | ================== 62 | 63 | * :ref:`genindex` 64 | * :ref:`modindex` 65 | * :ref:`search` 66 | 67 | -------------------------------------------------------------------------------- /docs/internals/worker.rst: -------------------------------------------------------------------------------- 1 | .. _internals-worker: 2 | 3 | ======================= 4 | Internals: The worker 5 | ======================= 6 | 7 | .. contents:: 8 | :local: 9 | 10 | Introduction 11 | ============ 12 | 13 | The worker consists of 4 main components: the consumer, the scheduler, 14 | the mediator and the task pool. All these components runs in parallel working 15 | with two data structures: the ready queue and the ETA schedule. 16 | 17 | Data structures 18 | =============== 19 | 20 | timer 21 | ----- 22 | 23 | The timer uses :mod:`heapq` to schedule internal functions. 24 | It's very efficient and can handle hundred of thousands of entries. 25 | 26 | 27 | Components 28 | ========== 29 | 30 | Consumer 31 | -------- 32 | 33 | Receives messages from the broker using `Kombu`_. 34 | 35 | .. _`Kombu`: http://pypi.python.org/pypi/kombu 36 | 37 | When a message is received it's converted into a 38 | :class:`celery.worker.request.Request` object. 39 | 40 | Tasks with an ETA, or rate-limit are entered into the `timer`, 41 | messages that can be immediately processed are sent to the execution pool. 42 | 43 | Timer 44 | ----- 45 | 46 | The timer schedules internal functions, like cleanup and internal monitoring, 47 | but also it schedules ETA tasks and rate limited tasks. 48 | If the scheduled tasks eta has passed it is moved to the execution pool. 49 | 50 | TaskPool 51 | -------- 52 | 53 | This is a slightly modified :class:`multiprocessing.Pool`. 54 | It mostly works the same way, except it makes sure all of the workers 55 | are running at all times. If a worker is missing, it replaces 56 | it with a new one. 57 | -------------------------------------------------------------------------------- /docs/includes/resources.txt: -------------------------------------------------------------------------------- 1 | .. _getting-help: 2 | 3 | Getting Help 4 | ============ 5 | 6 | .. _mailing-list: 7 | 8 | Mailing list 9 | ------------ 10 | 11 | For discussions about the usage, development, and future of celery, 12 | please join the `celery-users`_ mailing list. 13 | 14 | .. _`celery-users`: http://groups.google.com/group/celery-users/ 15 | 16 | .. _irc-channel: 17 | 18 | IRC 19 | --- 20 | 21 | Come chat with us on IRC. The **#celery** channel is located at the `Freenode`_ 22 | network. 23 | 24 | .. _`Freenode`: http://freenode.net 25 | 26 | .. _bug-tracker: 27 | 28 | Bug tracker 29 | =========== 30 | 31 | If you have any suggestions, bug reports or annoyances please report them 32 | to our issue tracker at https://github.com/celery/celery/issues/ 33 | 34 | .. _wiki: 35 | 36 | Wiki 37 | ==== 38 | 39 | http://wiki.github.com/celery/celery/ 40 | 41 | .. _contributing-short: 42 | 43 | Contributing 44 | ============ 45 | 46 | Development of `celery` happens at GitHub: https://github.com/celery/celery 47 | 48 | You are highly encouraged to participate in the development 49 | of `celery`. If you don't like GitHub (for some reason) you're welcome 50 | to send regular patches. 51 | 52 | Be sure to also read the `Contributing to Celery`_ section in the 53 | documentation. 54 | 55 | .. _`Contributing to Celery`: 56 | http://docs.celeryproject.org/en/master/contributing.html 57 | 58 | .. _license: 59 | 60 | License 61 | ======= 62 | 63 | This software is licensed under the `New BSD License`. See the :file:`LICENSE` 64 | file in the top distribution directory for the full license text. 65 | 66 | .. # vim: syntax=rst expandtab tabstop=4 shiftwidth=4 shiftround 67 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 2.7,pypy,3.4,3.5,pypy3,flake8,flakeplus,apicheck,configcheck 3 | 4 | [testenv] 5 | deps= 6 | -r{toxinidir}/requirements/default.txt 7 | -r{toxinidir}/requirements/test.txt 8 | 9 | 2.7: -r{toxinidir}/requirements/test-ci-default.txt 10 | 3.4,3.5: -r{toxinidir}/requirements/test-ci-default.txt 11 | pypy,pypy3: -r{toxinidir}/requirements/test-ci-base.txt 12 | 13 | linkcheck,apicheck,configcheck: -r{toxinidir}/requirements/docs.txt 14 | flake8,flakeplus: -r{toxinidir}/requirements/pkgutils.txt 15 | sitepackages = False 16 | recreate = False 17 | commands = pip install -U -r{toxinidir}/requirements/dev.txt 18 | nosetests -xsv --with-coverage \ 19 | --cover-inclusive --cover-min-percentage=92 --cover-erase [] 20 | 21 | basepython = 22 | 2.7,flake8,flakeplus,apicheck,linkcheck,configcheck: python2.7 23 | 3.4: python3.4 24 | 3.5: python3.5 25 | pypy: pypy 26 | pypy3: pypy3 27 | 28 | [testenv:apicheck] 29 | commands = 30 | pip install -U -r{toxinidir}/requirements/dev.txt 31 | sphinx-build -b apicheck -d {envtmpdir}/doctrees docs docs/_build/apicheck 32 | 33 | [testenv:configcheck] 34 | commands = 35 | pip install -U -r{toxinidir}/requirements/dev.txt 36 | sphinx-build -b configcheck -d {envtmpdir}/doctrees docs docs/_build/configcheck 37 | 38 | [testenv:linkcheck] 39 | commands = 40 | pip install -U -r{toxinidir}/requirements/dev.txt 41 | sphinx-build -W -b linkcheck -d {envtmpdir}/doctrees docs docs/_build/linkcheck 42 | 43 | [testenv:flake8] 44 | commands = 45 | flake8 --ignore=X999 {toxinidir}/celery 46 | 47 | [testenv:flakeplus] 48 | commands = 49 | flakeplus --2.7 {toxinidir}/celery 50 | -------------------------------------------------------------------------------- /docs/reference/index.rst: -------------------------------------------------------------------------------- 1 | .. _apiref: 2 | 3 | =============== 4 | API Reference 5 | =============== 6 | 7 | :Release: |version| 8 | :Date: |today| 9 | 10 | .. toctree:: 11 | :maxdepth: 1 12 | 13 | celery 14 | celery.app 15 | celery.app.task 16 | celery.app.amqp 17 | celery.app.defaults 18 | celery.app.control 19 | celery.app.registry 20 | celery.app.builtins 21 | celery.app.log 22 | celery.app.utils 23 | celery.bootsteps 24 | celery.result 25 | celery.schedules 26 | celery.signals 27 | celery.security 28 | celery.utils.debug 29 | celery.exceptions 30 | celery.loaders 31 | celery.loaders.app 32 | celery.loaders.default 33 | celery.loaders.base 34 | celery.states 35 | celery.contrib.abortable 36 | celery.contrib.migrate 37 | celery.contrib.sphinx 38 | celery.contrib.rdb 39 | celery.events 40 | celery.events.state 41 | celery.beat 42 | celery.apps.worker 43 | celery.apps.beat 44 | celery.worker 45 | celery.worker.request 46 | celery.worker.state 47 | celery.worker.strategy 48 | celery.worker.consumer 49 | celery.worker.consumer.agent 50 | celery.worker.consumer.connection 51 | celery.worker.consumer.consumer 52 | celery.worker.consumer.control 53 | celery.worker.consumer.events 54 | celery.worker.consumer.gossip 55 | celery.worker.consumer.heart 56 | celery.worker.consumer.mingle 57 | celery.worker.consumer.tasks 58 | celery.bin.base 59 | celery.bin.celery 60 | celery.bin.worker 61 | celery.bin.beat 62 | celery.bin.events 63 | celery.bin.logtool 64 | celery.bin.amqp 65 | celery.bin.multi 66 | celery.bin.graph 67 | -------------------------------------------------------------------------------- /examples/eventlet/README.rst: -------------------------------------------------------------------------------- 1 | ================================== 2 | Example using the Eventlet Pool 3 | ================================== 4 | 5 | Introduction 6 | ============ 7 | 8 | This is a Celery application containing two example tasks. 9 | 10 | First you need to install Eventlet, and also recommended is the `dnspython` 11 | module (when this is installed all name lookups will be asynchronous):: 12 | 13 | $ pip install eventlet 14 | $ pip install dnspython 15 | $ pip install requests 16 | 17 | Before you run any of the example tasks you need to start 18 | the worker:: 19 | 20 | $ cd examples/eventlet 21 | $ celery worker -l info --concurrency=500 --pool=eventlet 22 | 23 | As usual you need to have RabbitMQ running, see the Celery getting started 24 | guide if you haven't installed it yet. 25 | 26 | Tasks 27 | ===== 28 | 29 | * `tasks.urlopen` 30 | 31 | This task simply makes a request opening the URL and returns the size 32 | of the response body:: 33 | 34 | $ cd examples/eventlet 35 | $ python 36 | >>> from tasks import urlopen 37 | >>> urlopen.delay('http://www.google.com/').get() 38 | 9980 39 | 40 | To open several URLs at once you can do:: 41 | 42 | $ cd examples/eventlet 43 | $ python 44 | >>> from tasks import urlopen 45 | >>> from celery import group 46 | >>> result = group(urlopen.s(url) 47 | ... for url in LIST_OF_URLS).apply_async() 48 | >>> for incoming_result in result.iter_native(): 49 | ... print(incoming_result) 50 | 51 | * `webcrawler.crawl` 52 | 53 | This is a simple recursive web crawler. It will only crawl 54 | URLs for the current host name. Please see comments in the 55 | `webcrawler.py` file. 56 | -------------------------------------------------------------------------------- /celery/tests/backends/test_backends.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from celery import backends 4 | from celery.backends.amqp import AMQPBackend 5 | from celery.backends.cache import CacheBackend 6 | from celery.exceptions import ImproperlyConfigured 7 | from celery.tests.case import AppCase, depends_on_current_app, patch 8 | 9 | 10 | class test_backends(AppCase): 11 | 12 | def test_get_backend_aliases(self): 13 | expects = [('amqp://', AMQPBackend), 14 | ('cache+memory://', CacheBackend)] 15 | 16 | for url, expect_cls in expects: 17 | backend, url = backends.get_backend_by_url(url, self.app.loader) 18 | self.assertIsInstance( 19 | backend(app=self.app, url=url), 20 | expect_cls, 21 | ) 22 | 23 | def test_unknown_backend(self): 24 | with self.assertRaises(ImportError): 25 | backends.get_backend_cls('fasodaopjeqijwqe', self.app.loader) 26 | 27 | @depends_on_current_app 28 | def test_default_backend(self): 29 | self.assertEqual(backends.default_backend, self.app.backend) 30 | 31 | def test_backend_by_url(self, url='redis://localhost/1'): 32 | from celery.backends.redis import RedisBackend 33 | backend, url_ = backends.get_backend_by_url(url, self.app.loader) 34 | self.assertIs(backend, RedisBackend) 35 | self.assertEqual(url_, url) 36 | 37 | def test_sym_raises_ValuError(self): 38 | with patch('celery.backends.symbol_by_name') as sbn: 39 | sbn.side_effect = ValueError() 40 | with self.assertRaises(ImproperlyConfigured): 41 | backends.get_backend_cls('xxx.xxx:foo', self.app.loader) 42 | -------------------------------------------------------------------------------- /celery/tests/contrib/test_abortable.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from celery.contrib.abortable import AbortableTask, AbortableAsyncResult 4 | from celery.tests.case import AppCase 5 | 6 | 7 | class test_AbortableTask(AppCase): 8 | 9 | def setup(self): 10 | 11 | @self.app.task(base=AbortableTask, shared=False) 12 | def abortable(): 13 | return True 14 | self.abortable = abortable 15 | 16 | def test_async_result_is_abortable(self): 17 | result = self.abortable.apply_async() 18 | tid = result.id 19 | self.assertIsInstance( 20 | self.abortable.AsyncResult(tid), AbortableAsyncResult, 21 | ) 22 | 23 | def test_is_not_aborted(self): 24 | self.abortable.push_request() 25 | try: 26 | result = self.abortable.apply_async() 27 | tid = result.id 28 | self.assertFalse(self.abortable.is_aborted(task_id=tid)) 29 | finally: 30 | self.abortable.pop_request() 31 | 32 | def test_is_aborted_not_abort_result(self): 33 | self.abortable.AsyncResult = self.app.AsyncResult 34 | self.abortable.push_request() 35 | try: 36 | self.abortable.request.id = 'foo' 37 | self.assertFalse(self.abortable.is_aborted()) 38 | finally: 39 | self.abortable.pop_request() 40 | 41 | def test_abort_yields_aborted(self): 42 | self.abortable.push_request() 43 | try: 44 | result = self.abortable.apply_async() 45 | result.abort() 46 | tid = result.id 47 | self.assertTrue(self.abortable.is_aborted(task_id=tid)) 48 | finally: 49 | self.abortable.pop_request() 50 | -------------------------------------------------------------------------------- /celery/app/annotations.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | ``celery.app.annotations`` 4 | ~~~~~~~~~~~~~~~~~~~~~~~~~~ 5 | 6 | Annotations is a nice term for monkey-patching 7 | task classes in the configuration. 8 | 9 | This prepares and performs the annotations in the 10 | :setting:`task_annotations` setting. 11 | 12 | """ 13 | from __future__ import absolute_import, unicode_literals 14 | 15 | from celery.five import string_t 16 | from celery.utils.functional import firstmethod, mlazy 17 | from celery.utils.imports import instantiate 18 | 19 | _first_match = firstmethod('annotate') 20 | _first_match_any = firstmethod('annotate_any') 21 | 22 | __all__ = ['MapAnnotation', 'prepare', 'resolve_all'] 23 | 24 | 25 | class MapAnnotation(dict): 26 | 27 | def annotate_any(self): 28 | try: 29 | return dict(self['*']) 30 | except KeyError: 31 | pass 32 | 33 | def annotate(self, task): 34 | try: 35 | return dict(self[task.name]) 36 | except KeyError: 37 | pass 38 | 39 | 40 | def prepare(annotations): 41 | """Expands the :setting:`task_annotations` setting.""" 42 | 43 | def expand_annotation(annotation): 44 | if isinstance(annotation, dict): 45 | return MapAnnotation(annotation) 46 | elif isinstance(annotation, string_t): 47 | return mlazy(instantiate, annotation) 48 | return annotation 49 | 50 | if annotations is None: 51 | return () 52 | elif not isinstance(annotations, (list, tuple)): 53 | annotations = (annotations,) 54 | return [expand_annotation(anno) for anno in annotations] 55 | 56 | 57 | def resolve_all(anno, task): 58 | return (x for x in (_first_match(anno, task), _first_match_any(anno)) if x) 59 | -------------------------------------------------------------------------------- /celery/tests/app/test_annotations.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from celery.app.annotations import MapAnnotation, prepare 4 | from celery.utils.imports import qualname 5 | 6 | from celery.tests.case import AppCase 7 | 8 | 9 | class MyAnnotation(object): 10 | foo = 65 11 | 12 | 13 | class AnnotationCase(AppCase): 14 | 15 | def setup(self): 16 | @self.app.task(shared=False) 17 | def add(x, y): 18 | return x + y 19 | self.add = add 20 | 21 | @self.app.task(shared=False) 22 | def mul(x, y): 23 | return x * y 24 | self.mul = mul 25 | 26 | 27 | class test_MapAnnotation(AnnotationCase): 28 | 29 | def test_annotate(self): 30 | x = MapAnnotation({self.add.name: {'foo': 1}}) 31 | self.assertDictEqual(x.annotate(self.add), {'foo': 1}) 32 | self.assertIsNone(x.annotate(self.mul)) 33 | 34 | def test_annotate_any(self): 35 | x = MapAnnotation({'*': {'foo': 2}}) 36 | self.assertDictEqual(x.annotate_any(), {'foo': 2}) 37 | 38 | x = MapAnnotation() 39 | self.assertIsNone(x.annotate_any()) 40 | 41 | 42 | class test_prepare(AnnotationCase): 43 | 44 | def test_dict_to_MapAnnotation(self): 45 | x = prepare({self.add.name: {'foo': 3}}) 46 | self.assertIsInstance(x[0], MapAnnotation) 47 | 48 | def test_returns_list(self): 49 | self.assertListEqual(prepare(1), [1]) 50 | self.assertListEqual(prepare([1]), [1]) 51 | self.assertListEqual(prepare((1,)), [1]) 52 | self.assertEqual(prepare(None), ()) 53 | 54 | def test_evalutes_qualnames(self): 55 | self.assertEqual(prepare(qualname(MyAnnotation))[0]().foo, 65) 56 | self.assertEqual(prepare([qualname(MyAnnotation)])[0]().foo, 65) 57 | -------------------------------------------------------------------------------- /celery/tests/utils/test_pickle.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from celery.utils.serialization import pickle 4 | from celery.tests.case import Case 5 | 6 | 7 | class RegularException(Exception): 8 | pass 9 | 10 | 11 | class ArgOverrideException(Exception): 12 | 13 | def __init__(self, message, status_code=10): 14 | self.status_code = status_code 15 | Exception.__init__(self, message, status_code) 16 | 17 | 18 | class test_Pickle(Case): 19 | 20 | def test_pickle_regular_exception(self): 21 | exc = None 22 | try: 23 | raise RegularException('RegularException raised') 24 | except RegularException as exc_: 25 | exc = exc_ 26 | 27 | pickled = pickle.dumps({'exception': exc}) 28 | unpickled = pickle.loads(pickled) 29 | exception = unpickled.get('exception') 30 | self.assertTrue(exception) 31 | self.assertIsInstance(exception, RegularException) 32 | self.assertTupleEqual(exception.args, ('RegularException raised',)) 33 | 34 | def test_pickle_arg_override_exception(self): 35 | 36 | exc = None 37 | try: 38 | raise ArgOverrideException( 39 | 'ArgOverrideException raised', status_code=100, 40 | ) 41 | except ArgOverrideException as exc_: 42 | exc = exc_ 43 | 44 | pickled = pickle.dumps({'exception': exc}) 45 | unpickled = pickle.loads(pickled) 46 | exception = unpickled.get('exception') 47 | self.assertTrue(exception) 48 | self.assertIsInstance(exception, ArgOverrideException) 49 | self.assertTupleEqual(exception.args, ( 50 | 'ArgOverrideException raised', 100)) 51 | self.assertEqual(exception.status_code, 100) 52 | -------------------------------------------------------------------------------- /docs/getting-started/brokers/index.rst: -------------------------------------------------------------------------------- 1 | .. _brokers: 2 | 3 | ===================== 4 | Brokers 5 | ===================== 6 | 7 | :Release: |version| 8 | :Date: |today| 9 | 10 | Celery supports several message transport alternatives. 11 | 12 | .. _broker_toc: 13 | 14 | Broker Instructions 15 | =================== 16 | 17 | .. toctree:: 18 | :maxdepth: 1 19 | 20 | rabbitmq 21 | sqs 22 | 23 | .. _broker-overview: 24 | 25 | Broker Overview 26 | =============== 27 | 28 | This is comparison table of the different transports supports, 29 | more information can be found in the documentation for each 30 | individual transport (see :ref:`broker_toc`). 31 | 32 | +---------------+--------------+----------------+--------------------+ 33 | | **Name** | **Status** | **Monitoring** | **Remote Control** | 34 | +---------------+--------------+----------------+--------------------+ 35 | | *RabbitMQ* | Stable | Yes | Yes | 36 | +---------------+--------------+----------------+--------------------+ 37 | | *Amazon SQS* | Stable | No | No | 38 | +---------------+--------------+----------------+--------------------+ 39 | | *Zookeeper* | Experimental | No | No | 40 | +---------------+--------------+----------------+--------------------+ 41 | 42 | Experimental brokers may be functional but they do not have 43 | dedicated maintainers. 44 | 45 | Missing monitor support means that the transport does not 46 | implement events, and as such Flower, `celery events`, `celerymon` 47 | and other event-based monitoring tools will not work. 48 | 49 | Remote control means the ability to inspect and manage workers 50 | at runtime using the `celery inspect` and `celery control` commands 51 | (and other tools using the remote control API). 52 | -------------------------------------------------------------------------------- /examples/django/README.rst: -------------------------------------------------------------------------------- 1 | ============================================================== 2 | Example Django project using Celery 3 | ============================================================== 4 | 5 | Contents 6 | ======== 7 | 8 | ``proj/`` 9 | --------- 10 | 11 | This is the project iself, created using 12 | ``django-admin.py startproject proj``, and then the settings module 13 | (``proj/settings.py``) was modified to add ``demoapp`` to 14 | ``INSTALLED_APPS`` 15 | 16 | ``proj/celery.py`` 17 | ---------- 18 | 19 | This module contains the Celery application instance for this project, 20 | we take configuration from Django settings and use ``autodiscover_tasks`` to 21 | find task modules inside all packages listed in ``INSTALLED_APPS``. 22 | 23 | ``demoapp/`` 24 | ------------ 25 | 26 | Example generic app. This is decoupled from the rest of the project by using 27 | the ``@shared_task`` decorator. This decorator returns a proxy that always 28 | points to the currently active Celery instance. 29 | 30 | Installing requirements 31 | ======================= 32 | 33 | The settings file assumes that ``rabbitmq-server`` is running on ``localhost`` 34 | using the default ports. More information here: 35 | 36 | http://docs.celeryproject.org/en/latest/getting-started/brokers/rabbitmq.html 37 | 38 | In addition, some Python requirements must also be satisfied: 39 | 40 | .. code-block:: console 41 | 42 | $ pip install -r requirements.txt 43 | 44 | Starting the worker 45 | =================== 46 | 47 | .. code-block:: console 48 | 49 | $ celery -A proj worker -l info 50 | 51 | Running a task 52 | =================== 53 | 54 | .. code-block:: console 55 | 56 | $ python ./manage.py shell 57 | >>> from demoapp.tasks import add, mul, xsum 58 | >>> res = add.delay(2,3) 59 | >>> res.get() 60 | 5 61 | -------------------------------------------------------------------------------- /celery/utils/dispatch/license.txt: -------------------------------------------------------------------------------- 1 | django.dispatch was originally forked from PyDispatcher. 2 | 3 | PyDispatcher License: 4 | 5 | Copyright (c) 2001-2003, Patrick K. O'Brien and Contributors 6 | All rights reserved. 7 | 8 | Redistribution and use in source and binary forms, with or without 9 | modification, are permitted provided that the following conditions 10 | are met: 11 | 12 | Redistributions of source code must retain the above copyright 13 | notice, this list of conditions and the following disclaimer. 14 | 15 | Redistributions in binary form must reproduce the above 16 | copyright notice, this list of conditions and the following 17 | disclaimer in the documentation and/or other materials 18 | provided with the distribution. 19 | 20 | The name of Patrick K. O'Brien, or the name of any Contributor, 21 | may not be used to endorse or promote products derived from this 22 | software without specific prior written permission. 23 | 24 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 25 | ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 26 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 27 | FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 28 | COPYRIGHT HOLDERS AND CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 29 | INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 30 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 31 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 32 | HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 33 | STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 35 | OF THE POSSIBILITY OF SUCH DAMAGE. 36 | -------------------------------------------------------------------------------- /celery/tests/utils/test_imports.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from celery.five import bytes_if_py2 4 | 5 | from celery.utils.imports import ( 6 | NotAPackage, 7 | qualname, 8 | gen_task_name, 9 | reload_from_cwd, 10 | module_file, 11 | find_module, 12 | ) 13 | 14 | from celery.tests.case import Case, Mock, patch 15 | 16 | 17 | class test_import_utils(Case): 18 | 19 | def test_find_module(self): 20 | self.assertTrue(find_module('celery')) 21 | imp = Mock() 22 | imp.return_value = None 23 | with self.assertRaises(NotAPackage): 24 | find_module('foo.bar.baz', imp=imp) 25 | self.assertTrue(find_module('celery.worker.request')) 26 | 27 | def test_qualname(self): 28 | Class = type(bytes_if_py2('Fox'), (object,), { 29 | '__module__': 'quick.brown', 30 | }) 31 | self.assertEqual(qualname(Class), 'quick.brown.Fox') 32 | self.assertEqual(qualname(Class()), 'quick.brown.Fox') 33 | 34 | @patch('celery.utils.imports.reload') 35 | def test_reload_from_cwd(self, reload): 36 | reload_from_cwd('foo') 37 | reload.assert_called() 38 | 39 | def test_reload_from_cwd_custom_reloader(self): 40 | reload = Mock() 41 | reload_from_cwd('foo', reload) 42 | reload.assert_called() 43 | 44 | def test_module_file(self): 45 | m1 = Mock() 46 | m1.__file__ = '/opt/foo/xyz.pyc' 47 | self.assertEqual(module_file(m1), '/opt/foo/xyz.py') 48 | m2 = Mock() 49 | m2.__file__ = '/opt/foo/xyz.py' 50 | self.assertEqual(module_file(m1), '/opt/foo/xyz.py') 51 | 52 | 53 | class test_gen_task_name(Case): 54 | 55 | def test_no_module(self): 56 | app = Mock() 57 | app.name == '__main__' 58 | self.assertTrue(gen_task_name(app, 'foo', 'axsadaewe')) 59 | -------------------------------------------------------------------------------- /celery/worker/consumer/events.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from kombu.common import ignore_errors 4 | 5 | from celery import bootsteps 6 | 7 | from .connection import Connection 8 | 9 | __all__ = ['Events'] 10 | 11 | 12 | class Events(bootsteps.StartStopStep): 13 | 14 | requires = (Connection,) 15 | 16 | def __init__(self, c, send_events=True, 17 | without_heartbeat=False, without_gossip=False, **kwargs): 18 | self.groups = None if send_events else ['worker'] 19 | self.send_events = ( 20 | send_events or 21 | not without_gossip or 22 | not without_heartbeat 23 | ) 24 | c.event_dispatcher = None 25 | 26 | def start(self, c): 27 | # flush events sent while connection was down. 28 | prev = self._close(c) 29 | dis = c.event_dispatcher = c.app.events.Dispatcher( 30 | c.connect(), hostname=c.hostname, 31 | enabled=self.send_events, groups=self.groups, 32 | buffer_group=['task'] if c.hub else None, 33 | on_send_buffered=c.on_send_event_buffered if c.hub else None, 34 | ) 35 | if prev: 36 | dis.extend_buffer(prev) 37 | dis.flush() 38 | 39 | def stop(self, c): 40 | pass 41 | 42 | def _close(self, c): 43 | if c.event_dispatcher: 44 | dispatcher = c.event_dispatcher 45 | # remember changes from remote control commands: 46 | self.groups = dispatcher.groups 47 | 48 | # close custom connection 49 | if dispatcher.connection: 50 | ignore_errors(c, dispatcher.connection.close) 51 | ignore_errors(c, dispatcher.close) 52 | c.event_dispatcher = None 53 | return dispatcher 54 | 55 | def shutdown(self, c): 56 | self._close(c) 57 | -------------------------------------------------------------------------------- /celery/tests/compat_modules/test_compat.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from datetime import timedelta 4 | 5 | from celery.schedules import schedule 6 | from celery.task import ( 7 | periodic_task, 8 | PeriodicTask 9 | ) 10 | 11 | from celery.tests.case import AppCase, depends_on_current_app # noqa 12 | 13 | 14 | @depends_on_current_app 15 | class test_periodic_tasks(AppCase): 16 | 17 | def setup(self): 18 | @periodic_task(app=self.app, shared=False, 19 | run_every=schedule(timedelta(hours=1), app=self.app)) 20 | def my_periodic(): 21 | pass 22 | self.my_periodic = my_periodic 23 | 24 | def now(self): 25 | return self.app.now() 26 | 27 | def test_must_have_run_every(self): 28 | with self.assertRaises(NotImplementedError): 29 | type('Foo', (PeriodicTask,), {'__module__': __name__}) 30 | 31 | def test_remaining_estimate(self): 32 | s = self.my_periodic.run_every 33 | self.assertIsInstance( 34 | s.remaining_estimate(s.maybe_make_aware(self.now())), 35 | timedelta) 36 | 37 | def test_is_due_not_due(self): 38 | due, remaining = self.my_periodic.run_every.is_due(self.now()) 39 | self.assertFalse(due) 40 | # This assertion may fail if executed in the 41 | # first minute of an hour, thus 59 instead of 60 42 | self.assertGreater(remaining, 59) 43 | 44 | def test_is_due(self): 45 | p = self.my_periodic 46 | due, remaining = p.run_every.is_due( 47 | self.now() - p.run_every.run_every, 48 | ) 49 | self.assertTrue(due) 50 | self.assertEqual( 51 | remaining, p.run_every.run_every.total_seconds(), 52 | ) 53 | 54 | def test_schedule_repr(self): 55 | p = self.my_periodic 56 | self.assertTrue(repr(p.run_every)) 57 | -------------------------------------------------------------------------------- /celery/task/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | celery.task 4 | ~~~~~~~~~~~ 5 | 6 | This is the old task module, it should not be used anymore, 7 | import from the main 'celery' module instead. 8 | If you're looking for the decorator implementation then that's in 9 | ``celery.app.base.Celery.task``. 10 | 11 | """ 12 | from __future__ import absolute_import, unicode_literals 13 | 14 | from celery._state import current_app, current_task as current 15 | from celery.five import LazyModule, recreate_module 16 | from celery.local import Proxy 17 | 18 | __all__ = [ 19 | 'BaseTask', 'Task', 'PeriodicTask', 'task', 'periodic_task', 20 | 'group', 'chord', 'subtask', 21 | ] 22 | 23 | 24 | STATICA_HACK = True 25 | globals()['kcah_acitats'[::-1].upper()] = False 26 | if STATICA_HACK: # pragma: no cover 27 | # This is never executed, but tricks static analyzers (PyDev, PyCharm, 28 | # pylint, etc.) into knowing the types of these symbols, and what 29 | # they contain. 30 | from celery.canvas import group, chord, subtask 31 | from .base import BaseTask, Task, PeriodicTask, task, periodic_task 32 | 33 | 34 | class module(LazyModule): 35 | 36 | def __call__(self, *args, **kwargs): 37 | return self.task(*args, **kwargs) 38 | 39 | 40 | old_module, new_module = recreate_module( # pragma: no cover 41 | __name__, 42 | by_module={ 43 | 'celery.task.base': ['BaseTask', 'Task', 'PeriodicTask', 44 | 'task', 'periodic_task'], 45 | 'celery.canvas': ['group', 'chord', 'subtask'], 46 | }, 47 | base=module, 48 | __package__='celery.task', 49 | __file__=__file__, 50 | __path__=__path__, 51 | __doc__=__doc__, 52 | current=current, 53 | discard_all=Proxy(lambda: current_app.control.purge), 54 | backend_cleanup=Proxy( 55 | lambda: current_app.tasks['celery.backend_cleanup'] 56 | ), 57 | ) 58 | -------------------------------------------------------------------------------- /funtests/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | from __future__ import absolute_import, unicode_literals 4 | 5 | try: 6 | from setuptools import setup 7 | from setuptools.command.install import install 8 | except ImportError: 9 | from ez_setup import use_setuptools 10 | use_setuptools() 11 | from setuptools import setup # noqa 12 | from setuptools.command.install import install # noqa 13 | 14 | import os 15 | import sys 16 | 17 | sys.path.insert(0, os.getcwd()) 18 | sys.path.insert(0, os.path.join(os.getcwd(), os.pardir)) 19 | import suite # noqa 20 | 21 | 22 | class no_install(install): 23 | 24 | def run(self, *args, **kwargs): 25 | import sys 26 | sys.stderr.write(""" 27 | ------------------------------------------------------ 28 | The Celery functional test suite cannot be installed. 29 | ------------------------------------------------------ 30 | 31 | 32 | But you can execute the tests by running the command: 33 | 34 | $ python setup.py test 35 | 36 | 37 | """) 38 | 39 | 40 | setup( 41 | name='celery-funtests', 42 | version='DEV', 43 | description='Functional test suite for Celery', 44 | author='Ask Solem', 45 | author_email='ask@celeryproject.org', 46 | url='https://github.com/celery/celery', 47 | platforms=['any'], 48 | packages=[], 49 | data_files=[], 50 | zip_safe=False, 51 | cmdclass={'install': no_install}, 52 | test_suite='nose.collector', 53 | tests_require=[ 54 | 'unittest2>=0.4.0', 55 | 'simplejson', 56 | 'nose', 57 | 'redis', 58 | 'pymongo', 59 | ], 60 | classifiers=[ 61 | 'Operating System :: OS Independent', 62 | 'Programming Language :: Python', 63 | 'License :: OSI Approved :: BSD License', 64 | 'Intended Audience :: Developers', 65 | ], 66 | long_description='Do not install this package', 67 | ) 68 | -------------------------------------------------------------------------------- /celery/tests/tasks/test_states.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from celery import states 4 | 5 | from celery.tests.case import Case 6 | 7 | 8 | class test_state_precedence(Case): 9 | 10 | def test_gt(self): 11 | self.assertGreater( 12 | states.state(states.SUCCESS), states.state(states.PENDING), 13 | ) 14 | self.assertGreater( 15 | states.state(states.FAILURE), states.state(states.RECEIVED), 16 | ) 17 | self.assertGreater( 18 | states.state(states.REVOKED), states.state(states.STARTED), 19 | ) 20 | self.assertGreater( 21 | states.state(states.SUCCESS), states.state('CRASHED'), 22 | ) 23 | self.assertGreater( 24 | states.state(states.FAILURE), states.state('CRASHED'), 25 | ) 26 | self.assertLessEqual( 27 | states.state(states.REVOKED), states.state('CRASHED'), 28 | ) 29 | 30 | def test_lt(self): 31 | self.assertLess( 32 | states.state(states.PENDING), states.state(states.SUCCESS), 33 | ) 34 | self.assertLess( 35 | states.state(states.RECEIVED), states.state(states.FAILURE), 36 | ) 37 | self.assertLess( 38 | states.state(states.STARTED), states.state(states.REVOKED), 39 | ) 40 | self.assertLess( 41 | states.state('CRASHED'), states.state(states.SUCCESS), 42 | ) 43 | self.assertLess( 44 | states.state('CRASHED'), states.state(states.FAILURE), 45 | ) 46 | self.assertLess( 47 | states.state(states.REVOKED), states.state('CRASHED'), 48 | ) 49 | self.assertLessEqual( 50 | states.state(states.REVOKED), states.state('CRASHED'), 51 | ) 52 | self.assertGreaterEqual( 53 | states.state('CRASHED'), states.state(states.REVOKED), 54 | ) 55 | -------------------------------------------------------------------------------- /celery/loaders/default.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | celery.loaders.default 4 | ~~~~~~~~~~~~~~~~~~~~~~ 5 | 6 | The default loader used when no custom app has been initialized. 7 | 8 | """ 9 | from __future__ import absolute_import, unicode_literals 10 | 11 | import os 12 | import warnings 13 | 14 | from celery.exceptions import NotConfigured 15 | from celery.utils.collections import DictAttribute 16 | from celery.utils.serialization import strtobool 17 | 18 | from .base import BaseLoader 19 | 20 | __all__ = ['Loader', 'DEFAULT_CONFIG_MODULE'] 21 | 22 | DEFAULT_CONFIG_MODULE = 'celeryconfig' 23 | 24 | #: Warns if configuration file is missing if :envvar:`C_WNOCONF` is set. 25 | C_WNOCONF = strtobool(os.environ.get('C_WNOCONF', False)) 26 | 27 | 28 | class Loader(BaseLoader): 29 | """The loader used by the default app.""" 30 | 31 | def setup_settings(self, settingsdict): 32 | return DictAttribute(settingsdict) 33 | 34 | def read_configuration(self, fail_silently=True): 35 | """Read configuration from :file:`celeryconfig.py` and configure 36 | celery and Django so it can be used by regular Python.""" 37 | configname = os.environ.get('CELERY_CONFIG_MODULE', 38 | DEFAULT_CONFIG_MODULE) 39 | try: 40 | usercfg = self._import_config_module(configname) 41 | except ImportError: 42 | if not fail_silently: 43 | raise 44 | # billiard sets this if forked using execv 45 | if C_WNOCONF and not os.environ.get('FORKED_BY_MULTIPROCESSING'): 46 | warnings.warn(NotConfigured( 47 | 'No {module} module found! Please make sure it exists and ' 48 | 'is available to Python.'.format(module=configname))) 49 | return self.setup_settings({}) 50 | else: 51 | self.configured = True 52 | return self.setup_settings(usercfg) 53 | -------------------------------------------------------------------------------- /celery/worker/consumer/tasks.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from kombu.common import QoS, ignore_errors 4 | 5 | from celery import bootsteps 6 | from celery.utils.log import get_logger 7 | 8 | from .mingle import Mingle 9 | 10 | __all__ = ['Tasks'] 11 | logger = get_logger(__name__) 12 | debug = logger.debug 13 | 14 | 15 | class Tasks(bootsteps.StartStopStep): 16 | 17 | requires = (Mingle,) 18 | 19 | def __init__(self, c, **kwargs): 20 | c.task_consumer = c.qos = None 21 | 22 | def start(self, c): 23 | c.update_strategies() 24 | 25 | # - RabbitMQ 3.3 completely redefines how basic_qos works.. 26 | # This will detect if the new qos smenatics is in effect, 27 | # and if so make sure the 'apply_global' flag is set on qos updates. 28 | qos_global = not c.connection.qos_semantics_matches_spec 29 | 30 | # set initial prefetch count 31 | c.connection.default_channel.basic_qos( 32 | 0, c.initial_prefetch_count, qos_global, 33 | ) 34 | 35 | c.task_consumer = c.app.amqp.TaskConsumer( 36 | c.connection, on_decode_error=c.on_decode_error, 37 | ) 38 | 39 | def set_prefetch_count(prefetch_count): 40 | return c.task_consumer.qos( 41 | prefetch_count=prefetch_count, 42 | apply_global=qos_global, 43 | ) 44 | c.qos = QoS(set_prefetch_count, c.initial_prefetch_count) 45 | 46 | def stop(self, c): 47 | if c.task_consumer: 48 | debug('Canceling task consumer...') 49 | ignore_errors(c, c.task_consumer.cancel) 50 | 51 | def shutdown(self, c): 52 | if c.task_consumer: 53 | self.stop(c) 54 | debug('Closing consumer channel...') 55 | ignore_errors(c, c.task_consumer.close) 56 | c.task_consumer = None 57 | 58 | def info(self, c): 59 | return {'prefetch_count': c.qos.value if c.qos else 'N/A'} 60 | -------------------------------------------------------------------------------- /examples/eventlet/bulk_task_producer.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from eventlet import spawn_n, monkey_patch, Timeout 4 | from eventlet.queue import LightQueue 5 | from eventlet.event import Event 6 | 7 | monkey_patch() 8 | 9 | 10 | class Receipt(object): 11 | result = None 12 | 13 | def __init__(self, callback=None): 14 | self.callback = callback 15 | self.ready = Event() 16 | 17 | def finished(self, result): 18 | self.result = result 19 | if self.callback: 20 | self.callback(result) 21 | self.ready.send() 22 | 23 | def wait(self, timeout=None): 24 | with Timeout(timeout): 25 | return self.ready.wait() 26 | 27 | 28 | class ProducerPool(object): 29 | """Usage:: 30 | 31 | >>> app = Celery(broker='amqp://') 32 | >>> ProducerPool(app) 33 | 34 | """ 35 | Receipt = Receipt 36 | 37 | def __init__(self, app, size=20): 38 | self.app = app 39 | self.size = size 40 | self.inqueue = LightQueue() 41 | self._running = None 42 | self._producers = None 43 | 44 | def apply_async(self, task, args, kwargs, callback=None, **options): 45 | if self._running is None: 46 | self._running = spawn_n(self._run) 47 | receipt = self.Receipt(callback) 48 | self.inqueue.put((task, args, kwargs, options, receipt)) 49 | return receipt 50 | 51 | def _run(self): 52 | self._producers = [ 53 | spawn_n(self._producer) for _ in range(self.size) 54 | ] 55 | 56 | def _producer(self): 57 | inqueue = self.inqueue 58 | 59 | with self.app.producer_or_acquire() as producer: 60 | while 1: 61 | task, args, kwargs, options, receipt = inqueue.get() 62 | result = task.apply_async(args, kwargs, 63 | producer=producer, 64 | **options) 65 | receipt.finished(result) 66 | -------------------------------------------------------------------------------- /extra/appveyor/run_with_compiler.cmd: -------------------------------------------------------------------------------- 1 | :: To build extensions for 64 bit Python 3, we need to configure environment 2 | :: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of: 3 | :: MS Windows SDK for Windows 7 and .NET Framework 4 (SDK v7.1) 4 | :: 5 | :: To build extensions for 64 bit Python 2, we need to configure environment 6 | :: variables to use the MSVC 2008 C++ compilers from GRMSDKX_EN_DVD.iso of: 7 | :: MS Windows SDK for Windows 7 and .NET Framework 3.5 (SDK v7.0) 8 | :: 9 | :: 32 bit builds do not require specific environment configurations. 10 | :: 11 | :: Note: this script needs to be run with the /E:ON and /V:ON flags for the 12 | :: cmd interpreter, at least for (SDK v7.0) 13 | :: 14 | :: More details at: 15 | :: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows 16 | :: http://stackoverflow.com/a/13751649/163740 17 | :: 18 | :: Author: Olivier Grisel 19 | :: License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ 20 | @ECHO OFF 21 | 22 | SET COMMAND_TO_RUN=%* 23 | SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows 24 | 25 | SET MAJOR_PYTHON_VERSION="%PYTHON_VERSION:~0,1%" 26 | IF %MAJOR_PYTHON_VERSION% == "2" ( 27 | SET WINDOWS_SDK_VERSION="v7.0" 28 | ) ELSE IF %MAJOR_PYTHON_VERSION% == "3" ( 29 | SET WINDOWS_SDK_VERSION="v7.1" 30 | ) ELSE ( 31 | ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%" 32 | EXIT 1 33 | ) 34 | 35 | IF "%PYTHON_ARCH%"=="64" ( 36 | ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture 37 | SET DISTUTILS_USE_SDK=1 38 | SET MSSdk=1 39 | "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION% 40 | "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release 41 | ECHO Executing: %COMMAND_TO_RUN% 42 | call %COMMAND_TO_RUN% || EXIT 1 43 | ) ELSE ( 44 | ECHO Using default MSVC build environment for 32 bit architecture 45 | ECHO Executing: %COMMAND_TO_RUN% 46 | call %COMMAND_TO_RUN% || EXIT 1 47 | ) 48 | -------------------------------------------------------------------------------- /celery/app/registry.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | celery.app.registry 4 | ~~~~~~~~~~~~~~~~~~~ 5 | 6 | Registry of available tasks. 7 | 8 | """ 9 | from __future__ import absolute_import, unicode_literals 10 | 11 | import inspect 12 | 13 | from importlib import import_module 14 | 15 | from celery._state import get_current_app 16 | from celery.exceptions import NotRegistered 17 | from celery.five import items 18 | 19 | __all__ = ['TaskRegistry'] 20 | 21 | 22 | class TaskRegistry(dict): 23 | NotRegistered = NotRegistered 24 | 25 | def __missing__(self, key): 26 | raise self.NotRegistered(key) 27 | 28 | def register(self, task): 29 | """Register a task in the task registry. 30 | 31 | The task will be automatically instantiated if not already an 32 | instance. 33 | 34 | """ 35 | self[task.name] = inspect.isclass(task) and task() or task 36 | 37 | def unregister(self, name): 38 | """Unregister task by name. 39 | 40 | :param name: name of the task to unregister, or a 41 | :class:`celery.task.base.Task` with a valid `name` attribute. 42 | 43 | :raises celery.exceptions.NotRegistered: if the task has not 44 | been registered. 45 | 46 | """ 47 | try: 48 | self.pop(getattr(name, 'name', name)) 49 | except KeyError: 50 | raise self.NotRegistered(name) 51 | 52 | # -- these methods are irrelevant now and will be removed in 4.0 53 | def regular(self): 54 | return self.filter_types('regular') 55 | 56 | def periodic(self): 57 | return self.filter_types('periodic') 58 | 59 | def filter_types(self, type): 60 | return {name: task for name, task in items(self) 61 | if getattr(task, 'type', 'regular') == type} 62 | 63 | 64 | def _unpickle_task(name): 65 | return get_current_app().tasks[name] 66 | 67 | 68 | def _unpickle_task_v2(name, module=None): 69 | if module: 70 | import_module(module) 71 | return get_current_app().tasks[name] 72 | -------------------------------------------------------------------------------- /celery/tests/utils/test_graph.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from celery.five import WhateverIO, items 4 | from celery.utils.graph import DependencyGraph 5 | 6 | from celery.tests.case import Case, Mock 7 | 8 | 9 | class test_DependencyGraph(Case): 10 | 11 | def graph1(self): 12 | return DependencyGraph([ 13 | ('A', []), 14 | ('B', []), 15 | ('C', ['A']), 16 | ('D', ['C', 'B']), 17 | ]) 18 | 19 | def test_repr(self): 20 | self.assertTrue(repr(self.graph1())) 21 | 22 | def test_topsort(self): 23 | order = self.graph1().topsort() 24 | # C must start before D 25 | self.assertLess(order.index('C'), order.index('D')) 26 | # and B must start before D 27 | self.assertLess(order.index('B'), order.index('D')) 28 | # and A must start before C 29 | self.assertLess(order.index('A'), order.index('C')) 30 | 31 | def test_edges(self): 32 | self.assertItemsEqual( 33 | list(self.graph1().edges()), 34 | ['C', 'D'], 35 | ) 36 | 37 | def test_connect(self): 38 | x, y = self.graph1(), self.graph1() 39 | x.connect(y) 40 | 41 | def test_valency_of_when_missing(self): 42 | x = self.graph1() 43 | self.assertEqual(x.valency_of('foobarbaz'), 0) 44 | 45 | def test_format(self): 46 | x = self.graph1() 47 | x.formatter = Mock() 48 | obj = Mock() 49 | self.assertTrue(x.format(obj)) 50 | x.formatter.assert_called_with(obj) 51 | x.formatter = None 52 | self.assertIs(x.format(obj), obj) 53 | 54 | def test_items(self): 55 | self.assertDictEqual( 56 | dict(items(self.graph1())), 57 | {'A': [], 'B': [], 'C': ['A'], 'D': ['C', 'B']}, 58 | ) 59 | 60 | def test_repr_node(self): 61 | x = self.graph1() 62 | self.assertTrue(x.repr_node('fasdswewqewq')) 63 | 64 | def test_to_dot(self): 65 | s = WhateverIO() 66 | self.graph1().to_dot(s) 67 | self.assertTrue(s.getvalue()) 68 | -------------------------------------------------------------------------------- /docs/internals/reference/index.rst: -------------------------------------------------------------------------------- 1 | =========================== 2 | Internal Module Reference 3 | =========================== 4 | 5 | :Release: |version| 6 | :Date: |today| 7 | 8 | .. toctree:: 9 | :maxdepth: 1 10 | 11 | celery.worker.components 12 | celery.worker.loops 13 | celery.worker.heartbeat 14 | celery.worker.control 15 | celery.worker.pidbox 16 | celery.concurrency 17 | celery.concurrency.solo 18 | celery.concurrency.prefork 19 | celery.concurrency.eventlet 20 | celery.concurrency.gevent 21 | celery.concurrency.base 22 | celery.backends 23 | celery.backends.base 24 | celery.backends.async 25 | celery.backends.rpc 26 | celery.backends.database 27 | celery.backends.amqp 28 | celery.backends.cache 29 | celery.backends.consul 30 | celery.backends.couchdb 31 | celery.backends.mongodb 32 | celery.backends.elasticsearch 33 | celery.backends.redis 34 | celery.backends.riak 35 | celery.backends.cassandra 36 | celery.backends.couchbase 37 | celery.backends.filesystem 38 | celery.app.trace 39 | celery.app.annotations 40 | celery.app.routes 41 | celery.security.certificate 42 | celery.security.key 43 | celery.security.serialization 44 | celery.security.utils 45 | celery.events.snapshot 46 | celery.events.cursesmon 47 | celery.events.dumper 48 | celery.backends.database.models 49 | celery.backends.database.session 50 | celery.utils 51 | celery.utils.abstract 52 | celery.utils.collections 53 | celery.utils.nodenames 54 | celery.utils.deprecated 55 | celery.utils.functional 56 | celery.utils.graph 57 | celery.utils.objects 58 | celery.utils.term 59 | celery.utils.timeutils 60 | celery.utils.iso8601 61 | celery.utils.saferepr 62 | celery.utils.serialization 63 | celery.utils.sysinfo 64 | celery.utils.threads 65 | celery.utils.timer2 66 | celery.utils.imports 67 | celery.utils.log 68 | celery.utils.text 69 | celery.utils.dispatch 70 | celery.utils.dispatch.signal 71 | celery.utils.dispatch.saferef 72 | celery.platforms 73 | celery._state 74 | -------------------------------------------------------------------------------- /celery/tests/utils/test_deprecated.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from celery.tests.case import Case, patch 4 | 5 | from celery.utils import deprecated 6 | 7 | 8 | class test_deprecated_property(Case): 9 | 10 | @patch('celery.utils.deprecated.warn') 11 | def test_deprecated(self, warn): 12 | 13 | class X(object): 14 | _foo = None 15 | 16 | @deprecated.Property(deprecation='1.2') 17 | def foo(self): 18 | return self._foo 19 | 20 | @foo.setter 21 | def foo(self, value): 22 | self._foo = value 23 | 24 | @foo.deleter 25 | def foo(self): 26 | self._foo = None 27 | self.assertTrue(X.foo) 28 | self.assertTrue(X.foo.__set__(None, 1)) 29 | self.assertTrue(X.foo.__delete__(None)) 30 | x = X() 31 | x.foo = 10 32 | warn.assert_called_with( 33 | stacklevel=3, deprecation='1.2', alternative=None, 34 | description='foo', removal=None, 35 | ) 36 | warn.reset_mock() 37 | self.assertEqual(x.foo, 10) 38 | warn.assert_called_with( 39 | stacklevel=3, deprecation='1.2', alternative=None, 40 | description='foo', removal=None, 41 | ) 42 | warn.reset_mock() 43 | del(x.foo) 44 | warn.assert_called_with( 45 | stacklevel=3, deprecation='1.2', alternative=None, 46 | description='foo', removal=None, 47 | ) 48 | self.assertIsNone(x._foo) 49 | 50 | def test_deprecated_no_setter_or_deleter(self): 51 | class X(object): 52 | @deprecated.Property(deprecation='1.2') 53 | def foo(self): 54 | pass 55 | self.assertTrue(X.foo) 56 | x = X() 57 | with self.assertRaises(AttributeError): 58 | x.foo = 10 59 | with self.assertRaises(AttributeError): 60 | del(x.foo) 61 | 62 | 63 | class test_warn(Case): 64 | 65 | @patch('warnings.warn') 66 | def test_warn_deprecated(self, warn): 67 | deprecated.warn('Foo') 68 | warn.assert_called() 69 | -------------------------------------------------------------------------------- /celery/security/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | celery.security 4 | ~~~~~~~~~~~~~~~ 5 | 6 | Module implementing the signing message serializer. 7 | 8 | """ 9 | from __future__ import absolute_import, unicode_literals 10 | 11 | from kombu.serialization import ( 12 | registry, disable_insecure_serializers as _disable_insecure_serializers, 13 | ) 14 | 15 | from celery.exceptions import ImproperlyConfigured 16 | 17 | from .serialization import register_auth 18 | 19 | SSL_NOT_INSTALLED = """\ 20 | You need to install the pyOpenSSL library to use the auth serializer. 21 | Please install by: 22 | 23 | $ pip install pyOpenSSL 24 | """ 25 | 26 | SETTING_MISSING = """\ 27 | Sorry, but you have to configure the 28 | * security_key 29 | * security_certificate, and the 30 | * security_cert_storE 31 | configuration settings to use the auth serializer. 32 | 33 | Please see the configuration reference for more information. 34 | """ 35 | 36 | __all__ = ['setup_security'] 37 | 38 | 39 | def setup_security(allowed_serializers=None, key=None, cert=None, store=None, 40 | digest='sha1', serializer='json', app=None): 41 | """See :meth:`@Celery.setup_security`.""" 42 | if app is None: 43 | from celery import current_app 44 | app = current_app._get_current_object() 45 | 46 | _disable_insecure_serializers(allowed_serializers) 47 | 48 | conf = app.conf 49 | if conf.task_serializer != 'auth': 50 | return 51 | 52 | try: 53 | from OpenSSL import crypto # noqa 54 | except ImportError: 55 | raise ImproperlyConfigured(SSL_NOT_INSTALLED) 56 | 57 | key = key or conf.security_key 58 | cert = cert or conf.security_certificate 59 | store = store or conf.security_cert_store 60 | 61 | if not (key and cert and store): 62 | raise ImproperlyConfigured(SETTING_MISSING) 63 | 64 | with open(key) as kf: 65 | with open(cert) as cf: 66 | register_auth(kf.read(), cf.read(), store, digest, serializer) 67 | registry._set_default_serializer('auth') 68 | 69 | 70 | def disable_untrusted_serializers(whitelist=None): 71 | _disable_insecure_serializers(allowed=whitelist) 72 | -------------------------------------------------------------------------------- /celery/tests/worker/test_heartbeat.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from celery.worker.heartbeat import Heart 4 | from celery.tests.case import AppCase, Mock 5 | 6 | 7 | class MockDispatcher(object): 8 | heart = None 9 | next_iter = 0 10 | 11 | def __init__(self): 12 | self.sent = [] 13 | self.on_enabled = set() 14 | self.on_disabled = set() 15 | self.enabled = True 16 | 17 | def send(self, msg, **_fields): 18 | self.sent.append(msg) 19 | if self.heart: 20 | if self.next_iter > 10: 21 | self.heart._shutdown.set() 22 | self.next_iter += 1 23 | 24 | 25 | class MockTimer(object): 26 | 27 | def call_repeatedly(self, secs, fun, args=(), kwargs={}): 28 | 29 | class entry(tuple): 30 | canceled = False 31 | 32 | def cancel(self): 33 | self.canceled = True 34 | 35 | return entry((secs, fun, args, kwargs)) 36 | 37 | def cancel(self, entry): 38 | entry.cancel() 39 | 40 | 41 | class test_Heart(AppCase): 42 | 43 | def test_start_stop(self): 44 | timer = MockTimer() 45 | eventer = MockDispatcher() 46 | h = Heart(timer, eventer, interval=1) 47 | h.start() 48 | self.assertTrue(h.tref) 49 | h.stop() 50 | self.assertIsNone(h.tref) 51 | h.stop() 52 | 53 | def test_send_sends_signal(self): 54 | h = Heart(MockTimer(), MockDispatcher(), interval=1) 55 | h._send_sent_signal = None 56 | h._send('worker-heartbeat') 57 | h._send_sent_signal = Mock(name='send_sent_signal') 58 | h._send('worker') 59 | h._send_sent_signal.assert_called_with(sender=h) 60 | 61 | def test_start_when_disabled(self): 62 | timer = MockTimer() 63 | eventer = MockDispatcher() 64 | eventer.enabled = False 65 | h = Heart(timer, eventer) 66 | h.start() 67 | self.assertFalse(h.tref) 68 | 69 | def test_stop_when_disabled(self): 70 | timer = MockTimer() 71 | eventer = MockDispatcher() 72 | eventer.enabled = False 73 | h = Heart(timer, eventer) 74 | h.stop() 75 | -------------------------------------------------------------------------------- /celery/backends/database/session.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | celery.backends.database.session 4 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 5 | 6 | SQLAlchemy sessions. 7 | 8 | """ 9 | from __future__ import absolute_import, unicode_literals 10 | 11 | from sqlalchemy import create_engine 12 | from sqlalchemy.ext.declarative import declarative_base 13 | from sqlalchemy.orm import sessionmaker 14 | from sqlalchemy.pool import NullPool 15 | 16 | from kombu.utils import register_after_fork 17 | 18 | ResultModelBase = declarative_base() 19 | 20 | __all__ = ['SessionManager'] 21 | 22 | 23 | def _after_fork_cleanup_session(session): 24 | session._after_fork() 25 | 26 | 27 | class SessionManager(object): 28 | 29 | def __init__(self): 30 | self._engines = {} 31 | self._sessions = {} 32 | self.forked = False 33 | self.prepared = False 34 | if register_after_fork is not None: 35 | register_after_fork(self, _after_fork_cleanup_session) 36 | 37 | def _after_fork(self): 38 | self.forked = True 39 | 40 | def get_engine(self, dburi, **kwargs): 41 | if self.forked: 42 | try: 43 | return self._engines[dburi] 44 | except KeyError: 45 | engine = self._engines[dburi] = create_engine(dburi, **kwargs) 46 | return engine 47 | else: 48 | return create_engine(dburi, poolclass=NullPool) 49 | 50 | def create_session(self, dburi, short_lived_sessions=False, **kwargs): 51 | engine = self.get_engine(dburi, **kwargs) 52 | if self.forked: 53 | if short_lived_sessions or dburi not in self._sessions: 54 | self._sessions[dburi] = sessionmaker(bind=engine) 55 | return engine, self._sessions[dburi] 56 | else: 57 | return engine, sessionmaker(bind=engine) 58 | 59 | def prepare_models(self, engine): 60 | if not self.prepared: 61 | ResultModelBase.metadata.create_all(engine) 62 | self.prepared = True 63 | 64 | def session_factory(self, dburi, **kwargs): 65 | engine, session = self.create_session(dburi, **kwargs) 66 | self.prepare_models(engine) 67 | return session() 68 | -------------------------------------------------------------------------------- /celery/contrib/sphinx.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | ``celery.contrib.sphinx`` 4 | ========================= 5 | 6 | Sphinx documentation plugin 7 | 8 | **Usage** 9 | 10 | Add the extension to your :file:`docs/conf.py` configuration module: 11 | 12 | .. code-block:: python 13 | 14 | extensions = (..., 15 | 'celery.contrib.sphinx') 16 | 17 | If you would like to change the prefix for tasks in reference documentation 18 | then you can change the ``celery_task_prefix`` configuration value: 19 | 20 | .. code-block:: python 21 | 22 | celery_task_prefix = '(task)' # < default 23 | 24 | 25 | With the extension installed `autodoc` will automatically find 26 | task decorated objects and generate the correct (as well as 27 | add a ``(task)`` prefix), and you can also refer to the tasks 28 | using `:task:proj.tasks.add` syntax. 29 | 30 | Use ``.. autotask::`` to manually document a task. 31 | 32 | """ 33 | from __future__ import absolute_import, unicode_literals 34 | 35 | from inspect import formatargspec 36 | 37 | from sphinx.domains.python import PyModulelevel 38 | from sphinx.ext.autodoc import FunctionDocumenter 39 | 40 | from celery.app.task import BaseTask 41 | from celery.five import getfullargspec 42 | 43 | 44 | class TaskDocumenter(FunctionDocumenter): 45 | objtype = 'task' 46 | member_order = 11 47 | 48 | @classmethod 49 | def can_document_member(cls, member, membername, isattr, parent): 50 | return isinstance(member, BaseTask) and getattr(member, '__wrapped__') 51 | 52 | def format_args(self): 53 | wrapped = getattr(self.object, '__wrapped__') 54 | if wrapped is not None: 55 | argspec = getfullargspec(wrapped) 56 | fmt = formatargspec(*argspec) 57 | fmt = fmt.replace('\\', '\\\\') 58 | return fmt 59 | return '' 60 | 61 | def document_members(self, all_members=False): 62 | pass 63 | 64 | 65 | class TaskDirective(PyModulelevel): 66 | 67 | def get_signature_prefix(self, sig): 68 | return self.env.config.celery_task_prefix 69 | 70 | 71 | def setup(app): 72 | app.add_autodocumenter(TaskDocumenter) 73 | app.domains['py'].directives['task'] = TaskDirective 74 | app.add_config_value('celery_task_prefix', '(task)', True) 75 | -------------------------------------------------------------------------------- /celery/tests/app/test_utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from collections import Mapping, MutableMapping 4 | 5 | from celery.app.utils import Settings, filter_hidden_settings, bugreport 6 | 7 | from celery.tests.case import AppCase, Mock 8 | 9 | 10 | class test_Settings(AppCase): 11 | 12 | def test_is_mapping(self): 13 | """Settings should be a collections.Mapping""" 14 | self.assertTrue(issubclass(Settings, Mapping)) 15 | 16 | def test_is_mutable_mapping(self): 17 | """Settings should be a collections.MutableMapping""" 18 | self.assertTrue(issubclass(Settings, MutableMapping)) 19 | 20 | def test_find(self): 21 | self.assertTrue(self.app.conf.find_option('always_eager')) 22 | 23 | def test_get_by_parts(self): 24 | self.app.conf.task_do_this_and_that = 303 25 | self.assertEqual( 26 | self.app.conf.get_by_parts('task', 'do', 'this', 'and', 'that'), 27 | 303, 28 | ) 29 | 30 | def test_find_value_for_key(self): 31 | self.assertEqual( 32 | self.app.conf.find_value_for_key('always_eager'), 33 | False, 34 | ) 35 | 36 | def test_table(self): 37 | self.assertTrue(self.app.conf.table(with_defaults=True)) 38 | self.assertTrue(self.app.conf.table(with_defaults=False)) 39 | self.assertTrue(self.app.conf.table(censored=False)) 40 | self.assertTrue(self.app.conf.table(censored=True)) 41 | 42 | 43 | class test_filter_hidden_settings(AppCase): 44 | 45 | def test_handles_non_string_keys(self): 46 | """filter_hidden_settings shouldn't raise an exception when handling 47 | mappings with non-string keys""" 48 | conf = { 49 | 'STRING_KEY': 'VALUE1', 50 | ('NON', 'STRING', 'KEY'): 'VALUE2', 51 | 'STRING_KEY2': { 52 | 'STRING_KEY3': 1, 53 | ('NON', 'STRING', 'KEY', '2'): 2 54 | }, 55 | } 56 | filter_hidden_settings(conf) 57 | 58 | 59 | class test_bugreport(AppCase): 60 | 61 | def test_no_conn_driver_info(self): 62 | self.app.connection = Mock() 63 | conn = self.app.connection.return_value = Mock() 64 | conn.transport = None 65 | 66 | bugreport(self.app) 67 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import absolute_import, unicode_literals 3 | 4 | from sphinx_celery import conf 5 | 6 | globals().update(conf.build_config( 7 | 'celery', __file__, 8 | project='Celery', 9 | version_dev='4.0', 10 | version_stable='3.1', 11 | canonical_url='http://docs.celeryproject.org', 12 | webdomain='celeryproject.org', 13 | github_project='celery/celery', 14 | author='Ask Solem & contributors', 15 | author_name='Ask Solem', 16 | copyright='2009-2016', 17 | publisher='Celery Project', 18 | html_logo='images/celery_128.png', 19 | html_favicon='images/favicon.ico', 20 | html_prepend_sidebars=['sidebardonations.html'], 21 | extra_extensions=[ 22 | 'celery.contrib.sphinx', 23 | 'celerydocs', 24 | ], 25 | extra_intersphinx_mapping={ 26 | 'cyanide': ('https://cyanide.readthedocs.io/en/latest', None), 27 | }, 28 | apicheck_ignore_modules=[ 29 | 'celery.five', 30 | 'celery.__main__', 31 | 'celery.task', 32 | 'celery.task.base', 33 | 'celery.bin', 34 | 'celery.bin.celeryd_detach', 35 | 'celery.contrib', 36 | r'celery.fixups.*', 37 | 'celery.local', 38 | 'celery.app.base', 39 | 'celery.apps', 40 | 'celery.canvas', 41 | 'celery.concurrency.asynpool', 42 | 'celery.utils.encoding', 43 | ], 44 | )) 45 | 46 | settings = {} 47 | ignored_settings = { 48 | 'worker_agent', 49 | 'worker_pool_putlocks', 50 | 'broker_host', 51 | 'broker_user', 52 | 'broker_password', 53 | 'broker_vhost', 54 | 'broker_port', 55 | 'broker_transport', 56 | 'chord_propagates', 57 | 'mongodb_backend_settings', 58 | 'redis_host', 59 | 'redis_port', 60 | 'redis_db', 61 | 'redis_password', 62 | } 63 | 64 | 65 | def configcheck_project_settings(): 66 | from celery.app.defaults import NAMESPACES, flatten 67 | settings.update(dict(flatten(NAMESPACES))) 68 | return set(settings) 69 | 70 | 71 | def is_deprecated_setting(setting): 72 | try: 73 | return settings[setting].deprecate_by 74 | except KeyError: 75 | pass 76 | 77 | 78 | def configcheck_should_ignore(setting): 79 | return setting in ignored_settings or is_deprecated_setting(setting) 80 | -------------------------------------------------------------------------------- /celery/worker/heartbeat.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | celery.worker.heartbeat 4 | ~~~~~~~~~~~~~~~~~~~~~~~ 5 | 6 | This is the internal thread that sends heartbeat events 7 | at regular intervals. 8 | 9 | """ 10 | from __future__ import absolute_import, unicode_literals 11 | 12 | from celery.signals import heartbeat_sent 13 | from celery.utils.sysinfo import load_average 14 | 15 | from .state import SOFTWARE_INFO, active_requests, all_total_count 16 | 17 | __all__ = ['Heart'] 18 | 19 | 20 | class Heart(object): 21 | """Timer sending heartbeats at regular intervals. 22 | 23 | :param timer: Timer instance. 24 | :param eventer: Event dispatcher used to send the event. 25 | :keyword interval: Time in seconds between heartbeats. 26 | Default is 2 seconds. 27 | 28 | """ 29 | 30 | def __init__(self, timer, eventer, interval=None): 31 | self.timer = timer 32 | self.eventer = eventer 33 | self.interval = float(interval or 2.0) 34 | self.tref = None 35 | 36 | # Make event dispatcher start/stop us when enabled/disabled. 37 | self.eventer.on_enabled.add(self.start) 38 | self.eventer.on_disabled.add(self.stop) 39 | 40 | # Only send heartbeat_sent signal if it has receivers. 41 | self._send_sent_signal = ( 42 | heartbeat_sent.send if heartbeat_sent.receivers else None) 43 | 44 | def _send(self, event): 45 | if self._send_sent_signal is not None: 46 | self._send_sent_signal(sender=self) 47 | return self.eventer.send(event, freq=self.interval, 48 | active=len(active_requests), 49 | processed=all_total_count[0], 50 | loadavg=load_average(), 51 | **SOFTWARE_INFO) 52 | 53 | def start(self): 54 | if self.eventer.enabled: 55 | self._send('worker-online') 56 | self.tref = self.timer.call_repeatedly( 57 | self.interval, self._send, ('worker-heartbeat',), 58 | ) 59 | 60 | def stop(self): 61 | if self.tref is not None: 62 | self.timer.cancel(self.tref) 63 | self.tref = None 64 | if self.eventer.enabled: 65 | self._send('worker-offline') 66 | -------------------------------------------------------------------------------- /examples/eventlet/webcrawler.py: -------------------------------------------------------------------------------- 1 | """Recursive webcrawler example. 2 | 3 | For asynchronous DNS lookups install the `dnspython` package: 4 | 5 | $ pip install dnspython 6 | 7 | Requires the `pybloom` module for the bloom filter which is used 8 | to ensure a lower chance of recrawling a URL previously seen. 9 | 10 | Since the bloom filter is not shared, but only passed as an argument 11 | to each subtask, it would be much better to have this as a centralized 12 | service. Redis sets could also be a practical solution. 13 | 14 | A BloomFilter with a capacity of 100_000 members and an error rate 15 | of 0.001 is 2.8MB pickled, but if compressed with zlib it only takes 16 | up 2.9kB(!). 17 | 18 | We don't have to do compression manually, just set the tasks compression 19 | to "zlib", and the serializer to "pickle". 20 | 21 | """ 22 | from __future__ import absolute_import, unicode_literals 23 | 24 | import re 25 | 26 | try: 27 | from urllib.parse import urlsplit 28 | except ImportError: 29 | from urlparse import urlsplit # noqa 30 | 31 | import requests 32 | 33 | from celery import task, group 34 | from eventlet import Timeout 35 | 36 | from pybloom import BloomFilter 37 | 38 | # http://daringfireball.net/2009/11/liberal_regex_for_matching_urls 39 | url_regex = re.compile( 40 | r'\b(([\w-]+://?|www[.])[^\s()<>]+(?:\([\w\d]+\)|([^[:punct:]\s]|/)))') 41 | 42 | 43 | def domain(url): 44 | """Return the domain part of a URL.""" 45 | return urlsplit(url)[1].split(':')[0] 46 | 47 | 48 | @task(ignore_result=True, serializer='pickle', compression='zlib') 49 | def crawl(url, seen=None): 50 | print('crawling: {0}'.format(url)) 51 | if not seen: 52 | seen = BloomFilter(capacity=50000, error_rate=0.0001) 53 | 54 | with Timeout(5, False): 55 | try: 56 | response = requests.get(url) 57 | except Exception: 58 | return 59 | 60 | location = domain(url) 61 | wanted_urls = [] 62 | for url_match in url_regex.finditer(response.text): 63 | url = url_match.group(0) 64 | # To not destroy the internet, we only fetch URLs on the same domain. 65 | if url not in seen and location in domain(url): 66 | wanted_urls.append(url) 67 | seen.add(url) 68 | 69 | subtasks = group(crawl.s(url, seen) for url in wanted_urls) 70 | subtasks() 71 | -------------------------------------------------------------------------------- /celery/worker/consumer/mingle.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from celery import bootsteps 4 | from celery.five import items 5 | from celery.utils.log import get_logger 6 | 7 | from .events import Events 8 | 9 | __all__ = ['Mingle'] 10 | 11 | logger = get_logger(__name__) 12 | debug, info, exception = logger.debug, logger.info, logger.exception 13 | 14 | 15 | class Mingle(bootsteps.StartStopStep): 16 | 17 | label = 'Mingle' 18 | requires = (Events,) 19 | compatible_transports = {'amqp', 'redis'} 20 | 21 | def __init__(self, c, without_mingle=False, **kwargs): 22 | self.enabled = not without_mingle and self.compatible_transport(c.app) 23 | 24 | def compatible_transport(self, app): 25 | with app.connection_for_read() as conn: 26 | return conn.transport.driver_type in self.compatible_transports 27 | 28 | def start(self, c): 29 | info('mingle: searching for neighbors') 30 | I = c.app.control.inspect(timeout=1.0, connection=c.connection) 31 | our_revoked = c.controller.state.revoked 32 | replies = I.hello(c.hostname, our_revoked._data) or {} 33 | replies.pop(c.hostname, None) # delete my own response 34 | if replies: 35 | info('mingle: sync with %s nodes', 36 | len([reply for reply, value in items(replies) if value])) 37 | [self.on_node_reply(c, nodename, reply) 38 | for nodename, reply in items(replies) if reply] 39 | info('mingle: sync complete') 40 | else: 41 | info('mingle: all alone') 42 | 43 | def on_node_reply(self, c, nodename, reply): 44 | debug('mingle: processing reply from %s', nodename) 45 | try: 46 | self.sync_with_node(c, **reply) 47 | except MemoryError: 48 | raise 49 | except Exception as exc: 50 | exception('mingle: sync with %s failed: %r', nodename, exc) 51 | 52 | def sync_with_node(self, c, clock=None, revoked=None, **kwargs): 53 | self.on_clock_event(c, clock) 54 | self.on_revoked_received(c, revoked) 55 | 56 | def on_clock_event(self, c, clock): 57 | c.app.clock.adjust(clock) if clock else c.app.clock.forward() 58 | 59 | def on_revoked_received(self, c, revoked): 60 | if revoked: 61 | c.controller.state.revoked.update(revoked) 62 | -------------------------------------------------------------------------------- /celery/tests/app/test_defaults.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | import sys 4 | 5 | from importlib import import_module 6 | 7 | from celery.app.defaults import ( 8 | _OLD_DEFAULTS, _OLD_SETTING_KEYS, _TO_NEW_KEY, _TO_OLD_KEY, 9 | DEFAULTS, NAMESPACES, SETTING_KEYS 10 | ) 11 | from celery.five import values 12 | 13 | from celery.tests.case import AppCase, mock 14 | 15 | 16 | class test_defaults(AppCase): 17 | 18 | def setup(self): 19 | self._prev = sys.modules.pop('celery.app.defaults', None) 20 | 21 | def teardown(self): 22 | if self._prev: 23 | sys.modules['celery.app.defaults'] = self._prev 24 | 25 | def test_option_repr(self): 26 | self.assertTrue(repr(NAMESPACES['broker']['url'])) 27 | 28 | def test_any(self): 29 | val = object() 30 | self.assertIs(self.defaults.Option.typemap['any'](val), val) 31 | 32 | @mock.sys_platform('darwin') 33 | @mock.pypy_version((1, 4, 0)) 34 | def test_default_pool_pypy_14(self): 35 | self.assertEqual(self.defaults.DEFAULT_POOL, 'solo') 36 | 37 | @mock.sys_platform('darwin') 38 | @mock.pypy_version((1, 5, 0)) 39 | def test_default_pool_pypy_15(self): 40 | self.assertEqual(self.defaults.DEFAULT_POOL, 'prefork') 41 | 42 | def test_compat_indices(self): 43 | self.assertFalse(any(key.isupper() for key in DEFAULTS)) 44 | self.assertFalse(any(key.islower() for key in _OLD_DEFAULTS)) 45 | self.assertFalse(any(key.isupper() for key in _TO_OLD_KEY)) 46 | self.assertFalse(any(key.islower() for key in _TO_NEW_KEY)) 47 | self.assertFalse(any(key.isupper() for key in SETTING_KEYS)) 48 | self.assertFalse(any(key.islower() for key in _OLD_SETTING_KEYS)) 49 | self.assertFalse(any(value.isupper() for value in values(_TO_NEW_KEY))) 50 | self.assertFalse(any(value.islower() for value in values(_TO_OLD_KEY))) 51 | 52 | for key in _TO_NEW_KEY: 53 | self.assertIn(key, _OLD_SETTING_KEYS) 54 | for key in _TO_OLD_KEY: 55 | self.assertIn(key, SETTING_KEYS) 56 | 57 | def test_find(self): 58 | find = self.defaults.find 59 | 60 | self.assertEqual(find('default_queue')[2].default, 'celery') 61 | self.assertEqual(find('task_default_exchange')[2], 'celery') 62 | 63 | @property 64 | def defaults(self): 65 | return import_module('celery.app.defaults') 66 | -------------------------------------------------------------------------------- /celery/backends/amqp.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | ``celery.backends.amqp`` 4 | ~~~~~~~~~~~~~~~~~~~~~~~~ 5 | 6 | The AMQP result backend. 7 | 8 | This backend publishes results as messages. 9 | 10 | """ 11 | from __future__ import absolute_import, unicode_literals 12 | 13 | from .rpc import BaseRPCBackend 14 | 15 | from celery.utils import deprecated 16 | 17 | __all__ = ['AMQPBackend'] 18 | 19 | 20 | def repair_uuid(s): 21 | # Historically the dashes in UUIDS are removed from AMQ entity names, 22 | # but there is no known reason to. Hopefully we'll be able to fix 23 | # this in v4.0. 24 | return '%s-%s-%s-%s-%s' % (s[:8], s[8:12], s[12:16], s[16:20], s[20:]) 25 | 26 | 27 | class AMQPBackend(BaseRPCBackend): 28 | """Publishes results by sending messages.""" 29 | 30 | def __init__(self, *args, **kwargs): 31 | deprecated.warn( 32 | 'The AMQP backend', deprecation='4.0', removal='5.0', 33 | alternative='Please use RPC backend or a persistent backend.') 34 | super(AMQPBackend, self).__init__(*args, **kwargs) 35 | 36 | def _create_exchange(self, name, type='direct', delivery_mode=2): 37 | return self.Exchange(name=name, 38 | type=type, 39 | delivery_mode=delivery_mode, 40 | durable=self.persistent, 41 | auto_delete=False) 42 | 43 | def _create_binding(self, task_id): 44 | name = self.rkey(task_id) 45 | return self.Queue(name=name, 46 | exchange=self.exchange, 47 | routing_key=name, 48 | durable=self.persistent, 49 | auto_delete=self.auto_delete, 50 | queue_arguments=self.queue_arguments) 51 | 52 | def on_task_call(self, producer, task_id): 53 | pass 54 | 55 | def rkey(self, task_id): 56 | return task_id.replace('-', '') 57 | 58 | def destination_for(self, task_id, request): 59 | if request: 60 | return self.rkey(task_id), request.correlation_id or task_id 61 | return self.rkey(task_id), task_id 62 | 63 | def on_reply_declare(self, task_id): 64 | return [self._create_binding(task_id)] 65 | 66 | def on_result_fulfilled(self, result): 67 | self.result_consumer.cancel_for(result.id) 68 | 69 | def as_uri(self, include_password=True): 70 | return 'amqp://' 71 | -------------------------------------------------------------------------------- /celery/tests/utils/test_saferef.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | from celery.five import range 4 | from celery.utils.dispatch.saferef import safe_ref 5 | from celery.tests.case import Case 6 | 7 | 8 | class Class1(object): 9 | 10 | def x(self): 11 | pass 12 | 13 | 14 | def fun(obj): 15 | pass 16 | 17 | 18 | class Class2(object): 19 | 20 | def __call__(self, obj): 21 | pass 22 | 23 | 24 | class SaferefTests(Case): 25 | 26 | def setUp(self): 27 | ts = [] 28 | ss = [] 29 | for x in range(5000): 30 | t = Class1() 31 | ts.append(t) 32 | s = safe_ref(t.x, self._closure) 33 | ss.append(s) 34 | ts.append(fun) 35 | ss.append(safe_ref(fun, self._closure)) 36 | for x in range(30): 37 | t = Class2() 38 | ts.append(t) 39 | s = safe_ref(t, self._closure) 40 | ss.append(s) 41 | self.ts = ts 42 | self.ss = ss 43 | self.closureCount = 0 44 | 45 | def tearDown(self): 46 | del self.ts 47 | del self.ss 48 | 49 | def test_in(self): 50 | """test_in 51 | 52 | Test the "in" operator for safe references (cmp) 53 | 54 | """ 55 | for t in self.ts[:50]: 56 | self.assertIn(safe_ref(t.x), self.ss) 57 | 58 | def test_valid(self): 59 | """test_value 60 | 61 | Test that the references are valid (return instance methods) 62 | 63 | """ 64 | for s in self.ss: 65 | self.assertTrue(s()) 66 | 67 | def test_shortcircuit(self): 68 | """test_shortcircuit 69 | 70 | Test that creation short-circuits to reuse existing references 71 | 72 | """ 73 | sd = {} 74 | for s in self.ss: 75 | sd[s] = 1 76 | for t in self.ts: 77 | if hasattr(t, 'x'): 78 | self.assertIn(safe_ref(t.x), sd) 79 | else: 80 | self.assertIn(safe_ref(t), sd) 81 | 82 | def test_representation(self): 83 | """test_representation 84 | 85 | Test that the reference object's representation works 86 | 87 | XXX Doesn't currently check the results, just that no error 88 | is raised 89 | """ 90 | repr(self.ss[-1]) 91 | 92 | def _closure(self, ref): 93 | """Dumb utility mechanism to increment deletion counter""" 94 | self.closureCount += 1 95 | --------------------------------------------------------------------------------