├── requirements.txt ├── MANIFEST.in ├── .gitignore ├── AUTHORS ├── .travis.yml ├── INSTALL ├── celerytest ├── __init__.py ├── config.py ├── testcase.py ├── tests.py └── worker.py ├── setup.py ├── LICENSE └── README.md /requirements.txt: -------------------------------------------------------------------------------- 1 | celery>=3.0.19 2 | python-memcached>=1.48 3 | threadpool>=1.2.7 -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include AUTHORS README README.md INSTALL LICENSE 2 | recursive-include celerytest *.py 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # / 2 | *.pyc 3 | .DS_Store 4 | dist/ 5 | *.egg-info 6 | doc/__build/* 7 | build/ 8 | .build/ 9 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | ========= 2 | AUTHORS 3 | ========= 4 | :order: sorted 5 | 6 | Willem Bult -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | services: 3 | - memcached 4 | install: 5 | - pip install -r requirements.txt 6 | - pip install nose 7 | script: nosetests 8 | -------------------------------------------------------------------------------- /INSTALL: -------------------------------------------------------------------------------- 1 | Installation 2 | ============= 3 | 4 | Install the latest version of ``celerytest`` from PyPI: 5 | 6 | $ pip install celerytest 7 | 8 | Or clone the latest version of ``celerytest`` from GitHub:: 9 | 10 | $ git clone git://github.com/RentMethod/celerytest.git 11 | $ cd celerytest 12 | # python setup.py install # as root -------------------------------------------------------------------------------- /celerytest/__init__.py: -------------------------------------------------------------------------------- 1 | from celerytest.config import CELERY_TEST_CONFIG, CELERY_TEST_CONFIG_MEMORY 2 | from celerytest.worker import CeleryWorkerThread 3 | 4 | 5 | def setup_celery_worker(app, config=CELERY_TEST_CONFIG_MEMORY, concurrency=1): 6 | conf = dict(list(CELERY_TEST_CONFIG.__dict__.items()) + list(config.__dict__.items())) 7 | conf['CELERYD_CONCURRENCY'] = concurrency 8 | app.config_from_object(conf) 9 | 10 | 11 | def start_celery_worker(app, config=CELERY_TEST_CONFIG_MEMORY, concurrency=1): 12 | setup_celery_worker(app, config=config, concurrency=concurrency) 13 | 14 | worker = CeleryWorkerThread(app) 15 | worker.daemon = True 16 | worker.start() 17 | worker.ready.wait() 18 | return worker 19 | -------------------------------------------------------------------------------- /celerytest/config.py: -------------------------------------------------------------------------------- 1 | class CELERY_TEST_CONFIG(object): 2 | CELERY_SEND_EVENTS = True 3 | CELERYD_POOL = 'threads' 4 | CELERYD_CONCURRENCY = 1 5 | CELERYD_HIJACK_ROOT_LOGGER = False 6 | CELERYD_LOG_FORMAT = "%(message)s" 7 | 8 | class CELERY_TEST_CONFIG_MEMORY(object): 9 | BROKER_URL = 'memory://' 10 | BROKER_TRANSPORT_OPTIONS = {'polling_interval': .01} 11 | CELERY_RESULT_BACKEND = "cache" 12 | CELERY_CACHE_BACKEND = 'memcached://127.0.0.1:11211/' 13 | 14 | class CELERY_TEST_CONFIG_AMQP(object): 15 | BROKER_URL = 'amqp://' 16 | CELERY_RESULT_BACKEND = "amqp" 17 | CELERY_DEFAULT_QUEUE = 'test_default' 18 | CELERY_DEFAULT_EXCHANGE = 'test_default' 19 | CELERY_RESULT_EXCHANGE = 'test_celeryresults' 20 | CELERY_BROADCAST_QUEUE = 'test_celeryctl' 21 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from setuptools import setup, find_packages 3 | 4 | # convert the readme to pypi compatible rst 5 | try: 6 | from pypandoc import convert 7 | read_md = lambda f: convert(f, 'rst') 8 | except ImportError: 9 | print("warning: pypandoc module not found, could not convert Markdown to RST") 10 | read_md = lambda f: open(f, 'r').read() 11 | 12 | setup( 13 | name='celerytest', 14 | version='0.1.1', 15 | author=u'Willem Bult', 16 | author_email='willem.bult@gmail.com', 17 | packages=find_packages(), 18 | url='http://github.com/RentMethod/celerytest', 19 | download_url='https://github.com/RentMethod/celerytest/archive/0.1.1.tar.gz', 20 | keywords=['celery','testing','integration','test'], 21 | license='BSD license, see LICENSE', 22 | description='Run a monitored Celery worker for integration tests that depend on Celery tasks', 23 | long_description=read_md('README.md'), 24 | zip_safe=False, 25 | include_package_data=True, 26 | install_requires=[ 27 | 'celery>=3.0.19' 28 | ], 29 | test_suite='celerytest.tests' 30 | ) 31 | -------------------------------------------------------------------------------- /celerytest/testcase.py: -------------------------------------------------------------------------------- 1 | from . import start_celery_worker 2 | from celerytest.config import CELERY_TEST_CONFIG_MEMORY 3 | 4 | 5 | class CeleryTestCaseMixin(object): 6 | ''' 7 | Use to run a celery worker in the background for this TestCase. 8 | 9 | Worker is started and stopped on class setup/teardown. 10 | Use self.worker.idle.wait() to make sure tasks have stopped executing. 11 | ''' 12 | celery_config = CELERY_TEST_CONFIG_MEMORY 13 | celery_app = None 14 | celery_concurrency = 1 15 | celery_share_worker = True 16 | 17 | @classmethod 18 | def start_worker(cls): 19 | return start_celery_worker(cls.celery_app, 20 | config=cls.celery_config, 21 | concurrency=cls.celery_concurrency) 22 | 23 | @classmethod 24 | def setup_class(cls): 25 | cls.worker = cls.start_worker() 26 | cls.shared_worker = cls.worker 27 | 28 | @classmethod 29 | def teardown_class(cls): 30 | cls.worker.stop() 31 | cls.worker.join() 32 | 33 | def setUp(self): 34 | if not getattr(self, 'shared_worker', False): 35 | self.worker = self.start_worker() 36 | 37 | def tearDown(self): 38 | if not getattr(self, 'shared_worker', False): 39 | self.worker.stop() 40 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014, RentMethod Inc. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | 13 | Neither the name of RentMethod Inc. nor the names of its contributors may be used 14 | to endorse or promote products derived from this software without specific 15 | prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 19 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 | PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS 21 | BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/RentMethod/celerytest.svg?branch=master)](https://travis-ci.org/RentMethod/celerytest) 2 | 3 | # celerytest - Integration testing with Celery 4 | Writing (integration) tests that depend on Celery tasks is problematic. When you manually run a Celery worker together with your tests, it runs in a separate process and there's no clean way to address objects targeted by Celery from your tests. When you use a separate test database (as with Django for example), you'll have to duplicate configuration code so your Celery worker accesses the same database. 5 | 6 | celerytest provides the ability to run a Celery worker in the background from your tests. It also allows your tests to monitor the worker and pause until Celery tasks are completed. 7 | 8 | ## Using celerytest 9 | 10 | To start a Celery worker in a separate thread, use: 11 | 12 | ```python 13 | app = Celery() # your Celery app 14 | worker = start_celery_worker(app) # configure the app for our celery worker 15 | ``` 16 | 17 | To wait for the worker to finish executing tasks, use: 18 | 19 | ```python 20 | result = some_celery_task.delay() 21 | worker.idle.wait() # optionally specify time-out 22 | ``` 23 | 24 | ### Django 25 | 26 | To use this with your django app through django-celery, get your app as such: 27 | 28 | ```python 29 | from djcelery.app import app 30 | worker = start_celery_worker(app) 31 | ``` 32 | 33 | ### TestCase 34 | 35 | If you want to use this in a unittest TestCase, you can use CeleryTestCaseMixin. If you're writing unit tests that depend on a celery worker, though, you're doing it wrong. For unit tests, you'll want to mock your Celery methods and test them separately. You could use CeleryTestCaseMixin to write integration tests with Celery tasks, though. 36 | 37 | ```python 38 | from unittest import TestCase 39 | from celerytest.testcase import CeleryTestCaseMixin, setup_celery_worker 40 | import time 41 | 42 | app = Celery() 43 | setup_celery_worker(app) # need to setup worker outside 44 | 45 | class SomeTestCase(CeleryTestCaseMixin, TestCase): 46 | celery_app = app 47 | celery_concurrency = 4 48 | 49 | def test_something(self): 50 | result = multiply.delay(2,3) 51 | self.worker.idle.wait() 52 | self.assertEqual(result.get(), 6) 53 | ``` 54 | 55 | ### Lettuce 56 | 57 | To automatically launch a worker in the background while running a Lettuce integration test suite, add to ``terrain.py``: 58 | 59 | ```python 60 | # my_celery_app.py 61 | app = Celery('my_celery_app', broker='amqp://') 62 | 63 | # terrain.py 64 | from lettuce import * 65 | from celerytest import start_celery_worker 66 | 67 | # replace this with an import of your actual app 68 | from my_celery_app import app 69 | 70 | @before.harvest 71 | def initial_setup(server): 72 | # memory transport may not work here 73 | world.celery = start_celery_worker(app, config="amqp") 74 | 75 | @after.harvest 76 | def cleanup(server): 77 | world.celery.stop() 78 | 79 | @after.each_step 80 | def after_step(step): 81 | # make sure we've received any scheduled tasks 82 | world.celery.active.wait(.05) 83 | # allow tasks to complete 84 | world.celery.idle.wait(5) 85 | ``` 86 | 87 | 88 | ## Installation 89 | 90 | Install the latest version of ``celerytest`` from PyPI: 91 | 92 | $ pip install celerytest 93 | 94 | Or, clone the latest version of ``celerytest`` from GitHub and run setup: 95 | 96 | $ git clone git://github.com/RentMethod/celerytest.git 97 | $ cd celerytest 98 | $ ./setup.py install # as root -------------------------------------------------------------------------------- /celerytest/tests.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from . import setup_celery_worker 4 | from .testcase import CeleryTestCaseMixin 5 | from unittest import TestCase 6 | 7 | # set up a test app with some tasks 8 | from celery import Celery 9 | app = Celery() 10 | 11 | setup_celery_worker(app) 12 | 13 | @app.task 14 | def wait_a_second(delay=1): 15 | time.sleep(delay) 16 | return delay 17 | 18 | @app.task 19 | def multiply(a, b): 20 | return a*b 21 | 22 | class WorkerThreadTestCase(CeleryTestCaseMixin, TestCase): 23 | celery_app = app 24 | celery_concurrency = 4 25 | 26 | delay = .1 27 | overhead_time = 0.05 28 | 29 | def test_sequential(self): 30 | x, base, pwr = 1, 2, 5 31 | for i in range(0,pwr): 32 | result = multiply.delay(x, base) 33 | t1 = time.time() 34 | self.worker.active.wait() 35 | t2 = time.time() 36 | self.worker.idle.wait() 37 | t3 = time.time() 38 | self.assertTrue(result.ready()) 39 | x = result.get() 40 | self.assertEqual(x, pow(base,i+1)) 41 | t4 = time.time() 42 | 43 | # see that we started in reasonable time 44 | self.assertTrue(t2-t1 < self.overhead_time) 45 | # see that we finished in reasonable time 46 | self.assertTrue(t3-t2 < self.overhead_time * 2) 47 | # see that we got the result in reasonable time 48 | self.assertTrue(t4-t3 < self.overhead_time) 49 | 50 | self.assertEqual(x, pow(base,pwr)) 51 | 52 | def test_parallel(self): 53 | count = self.celery_concurrency 54 | results = [wait_a_second.delay(self.delay) for i in range(0,count)] 55 | 56 | # make sure we're actually testing parralism here 57 | # total expected time should be less than repeated delay 58 | self.assertTrue(self.delay * count > self.delay + (self.overhead_time * count)) 59 | 60 | t1 = time.time() 61 | 62 | # wait until we start 63 | self.worker.active.wait() 64 | t2 = time.time() 65 | 66 | # then wait until we're done 67 | self.worker.idle.wait() 68 | t3 = time.time() 69 | 70 | # should have all results now 71 | self.assertFalse(False in [r.ready() for r in results]) 72 | t4 = time.time() 73 | 74 | # see that we started in reasonable time 75 | self.assertTrue(t2-t1 < self.overhead_time) 76 | # see that we finished in reasonable time (should have run in parallel) 77 | self.assertTrue(t3-t2 < self.delay + self.overhead_time * count) 78 | # see that we got the results in reasonable time 79 | self.assertTrue(t4-t3 < self.overhead_time) 80 | 81 | def test_scheduled_task(self): 82 | result = multiply.apply_async((2,3), countdown=self.delay) # schedule this for the future 83 | t1 = time.time() 84 | self.worker.active.wait(self.overhead_time) 85 | t2 = time.time() 86 | self.worker.idle.wait() 87 | t3 = time.time() 88 | 89 | # shouldn't have executed yet 90 | self.assertTrue(t2-t1 < self.overhead_time * 2) 91 | self.assertTrue(t3-t2 < self.overhead_time) 92 | self.assertFalse(result.ready()) 93 | 94 | # should become active again soon 95 | self.worker.active.wait(2) 96 | t4 = time.time() 97 | self.worker.idle.wait() 98 | t5 = time.time() 99 | self.assertTrue(result.ready()) 100 | 101 | self.assertTrue(t5-t4 < self.overhead_time) -------------------------------------------------------------------------------- /celerytest/worker.py: -------------------------------------------------------------------------------- 1 | import threading 2 | import socket 3 | from datetime import datetime 4 | from celery import signals, states 5 | 6 | class CeleryWorkerThread(threading.Thread): 7 | ''' 8 | Thread that runs a celery worker while monitoring its state. 9 | 10 | Useful for testing purposes. Use the idle and active signals 11 | to wait for tasks to complete. 12 | ''' 13 | def __init__(self, app): 14 | super(CeleryWorkerThread, self).__init__() 15 | 16 | self.app = app 17 | self.workers = [] 18 | self.consumers = [] 19 | self.monitor = CeleryMonitorThread(app) 20 | 21 | self.ready = threading.Event() 22 | self.active = self.monitor.active 23 | self.idle = self.monitor.idle 24 | 25 | def on_worker_init(self, sender=None, **kwargs): 26 | self.workers.append(sender) 27 | 28 | def on_worker_ready(self, sender=None, **kwargs): 29 | if not self.ready.is_set(): 30 | self.ready.set() 31 | 32 | self.consumers.append(sender) 33 | 34 | def run(self): 35 | signals.worker_init.connect(self.on_worker_init) 36 | signals.worker_ready.connect(self.on_worker_ready) 37 | 38 | self.monitor.daemon = self.daemon 39 | self.monitor.start() 40 | 41 | worker = self.app.Worker() 42 | if hasattr(worker, 'start'): 43 | worker.start() 44 | elif hasattr(worker, 'run'): 45 | worker.run() 46 | else: 47 | raise Exception("Don't know how to start worker. Incompatible Celery?") 48 | 49 | def stop(self): 50 | self.monitor.stop() 51 | 52 | for c in self.consumers: 53 | c.stop() 54 | 55 | for w in self.workers: 56 | w.terminate() 57 | 58 | signals.worker_init.disconnect(self.on_worker_init) 59 | signals.worker_ready.disconnect(self.on_worker_ready) 60 | 61 | def join(self, *args, **kwargs): 62 | self.monitor.join(*args, **kwargs) 63 | super(CeleryWorkerThread, self).join(*args, **kwargs) 64 | 65 | 66 | class CeleryMonitorThread(threading.Thread): 67 | ''' 68 | Monitors a Celery app. Keeps track of pending tasks. 69 | Exposes idle and active events. 70 | ''' 71 | 72 | def __init__(self, app): 73 | super(CeleryMonitorThread, self).__init__() 74 | 75 | self.app = app 76 | self.state = app.events.State() 77 | self.stop_requested = False 78 | 79 | self.pending = 0 80 | self.idle = threading.Event() 81 | self.idle.set() 82 | self.active = threading.Event() 83 | 84 | def on_event(self, event): 85 | # maintain state 86 | self.state.event(event) 87 | 88 | # only need to update state when something relevant to pending tasks is happening 89 | check_states = ['task-received','task-started','task-succeeded','task-failed','task-revoked'] 90 | if not event['type'] in check_states: 91 | return 92 | 93 | active = len(self.immediate_pending_tasks) > 0 94 | 95 | # switch signals if needed 96 | if active and self.idle.is_set(): 97 | self.idle.clear() 98 | self.active.set() 99 | elif not active and self.active.is_set(): 100 | self.idle.set() 101 | self.active.clear() 102 | 103 | @property 104 | def pending_tasks(self): 105 | tasks = self.state.tasks.values() 106 | return [t for t in tasks if t.state in states.UNREADY_STATES] 107 | 108 | @property 109 | def immediate_pending_tasks(self): 110 | now = datetime.utcnow().isoformat() 111 | return [t for t in self.pending_tasks if not t.eta or t.eta < now] 112 | 113 | def run(self): 114 | with self.app.connection() as connection: 115 | recv = self.app.events.Receiver(connection, handlers={ 116 | '*': self.on_event, 117 | }) 118 | 119 | # we want to be able to stop from outside 120 | while not self.stop_requested: 121 | try: 122 | # use timeout so we can monitor if we should stop 123 | recv.capture(limit=None, timeout=.5, wakeup=False) 124 | except socket.timeout: 125 | pass 126 | 127 | def stop(self): 128 | self.stop_requested = True --------------------------------------------------------------------------------