├── .gitignore ├── MANIFEST.in ├── README.rst ├── celery_api.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.egg-info 2 | *.pyc 3 | .*.swp 4 | *~ 5 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst 2 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Celery API discovery module 2 | --------------------------- 3 | 4 | Given the celery instance, it inspects all available celery workers to get 5 | the information about the queues they serve and tasks they know about. 6 | 7 | Then it creates a chain of attributes allowing to execute any task as 8 | ``queue_name.full_task_name.delay``. 9 | 10 | With help of this class you can turn your Celery installation to a set of 11 | independent modules, each of which "exposes" its own "Celery API". 12 | 13 | To make it more clear, the analogy with a random HTTP-based API available 14 | at http://example.com/users/get?email=john@example.com can be like: 15 | 16 | - Celery object (including broker URL, result backend settings, etc) is the 17 | analogue of the protocol (http://) 18 | - Queue name is the analogue of the hostname (example.com) 19 | - Task name is the analogue of the URL path (/users/get) 20 | - Task parameters the the analogue the querystring (?email=john@example.com) 21 | 22 | 23 | Usage example. 24 | 25 | If we have a Celery installation with two queues: 26 | "download" (knows how to execute "downloader.download_url" task) and 27 | "parse" (knows how to execute "parser.parse_html"), we can instantiate 28 | API and work with it the following way: 29 | 30 | 31 | .. code-block:: python 32 | 33 | >>> api = celery_api.CeleryApi(celery) 34 | >>> html_page = api.download.downloader.download_html.delay('http://example.com').get() 35 | >>> html_tree = api.parse.parser.parse_html.delay(html_page).get() 36 | 37 | .. note:: 38 | Ensure that workers are up and available from clients for inspection. 39 | You may re-discover your installation after object creation by executing 40 | ``api._discover()``. 41 | 42 | .. note:: 43 | If you can't launch the task and get a NotRegistered exception instead, 44 | it's most likely you have the ``CELERY_ALWAYS_EAGER`` config option set to 45 | ``True``. 46 | -------------------------------------------------------------------------------- /celery_api.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Celery API discovery module. 4 | """ 5 | from celery import Celery 6 | 7 | 8 | def make_api(*args, **kwargs): 9 | """ 10 | Little shortcut for api discovery 11 | """ 12 | celery = Celery(*args, **kwargs) 13 | return CeleryApi(celery) 14 | 15 | 16 | class CeleryApi(object): 17 | """ 18 | Celery API discovery class. 19 | 20 | Given the celery instance, it inspects all available celery workers to get 21 | the information about the queues they serve and tasks they know about. 22 | 23 | Then it creates a chain of attributes allowing to execute any task as 24 | ``queue_name.full_task_name.delay``. 25 | 26 | With help of this class you can turn your Celery installation to a set of 27 | independent modules, each of which "exposes" its own "Celery API". 28 | 29 | To make it more clear, the analogy with a random HTTP-based API available 30 | at http://example.com/users/get?email=john@example.com can be like: 31 | 32 | - Celery object (including broker URL, result backend settings, etc) is the 33 | analogue of the protocol (http://) 34 | - Queue name is the analogue of the hostname (example.com) 35 | - Task name is the analogue of the URL path (/users/get) 36 | - Task parameters the the analogue the querystring (?email=john@example.com) 37 | 38 | Usage example. 39 | 40 | If we have a Celery installation with two queues: 41 | "download" (knows how to execute "downloader.download_url" task) and 42 | "parse" (knows how to execute "parser.parse_html"), we can instantiate 43 | API and work with it the following way: 44 | 45 | .. code-block:: python 46 | 47 | >>> api = celery_api.CeleryApi(celery) 48 | >>> html_page = api.download.downloader.download_html.delay('http://example.com').get() 49 | >>> html_tree = api.parse.parser.parse_html.delay(html_page).get() 50 | 51 | .. note:: 52 | Ensure that workers are up and available from clients for inspection. 53 | You may re-discover your installation after object creation by executing 54 | ``api._discover()``. 55 | """ 56 | 57 | def __init__(self, celery): 58 | self._celery = celery 59 | self._inspect = self._celery.control.inspect() 60 | self._discover() 61 | 62 | def _discover(self): 63 | """ 64 | Discover and expose the API by creating a chain of object attributes 65 | """ 66 | for queue_name, task_list in self._get_tasks().items(): 67 | if not hasattr(self, queue_name): 68 | setattr(self, queue_name, TaskProxy()) 69 | queue = getattr(self, queue_name) 70 | 71 | for task_name in task_list: 72 | chunks = task_name.split('.') # ['foo', 'bar', 'do_smth'] 73 | current = queue 74 | for chunk in chunks[:-1]: 75 | if not hasattr(current, chunk): 76 | setattr(current, chunk, TaskProxy()) 77 | current = getattr(current, chunk) 78 | task = self._celery.Task() 79 | task.name = task_name 80 | setattr(current, chunks[-1], task) 81 | 82 | def _get_tasks(self): 83 | """ 84 | Helper function: get the dict: queue_name -> [task name, ...] 85 | """ 86 | queues = {} 87 | # get the list of registered tasks: worker -> [task1, task2, ...] 88 | registered = self._inspect.registered() 89 | 90 | active_queues = self._inspect.active_queues() or {} 91 | for worker_name, queue_info_list in active_queues.items(): 92 | for queue_info in queue_info_list: 93 | queue_name = queue_info['name'] 94 | queues.setdefault(queue_name, set()) 95 | worker_tasks = registered.get(worker_name, []) 96 | queues[queue_name].update(worker_tasks) 97 | 98 | return queues 99 | 100 | 101 | class TaskProxy(object): 102 | pass 103 | 104 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | from setuptools import setup 4 | 5 | 6 | def read(fname): 7 | try: 8 | return open(os.path.join(os.path.dirname(__file__), fname)).read() 9 | except: 10 | return '' 11 | 12 | 13 | setup( 14 | name='celery-api', 15 | version='0.1', 16 | py_modules=['celery_api'], 17 | url='https://github.com/imankulov/celery-api', 18 | license='BSD', 19 | author='Roman Imankulov', 20 | author_email='roman.imankulov@gmail.com', 21 | description='Celery API discovery module', 22 | long_description=read('README.rst'), 23 | classifiers=( 24 | 'Development Status :: 3 - Alpha', 25 | 'Intended Audience :: Developers', 26 | 'License :: OSI Approved :: BSD License', 27 | 'Programming Language :: Python', 28 | ), 29 | install_requires=[ 30 | 'celery>=3.0', 31 | ], 32 | ) 33 | --------------------------------------------------------------------------------