├── .gitignore ├── .travis.yml ├── AUTHORS ├── Changelog ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README ├── README.rst ├── cell ├── __init__.py ├── actors.py ├── agents.py ├── bin │ ├── __init__.py │ ├── base.py │ └── cell.py ├── exceptions.py ├── g │ ├── __init__.py │ └── eventlet.py ├── groups.py ├── models.py ├── presence.py ├── results.py ├── tests │ ├── __init__.py │ ├── actors │ │ ├── __init__.py │ │ ├── test_actors.py │ │ ├── test_agents.py │ │ └── test_results.py │ └── utils.py ├── utils │ ├── __init__.py │ ├── custom_operators.py │ └── utils.py └── workflow │ ├── __init__.py │ ├── common.py │ ├── entities.py │ └── monads.py ├── docs ├── .static │ └── .keep ├── .templates │ ├── page.html │ ├── sidebarintro.html │ └── sidebarlogo.html ├── Makefile ├── _ext │ ├── applyxrefs.py │ └── literals_to_xrefs.py ├── _theme │ └── celery │ │ ├── static │ │ └── celery.css_t │ │ └── theme.conf ├── changelog.rst ├── conf.py ├── getting-started │ ├── hello-world-example.rst │ ├── index.rst │ └── more-examples.rst ├── images │ ├── celery-icon-128.png │ ├── celery-icon-32.png │ ├── celery-icon-64.png │ ├── celery_128.png │ ├── celery_512.png │ ├── celery_favicon_128.png │ ├── favicon.ico │ └── favicon.png ├── index.rst ├── introduction.rst ├── reference │ ├── cell.actors.rst │ ├── cell.agents.rst │ ├── cell.bin.base.rst │ ├── cell.bin.cell.rst │ ├── cell.exceptions.rst │ ├── cell.g.eventlet.rst │ ├── cell.g.rst │ ├── cell.models.rst │ ├── cell.presence.rst │ ├── cell.results.rst │ ├── cell.utils.rst │ └── index.rst └── user-guide │ ├── delivery-options.png │ └── index.rst ├── examples ├── __init__.py ├── adder.py ├── chat.py ├── clex.py ├── distributed_cache.py ├── flowlet.py ├── hello.py ├── map_reduce.py ├── tasks.py └── workflow.py ├── extra └── release │ ├── bump_version.py │ ├── doc4allmods │ ├── flakeplus.py │ ├── prepy3ktest │ ├── py3k-run-tests │ ├── removepyc.sh │ └── verify-reference-index.sh ├── requirements ├── default.txt ├── docs.txt ├── pkgutils.txt └── test.txt ├── setup.cfg ├── setup.py └── tox.ini /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.pyc 3 | *$py.class 4 | *~ 5 | *.sqlite 6 | *.sqlite-journal 7 | settings_local.py 8 | .build 9 | build 10 | .*.sw[po] 11 | dist/ 12 | *.egg-info 13 | doc/__build/* 14 | pip-log.txt 15 | devdatabase.db 16 | ^parts 17 | ^eggs 18 | ^bin 19 | developer-eggs 20 | downloads 21 | Documentation/* 22 | docs/.build/* 23 | .tox/ 24 | nosetests.xml 25 | kombu/tests/cover 26 | kombu/tests/coverage.xml 27 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | sudo: required 3 | dist: xenial 4 | cache: pip 5 | 6 | env: 7 | global: 8 | PYTHONUNBUFFERED=yes 9 | 10 | 11 | matrix: 12 | include: 13 | - python: 3.7 14 | env: TOXENV=3.7 15 | - python: 3.6 16 | env: TOXENV=3.6 17 | 18 | 19 | install: 20 | - pip install -U pip setuptools wheel | cat 21 | - pip install -U tox | cat 22 | script: tox -v -- -v 23 | after_success: 24 | - .tox/$TRAVIS_PYTHON_VERSION/bin/coverage xml 25 | - .tox/$TRAVIS_PYTHON_VERSION/bin/codecov -e TOXENV 26 | notifications: 27 | irc: 28 | channels: 29 | - "chat.freenode.net#celery" 30 | on_success: change 31 | on_failure: change -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | ========= 2 | AUTHORS 3 | ========= 4 | 5 | Ask Solem 6 | Asif Saif Uddin 7 | Rumyana Neykova 8 | -------------------------------------------------------------------------------- /Changelog: -------------------------------------------------------------------------------- 1 | ================ 2 | Change history 3 | ================ 4 | 5 | .. _version-0.0.1: 6 | 7 | 0.0.1 8 | ===== 9 | :release-date: TBA 10 | 11 | * Initial version. 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011-2013 GoPivotal, Inc. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | 13 | Neither the name of GoPivotal, Inc. nor the names of its contributors may be used 14 | to endorse or promote products derived from this software without specific 15 | prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 19 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 | PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS 21 | BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include AUTHORS 2 | include Changelog 3 | include LICENSE 4 | include MANIFEST.in 5 | include README.rst 6 | include README 7 | include setup.cfg 8 | recursive-include cell *.py 9 | recursive-include requirements *.txt 10 | recursive-include examples *.py 11 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PYTHON=python 2 | SPHINX_DIR="docs/" 3 | SPHINX_BUILDDIR="${SPHINX_DIR}/.build" 4 | README="README.rst" 5 | README_SRC="docs/templates/readme.txt" 6 | CONTRIBUTING_SRC="docs/contributing.rst" 7 | SPHINX2RST="extra/release/sphinx-to-rst.py" 8 | 9 | SPHINX_HTMLDIR = "${SPHINX_BUILDDIR}/html" 10 | 11 | html: 12 | (cd "$(SPHINX_DIR)"; make html) 13 | mv "$(SPHINX_HTMLDIR)" Documentation 14 | 15 | docsclean: 16 | -rm -rf "$(SPHINX_BUILDDIR)" 17 | 18 | htmlclean: 19 | -rm -rf "$(SPHINX)" 20 | 21 | apicheck: 22 | extra/release/doc4allmods cell 23 | 24 | indexcheck: 25 | extra/release/verify-reference-index.sh 26 | 27 | configcheck: 28 | PYTHONPATH=. $(PYTHON) extra/release/verify_config_reference.py $(CONFIGREF_SRC) 29 | 30 | flakecheck: 31 | flake8 cell 32 | 33 | flakediag: 34 | -$(MAKE) flakecheck 35 | 36 | flakepluscheck: 37 | flakeplus cell --2.6 38 | 39 | flakeplusdiag: 40 | -$(MAKE) flakepluscheck 41 | 42 | flakes: flakediag flakeplusdiag 43 | 44 | readmeclean: 45 | -rm -f $(README) 46 | 47 | readmecheck: 48 | iconv -f ascii -t ascii $(README) >/dev/null 49 | 50 | $(README): 51 | $(PYTHON) $(SPHINX2RST) $(README_SRC) --ascii > $@ 52 | 53 | readme: readmeclean $(README) readmecheck 54 | 55 | test: 56 | nosetests -xv cell.tests 57 | 58 | cov: 59 | nosetests -xv cell.tests --with-coverage --cover-html --cover-branch 60 | 61 | removepyc: 62 | -find . -type f -a \( -name "*.pyc" -o -name "*$$py.class" \) | xargs rm 63 | -find . -type d -name "__pycache__" | xargs rm -r 64 | 65 | gitclean: 66 | git clean -xdn 67 | 68 | gitcleanforce: 69 | git clean -xdf 70 | 71 | bump_version: 72 | $(PYTHON) extra/release/bump_version.py cell/__init__.py README.rst 73 | 74 | distcheck: flakecheck apicheck indexcheck configcheck readmecheck test gitclean 75 | 76 | dist: readme docsclean gitcleanforce removepyc 77 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | README.rst -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ############################################# 2 | cell - Actor framework 3 | ############################################# 4 | 5 | :Version: 0.0.3 6 | 7 | Synopsis 8 | ======== 9 | 10 | `cell` is an actor framework for `Kombu`_ and `celery`_ . 11 | 12 | .. _`Kombu`: http://pypi.python.org/pypi/kombu 13 | .. _`celery`: http://pypi.python.org/pypi/celery 14 | 15 | 16 | What is an Actor 17 | ================ 18 | 19 | The actor model was first proposed by Carl Hewitt in 1973 `[1]`_ and was improved, among others, 20 | by Gul Agha `[2]`_. 21 | 22 | .. _`[1]`: http://dl.acm.org/citation.cfm?id=1624804 23 | .. _`[2]`: http://dl.acm.org/citation.cfm?id=7929 24 | 25 | An Actor is an entity (a class in cell), that has a mailbox and a behaviour. Actors communicate between each other only by exchanging messages. 26 | Upon receiving a message, the behaviour of the actor is executed, upon which the actor can send a number of messages to other actors, 27 | create a number of actors or change its internal state. 28 | 29 | Cell supports: 30 | 31 | * distributed actors (actors are started on celery workers) 32 | * Remoting: Communicating with actors running on other hosts 33 | * Routers: it supports round-robin, direct and broadcast delivery of actor messages. You can create custom routers on top of it, implementing Actors as routers (joiner, collector, gatherer). 34 | 35 | Why should I use it? 36 | ==================== 37 | 38 | In a nutshell: 39 | 40 | * Horizontal scalability with actors across multiple nodes 41 | * You get asynchronous message passing for free 42 | * If you are already using celery, all comes for free, no additional setup is required 43 | * Control over the tasks distribution 44 | * More flexible configurations of nodes 45 | * Well known abstraction 46 | * Easy learning curve (check teh 30 sec video to get you started) 47 | 48 | if you are a celery user: 49 | ------------------------- 50 | * You can use Actors, instead of task-based classes: 51 | (You can program with classes and not tasks) 52 | 53 | * Stateful execution. You can link actors together and their execution, creating complex workflows. 54 | You can control the execution per actor/not per worker. 55 | 56 | * Better control over work distribution (You can target the same worker for a given task): 57 | 58 | .. code-block:: python 59 | 60 | adder.send.add(2, 2) 61 | adder.send.add(2, 2) 62 | 63 | .. If you are an actor frameworks user: 64 | .. ----------------------------------- 65 | 66 | .. Cell supports different routers, distributed actors, supervision strategies (retry, exceptions). 67 | .. Typing actors, workflows with actors and checks on the workflow. 68 | .. Workflow in cell: 69 | .. Distributed work: 70 | .. Flexible configuration: (with actors you can implement routing behaviour that is needed) 71 | 72 | If you are a general Pythonist 73 | ------------------------------ 74 | Having a framework for distributed actor management in your toolbox is a must, bacause: 75 | 76 | * simplify the distributed processing of tasks. 77 | * vertical scalability: 78 | * Fair work distribution, load balancing, sticky routing 79 | 80 | .. Features you will love: 81 | .. ~~~~~~~~~~~~~~~~~~~~~~~ 82 | .. 83 | .. - exceptions happened during background processing are collected and can be browsed later 84 | .. - workflows: run Task1, pass it's results to Task2, Task3 and Task4 which are run in parallel, collect their results and pass to Task5 85 | .. - How do you handle tasks 86 | 87 | 88 | Installation 89 | ============ 90 | 91 | You can install `cell` either via the Python Package Index (PyPI) 92 | or from source. 93 | 94 | To install using `pip`,:: 95 | 96 | $ pip install cell 97 | 98 | To install using `easy_install`,:: 99 | 100 | $ easy_install cell 101 | 102 | If you have downloaded a source tarball you can install it 103 | by doing the following,:: 104 | 105 | $ python setup.py build 106 | # python setup.py install # as root 107 | 108 | 109 | Quick how-to 110 | ============ 111 | If you are too impatient to start, here are the 3 quick steps you need to run 'Hello, world!' in cell: 112 | (You can also check the `Demo`_ video) 113 | 114 | .. _`Demo`: http://www.doc.ic.ac.uk/~rn710/videos/FirstSteps.mp4 115 | 116 | * Define an Actor 117 | 118 | .. code-block:: python 119 | 120 | from cell.actors import Actor 121 | 122 | class GreetingActor(Actor): 123 | class state: 124 | def greet(self, who='world'): 125 | print 'Hello %s' % who 126 | 127 | 128 | 129 | * Start celery with an amqp broker support 130 | 131 | .. code-block:: python 132 | 133 | >>> celery worker -b 'pyamqp://guest@localhost' 134 | 135 | * Invoke a method on an actor instance: 136 | .. code-block:: python 137 | 138 | from cell.agents import dAgent 139 | from kombu import Connection 140 | from examples.greeting import GreetingActor 141 | 142 | connection = Connection('amqp://guest:guest@localhost:5672//') 143 | agent = dAgent(connection) 144 | greeter = agent.spawn(GreetingActor) 145 | greeter.call('greet') 146 | 147 | The full source code of the example from :py:mod:`examples` module. 148 | To understand what is going on check the :ref:`Getting started ` section. 149 | 150 | 151 | Getting Help 152 | ============ 153 | 154 | Mailing list 155 | ------------ 156 | 157 | Join the `celery-users`_ mailing list. 158 | 159 | .. _`celery-users`: http://groups.google.com/group/celery-users/ 160 | 161 | Bug tracker 162 | =========== 163 | 164 | If you have any suggestions, bug reports or annoyances please report them 165 | to our issue tracker at http://github.com/celery/cell/issues/ 166 | 167 | Contributing 168 | ============ 169 | 170 | Development of `cell` happens at Github: http://github.com/celery/cell 171 | 172 | You are highly encouraged to participate in the development. If you don't 173 | like Github (for some reason) you're welcome to send regular patches. 174 | 175 | License 176 | ======= 177 | 178 | This software is licensed under the `New BSD License`. See the `LICENSE` 179 | file in the top distribution directory for the full license text. 180 | 181 | Copyright 182 | ========= 183 | 184 | Copyright (C) 2011-2013 GoPivotal, Inc. 185 | 186 | cell as part of the Tidelift Subscription 187 | ========= 188 | 189 | The maintainers of cell and thousands of other packages are working with Tidelift to deliver commercial support and maintenance for the open source dependencies you use to build your applications. Save time, reduce risk, and improve code health, while paying the maintainers of the exact dependencies you use. [Learn more.](https://tidelift.com/subscription/pkg/pypi-cell?utm_source=pypi-cell&utm_medium=referral&utm_campaign=readme&utm_term=repo) 190 | 191 | -------------------------------------------------------------------------------- /cell/__init__.py: -------------------------------------------------------------------------------- 1 | """Kombu actor framework""" 2 | import sys 3 | # Lazy loading. 4 | # - See werkzeug/__init__.py for the rationale behind this. 5 | from types import ModuleType 6 | 7 | 8 | VERSION = (0, 0, 3) 9 | __version__ = '.'.join(map(str, VERSION[0:3])) + ''.join(VERSION[3:]) 10 | __author__ = 'Ask Solem' 11 | __contact__ = 'ask@celeryproject.org' 12 | __homepage__ = 'http://github.com/celery/cell/' 13 | __docformat__ = 'restructuredtext en' 14 | 15 | # -eof meta- 16 | 17 | 18 | all_by_module = { 19 | 'cell.actors': ['Actor'], 20 | 'cell.agents': ['Agent'], 21 | } 22 | 23 | object_origins = {} 24 | for module, items in all_by_module.items(): 25 | for item in items: 26 | object_origins[item] = module 27 | 28 | 29 | class module(ModuleType): 30 | 31 | def __getattr__(self, name): 32 | if name in object_origins: 33 | module = __import__(object_origins[name], None, None, [name]) 34 | for extra_name in all_by_module[module.__name__]: 35 | setattr(self, extra_name, getattr(module, extra_name)) 36 | return getattr(module, name) 37 | return ModuleType.__getattribute__(self, name) 38 | 39 | def __dir__(self): 40 | result = list(new_module.__all__) 41 | result.extend(('__file__', '__path__', '__doc__', '__all__', 42 | '__docformat__', '__name__', '__path__', 'VERSION', 43 | '__package__', '__version__', '__author__', 44 | '__contact__', '__homepage__', '__docformat__')) 45 | return result 46 | 47 | 48 | # keep a reference to this module so that it's not garbage collected 49 | old_module = sys.modules[__name__] 50 | 51 | new_module = sys.modules[__name__] = module(__name__) 52 | new_module.__dict__.update({ 53 | '__file__': __file__, 54 | '__path__': __path__, 55 | '__doc__': __doc__, 56 | '__all__': tuple(object_origins), 57 | '__version__': __version__, 58 | '__author__': __author__, 59 | '__contact__': __contact__, 60 | '__homepage__': __homepage__, 61 | '__docformat__': __docformat__, 62 | 'VERSION': VERSION}) 63 | -------------------------------------------------------------------------------- /cell/agents.py: -------------------------------------------------------------------------------- 1 | """cell.agents""" 2 | 3 | from inspect import isclass 4 | from operator import itemgetter 5 | import weakref 6 | 7 | from kombu.common import uuid, ignore_errors 8 | from kombu.five import items, values 9 | from kombu.log import get_logger, setup_logging 10 | from kombu.mixins import ConsumerMixin 11 | from kombu.utils import symbol_by_name 12 | 13 | from .actors import Actor, ActorProxy, ACTOR_TYPE 14 | from .utils import qualname, first_reply 15 | 16 | __all__ = ['Agent', 'dAgent'] 17 | 18 | logger = get_logger(__name__) 19 | debug, warn, error = logger.debug, logger.warn, logger.error 20 | 21 | 22 | class dAgent(Actor): 23 | types = (ACTOR_TYPE.RR, ACTOR_TYPE.SCATTER, ACTOR_TYPE.DIRECT) 24 | MAX_ACTORS = 2 25 | 26 | class state: 27 | def __init__(self): 28 | self.registry = {} 29 | 30 | def _start_actor_consumer(self, actor): 31 | actor.consumer = actor.Consumer(self.connection.channel()) 32 | actor.consumer.consume() 33 | self.registry[actor.id] = actor 34 | actor.agent = weakref.proxy(self.agent) 35 | actor.on_agent_ready() 36 | 37 | def spawn(self, cls, id, kwargs={}): 38 | """Add actor to the registry and start the actor's main method.""" 39 | try: 40 | actor = symbol_by_name(cls)( 41 | connection=self.connection, id=id, **kwargs) 42 | 43 | if actor.id in self.registry: 44 | warn('Actor id %r already exists', actor.id) 45 | self._start_actor_consumer(actor) 46 | debug('Actor registered: %s', cls) 47 | return actor.id 48 | except Exception as exc: 49 | error('Cannot start actor: %r', exc, exc_info=True) 50 | 51 | def stop_all(self): 52 | self.agent.shutdown() 53 | 54 | def reset(self): 55 | debug('Resetting active actors') 56 | for actor in values(self.registry): 57 | if actor.consumer: 58 | ignore_errors(self.connection, actor.consumer.cancel) 59 | actor.connection = self.connection 60 | self._start_actor_consumer(actor) 61 | 62 | def kill(self, actor_id): 63 | if actor_id not in self.registry: 64 | raise Actor.Next() 65 | else: 66 | actor = self.registry.pop(actor_id) 67 | if actor.consumer and actor.consumer.channel: 68 | ignore_errors(self.connection, actor.consumer.cancel) 69 | 70 | def select(self, cls): 71 | for key, val in items(self.registry): 72 | if qualname(val.__class__) == cls: 73 | return key 74 | # delegate to next agent. 75 | raise Actor.Next() 76 | 77 | def _shutdown(self, cancel=True, close=True, clear=True): 78 | try: 79 | for actor in values(self.registry): 80 | if actor and actor.consumer: 81 | if cancel: 82 | ignore_errors(self.connection, 83 | actor.consumer.cancel) 84 | if close and actor.consumer.channel: 85 | ignore_errors(self.connection, 86 | actor.consumer.channel.close) 87 | finally: 88 | if clear: 89 | self.registry.clear() 90 | 91 | def __init__(self, connection, id=None): 92 | self.registry = {} 93 | Actor.__init__(self, connection=connection, id=id, agent=self) 94 | 95 | def spawn_group(self, group, cls, n=1, nowait=False): 96 | return self.spawn( 97 | group, {'act_type': qualname(cls), 'number': n}, nowait) 98 | 99 | def spawn(self, cls, kwargs={}, nowait=False): 100 | """Spawn a new actor on a celery worker by sending 101 | a remote command to the worker. 102 | 103 | :param cls: the name of the :class:`~.cell.actors.Actor` class or its 104 | derivative. 105 | 106 | :keyword kwargs: The keyword arguments to pass on to 107 | actor __init__ (a :class:`dict`) 108 | 109 | :keyword nowait: If set to True (default) the call waits for the 110 | result of spawning the actor. if False, the spawning 111 | is asynchronous. 112 | 113 | :returns :class:`~.cell.actors.ActorProxy`:, 114 | holding the id of the spawned actor. 115 | """ 116 | 117 | actor_id = uuid() 118 | 119 | if str(qualname(cls)) == '__builtin__.unicode': 120 | name = cls 121 | else: 122 | name = qualname(cls) 123 | 124 | res = self.call('spawn', {'cls': name, 'id': actor_id, 125 | 'kwargs': kwargs}, 126 | type=ACTOR_TYPE.RR, nowait=nowait) 127 | return ActorProxy(name, actor_id, res, agent=self, 128 | connection=self.connection, **kwargs) 129 | 130 | def select(self, cls, **kwargs): 131 | """Get the id of already spawned actor 132 | 133 | :keyword actor: the name of the :class:`Actor` class 134 | """ 135 | name = qualname(cls) 136 | id = first_reply( 137 | self.scatter('select', {'cls': name}, limit=1), cls) 138 | return ActorProxy(name, id, agent=self, 139 | connection=self.connection, **kwargs) 140 | 141 | def kill(self, actor_id, nowait=False): 142 | return self.scatter('kill', {'actor_id': actor_id}, 143 | nowait=nowait) 144 | 145 | # ------------------------------------------------------------ 146 | # Control methods. To be invoked locally 147 | # ------------------------------------------------------------ 148 | 149 | def start(self): 150 | debug('Starting agent %s', self.id) 151 | consumer = self.Consumer(self.connection.channel()) 152 | consumer.consume() 153 | self.state.reset() 154 | 155 | def stop(self): 156 | 157 | debug('Stopping agent %s', self.id) 158 | self.state._shutdown(clear=False) 159 | 160 | def shutdown(self): 161 | debug('Shutdown agent %s', self.id) 162 | self.state._shutdown(cancel=False) 163 | 164 | def process_message(self, actor, body, message): 165 | """Process actor message depending depending on the the worker settings. 166 | 167 | If greenlets are enabled in the worker, the actor message is processed 168 | in a greenlet from the greenlet pool, 169 | Otherwise, the message is processed by the same thread. 170 | The method is invoked from the callback `cell.actors.Actor.on_message` 171 | upon receiving of a message. 172 | 173 | :keyword actor: instance of :class:`Actor` or its derivative. 174 | The actor instance to process the message. 175 | 176 | For the full list of arguments see 177 | :meth:`cell.actors.Actor._on_message`. 178 | 179 | """ 180 | if actor is not self and self.is_green(): 181 | self.pool.spawn_n(actor._on_message, body, message) 182 | else: 183 | if not self.is_green() and message.properties.get('reply_to'): 184 | warn('Starting a blocking call (%s) on actor (%s) ' 185 | 'when greenlets are disabled.', 186 | itemgetter('method')(body), actor.__class__) 187 | actor._on_message(body, message) 188 | 189 | def is_green(self): 190 | return self.pool is not None and self.pool.is_green 191 | 192 | def get_default_scatter_limit(self): 193 | return None 194 | 195 | 196 | class Agent(ConsumerMixin): 197 | actors = [] 198 | 199 | def __init__(self, connection, id=None, actors=None): 200 | self.connection = connection 201 | self.id = id or uuid() 202 | if actors is not None: 203 | self.actors = actors 204 | self.actors = self.prepare_actors() 205 | 206 | def on_run(self): 207 | pass 208 | 209 | def run(self): 210 | self.info('Agent on behalf of [%s] starting...', 211 | ', '.join(actor.name for actor in self.actors)) 212 | self.on_run() 213 | super().run() 214 | 215 | def stop(self): 216 | pass 217 | 218 | def on_consume_ready(self, *args, **kwargs): 219 | for actor in self.actors: 220 | actor.on_agent_ready() 221 | 222 | def run_from_commandline(self, loglevel='INFO', logfile=None): 223 | setup_logging(loglevel, logfile) 224 | try: 225 | self.run() 226 | except KeyboardInterrupt: 227 | self.info('[Quit requested by user]') 228 | 229 | def _maybe_actor(self, actor): 230 | if isclass(actor): 231 | return actor(self.connection) 232 | return actor 233 | 234 | def prepare_actors(self): 235 | return [self._maybe_actor(actor).bind(self.connection, self) 236 | for actor in self.actors] 237 | 238 | def get_consumers(self, Consumer, channel): 239 | return [actor.Consumer(channel) for actor in self.actors] 240 | 241 | def get_default_scatter_limit(self, actor): 242 | return None 243 | -------------------------------------------------------------------------------- /cell/bin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/auvipy/cell/54e474592112b90ebb63aec1e6ef1856619e0e14/cell/bin/__init__.py -------------------------------------------------------------------------------- /cell/bin/base.py: -------------------------------------------------------------------------------- 1 | """cell.bin.base""" 2 | 3 | import optparse 4 | import os 5 | import sys 6 | 7 | from cell import __version__ 8 | 9 | __all__ = ['Option', 'Command'] 10 | Option = optparse.make_option 11 | 12 | 13 | class Command(object): 14 | Parser = optparse.OptionParser 15 | 16 | args = '' 17 | version = __version__ 18 | option_list = () 19 | prog_name = None 20 | 21 | def run(self, *args, **options): 22 | raise NotImplementedError('subclass responsibility') 23 | 24 | def execute_from_commandline(self, argv=None): 25 | """Execute application from command line. 26 | 27 | :keyword argv: The list of command line arguments. 28 | Defaults to ``sys.argv``. 29 | 30 | """ 31 | if argv is None: 32 | argv = list(sys.argv) 33 | self.prog_name = os.path.basename(argv[0]) 34 | return self.handle_argv(self.prog_name, argv[1:]) 35 | 36 | def usage(self): 37 | """Returns the command-line usage string for this app.""" 38 | return '%%prog [options] %s' % (self.args, ) 39 | 40 | def get_options(self): 41 | """Get supported command line options.""" 42 | return self.option_list 43 | 44 | def handle_argv(self, prog_name, argv): 45 | """Parses command line arguments from ``argv`` and dispatches 46 | to :meth:`run`. 47 | 48 | :param prog_name: The program name (``argv[0]``). 49 | :param argv: Command arguments. 50 | 51 | """ 52 | options, args = self.parse_options(prog_name, argv) 53 | return self.run(*args, **vars(options)) 54 | 55 | def exit(self, v=0): 56 | sys.exit(v) 57 | 58 | def exit_status(self, msg, status=0, fh=sys.stderr): 59 | fh.write('%s\n' % (msg, )) 60 | self.exit(status) 61 | 62 | def exit_usage(self, msg): 63 | sys.stderr.write('ERROR: %s\n\n' % (msg, )) 64 | self.exit_status('Usage: %s' % ( 65 | self.usage().replace('%prog', self.prog_name), )) 66 | 67 | def parse_options(self, prog_name, arguments): 68 | """Parse the available options.""" 69 | # Don't want to load configuration to just print the version, 70 | # so we handle --version manually here. 71 | if '--version' in arguments: 72 | self.exit_status(self.version, fh=sys.stdout) 73 | parser = self.create_parser(prog_name) 74 | options, args = parser.parse_args(arguments) 75 | return options, args 76 | 77 | def create_parser(self, prog_name): 78 | return self.Parser(prog=prog_name, 79 | usage=self.usage(), 80 | version=self.version, 81 | option_list=self.get_options()) 82 | -------------------------------------------------------------------------------- /cell/bin/cell.py: -------------------------------------------------------------------------------- 1 | """cell.bin.cell""" 2 | 3 | from kombu import Connection 4 | 5 | from .base import Command, Option 6 | from cell import Agent 7 | from cell.utils import instantiate 8 | 9 | __all__ = ['cell', 'main'] 10 | 11 | DEFAULT_BROKER_URL = 'amqp://guest:guest@localhost:5672//' 12 | 13 | 14 | class cell(Command): 15 | args = '' 16 | 17 | option_list = ( 18 | Option('-i', '--id', 19 | default=None, action='store', dest='id', 20 | help='Id of the agent (or automatically generated).'), 21 | Option('-l', '--loglevel', 22 | default=None, action='store', dest='loglevel', 23 | help='Loglevel (CRITICAL/ERROR/WARNING/INFO/DEBUG).'), 24 | Option('-f', '--logfile', 25 | default=None, action='store', dest='logfile', 26 | help='Logfile. Default is stderr.'), 27 | Option('-b', '--broker', 28 | default=DEFAULT_BROKER_URL, action='store', dest='broker', 29 | help='Broker URL. Default is %s' % (DEFAULT_BROKER_URL, )), 30 | ) 31 | 32 | def run(self, *actors, **kwargs): 33 | if not actors: 34 | self.exit_usage('No actor specified') 35 | 36 | actors = [instantiate(actor) for actor in list(actors)] 37 | 38 | connection = Connection(kwargs.get('broker')) 39 | agent = Agent(connection, actors=actors, id=kwargs.get('id')) 40 | agent.run_from_commandline(loglevel=kwargs.get('loglevel'), 41 | logfile=kwargs.get('logfile')) 42 | 43 | 44 | def main(argv=None): 45 | return cell().execute_from_commandline(argv) 46 | 47 | 48 | if __name__ == '__main__': 49 | main() 50 | -------------------------------------------------------------------------------- /cell/exceptions.py: -------------------------------------------------------------------------------- 1 | """cell.exceptions""" 2 | 3 | 4 | __all__ = ['CellError', 'Next', 'NoReplyError', 'NotBoundError'] 5 | 6 | FRIENDLY_ERROR_FMT = """ 7 | Remote method raised exception: 8 | ------------------------------------ 9 | %s 10 | """ 11 | 12 | 13 | class CellError(Exception): 14 | """Remote method raised exception.""" 15 | exc = None 16 | traceback = None 17 | 18 | def __init__(self, exc=None, traceback=None): 19 | self.exc = exc 20 | self.traceback = traceback 21 | Exception.__init__(self, exc, traceback) 22 | 23 | def __str__(self): 24 | return FRIENDLY_ERROR_FMT % (self.traceback, ) 25 | 26 | 27 | class Next(Exception): 28 | """Used in a gather scenario to signify that no reply should be sent, 29 | to give another agent the chance to reply.""" 30 | pass 31 | 32 | 33 | class NoReplyError(Exception): 34 | """No reply received within time constraint""" 35 | pass 36 | 37 | 38 | class NotBoundError(Exception): 39 | """Object is not bound to a connection.""" 40 | pass 41 | 42 | 43 | class NoRouteError(Exception): 44 | """Presence: No known route for wanted item.""" 45 | pass 46 | 47 | 48 | class WrongNumberOfArguments(Exception): 49 | """An actor call method is invoked without arguments""" 50 | pass 51 | -------------------------------------------------------------------------------- /cell/g/__init__.py: -------------------------------------------------------------------------------- 1 | from kombu.five import keys 2 | from kombu.syn import detect_environment 3 | 4 | from cell.utils import cached_property 5 | 6 | G_NOT_FOUND = """\ 7 | cell does not currently support {0!r}, please use one of {1}\ 8 | """ 9 | 10 | 11 | class G(object): 12 | map = {'eventlet': '_eventlet'} 13 | 14 | def spawn(self, fun, *args, **kwargs): 15 | return self.current.spawn(fun, *args, **kwargs) 16 | 17 | def timer(self, interval, fun, *args, **kwargs): 18 | return self.current.timer(interval, fun, *args, **kwargs) 19 | 20 | def blocking(self, fun, *args, **kwargs): 21 | return self.current.blocking(fun, *args, **kwargs) 22 | 23 | def Queue(self, *args, **kwargs): 24 | return self.current.Queue(*args, **kwargs) 25 | 26 | def Event(self, *args, **kwargs): 27 | return self.current.Event(*args, **kwargs) 28 | 29 | @cached_property 30 | def _eventlet(self): 31 | from . import eventlet 32 | return eventlet 33 | 34 | @cached_property 35 | def current(self): 36 | type = detect_environment() 37 | try: 38 | return getattr(self, self.map[type]) 39 | except KeyError: 40 | raise KeyError(G_NOT_FOUND.format( 41 | type, ', '.join(keys(self.map)))) 42 | 43 | 44 | g = G() 45 | blocking = g.blocking 46 | spawn = g.spawn 47 | timer = g.timer 48 | Queue = g.Queue 49 | Event = g.Event 50 | -------------------------------------------------------------------------------- /cell/g/eventlet.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | 3 | from eventlet import Timeout # noqa 4 | from eventlet import event 5 | from eventlet import greenthread 6 | from eventlet import queue 7 | from greenlet import GreenletExit 8 | from kombu import syn 9 | 10 | blocking = syn.blocking 11 | spawn = greenthread.spawn 12 | Queue = queue.LightQueue 13 | Event = event.Event 14 | 15 | 16 | class Entry(object): 17 | g = None 18 | 19 | def __init__(self, interval, fun, *args, **kwargs): 20 | self.interval = interval 21 | self.fun = fun 22 | self.args = args 23 | self.kwargs = kwargs 24 | self.cancelled = False 25 | self._spawn() 26 | 27 | def _spawn(self): 28 | self.g = greenthread.spawn_after_local(self.interval, self) 29 | self.g.link(self._exit) 30 | 31 | def __call__(self): 32 | try: 33 | return blocking(self.fun, *self.args, **self.kwargs) 34 | except Exception as exc: 35 | warnings.warn('Periodic timer %r raised: %r' % (self.fun, exc)) 36 | finally: 37 | self._spawn() 38 | 39 | def _exit(self, g): 40 | try: 41 | self.g.wait() 42 | except GreenletExit: 43 | self.cancel() 44 | 45 | def cancel(self): 46 | if self.g and not self.cancelled: 47 | self.g.cancel() 48 | self.cancelled = True 49 | 50 | def kill(self): 51 | if self.g: 52 | try: 53 | self.g.kill() 54 | except GreenletExit: 55 | pass 56 | 57 | def __repr__(self): 58 | return '' % ( 59 | self.fun, 'cancelled' if self.cancelled else 'alive') 60 | 61 | 62 | def timer(interval, fun, *args, **kwargs): 63 | return Entry(interval, fun, *args, **kwargs) 64 | -------------------------------------------------------------------------------- /cell/groups.py: -------------------------------------------------------------------------------- 1 | from cell.actors import Actor 2 | from cell.agents import dAgent 3 | from kombu.entity import Exchange 4 | 5 | __author__ = 'rumi' 6 | 7 | 8 | class Group(Actor): 9 | """Convenience class used to spawn group of actors of the same type. 10 | 11 | **Example usage** 12 | Here we spawn two groups, of 10 :class:`Logger` actors each. 13 | 14 | .. code-block:: python 15 | 16 | >>> exception_group = agent.spawn(Group, Logger, 10) 17 | >>> warning_group = agent.spawn(Group, Logger, 10) 18 | >>> exception_group.scatter('log_msg', 'some exception msg...') 19 | >>> warning_group.scatter('log_msg', 'some exception msg...') 20 | 21 | :param act_type: the actor to spawn. 22 | :param number: the number of actor instances to spawn. 23 | 24 | """ 25 | def __init__(self, act_type, number, **kwargs): 26 | super().__init__(**kwargs) 27 | self.state.act_type = act_type 28 | self.state.number = number 29 | 30 | class state: 31 | def config(self, act_type, number): 32 | agent = dAgent(self.actor.connection) 33 | 34 | for _ in range(0, number): 35 | agent.spawn( 36 | act_type, 37 | {'group_exchange': self.actor.inbox_scatter.name}) 38 | 39 | def get_scatter_exchange(self): 40 | """Returns a :class:'kombu.Exchange' for type fanout""" 41 | return Exchange('cl.scatter.%s.%s' % (self.name, self.id), 42 | 'fanout', auto_delete=True) 43 | 44 | def on_agent_ready(self): 45 | self.state.config(self.state.act_type, self.state.number) 46 | 47 | def get_queues(self): 48 | return [] 49 | -------------------------------------------------------------------------------- /cell/models.py: -------------------------------------------------------------------------------- 1 | """cell.models""" 2 | 3 | from kombu import Consumer, Queue 4 | from komub.five import items, string_t 5 | from kombu.utils import gen_unique_id 6 | 7 | from . import Actor 8 | 9 | _all__ = ['ModelActor', 'ModelConsumer'] 10 | 11 | 12 | class ModelConsumer(Consumer): 13 | model = None 14 | field = 'name' 15 | auto_delete = True 16 | 17 | def __init__(self, channel, exchange, *args, **kwargs): 18 | model = kwargs.pop('model', None) 19 | self.model = model if model is not None else self.model 20 | self.exchange = exchange 21 | self.prepare_signals(kwargs.pop('sigmap', None)) 22 | queues = self.sync_queues(kwargs.pop('queues', [])) 23 | super().__init__(channel, queues, *args, **kwargs) 24 | 25 | def prepare_signals(self, sigmap=None): 26 | for callback, connect in items(sigmap or {}): 27 | if isinstance(callback, string_t): 28 | callback = getattr(self, callback) 29 | connect(callback) 30 | 31 | def create_queue(self, field_value): 32 | return Queue(gen_unique_id(), self.exchange, field_value, 33 | auto_delete=self.auto_delete) 34 | 35 | def sync_queues(self, keep_queues=[]): 36 | expected = [getattr(obj, self.field) 37 | for obj in self.model._default_manager.enabled()] 38 | queues = set() 39 | create = self.create_queue 40 | 41 | for v in expected: 42 | queues.add(create(v)) 43 | for queue in queues: 44 | if queue.routing_key not in expected: 45 | queues.discard(v) 46 | return list(keep_queues) + list(queues) 47 | 48 | def on_create(self, instance=None, **kwargs): 49 | fv = getattr(instance, self.field) 50 | if not self.find_queue_by_rkey(fv): 51 | self.add_queue(self.create_queue(fv)) 52 | self.consume() 53 | 54 | def on_delete(self, instance=None, **kwargs): 55 | fv = getattr(instance, self.field) 56 | queue = self.find_queue_by_rkey(fv) 57 | if queue: 58 | self.cancel_by_queue(queue.name) 59 | 60 | def find_queue_by_rkey(self, rkey): 61 | for queue in self.queues: 62 | if queue.routing_key == rkey: 63 | return queue 64 | 65 | 66 | class ModelActor(Actor): 67 | #: The model this actor is a controller for (*required*). 68 | model = None 69 | 70 | #: Map of signals to connect and corresponding actions. 71 | sigmap = {} 72 | 73 | def __init__(self, connection=None, id=None, name=None, *args, **kwargs): 74 | if self.model is None: 75 | raise NotImplementedError( 76 | "ModelActors must define the 'model' attribute!") 77 | if not name or self.name: 78 | name = self.model.__name__ 79 | 80 | super().__init__(connection, id, name, *args, **kwargs) 81 | 82 | def Consumer(self, channel, **kwargs): 83 | return ModelConsumer(channel, self.exchange, 84 | callbacks=[self.on_message], 85 | sigmap=self.sigmap, model=self.model, 86 | queues=[self.get_scatter_queue(), 87 | self.get_rr_queue()], 88 | **kwargs) 89 | -------------------------------------------------------------------------------- /cell/presence.py: -------------------------------------------------------------------------------- 1 | """cell.presence""" 2 | 3 | import warnings 4 | 5 | from collections import defaultdict 6 | from contextlib import contextmanager 7 | from functools import wraps 8 | from random import shuffle 9 | from time import time, sleep 10 | 11 | from kombu import Exchange, Queue 12 | from kombu.common import ipublish 13 | from kombu.five import items 14 | from kombu.log import LogMixin 15 | from kombu.mixins import ConsumerMixin 16 | from kombu.pools import producers 17 | from kombu.utils.functional import promise 18 | 19 | from .agents import Agent 20 | from .g import spawn, timer 21 | from .utils import cached_property, first_or_raise, shortuuid 22 | 23 | 24 | class State(LogMixin): 25 | logger_name = 'cell.presence.state' 26 | 27 | def __init__(self, presence): 28 | self.presence = presence 29 | self._agents = defaultdict(dict) 30 | self.heartbeat_expire = self.presence.interval * 2.5 31 | self.handlers = {'online': self.when_online, 32 | 'offline': self.when_offline, 33 | 'heartbeat': self.when_heartbeat, 34 | 'wakeup': self.when_wakeup} 35 | 36 | def can(self, actor): 37 | able = set() 38 | for id, state in items(self.agents): 39 | if actor in state['actors']: 40 | # remove the . from the agent, which means that the 41 | # agent is a clone of another agent. 42 | able.add(id.partition('.')[0]) 43 | return able 44 | 45 | def meta_for(self, actor): 46 | return self._agents['meta'][actor] 47 | 48 | def update_meta_for(self, agent, meta): 49 | self._agents[agent].update(meta=meta) 50 | 51 | def agents_by_meta(self, predicate, *sections): 52 | agents = self._agents 53 | agent_ids = list(agents.keys()) 54 | # shuffle the agents so we don't get the same agent every time. 55 | shuffle(agent_ids) 56 | for agent in agent_ids: 57 | d = agents[agent]['meta'] 58 | for i, section in enumerate(sections): 59 | d = d[section] 60 | if predicate(d): 61 | yield agent 62 | 63 | def first_agent_by_meta(self, predicate, *sections): 64 | for agent in self.agents_by_meta(predicate, *sections): 65 | return agent 66 | raise KeyError() 67 | 68 | def on_message(self, body, message): 69 | event = body['event'] 70 | self.handlers[event](**body) 71 | self.debug('agents after event recv: %s', promise(lambda: self.agents)) 72 | 73 | def when_online(self, agent=None, **kw): 74 | self._update_agent(agent, kw) 75 | 76 | def when_wakeup(self, **kw): 77 | self.presence.send_heartbeat() 78 | 79 | def when_heartbeat(self, agent=None, **kw): 80 | self._update_agent(agent, kw) 81 | 82 | def when_offline(self, agent=None, **kw): 83 | self._remove_agent(agent) 84 | 85 | def expire_agents(self): 86 | expired = set() 87 | for id, state in items(self._agents): 88 | if state and state.get('ts'): 89 | if time() > state['ts'] + self.heartbeat_expire: 90 | expired.add(id) 91 | 92 | for id in expired: 93 | self._remove_agent(id) 94 | return self._agents 95 | 96 | def update_agent(self, agent=None, **kw): 97 | return self._update_agent(agent, kw) 98 | 99 | def _update_agent(self, agent, kw): 100 | kw = dict(kw) 101 | meta = kw.pop('meta', None) 102 | if meta: 103 | self.update_meta_for(agent, meta) 104 | self._agents[agent].update(kw) 105 | 106 | def _remove_agent(self, agent): 107 | self._agents[agent].clear() 108 | 109 | def neighbors(self): 110 | return {'agents': list(self.agents.keys())} 111 | 112 | @property 113 | def agents(self): 114 | return self.expire_agents() 115 | 116 | 117 | class Event(dict): 118 | pass 119 | 120 | 121 | class Presence(ConsumerMixin): 122 | Event = Event 123 | State = State 124 | 125 | exchange = Exchange('cl.agents', type='topic', auto_delete=True) 126 | interval = 10 127 | _channel = None 128 | g = None 129 | 130 | def __init__(self, agent, interval=None, on_awake=None): 131 | self.agent = agent 132 | self.state = self.State(self) 133 | self.interval = interval or self.interval 134 | self.connection = agent.connection 135 | self.on_awake = on_awake 136 | 137 | def get_queue(self): 138 | return Queue('cl.agents.%s' % (self.agent.id, ), self.exchange, 139 | routing_key='#', auto_delete=True) 140 | 141 | def get_consumers(self, Consumer, channel): 142 | return [Consumer(self.get_queue(), 143 | callbacks=[self.state.on_message], no_ack=True)] 144 | 145 | def create_event(self, type): 146 | return self.Event(agent=self.agent.id, 147 | event=type, 148 | actors=[actor.name for actor in self.agent.actors], 149 | meta=self.meta(), 150 | ts=time(), 151 | neighbors=self.state.neighbors()) 152 | 153 | def meta(self): 154 | return {actor.name: actor.meta for actor in self.agent.actors} 155 | 156 | @contextmanager 157 | def extra_context(self, connection, channel): 158 | self.send_online() 159 | self.wakeup() 160 | sleep(1.0) 161 | if self.on_awake: 162 | self.on_awake() 163 | timer(self.interval, self.send_heartbeat) 164 | self.agent.on_presence_ready() 165 | yield 166 | self.send_offline() 167 | 168 | def _announce(self, event, producer=None): 169 | producer.publish(event, exchange=self.exchange.name, 170 | routing_key=self.agent.id) 171 | 172 | def announce(self, event, **retry_policy): 173 | return ipublish(producers[self.agent.connection], 174 | self._announce, (event, ), **retry_policy) 175 | 176 | def start(self): 177 | self.g = spawn(self.run) 178 | 179 | def send_online(self): 180 | return self.announce(self.create_event('online')) 181 | 182 | def send_heartbeat(self): 183 | return self.announce(self.create_event('heartbeat')) 184 | 185 | def send_offline(self): 186 | return self.announce(self.create_event('offline')) 187 | 188 | def wakeup(self): 189 | event = self.create_event('wakeup') 190 | self.state.update_agent(**event) 191 | return self.announce(event) 192 | 193 | def can(self, actor): 194 | return self.state.can(actor) 195 | 196 | @property 197 | def logger_name(self): 198 | return 'Presence#%s' % (shortuuid(self.agent.id), ) 199 | 200 | @property 201 | def should_stop(self): 202 | return self.agent.should_stop 203 | 204 | 205 | class AwareAgent(Agent): 206 | 207 | def on_run(self): 208 | self.presence.start() 209 | 210 | def get_default_scatter_limit(self, actor): 211 | able = self.presence.can(actor) 212 | if not able: 213 | warnings.warn('Presence running, but no agents available?!?') 214 | return len(able) if able else None 215 | 216 | def on_awake(self): 217 | pass 218 | 219 | def on_presence_ready(self): 220 | pass 221 | 222 | def lookup_agent(self, pred, *sections): 223 | return self.presence.state.first_agent_by_meta(pred, *sections) 224 | 225 | def lookup_agents(self, pred, *sections): 226 | return self.presence.state.agents_by_meta(pred, *sections) 227 | 228 | @cached_property 229 | def presence(self): 230 | return Presence(self, on_awake=self.on_awake) 231 | 232 | 233 | class AwareActorMixin: 234 | meta_lookup_section = None 235 | 236 | def lookup(self, value): 237 | if self.agent: 238 | return self.agent.lookup_agent(lambda values: value in values, 239 | self.name, self.meta_lookup_section) 240 | 241 | def send_to_able(self, method, args={}, to=None, **kwargs): 242 | actor = None 243 | try: 244 | actor = self.lookup(to) 245 | except KeyError: 246 | raise self.NoRouteError(to) 247 | 248 | if actor: 249 | return self.send(method, args, to=actor, **kwargs) 250 | r = self.scatter(method, args, propagate=True, **kwargs) 251 | if r: 252 | return first_or_raise(r, self.NoRouteError(to)) 253 | 254 | def wakeup_all_agents(self): 255 | if self.agent: 256 | self.log.info('presence wakeup others') 257 | self.agent.presence.wakeup() 258 | 259 | 260 | def announce_after(fun): 261 | 262 | @wraps(fun) 263 | def _inner(self, *args, **kwargs): 264 | try: 265 | return fun(self, *args, **kwargs) 266 | finally: 267 | self.actor.wakeup_all_agents() 268 | return _inner 269 | -------------------------------------------------------------------------------- /cell/results.py: -------------------------------------------------------------------------------- 1 | """cell.result""" 2 | from kombu.pools import producers 3 | 4 | from .exceptions import CellError, NoReplyError 5 | 6 | __all__ = ['AsyncResult'] 7 | 8 | 9 | class AsyncResult: 10 | Error = CellError 11 | NoReplyError = NoReplyError 12 | 13 | def __init__(self, ticket, actor): 14 | self.ticket = ticket 15 | self.actor = actor 16 | self._result = None 17 | 18 | def _first(self, replies): 19 | if replies is not None: 20 | replies = list(replies) 21 | if replies: 22 | return replies[0] 23 | raise self.NoReplyError('No reply received within time constraint') 24 | 25 | def result(self, **kwargs): 26 | if not self._result: 27 | self._result = self.get(**kwargs) 28 | return self._result 29 | 30 | def get(self, **kwargs): 31 | "What kind of arguments should be pass here" 32 | kwargs.setdefault('limit', 1) 33 | return self._first(self.gather(**kwargs)) 34 | 35 | def gather(self, propagate=True, **kwargs): 36 | # mock collect_replies. 37 | # check to_python is invoked for every result 38 | # check collect_replies is called with teh exact parameters 39 | # test collect_replies separately 40 | connection = self.actor.connection 41 | gather = self._gather 42 | with producers[connection].acquire(block=True) as producer: 43 | for r in gather(producer.connection, producer.channel, self.ticket, 44 | propagate=propagate, **kwargs): 45 | yield r 46 | 47 | def _gather(self, *args, **kwargs): 48 | """Generator over the results 49 | """ 50 | propagate = kwargs.pop('propagate', True) 51 | return (self.to_python(reply, propagate=propagate) 52 | for reply in self.actor._collect_replies(*args, **kwargs)) 53 | 54 | def to_python(self, reply, propagate=True): 55 | """Extracts the value out of the reply message. 56 | 57 | :param reply: In the case of a successful call the reply message 58 | will be:: 59 | 60 | {'ok': return_value, **default_fields} 61 | 62 | Therefore the method returns: return_value, **default_fields 63 | 64 | If the method raises an exception the reply message 65 | will be:: 66 | 67 | {'nok': [repr exc, str traceback], **default_fields} 68 | 69 | :keyword propagate - Propagate exceptions raised instead of returning 70 | a result representation of the error. 71 | 72 | """ 73 | try: 74 | return reply['ok'] 75 | except KeyError: 76 | error = self.Error(*reply.get('nok') or ()) 77 | if propagate: 78 | raise error 79 | return error 80 | -------------------------------------------------------------------------------- /cell/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/auvipy/cell/54e474592112b90ebb63aec1e6ef1856619e0e14/cell/tests/__init__.py -------------------------------------------------------------------------------- /cell/tests/actors/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'rumi' 2 | -------------------------------------------------------------------------------- /cell/tests/actors/test_agents.py: -------------------------------------------------------------------------------- 1 | from mock import patch, Mock, ANY 2 | from cell.actors import Actor, ActorProxy, ACTOR_TYPE 3 | from cell.agents import dAgent 4 | from cell.tests.utils import Case, with_in_memory_connection 5 | from cell.utils import qualname 6 | from kombu.utils import uuid 7 | 8 | 9 | class dA(dAgent): 10 | pass 11 | 12 | 13 | class A(Actor): 14 | pass 15 | 16 | 17 | class test_dAgent(Case): 18 | 19 | @patch('cell.actors.Actor') 20 | @patch('kombu.Connection') 21 | def test_init(self, conn, actor): 22 | id = uuid() 23 | a = dA(conn, id) 24 | self.assertEqual(a.connection, conn) 25 | self.assertEqual(a.id, id) 26 | self.assertEqual(a.agent, a) 27 | 28 | @with_in_memory_connection 29 | @patch('cell.agents.uuid', return_value=uuid()) 30 | def test_kill_actor_by_id(self, conn, static_id): 31 | ag = dA(conn) 32 | ag.cast = Mock() 33 | 34 | ag.kill(static_id) 35 | 36 | ag.cast.assert_called_once_with( 37 | 'kill', {'actor_id': static_id}, 38 | reply_to=ANY, type=ACTOR_TYPE.SCATTER, declare=ANY, 39 | timeout=ag.default_timeout) 40 | 41 | @with_in_memory_connection 42 | @patch('cell.actors.Actor.Consumer', return_value=Mock()) 43 | def test_state_spawn_when_id_not_in_registry(self, conn, consumer): 44 | ag, a, id = dA(conn), A(), uuid() 45 | 46 | self.assertEquals(ag.state.registry, {}) 47 | ag.state.spawn(qualname(a), id) 48 | 49 | self.assertEquals(len(ag.state.registry), 1) 50 | actor = ag.state.registry[id] 51 | self.assertIs(type(actor), A) 52 | self.assertIsNotNone(actor.consumer) 53 | actor.consumer.consume.assert_called_once_with() 54 | 55 | @with_in_memory_connection 56 | @patch('cell.agents.warn', return_value=Mock()) 57 | def test_state_spawn_when_id_in_registry(self, conn, warn): 58 | ag, a1 = dA(conn), A(conn) 59 | ag.state.registry[a1.id] = a1 60 | 61 | ag.state.spawn(qualname(a1), a1.id) 62 | 63 | warn.assert_called_once_with(ANY, a1.id) 64 | 65 | @with_in_memory_connection 66 | @patch('cell.agents.error', return_value=Mock()) 67 | def test_state_spawn_when_error_occurs(self, conn, error): 68 | ag, a1 = dA(conn), A(conn) 69 | ag.state.registry[a1.id] = a1 70 | ag.state._start_actor_consumer = Mock( 71 | side_effect=Exception('FooError')) 72 | 73 | ag.state.spawn(qualname(a1), a1.id) 74 | error.called_once_with('Cannot start actor: %r', 75 | Exception('FooError'), ANY) 76 | 77 | ag.state._start_actor_consumer.reset_mock() 78 | ag.state._start_actor_consumer = Mock() 79 | error.reset_mock() 80 | ag.state.spawn('Ala Bala', a1.id) 81 | error.called_once_with('Cannot start actor: %r', 'ihu', ANY) 82 | 83 | @patch('cell.actors.ActorProxy', return_value=Mock()) 84 | @patch('cell.actors.uuid', return_value=uuid()) 85 | @patch('cell.agents.uuid', return_value=uuid()) 86 | @with_in_memory_connection 87 | def test_spawn(self, conn, actor_static_id, ticket_static_id, proxy): 88 | # Ensure the ActorProxy is returned 89 | # Ensure cast is invoked with the correct arguments 90 | 91 | ag, a = dA(conn), A() 92 | ag.cast = Mock() 93 | 94 | proxy = ag.spawn(A) 95 | 96 | ag.cast.assert_called_once_with( 97 | 'spawn', 98 | {'cls': qualname(a), 'id': actor_static_id.return_value, 99 | 'kwargs': {}}, 100 | reply_to=ticket_static_id.return_value, declare=ANY, 101 | type=ACTOR_TYPE.RR, nowait=False) 102 | 103 | # Check ActorProxy initialisation 104 | self.assertIsInstance(proxy, ActorProxy) 105 | self.assertEqual(proxy.id, actor_static_id.return_value) 106 | self.assertIsInstance(proxy._actor, A) 107 | self.assertEqual(proxy._actor.name, A().__class__.__name__) 108 | self.assertEqual(proxy._actor.connection, conn) 109 | self.assertEqual(proxy._actor.agent, ag) 110 | self.assertEqual(proxy.async_start_result.ticket, 111 | ticket_static_id.return_value) 112 | 113 | # Agent state is not affected by the remote spawn call 114 | self.assertDictEqual(ag.state.registry, {}) 115 | 116 | @with_in_memory_connection 117 | @patch('cell.actors.Actor.Consumer', return_value=Mock()) 118 | def test_state_stop_actor_by_id(self, conn, consumer): 119 | ag, a, id = dA(conn), A(), uuid() 120 | ag.state.spawn(qualname(a), id) 121 | self.assertEquals(len(ag.state.registry), 1) 122 | actor = ag.state.registry[id] 123 | 124 | ag.state.kill(id) 125 | 126 | self.assertEquals(ag.state.registry, {}) 127 | actor.consumer.cancel.assert_called_once_with() 128 | 129 | @with_in_memory_connection 130 | def test_state_stop_all(self, conn): 131 | ag, a = dA(conn), A() 132 | id1, id2 = uuid(), uuid() 133 | ag.state.spawn(qualname(a), id1) 134 | ag.state.spawn(qualname(a), id2) 135 | self.assertEquals(len(ag.state.registry), 2) 136 | actor1, actor2 = ag.state.registry[id1], ag.state.registry[id2] 137 | 138 | ag.state.stop_all() 139 | 140 | self.assertEquals(ag.state.registry, {}) 141 | self.assertEquals(actor1.consumer.channel.queues, {}) 142 | self.assertEquals(actor2.consumer.channel.queues, {}) 143 | 144 | @with_in_memory_connection 145 | def test_stop(self, conn): 146 | ag, a, id1, id2 = dA(conn), A(), uuid(), uuid() 147 | ag.state.spawn(qualname(a), id1) 148 | ag.state.spawn(qualname(a), id2) 149 | self.assertEquals(len(ag.state.registry), 2) 150 | actor1, actor2 = ag.state.registry[id1], ag.state.registry[id2] 151 | 152 | ag.stop() 153 | 154 | self.assertEquals(len(ag.state.registry), 2) 155 | self.assertEquals(actor1.consumer.channel.queues, {}) 156 | self.assertEquals(actor2.consumer.channel.queues, {}) 157 | 158 | @with_in_memory_connection 159 | def test_start_when_actors_are_already_in_the_registry(self, conn): 160 | ag, a1, a2 = dA(conn), A(conn), A(conn) 161 | ag.state.registry.update({a1.id: a1, a2.id: a2}) 162 | 163 | ag.start() 164 | 165 | self.assertIsNotNone(a1.consumer) 166 | self.assertIsNotNone(a2.consumer) 167 | self.assertEqual(len(ag.state.registry), 2) 168 | 169 | @with_in_memory_connection 170 | @patch('cell.actors.Actor.Consumer', return_value=Mock()) 171 | def test_reset(self, conn, consumer): 172 | ag, a1 = dA(conn), A() 173 | ag.state.spawn(qualname(a1), a1.id) 174 | a1 = ag.state.registry[a1.id] 175 | 176 | ag.state.reset() 177 | 178 | self.assertIsNotNone(a1.consumer) 179 | self.assertEqual(len(ag.state.registry), 1) 180 | self.assertEqual(a1.consumer.cancel.call_count, 1) 181 | 182 | @with_in_memory_connection 183 | def test_stop_actor_when_id_not_in_registry(self, conn): 184 | ag, a1 = dA(conn), A(conn) 185 | self.assertEqual(ag.state.registry, {}) 186 | 187 | with self.assertRaises(Actor.Next): 188 | ag.state.kill(a1.id) 189 | 190 | @with_in_memory_connection 191 | def test_select_returns_scatter_results(self, conn): 192 | id1, id2 = uuid(), uuid() 193 | 194 | def scatter_result(): 195 | yield id1 196 | yield id2 197 | 198 | ag = dAgent(conn) 199 | ag.scatter = Mock(return_value=scatter_result()) 200 | 201 | proxy = ag.select(A) 202 | 203 | ag.scatter.assert_called_once_with( 204 | 'select', {'cls': qualname(A)}, limit=1) 205 | 206 | # Check ActorProxy initialisation 207 | self.assertIsInstance(proxy, ActorProxy) 208 | self.assertEqual(proxy.id, id1) 209 | self.assertIsInstance(proxy._actor, A) 210 | self.assertEqual(proxy._actor.name, A().__class__.__name__) 211 | self.assertEqual(proxy._actor.connection, conn) 212 | self.assertEqual(proxy._actor.agent, ag) 213 | self.assertIsNone(proxy.async_start_result) 214 | 215 | @with_in_memory_connection 216 | def test_select_returns_error_when_no_result_found(self, conn): 217 | 218 | def scatter_result(): 219 | yield None 220 | 221 | gen = scatter_result() 222 | next(gen) 223 | ag = dAgent(conn) 224 | ag.scatter = Mock(return_value=gen) 225 | 226 | with self.assertRaises(KeyError): 227 | ag.select(A) 228 | 229 | @with_in_memory_connection 230 | def test_state_select_returns_from_registry(self, conn): 231 | class B(Actor): 232 | pass 233 | 234 | ag = dAgent(conn) 235 | id1, id2 = uuid(), uuid() 236 | 237 | with self.assertRaises(Actor.Next): 238 | ag.state.select(qualname(A)) 239 | 240 | ag.state.registry[id1] = A() 241 | key = ag.state.select(qualname(A)) 242 | 243 | self.assertEqual(key, id1) 244 | 245 | ag.state.registry[id2] = B(conn) 246 | keyA = ag.state.select(qualname(A)) 247 | keyB = ag.state.select(qualname(B)) 248 | self.assertEqual(keyA, id1) 249 | self.assertEqual(keyB, id2) 250 | 251 | @with_in_memory_connection 252 | def test_messages_processing_when_greenlets_are_enabled(self, conn): 253 | 254 | ag = dAgent(conn) 255 | ag.pool = Mock() 256 | ag.pool.is_green = True 257 | al, body, message = Mock(), Mock(), Mock() 258 | self.assertEqual(ag.is_green(), True) 259 | 260 | # message is processed in a separate pool if 261 | # greenlets are enabled and the sending actor is not an agent 262 | ag.process_message(al, body, message) 263 | ag.pool.spawn_n.assert_called_once_with(al._on_message, body, message) 264 | ag.pool.reset_mock() 265 | 266 | # message is always processed in a the same thread 267 | # if the sending actor is an agent 268 | ag._on_message = Mock() 269 | ag.process_message(ag, body, message) 270 | self.assertEqual(ag.pool.spawn_n.call_count, 0) 271 | ag._on_message.assert_called_once_with(body, message) 272 | 273 | @with_in_memory_connection 274 | def test_message_processing_when_greenlets_are_disabled(self, conn): 275 | ag = dAgent(conn) 276 | ag.pool = Mock() 277 | al = Mock() 278 | ag.pool.is_green = False 279 | body, message = Mock(), Mock() 280 | 281 | ag.process_message(al, body, message) 282 | 283 | al._on_message.assert_called_once_with(body, message) 284 | 285 | ag._on_message = Mock() 286 | ag.process_message(ag, body, message) 287 | self.assertEqual(ag.pool.spawn_n.call_count, 0) 288 | ag._on_message.assert_called_once_with(body, message) 289 | 290 | @with_in_memory_connection 291 | @patch('cell.agents.warn', return_value=Mock()) 292 | def test_message_processing_warning(self, conn, warn): 293 | 294 | ag, al = dAgent(conn), A(conn) 295 | ag.pool = Mock() 296 | al._on_message = Mock() 297 | body, message = Mock(), Mock() 298 | 299 | # warning is not triggered when greenlets are disabled 300 | ag.pool.is_green = True 301 | ag.process_message(al, body, message) 302 | self.assertEquals(warn.call_count, 0) 303 | warn.reset_mock() 304 | 305 | # warning is not triggered when greenlets are enabled 306 | # but the call is not blocking 307 | ag.pool.is_green = True 308 | ag.process_message(al, body, message) 309 | self.assertEquals(warn.call_count, 0) 310 | warn.reset_mock() 311 | 312 | # warning is not triggered when greenlets are disables 313 | # and teh call is blocking 314 | ag.pool.is_green = False 315 | 316 | import cell 317 | cell.agents.itemgetter = Mock() 318 | message.properties = {'reply_to': '1234'} 319 | ag.process_message(al, body, message) 320 | warn.assert_called_once_with( 321 | 'Starting a blocking call (%s) on actor (%s) ' 322 | 'when greenlets are disabled.', 323 | ANY, al.__class__) 324 | 325 | cell.agents.itemgetter.called_once_with('method') 326 | -------------------------------------------------------------------------------- /cell/tests/actors/test_results.py: -------------------------------------------------------------------------------- 1 | from mock import ANY, patch 2 | from cell.actors import Actor 3 | from cell.agents import dAgent 4 | from cell.exceptions import CellError, NoReplyError 5 | from cell.results import AsyncResult 6 | from cell.tests.utils import Case, Mock, with_in_memory_connection 7 | from kombu.utils import uuid 8 | 9 | __author__ = 'rumi' 10 | 11 | 12 | class A(Actor): 13 | pass 14 | 15 | 16 | class Ag(dAgent): 17 | def get_default_scatter_limit(self): 18 | return 5 19 | 20 | 21 | class test_AsyncResuls(Case): 22 | def get_async_result(self): 23 | ticket = uuid() 24 | actor = Mock() 25 | ares = AsyncResult(ticket, actor) 26 | return ares 27 | 28 | def test_init(self): 29 | ticket = uuid() 30 | actor = Mock() 31 | ares = AsyncResult(ticket, actor) 32 | 33 | self.assertEquals(ares.ticket, ticket) 34 | self.assertEqual(ares.actor, actor) 35 | self.assertIsNone(ares._result) 36 | self.assertEqual(ares.Error, CellError) 37 | self.assertEqual(ares.NoReplyError, NoReplyError) 38 | 39 | with self.assertRaises(TypeError): 40 | AsyncResult(ticket) 41 | 42 | def test_result_when_result_is_set(self): 43 | val = 'the quick brown fox' 44 | ares = self.get_async_result() 45 | ares.get = Mock() 46 | ares._result = val 47 | 48 | res = ares.result() 49 | 50 | self.assertEqual(res, val) 51 | self.assertEqual(ares.get.call_count, 0) 52 | 53 | def test_result_when_result_is_not_set(self): 54 | val = 'the quick brown fox' 55 | ares = self.get_async_result() 56 | ares.get = Mock(return_value=val) 57 | 58 | res = ares.result() 59 | 60 | self.assertEqual(res, val) 61 | self.assertEqual(ares._result, res) 62 | ares.get.assert_called_once_with() 63 | 64 | def test_to_python(self): 65 | ok_message = {'ok': 'the quick_brown_fox'} 66 | ares = self.get_async_result() 67 | 68 | # ------------------------------ 69 | # reply is a successful message 70 | # ------------------------------ 71 | 72 | # correct format 73 | res = ares.to_python(ok_message) 74 | 75 | self.assertEqual(res, ok_message['ok']) 76 | 77 | # correct format with multiple keys in the reply dict 78 | ok_message = {'ok': 'the quick_brown_fox', 79 | 'foo': 'the quick_brown_fox'} 80 | res = ares.to_python(ok_message) 81 | self.assertEqual(res, ok_message['ok']) 82 | 83 | # contains both ok and nok 84 | ok_message = {'ok': 'the quick_brown_fox', 85 | 'nok': 'the quick_brown_fox'} 86 | res = ares.to_python(ok_message) 87 | self.assertEqual(res, ok_message['ok']) 88 | 89 | # --------------------------- 90 | # reply is an error message 91 | # --------------------------- 92 | 93 | # correct error format with to propagate param set 94 | error_message = {'nok': [Exception('jump over')]} 95 | with self.assertRaises(ares.Error): 96 | ares.to_python(error_message) 97 | 98 | # correct error format with to propagate set to True 99 | with self.assertRaises(ares.Error): 100 | ares.to_python(error_message, propagate=True) 101 | 102 | # correct error format with to propagate set to False 103 | error_message = {'nok': ['jump over', None]} 104 | res = ares.to_python(error_message, propagate=False) 105 | self.assertEquals(res.__dict__, 106 | ares.Error(*error_message.get('nok')).__dict__) 107 | 108 | # neither nok or ok message given 109 | error_message = {'foo': ['jump over']} 110 | with self.assertRaises(ares.Error): 111 | ares.to_python(error_message) 112 | 113 | # multiple keys in the reply dics given, one of teh eks is nok 114 | error_message = {'foo': 'the quick_brown_fox', 115 | 'nok': ['jump over']} 116 | res = ares.to_python(error_message, propagate=False) 117 | self.assertEqual(res.__dict__, 118 | ares.Error(*error_message['nok']).__dict__) 119 | 120 | def test_get(self): 121 | id1, id2 = uuid(), uuid() 122 | 123 | def gather(): 124 | yield id1 125 | yield id2 126 | 127 | # test that it calls gather with limit = 1 and kwargs 128 | ares = self.get_async_result() 129 | ares.gather = Mock(return_value=['1']) 130 | 131 | ares.get() 132 | ares.gather.assert_called_once_with(limit=1) 133 | ares.gather.reset_mock() 134 | 135 | kwargs = {'timeout': 100, 'ignore_timeout': False, 136 | 'foo': 'bar', 'propaget': True} 137 | ares.get(**kwargs) 138 | ares.gather.assert_called_once_with(**dict(kwargs, limit=1)) 139 | ares.gather.reset_mock() 140 | 141 | kwargs = {'timeout': 100, 'ignore_timeout': False, 'limit': 10} 142 | ares.get(**kwargs) 143 | ares.gather.assert_called_once_with(**kwargs) 144 | ares.gather.reset_mock() 145 | 146 | # it returns the first value of whatever gather returns 147 | ares.gather = Mock(return_value=gather()) 148 | res = ares.get() 149 | self.assertEqual(res, id1) 150 | 151 | # if gather does not return result: 152 | # self.NoReplyError('No reply received within time constraint') 153 | ares.gather = Mock(return_value=None) 154 | 155 | with self.assertRaises(ares.NoReplyError): 156 | ares.get() 157 | 158 | ares.gather.reset_mock() 159 | ares.gather = Mock(return_value={}) 160 | 161 | with self.assertRaises(ares.NoReplyError): 162 | ares.get() 163 | 164 | @with_in_memory_connection 165 | def test_gather(self, conn): 166 | def collect_replies(): 167 | yield 1 168 | yield 2 169 | yield 3 170 | 171 | ticket = uuid() 172 | actor = Actor(conn) 173 | actor._collect_replies = Mock(return_value=collect_replies()) 174 | 175 | ares = AsyncResult(ticket, actor) 176 | ares.to_python = Mock() 177 | 178 | all = ares.gather() 179 | list(all) 180 | 181 | actor._collect_replies.assert_caleld_once_with(conn, ANY, ticket) 182 | self.assertEqual(ares.to_python.call_count, 183 | len(list(collect_replies()))) 184 | 185 | # test that the to_python is applied to all results 186 | actor._collect_replies.reset_mock() 187 | actor._collect_replies = Mock(return_value=collect_replies()) 188 | prev_to_python = ares.to_python 189 | new_to_python = lambda x, propagate = True: 'called_%s' % x 190 | ares.to_python = new_to_python 191 | 192 | all = ares.gather() 193 | vals = list(all) 194 | 195 | expected_vals = [new_to_python(i) for i in collect_replies()] 196 | 197 | actor._collect_replies.assert_caleld_once_with(conn, ANY, ticket) 198 | self.assertEqual(vals, expected_vals) 199 | ares.to_python = prev_to_python 200 | 201 | # test kwargs 202 | 203 | @patch('cell.actors.collect_replies') 204 | @with_in_memory_connection 205 | def test_gather_kwargs(self, conn, collect): 206 | actor = Actor(conn) 207 | ares = AsyncResult(uuid(), actor) 208 | prev_to_python = ares.to_python 209 | new_to_python = lambda x, propagate = True: x 210 | ares.to_python = new_to_python 211 | 212 | # Test default kwargs, 213 | # nothing is passed, the actor does not have agent assigned 214 | self.assert_gather_kwargs( 215 | ares, collect, {}, 216 | timeout=actor.default_timeout, ignore_timeout=False) 217 | 218 | # limit - set the default agent limit if NONE is set 219 | # Test default kwargs, nothing is passed, 220 | # the actor does have default agent assigned 221 | actor.agent = dAgent(conn) 222 | self.assert_gather_kwargs( 223 | ares, collect, {}, 224 | timeout=actor.default_timeout, limit=None, ignore_timeout=False) 225 | 226 | # limit - set the default agent limit if NONE is set 227 | # Test default kwargs, nothing is passed, 228 | # the actor does have agent with custom scatter limit assigned 229 | ag = Ag(conn) 230 | actor.agent = ag 231 | self.assert_gather_kwargs( 232 | ares, collect, {}, timeout=actor.default_timeout, 233 | limit=ag.get_default_scatter_limit()) 234 | 235 | # pass all args 236 | actor.agent = Ag(conn) 237 | timeout, ignore_timeout, limit = 200.0, False, uuid() 238 | 239 | self.assert_gather_kwargs( 240 | ares, collect, 241 | {'timeout': timeout, 'ignore_timeout': ignore_timeout, 242 | 'limit': limit}, 243 | timeout=timeout, limit=limit, ignore_timeout=ignore_timeout) 244 | 245 | # ig ignore_tiemout is passed, 246 | # the custom logic for limit is not applies 247 | actor.agent = None 248 | timeout, ignore_timeout = 200.0, True 249 | self.assert_gather_kwargs( 250 | ares, collect, 251 | {'timeout': timeout, 'ignore_timeout': ignore_timeout}, 252 | timeout=timeout, ignore_timeout=ignore_timeout) 253 | 254 | ares.to_python = prev_to_python 255 | 256 | def assert_gather_kwargs(self, ares, collect, args, **kwargs): 257 | def drain(): 258 | yield 1 259 | yield 2 260 | collect.return_value = drain() 261 | all = ares.gather(**args) 262 | 263 | self.assertEqual(list(all), list(drain())) 264 | collect.assert_called_once_with(ANY, ANY, ANY, **kwargs) 265 | collect.reset_mock() 266 | -------------------------------------------------------------------------------- /cell/tests/utils.py: -------------------------------------------------------------------------------- 1 | from kombu.connection import Connection 2 | 3 | try: 4 | import unittest 5 | unittest.skip 6 | from unittest.util import safe_repr, unorderable_list_difference 7 | except AttributeError: 8 | import unittest2 as unittest 9 | from unittest2.util import safe_repr, unorderable_list_difference # noqa 10 | 11 | import importlib 12 | import os 13 | import platform 14 | import re 15 | import sys 16 | import warnings 17 | 18 | from contextlib import contextmanager 19 | from functools import partial, wraps 20 | from types import ModuleType 21 | 22 | import mock 23 | from nose import SkipTest 24 | from kombu.five import items, string_t, reraise, values, WhateverIO 25 | from kombu.utils import nested 26 | 27 | 28 | class Mock(mock.Mock): 29 | 30 | def __init__(self, *args, **kwargs): 31 | attrs = kwargs.pop('attrs', None) or {} 32 | super(Mock, self).__init__(*args, **kwargs) 33 | for attr_name, attr_value in items(attrs): 34 | setattr(self, attr_name, attr_value) 35 | 36 | 37 | def skip_unless_module(module): 38 | 39 | def _inner(fun): 40 | 41 | @wraps(fun) 42 | def __inner(*args, **kwargs): 43 | try: 44 | importlib.import_module(module) 45 | except ImportError: 46 | raise SkipTest('Does not have %s' % (module, )) 47 | 48 | return fun(*args, **kwargs) 49 | 50 | return __inner 51 | return _inner 52 | 53 | 54 | # -- adds assertWarns from recent unittest2, not in Python 2.7. 55 | 56 | class _AssertRaisesBaseContext(object): 57 | 58 | def __init__(self, expected, test_case, callable_obj=None, 59 | expected_regex=None): 60 | self.expected = expected 61 | self.failureException = test_case.failureException 62 | self.obj_name = None 63 | if isinstance(expected_regex, string_t): 64 | expected_regex = re.compile(expected_regex) 65 | self.expected_regex = expected_regex 66 | 67 | 68 | class _AssertWarnsContext(_AssertRaisesBaseContext): 69 | """A context manager used to implement TestCase.assertWarns* methods.""" 70 | 71 | def __enter__(self): 72 | # The __warningregistry__'s need to be in a pristine state for tests 73 | # to work properly. 74 | warnings.resetwarnings() 75 | for v in values(sys.modules): 76 | if getattr(v, '__warningregistry__', None): 77 | v.__warningregistry__ = {} 78 | self.warnings_manager = warnings.catch_warnings(record=True) 79 | self.warnings = self.warnings_manager.__enter__() 80 | warnings.simplefilter('always', self.expected) 81 | return self 82 | 83 | def __exit__(self, exc_type, exc_value, tb): 84 | self.warnings_manager.__exit__(exc_type, exc_value, tb) 85 | if exc_type is not None: 86 | # let unexpected exceptions pass through 87 | return 88 | try: 89 | exc_name = self.expected.__name__ 90 | except AttributeError: 91 | exc_name = str(self.expected) 92 | first_matching = None 93 | for m in self.warnings: 94 | w = m.message 95 | if not isinstance(w, self.expected): 96 | continue 97 | if first_matching is None: 98 | first_matching = w 99 | if (self.expected_regex is not None and 100 | not self.expected_regex.search(str(w))): 101 | continue 102 | # store warning for later retrieval 103 | self.warning = w 104 | self.filename = m.filename 105 | self.lineno = m.lineno 106 | return 107 | # Now we simply try to choose a helpful failure message 108 | if first_matching is not None: 109 | raise self.failureException( 110 | '%r does not match %r' 111 | % (self.expected_regex.pattern, str(first_matching))) 112 | if self.obj_name: 113 | raise self.failureException('%s not triggered by %s' 114 | % (exc_name, self.obj_name)) 115 | else: 116 | raise self.failureException('%s not triggered' 117 | % exc_name) 118 | 119 | 120 | class Case(unittest.TestCase): 121 | 122 | def assertWarns(self, expected_warning): 123 | return _AssertWarnsContext(expected_warning, self, None) 124 | 125 | def assertWarnsRegex(self, expected_warning, expected_regex): 126 | return _AssertWarnsContext(expected_warning, self, 127 | None, expected_regex) 128 | 129 | def assertDictContainsSubset(self, expected, actual, msg=None): 130 | missing, mismatched = [], [] 131 | 132 | for key, value in items(expected): 133 | if key not in actual: 134 | missing.append(key) 135 | elif value != actual[key]: 136 | mismatched.append('%s, expected: %s, actual: %s' % ( 137 | safe_repr(key), safe_repr(value), 138 | safe_repr(actual[key]))) 139 | 140 | if not (missing or mismatched): 141 | return 142 | 143 | standard_msg = '' 144 | if missing: 145 | standard_msg = 'Missing: %s' % ','.join(map(safe_repr, missing)) 146 | 147 | if mismatched: 148 | if standard_msg: 149 | standard_msg += '; ' 150 | standard_msg += 'Mismatched values: %s' % ( 151 | ','.join(mismatched)) 152 | 153 | self.fail(self._formatMessage(msg, standard_msg)) 154 | 155 | def assertItemsEqual(self, expected_seq, actual_seq, msg=None): 156 | try: 157 | expected = sorted(expected_seq) 158 | actual = sorted(actual_seq) 159 | except TypeError: 160 | # Unsortable items (example: set(), complex(), ...) 161 | expected = list(expected_seq) 162 | actual = list(actual_seq) 163 | missing, unexpected = unorderable_list_difference( 164 | expected, actual) 165 | else: 166 | return self.assertSequenceEqual(expected, actual, msg=msg) 167 | 168 | errors = [] 169 | if missing: 170 | errors.append('Expected, but missing:\n %s' % ( 171 | safe_repr(missing))) 172 | if unexpected: 173 | errors.append('Unexpected, but present:\n %s' % ( 174 | safe_repr(unexpected))) 175 | if errors: 176 | standardMsg = '\n'.join(errors) 177 | self.fail(self._formatMessage(msg, standardMsg)) 178 | 179 | 180 | def with_in_memory_connection(fn): 181 | 182 | @wraps(fn) 183 | def wrapper(self, *args, **kwargs): 184 | with Connection('memory://') as conn: 185 | fn(self, conn, *args, **kwargs) 186 | return wrapper 187 | 188 | 189 | def with_environ(env_name, env_value): 190 | 191 | def _envpatched(fun): 192 | 193 | @wraps(fun) 194 | def _patch_environ(*args, **kwargs): 195 | prev_val = os.environ.get(env_name) 196 | os.environ[env_name] = env_value 197 | try: 198 | return fun(*args, **kwargs) 199 | finally: 200 | if prev_val is not None: 201 | os.environ[env_name] = prev_val 202 | 203 | return _patch_environ 204 | return _envpatched 205 | 206 | 207 | def patch(module, name, mocked): 208 | module = importlib.import_module(module) 209 | 210 | def _patch(fun): 211 | 212 | @wraps(fun) 213 | def __patched(*args, **kwargs): 214 | prev = getattr(module, name) 215 | setattr(module, name, mocked) 216 | try: 217 | return fun(*args, **kwargs) 218 | finally: 219 | setattr(module, name, prev) 220 | return __patched 221 | return _patch 222 | 223 | 224 | @contextmanager 225 | def replace_module_value(module, name, value=None): 226 | has_prev = hasattr(module, name) 227 | prev = getattr(module, name, None) 228 | if value: 229 | setattr(module, name, value) 230 | else: 231 | try: 232 | delattr(module, name) 233 | except AttributeError: 234 | pass 235 | yield 236 | if prev is not None: 237 | setattr(sys, name, prev) 238 | if not has_prev: 239 | try: 240 | delattr(module, name) 241 | except AttributeError: 242 | pass 243 | 244 | 245 | pypy_version = partial( 246 | replace_module_value, sys, 'pypy_version_info', 247 | ) 248 | platform_pyimp = partial( 249 | replace_module_value, platform, 'python_implementation', 250 | ) 251 | 252 | 253 | @contextmanager 254 | def sys_platform(value): 255 | prev, sys.platform = sys.platform, value 256 | yield 257 | sys.platform = prev 258 | 259 | 260 | @contextmanager 261 | def mock_module(*names): 262 | prev = {} 263 | 264 | class MockModule(ModuleType): 265 | 266 | def __getattr__(self, attr): 267 | setattr(self, attr, Mock()) 268 | return ModuleType.__getattribute__(self, attr) 269 | 270 | mods = [] 271 | for name in names: 272 | try: 273 | prev[name] = sys.modules[name] 274 | except KeyError: 275 | pass 276 | mod = sys.modules[name] = MockModule(name) 277 | mods.append(mod) 278 | try: 279 | yield mods 280 | finally: 281 | for name in names: 282 | try: 283 | sys.modules[name] = prev[name] 284 | except KeyError: 285 | try: 286 | del(sys.modules[name]) 287 | except KeyError: 288 | pass 289 | 290 | 291 | @contextmanager 292 | def mock_context(mock, typ=Mock): 293 | context = mock.return_value = Mock() 294 | context.__enter__ = typ() 295 | context.__exit__ = typ() 296 | 297 | def on_exit(*x): 298 | if x[0]: 299 | reraise(x[0], x[1], x[2]) 300 | context.__exit__.side_effect = on_exit 301 | context.__enter__.return_value = context 302 | yield context 303 | context.reset() 304 | 305 | 306 | @contextmanager 307 | def mock_open(typ=WhateverIO, side_effect=None): 308 | with mock.patch('__builtin__.open') as open_: 309 | with mock_context(open_) as context: 310 | if side_effect is not None: 311 | context.__enter__.side_effect = side_effect 312 | val = context.__enter__.return_value = typ() 313 | val.__exit__ = Mock() 314 | yield val 315 | 316 | 317 | def patch_many(*targets): 318 | return nested(*[mock.patch(target) for target in targets]) 319 | 320 | 321 | def skip_if_pypy(fun): 322 | 323 | @wraps(fun) 324 | def _inner(*args, **kwargs): 325 | if getattr(sys, 'pypy_version_info', None): 326 | raise SkipTest('does not work on PyPy') 327 | return fun(*args, **kwargs) 328 | return _inner 329 | 330 | 331 | def skip_if_jython(fun): 332 | 333 | @wraps(fun) 334 | def _inner(*args, **kwargs): 335 | if sys.platform.startswith('java'): 336 | raise SkipTest('does not work on Jython') 337 | return fun(*args, **kwargs) 338 | return _inner 339 | -------------------------------------------------------------------------------- /cell/utils/__init__.py: -------------------------------------------------------------------------------- 1 | """cl.utils""" 2 | import operator 3 | 4 | from collections import namedtuple 5 | 6 | from kombu.five import map, zip 7 | from kombu.utils import cached_property, symbol_by_name # noqa 8 | 9 | __all__ = ['force_list', 'flatten', 10 | 'instantiate', 'cached_property'] 11 | 12 | 13 | def enum(**alt): 14 | keys, values = zip(*alt.items()) 15 | return namedtuple('Enum', keys)(*values) 16 | 17 | 18 | def setattr_default(obj, attr, value): 19 | if not hasattr(obj, attr): 20 | setattr(obj, attr, value) 21 | 22 | 23 | def force_list(obj): 24 | if not hasattr(obj, '__iter__'): 25 | return [obj] 26 | return obj 27 | 28 | 29 | def flatten(it): 30 | if it: 31 | try: 32 | return reduce(operator.add, 33 | map(force_list, (x for x in it if x))) 34 | except TypeError: 35 | return [] 36 | return it 37 | 38 | 39 | def first(it, default=None): 40 | try: 41 | next(it) 42 | except StopIteration: 43 | return default 44 | 45 | 46 | def first_or_raise(it, exc): 47 | for reply in it: 48 | if not isinstance(reply, Exception): 49 | return reply 50 | raise exc 51 | 52 | 53 | def instantiate(name, *args, **kwargs): 54 | """Instantiate class by name. 55 | 56 | See :func:`get_cls_by_name`. 57 | 58 | """ 59 | return symbol_by_name(name)(*args, **kwargs) 60 | 61 | 62 | def abbr(S, max, ellipsis='...'): 63 | if S and len(S) > max: 64 | return ellipsis and (S[:max - len(ellipsis)] + ellipsis) or S[:max] 65 | return S 66 | 67 | 68 | def shortuuid(u): 69 | if '-' in u: 70 | return u[:u.index('-')] 71 | return abbr(u, 16) 72 | 73 | 74 | def qualname(obj): # noqa 75 | if not hasattr(obj, '__name__') and hasattr(obj, '__class__'): 76 | obj = obj.__class__ 77 | return '%s.%s' % (obj.__module__, obj.__name__) 78 | 79 | 80 | def first_reply(replies, key): 81 | try: 82 | return next(replies) 83 | except StopIteration: 84 | raise KeyError(key) 85 | -------------------------------------------------------------------------------- /cell/utils/custom_operators.py: -------------------------------------------------------------------------------- 1 | class Infix: 2 | 3 | def __init__(self, function): 4 | self.function = function 5 | 6 | def __ror__(self, other): 7 | return Infix(lambda x, self=self, other=other: self.function(other, x)) 8 | 9 | def __or__(self, other): 10 | return self.function(other) 11 | 12 | def __rlshift__(self, other): 13 | return Infix(lambda x, self=self, other=other: self.function(other, x)) 14 | 15 | def __rshift__(self, other): 16 | return self.function(other) 17 | 18 | def __call__(self, value1, value2): 19 | return self.function(value1, value2) 20 | 21 | # Examples 22 | 23 | 24 | # simple multiplication 25 | send = Infix(lambda channel, task: channel * task) 26 | recv = Infix(lambda channel, task: channel * task) 27 | to = Infix( 28 | lambda in_actor, out_actor: connect_actor_ports(in_actor, out_actor) 29 | ) 30 | 31 | 32 | def connect_actor_ports(in_actor, out_actor): 33 | in_actor.connect_out(out_actor.in_ports['default']) 34 | -------------------------------------------------------------------------------- /cell/utils/utils.py: -------------------------------------------------------------------------------- 1 | 2 | def lazy_property(property_name, property_factory, doc=None): 3 | 4 | def get(self): 5 | if not hasattr(self, property_name): 6 | setattr(self, property_name, property_factory(self)) 7 | return getattr(self, property_name) 8 | return property(get) 9 | -------------------------------------------------------------------------------- /cell/workflow/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/auvipy/cell/54e474592112b90ebb63aec1e6ef1856619e0e14/cell/workflow/__init__.py -------------------------------------------------------------------------------- /cell/workflow/common.py: -------------------------------------------------------------------------------- 1 | from collections import deque 2 | 3 | from .monads import callcc, done, ContinuationMonad, do 4 | 5 | 6 | class Mailbox: 7 | 8 | def __init__(self, name=None): 9 | self.name = name 10 | self.messages = deque() 11 | self.handlers = deque() 12 | 13 | def send(self, message): 14 | try: 15 | handler = self.handlers.popleft() 16 | except IndexError: 17 | self.messages.append(message) 18 | else: 19 | handler(message)() 20 | 21 | def receive(self): 22 | return callcc(self.react) 23 | 24 | @do(ContinuationMonad) 25 | def react(self, handler): 26 | try: 27 | message = self.messages.popleft() 28 | except IndexError: 29 | self.handlers.append(handler) 30 | done(ContinuationMonad.zero()) 31 | else: 32 | yield handler(message) 33 | -------------------------------------------------------------------------------- /cell/workflow/entities.py: -------------------------------------------------------------------------------- 1 | from kombu.common import uuid 2 | 3 | from cell.results import AsyncResult 4 | from cell.actors import Actor 5 | 6 | from .monads import mreturn, MonadReturn 7 | 8 | __all__ = ['Workflow'] 9 | 10 | 11 | class Workflow: 12 | 13 | def __init__(self, protocol, id=None): 14 | self._wf_table = {} 15 | self.protocol = protocol 16 | self.id = id if id else self._build_conv_id() 17 | 18 | def __getitem__(self, to_role): 19 | print("In._get_from_conv_table") 20 | self._wf_table.setdefault(to_role, AsyncResult()) 21 | if isinstance(self._wf_table[to_role], AsyncResult): 22 | # @TODO. Need timeout for the AsyncResult 23 | print("Wait on the Async Result") 24 | to_role_addr = self._wf_table[to_role].get() 25 | print("get the Async Result, value is:%s" % to_role_addr) 26 | self._wf_table[to_role] = to_role_addr 27 | return self._wf_table[to_role] 28 | 29 | def __setitem__(self, to_role, to_role_addr): 30 | print("Conv._add_to_conv_table: to_role:%s, to_role_addr:%s" % ( 31 | to_role, to_role_addr)) 32 | if to_role in self._conv_table and \ 33 | isinstance(self._conv_table[to_role], AsyncResult): 34 | self._wf_table[to_role].set(to_role_addr) 35 | else: 36 | self._wf_table[to_role] = to_role_addr 37 | 38 | def has_role(self, role): 39 | return role in self._conv_table 40 | 41 | # TODO: Why we need the counter here?. 42 | # This is a copy from endpoint.py, it should be changed 43 | def _build_workflow_id(self): 44 | """ 45 | Builds a unique conversation id. 46 | """ 47 | return uuid() 48 | 49 | 50 | class Server(Actor): 51 | """An actor which responds to the call protocol by looking for the 52 | specified method and calling it. 53 | 54 | Also, Server provides start and stop methods which can be overridden 55 | to customize setup. 56 | """ 57 | 58 | def get_handler(self, message): 59 | if message.properties.get('reply_to'): 60 | handler = self.handle_call 61 | else: 62 | handler = self.handle_cast 63 | return handler() 64 | 65 | def start(self, *args, **kwargs): 66 | """Override to be notified when the server starts.""" 67 | pass 68 | 69 | def stop(self, *args, **kwargs): 70 | """Override to be notified when the server stops.""" 71 | pass 72 | 73 | def main(self, *args, **kwargs): 74 | """Implement the actor main loop by waiting forever for messages.""" 75 | self.start(*args, **kwargs) 76 | try: 77 | while 1: 78 | body, message = yield self.receive() 79 | handler = self.get_handler(message) 80 | handler(body, message) 81 | finally: 82 | self.stop(*args, **kwargs) 83 | 84 | 85 | class RPCClient(Actor): 86 | 87 | def __init__(self, server): 88 | self.server = server 89 | 90 | def request_internal(self, method, args): 91 | self.server.send({'method': method, 'args': args}, nowait=True) 92 | result = (yield self.server.receive()) 93 | mreturn(result) 94 | 95 | def request(self, method, args): 96 | try: 97 | self.request_internal(method, args) 98 | except MonadReturn as val: 99 | return val 100 | -------------------------------------------------------------------------------- /cell/workflow/monads.py: -------------------------------------------------------------------------------- 1 | import types 2 | 3 | from collections import deque 4 | from functools import wraps 5 | from queue import Queue 6 | 7 | # ##### Base Monad and @do syntax ######### 8 | 9 | 10 | class Monad: 11 | 12 | def bind(self, fun): 13 | raise NotImplementedError('bind') 14 | 15 | def __rshift__(self, bindee): 16 | return self.bind(bindee) 17 | 18 | def __add__(self, nullary_bindee): 19 | return self.bind(lambda _: nullary_bindee()) 20 | 21 | 22 | def make_decorator(fun, *dec_args): 23 | 24 | @wraps(fun) 25 | def decorator(plain): 26 | 27 | @wraps(plain) 28 | def decorated(*args, **kwargs): 29 | return fun(plain, args, kwargs, *dec_args) 30 | return decorated 31 | return decorator 32 | 33 | 34 | def make_decorator_with_args(fun): 35 | 36 | def decorator_with_args(*dec_args): 37 | return make_decorator(fun, *dec_args) 38 | return decorator_with_args 39 | 40 | 41 | decorator = make_decorator 42 | decorator_with_args = make_decorator_with_args 43 | 44 | 45 | @decorator_with_args 46 | def do(fun, args, kwargs, Monad): 47 | 48 | @handle_monadic_throws(Monad) 49 | def run_maybe_iterator(): 50 | it = fun(*args, **kwargs) 51 | 52 | if isinstance(it, types.GeneratorType): 53 | 54 | @handle_monadic_throws(Monad) 55 | def send(val): 56 | try: 57 | # here's the real magic 58 | # --what magic? please explain or the comment goes :) [ask] 59 | monad = it.send(val) 60 | return monad.bind(send) 61 | except StopIteration: 62 | return Monad.unit(None) 63 | return send(None) 64 | else: 65 | # not a generator 66 | return Monad.unit(None) if it is None else it 67 | 68 | return run_maybe_iterator() 69 | 70 | 71 | @decorator_with_args 72 | def handle_monadic_throws(fun, args, kwargs, Monad): 73 | try: 74 | return fun(*args, **kwargs) 75 | except MonadReturn as ret: 76 | return Monad.unit(ret.value) 77 | except Done as done: 78 | assert isinstance(done.monad, Monad) 79 | return done.monad 80 | 81 | 82 | class MonadReturn(Exception): 83 | 84 | def __init__(self, value): 85 | self.value = value 86 | Exception.__init__(self, value) 87 | 88 | 89 | class Done(Exception): 90 | 91 | def __init__(self, monad): 92 | self.monad = monad 93 | Exception.__init__(self, monad) 94 | 95 | 96 | def mreturn(val): 97 | raise MonadReturn(val) 98 | 99 | 100 | def done(val): 101 | raise Done(val) 102 | 103 | 104 | def fid(val): 105 | return val 106 | 107 | # #### Failable Monad ###### 108 | 109 | 110 | class Failable(Monad): 111 | 112 | def __init__(self, value, success): 113 | self.value = value 114 | self.success = success 115 | 116 | def __repr__(self): 117 | fmt = 'Success(%r)' if self.success else 'Failure(%r)' 118 | return fmt % (self.value, ) 119 | 120 | def bind(self, bindee): 121 | return bindee(self.value) if self.success else self 122 | 123 | @classmethod 124 | def unit(cls, val): 125 | return cls(val, True) 126 | 127 | 128 | class Success(Failable): 129 | 130 | def __init__(self, value): 131 | super(Success, self).__init__(self, value, True) 132 | 133 | 134 | class Failure(Failable): 135 | 136 | def __init__(self, value): 137 | super(Success, self).__init__(self, value, True) 138 | 139 | 140 | # ##### StateChanger Monad ######### 141 | 142 | 143 | class StateChanger(Monad): 144 | 145 | def __init__(self, run): 146 | self.run = run 147 | 148 | def bind(self, bindee): 149 | run0 = self.run 150 | 151 | def run1(state0): 152 | result, state1 = run0(state0) 153 | return bindee(result).run(state1) 154 | 155 | return StateChanger(run1) 156 | 157 | @classmethod 158 | def unit(cls, val): 159 | return cls(lambda state: (val, state)) 160 | 161 | 162 | def get_state(view=fid): 163 | return change_state(fid, view) 164 | 165 | 166 | def change_state(changer, view=fid): 167 | 168 | def make_new_state(old_state): 169 | new_state = changer(old_state) 170 | viewed_state = view(old_state) 171 | return viewed_state, new_state 172 | return StateChanger(make_new_state) 173 | 174 | # ##### Continuation Monad ######### 175 | 176 | 177 | class ContinuationMonad(Monad): 178 | 179 | def __init__(self, run): 180 | self.run = run 181 | 182 | def __call__(self, cont=fid): 183 | return self.run(cont) 184 | 185 | def bind(self, bindee): 186 | return ContinuationMonad( 187 | lambda cont: self.run( 188 | lambda val: bindee(val).run(cont)) 189 | ) 190 | 191 | @classmethod 192 | def unit(cls, val): 193 | return cls(lambda cont: cont(val)) 194 | 195 | @classmethod 196 | def zero(cls): 197 | return cls(lambda cont: None) 198 | 199 | 200 | def callcc(usecc): 201 | return ContinuationMonad( 202 | lambda cont: usecc( 203 | lambda val: ContinuationMonad( 204 | lambda _: cont(val)) 205 | ).run(cont) 206 | ) 207 | 208 | 209 | class AgentRole: 210 | 211 | def receive(self, sender): 212 | yield self.receiver.receive() 213 | 214 | def send(self, recepient, message): 215 | recepient.send(message) 216 | 217 | def add_role(self, mailbox): 218 | self.roles.append(mailbox) 219 | 220 | 221 | class Mailbox: 222 | 223 | def __init__(self): 224 | self.messages = deque() 225 | self.handlers = deque() 226 | 227 | def send(self, message): 228 | try: 229 | handler = self.handlers.popleft() 230 | except IndexError: 231 | self.messages.append(message) 232 | else: 233 | handler(message)() 234 | 235 | def receive(self): 236 | return callcc(self.react) 237 | 238 | @do(ContinuationMonad) 239 | def react(self, handler): 240 | try: 241 | message = self.messages.popleft() 242 | except IndexError: 243 | self.handlers.append(handler) 244 | done(ContinuationMonad.zero()) 245 | else: 246 | yield handler(message) 247 | 248 | 249 | class RemoteMailbox(Mailbox): 250 | #: data for this receive channel 251 | queue = None 252 | _consumer_tag = None 253 | 254 | #: name this receiving channel is receiving on - 255 | #: a tuple of ``(exchange, queue)`` 256 | _recv_name = None 257 | 258 | #: binding this queue is listening on (set via :meth:`_bind`) 259 | _recv_binding = None 260 | 261 | def __init__(self, name, binding): 262 | self.queue = Queue() 263 | self._recv_name = name 264 | self._recv_binding = binding 265 | super().__init__() 266 | 267 | 268 | if __name__ == "__main__": 269 | 270 | def failable_monad_example(): 271 | 272 | def fdiv(a, b): 273 | if b == 0: 274 | return Failure("divide by zero") 275 | else: 276 | return Success(a / b) 277 | 278 | @do(Failable) 279 | def with_failable(first_divisor): 280 | val1 = yield fdiv(2.0, first_divisor) 281 | val2 = yield fdiv(3.0, 1.0) 282 | val3 = yield fdiv(val1, val2) 283 | mreturn(val3) 284 | 285 | print(with_failable(0.0)) 286 | print(with_failable(1.0)) 287 | 288 | def state_changer_monad_example(): 289 | 290 | @do(StateChanger) 291 | def dict_state_copy(key1, key2): 292 | val = yield dict_state_get(key1) 293 | yield dict_state_set(key2, val) 294 | mreturn(val) 295 | 296 | @do(StateChanger) 297 | def dict_state_get(key, default=None): 298 | dct = yield get_state() 299 | val = dct.get(key, default) 300 | mreturn(val) 301 | 302 | @do(StateChanger) 303 | def dict_state_set(key, val): 304 | 305 | def setitem(dct, key, val): 306 | dct[key] = val 307 | return dct 308 | 309 | yield change_state(lambda dct: setitem(dct, key, val)) 310 | mreturn(val) 311 | 312 | @do(StateChanger) 313 | def with_dict_state(): 314 | val2 = yield dict_state_set("a", 2) 315 | yield dict_state_copy("a", "b") 316 | yield get_state() 317 | mreturn(val2) 318 | 319 | print(with_dict_state().run({})) # (2, {"a" : 2, "b" : 2})) 320 | 321 | def continuation_monad_example(): 322 | 323 | @do(ContinuationMonad) 324 | def insert(mb, values): 325 | for val in values: 326 | mb.send(val) 327 | 328 | # This is a multiplier flowlet 329 | @do(ContinuationMonad) 330 | def multiply(mbin, mbout, factor): 331 | while 1: 332 | val = (yield mbin.receive()) 333 | mbout.send(val * factor) 334 | 335 | # This is a printer flowlet 336 | @do(ContinuationMonad) 337 | def print_all(mb): 338 | while 1: 339 | print(yield mb.receive()) 340 | 341 | # This is a flowlet for filtering 342 | @do(ContinuationMonad) 343 | def filter(mbin, mbout, cond): 344 | print('Entering filter') 345 | while 1: 346 | val = (yield mbin.receive()) 347 | print('In filter: {0}'.format(val)) 348 | res = cond(val) 349 | mbout.send((val, res)) 350 | 351 | @do(ContinuationMonad) 352 | def recv(mbin, collector=None, is_rec=False): 353 | val = (yield mbin.receive()) 354 | if collector: 355 | collector.send(val) 356 | if val: 357 | print(val) 358 | while is_rec: 359 | val = (yield mbin.receive()) 360 | if collector: 361 | collector.send(val) 362 | if val: 363 | print(val) 364 | 365 | @do(ContinuationMonad) 366 | def join(l, waiter): 367 | collector = Mailbox() 368 | par(l, collector) 369 | for c, count in enumerate(l): 370 | val = (yield collector.receive()) 371 | print('Inside join: {0}'.format(val)) 372 | print('Counter is: {0}'.format(c)) 373 | waiter.send('Done') 374 | 375 | def csend(actor, task): 376 | actor.send(task) 377 | 378 | @do(ContinuationMonad) 379 | def creceive(actor): 380 | val = (yield actor.receive()) 381 | mreturn(val) 382 | 383 | @do(ContinuationMonad) 384 | def execute(sink1, sink2, waiter): 385 | join([sink1, sink2], waiter)() 386 | add_callback(waiter)() 387 | print('After join') 388 | sink1.send(1) 389 | sink2.send(2) 390 | print('After sink send') 391 | 392 | def coroutine(fun): 393 | 394 | def start(*args, **kwargs): 395 | cr = fun(*args, **kwargs) 396 | next(cr) 397 | return cr 398 | return start 399 | 400 | def par(l, collector=None): 401 | for mbin in l: 402 | recv(mbin, collector, True)() 403 | 404 | def choice(cond, tasks, mbin): 405 | for task in tasks: 406 | if cond(task): 407 | mbin.send(task) 408 | break 409 | 410 | def rec(): 411 | pass 412 | 413 | @do(ContinuationMonad) 414 | def add_callback(waiter): 415 | print('In do computation') 416 | s = (yield waiter.receive()) 417 | print('Time to end this wonderful jurney: {0}'.format(s)) 418 | # nooooooo 419 | 420 | # worker@allice@workflow 421 | # create a conversation channel 422 | # workflow is a combination of work(flow)lets 423 | # It specifies whether the actors are local or remote, 424 | # provide a runtime guarantee if somethink is not right 425 | # from the pool of workers, assign a handler to one of it 426 | # if you have receive, it's you 427 | original = Mailbox() # channels 428 | multiplied = Mailbox() # channels, whenever it is send 429 | multiplied1 = Mailbox() 430 | sink = Mailbox() 431 | sink1 = Mailbox() 432 | sink2 = Mailbox() 433 | waiter = Mailbox() 434 | val = creceive(sink1) 435 | print(val) 436 | # spawn an Actor on their behalf ... start doing it, cannot join here 437 | execute(sink1, sink2, original) 438 | insert(original, [1, 2, 3])() 439 | multiply(original, multiplied, 3)() 440 | multiply(original, multiplied1, 4)() 441 | filter(multiplied1, sink, lambda x: x % 2 == 0)() 442 | # Note that here we wait on a collector, what does it means, 443 | # that we want the collector to be triggered 444 | # par([multiplied, sink1], collector) 445 | join([sink1, sink2], waiter)() 446 | add_callback(waiter)() 447 | print('After join') 448 | sink1.send(1) 449 | sink2.send(2) 450 | print('After sink send') 451 | print_all(sink)() 452 | 453 | # So every workflow always has a collcetor, 454 | # so the continuation only adds 455 | 456 | @do(ContinuationMonad) 457 | def multiply_print(values, factor, orig, mult, me): 458 | # tests for actor framework 459 | for val in values: 460 | # send values to the originator 461 | me.to(orig).send(val) 462 | # originator receive the values, process them 463 | # and send them to the multiplier 464 | result = val * factor 465 | orig.to(mult).send(result) 466 | # then the multiplier returns them to me 467 | mult.to(me).send(result) 468 | # Note: we have data paralellism here,, not task parallelism 469 | 470 | # inserter [1, 2, 3]| multiplier | printer 471 | -------------------------------------------------------------------------------- /docs/.static/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/auvipy/cell/54e474592112b90ebb63aec1e6ef1856619e0e14/docs/.static/.keep -------------------------------------------------------------------------------- /docs/.templates/page.html: -------------------------------------------------------------------------------- 1 | {% extends "layout.html" %} 2 | {% block body %} 3 |
4 | 5 | {% if version == "0.2" %} 6 |

7 | This document is for cell's development version, which can be 8 | significantly different from previous releases. Get old docs here: 9 | 10 | 2.1. 11 |

12 | {% else %} 13 |

14 | This document describes cell {{ version }}. For development docs, 15 | go here. 16 |

17 | {% endif %} 18 | 19 |
20 | {{ body }} 21 | {% endblock %} 22 | -------------------------------------------------------------------------------- /docs/.templates/sidebarintro.html: -------------------------------------------------------------------------------- 1 | 4 | 5 | -------------------------------------------------------------------------------- /docs/.templates/sidebarlogo.html: -------------------------------------------------------------------------------- 1 | 4 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | 9 | # Internal variables. 10 | PAPEROPT_a4 = -D latex_paper_size=a4 11 | PAPEROPT_letter = -D latex_paper_size=letter 12 | ALLSPHINXOPTS = -d .build/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 13 | 14 | .PHONY: help clean html web pickle htmlhelp latex changes linkcheck 15 | 16 | help: 17 | @echo "Please use \`make ' where is one of" 18 | @echo " html to make standalone HTML files" 19 | @echo " pickle to make pickle files" 20 | @echo " json to make JSON files" 21 | @echo " htmlhelp to make HTML files and a HTML help project" 22 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 23 | @echo " changes to make an overview over all changed/added/deprecated items" 24 | @echo " linkcheck to check all external links for integrity" 25 | 26 | clean: 27 | -rm -rf .build/* 28 | 29 | html: 30 | mkdir -p .build/html .build/doctrees 31 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) .build/html 32 | @echo 33 | @echo "Build finished. The HTML pages are in .build/html." 34 | 35 | coverage: 36 | mkdir -p .build/coverage .build/doctrees 37 | $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) .build/coverage 38 | @echo 39 | @echo "Build finished." 40 | 41 | pickle: 42 | mkdir -p .build/pickle .build/doctrees 43 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) .build/pickle 44 | @echo 45 | @echo "Build finished; now you can process the pickle files." 46 | 47 | web: pickle 48 | 49 | json: 50 | mkdir -p .build/json .build/doctrees 51 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) .build/json 52 | @echo 53 | @echo "Build finished; now you can process the JSON files." 54 | 55 | htmlhelp: 56 | mkdir -p .build/htmlhelp .build/doctrees 57 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) .build/htmlhelp 58 | @echo 59 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 60 | ".hhp project file in .build/htmlhelp." 61 | 62 | latex: 63 | mkdir -p .build/latex .build/doctrees 64 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) .build/latex 65 | @echo 66 | @echo "Build finished; the LaTeX files are in .build/latex." 67 | @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ 68 | "run these through (pdf)latex." 69 | 70 | changes: 71 | mkdir -p .build/changes .build/doctrees 72 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) .build/changes 73 | @echo 74 | @echo "The overview file is in .build/changes." 75 | 76 | linkcheck: 77 | mkdir -p .build/linkcheck .build/doctrees 78 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) .build/linkcheck 79 | @echo 80 | @echo "Link check complete; look for any errors in the above output " \ 81 | "or in .build/linkcheck/output.txt." 82 | -------------------------------------------------------------------------------- /docs/_ext/applyxrefs.py: -------------------------------------------------------------------------------- 1 | """Adds xref targets to the top of files.""" 2 | 3 | import sys 4 | import os 5 | 6 | testing = False 7 | 8 | DONT_TOUCH = ( 9 | './index.txt', 10 | ) 11 | 12 | 13 | def target_name(fn): 14 | if fn.endswith('.txt'): 15 | fn = fn[:-4] 16 | return '_' + fn.lstrip('./').replace('/', '-') 17 | 18 | 19 | def process_file(fn, lines): 20 | lines.insert(0, '\n') 21 | lines.insert(0, '.. %s:\n' % target_name(fn)) 22 | try: 23 | f = open(fn, 'w') 24 | except IOError: 25 | print("Can't open %s for writing. Not touching it." % fn) 26 | return 27 | try: 28 | f.writelines(lines) 29 | except IOError: 30 | print("Can't write to %s. Not touching it." % fn) 31 | finally: 32 | f.close() 33 | 34 | 35 | def has_target(fn): 36 | try: 37 | f = open(fn, 'r') 38 | except IOError: 39 | print("Can't open %s. Not touching it." % fn) 40 | return (True, None) 41 | readok = True 42 | try: 43 | lines = f.readlines() 44 | except IOError: 45 | print("Can't read %s. Not touching it." % fn) 46 | readok = False 47 | finally: 48 | f.close() 49 | if not readok: 50 | return (True, None) 51 | 52 | #print fn, len(lines) 53 | if len(lines) < 1: 54 | print("Not touching empty file %s." % fn) 55 | return (True, None) 56 | if lines[0].startswith('.. _'): 57 | return (True, None) 58 | return (False, lines) 59 | 60 | 61 | def main(argv=None): 62 | if argv is None: 63 | argv = sys.argv 64 | 65 | if len(argv) == 1: 66 | argv.extend('.') 67 | 68 | files = [] 69 | for root in argv[1:]: 70 | for (dirpath, dirnames, filenames) in os.walk(root): 71 | files.extend([(dirpath, f) for f in filenames]) 72 | files.sort() 73 | files = [os.path.join(p, fn) for p, fn in files if fn.endswith('.txt')] 74 | #print files 75 | 76 | for fn in files: 77 | if fn in DONT_TOUCH: 78 | print("Skipping blacklisted file %s." % fn) 79 | continue 80 | 81 | target_found, lines = has_target(fn) 82 | if not target_found: 83 | if testing: 84 | print '%s: %s' % (fn, lines[0]), 85 | else: 86 | print "Adding xref to %s" % fn 87 | process_file(fn, lines) 88 | else: 89 | print "Skipping %s: already has a xref" % fn 90 | 91 | if __name__ == '__main__': 92 | sys.exit(main()) 93 | -------------------------------------------------------------------------------- /docs/_ext/literals_to_xrefs.py: -------------------------------------------------------------------------------- 1 | """ 2 | Runs through a reST file looking for old-style literals, and helps replace them 3 | with new-style references. 4 | """ 5 | 6 | import re 7 | import sys 8 | import shelve 9 | 10 | try: 11 | input = input 12 | except NameError: 13 | input = raw_input # noqa 14 | 15 | refre = re.compile(r'``([^`\s]+?)``') 16 | 17 | ROLES = ( 18 | 'attr', 19 | 'class', 20 | "djadmin", 21 | 'data', 22 | 'exc', 23 | 'file', 24 | 'func', 25 | 'lookup', 26 | 'meth', 27 | 'mod', 28 | "djadminopt", 29 | "ref", 30 | "setting", 31 | "term", 32 | "tfilter", 33 | "ttag", 34 | 35 | # special 36 | "skip", 37 | ) 38 | 39 | ALWAYS_SKIP = [ 40 | "NULL", 41 | "True", 42 | "False", 43 | ] 44 | 45 | 46 | def fixliterals(fname): 47 | data = open(fname).read() 48 | 49 | last = 0 50 | new = [] 51 | storage = shelve.open("/tmp/literals_to_xref.shelve") 52 | lastvalues = storage.get("lastvalues", {}) 53 | 54 | for m in refre.finditer(data): 55 | 56 | new.append(data[last:m.start()]) 57 | last = m.end() 58 | 59 | line_start = data.rfind("\n", 0, m.start()) 60 | line_end = data.find("\n", m.end()) 61 | prev_start = data.rfind("\n", 0, line_start) 62 | next_end = data.find("\n", line_end + 1) 63 | 64 | # Skip always-skip stuff 65 | if m.group(1) in ALWAYS_SKIP: 66 | new.append(m.group(0)) 67 | continue 68 | 69 | # skip when the next line is a title 70 | next_line = data[m.end():next_end].strip() 71 | if next_line[0] in "!-/:-@[-`{-~" and \ 72 | all(c == next_line[0] for c in next_line): 73 | new.append(m.group(0)) 74 | continue 75 | 76 | sys.stdout.write("\n" + "-" * 80 + "\n") 77 | sys.stdout.write(data[prev_start + 1:m.start()]) 78 | sys.stdout.write(colorize(m.group(0), fg="red")) 79 | sys.stdout.write(data[m.end():next_end]) 80 | sys.stdout.write("\n\n") 81 | 82 | replace_type = None 83 | while replace_type is None: 84 | replace_type = input( 85 | colorize("Replace role: ", fg="yellow")).strip().lower() 86 | if replace_type and replace_type not in ROLES: 87 | replace_type = None 88 | 89 | if replace_type == "": 90 | new.append(m.group(0)) 91 | continue 92 | 93 | if replace_type == "skip": 94 | new.append(m.group(0)) 95 | ALWAYS_SKIP.append(m.group(1)) 96 | continue 97 | 98 | default = lastvalues.get(m.group(1), m.group(1)) 99 | if default.endswith("()") and \ 100 | replace_type in ("class", "func", "meth"): 101 | default = default[:-2] 102 | replace_value = input( 103 | colorize("Text [", fg="yellow") + 104 | default + colorize("]: ", fg="yellow"), 105 | ).strip() 106 | if not replace_value: 107 | replace_value = default 108 | new.append(":%s:`%s`" % (replace_type, replace_value)) 109 | lastvalues[m.group(1)] = replace_value 110 | 111 | new.append(data[last:]) 112 | open(fname, "w").write("".join(new)) 113 | 114 | storage["lastvalues"] = lastvalues 115 | storage.close() 116 | 117 | 118 | def colorize(text='', opts=(), **kwargs): 119 | """ 120 | Returns your text, enclosed in ANSI graphics codes. 121 | 122 | Depends on the keyword arguments 'fg' and 'bg', and the contents of 123 | the opts tuple/list. 124 | 125 | Returns the RESET code if no parameters are given. 126 | 127 | Valid colors: 128 | 'black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white' 129 | 130 | Valid options: 131 | 'bold' 132 | 'underscore' 133 | 'blink' 134 | 'reverse' 135 | 'conceal' 136 | 'noreset' - string will not be auto-terminated with the RESET code 137 | 138 | Examples: 139 | colorize('hello', fg='red', bg='blue', opts=('blink',)) 140 | colorize() 141 | colorize('goodbye', opts=('underscore',)) 142 | print colorize('first line', fg='red', opts=('noreset',)) 143 | print 'this should be red too' 144 | print colorize('and so should this') 145 | print 'this should not be red' 146 | """ 147 | color_names = ('black', 'red', 'green', 'yellow', 148 | 'blue', 'magenta', 'cyan', 'white') 149 | foreground = dict([(color_names[x], '3%s' % x) for x in range(8)]) 150 | background = dict([(color_names[x], '4%s' % x) for x in range(8)]) 151 | 152 | RESET = '0' 153 | opt_dict = {'bold': '1', 154 | 'underscore': '4', 155 | 'blink': '5', 156 | 'reverse': '7', 157 | 'conceal': '8'} 158 | 159 | text = str(text) 160 | code_list = [] 161 | if text == '' and len(opts) == 1 and opts[0] == 'reset': 162 | return '\x1b[%sm' % RESET 163 | for k, v in kwargs.items(): 164 | if k == 'fg': 165 | code_list.append(foreground[v]) 166 | elif k == 'bg': 167 | code_list.append(background[v]) 168 | for o in opts: 169 | if o in opt_dict: 170 | code_list.append(opt_dict[o]) 171 | if 'noreset' not in opts: 172 | text = text + '\x1b[%sm' % RESET 173 | return ('\x1b[%sm' % ';'.join(code_list)) + text 174 | 175 | if __name__ == '__main__': 176 | try: 177 | fixliterals(sys.argv[1]) 178 | except (KeyboardInterrupt, SystemExit): 179 | print 180 | -------------------------------------------------------------------------------- /docs/_theme/celery/static/celery.css_t: -------------------------------------------------------------------------------- 1 | /* 2 | * celery.css_t 3 | * ~~~~~~~~~~~~ 4 | * 5 | * :copyright: Copyright 2010 by Armin Ronacher. 6 | * :license: BSD, see LICENSE for details. 7 | */ 8 | 9 | {% set page_width = 940 %} 10 | {% set sidebar_width = 220 %} 11 | {% set body_font_stack = 'Optima, Segoe, "Segoe UI", Candara, Calibri, Arial, sans-serif' %} 12 | {% set headline_font_stack = 'Futura, "Trebuchet MS", Arial, sans-serif' %} 13 | {% set code_font_stack = "'Consolas', 'Menlo', 'Deja Vu Sans Mono', 'Bitstream Vera Sans Mono', monospace" %} 14 | 15 | @import url("basic.css"); 16 | 17 | /* -- page layout ----------------------------------------------------------- */ 18 | 19 | body { 20 | font-family: {{ body_font_stack }}; 21 | font-size: 17px; 22 | background-color: white; 23 | color: #000; 24 | margin: 30px 0 0 0; 25 | padding: 0; 26 | } 27 | 28 | div.document { 29 | width: {{ page_width }}px; 30 | margin: 0 auto; 31 | } 32 | 33 | div.deck { 34 | font-size: 18px; 35 | } 36 | 37 | p.developmentversion { 38 | color: red; 39 | } 40 | 41 | div.related { 42 | width: {{ page_width - 20 }}px; 43 | padding: 5px 10px; 44 | background: #F2FCEE; 45 | margin: 15px auto 15px auto; 46 | } 47 | 48 | div.documentwrapper { 49 | float: left; 50 | width: 100%; 51 | } 52 | 53 | div.bodywrapper { 54 | margin: 0 0 0 {{ sidebar_width }}px; 55 | } 56 | 57 | div.sphinxsidebar { 58 | width: {{ sidebar_width }}px; 59 | } 60 | 61 | hr { 62 | border: 1px solid #B1B4B6; 63 | } 64 | 65 | div.body { 66 | background-color: #ffffff; 67 | color: #3E4349; 68 | padding: 0 30px 0 30px; 69 | } 70 | 71 | img.celerylogo { 72 | padding: 0 0 10px 10px; 73 | float: right; 74 | } 75 | 76 | div.footer { 77 | width: {{ page_width - 15 }}px; 78 | margin: 10px auto 30px auto; 79 | padding-right: 15px; 80 | font-size: 14px; 81 | color: #888; 82 | text-align: right; 83 | } 84 | 85 | div.footer a { 86 | color: #888; 87 | } 88 | 89 | div.sphinxsidebar a { 90 | color: #444; 91 | text-decoration: none; 92 | border-bottom: 1px dashed #DCF0D5; 93 | } 94 | 95 | div.sphinxsidebar a:hover { 96 | border-bottom: 1px solid #999; 97 | } 98 | 99 | div.sphinxsidebar { 100 | font-size: 14px; 101 | line-height: 1.5; 102 | } 103 | 104 | div.sphinxsidebarwrapper { 105 | padding: 7px 10px; 106 | } 107 | 108 | div.sphinxsidebarwrapper p.logo { 109 | padding: 0 0 20px 0; 110 | margin: 0; 111 | } 112 | 113 | div.sphinxsidebar h3, 114 | div.sphinxsidebar h4 { 115 | font-family: {{ headline_font_stack }}; 116 | color: #444; 117 | font-size: 24px; 118 | font-weight: normal; 119 | margin: 0 0 5px 0; 120 | padding: 0; 121 | } 122 | 123 | div.sphinxsidebar h4 { 124 | font-size: 20px; 125 | } 126 | 127 | div.sphinxsidebar h3 a { 128 | color: #444; 129 | } 130 | 131 | div.sphinxsidebar p.logo a, 132 | div.sphinxsidebar h3 a, 133 | div.sphinxsidebar p.logo a:hover, 134 | div.sphinxsidebar h3 a:hover { 135 | border: none; 136 | } 137 | 138 | div.sphinxsidebar p { 139 | color: #555; 140 | margin: 10px 0; 141 | } 142 | 143 | div.sphinxsidebar ul { 144 | margin: 10px 0; 145 | padding: 0; 146 | color: #000; 147 | } 148 | 149 | div.sphinxsidebar input { 150 | border: 1px solid #ccc; 151 | font-family: {{ body_font_stack }}; 152 | font-size: 1em; 153 | } 154 | 155 | /* -- body styles ----------------------------------------------------------- */ 156 | 157 | a { 158 | color: #348613; 159 | text-decoration: underline; 160 | } 161 | 162 | a:hover { 163 | color: #59B833; 164 | text-decoration: underline; 165 | } 166 | 167 | div.body h1, 168 | div.body h2, 169 | div.body h3, 170 | div.body h4, 171 | div.body h5, 172 | div.body h6 { 173 | font-family: {{ headline_font_stack }}; 174 | font-weight: normal; 175 | margin: 30px 0px 10px 0px; 176 | padding: 0; 177 | } 178 | 179 | div.body h1 { margin-top: 0; padding-top: 0; font-size: 200%; } 180 | div.body h2 { font-size: 180%; } 181 | div.body h3 { font-size: 150%; } 182 | div.body h4 { font-size: 130%; } 183 | div.body h5 { font-size: 100%; } 184 | div.body h6 { font-size: 100%; } 185 | 186 | div.body h1 a.toc-backref, 187 | div.body h2 a.toc-backref, 188 | div.body h3 a.toc-backref, 189 | div.body h4 a.toc-backref, 190 | div.body h5 a.toc-backref, 191 | div.body h6 a.toc-backref { 192 | color: inherit!important; 193 | text-decoration: none; 194 | } 195 | 196 | a.headerlink { 197 | color: #ddd; 198 | padding: 0 4px; 199 | text-decoration: none; 200 | } 201 | 202 | a.headerlink:hover { 203 | color: #444; 204 | background: #eaeaea; 205 | } 206 | 207 | div.body p, div.body dd, div.body li { 208 | line-height: 1.4em; 209 | } 210 | 211 | div.admonition { 212 | background: #fafafa; 213 | margin: 20px -30px; 214 | padding: 10px 30px; 215 | border-top: 1px solid #ccc; 216 | border-bottom: 1px solid #ccc; 217 | } 218 | 219 | div.admonition p.admonition-title { 220 | font-family: {{ headline_font_stack }}; 221 | font-weight: normal; 222 | font-size: 24px; 223 | margin: 0 0 10px 0; 224 | padding: 0; 225 | line-height: 1; 226 | } 227 | 228 | div.admonition p.last { 229 | margin-bottom: 0; 230 | } 231 | 232 | div.highlight{ 233 | background-color: white; 234 | } 235 | 236 | dt:target, .highlight { 237 | background: #FAF3E8; 238 | } 239 | 240 | div.note { 241 | background-color: #eee; 242 | border: 1px solid #ccc; 243 | } 244 | 245 | div.seealso { 246 | background-color: #ffc; 247 | border: 1px solid #ff6; 248 | } 249 | 250 | div.topic { 251 | background-color: #eee; 252 | } 253 | 254 | div.warning { 255 | background-color: #ffe4e4; 256 | border: 1px solid #f66; 257 | } 258 | 259 | p.admonition-title { 260 | display: inline; 261 | } 262 | 263 | p.admonition-title:after { 264 | content: ":"; 265 | } 266 | 267 | pre, tt { 268 | font-family: {{ code_font_stack }}; 269 | font-size: 0.9em; 270 | } 271 | 272 | img.screenshot { 273 | } 274 | 275 | tt.descname, tt.descclassname { 276 | font-size: 0.95em; 277 | } 278 | 279 | tt.descname { 280 | padding-right: 0.08em; 281 | } 282 | 283 | img.screenshot { 284 | -moz-box-shadow: 2px 2px 4px #eee; 285 | -webkit-box-shadow: 2px 2px 4px #eee; 286 | box-shadow: 2px 2px 4px #eee; 287 | } 288 | 289 | table.docutils { 290 | border: 1px solid #888; 291 | -moz-box-shadow: 2px 2px 4px #eee; 292 | -webkit-box-shadow: 2px 2px 4px #eee; 293 | box-shadow: 2px 2px 4px #eee; 294 | } 295 | 296 | table.docutils td, table.docutils th { 297 | border: 1px solid #888; 298 | padding: 0.25em 0.7em; 299 | } 300 | 301 | table.field-list, table.footnote { 302 | border: none; 303 | -moz-box-shadow: none; 304 | -webkit-box-shadow: none; 305 | box-shadow: none; 306 | } 307 | 308 | table.footnote { 309 | margin: 15px 0; 310 | width: 100%; 311 | border: 1px solid #eee; 312 | background: #fdfdfd; 313 | font-size: 0.9em; 314 | } 315 | 316 | table.footnote + table.footnote { 317 | margin-top: -15px; 318 | border-top: none; 319 | } 320 | 321 | table.field-list th { 322 | padding: 0 0.8em 0 0; 323 | } 324 | 325 | table.field-list td { 326 | padding: 0; 327 | } 328 | 329 | table.footnote td.label { 330 | width: 0px; 331 | padding: 0.3em 0 0.3em 0.5em; 332 | } 333 | 334 | table.footnote td { 335 | padding: 0.3em 0.5em; 336 | } 337 | 338 | dl { 339 | margin: 0; 340 | padding: 0; 341 | } 342 | 343 | dl dd { 344 | margin-left: 30px; 345 | } 346 | 347 | blockquote { 348 | margin: 0 0 0 30px; 349 | padding: 0; 350 | } 351 | 352 | ul { 353 | margin: 10px 0 10px 30px; 354 | padding: 0; 355 | } 356 | 357 | pre { 358 | background: #F0FFEB; 359 | padding: 7px 10px; 360 | margin: 15px 0; 361 | border: 1px solid #C7ECB8; 362 | border-radius: 2px; 363 | -moz-border-radius: 2px; 364 | -webkit-border-radius: 2px; 365 | line-height: 1.3em; 366 | } 367 | 368 | tt { 369 | background: #F0FFEB; 370 | color: #222; 371 | /* padding: 1px 2px; */ 372 | } 373 | 374 | tt.xref, a tt { 375 | background: #F0FFEB; 376 | border-bottom: 1px solid white; 377 | } 378 | 379 | a.reference { 380 | text-decoration: none; 381 | border-bottom: 1px dashed #DCF0D5; 382 | } 383 | 384 | a.reference:hover { 385 | border-bottom: 1px solid #6D4100; 386 | } 387 | 388 | a.footnote-reference { 389 | text-decoration: none; 390 | font-size: 0.7em; 391 | vertical-align: top; 392 | border-bottom: 1px dashed #DCF0D5; 393 | } 394 | 395 | a.footnote-reference:hover { 396 | border-bottom: 1px solid #6D4100; 397 | } 398 | 399 | a:hover tt { 400 | background: #EEE; 401 | } 402 | -------------------------------------------------------------------------------- /docs/_theme/celery/theme.conf: -------------------------------------------------------------------------------- 1 | [theme] 2 | inherit = basic 3 | stylesheet = celery.css 4 | 5 | [options] 6 | -------------------------------------------------------------------------------- /docs/changelog.rst: -------------------------------------------------------------------------------- 1 | ../Changelog -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import sys 4 | import os 5 | 6 | # eventlet/gevent should not monkey patch anything. 7 | os.environ["GEVENT_NOPATCH"] = "yes" 8 | os.environ["EVENTLET_NOPATCH"] = "yes" 9 | 10 | root_dir = os.path.sep.join(os.path.realpath(__file__).split(os.path.sep)[:-2]) 11 | sys.path.insert(0, root_dir) 12 | 13 | this = os.path.dirname(os.path.abspath(__file__)) 14 | 15 | # If your extensions are in another directory, add it here. If the directory 16 | # is relative to the documentation root, use os.path.abspath to make it 17 | # absolute, like shown here. 18 | sys.path.append(os.path.join(os.pardir, "tests")) 19 | sys.path.append(os.path.join(this, "_ext")) 20 | import cell 21 | 22 | # General configuration 23 | # --------------------- 24 | 25 | extensions = ['sphinx.ext.autodoc', 26 | 'sphinx.ext.coverage', 27 | 'sphinx.ext.pngmath', 28 | 'sphinxcontrib.issuetracker'] 29 | 30 | # Add any paths that contain templates here, relative to this directory. 31 | templates_path = ['.templates'] 32 | 33 | # The suffix of source filenames. 34 | source_suffix = '.rst' 35 | 36 | # The master toctree document. 37 | master_doc = 'index' 38 | 39 | # General information about the project. 40 | project = u'cell' 41 | copyright = u'2011-2012, Ask Solem & Contributors' 42 | 43 | # The version info for the project you're documenting, acts as replacement for 44 | # |version| and |release|, also used in various other places throughout the 45 | # built documents. 46 | # 47 | # The short X.Y version. 48 | version = ".".join(map(str, cell.VERSION[0:2])) 49 | # The full version, including alpha/beta/rc tags. 50 | release = cell.__version__ 51 | 52 | exclude_trees = ['.build'] 53 | 54 | # If true, '()' will be appended to :func: etc. cross-reference text. 55 | add_function_parentheses = True 56 | 57 | # The name of the Pygments (syntax highlighting) style to use. 58 | pygments_style = 'trac' 59 | 60 | # Add any paths that contain custom static files (such as style sheets) here, 61 | # relative to this directory. They are copied after the builtin static files, 62 | # so a file named "default.css" will overwrite the builtin "default.css". 63 | html_static_path = ['.static'] 64 | 65 | html_use_smartypants = True 66 | 67 | # If false, no module index is generated. 68 | html_use_modindex = True 69 | 70 | # If false, no index is generated. 71 | html_use_index = True 72 | 73 | latex_documents = [ 74 | ('index', 'cell.tex', ur'cell Documentation', 75 | ur'Ask Solem & Contributors', 'manual'), 76 | ] 77 | 78 | html_theme = "celery" 79 | html_theme_path = ["_theme"] 80 | html_sidebars = { 81 | 'index': ['sidebarintro.html', 'sourcelink.html', 'searchbox.html'], 82 | '**': ['sidebarlogo.html', 'relations.html', 83 | 'sourcelink.html', 'searchbox.html'], 84 | } 85 | 86 | ### Issuetracker 87 | 88 | issuetracker = "github" 89 | issuetracker_project = "celery/cell" 90 | issuetracker_issue_pattern = r'[Ii]ssue #(\d+)' 91 | -------------------------------------------------------------------------------- /docs/getting-started/hello-world-example.rst: -------------------------------------------------------------------------------- 1 | 2 | Hello World 3 | =================== 4 | Here we will demonstrate the main cell features starting with a simple Hello World example. 5 | To understand what is going on, we will break the example into pieces. 6 | 7 | .. contents:: 8 | :local: 9 | 10 | 11 | Defining an actor 12 | ~~~~~~~~~~~~~~~~~ 13 | 14 | Actors are implemented by extending the :py:class:`~.actors.Actor` class and then defining a series of supported methods. 15 | The supported methods should be encapsulated in the Actor's internal :py:class:`~.Actor.state` class. 16 | Our first actor implements one method called greet. 17 | 18 | .. code-block:: python 19 | 20 | from cell.actors import Actor 21 | 22 | class GreetingActor(Actor): 23 | class state: 24 | def greet(self, who='world'): 25 | print 'Hello %s' % who 26 | 27 | Starting an actor 28 | ~~~~~~~~~~~~~~~~~~~~~~~ 29 | Cell targets distributed actors management. Therefore, creating an actor means spawning an actor consumer 30 | remotely by sending a command to a celery worker agent. That is why before spawning an actor, you need to have a celery worker running: 31 | 32 | .. code-block:: bash 33 | 34 | $ celery worker 35 | 36 | Actors are created via :py:meth:`~.agents.dAgent.spawn` method, called on an instance of :py:class:`~.agents.dAgent` (distributed agent). 37 | Agents are components responsible for creating and stopping actors on celery workers. 38 | Each celery worker has embedded agent component :py:class:`dAgent` (distributed agent) listening for commands. 39 | :py:meth:`~.agents.dAgent.spawn` is invoked with a class of type :py:class:`Actor` or its derivative 40 | and starts an actor of that type remotely (in the celery worker). 41 | The method returns a proxy (an instance of :py:class:`~.actors.ActorProxy`) for the remotely started actor. 42 | The proxy holds the unique identifier, which ensures the messages can be delivered unambiguously to the same remote actor. 43 | 44 | The code below starts a :py:class:`GreetingActor` and then invokes its py:meth:`greeting` method. 45 | 46 | .. code-block:: python 47 | 48 | from cell.agents import dAgent 49 | from kombu import Connection 50 | 51 | connection = Connection() 52 | # STEP 1: Create an agent 53 | agent = dAgent(connection) 54 | # STEP 2: Pass the actor type to spawn method 55 | greeter = agent.spawn(GreetingActor) 56 | 57 | # STEP 3: Use actor proxy to call methods on the remote actor 58 | greeter.send('greet') 59 | 60 | 61 | Calling a method 62 | ~~~~~~~~~~~~~~~~~ 63 | The cell actor model comes with few build-in delivery policies. 64 | Here we demonstrate direct delivery - the message is send to a particular actor instance. 65 | (Look at the end of the section for the other delivery options.) 66 | 67 | .. code-block:: python 68 | 69 | actor.send('greet') 70 | 71 | The :py:meth:`greet` method has been executed and you can verify that by looking at the workers console output. 72 | The remote actor of type GreetingActor you created earlier handle the method. 73 | The message 'Hello world' should be printed on the celery worker console. 74 | Note that the call is asynchronous and since 'great' is a void method no result will be returned. 75 | For getting results back and invoking methods synchronously check :ref:`Getting a result back section` 76 | 77 | The basic Actor API expose three more methods for sending a message: 78 | * :py:meth:`~.actors.Actor.send` - sends to a particular actor instance 79 | * :py:meth:`~.actors.Actor.throw` - sends to an actor instance of the same type 80 | * :py:meth:`~.actors.Actor.scatter` - sends to all actor instances of the same type 81 | 82 | For more information on the above options, see the :ref:`Delivery options` section 83 | 84 | Calling a method with parameters 85 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 86 | Here is an example how to call the method :py:meth:`~.examples.hello.GreeingActor.greet` 87 | with an argument :py:attr:`who` 88 | 89 | .. code-block:: python 90 | 91 | actor.send('greet', {'who':'everyone'}) 92 | 93 | Getting a result back 94 | ~~~~~~~~~~~~~~~~~~~~~ 95 | Let's add another method to the :py:class:`GreetingActor` class 96 | :py:meth:`how_are_you` that returns a result. 97 | 98 | .. code-block:: python 99 | 100 | from cell.actors import Actor 101 | 102 | class GreetingActor(Actor): 103 | class state: 104 | def greet(self, who='world'): 105 | print 'Hello %s' % who 106 | 107 | def how_are_you(self): 108 | return 'Fine!' 109 | 110 | 111 | We can get the result in two ways: 112 | 113 | * using a **blocking call** (set the nowait parameter to True), it blocks the execution until a result is delivered or a timeout is reached: 114 | 115 | .. code-block:: python 116 | 117 | result = actor.send('greet', {'who':'everyone'}, nowait=True) 118 | 119 | d.. warning:: If you are using blocking calls, greenlets should be enabled in the celery worker: 120 | 121 | Greenlets can be enabled either by using eventlet or using gevent: 122 | 123 | .. code-block:: python 124 | 125 | >>> celery worker -P eventlet -c 100 126 | 127 | or 128 | 129 | .. code-block:: python 130 | 131 | >>> celery worker -P gevent -c 100 132 | 133 | You can read more about concurrency in celery in `here`_ 134 | 135 | .. _`here`: http://docs.celeryproject.org/en/latest/userguide/concurrency/index.html 136 | 137 | * using a **non-blocking call** (set the nowait parameter to False, the default), it returns an an :py:class:`~.AsyncResult` instance. 138 | :py:class:`~.AsyncResult` can be used to check the state of the result, get the return value or if the method failed, the exception and traceback). 139 | 140 | .. code-block:: python 141 | 142 | result = actor.send('greet', {'who':'everyone'}, nowait=False) 143 | 144 | The :meth:`~@AsyncResult.result` returns the result if it is ready or wait for the result to complete 145 | 146 | .. code-block:: python 147 | 148 | result = actor.send('greet', {'who':'everyone'}, nowait=False) 149 | print result.result 150 | 151 | See cell.result for the complete result object reference. 152 | 153 | Stopping an actor 154 | ~~~~~~~~~~~~~~~~~~ 155 | We can stop an actor if we know its id. 156 | 157 | .. code-block:: python 158 | 159 | agent.kill(actor.id) 160 | 161 | :py:meth:`agents.dAgent.kill` is a broadcast command sent to all agents. 162 | If an agents doesn't have in its registry the given actor.id, it will dismiss the command, 163 | otherwise it will gently kill the actor and delete it from its registry. 164 | 165 | Where to go from here 166 | ~~~~~~~~~~~~~~~~~~~~~ 167 | If you want to learn more you should explore the examples in the :py:mod:`examples` module in the cell codebase 168 | and/or study the :ref:`User Guide `. -------------------------------------------------------------------------------- /docs/getting-started/index.rst: -------------------------------------------------------------------------------- 1 | .. _getting-started: 2 | 3 | Getting started examples 4 | ================== 5 | 6 | .. toctree:: 7 | :maxdepth: 2 8 | 9 | hello-world-example 10 | more-examples 11 | 12 | -------------------------------------------------------------------------------- /docs/getting-started/more-examples.rst: -------------------------------------------------------------------------------- 1 | Adder 2 | ===== 3 | 4 | Actor that can add one to a given number. 5 | Adder actor can be used to implement a Counter. 6 | 7 | .. code-block:: python 8 | 9 | from kombu import Connection 10 | connection = Connection() 11 | agent = dAgent(connection) 12 | 13 | class Adder(Actor): 14 | class state(): 15 | def add_one(self, i): 16 | print 'Increasing %s with 1' % i 17 | return i + 1 18 | 19 | if __name__=='__main__': 20 | import examples.adder 21 | adder = agent.spawn(Adder) 22 | 23 | adder.call('add-one', {'i':10}) 24 | 25 | Chat-users 26 | ========== 27 | 28 | .. code-block:: python 29 | 30 | from cell.actors import Actor 31 | from cell.agents import dAgent 32 | 33 | connection = Connection() 34 | 35 | class User(Actor): 36 | class state(): 37 | 38 | def post(self, msg): 39 | print msg 40 | 41 | def post(self, msg): 42 | msg = 'Posting on the wall: %s' % msg 43 | self.scatter('post', {'msg': msg}) 44 | 45 | def message_to(self, actor, msg): 46 | a = User(id = actor, connection = self.connection) 47 | msg = 'Actor %s is sending you a message: %s' %(self.id, msg) 48 | a.call('post', {'msg':msg}) 49 | 50 | def connect(self): 51 | if not agent: 52 | agent = dAgent(self.connection) 53 | return self.agent.spawn(self) 54 | 55 | if __name__=='__main__': 56 | import examples.chat 57 | rumi = examples.chat.User(connection).connect() 58 | rumi.post('Hello everyone') 59 | 60 | ask = examples.chat.User(connection).connect() 61 | ask.post('Hello everyone') 62 | rumi.message_to(ask.id, 'How are you?') 63 | ask.message_to(rumi.id, 'Fine.You?') 64 | 65 | Map-reduce 66 | ========== 67 | 68 | .. code-block:: python 69 | 70 | import celery 71 | from cell.actors import Actor 72 | from cell.agents import dAgent 73 | 74 | my_app = celery.Celery(broker='pyamqp://guest@localhost//') 75 | agent = dAgent(connection=my_app.broker_connection()) 76 | 77 | 78 | class Aggregator(Actor): 79 | 80 | def __init__(self, barrier=None, **kwargs): 81 | self.barrier = barrier 82 | super(Aggregator, self).__init__(**kwargs) 83 | 84 | class state(Actor.state): 85 | def __init__(self): 86 | self.result = {} 87 | super(Aggregator.state, self).__init__() 88 | 89 | def aggregate(self, words): 90 | for word, n in words.items(): 91 | self.result.setdefault(word, 0) 92 | self.result[word] += n 93 | 94 | self.actor.barrier -= 1 95 | if self.actor.barrier <= 0: 96 | self.print_result() 97 | 98 | def print_result(self): 99 | for (key, val) in self.result.items(): 100 | print "%s:%s" % (key, val) 101 | 102 | 103 | class Reducer(Actor): 104 | 105 | class state(Actor.state): 106 | def __init__(self): 107 | self.aggregator = None 108 | super(Reducer.state, self).__init__() 109 | 110 | def on_agent_ready(self): 111 | self.aggregator = Aggregator(connection=self.actor.connection) 112 | 113 | def count_lines(self, line, aggregator): 114 | words = {} 115 | for word in line.split(" "): 116 | words.setdefault(word, 0) 117 | words[word] += 1 118 | self.aggregator.id = aggregator 119 | self.aggregator.call('aggregate', {'words': words}) 120 | 121 | def on_agent_ready(self): 122 | self.state.on_agent_ready() 123 | 124 | 125 | class Mapper(Actor): 126 | 127 | class state(Actor.state): 128 | REDUCERS = 10 129 | 130 | def on_agent_ready(self): 131 | self.pool = [] 132 | for i in range(self.REDUCERS): 133 | reducer = self.actor.agent.spawn(Reducer) 134 | self.pool.append(reducer) 135 | 136 | def count_document(self, file): 137 | with open(file) as f: 138 | lines = f.readlines() 139 | count = 0 140 | self.aggregator = agent.spawn(Aggregator, barrier=len(lines)) 141 | for line in lines: 142 | reducer = self.pool[count % self.REDUCERS] 143 | reducer.cast('count_lines', 144 | {'line': line, 145 | 'aggregator': self.aggregator.id}) 146 | 147 | def on_agent_ready(self): 148 | self.state.on_agent_ready() 149 | 150 | if __name__ == '__main__': 151 | import examples.map_reduce 152 | file = "map_reduce_test.txt" 153 | mapper = agent.spawn(examples.map_reduce.Mapper) 154 | mapper.call('count_document', {'file': file}) 155 | -------------------------------------------------------------------------------- /docs/images/celery-icon-128.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/auvipy/cell/54e474592112b90ebb63aec1e6ef1856619e0e14/docs/images/celery-icon-128.png -------------------------------------------------------------------------------- /docs/images/celery-icon-32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/auvipy/cell/54e474592112b90ebb63aec1e6ef1856619e0e14/docs/images/celery-icon-32.png -------------------------------------------------------------------------------- /docs/images/celery-icon-64.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/auvipy/cell/54e474592112b90ebb63aec1e6ef1856619e0e14/docs/images/celery-icon-64.png -------------------------------------------------------------------------------- /docs/images/celery_128.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/auvipy/cell/54e474592112b90ebb63aec1e6ef1856619e0e14/docs/images/celery_128.png -------------------------------------------------------------------------------- /docs/images/celery_512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/auvipy/cell/54e474592112b90ebb63aec1e6ef1856619e0e14/docs/images/celery_512.png -------------------------------------------------------------------------------- /docs/images/celery_favicon_128.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/auvipy/cell/54e474592112b90ebb63aec1e6ef1856619e0e14/docs/images/celery_favicon_128.png -------------------------------------------------------------------------------- /docs/images/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/auvipy/cell/54e474592112b90ebb63aec1e6ef1856619e0e14/docs/images/favicon.ico -------------------------------------------------------------------------------- /docs/images/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/auvipy/cell/54e474592112b90ebb63aec1e6ef1856619e0e14/docs/images/favicon.png -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | ========================= 2 | cell - Actors for Celery 3 | ========================= 4 | 5 | Contents: 6 | 7 | .. toctree:: 8 | :maxdepth: 2 9 | 10 | introduction 11 | getting-started/index 12 | user-guide/index 13 | 14 | 15 | .. toctree:: 16 | :maxdepth: 1 17 | 18 | changelog 19 | reference/index 20 | 21 | 22 | Indices and tables 23 | ================== 24 | 25 | * :ref:`genindex` 26 | * :ref:`modindex` 27 | * :ref:`search` 28 | 29 | -------------------------------------------------------------------------------- /docs/introduction.rst: -------------------------------------------------------------------------------- 1 | ../README.rst -------------------------------------------------------------------------------- /docs/reference/cell.actors.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | cell.actors 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cell.actors 8 | 9 | .. automodule:: cell.actors 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cell.agents.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | cell.agents 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cell.agents 8 | 9 | .. automodule:: cell.agents 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cell.bin.base.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | cell.bin.base 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cell.bin.base 8 | 9 | .. automodule:: cell.bin.base 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cell.bin.cell.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | cell.bin.cell 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cell.bin.cell 8 | 9 | .. automodule:: cell.bin.cell 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cell.exceptions.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | cell.exceptions 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cell.exceptions 8 | 9 | .. automodule:: cell.exceptions 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cell.g.eventlet.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | cell.g.eventlet 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cell.g.eventlet 8 | 9 | .. automodule:: cell.g.eventlet 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cell.g.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | cell.g 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cell.g 8 | 9 | .. automodule:: cell.g 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cell.models.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | cell.models 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cell.models 8 | 9 | .. automodule:: cell.models 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cell.presence.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | cell.presence 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cell.presence 8 | 9 | .. automodule:: cell.presence 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cell.results.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | cell.results 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cell.results 8 | 9 | .. automodule:: cell.results 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/cell.utils.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | cell.utils 3 | ======================== 4 | 5 | .. contents:: 6 | :local: 7 | .. currentmodule:: cell.utils 8 | 9 | .. automodule:: cell.utils 10 | :members: 11 | :undoc-members: 12 | -------------------------------------------------------------------------------- /docs/reference/index.rst: -------------------------------------------------------------------------------- 1 | =============== 2 | API Reference 3 | =============== 4 | 5 | :Release: |version| 6 | :Date: |today| 7 | 8 | .. toctree:: 9 | :maxdepth: 1 10 | 11 | cell.actors 12 | cell.agents 13 | cell.results 14 | cell.exceptions 15 | cell.presence 16 | cell.models 17 | cell.g 18 | cell.g.eventlet 19 | cell.utils 20 | cell.bin.cell 21 | cell.bin.base 22 | -------------------------------------------------------------------------------- /docs/user-guide/delivery-options.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/auvipy/cell/54e474592112b90ebb63aec1e6ef1856619e0e14/docs/user-guide/delivery-options.png -------------------------------------------------------------------------------- /docs/user-guide/index.rst: -------------------------------------------------------------------------------- 1 | .. _guide: 2 | 3 | User Guide 4 | ============ 5 | 6 | “We are all merely Actors” Ask Solem 7 | 8 | .. contents:: 9 | :local: 10 | :depth: 1 11 | 12 | 13 | Basics 14 | ~~~~~~ 15 | 16 | .. _calling-cheat: 17 | 18 | .. topic:: Quick Cheat Sheet for Agents 19 | 20 | - ``a.spawn(cls, kwarg=value)`` 21 | start a remote actor instance 22 | 23 | - ``a.select(cls)`` 24 | returns a remote actor instance if actor of type cls is already started 25 | 26 | - ``a.kill(id)`` 27 | stops an actor by id 28 | 29 | .. topic:: Quick Cheat Sheet for Actors 30 | 31 | - ``a.send.method(kwarg=value, nowait=False)`` 32 | invoke method on the remote actor instance asynchronously 33 | returns AsyncResult 34 | 35 | - ``a.send.method(kwarg=value, nowait=False)`` 36 | invoke method on a remote actor instance synchronously 37 | 38 | - ``a.throw.method(kwarg=value)`` 39 | invoke method on a remote actor of type a instance asynchronously 40 | 41 | - ``a.scatter.method(kwarg=value)`` 42 | invoke method on all remote actors of type a 43 | 44 | Create an Actor 45 | ~~~~~~~~~~~~~~~~~ 46 | To implement a new actor extend :py:class:`~cell.actors.Actor` class. 47 | Actor behaviour (all supported methods) should be implemented in the internal 48 | :py:class:`~cell.actors.Actor.state` class. 49 | 50 | .. code-block:: python 51 | 52 | class Logger(Actor): 53 | class state(Actor.state): 54 | def log(self. msg): 55 | print msg 56 | 57 | Then the actor can be started (spawned) remotely using :py:meth:`cell.agents.dAgent.spawn` 58 | 59 | .. code-block:: python 60 | 61 | from kombu import Connection 62 | from cell.agents import dAgent 63 | 64 | agent = dAgent.Connection() 65 | logger_ref = agent.spawn(Logger) 66 | 67 | Actors and ActorProxy 68 | ~~~~~~~~~~~~~~~~~~~~~ 69 | We do not create instances of actors directly, instead we ask an :py:class:`~cell.agents.dAgent` 70 | to spawn (instantiate) an Actor of given type on a remote celery worker. 71 | 72 | .. code-block:: python 73 | 74 | logger_ref = agent.spawn(Logger) 75 | 76 | The returned object (logger_ref) is not of Logger type like our actor, it is not even an Actor. 77 | It is an instance of :py:class:`~cell.actrors.ActorProxy`, which is a wrapper (proxy) around an actor: 78 | The actual actor can be deployed on a different machine on different celery worker. 79 | 80 | :py:class:`~cell.actrors.ActorProxy` transparently and invisibly to the client sends messages over the wire to the correct worker(s). 81 | It wraps all method defined in the `Actor.state` internal class. 82 | 83 | Select an existing actor 84 | ~~~~~~~~~~~~~~~~~~~~~~~~ 85 | If you know that an actor of the type you need is already spawned, 86 | but you don't know its id, you can get a proxy for it as follows: 87 | 88 | .. code-block:: python 89 | 90 | from examples.logger import Logger 91 | try: 92 | logger = agent.select(Logger) 93 | except KeyError: 94 | logger = agent.spawn(Logger) 95 | 96 | In the above example we check if an actor is already spawned in any of the workers. 97 | If Logger is found in any of the workers, the :py:meth:`agents.Agent.select` will throw 98 | an exception of type :py:class:`KeyError`. 99 | 100 | Actor Delivery Types 101 | ~~~~~~~~~~~~~~~~~~~~ 102 | Here we create two actors to use throughout the section. 103 | 104 | .. code-block:: python 105 | 106 | logger1 = agent.spawn(Logger) 107 | logger2 = agent.spawn(Logger) 108 | 109 | logger1 and logger2 are ActorProxies for the actors started remotely as a result of the spawn command. 110 | The actors are of the same type (Logger), but have different identifiers. 111 | 112 | Cell supports few sending primitives, implementing the different delivery policies: 113 | 114 | * direct (using :py:meth:`~.actor.Actor.send`) - sends a message to a concrete actor (to the invoker) 115 | .. code-block:: python 116 | 117 | logger1.send('log', {'msg':'the quick brown fox ...'}) 118 | 119 | The message is delivered to the remote counterpart of logger1. 120 | 121 | * round-robin (using :py:meth:`~.actor.Actor.throw`) - sends a message to an arbitrary actor with the same actor type as the invoker 122 | 123 | .. code-block:: python 124 | 125 | logger1.throw('log', {'msg':'the quick brown fox ...'}) 126 | 127 | The message is delivered to the remote counterparts of either logger1, or logger2. 128 | 129 | * broadcast (using :py:meth:`~.actor.Actor.scatter`) - sends a message to all running actors, having the same actor type as the invoker 130 | 131 | .. code-block:: python 132 | 133 | logger1.scatter('log', {'msg':'the quick brown fox ...'}) 134 | 135 | The message is delivered to both logger 1 and logger 2. 136 | All running actors, having a type Logger will receive and process the message. 137 | 138 | The picture below sketches the three delivery options for actors. It shows what each primitive does and how each delivery option 139 | is implemented in terms of transport entities (exchanges and queues). Each primitive (depending on its type and the type of the actor) 140 | has an exchange and a routing key to use when sending. 141 | 142 | .. image:: delivery-options.* 143 | 144 | Emit method or how to bind actors together 145 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 146 | In addition to the inboxes (all exchanges, explained in the :ref:`Actor Delivery Types`), each actor also have an outbox. 147 | Outboxes are used when we want to bind actors together. An example is forwarding messages from one actor to another. 148 | This means that the original sender address/reference is maintained even 149 | though the message is going through a 'mediator'. 150 | This can be useful when writing actors that work as routers, load-balancers, replicators etc. 151 | They are also useful for returning result back to the caller. 152 | 153 | How it works? 154 | The :py:meth:`~.cell.actors.Actor.emit` method explicitly send a message to its actor outbox. 155 | By default, no one is listening to the actor outbox. 156 | The binding can be added and removed dynamically when needed by the application. 157 | (See :py:meth:`~.actors.add_binding` and :py:meth:`~.actors.remove_binding` for more information) 158 | The |forward| operator is a handy wrapper around the :py:meth:`~.actors.add_binding` method. 159 | Fir example The code below binds the outbox of logger1 to the inbox of logger2 160 | (logger1 |forward| logger 2) 161 | Thus, all messages that are send to logger1 (via :py:meth:`~.actors.emit`) will be 162 | received by logger 2. 163 | 164 | .. code-block:: python 165 | 166 | logger1 = agent.spawn(Logger) 167 | logger2 = agent.spawn(Logger) 168 | 169 | logger1 |forward| logger2 170 | logger1.emit('log', {'msg':'I will be printed from logger2'}) 171 | logger1 |stop_forward| logger2 172 | logger1.emit('log', {'msg':'I will be printed from logger1'}) 173 | 174 | 175 | 176 | Type your method calls 177 | ~~~~~~~~~~~~~~~~~~~~~~~~~~ 178 | Passing actor methods as a string is often not a convenient option. 179 | That's why we provide an API to call the method directly. The ActorProxy class returns a partial when any of the Actor API methods is called (call, send, throw, scatter) 180 | actor.api_method.state_method_to_be_called returns a partial application of actor.api_method with firtsh argument set to method_to_be_called. 181 | Therefore, the following pair of executions on the instance of :py:class: `ActorProxy` are teh same 182 | Note that if the state_method_to_be_called does not exist in the :py:class: `ActorProxy.state` an exception ( :py:meth:`AttributeError`) will be thrown. 183 | 184 | .. code-block:: python 185 | 186 | logger.send.log({'msg':'Hello'}, nowait=False) 187 | logger.send('log', {'msg':'Hello'}, nowait=False) 188 | 189 | logger.call.log({'msg':'Hello'}, nowait=False) 190 | logger.call('log', {'msg':'Hello'}, nowait=False) 191 | 192 | logger.throw.log({'msg':'Hello'}, nowait=False) 193 | logger.throw('log', {'msg':'Hello'}, nowait=False) 194 | 195 | logger.scatter.log({'msg':'Hello'}, nowait=False) 196 | logger.scatter('log', {'msg':'Hello'}, nowait=False) 197 | 198 | # throws `AttributeError` on the sender 199 | # and the message is not sent 200 | logger.scatter.my_log({'msg':'Hello'} 201 | 202 | Getting the result back 203 | ~~~~~~~~~~~~~~~~~~~~~~ 204 | Cell supports several types of return patterns: 205 | 206 | * fire and forget - whenever the nowait parameter is set to True, apply to all :py:meth:`cell.actors.Actor`) methods 207 | * returning future - apply only to :py:meth:`cell.Actors.call`. It returns :py:class:`cell.Actors.AsyncResults` instance when invoked with nowait is False. The result can be accessed when needed via :py:meth:`cell.Actors.AsyncResults.result`. 208 | * blocking call (returning the result) - apply to :py:meth:`cell.Actors.send` and :py:meth:`cell.Actors.throw` when invoked with nowait = False 209 | * generator - apply to :py:meth:`cell.Actors.scatter` when invoked with nowait = False 210 | 211 | ScatterGatherFirstCompletedActor 212 | ------------------------------- 213 | Here is how you can send a broadcast message to all actors of a given type and wait for the first message that 214 | is received. 215 | 216 | .. code-block:: python 217 | 218 | from cell.agents import first_reply 219 | first_reply(actor.scatter('greet', limit=1) 220 | 221 | You can implement you own :py:meth:`first_reply` function. Remember that the scatter method returns generator. 222 | Then all you need to do is call its next() method only once: 223 | 224 | .. code-block:: python 225 | 226 | def first_reply(replies, key): 227 | try: 228 | return replies.next() 229 | except StopIteration: 230 | raise KeyError(key) 231 | 232 | Collect all replies 233 | ------------------- 234 | if limit is not specifies, the 235 | 236 | .. code-block:: python 237 | 238 | # returns a generator 239 | res = actor.scatter('greet', timeout = 5) 240 | 241 | # Iterate over all the results until a timeout is reached 242 | for i in res: 243 | print i 244 | 245 | Request-reply pattern 246 | ~~~~~~~~~~~~~~~~~~~~~~ 247 | 248 | .. note:: When using actors, always start celery with greenlet support enabled! 249 | (see `Greenlets in celery`_ for more information) 250 | 251 | Depending on your environment and requirements, you can start green workers in one of these ways: 252 | 253 | .. code-block:: bash 254 | $ celery worker -P eventlet -c 1000 255 | 256 | or 257 | 258 | .. code-block:: bash 259 | $ celery -P gevent -c 1000 260 | 261 | .. _`Greenlets in celery`_ http://docs.celeryproject.org/en/latest/userguide/concurrency/eventlet.html 262 | 263 | When greenlet is enbaled, each method is executed in its own greenlet. 264 | 265 | Actor model prescribes that an actor should not block and wait for the result of another actor. Therefore, the result should always be passed 266 | via callback. However, if you are not a fen of the CPS (continuation passing style) and want to preserve your control flow, you can use greenlets. 267 | 268 | Below, the two options (callbacks and greenlets) are explained in more details: 269 | 270 | * **via greenlets** 271 | 272 | .. warning:: To use this option, GREENLETS SHOULD BE ENABLED IN THE CELERY WORKERS running the actors. If not, a deadlock is possible. 273 | 274 | Below is an example of Counter actor implementation. To count to a given target, the Counter calls the Incrementer inc method in a loop. 275 | The Incrementer advance the number by one and returned the incremented value. 276 | The loop continues until the final count target is reached. 277 | 278 | .. code-block:: python 279 | 280 | class Incrementer(Actor): 281 | class state: 282 | def inc(self, n) 283 | return n + 1 284 | 285 | def inc(self, n): 286 | self.send('inc', {'n':n}, nowait=False) 287 | 288 | class Counter(Actor): 289 | class state: 290 | def count_to(self, target) 291 | incrementer = self.agent.spawn(Incrementer) 292 | next = 0 293 | while target: 294 | print next 295 | next = incrementer.inc(next) 296 | target -= 1 297 | 298 | The actors (Counter and Incrementer) can run in the same worker or can run in a different workers and the above code 299 | will work in both cases. 300 | 301 | *What will happen if celery workers are not greenlet enabled?* 302 | 303 | If the actors are in the same worker and this worker is not started with a greenlet support 304 | the Counter worker will be blocked, waiting for the result of the Incrementer, preventing the Incrementer 305 | from receiving commands and therefore causing a dealock. 306 | If the worker supports greenlets, only the Counter greenlet will block, allowing the worker execution flow to continue. 307 | 308 | * **via actor outboxes** 309 | 310 | .. code-block:: python 311 | 312 | class Incrementer(Actor): 313 | class state: 314 | def inc(self, i, token=None): 315 | print 'Increasing %s with one' % i 316 | res = i + 1 317 | # Emit sends messages to the actor outbox 318 | # The actor outbox is bound to the Counter inbox 319 | # Thus, the message is send to teh Counter 320 | # and its count message is invoked. 321 | self.actor.emit('count', {'res': res, 'token': token}) 322 | return res 323 | 324 | def inc(self, n): 325 | self.send('inc', {'n':n}, nowait=False) 326 | 327 | class Counter(Actor): 328 | class state: 329 | def __init__(self): 330 | self.targets = {} 331 | self.adder = None 332 | 333 | # Here we bind the outbox of Adder to the inbox of Counter. 334 | # All messages emitted to Adder are delegated to the Counter inbox. 335 | def on_agent_ready(self): 336 | ra = Adder(self.actor.connection) 337 | self.adder = self.actor.agent.add_actor(ra) 338 | self.adder |forward| self.actor 339 | 340 | def count(self, res, token): 341 | if res < target: 342 | self.adder.throw('add_one', {'i': res, 'token': token}) 343 | else: 344 | print 'Done with counting' 345 | 346 | def count_to(self, target): 347 | self.adder.throw('add_one', {'i': 0, 'token': token}) 348 | 349 | def on_agent_ready(self): 350 | self.state.on_agent_ready() 351 | 352 | The above example uses the outbox of an actor to send back the result. 353 | All operations are asynchronous. Note that as a result of asynchrony, the counting might not be in order. 354 | Different measures should be takes to preserve the order. For example, a token can be assigned to each request 355 | and used to order the results. 356 | 357 | Scheduling Actors in celery 358 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~ 359 | 360 | * when greenlets are disabled 361 | 362 | All messages are handled by the same thread and processed in order of delivery. 363 | Thus, it is up to the broker in what order the messages will be delivered and processed. 364 | If one actor blocks, the whole thread will be blocked. 365 | 366 | * when greenlets are enabled 367 | Each message is processed in a separate greenlet. 368 | If one greenlet/actor blocks, the execution is passed to the next greenlet and the 369 | (system) thread as a whole is not blocked. 370 | 371 | Receiving arbitrary messages 372 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 373 | An actor message has a name (label) and arguments. 374 | Each message name should have a corresponding method in the Actor's internal state class. 375 | Otherwise, an error code is returned as a result of the message call. 376 | However, if fire and forget called is used (call with nowait argument set to True), 377 | no error code will be returned. You can find the error expecting the worker log or the the worker console. 378 | 379 | If you want to implement your own pattern matching on messages and/or want to accept generic method names, 380 | you can override the :py:meth:`~.cell.actors.Actor.default_receive` method. 381 | 382 | Exceptions 383 | ~~~~~~~~~~ 384 | It can happen that while a message is being processed by an actor, that some kind of exception is thrown, e.g. a database exception. 385 | 386 | What happens to the Message 387 | --------------------------- 388 | 389 | If an exception is thrown while a message is being processed (i.e. taken out of its queue and handed over to the current behavior), 390 | then this message will be lost. It is important to understand that it is not put back on the queue. 391 | So if you want to retry processing of a message, you need to deal with it yourself by catching the exception and retry your flow. 392 | 393 | What happens to the actor 394 | ------------------------- 395 | Actor is not stopped, either restarted, it can continue receiving other messages. 396 | 397 | -------------------------------------------------------------------------------- /examples/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'rumi' 2 | -------------------------------------------------------------------------------- /examples/adder.py: -------------------------------------------------------------------------------- 1 | import celery 2 | from cell.actors import Actor 3 | from cell.agents import dAgent 4 | from kombu.utils import uuid 5 | from examples.workflow import forward 6 | 7 | my_app = celery.Celery(broker='pyamqp://guest@localhost//') 8 | agent = dAgent(connection=my_app.broker_connection()) 9 | 10 | 11 | class Adder(Actor): 12 | def __init__(self, connection=None, *args, **kwargs): 13 | super().__init__( 14 | connection or my_app.broker_connection(), *args, **kwargs) 15 | 16 | class state(): 17 | def add_one(self, i, token=None): 18 | print('Increasing %s with one' % i) 19 | res = i + 1 20 | self.actor.emit('count', {'res': res, 'token': token}) 21 | return res 22 | 23 | 24 | class Counter(Actor): 25 | def __init__(self, connection=None, *args, **kwargs): 26 | super().__init__( 27 | connection or my_app.broker_connection(), *args, **kwargs) 28 | 29 | class state(): 30 | def __init__(self): 31 | self.targets = {} 32 | self.adder = None 33 | 34 | def on_agent_ready(self): 35 | ra = Adder(self.actor.connection) 36 | self.adder = self.actor.agent.add_actor(ra) 37 | self.adder | forward | self.actor 38 | 39 | def count(self, res, token): 40 | (target, cur) = self.targets.get(token) 41 | if cur < res < target: 42 | self.adder.call('add_one', {'i': res, 'token': token}, 43 | nowait=True) 44 | elif res >= target: 45 | self.targets.pop(token) 46 | 47 | def count_to(self, target): 48 | token = uuid() 49 | init = 0 50 | self.targets[token] = (target, init) 51 | self.adder.throw('add_one', {'i': init, 'token': token}, 52 | nowait=True, callback='count', 53 | ckwargs={'token': token}) 54 | 55 | def on_agent_ready(self): 56 | self.state.on_agent_ready() 57 | 58 | 59 | class gCounter(Actor): 60 | def __init__(self, connection=None, *args, **kwargs): 61 | super().__init__( 62 | connection or my_app.broker_connection(), *args, **kwargs) 63 | 64 | class state(): 65 | def __init__(self): 66 | self.targets = {} 67 | self.adder = None 68 | 69 | def on_agent_ready(self): 70 | adder = Adder(self.actor.connection) 71 | self.adder = self.actor.agent.add_actor(adder) 72 | 73 | def count_to(self, target): 74 | i = 0 75 | while i <= target: 76 | i = self.adder.send('add_one', {'i': i, 'token': 0}) 77 | return i 78 | 79 | def on_agent_ready(self): 80 | self.state.on_agent_ready() 81 | 82 | 83 | if __name__ == '__main__': 84 | import examples.adder 85 | rb = agent.spawn(examples.adder.Counter.__class__) 86 | rb.call('count_to', {'target': 10}) 87 | -------------------------------------------------------------------------------- /examples/chat.py: -------------------------------------------------------------------------------- 1 | import celery 2 | from cell.actors import Actor 3 | from cell.agents import dAgent 4 | 5 | my_app = celery.Celery(broker='pyamqp://guest@localhost//') 6 | agent = dAgent(connection=my_app.broker_connection()) 7 | 8 | 9 | class User(Actor): 10 | def __init__(self, connection=None, *args, **kwargs): 11 | super().__init__( 12 | connection or my_app.broker_connection(), *args, **kwargs) 13 | 14 | class state(): 15 | 16 | def post(self, msg): 17 | print msg 18 | 19 | def connect(self): 20 | return agent.spawn(self.__class__) 21 | 22 | def connect(self, nickname): 23 | self.call('connect', {'name': nickname}) 24 | 25 | def post(self, msg): 26 | msg = 'Posting on the wall: %s' % msg 27 | self.scatter('post', {'msg': msg}) 28 | 29 | def message_to(self, actor, msg): 30 | a = User(id=actor, connection=self.connection) 31 | msg = 'Actor %s is sending you a message: %s' % (self.id, msg) 32 | a.call('post', {'msg': msg}) 33 | 34 | 35 | if __name__ == '__main__': 36 | import examples.chat 37 | rumi = examples.chat.User().connect() 38 | rumi.post('Hello everyone') 39 | 40 | ask = examples.chat.User().connect() 41 | ask.post('Hello everyone') 42 | rumi.message_to(ask.id, 'How are you?') 43 | ask.message_to(rumi.id, 'Fine.You?') 44 | -------------------------------------------------------------------------------- /examples/clex.py: -------------------------------------------------------------------------------- 1 | import cell 2 | 3 | from celery import current_app as celery 4 | 5 | 6 | class BlenderActor(cell.Actor): 7 | types = ('direct', 'round-robin') 8 | 9 | def __init__(self, connection=None, *args, **kwargs): 10 | # - use celery's connection by default, 11 | # - means the agent must be started from where it has access 12 | # - to the celery config. 13 | super().__init__( 14 | connection or celery.broker_connection(), *args, **kwargs) 15 | 16 | class state: 17 | # - The state class defines the messages the actor can handle, 18 | # - so a message with method 'render' will be dispatched to 19 | # - the render method. 20 | 21 | def render(self, blabla): 22 | print('communicates with the blender process here') 23 | return 'value' 24 | 25 | # - It is good practice to provide helper methods, so that 26 | # - clients don't have to use .call, etc directly 27 | 28 | def render(self, blabla, nowait=False): 29 | return self.throw('render', {'blabla': blabla}, nowait=nowait) 30 | 31 | 32 | blender = BlenderActor() 33 | 34 | 35 | class Agent(cell.Agent): 36 | actors = [blender] 37 | 38 | def __init__(self, connection=None, *args, **kwargs): 39 | # - use celery's connection by default 40 | super().__init__( 41 | connection or celery.broker_connection(), *args, **kwargs) 42 | 43 | 44 | if __name__ == '__main__': 45 | Agent().run_from_commandline() 46 | 47 | 48 | #### 49 | # Run this script in one console, and in another run the following: 50 | # 51 | # >>> from x import blender 52 | # 53 | # >>> # call and wait for result 54 | # >>> blender.render('foo').get() 55 | # 'value' 56 | # 57 | # >>> # send and don't care about result 58 | # >>> blender.render('foo', nowait=True) 59 | -------------------------------------------------------------------------------- /examples/distributed_cache.py: -------------------------------------------------------------------------------- 1 | from UserDict import DictMixin 2 | 3 | from cell import Actor, Agent 4 | from cell.utils import flatten 5 | 6 | 7 | def first_reply(replies, key): 8 | try: 9 | return replies.next() 10 | except StopIteration: 11 | raise KeyError(key) 12 | 13 | 14 | class Cache(Actor, DictMixin): 15 | types = ('scatter', 'round-robin') 16 | default_timeout = 1 17 | 18 | class state: 19 | 20 | def __init__(self, data=None): 21 | self.data = {} 22 | 23 | def get(self, key): 24 | if key not in self.data: 25 | # delegate to next agent. 26 | raise Actor.Next() 27 | return self.data[key] 28 | 29 | def delete(self, key): 30 | if key not in self.data: 31 | raise Actor.Next() 32 | return self.data.pop(key, None) 33 | 34 | def set(self, key, value): 35 | self.data[key] = value 36 | 37 | def keys(self): 38 | return self.data.keys() 39 | 40 | def __getitem__(self, key): 41 | return first_reply(self.scatter('get', {'key': key}), key) 42 | 43 | def __delitem__(self, key): 44 | return first_reply(self.scatter('delete', {'key': key}), key) 45 | 46 | def __setitem__(self, key, value): 47 | return self.throw('set', {'key': key, 'value': value}) 48 | 49 | def keys(self): 50 | return flatten(self.scatter('keys')) 51 | 52 | 53 | class CacheAgent(Agent): 54 | actors = [Cache()] 55 | 56 | 57 | if __name__ == '__main__': 58 | from kombu import Connection 59 | CacheAgent(Connection()).run_from_commandline() 60 | -------------------------------------------------------------------------------- /examples/flowlet.py: -------------------------------------------------------------------------------- 1 | import time 2 | from math import sqrt 3 | from cell.actors import Actor 4 | from cell.agents import dAgent 5 | from cell.results import AsyncResult 6 | from kombu.connection import Connection 7 | 8 | 9 | class Abs(Actor): 10 | class state: 11 | 12 | def calc(self, val): 13 | if isinstance(val, AsyncResult): 14 | val = val.get() 15 | new_val = abs(val) 16 | print('The abs result is:', new_val) 17 | return new_val 18 | 19 | 20 | class Square(Actor): 21 | class state: 22 | 23 | def calc(self, val): 24 | if isinstance(val, AsyncResult): 25 | val = val.get() 26 | new_val = val * val 27 | print('The square result is:', new_val) 28 | return new_val 29 | 30 | 31 | class SquareRoot(Actor): 32 | class state: 33 | 34 | def calc(self, val): 35 | if isinstance(val, AsyncResult): 36 | val = val.get() 37 | new_val = sqrt(val) 38 | print('The sqrt result is:', new_val) 39 | return new_val 40 | 41 | 42 | class Printer(Actor): 43 | def __init__(self, **kwargs): 44 | super(Printer, self).__init__(**kwargs) 45 | 46 | class state(object): 47 | def calc(self, val): 48 | if isinstance(val, AsyncResult): 49 | print 'receiving AsyncResult' 50 | val = val.get() 51 | print('The printer result is:', val) 52 | return val 53 | 54 | 55 | class Calculator: 56 | def __init__(self): 57 | self.agent = agent = dAgent(Connection()) 58 | self.abs = agent.spawn(Abs) 59 | self.sqrt = agent.spawn(SquareRoot) 60 | self.square = agent.spawn(Square) 61 | self.printer = agent.spawn(Printer) 62 | 63 | def run(self, val): 64 | start = time.time() 65 | val = self.abs.send.calc({'val': val}) 66 | val = self.sqrt.send.calc({'val': val}) 67 | self.printer.call.send({'val': val}) 68 | total = start - time.time() 69 | print('Finish in:', total) 70 | 71 | 72 | if __name__ == '__main__': 73 | import sys 74 | print sys.path 75 | calc = Calculator() 76 | calc.run(10) 77 | -------------------------------------------------------------------------------- /examples/hello.py: -------------------------------------------------------------------------------- 1 | from kombu.common import uuid 2 | from cell.actors import Actor, ActorProxy 3 | from cell.agents import dAgent 4 | from kombu.connection import Connection 5 | 6 | 7 | class GreetingActor(Actor): 8 | class state(Actor.state): 9 | def greet(self, who='world'): 10 | return 'Hello %s' % who 11 | 12 | 13 | class ByeActor(Actor): 14 | class state(GreetingActor.state): 15 | def bye(self, who='world'): 16 | print('Bye %s' % who) 17 | 18 | 19 | # Run from the command line: 20 | """ 21 | from kombu import Connection 22 | from examples.hello import GreetingActor 23 | from cell.agents import dAgent 24 | 25 | 26 | agent = dAgent(Connection()) 27 | greeter = agent.spawn(GreetingActor) 28 | greeter.call('greet') 29 | greeter = agent.select(GreetingActor) 30 | from examples.workflow import Printer 31 | id = agent.select(Printer) 32 | """ 33 | 34 | 35 | if __name__ == '__main__': 36 | """agent = dAgent(Connection()) 37 | actor = agent.spawn(GreetingActor) 38 | 39 | actor.send.greet({'who':'hello'}) 40 | actor.send.greet({'who':'hello'}) 41 | first_reply(actor.scatter.greet({'who':'hello'})) 42 | """ 43 | -------------------------------------------------------------------------------- /examples/map_reduce.py: -------------------------------------------------------------------------------- 1 | import celery 2 | from cell.actors import Actor 3 | from cell.agents import dAgent 4 | 5 | my_app = celery.Celery(broker='pyamqp://guest@localhost//') 6 | agent = dAgent(connection=my_app.broker_connection()) 7 | 8 | 9 | class Aggregator(Actor): 10 | 11 | def __init__(self, barrier=None, **kwargs): 12 | self.barrier = barrier 13 | super().__init__(**kwargs) 14 | 15 | class state(Actor.state): 16 | def __init__(self): 17 | self.result = {} 18 | super(Aggregator.state, self).__init__() 19 | 20 | def aggregate(self, words): 21 | for word, n in words.items(): 22 | self.result.setdefault(word, 0) 23 | self.result[word] += n 24 | 25 | self.actor.barrier -= 1 26 | if self.actor.barrier <= 0: 27 | self.print_result() 28 | 29 | def print_result(self): 30 | for (key, val) in self.result.items(): 31 | print("%s:%s" % (key, val)) 32 | 33 | 34 | class Reducer(Actor): 35 | 36 | class state(Actor.state): 37 | def __init__(self): 38 | self.aggregator = None 39 | super(Reducer.state, self).__init__() 40 | 41 | def on_agent_ready(self): 42 | self.aggregator = Aggregator(connection=self.actor.connection) 43 | 44 | def count_lines(self, line, aggregator): 45 | words = {} 46 | for word in line.split(" "): 47 | words.setdefault(word, 0) 48 | words[word] += 1 49 | self.aggregator.id = aggregator 50 | self.aggregator.call('aggregate', {'words': words}) 51 | 52 | def on_agent_ready(self): 53 | self.state.on_agent_ready() 54 | 55 | 56 | class Mapper(Actor): 57 | 58 | class state(Actor.state): 59 | REDUCERS = 10 60 | 61 | def on_agent_ready(self): 62 | self.pool = [] 63 | for i in range(self.REDUCERS): 64 | reducer = self.actor.agent.spawn(Reducer) 65 | self.pool.append(reducer) 66 | 67 | def count_document(self, file): 68 | with open(file) as f: 69 | lines = f.readlines() 70 | count = 0 71 | self.aggregator = agent.spawn(Aggregator, barrier=len(lines)) 72 | for line in lines: 73 | reducer = self.pool[count % self.REDUCERS] 74 | reducer.cast('count_lines', 75 | {'line': line, 76 | 'aggregator': self.aggregator.id}) 77 | 78 | def on_agent_ready(self): 79 | self.state.on_agent_ready() 80 | 81 | 82 | if __name__ == '__main__': 83 | import examples.map_reduce 84 | file = "/home/rumi/Dev/code/test/map_reduce.txt" 85 | mapper = agent.spawn(examples.map_reduce.Mapper) 86 | mapper.call('count_document', {'file': file}) 87 | -------------------------------------------------------------------------------- /examples/tasks.py: -------------------------------------------------------------------------------- 1 | from celery import Celery 2 | 3 | celery = Celery('tasks', broker='amqp://') 4 | 5 | 6 | @celery.task() 7 | def add(x, y): 8 | return x + y 9 | 10 | 11 | if __name__ == '__main__': 12 | celery.start() 13 | __author__ = 'rumi' 14 | -------------------------------------------------------------------------------- /examples/workflow.py: -------------------------------------------------------------------------------- 1 | import celery 2 | from cell.actors import Actor 3 | from cell.agents import dAgent 4 | from cell.utils.custom_operators import Infix 5 | import time 6 | from celery.utils.imports import instantiate 7 | 8 | my_app = celery.Celery(broker='pyamqp://guest@localhost//') 9 | 10 | # celery.Celery().control.broadcast('shutdown') 11 | #from examples.workflow import FilterExample 12 | #from examples.workflow import actors_mng 13 | #f = FilterExample() 14 | # f.start() 15 | #from examples.workflow import TestActor 16 | #t = TestActor() 17 | 18 | """ Simple scenario. 19 | We have a Filter that filter collections and we want every result 20 | to be send to Logger that do intensive computation on the filtered result 21 | and to a Printer that do diverse visualizations. 22 | The topology looks like that: 23 | Filter -> (Logger | Printer) 24 | """ 25 | 26 | 27 | class WorkflowActor(Actor): 28 | def __init__(self, connection=None, *args, **kwargs): 29 | super().__init__( 30 | connection or my_app.broker_connection(), *args, **kwargs) 31 | 32 | class state(Actor.state): 33 | pass 34 | 35 | def become_remote(self, actor): 36 | return self.add_actor(actor) 37 | 38 | 39 | class TrueFilter(WorkflowActor): 40 | 41 | class state(WorkflowActor.state): 42 | def filter(self, msg): 43 | print('Msg:%s received in filter' % (msg)) 44 | self.actor.emit('notify', {'msg': msg}) 45 | 46 | 47 | class FalseFilter(WorkflowActor): 48 | 49 | class state(WorkflowActor.state): 50 | def filter(self, msg): 51 | print('Msg:%s received in filter' % (msg)) 52 | self.actor.emit('notify', {'msg': msg}) 53 | 54 | 55 | class Joiner(WorkflowActor): 56 | def __init__(self, connection=None, *args, **kwargs): 57 | super().__init__( 58 | connection or my_app.broker_connection(), *args, **kwargs) 59 | 60 | def is_mutable(self): 61 | return False 62 | 63 | class state(WorkflowActor.state): 64 | def set_sources(self, sources): 65 | print('In set_source, Collector. Count limit is', len(sources)) 66 | self.count = 0 67 | self.sources = sources 68 | 69 | def notify(self, msg): 70 | print('In notify with count: %s and msg:%s' % (self.count, 71 | msg)) 72 | self.count += 1 73 | if self.count == len(self.sources): 74 | print 'I am sending the message to whoever is subscribed' 75 | self.actor.emit('set_ready', {'msg': 'ready'}) 76 | self.count = 0 77 | 78 | 79 | class GuardedActor(WorkflowActor): 80 | def __init__(self, connection=None, *args, **kwargs): 81 | self.ready = False 82 | super().__init__( 83 | connection or my_app.broker_connection(), *args, **kwargs) 84 | 85 | class state(WorkflowActor.state): 86 | def set_ready(self, msg): 87 | self.ready = True 88 | self.do_smth() 89 | 90 | def do_smth(self): 91 | print('I have finally received all messages.') 92 | 93 | 94 | class Printer(GuardedActor): 95 | types = ('scatter', 'round-robin', 'direct') 96 | 97 | def __init__(self, **kwargs): 98 | super().__init__(**kwargs) 99 | 100 | class state(GuardedActor.state): 101 | 102 | def do_smth(self): 103 | print('I am a printer') 104 | 105 | 106 | class Logger(GuardedActor): 107 | 108 | def default_receive(self, msg): 109 | print msg 110 | 111 | class state(GuardedActor.state): 112 | pass 113 | 114 | 115 | class Workflow(object): 116 | actors = [] 117 | 118 | def __init__(self, actors): 119 | self.actors_mng = dAgent( 120 | connection=my_app.broker_connection(), 121 | app=my_app) 122 | self.actors = actors 123 | 124 | def start(self): 125 | for actor in self.actors: 126 | yield self.actors_mng.spawn(actor) 127 | 128 | 129 | def join(outboxes, inbox): 130 | inbox.wait_to_start() 131 | inbox.call('set_sources', 132 | {'sources': [outbox.name for outbox in outboxes]}, 133 | nowait=False) 134 | print 'send set_sources to collector' 135 | for outbox in outboxes: 136 | inbox.add_binding(outbox.outbox, 137 | routing_key=outbox.routing_key, 138 | inbox_type='direct') 139 | 140 | 141 | def forward(source_actor, dest_actor): 142 | dest_actor.add_binding(source_actor.outbox, 143 | routing_key=source_actor.routing_key, 144 | inbox_type='scatter') 145 | 146 | 147 | def stop_forward(source_actor, dest_actor): 148 | dest_actor.remove_binding(source_actor.outbox, 149 | routing_key=source_actor.routing_key, 150 | inbox_type='direct') 151 | 152 | 153 | def multilplex(outbox, inboxes): 154 | for inbox in inboxes: 155 | inbox.wait_to_start() 156 | inbox.add_binding(outbox.outbox, 157 | routing_key=outbox.routing_key, 158 | inbox_type='direct') 159 | 160 | 161 | join = Infix(join) 162 | 163 | forward = Infix(forward) 164 | 165 | multiplex = Infix(multilplex) 166 | 167 | stop_forward = Infix(stop_forward) 168 | 169 | 170 | def start_group(actor_type, count): 171 | actor_group = [] 172 | [actor_group.append(instantiate(actor_type)) for _ in range(0, count)] 173 | wf = Workflow(actor_group) 174 | remote_group = list(wf.start()) 175 | return remote_group 176 | 177 | 178 | class FilterExample: 179 | def start(self): 180 | filter1, filter2, printer = TrueFilter(), FalseFilter(), Printer(), 181 | logger, collector = Logger(), Joiner() 182 | print('collector_id before start:' + collector.id) 183 | wf = Workflow([filter1, filter2, printer, logger, collector]) 184 | [filter1, filter2, printer, logger, collector] = list(wf.start()) 185 | print('collector_id after start:' + collector.id) 186 | 187 | time.sleep(2) 188 | 189 | [filter1, filter2] | join | collector 190 | collector | multiplex | [printer, logger] 191 | 192 | filter1.call('filter', {'msg': 'Ihu'}) 193 | filter2.call('filter', {'msg': 'Ahu'}) 194 | 195 | 196 | printer_name = 'examples.workflow.Printer' 197 | """actors_mng = ActorsManager(connection = my_app.broker_connection(), 198 | app = my_app) 199 | """ 200 | 201 | agent = dAgent(connection=my_app.broker_connection()) 202 | 203 | if __name__ == '__main__': 204 | printer = Printer() 205 | agent.spawn(printer) 206 | 207 | """Example usage: 208 | >>from examples.workflow import Printer, Logger, actors_mng 209 | Start 2 actors of type Printer remotely 210 | >>rpr1 = agent.spawn(Printer) 211 | >>rpr2 = agent.spawn(Printer) 212 | Use remote actor 213 | >>rpr.call('do_smth') 214 | >>rpr.scatter('do_smth') 215 | >>rpr.throw('do_smth') 216 | Start another actor 217 | >>rlog = agent.spawn(Log) 218 | Bind two actors together 219 | >>from examples.workflow import forward, stop_forward 220 | >>rlog |forward| rpr1 221 | Send to the output of ane actor and checks the binded actor receives is 222 | >>rlog.emit('do_smth') # here do_smth of rpt1 should be invoked 223 | Unbind actors 224 | >>rlog |stop_forward| rpr1 225 | Stop actors 226 | >>agent.stop_actor_by_id(rlog.id) 227 | >>agent.stop_actor_by_id(rpr1.id) 228 | >>agent.stop_actor_by_id(rpr2.id) 229 | """ 230 | -------------------------------------------------------------------------------- /extra/release/bump_version.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import absolute_import 3 | 4 | import errno 5 | import os 6 | import re 7 | import sys 8 | import subprocess 9 | 10 | from contextlib import contextmanager 11 | from tempfile import NamedTemporaryFile 12 | 13 | rq = lambda s: s.strip("\"'") 14 | str_t = str if sys.version_info[0] >= 3 else basestring 15 | 16 | 17 | def cmd(*args): 18 | return subprocess.Popen(args, stdout=subprocess.PIPE).communicate()[0] 19 | 20 | 21 | @contextmanager 22 | def no_enoent(): 23 | try: 24 | yield 25 | except OSError as exc: 26 | if exc.errno != errno.ENOENT: 27 | raise 28 | 29 | 30 | class StringVersion(object): 31 | 32 | def decode(self, s): 33 | s = rq(s) 34 | text = "" 35 | major, minor, release = s.split(".") 36 | if not release.isdigit(): 37 | pos = release.index(re.split("\d+", release)[1][0]) 38 | release, text = release[:pos], release[pos:] 39 | return int(major), int(minor), int(release), text 40 | 41 | def encode(self, v): 42 | return ".".join(map(str, v[:3])) + v[3] 43 | to_str = StringVersion().encode 44 | from_str = StringVersion().decode 45 | 46 | 47 | class TupleVersion(object): 48 | 49 | def decode(self, s): 50 | v = list(map(rq, s.split(", "))) 51 | return (tuple(map(int, v[0:3])) + 52 | tuple(["".join(v[3:])])) 53 | 54 | def encode(self, v): 55 | v = list(v) 56 | 57 | def quote(lit): 58 | if isinstance(lit, str_t): 59 | return '"%s"' % (lit, ) 60 | return str(lit) 61 | 62 | if not v[-1]: 63 | v.pop() 64 | return ", ".join(map(quote, v)) 65 | 66 | 67 | class VersionFile(object): 68 | 69 | def __init__(self, filename): 70 | self.filename = filename 71 | self._kept = None 72 | 73 | def _as_orig(self, version): 74 | return self.wb % {"version": self.type.encode(version), 75 | "kept": self._kept} 76 | 77 | def write(self, version): 78 | pattern = self.regex 79 | with no_enoent(): 80 | with NamedTemporaryFile() as dest: 81 | with open(self.filename) as orig: 82 | for line in orig: 83 | if pattern.match(line): 84 | dest.write(self._as_orig(version)) 85 | else: 86 | dest.write(line) 87 | os.rename(dest.name, self.filename) 88 | 89 | def parse(self): 90 | pattern = self.regex 91 | gpos = 0 92 | with open(self.filename) as fh: 93 | for line in fh: 94 | m = pattern.match(line) 95 | if m: 96 | if "?P" in pattern.pattern: 97 | self._kept, gpos = m.groupdict()["keep"], 1 98 | return self.type.decode(m.groups()[gpos]) 99 | 100 | 101 | class PyVersion(VersionFile): 102 | regex = re.compile(r'^VERSION\s*=\s*\((.+?)\)') 103 | wb = "VERSION = (%(version)s)\n" 104 | type = TupleVersion() 105 | 106 | 107 | class SphinxVersion(VersionFile): 108 | regex = re.compile(r'^:[Vv]ersion:\s*(.+?)$') 109 | wb = ':Version: %(version)s\n' 110 | type = StringVersion() 111 | 112 | 113 | class CPPVersion(VersionFile): 114 | regex = re.compile(r'^\#\s*define\s*(?P\w*)VERSION\s+(.+)') 115 | wb = '#define %(kept)sVERSION "%(version)s"\n' 116 | type = StringVersion() 117 | 118 | 119 | _filetype_to_type = {"py": PyVersion, 120 | "rst": SphinxVersion, 121 | "c": CPPVersion, 122 | "h": CPPVersion} 123 | 124 | 125 | def filetype_to_type(filename): 126 | _, _, suffix = filename.rpartition(".") 127 | return _filetype_to_type[suffix](filename) 128 | 129 | 130 | def bump(*files, **kwargs): 131 | version = kwargs.get("version") 132 | files = [filetype_to_type(f) for f in files] 133 | versions = [v.parse() for v in files] 134 | current = list(reversed(sorted(versions)))[0] # find highest 135 | 136 | if version: 137 | next = from_str(version) 138 | else: 139 | major, minor, release, text = current 140 | if text: 141 | raise Exception("Can't bump alpha releases") 142 | next = (major, minor, release + 1, text) 143 | 144 | print("Bump version from %s -> %s" % (to_str(current), to_str(next))) 145 | 146 | for v in files: 147 | print(" writing %r..." % (v.filename, )) 148 | v.write(next) 149 | 150 | print(cmd("git", "commit", "-m", "Bumps version to %s" % (to_str(next), ), 151 | *[f.filename for f in files])) 152 | print(cmd("git", "tag", "v%s" % (to_str(next), ))) 153 | 154 | 155 | def main(argv=sys.argv, version=None): 156 | if not len(argv) > 1: 157 | print("Usage: distdir [docfile] -- ") 158 | sys.exit(0) 159 | if "--" in argv: 160 | c = argv.index('--') 161 | version = argv[c + 1] 162 | argv = argv[:c] 163 | bump(*argv[1:], version=version) 164 | 165 | if __name__ == "__main__": 166 | main() 167 | -------------------------------------------------------------------------------- /extra/release/doc4allmods: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | PACKAGE="$1" 4 | SKIP_PACKAGES="$PACKAGE tests" 5 | SKIP_FILES="cell.bin.rst" 6 | 7 | modules=$(find "$PACKAGE" -name "*.py") 8 | 9 | failed=0 10 | for module in $modules; do 11 | dotted=$(echo $module | sed 's/\//\./g') 12 | name=${dotted%.__init__.py} 13 | name=${name%.py} 14 | rst=$name.rst 15 | skip=0 16 | for skip_package in $SKIP_PACKAGES; do 17 | [ $(echo "$name" | cut -d. -f 2) == "$skip_package" ] && skip=1 18 | done 19 | for skip_file in $SKIP_FILES; do 20 | [ "$skip_file" == "$rst" ] && skip=1 21 | done 22 | 23 | if [ $skip -eq 0 ]; then 24 | if [ ! -f "docs/reference/$rst" ]; then 25 | echo $rst :: FAIL 26 | failed=1 27 | fi 28 | fi 29 | done 30 | 31 | exit $failed 32 | -------------------------------------------------------------------------------- /extra/release/flakeplus.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import absolute_import 3 | from __future__ import with_statement 4 | 5 | import os 6 | import re 7 | import sys 8 | 9 | from collections import defaultdict 10 | from unipath import Path 11 | 12 | RE_COMMENT = r'^\s*\#' 13 | RE_NOQA = r'.+?\#\s+noqa+' 14 | RE_MULTILINE_COMMENT_O = r'^\s*(?:\'\'\'|""").+?(?:\'\'\'|""")' 15 | RE_MULTILINE_COMMENT_S = r'^\s*(?:\'\'\'|""")' 16 | RE_MULTILINE_COMMENT_E = r'(?:^|.+?)(?:\'\'\'|""")' 17 | RE_WITH = r'(?:^|\s+)with\s+' 18 | RE_WITH_IMPORT = r'''from\s+ __future__\s+ import\s+ with_statement''' 19 | RE_PRINT = r'''(?:^|\s+)print\((?:"|')(?:\W+?)?[A-Z0-9:]{2,}''' 20 | RE_ABS_IMPORT = r'''from\s+ __future__\s+ import\s+ absolute_import''' 21 | 22 | acc = defaultdict(lambda: {"abs": False, "print": False}) 23 | 24 | 25 | def compile(regex): 26 | return re.compile(regex, re.VERBOSE) 27 | 28 | 29 | class FlakePP(object): 30 | re_comment = compile(RE_COMMENT) 31 | re_ml_comment_o = compile(RE_MULTILINE_COMMENT_O) 32 | re_ml_comment_s = compile(RE_MULTILINE_COMMENT_S) 33 | re_ml_comment_e = compile(RE_MULTILINE_COMMENT_E) 34 | re_abs_import = compile(RE_ABS_IMPORT) 35 | re_print = compile(RE_PRINT) 36 | re_with_import = compile(RE_WITH_IMPORT) 37 | re_with = compile(RE_WITH) 38 | re_noqa = compile(RE_NOQA) 39 | map = {"abs": False, "print": False, 40 | "with": False, "with-used": False} 41 | 42 | def __init__(self, verbose=False): 43 | self.verbose = verbose 44 | self.steps = (("abs", self.re_abs_import), 45 | ("with", self.re_with_import), 46 | ("with-used", self.re_with), 47 | ("print", self.re_print)) 48 | 49 | def analyze_fh(self, fh): 50 | steps = self.steps 51 | filename = fh.name 52 | acc = dict(self.map) 53 | index = 0 54 | errors = [0] 55 | 56 | def error(fmt, **kwargs): 57 | errors[0] += 1 58 | self.announce(fmt, **dict(kwargs, filename=filename)) 59 | 60 | for index, line in enumerate(self.strip_comments(fh)): 61 | for key, pattern in steps: 62 | if pattern.match(line): 63 | acc[key] = True 64 | if index: 65 | if not acc["abs"]: 66 | error("%(filename)s: missing abs import") 67 | if acc["with-used"] and not acc["with"]: 68 | error("%(filename)s: missing with import") 69 | if acc["print"]: 70 | error("%(filename)s: left over print statement") 71 | 72 | return filename, errors[0], acc 73 | 74 | def analyze_file(self, filename): 75 | with open(filename) as fh: 76 | return self.analyze_fh(fh) 77 | 78 | def analyze_tree(self, dir): 79 | for dirpath, _, filenames in os.walk(dir): 80 | for path in (Path(dirpath, f) for f in filenames): 81 | if path.endswith(".py"): 82 | yield self.analyze_file(path) 83 | 84 | def analyze(self, *paths): 85 | for path in map(Path, paths): 86 | if path.isdir(): 87 | for res in self.analyze_tree(path): 88 | yield res 89 | else: 90 | yield self.analyze_file(path) 91 | 92 | def strip_comments(self, fh): 93 | re_comment = self.re_comment 94 | re_ml_comment_o = self.re_ml_comment_o 95 | re_ml_comment_s = self.re_ml_comment_s 96 | re_ml_comment_e = self.re_ml_comment_e 97 | re_noqa = self.re_noqa 98 | in_ml = False 99 | 100 | for line in fh.readlines(): 101 | if in_ml: 102 | if re_ml_comment_e.match(line): 103 | in_ml = False 104 | else: 105 | if re_noqa.match(line) or re_ml_comment_o.match(line): 106 | pass 107 | elif re_ml_comment_s.match(line): 108 | in_ml = True 109 | elif re_comment.match(line): 110 | pass 111 | else: 112 | yield line 113 | 114 | def announce(self, fmt, **kwargs): 115 | sys.stderr.write((fmt + "\n") % kwargs) 116 | 117 | 118 | def main(argv=sys.argv, exitcode=0): 119 | for _, errors, _ in FlakePP(verbose=True).analyze(*argv[1:]): 120 | if errors: 121 | exitcode = 1 122 | return exitcode 123 | 124 | 125 | if __name__ == "__main__": 126 | sys.exit(main()) 127 | -------------------------------------------------------------------------------- /extra/release/prepy3ktest: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | (cd "${1:-.}/.tox/py32/src/kombu"; 3 | 2to3 --no-diff -w --nobackups kombu) 4 | -------------------------------------------------------------------------------- /extra/release/py3k-run-tests: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | base=${1:-.} 3 | nosetests -vd cell.tests \ 4 | --with-coverage3 \ 5 | --cover3-branch \ 6 | --cover3-xml \ 7 | --cover3-xml-file="$base/coverage.xml" \ 8 | --cover3-html \ 9 | --cover3-html-dir="$base/cover" \ 10 | --cover3-package=cell \ 11 | --cover3-exclude=" \ 12 | cell \ 13 | cell.tests.* \ 14 | --with-xunit \ 15 | --xunit-file="$base/nosetests.xml" 16 | -------------------------------------------------------------------------------- /extra/release/removepyc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | (cd "${1:-.}"; 3 | find . -name "*.pyc" | xargs rm -- 2>/dev/null) || echo "ok" 4 | -------------------------------------------------------------------------------- /extra/release/verify-reference-index.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | verify_index() { 4 | modules=$(grep "cell." "$1" | \ 5 | perl -ple's/^\s*|\s*$//g;s{\.}{/}g;') 6 | retval=0 7 | for module in $modules; do 8 | if [ ! -f "$module.py" ]; then 9 | if [ ! -f "$module/__init__.py" ]; then 10 | echo "Outdated reference: $module" 11 | retval=1 12 | fi 13 | fi 14 | done 15 | 16 | return $retval 17 | } 18 | 19 | verify_index docs/reference/index.rst 20 | -------------------------------------------------------------------------------- /requirements/default.txt: -------------------------------------------------------------------------------- 1 | kombu>=4.2 2 | -------------------------------------------------------------------------------- /requirements/docs.txt: -------------------------------------------------------------------------------- 1 | Sphinx 2 | sphinxcontrib-issuetracker>=0.9 -------------------------------------------------------------------------------- /requirements/pkgutils.txt: -------------------------------------------------------------------------------- 1 | flake8 2 | -------------------------------------------------------------------------------- /requirements/test.txt: -------------------------------------------------------------------------------- 1 | mock 2 | nose 3 | nose-cover3 4 | unittest2>=0.5.0 5 | coverage>=3.0 6 | simplejson 7 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [nosetests] 2 | verbosity = 1 3 | detailed-errors = 1 4 | where = cell/tests 5 | cover3-branch = 1 6 | cover3-html = 1 7 | cover3-package = cell 8 | cover3-exclude = cell 9 | 10 | [build_sphinx] 11 | source-dir = docs/ 12 | build-dir = docs/.build 13 | all_files = 1 14 | 15 | [upload_sphinx] 16 | upload-dir = docs/.build/html 17 | 18 | 19 | [bdist_rpm] 20 | requires = kombu >= 4.2 21 | 22 | [wheel] 23 | universal = 1 24 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | import os 4 | import sys 5 | import codecs 6 | 7 | extra = {} 8 | tests_require = ['nose', 'nose-cover3'] 9 | 10 | if sys.version_info < (2, 7): 11 | raise Exception('cell requires Python 2.7 or higher.') 12 | 13 | try: 14 | from setuptools import setup 15 | except ImportError: 16 | from distutils.core import setup # noqa 17 | 18 | from distutils.command.install import INSTALL_SCHEMES 19 | 20 | # -- Parse meta 21 | import re 22 | re_meta = re.compile(r'__(\w+?)__\s*=\s*(.*)') 23 | re_vers = re.compile(r'VERSION\s*=\s*\((.*?)\)') 24 | re_doc = re.compile(r'^"""(.+?)"""') 25 | rq = lambda s: s.strip("\"'") 26 | 27 | def add_default(m): 28 | attr_name, attr_value = m.groups() 29 | return ((attr_name, rq(attr_value)), ) 30 | 31 | 32 | def add_version(m): 33 | v = list(map(rq, m.groups()[0].split(', '))) 34 | return (('VERSION', '.'.join(v[0:3]) + ''.join(v[3:])), ) 35 | 36 | 37 | def add_doc(m): 38 | return (('doc', m.groups()[0]), ) 39 | 40 | pats = {re_meta: add_default, 41 | re_vers: add_version, 42 | re_doc: add_doc} 43 | here = os.path.abspath(os.path.dirname(__file__)) 44 | meta_fh = open(os.path.join(here, 'cell/__init__.py')) 45 | try: 46 | meta = {} 47 | for line in meta_fh: 48 | if line.strip() == '# -eof meta-': 49 | break 50 | for pattern, handler in pats.items(): 51 | m = pattern.match(line.strip()) 52 | if m: 53 | meta.update(handler(m)) 54 | finally: 55 | meta_fh.close() 56 | # -- 57 | 58 | packages, data_files = [], [] 59 | root_dir = os.path.dirname(__file__) 60 | if root_dir != '': 61 | os.chdir(root_dir) 62 | src_dir = 'cell' 63 | 64 | 65 | def fullsplit(path, result=None): 66 | if result is None: 67 | result = [] 68 | head, tail = os.path.split(path) 69 | if head == '': 70 | return [tail] + result 71 | if head == path: 72 | return result 73 | return fullsplit(head, [tail] + result) 74 | 75 | 76 | for scheme in list(INSTALL_SCHEMES.values()): 77 | scheme['data'] = scheme['purelib'] 78 | 79 | for dirpath, dirnames, filenames in os.walk(src_dir): 80 | # Ignore dirnames that start with '.' 81 | for i, dirname in enumerate(dirnames): 82 | if dirname.startswith('.'): 83 | del dirnames[i] 84 | for filename in filenames: 85 | if filename.endswith('.py'): 86 | packages.append('.'.join(fullsplit(dirpath))) 87 | else: 88 | data_files.append([dirpath, [os.path.join(dirpath, f) for f in 89 | filenames]]) 90 | 91 | if os.path.exists('README.rst'): 92 | long_description = codecs.open('README.rst', 'r', 'utf-8').read() 93 | else: 94 | long_description = 'See http://pypi.python.org/pypi/cell' 95 | 96 | install_requires = ['kombu>=4.2'] 97 | setup( 98 | name='cell', 99 | version=meta['VERSION'], 100 | description=meta['doc'], 101 | author=meta['author'], 102 | author_email=meta['contact'], 103 | url=meta['homepage'], 104 | platforms=['any'], 105 | packages=packages, 106 | data_files=data_files, 107 | zip_safe=False, 108 | test_suite='nose.collector', 109 | install_requires=install_requires, 110 | tests_require=tests_require, 111 | classifiers=[ 112 | 'Development Status :: 3 - Alpha', 113 | 'License :: OSI Approved :: BSD License', 114 | 'Operating System :: OS Independent', 115 | 'Programming Language :: Python', 116 | 'Programming Language :: Python :: 3', 117 | 'Programming Language :: Python :: 3.7', 118 | 'Programming Language :: Python :: 3.6', 119 | 'Programming Language :: Python :: 3.5', 120 | 'Intended Audience :: Developers', 121 | 'Topic :: Communications', 122 | 'Topic :: System :: Distributed Computing', 123 | 'Topic :: System :: Networking', 124 | 'Topic :: Software Development :: Libraries :: Python Modules', 125 | ], 126 | entry_points={ 127 | 'console_scripts': ['cell = cell.bin.cell:main'], 128 | }, 129 | long_description=long_description, 130 | **extra) 131 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py36,py37 3 | 4 | [testenv] 5 | distribute = True 6 | sitepackages = False 7 | commands = nosetests 8 | 9 | [testenv:py36] 10 | basepython = python3.6 11 | commands = pip -E {envdir} install -r requirements/default.txt 12 | pip -E {envdir} install -r requirements/test.txt 13 | nosetests --with-xunit --xunit-file=nosetests.xml --with-coverage3 14 | 15 | [testenv:py37] 16 | basepython = python3.7 17 | commands = pip -E {envdir} install -r requirements/default.txt 18 | pip -E {envdir} install -r requirements/test.txt 19 | nosetests --with-xunit --xunit-file=nosetests.xml --with-coverage3 20 | --------------------------------------------------------------------------------