├── .coveragerc ├── .gitignore ├── CHANGELOG.rst ├── CONTRIBUTING.rst ├── LICENSE.txt ├── README.rst ├── botoflow ├── __init__.py ├── activity_retrying.py ├── constants.py ├── context │ ├── __init__.py │ ├── activity_context.py │ ├── context_base.py │ ├── decision_context.py │ └── start_workflow_context.py ├── core │ ├── __init__.py │ ├── async_context.py │ ├── async_event_loop.py │ ├── async_root_task_context.py │ ├── async_task.py │ ├── async_task_context.py │ ├── async_traceback.py │ ├── base_future.py │ ├── decorators.py │ ├── exceptions.py │ ├── future.py │ └── utils.py ├── data_converter │ ├── __init__.py │ ├── abstract_data_converter.py │ ├── json_data_converter.py │ └── pickle_data_converter.py ├── decider │ ├── __init__.py │ ├── activity_future.py │ ├── activity_task_handler.py │ ├── child_workflow_execution_handler.py │ ├── decider.py │ ├── decision_task_poller.py │ ├── external_workflow_handler.py │ ├── timer_handler.py │ ├── workflow_execution_handler.py │ └── workflow_replayer.py ├── decisions │ ├── __init__.py │ ├── decision_bases.py │ ├── decision_list.py │ └── decisions.py ├── decorator_descriptors.py ├── decorators.py ├── exceptions.py ├── flow_types │ ├── __init__.py │ ├── activity_type.py │ ├── base_flow_type.py │ ├── signal_type.py │ └── workflow_type.py ├── history_events │ ├── __init__.py │ ├── event_bases.py │ └── events.py ├── logging_filters.py ├── manual_activity_completion_client.py ├── options.py ├── swf_exceptions.py ├── test │ ├── __init__.py │ └── workflow_testing_context.py ├── utils.py ├── workers │ ├── __init__.py │ ├── activity_task.py │ ├── activity_worker.py │ ├── base_worker.py │ ├── multiprocessing_activity_executor.py │ ├── multiprocessing_executor.py │ ├── multiprocessing_workflow_executor.py │ ├── swf_op_callable.py │ ├── threaded_activity_executor.py │ ├── threaded_executor.py │ ├── threaded_workflow_executor.py │ └── workflow_worker.py ├── workflow_definition.py ├── workflow_execution.py ├── workflow_starting.py └── workflow_time.py ├── circle.yml ├── codecov.yml ├── docs ├── Makefile └── source │ ├── _static │ ├── botoflow_favico.png │ ├── botoflow_logo.svg │ ├── li_bullet.gif │ ├── style.css │ └── ul_bullet.png │ ├── _templates │ ├── layout.html │ └── logo-text.html │ ├── changelog.rst │ ├── concepts.rst │ ├── conf.py │ ├── feature_details.rst │ ├── index.rst │ ├── internal_reference │ ├── activity_retrying.rst │ ├── core.rst │ ├── decider.rst │ ├── decisions.rst │ ├── decorator_descriptors.rst │ ├── flow_types.rst │ ├── history_events.rst │ ├── index.rst │ ├── utils.rst │ └── workers_internal.rst │ ├── overview.rst │ ├── reference │ ├── constants.rst │ ├── context.rst │ ├── data_converter.rst │ ├── decorators.rst │ ├── exceptions.rst │ ├── index.rst │ ├── logging_filters.rst │ ├── options.rst │ ├── workers.rst │ ├── workflow_definition.rst │ ├── workflow_execution.rst │ ├── workflow_starter.rst │ └── workflow_time.rst │ └── setting_up_dev_env.rst ├── examples └── helloworld │ ├── README.rst │ ├── helloworld.py │ └── sample.config.ini ├── pytest.ini ├── requirements-docs.txt ├── requirements.txt ├── scripts ├── ci │ ├── fix-coverage-path │ └── install └── new-change ├── setup.cfg ├── setup.py ├── test ├── integration │ ├── conftest.py │ ├── multiprocessing_workflows.py │ ├── test_cancellation.py │ ├── test_child_workflows.py │ ├── test_generic_workflows.py │ ├── test_manual_activities.py │ ├── test_multi_workflow.py │ ├── test_multiprocessing_workers.py │ ├── test_retrying_workflows.py │ ├── test_signaling_workflows.py │ ├── test_simple_workflows.py │ ├── utils.py │ └── various_activities.py └── unit │ ├── core │ ├── conftest.py │ ├── test_async_event_loop.py │ ├── test_async_task.py │ ├── test_async_traceback.py │ ├── test_base_future.py │ └── test_future.py │ ├── data_converter │ └── test_json_data_converter.py │ ├── decider │ ├── test_activity_future.py │ ├── test_activity_task_handler.py │ ├── test_decider.py │ └── test_workflow_execution_handler.py │ ├── decisions │ ├── test_decision_list.py │ └── test_decisions.py │ ├── test │ └── test_simple_workflow_testing.py │ ├── test_decorators.py │ ├── test_options.py │ ├── test_swf_exceptions.py │ ├── test_utils.py │ ├── test_workflow_definition.py │ └── test_workflow_time.py └── tox.ini /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | omit = 4 | botoflow/* 5 | 6 | [report] 7 | exclude_lines = 8 | raise NotImplementedError.* 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.py[co] 3 | 4 | # Packages 5 | *.egg 6 | *.egg-info 7 | dist 8 | build 9 | eggs 10 | parts 11 | var 12 | sdist 13 | develop-eggs 14 | .installed.cfg 15 | 16 | # Installer logs 17 | pip-log.txt 18 | 19 | # Unit test / coverage reports 20 | .coverage 21 | .tox 22 | 23 | #Translations 24 | *.mo 25 | 26 | #Mr Developer 27 | .mr.developer.cfg 28 | 29 | # Emacs backup files 30 | *~ 31 | 32 | # IDEA / PyCharm IDE 33 | .idea/ 34 | 35 | # Virtualenvs 36 | env 37 | env2 38 | env3 39 | 40 | # Emacs 41 | *~ 42 | \#*\# 43 | .\#* 44 | 45 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | Contributing Code 2 | ----------------- 3 | A good pull request: 4 | 5 | - Is clear. 6 | - Works across all supported versions of Python (2.7, 3.4+). 7 | - Follows the existing style of the code base (PEP-8). 8 | - Has comments included as needed. 9 | 10 | - A test case that demonstrates the previous flaw that now passes with 11 | the included patch, or demonstrates the newly added feature. 12 | - If it adds/changes a public API, it must also include documentation 13 | for those changes. 14 | - Must be appropriately licensed (Apache 2.0). 15 | 16 | Reporting An Issue/Feature 17 | -------------------------- 18 | First, check to see if there's an existing issue/pull request for the 19 | bug/feature. All issues are at 20 | https://github.com/boto/botoflow/issues and pull requests are at 21 | https://github.com/boto/botoflow/pulls. 22 | 23 | If there isn't an existing issue there, please file an issue. The 24 | ideal report includes: 25 | 26 | - A description of the problem/suggestion. 27 | - How to recreate the bug. 28 | - If relevant, including the versions of your: 29 | 30 | - Python interpreter 31 | - Botoflow 32 | - Optionally of the other dependencies involved (e.g. Botocore, Dill, etc.) 33 | 34 | - If possible, create a pull request with a (failing) test case 35 | demonstrating what's wrong. This makes the process for fixing bugs 36 | quicker & gets issues resolved sooner. 37 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ================================================ 2 | Botoflow - Asynchronous Framework for Amazon SWF 3 | ================================================ 4 | 5 | |Version| |Documentation| |Build Status| |Coverage| 6 | 7 | Botoflow is an asynchronous framework for `Amazon SWF`_ that helps you 8 | build SWF applications using Python. You can find the latest, most 9 | up to date, documentation at `Read the Docs`_ including the "Getting Started Guide". 10 | 11 | Under the hood it uses `botocore`_ low level interface to interact with `Amazon SWF`_. 12 | 13 | .. _`botocore`: https://github.com/boto/botocore 14 | .. _`Read the Docs`: https://botoflow.readthedocs.io/en/latest/ 15 | .. _`Amazon SWF`: https://aws.amazon.com/swf/ 16 | .. |Version| image:: https://img.shields.io/pypi/v/botoflow.svg 17 | :target: https://pypi.python.org/pypi/botoflow 18 | :alt: Version 19 | .. |Documentation| image:: https://readthedocs.org/projects/botoflow/badge/?version=latest 20 | :target: https://botoflow.readthedocs.io 21 | :alt: Documentation 22 | .. |Build Status| image:: https://img.shields.io/circleci/project/boto/botoflow.svg 23 | :target: https://circleci.com/gh/boto/botoflow 24 | :alt: Build Status 25 | .. |Coverage| image:: https://img.shields.io/codecov/c/github/boto/botoflow.svg?maxAge=2592000 26 | :target: https://codecov.io/gh/boto/botoflow 27 | :alt: Coverage 28 | 29 | Issue tracker 30 | ------------- 31 | 32 | Please report any bugs or enhancement ideas using our issue tracker: 33 | https://github.com/boto/botoflow/issues . Also, feel free to ask any 34 | other project related questions there. 35 | 36 | 37 | Development 38 | ----------- 39 | 40 | Getting Started 41 | ~~~~~~~~~~~~~~~ 42 | Assuming that you have Python and ``virtualenv`` installed, set up your 43 | environment and install the required dependencies: 44 | 45 | .. code-block:: sh 46 | 47 | $ git clone https://github.com/boto/botoflow.git 48 | $ cd botoflow 49 | $ virtualenv venv 50 | ... 51 | $ . venv/bin/activate 52 | $ pip install -r requirements.txt 53 | $ pip install -e . 54 | 55 | Running Tests 56 | ~~~~~~~~~~~~~ 57 | You can run tests in all supported Python versions using ``tox``. By default, 58 | it will run all of the unit tests, but you can also specify your own 59 | ``pytest`` options. Note that this requires that you have all supported 60 | versions of Python installed, otherwise you must pass ``-e`` or run the 61 | ``pytest`` command directly: 62 | 63 | .. code-block:: sh 64 | 65 | $ tox 66 | $ tox test/unit/test_workflow_time.py 67 | $ tox -e py27,py34 test/integration 68 | 69 | You can also run individual tests with your default Python version: 70 | 71 | .. code-block:: sh 72 | 73 | $ py.test -v test/unit 74 | 75 | Generating Documentation 76 | ~~~~~~~~~~~~~~~~~~~~~~~~ 77 | Sphinx is used for documentation. You can generate HTML locally with the 78 | following: 79 | 80 | .. code-block:: sh 81 | 82 | $ pip install -r requirements-docs.txt 83 | $ cd docs 84 | $ make html 85 | -------------------------------------------------------------------------------- /botoflow/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | from .core import coroutine, Return, return_, Future 15 | from .context import get_context, set_context 16 | from .workers import (GenericWorkflowWorker, WorkflowWorker, ActivityWorker, ThreadedWorkflowExecutor, 17 | ThreadedActivityExecutor, MultiprocessingWorkflowExecutor, MultiprocessingActivityExecutor) 18 | from .decorators import workflow, execute, activity, manual_activity, activities, signal, retry_activity 19 | from .activity_retrying import retry_on_exception 20 | from .options import workflow_options, activity_options 21 | from .workflow_definition import WorkflowDefinition 22 | from .workflow_starting import workflow_starter 23 | from .flow_types import ActivityType, SignalType, WorkflowType 24 | 25 | __version__ = '0.8' 26 | -------------------------------------------------------------------------------- /botoflow/constants.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | """This module contains various workflow, activity and time constants. 15 | 16 | Tasklist settings 17 | +++++++++++++++++ 18 | 19 | .. py:data:: USE_WORKER_TASK_LIST 20 | 21 | Use task list of the ActivityWorker or WorkflowWorker that is used to register activity or workflow as the default 22 | task list for the activity or workflow type. 23 | 24 | .. py:data:: NO_DEFAULT_TASK_LIST 25 | 26 | Do not specify task list on registration. Which means that task list is required when scheduling activity. 27 | 28 | 29 | Child workflow termination policy settings 30 | ++++++++++++++++++++++++++++++++++++++++++ 31 | 32 | You can learn more about *Child Workflows* from the official 33 | `SWF Developer Guide `_. 34 | 35 | .. py:data:: CHILD_TERMINATE 36 | 37 | The child executions will be terminated. 38 | 39 | .. py:data:: CHILD_REQUEST_CANCEL 40 | 41 | Request to cancel will be attempted for each child execution by recording a 42 | :py:class:`~botoflow.history_events.events.WorkflowExecutionCancelRequested` event in its history. It is up to the 43 | decider to take appropriate actions when it receives an execution history with this event. 44 | 45 | .. py:data:: CHILD_ABANDON 46 | 47 | Child policy to abandon the parent workflow. If there are any child workflows still running the will be allowed 48 | to continue without notice. 49 | 50 | 51 | Time multipliers 52 | ++++++++++++++++ 53 | 54 | The time multiplier constants are just an attempt at making setting various workflow or activity timeouts more readable. 55 | 56 | Consider the following examples and their readability: 57 | 58 | .. code-block:: python 59 | 60 | @activities(schedule_to_start_timeout=120, 61 | start_to_close_timeout=23400) 62 | class ImportantBusinessActivities(object): ... 63 | 64 | # using the time multiplier constants 65 | from botoflow.constants import MINUTES, HOURS 66 | 67 | @activities(schedule_to_start_timeout=2*MINUTES, 68 | start_to_close_timeout=30*MINUTES + 6*HOURS) 69 | class ImportantBusinessActivities(object): ... 70 | 71 | 72 | .. py:data:: SECONDS 73 | 74 | ``2*SECONDS = 2`` 75 | 76 | .. py:data:: MINUTES 77 | 78 | ``2*MINUTES = 120`` 79 | 80 | 81 | .. py:data:: HOURS 82 | 83 | ``2*HOURS = 7200`` 84 | 85 | 86 | .. py:data:: DAYS 87 | 88 | ``2*DAYS = 172800`` 89 | 90 | 91 | .. py:data:: WEEKS 92 | 93 | ``2*WEEKS = 1209600`` 94 | 95 | """ 96 | 97 | # TASK LIST SETTINGS 98 | USE_WORKER_TASK_LIST = "USE_WORKER_TASK_LIST" 99 | NO_DEFAULT_TASK_LIST = "NO_DEFAULT_TASK_LIST" 100 | 101 | # CHILD POLICIES 102 | CHILD_TERMINATE = "TERMINATE" 103 | CHILD_REQUEST_CANCEL = "REQUEST_CANCEL" 104 | CHILD_ABANDON = "ABANDON" 105 | 106 | # TIME MULTIPLIERS 107 | SECONDS = 1 108 | MINUTES = 60 109 | HOURS = 3600 110 | DAYS = 86400 111 | WEEKS = 604800 112 | -------------------------------------------------------------------------------- /botoflow/context/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | # XXX docs 15 | 16 | 17 | from .context_base import ContextBase 18 | from .start_workflow_context import StartWorkflowContext 19 | from .activity_context import ActivityContext 20 | from .decision_context import DecisionContext 21 | 22 | __all__ = ('get_context', 'set_context', 'StartWorkflowContext', 23 | 'ActivityContext', 'DecisionContext') 24 | 25 | 26 | get_context = ContextBase.get_context 27 | set_context = ContextBase.set_context 28 | -------------------------------------------------------------------------------- /botoflow/context/activity_context.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | from botoflow.core.exceptions import CancellationError 15 | 16 | from .context_base import ContextBase 17 | 18 | 19 | class ActivityContext(ContextBase): 20 | """ActivityContext is accessible from within activities via 21 | :py:func:`botoflow.get_context` and provides the ability to 22 | retrieve information about the workflow as well as access to the 23 | :py:meth:`heartbeat` for heartbeating the execution of activity 24 | """ 25 | 26 | def __init__(self, worker, task): 27 | """ 28 | 29 | :param worker: 30 | :type worker: botoflow.workers.activity_worker.ActivityWorker 31 | :param task: 32 | :type task: botoflow.workers.activity_task.ActivityTask 33 | """ 34 | 35 | self.worker = worker 36 | self.task = task 37 | 38 | def heartbeat(self, details=None): 39 | """Heartbeats current activity, raising CancellationError if cancel requested. 40 | 41 | Ignore request by catching the exception, or let it raise to cancel. 42 | 43 | :param details: If specified, contains details about the progress of the task. 44 | :type details: str 45 | :raises CancellationError: if uncaught, will record this activity as cancelled 46 | in SWF history, and bubble up to the decider, where it will cancel the 47 | workflow. 48 | """ 49 | result = self.worker.request_heartbeat(self.task, details) 50 | if result['cancelRequested']: 51 | raise CancellationError('Cancel was requested during activity heartbeat') 52 | 53 | @property 54 | def workflow_execution(self): 55 | """ 56 | :returns: the information about current workflow that the activity is handling 57 | :rtype: botoflow.workflow_execution.WorkflowExecution 58 | """ 59 | return self.task.workflow_execution 60 | -------------------------------------------------------------------------------- /botoflow/context/context_base.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | import threading 15 | import logging 16 | 17 | log = logging.getLogger(__name__) 18 | 19 | DEBUG = False 20 | 21 | 22 | class ContextBase(object): 23 | 24 | thread_local = threading.local() 25 | 26 | # Python has no good support for class properties 27 | @classmethod 28 | def get_context(cls): 29 | context = cls.thread_local.flow_current_context 30 | if DEBUG: 31 | log.debug("Current context: %s", context) 32 | return context 33 | 34 | @classmethod 35 | def set_context(cls, context): 36 | if DEBUG: 37 | log.debug("Setting context: %s", context) 38 | cls.thread_local.flow_current_context = context 39 | -------------------------------------------------------------------------------- /botoflow/context/decision_context.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | from .context_base import ContextBase 15 | from ..workflow_execution import WorkflowExecution 16 | 17 | 18 | class DecisionContext(ContextBase): 19 | """Decision context provides information about current workflow execution 20 | """ 21 | 22 | def __init__(self, decider): 23 | self.decider = decider 24 | 25 | self._workflow_time = 0 26 | self._replaying = True 27 | 28 | self._activity_options_overrides = dict() 29 | self._workflow_options_overrides = dict() 30 | 31 | self._workflow_instance = None 32 | self._workflow_execution = WorkflowExecution(None, None) 33 | 34 | @property 35 | def _replaying(self): 36 | """Do not use directly, instead please use 37 | ``botoflow.workflow_time.is_replaying`` 38 | """ 39 | return self.__replaying 40 | 41 | @_replaying.setter 42 | def _replaying(self, value): 43 | self.__replaying = value 44 | 45 | @property 46 | def workflow_execution(self): 47 | """ 48 | :returns: the current workflow execution information 49 | :rtype: botoflow.workflow_execution.WorkflowExecution 50 | """ 51 | return self.__workflow_execution 52 | 53 | @workflow_execution.setter 54 | def workflow_execution(self, value): 55 | self.__workflow_execution = value 56 | 57 | @property 58 | def _workflow_time(self): 59 | """Do not use directly, instead please use 60 | ``botoflow.workflow_time.time`` 61 | 62 | :returns: workflow time 63 | :rtype: datetime.datetime 64 | """ 65 | return self.__time 66 | 67 | @_workflow_time.setter 68 | def _workflow_time(self, value): 69 | """INTERNAL: Never set the time yourself 70 | """ 71 | self.__time = value 72 | 73 | @property 74 | def _workflow_instance(self): 75 | """Returns the currently executing workflow instance 76 | 77 | :rtype: awsflow.workflow_definition.WorkflowDefinition 78 | """ 79 | return self.__workflow_instance 80 | 81 | @_workflow_instance.setter 82 | def _workflow_instance(self, value): 83 | self.__workflow_instance = value 84 | -------------------------------------------------------------------------------- /botoflow/context/start_workflow_context.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | from .context_base import ContextBase 15 | 16 | 17 | class StartWorkflowContext(ContextBase): 18 | """Context provided when running within :py:class:`botoflow.workflow_starter.WorkflowStarter`. 19 | 20 | This gives you an opportunity to get access to the worker and workflow_execution. Generally though it's used 21 | for storing internal states. 22 | 23 | .. py:attribute:: workflow_starter 24 | 25 | :rtype: botoflow.workflow_starting.workflow_starter 26 | 27 | """ 28 | 29 | def __init__(self, worker): 30 | self.worker = worker 31 | 32 | self._workflow_options_overrides = dict() 33 | -------------------------------------------------------------------------------- /botoflow/core/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | from .decorators import coroutine, daemon_coroutine, task, daemon_task 15 | from .async_event_loop import AsyncEventLoop 16 | from .async_context import get_async_context 17 | from .base_future import BaseFuture, Return, return_ 18 | from .future import Future, AnyFuture, AllFuture 19 | from .exceptions import CancelledError 20 | -------------------------------------------------------------------------------- /botoflow/core/async_context.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | import threading 15 | import logging 16 | 17 | log = logging.getLogger(__name__) 18 | 19 | # change this to enable a ton of debug printing 20 | DEBUG = False 21 | 22 | 23 | class AsyncContext(object): 24 | 25 | thread_local = threading.local() 26 | 27 | # Python has no good support for class properties 28 | @classmethod 29 | def get_async_context(cls): 30 | context = cls.thread_local.async_current_context 31 | if DEBUG: 32 | log.debug("Current async context: %r", context) 33 | return context 34 | 35 | @classmethod 36 | def set_async_context(cls, context): 37 | if DEBUG: 38 | log.debug("Setting async context: %r", context) 39 | cls.thread_local.async_current_context = context 40 | 41 | get_async_context = AsyncContext.get_async_context 42 | set_async_context = AsyncContext.set_async_context 43 | -------------------------------------------------------------------------------- /botoflow/core/async_event_loop.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | import logging 15 | import weakref 16 | 17 | from collections import deque 18 | 19 | from .async_root_task_context import AsyncRootTaskContext 20 | from .async_context import set_async_context 21 | 22 | log = logging.getLogger(__name__) 23 | 24 | # change this to enable a ton of debug printing 25 | DEBUG = False 26 | 27 | 28 | class AsyncEventLoop(object): 29 | """ 30 | TODO Document 31 | """ 32 | 33 | def __init__(self): 34 | self.tasks = deque() 35 | self.root_context = AsyncRootTaskContext(weakref.proxy(self)) 36 | 37 | def __enter__(self): 38 | set_async_context(self.root_context) 39 | return self.root_context 40 | 41 | # noinspection PyUnusedLocal 42 | def __exit__(self, exc_type, err, tb): 43 | set_async_context(None) 44 | 45 | def execute(self, task): 46 | if DEBUG: 47 | log.debug("Adding task: %s", task) 48 | self.tasks.append(task) 49 | 50 | def execute_now(self, task): 51 | if DEBUG: 52 | log.debug("Prepending task: %s", task) 53 | self.tasks.appendleft(task) 54 | 55 | def execute_all_tasks(self): 56 | while self.execute_queued_task(): 57 | pass 58 | 59 | def execute_queued_task(self): 60 | if DEBUG: 61 | log.debug("Task queue: %s", self.tasks) 62 | try: 63 | task = self.tasks.popleft() 64 | if not task.done: 65 | if DEBUG: 66 | log.debug("Running task: %s", task) 67 | task.run() 68 | except IndexError: # no more tasks 69 | return False 70 | return True 71 | -------------------------------------------------------------------------------- /botoflow/core/async_root_task_context.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | from .async_task_context import AsyncTaskContext 15 | 16 | 17 | class AsyncRootTaskContext(AsyncTaskContext): 18 | 19 | # noinspection PyMissingConstructor 20 | def __init__(self, eventloop): 21 | self._setup() 22 | self.eventloop = eventloop 23 | self.parent = None 24 | self.name = "Root" 25 | 26 | def update_parent(self): 27 | self.exception = None 28 | -------------------------------------------------------------------------------- /botoflow/core/async_task.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | import sys 15 | import traceback 16 | import logging 17 | 18 | import six 19 | 20 | from .async_context import get_async_context, set_async_context 21 | from .async_task_context import AsyncTaskContext 22 | from .exceptions import CancellationError 23 | from .utils import extract_stacks_from_contexts, filter_framework_frames 24 | 25 | log = logging.getLogger(__name__) 26 | 27 | # change this to enable a ton of debug printing 28 | DEBUG = False 29 | 30 | 31 | class AsyncTask(object): 32 | """ 33 | AsyncTask contains a method with arguments to be run at some time 34 | """ 35 | 36 | def __init__(self, function, args=tuple(), kwargs=None, daemon=False, context=None, name=None): 37 | """ 38 | :param function: function to call at some time. 39 | :type function: FunctionType 40 | :param args: arguments to pass into the function 41 | :type args: tuple 42 | :param kwargs: keyword arguments to pass into the function 43 | :type kwargs: dict 44 | :param daemon: If True, task is a daemon task 45 | :type daemon: True/False 46 | :param context: context to use instead of creating a new 47 | :type context: AsyncTaskContext 48 | 49 | Example:: 50 | task = Task(sum, (1,2)) 51 | task.run() 52 | """ 53 | if not kwargs: 54 | kwargs = {} 55 | 56 | self.cancellable = True 57 | self.cancelled = False 58 | self.exception = None 59 | self.done = False 60 | self.function = function 61 | self.args = args 62 | self.kwargs = kwargs 63 | self.name = name 64 | 65 | self.daemon = daemon 66 | if context: 67 | self.context = context 68 | else: 69 | self.context = AsyncTaskContext(self.daemon, get_async_context()) 70 | 71 | def __enter__(self): 72 | set_async_context(self.context) 73 | return self.context 74 | 75 | # noinspection PyUnusedLocal 76 | def __exit__(self, exc_type, err, tb): 77 | set_async_context(self.context.parent) 78 | 79 | def _run(self): 80 | if self.cancelled: 81 | raise CancellationError() 82 | self.function(*self.args, **self.kwargs) 83 | if self.cancelled: 84 | raise CancellationError() 85 | 86 | def run(self): 87 | """ 88 | Executes the function. This method does not return values from the 89 | functionran. 90 | """ 91 | with self: 92 | try: 93 | self._run() 94 | except Exception as err: 95 | self.exception = err 96 | _, _, tb = sys.exc_info() 97 | stacks = extract_stacks_from_contexts(self.context) 98 | tb_list = list() 99 | 100 | for stack in stacks: 101 | tb_list.extend(stack) 102 | tb_list.append((None, 1, 'flow.core', 103 | '---continuation---')) 104 | 105 | tb_list.extend( 106 | filter_framework_frames(traceback.extract_tb(tb))) 107 | 108 | self.context.handle_exception(err, tb_list) 109 | finally: 110 | self.context.remove_child(self) 111 | self.done = True 112 | self.context = None 113 | 114 | def execute(self): 115 | self.context.add_child(self) 116 | self.schedule() 117 | 118 | def execute_now(self): 119 | self.context.add_child(self) 120 | self.schedule(now=True) 121 | 122 | def schedule(self, now=False): 123 | if not self.done or self.exception is None: 124 | self.context.schedule_task(self, now) 125 | 126 | def cancel(self): 127 | if DEBUG: 128 | log.debug("Task canceling: %r", self) 129 | 130 | if self.exception is None and self.cancellable: 131 | self.cancelled = True 132 | 133 | def __repr__(self): 134 | func_str = '' 135 | if self.name is not None: 136 | func_str = self.name 137 | 138 | args_str = '' 139 | if hasattr(self.function, 'im_self'): 140 | if self.function.im_self: 141 | args_str = 'self' 142 | if self.args or self.kwargs: 143 | args_str = 'self, ' 144 | 145 | if self.args: 146 | args_str += ", ".join((repr(arg) for arg in self.args)) 147 | 148 | if self.args and self.kwargs: 149 | args_str += ', ' 150 | 151 | if self.kwargs: 152 | args_str += ", ".join(('%s=%s' % (key, repr(val)) 153 | for key, val in six.iteritems(self.kwargs))) 154 | 155 | return "<%s at %s %s(%s)>" % (self.__class__.__name__, hex(id(self)), 156 | func_str, args_str) 157 | -------------------------------------------------------------------------------- /botoflow/core/async_traceback.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | """ 15 | This module simulates some functions from the original traceback module 16 | specifically for printing tracebacks. It tidies up the traceback, hiding the 17 | underlying asynchronous framework, so the tracebacks are much more readable. 18 | """ 19 | 20 | import sys 21 | import traceback 22 | from .async_context import get_async_context 23 | from .utils import get_context_with_traceback 24 | 25 | 26 | def print_exc(limit=None, file=None): 27 | """ 28 | Print exception information and up to limit stack trace entries to file 29 | """ 30 | if file is None: 31 | file = sys.stderr 32 | 33 | formatted_exc = format_exc(limit) 34 | if formatted_exc is not None: 35 | for line in formatted_exc: 36 | file.write(line) 37 | 38 | 39 | def format_exc(limit=None, exception=None, tb_list=None): 40 | """ 41 | This is like print_exc(limit) but returns a string instead of printing to a 42 | file. 43 | """ 44 | result = ["Traceback (most recent call last):\n"] 45 | if exception is None: 46 | exception = get_context_with_traceback(get_async_context()).exception 47 | 48 | if tb_list is None: 49 | tb_list = extract_tb(limit) 50 | 51 | if tb_list: 52 | result.extend(traceback.format_list(tb_list)) 53 | result.extend(traceback.format_exception_only(exception.__class__, 54 | exception)) 55 | return result 56 | else: 57 | return None 58 | 59 | 60 | def extract_tb(limit=None): 61 | """ 62 | Return list of up to limit pre-processed entries from traceback. 63 | 64 | This is useful for alternate formatting of stack traces. If 65 | 'limit' is omitted or None, all entries are extracted. A 66 | pre-processed stack trace entry is a quadruple (filename, line 67 | number, function name, text) representing the information that is 68 | usually printed for a stack trace. The text is a string with 69 | leading and trailing whitespace stripped; if the source is not 70 | available it is None. 71 | """ 72 | prev_tb = None 73 | try: 74 | prev_tb = sys.exc_info()[2] 75 | except AttributeError: 76 | pass 77 | 78 | try: 79 | context = get_context_with_traceback(get_async_context()) 80 | except Exception: 81 | context = None 82 | 83 | if context is not None: 84 | tb_list = context.tb_list 85 | if tb_list is not None: 86 | if limit is not None: 87 | return tb_list[-limit:] 88 | return tb_list 89 | else: 90 | return traceback.extract_tb(prev_tb) 91 | -------------------------------------------------------------------------------- /botoflow/core/exceptions.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | 15 | class CancellationError(Exception): 16 | """ 17 | Cancellation error exception 18 | """ 19 | pass 20 | 21 | 22 | class CancelledError(Exception): 23 | """ 24 | The Future was cancelled 25 | """ 26 | 27 | @property 28 | def cause(self): 29 | return self 30 | -------------------------------------------------------------------------------- /botoflow/core/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | """ 14 | Various helper utils for the core 15 | """ 16 | 17 | 18 | def split_stack(stack): 19 | """ 20 | Splits the stack into two, before and after the framework 21 | """ 22 | stack_before, stack_after = list(), list() 23 | in_before = True 24 | for frame in stack: 25 | if 'flow/core' in frame[0]: 26 | in_before = False 27 | else: 28 | if in_before: 29 | stack_before.append(frame) 30 | else: 31 | stack_after.append(frame) 32 | return stack_before, stack_after 33 | 34 | 35 | def filter_framework_frames(stack): 36 | """ 37 | Returns a stack clean of framework frames 38 | """ 39 | result = list() 40 | for frame in stack: 41 | # XXX Windows? 42 | if 'flow/core' not in frame[0]: 43 | result.append(frame) 44 | return result 45 | 46 | 47 | def get_context_with_traceback(context): 48 | """ 49 | Returns the very first context that contains traceback information 50 | """ 51 | if context.tb_list: 52 | return context 53 | if context.parent: 54 | return get_context_with_traceback(context.parent) 55 | 56 | 57 | def extract_stacks_from_contexts(context, stacks_list=None): 58 | """ 59 | Returns a list of stacks extracted from the AsyncTaskContext in the right 60 | order 61 | """ 62 | if stacks_list is None: 63 | stacks_list = list() 64 | 65 | if context.stack_list: 66 | stacks_list.append(context.stack_list) 67 | 68 | if context.parent is not None: 69 | return extract_stacks_from_contexts(context.parent, stacks_list) 70 | else: 71 | stacks_list.reverse() 72 | return stacks_list 73 | 74 | 75 | def log_task_context(context, logger): 76 | """ 77 | A helper for printing contexts as a tree 78 | """ 79 | from .async_root_task_context import AsyncRootTaskContext 80 | 81 | root_context = None 82 | while root_context is None: 83 | if isinstance(context, AsyncRootTaskContext): 84 | root_context = context 85 | context = context.parent 86 | _log_task_context(root_context, logger) 87 | 88 | 89 | def _log_task_context(context, logger, indent=0): 90 | logger.debug(" " * indent + '%r', context) 91 | indent += 2 92 | if not hasattr(context, 'children'): 93 | return 94 | for sub_context in context.children.union(context.daemon_children): 95 | _log_task_context(sub_context, logger, indent) 96 | -------------------------------------------------------------------------------- /botoflow/data_converter/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | from .abstract_data_converter import AbstractDataConverter 15 | from .pickle_data_converter import PickleDataConverter 16 | from .json_data_converter import JSONDataConverter 17 | -------------------------------------------------------------------------------- /botoflow/data_converter/abstract_data_converter.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | import abc 15 | 16 | 17 | class AbstractDataConverter(object): 18 | """ 19 | Subclasses of this data converter are used by the framework to 20 | serialize/deserialize method parameters that need to be sent over the wire. 21 | """ 22 | __metaclass__ = abc.ABCMeta 23 | 24 | @abc.abstractmethod 25 | def dumps(self, obj): 26 | """ 27 | Should return serialized string data 28 | """ 29 | raise NotImplementedError 30 | 31 | @abc.abstractmethod 32 | def loads(self, data): 33 | """ 34 | Should return deserialized string data 35 | """ 36 | raise NotImplementedError 37 | -------------------------------------------------------------------------------- /botoflow/data_converter/pickle_data_converter.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | import pickle 15 | 16 | from base64 import b64encode, b64decode 17 | 18 | from .abstract_data_converter import AbstractDataConverter 19 | 20 | 21 | class PickleDataConverter(AbstractDataConverter): 22 | """This is a *pickling* data converter. The data passed around 23 | with SWF will be in the pickle format. In addition, if the 24 | protocol version is not 0, the data will be base64 encoded (as any 25 | version other than 0 is binary). 26 | 27 | .. warning:: 28 | 29 | This data converter is **NOT** recomended as it does not 30 | serialize exceptions well and can cause various hard to debug 31 | issues. 32 | """ 33 | 34 | def __init__(self, protocol=0): 35 | """ 36 | 37 | :param protocol: Pickle protocol version 38 | """ 39 | self._protocol = protocol 40 | 41 | def dumps(self, obj): 42 | """Dumps object as pickle then base64 encodes it depending on 43 | the protocol version. 44 | 45 | :param obj: object to serialize. 46 | """ 47 | if self._protocol == 0: 48 | return pickle.dumps(obj, 0) 49 | return b64encode(pickle.dumps(obj, self._protocol)) 50 | 51 | def loads(self, data): 52 | """loads the pickle data 53 | 54 | :param data: data to deserialize. 55 | """ 56 | if self._protocol == 0: 57 | return pickle.loads(data) 58 | return pickle.loads(b64decode(data)) 59 | -------------------------------------------------------------------------------- /botoflow/decider/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | from .decider import Decider 15 | from .workflow_replayer import WorkflowReplayer 16 | -------------------------------------------------------------------------------- /botoflow/decider/activity_future.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | 15 | from ..core import BaseFuture, AnyFuture, AllFuture, CancelledError 16 | from ..core.async_task import AsyncTask 17 | 18 | 19 | class ActivityFuture(BaseFuture): 20 | 21 | def __init__(self, future, activity_task_handler, activity_id): 22 | """ 23 | 24 | :param future: 25 | :type future: Future 26 | :param activity_task_handler: 27 | :type activity_task_handler: awsflow.decider.activity_task_handler.ActivityTaskHandler 28 | :param activity_id: 29 | :type activity_id: str 30 | :return: 31 | """ 32 | super(ActivityFuture, self).__init__() 33 | 34 | self._future = future 35 | self._activity_task_handler = activity_task_handler 36 | self._activity_id = activity_id 37 | self._cancellation_future = None # may be set to a Future if cancel was requested 38 | 39 | task = AsyncTask(self._future_callback, (future,), 40 | name=self._future_callback.__name__) 41 | task.cancellable = False 42 | future.add_task(task) 43 | 44 | def _future_callback(self, future): 45 | if self.done(): 46 | return 47 | 48 | if future.exception() is not None: 49 | self.set_exception(future.exception(), 50 | future.traceback()) 51 | else: 52 | self.set_result(future.result()) 53 | 54 | if self._cancellation_future is not None: 55 | self._cancellation_future.set_result(None) 56 | 57 | def cancel(self): 58 | """Requests cancellation of activity. 59 | 60 | :return: Cancellation future 61 | :rtype: awsflow.Future 62 | """ 63 | if not super(ActivityFuture, self).cancelled(): 64 | self._cancellation_future = self._activity_task_handler.request_cancel_activity_task( 65 | self, self._activity_id) 66 | 67 | return self._cancellation_future 68 | 69 | def cancelled(self): 70 | """ 71 | Returns True if activity was cancelled 72 | :return: 73 | """ 74 | if isinstance(self._exception, CancelledError): 75 | return True 76 | return False 77 | 78 | def exception(self): 79 | """ 80 | Returns the exception if available, or a ValueError if a result is 81 | not yet set 82 | """ 83 | if self.done(): 84 | return self._exception 85 | else: 86 | raise ValueError("Exception was not yet set") 87 | 88 | def traceback(self): 89 | if self.done(): 90 | return self._traceback 91 | else: 92 | raise ValueError("Exception is not yet set") 93 | 94 | def result(self): 95 | """ 96 | Return the result 97 | 98 | :raises Exception: Any exception raised from the call will be raised. 99 | :raises ValueError: if a result was not yet set 100 | """ 101 | if self.done(): 102 | return self._get_result() 103 | else: 104 | raise ValueError("Result is not yet set") 105 | 106 | def __or__(self, other): 107 | if isinstance(other, BaseFuture): 108 | return AnyFuture(self, other) 109 | elif isinstance(other, AnyFuture): 110 | other.add_future(self) 111 | return other 112 | 113 | raise TypeError("unsupported operand type(s) for " 114 | "|: '%s' and '%s'" % (self.__class__.__name__, 115 | other.__class__.__name__)) 116 | 117 | def __and__(self, other): 118 | if isinstance(other, BaseFuture): 119 | return AllFuture(self, other) 120 | elif isinstance(other, AllFuture): 121 | other.add_future(self) 122 | return other 123 | 124 | raise TypeError("unsupported operand type(s) for " 125 | "&: '%s' and '%s'" % (self.__class__.__name__, 126 | other.__class__.__name__)) 127 | -------------------------------------------------------------------------------- /botoflow/decider/decision_task_poller.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | import sys 15 | import time 16 | 17 | import six 18 | 19 | from ..history_events import swf_event_to_object 20 | 21 | 22 | class EventsIterator(six.Iterator): 23 | 24 | def __init__(self, poller, decision_dict): 25 | self.poller = poller 26 | self.decision_dict = decision_dict 27 | self.cur_event_pos = -1 28 | self.event_len = len(decision_dict['events']) 29 | 30 | def __next__(self): 31 | self.cur_event_pos += 1 32 | if self.cur_event_pos >= self.event_len: 33 | if 'nextPageToken' in self.decision_dict: 34 | self.decision_dict = self.poller.single_poll( 35 | self.decision_dict['nextPageToken']) 36 | self.cur_event_pos = 0 37 | self.event_len = len(self.decision_dict['events']) 38 | else: 39 | raise StopIteration() 40 | 41 | return swf_event_to_object( 42 | self.decision_dict['events'][self.cur_event_pos]) 43 | 44 | def contains(self, event_type): 45 | """ 46 | :param event_type: type of event to search for 47 | :type event_type: awsflow.history_events.event_bases.EventBase 48 | :return: True if given event type exists among events 49 | :rtype: bool 50 | """ 51 | return any(isinstance(swf_event_to_object(event), event_type) 52 | for event in self.decision_dict['events']) 53 | 54 | 55 | class DecisionTask(object): 56 | 57 | def __init__(self, poller, decision_dict): 58 | self._poller = poller 59 | 60 | self._decision_dict = decision_dict 61 | self.started_event_id = decision_dict['startedEventId'] 62 | self.task_token = decision_dict['taskToken'] 63 | self.previous_started_event_id = decision_dict[ 64 | 'previousStartedEventId'] 65 | 66 | self.workflow_id = decision_dict['workflowExecution']['workflowId'] 67 | self.run_id = decision_dict['workflowExecution']['runId'] 68 | self.workflow_name = decision_dict['workflowType']['name'] 69 | self.workflow_version = decision_dict['workflowType']['version'] 70 | 71 | def __repr__(self): 72 | return ("<{0} workflow_name={1.workflow_name}, workflow_version={1.workflow_version}, " 73 | "started_event_id={1.started_event_id}, previous_started_event_id={1.previous_started_event_id} " 74 | "workflow_id={1.workflow_id}, run_id={1.run_id}>").format(self.__class__.__name__, self) 75 | 76 | @property 77 | def events(self): 78 | return EventsIterator(self._poller, self._decision_dict) 79 | 80 | 81 | class DecisionTaskPoller(object): 82 | """ 83 | Polls for decisions 84 | """ 85 | 86 | def __init__(self, worker, domain, task_list, identity): 87 | self.worker = worker 88 | self.domain = domain 89 | self.task_list = task_list 90 | self.identity = identity 91 | 92 | def single_poll(self, next_page_token=None): 93 | poll_time = time.time() 94 | try: 95 | kwargs = {'domain': self.domain, 96 | 'taskList': {'name': self.task_list}, 97 | 'identity': self.identity} 98 | if next_page_token is not None: 99 | kwargs['nextPageToken'] = next_page_token 100 | # old botocore throws TypeError when unable to establish SWF connection 101 | return self.worker.client.poll_for_decision_task(**kwargs) 102 | 103 | except KeyboardInterrupt: 104 | # sleep before actually exiting as the connection is not yet closed 105 | # on the other end 106 | sleep_time = 60 - (time.time() - poll_time) 107 | six.print_("Exiting in {0}...".format(sleep_time), file=sys.stderr) 108 | time.sleep(sleep_time) 109 | raise 110 | 111 | def poll(self): 112 | """ 113 | Returns a paginating DecisionTask generator 114 | """ 115 | decision_dict = self.single_poll() 116 | # from pprint import pprint 117 | # pprint(decision_dict) 118 | if decision_dict['startedEventId'] == 0: 119 | return None 120 | else: 121 | return DecisionTask(self, decision_dict) 122 | -------------------------------------------------------------------------------- /botoflow/decider/external_workflow_handler.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | from collections import defaultdict 14 | import logging 15 | 16 | import six 17 | 18 | from ..core import Future, get_async_context 19 | from ..core.async_task_context import AsyncTaskContext 20 | from ..decisions import RequestCancelExternalWorkflowExecution 21 | from ..exceptions import RequestCancelExternalWorkflowExecutionFailedError 22 | from ..workflow_execution import workflow_execution_from_swf_event 23 | from ..history_events import (ExternalWorkflowExecutionCancelRequested, 24 | RequestCancelExternalWorkflowExecutionInitiated, 25 | RequestCancelExternalWorkflowExecutionFailed) 26 | 27 | log = logging.getLogger(__name__) 28 | 29 | 30 | class ExternalWorkflowHandler(object): 31 | 32 | responds_to = (ExternalWorkflowExecutionCancelRequested, 33 | RequestCancelExternalWorkflowExecutionInitiated, 34 | RequestCancelExternalWorkflowExecutionFailed) 35 | 36 | def __init__(self, decider, task_list): 37 | self._decider = decider 38 | self._open_cancel_requests = defaultdict(dict) 39 | self._task_list = task_list 40 | 41 | def handle_event(self, event): 42 | external_workflow_execution = workflow_execution_from_swf_event(event) 43 | self._open_cancel_requests[external_workflow_execution]['handler'].send(event) 44 | 45 | def request_cancel_external_workflow_execution(self, external_workflow_execution): 46 | """Requests cancellation of another workflow. 47 | 48 | :param external_workflow_execution: details of target workflow to cancel 49 | :type external_workflow_execution: botoflow.workflow_execution.WorkflowExecution 50 | :return: cancel Future 51 | :rtype: awsflow.core.future.Future 52 | """ 53 | self._decider._decisions.append(RequestCancelExternalWorkflowExecution( 54 | workflow_id=external_workflow_execution.workflow_id, 55 | run_id=external_workflow_execution.run_id)) 56 | 57 | cancel_future = Future() 58 | context = AsyncTaskContext(False, get_async_context()) 59 | cancel_future.context = context 60 | 61 | handler = self._handle_external_workflow_event(external_workflow_execution, cancel_future) 62 | six.next(handler) 63 | self._open_cancel_requests[external_workflow_execution] = {'handler': handler} 64 | return cancel_future 65 | 66 | def _handle_external_workflow_event(self, external_workflow_execution, cancel_future): 67 | """Handles external workflow events and resolves open handler(s). 68 | 69 | Events handled: 70 | RequestCancelExternalWorkflowExecutionInitiated 71 | - SWF has received decision of canceling an external workflow 72 | RequestCancelExternalWorkflowExecutionFailed 73 | - SWF could not send cancel request to external workflow due to invalid workflowID 74 | ExternalWorkflowExecutionCancelRequested 75 | - SWF successfully sent cancel request to target external workflow 76 | 77 | :param external_workflow_execution: details of target workflow to cancel 78 | :type external_workflow_execution: botoflow.workflow_execution.WorkflowExecution 79 | :param cancel_future: 80 | :type cancel_future: awsflow.core.future.Future 81 | :return: 82 | """ 83 | event = (yield) 84 | 85 | if isinstance(event, RequestCancelExternalWorkflowExecutionInitiated): 86 | self._decider._decisions.delete_decision(RequestCancelExternalWorkflowExecution, 87 | external_workflow_execution) 88 | event = (yield) 89 | 90 | if isinstance(event, ExternalWorkflowExecutionCancelRequested): 91 | cancel_future.set_result(None) 92 | 93 | elif isinstance(event, RequestCancelExternalWorkflowExecutionFailed): 94 | attributes = event.attributes 95 | exception = RequestCancelExternalWorkflowExecutionFailedError( 96 | attributes['decisionTaskCompletedEventId'], 97 | attributes['initiatedEventId'], 98 | attributes['runId'], 99 | attributes['workflowId'], 100 | attributes['cause']) 101 | cancel_future.set_exception(exception) 102 | 103 | del self._open_cancel_requests[external_workflow_execution] 104 | -------------------------------------------------------------------------------- /botoflow/decider/timer_handler.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | import six 15 | import logging 16 | 17 | from ..core import Future, coroutine, CancelledError 18 | from ..decisions import StartTimer 19 | from ..history_events import (StartTimerFailed, TimerFired, TimerStarted, TimerCanceled) 20 | log = logging.getLogger(__name__) 21 | 22 | 23 | class TimerHandler(object): 24 | 25 | responds_to = (StartTimerFailed, TimerFired, TimerStarted, TimerCanceled) 26 | 27 | def __init__(self, decider, task_list): 28 | 29 | self._decider = decider 30 | self._open_timers = {} 31 | self._task_list = task_list 32 | 33 | def handle_execute_timer(self, seconds): 34 | decision_id = self._decider.get_next_id() 35 | timer_decision = StartTimer(decision_id, str(int(seconds))) 36 | self._decider._decisions.append(timer_decision) 37 | 38 | timer_future = Future() 39 | 40 | handler = self._handler_fsm(decision_id, timer_future) 41 | six.next(handler) # arm 42 | self._open_timers[decision_id] = {'future': timer_future, 'handler': handler} 43 | 44 | @coroutine 45 | def wait_for_timer(): 46 | yield timer_future 47 | 48 | return wait_for_timer() 49 | 50 | def handle_event(self, event): 51 | if isinstance(event, (StartTimerFailed, TimerFired, TimerStarted, TimerCanceled)): 52 | timer_id = event.attributes['timerId'] 53 | self._open_timers[timer_id]['handler'].send(event) 54 | else: 55 | log.warn("Tried to handle timer event, but a handler is missing: %r", event) 56 | 57 | def _handler_fsm(self, timer_id, timer_future): 58 | event = (yield) 59 | 60 | if isinstance(event, (StartTimerFailed, TimerStarted)): 61 | self._decider._decisions.delete_decision(StartTimer, timer_id) 62 | 63 | if isinstance(event, StartTimerFailed): 64 | # TODO Throw an exception on startTimerFailed 65 | pass 66 | elif isinstance(event, TimerStarted): 67 | event = (yield) 68 | 69 | if isinstance(event, TimerFired): 70 | timer_future.set_result(None) 71 | elif isinstance(event, TimerCanceled): 72 | timer_future.set_exception(CancelledError("Timer Cancelled")) 73 | else: 74 | raise RuntimeError("Unexpected event/state: %s", event) 75 | 76 | del self._open_timers[timer_id] 77 | -------------------------------------------------------------------------------- /botoflow/decider/workflow_replayer.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | import json 15 | 16 | from .decider import Decider 17 | from .decision_task_poller import DecisionTaskPoller 18 | from ..utils import extract_workflows_dict 19 | 20 | 21 | class ReplayingDecisionTaskPoller(DecisionTaskPoller): 22 | 23 | def single_poll(self, next_page_token=None): 24 | try: 25 | result = json.loads(self.history_pages[self.position]) 26 | result['startedEventId'] = '1' 27 | result['previousStartedEventId'] = '0' 28 | result['taskToken'] = 'test_token' 29 | result['workflowExecution'] = self.workflow_description[ 30 | 'executionInfo']['execution'] 31 | result['workflowType'] = self.workflow_description[ 32 | 'executionInfo']['workflowType'] 33 | return result 34 | finally: 35 | self.position += 1 36 | 37 | 38 | class WorkflowReplayer(object): 39 | 40 | def from_history_dump(self, workflows, workflow_description, 41 | history_pages): 42 | self._history_pages = history_pages 43 | self._workflows = extract_workflows_dict(workflows) 44 | 45 | self._workflow_description = json.loads(workflow_description) 46 | 47 | self._poller = ReplayingDecisionTaskPoller 48 | self._poller.workflow_description = self._workflow_description 49 | self._poller.history_pages = history_pages 50 | self._poller.position = 0 51 | return self 52 | 53 | def replay(self): 54 | decider = Decider( 55 | swf_client=None, 56 | domain='replaying_domain', 57 | task_list=self._workflow_description[ 58 | 'executionConfiguration']['taskList']['name'], 59 | workflows=self._workflows, 60 | identity='replaying_decider_ident', 61 | _Poller=self._poller 62 | ) 63 | 64 | while True: 65 | decider.decide() 66 | -------------------------------------------------------------------------------- /botoflow/decisions/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | ### 15 | 16 | from .decisions import * 17 | from .decision_list import DecisionList 18 | -------------------------------------------------------------------------------- /botoflow/decisions/decision_bases.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | """ 14 | Decision bases are base classes for various decision classes, essentially grouping and reusing common bits between 15 | actual decisions. 16 | 17 | """ 18 | 19 | 20 | class DecisionBase(object): 21 | """Every decision or decision base should inherit from this class 22 | 23 | .. py:data:: decision 24 | 25 | Contains a dictionary of decision attributes 26 | """ 27 | 28 | def __init__(self): 29 | self.decision = {} 30 | 31 | 32 | class ActivityDecisionBase(DecisionBase): 33 | """Base for Activity decisions 34 | """ 35 | 36 | def __init__(self, decision_id): 37 | """ 38 | :param str decision_id: Decision ID 39 | """ 40 | super(ActivityDecisionBase, self).__init__() 41 | self.decision_id = decision_id 42 | 43 | def __repr__(self): 44 | return "<%s decision activity_id=%s details=%s>" % ( 45 | self.__class__.__name__, self.decision_id, self.decision) 46 | 47 | 48 | class RequestCancelExternalWorkflowDecisionBase(DecisionBase): 49 | """Base for requesting a cancellation of external workflow 50 | """ 51 | 52 | def __init__(self, workflow_id, run_id): 53 | """ 54 | :param str workflow_id: Workflow ID 55 | :param str run_id: Run ID 56 | """ 57 | super(RequestCancelExternalWorkflowDecisionBase, self).__init__() 58 | self.decision_id = (workflow_id, run_id) 59 | 60 | def __repr__(self): 61 | return "<%s decision workflow_id=%s run_id=%s details=%s>" % ( 62 | self.__class__.__name__, self.decision_id[0], 63 | self.decision_id[1], self.decision) 64 | 65 | 66 | class RecordMarkerDecisionBase(DecisionBase): 67 | """Record marker decision base""" 68 | 69 | def __init__(self, decision_id): 70 | """ 71 | :param str decision_id: Decision ID 72 | """ 73 | super(RecordMarkerDecisionBase, self).__init__() 74 | self.decision_id = decision_id 75 | 76 | def __repr__(self): 77 | return "<%s decision marker_id=%s details=%s>" % ( 78 | self.__class__.__name__, self.decision_id, self.decision) 79 | 80 | 81 | class SignalExternalWorkflowExecutionDecisionBase(DecisionBase): 82 | """Signalling for external workflow 83 | """ 84 | 85 | def __init__(self, workflow_id, run_id, signal_name): 86 | """ 87 | :param str workflow_id: Workflow ID 88 | :param str run_id: Run ID 89 | :param str signal_name: name of the signal to execute 90 | """ 91 | super(SignalExternalWorkflowExecutionDecisionBase, self).__init__() 92 | self.decision_id = (workflow_id, run_id, signal_name) 93 | 94 | def __repr__(self): 95 | return ("<%s decision workflow_id=%s run_id=%s signal_name " 96 | "%s details=%s>" % ( 97 | self.__class__.__name__, self.decision_id[0], 98 | self.decision_id[1], self.decision_id[2], self.decision)) 99 | 100 | 101 | class StartChildWorkflowExecutionDecisionBase(DecisionBase): 102 | """Starting child workflow base 103 | """ 104 | 105 | def __init__(self, workflow_type_name, workflow_type_version, workflow_id): 106 | """ 107 | :param str workflow_type_name: Workflow type name 108 | :param str workflow_type_version: version of the workflow 109 | :param str workflow_id: Workflow ID 110 | """ 111 | super(StartChildWorkflowExecutionDecisionBase, self).__init__() 112 | self.decision_id = (workflow_type_name, workflow_type_version, 113 | workflow_id) 114 | 115 | def __repr__(self): 116 | return ("<%s decision workflow_type_name=%s, " 117 | "workflow_type_version=%s, workflow_id=%s details=%s>" % ( 118 | self.__class__.__name__, self.decision_id[0], 119 | self.decision_id[1], self.decision_id[2], self.decision)) 120 | 121 | 122 | class TimerDecisionBase(DecisionBase): 123 | """Timer decisions base 124 | """ 125 | 126 | def __init__(self, decision_id): 127 | """ 128 | :param str decision_id: Decision ID 129 | """ 130 | super(TimerDecisionBase, self).__init__() 131 | self.decision_id = decision_id 132 | 133 | def __repr__(self): 134 | return "<%s decision timer_id=%s details=%s>" % ( 135 | self.__class__.__name__, self.decision_id, self.decision) 136 | 137 | 138 | class WorkflowDecisionBase(DecisionBase): 139 | """Workflow decision base""" 140 | 141 | def __init__(self): 142 | super(WorkflowDecisionBase, self).__init__() 143 | 144 | def __repr__(self): 145 | return "<%s decision details=%s>" % ( 146 | self.__class__.__name__, self.decision) 147 | -------------------------------------------------------------------------------- /botoflow/decisions/decision_list.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | 15 | class DecisionList(list): 16 | """ 17 | DecisionList is just like a regular list with a few additional methods 18 | """ 19 | 20 | # TODO: validation of inputs 21 | 22 | def delete_decision(self, decision_type, decision_id): 23 | """delete a decision 24 | 25 | :returns: True if the decision was found and removed, False otherwise 26 | :rtype: bool 27 | """ 28 | for decision in self: 29 | if not isinstance(decision, decision_type): 30 | continue 31 | 32 | if decision.decision_id == decision_id: 33 | self.remove(decision) 34 | return True 35 | 36 | return False 37 | 38 | def has_decision_type(self, *args): 39 | for decision in self: 40 | if isinstance(decision, args): 41 | return True 42 | return False 43 | 44 | def to_swf(self): 45 | """ 46 | Returns a list of decisions ready to be consumend by swf api 47 | """ 48 | swf_decisions = list() 49 | for decision in self: 50 | swf_decisions.append(decision.decision) 51 | return swf_decisions 52 | -------------------------------------------------------------------------------- /botoflow/flow_types/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | 15 | from .activity_type import ActivityType 16 | from .signal_type import SignalType 17 | from .workflow_type import WorkflowType 18 | -------------------------------------------------------------------------------- /botoflow/flow_types/base_flow_type.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Darjus Loktevic 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import abc 16 | 17 | 18 | class BaseFlowType(object): 19 | 20 | __metaclass__ = abc.ABCMeta 21 | 22 | @abc.abstractmethod 23 | def to_decision_dict(self, *args, **kwargs): 24 | raise NotImplementedError() 25 | 26 | @abc.abstractmethod 27 | def to_registration_options_dict(self, *args, **kwargs): 28 | raise NotImplementedError() 29 | 30 | @abc.abstractmethod 31 | def __call__(self, *args, **kwargs): 32 | raise NotImplementedError() 33 | 34 | @abc.abstractmethod 35 | def _reset_name(self, *args, **kwargs): 36 | raise NotImplementedError() 37 | -------------------------------------------------------------------------------- /botoflow/flow_types/signal_type.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Darjus Loktevic 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from ..context import get_context, StartWorkflowContext, ActivityContext 16 | from ..swf_exceptions import swf_exception_wrapper 17 | 18 | from .base_flow_type import BaseFlowType 19 | 20 | 21 | class SignalType(BaseFlowType): 22 | 23 | def __init__(self, name, data_converter=None): 24 | """ 25 | :param data_converter: (optional) Serializer to use for serializing inputs 26 | :type: botoflow.data_converter.AbstractDataConverter 27 | """ 28 | self.name = name 29 | self.data_converter = data_converter 30 | self.workflow_execution = None 31 | 32 | def to_decision_dict(self): 33 | raise NotImplementedError("Not applicable to SignalType") 34 | 35 | def to_registration_options_dict(self): 36 | raise NotImplementedError("Not applicable to SignalType") 37 | 38 | def _reset_name(self): 39 | raise NotImplementedError("Not applicable to SignalType") 40 | 41 | def __call__(self, *args, **kwargs): 42 | """ 43 | Records a WorkflowExecutionSignaled event in the workflow execution 44 | history and creates a decision task for the workflow execution 45 | identified by the given domain, workflow_execution. 46 | The event is recorded with the specified user defined name 47 | and input (if provided). 48 | 49 | :returns: Signals do not return anything 50 | :rtype: None 51 | 52 | :raises: UnknownResourceFault, OperationNotPermittedFault, RuntimeError 53 | """ 54 | serialized_input = self.data_converter.dumps([args, kwargs]) 55 | workflow_execution = self.workflow_execution 56 | 57 | context = get_context() 58 | if not isinstance(context, (StartWorkflowContext, ActivityContext)): 59 | raise RuntimeError( 60 | "Unsupported context for this call: %r" % context) 61 | 62 | with swf_exception_wrapper(): 63 | context.worker.client.signal_workflow_execution( 64 | domain=context.worker.domain, signalName=self.name, 65 | workflowId=workflow_execution.workflow_id, 66 | runId=workflow_execution.run_id, 67 | input=serialized_input) 68 | 69 | def __repr__(self): 70 | return "<{0}(name={1.name}, data_converter={1.data_converter}, " \ 71 | "workflow_execution={1.workflow_execution}".format(self.__class__.__name__, self) 72 | -------------------------------------------------------------------------------- /botoflow/history_events/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | 15 | from .events import * 16 | -------------------------------------------------------------------------------- /botoflow/history_events/event_bases.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | 15 | class EventBase(object): 16 | 17 | def __init__(self, event_id, datetime, attributes): 18 | """ 19 | :param event_id: event id 20 | :type event_id: int 21 | :param datetime: datetime of event 22 | :type datetime: datetime.datetime 23 | :param attributes: event attributes 24 | :type attributes: dict 25 | """ 26 | self.id = event_id 27 | self.datetime = datetime 28 | self.attributes = attributes 29 | 30 | def __repr__(self): 31 | return "<{0} id={1}, time={2}, attributes={3}>".format( 32 | self.__class__.__name__, self.id, self.datetime, 33 | self.attributes) 34 | 35 | 36 | class ActivityEventBase(EventBase): 37 | pass 38 | 39 | 40 | class ChildWorkflowEventBase(EventBase): 41 | pass 42 | 43 | 44 | class DecisionEventBase(EventBase): 45 | """ 46 | To be used as a mixin with events that represent decisions 47 | """ 48 | pass 49 | 50 | 51 | class DecisionTaskEventBase(EventBase): 52 | pass 53 | 54 | 55 | class ExternalWorkflowEventBase(EventBase): 56 | pass 57 | 58 | 59 | class MarkerEventBase(EventBase): 60 | pass 61 | 62 | 63 | class TimerEventBase(EventBase): 64 | pass 65 | 66 | 67 | class WorkflowEventBase(EventBase): 68 | pass 69 | -------------------------------------------------------------------------------- /botoflow/logging_filters.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | import logging 15 | 16 | import botoflow 17 | 18 | 19 | class BotoflowFilter(logging.Filter): 20 | """You can use this filter with Python's :py:mod:`logging` module to filter out 21 | botoflow logs that are being replayed by the decider. 22 | 23 | For example:: 24 | 25 | import logging 26 | from botoflow.logging_filters import BotoflowFilter 27 | 28 | logging.basicConfig(level=logging.DEBUG, 29 | format='%(filename)s:%(lineno)d (%(funcName)s) - %(message)s') 30 | 31 | logging.getLogger('botoflow').addFilter(BotoflowFilter()) 32 | 33 | """ 34 | 35 | def __init__(self, name='', filter_replaying=True): 36 | super(BotoflowFilter, self).__init__(name) 37 | self._filter_replaying = filter_replaying 38 | 39 | def filter(self, record): 40 | try: 41 | if self._filter_replaying and botoflow.get_context().replaying: 42 | return 0 43 | except AttributeError: 44 | pass 45 | return 1 46 | -------------------------------------------------------------------------------- /botoflow/manual_activity_completion_client.py: -------------------------------------------------------------------------------- 1 | from botoflow.swf_exceptions import swf_exception_wrapper 2 | from botoflow.data_converter import JSONDataConverter 3 | from botoflow.core.exceptions import CancellationError 4 | 5 | 6 | class ManualActivityCompletionClient(object): 7 | def __init__(self, swf_client, data_converter=JSONDataConverter()): 8 | """Helper class to work with manual activities 9 | 10 | :param swf_client: botocore SWF client 11 | :type swf_client: botocore.clients.Client 12 | :param data_converter: DataConverter to use for marshaling data 13 | :type data_converter: botoflow.data_converter.BaseDataConverter 14 | """ 15 | self._swf_client = swf_client 16 | self.data_converter = data_converter 17 | 18 | def complete(self, result, task_token): 19 | result = self.data_converter.dumps(result) 20 | with swf_exception_wrapper(): 21 | self._swf_client.respond_activity_task_completed(result=result, taskToken=task_token) 22 | 23 | def fail(self, details, task_token, reason=''): 24 | details = self.data_converter.dumps(details) 25 | with swf_exception_wrapper(): 26 | self._swf_client.respond_activity_task_failed(details=details, reason=reason, taskToken=task_token) 27 | 28 | def cancel(self, details, task_token): 29 | with swf_exception_wrapper(): 30 | self._swf_client.respond_activity_task_cancelled(details=details, taskToken=task_token) 31 | 32 | def record_heartbeat(self, details, task_token): 33 | with swf_exception_wrapper(): 34 | response_data = self._swf_client.record_activity_task_hearbeat(details=details, taskToken=task_token) 35 | if response_data['cancelRequested']: 36 | raise CancellationError() 37 | -------------------------------------------------------------------------------- /botoflow/swf_exceptions.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | """Exceptions from the SWF service 15 | """ 16 | 17 | from contextlib import contextmanager 18 | from botocore.client import ClientError 19 | 20 | 21 | class SWFResponseError(Exception): 22 | """Base exception for SWF Errors""" 23 | pass 24 | 25 | 26 | class DomainDeprecatedError(SWFResponseError): 27 | """Returned when the specified domain has been deprecated. 28 | """ 29 | pass 30 | 31 | 32 | class DomainAlreadyExistsError(SWFResponseError): 33 | """Returned if the specified domain already exists. You will get this fault 34 | even if the existing domain is in deprecated status. 35 | """ 36 | pass 37 | 38 | 39 | class DefaultUndefinedError(SWFResponseError): 40 | """Constructs a new DefaultUndefinedException with the specified error 41 | message. 42 | """ 43 | pass 44 | 45 | 46 | class LimitExceededError(SWFResponseError): 47 | """Returned by any operation if a system imposed limitation has been 48 | reached. 49 | 50 | To address this fault you should either clean up unused resources or 51 | increase the limit by contacting AWS. 52 | """ 53 | pass 54 | 55 | 56 | class WorkflowExecutionAlreadyStartedError(SWFResponseError): 57 | """Returned by StartWorkflowExecution when an open execution with the same 58 | workflowId is already running in the specified domain. 59 | """ 60 | 61 | 62 | class TypeDeprecatedError(SWFResponseError): 63 | """Returned when the specified activity or workflow type was already 64 | deprecated. 65 | """ 66 | pass 67 | 68 | 69 | class TypeAlreadyExistsError(SWFResponseError): 70 | """Returned if the type already exists in the specified domain. 71 | 72 | You will get this fault even if the existing type is in deprecated 73 | status. You can specify another version if the intent is to create a new 74 | distinct version of the type. 75 | """ 76 | pass 77 | 78 | 79 | class OperationNotPermittedError(SWFResponseError): 80 | """Returned when the requester does not have the required permissions to 81 | perform the requested operation. 82 | """ 83 | pass 84 | 85 | 86 | class UnknownResourceError(SWFResponseError): 87 | """Returned when the named resource cannot be found with in the scope of 88 | this operation (region or domain). 89 | 90 | This could happen if the named resource was never created or is no longer 91 | available for this operation. 92 | """ 93 | pass 94 | 95 | 96 | class UnrecognizedClientException(SWFResponseError): 97 | """Raised when the client is not authenticated by SWF""" 98 | pass 99 | 100 | 101 | class ThrottlingException(SWFResponseError): 102 | pass 103 | 104 | 105 | class ValidationException(SWFResponseError): 106 | pass 107 | 108 | 109 | class InternalFailureError(SWFResponseError): 110 | """Raised when there's an internal SWF failure""" 111 | pass 112 | 113 | 114 | # SWF __type/fault string to botoflow exception mapping 115 | _swf_fault_exception = { 116 | 'DomainDeprecatedFault': DomainDeprecatedError, 117 | 'DomainAlreadyExistsFault': DomainAlreadyExistsError, 118 | 'DefaultUndefinedFault': DefaultUndefinedError, 119 | 'LimitExceededFault': LimitExceededError, 120 | 'WorkflowExecutionAlreadyStartedFault': WorkflowExecutionAlreadyStartedError, 121 | 'TypeDeprecatedFault': TypeDeprecatedError, 122 | 'TypeAlreadyExistsFault': TypeAlreadyExistsError, 123 | 'OperationNotPermittedFault': OperationNotPermittedError, 124 | 'UnknownResourceFault': UnknownResourceError, 125 | 'SWFResponseError': SWFResponseError, 126 | 'ThrottlingException': ThrottlingException, 127 | 'ValidationException': ValidationException, 128 | 'UnrecognizedClientException': UnrecognizedClientException, 129 | 'InternalFailure': InternalFailureError 130 | } 131 | 132 | 133 | @contextmanager 134 | def swf_exception_wrapper(): 135 | try: 136 | yield 137 | except ClientError as err: 138 | err_type = err.response['Error'].get('Code', 'SWFResponseError') 139 | err_msg = err.response['Error'].get( 140 | 'Message', 'No error message provided...') 141 | 142 | raise _swf_fault_exception.get(err_type, SWFResponseError)(err_msg) 143 | -------------------------------------------------------------------------------- /botoflow/test/__init__.py: -------------------------------------------------------------------------------- 1 | from .workflow_testing_context import WorkflowTestingContext 2 | -------------------------------------------------------------------------------- /botoflow/test/workflow_testing_context.py: -------------------------------------------------------------------------------- 1 | from botoflow.core import AsyncEventLoop 2 | 3 | from botoflow.context import ContextBase 4 | 5 | 6 | class WorkflowTestingContext(ContextBase): 7 | 8 | def __init__(self): 9 | self._event_loop = AsyncEventLoop() 10 | 11 | def __enter__(self): 12 | try: 13 | self._context = self.get_context() 14 | except AttributeError: 15 | self._context = None 16 | self.set_context(self) 17 | self._event_loop.__enter__() 18 | 19 | def __exit__(self, exc_type, exc_val, exc_tb): 20 | if exc_type is None: 21 | self._event_loop.execute_all_tasks() 22 | self._event_loop.__exit__(exc_type, exc_val, exc_tb) 23 | -------------------------------------------------------------------------------- /botoflow/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | """**INTERNAL** 15 | """ 16 | import re 17 | import hashlib 18 | import time 19 | import random 20 | 21 | from collections import namedtuple 22 | 23 | import six 24 | 25 | from .workflow_definition import WorkflowDefinition 26 | 27 | WorkflowDetails = namedtuple('WorkflowDetails', 28 | 'name version skip_registration ' 29 | 'registration_options') 30 | 31 | 32 | # noinspection PyPep8Naming 33 | def str_or_NONE(value): 34 | """If the `value` is `None` returns a "NONE"", 35 | otherwise returns `str(value)` 36 | """ 37 | if value is None: 38 | return 'NONE' 39 | return str(value) 40 | 41 | 42 | def random_sha1_hash(): 43 | hash_ = hashlib.sha1() 44 | hash_.update(six.b(str(time.time()))) 45 | hash_.update(six.b(str(random.random()))) 46 | return hash_.hexdigest() 47 | 48 | 49 | def extract_workflow_details_from_class(cls): 50 | workflow_options = cls.swf_options['workflow'] 51 | if workflow_options['version'] is None: 52 | raise AttributeError("workflow version must be set") 53 | 54 | name = workflow_options['name'] 55 | version = workflow_options['version'] 56 | skip_registration = workflow_options['skip_registration'] 57 | 58 | # generate a default workflow name if one is missing 59 | if workflow_options['name'] is None: 60 | name = cls.__name__ 61 | 62 | return WorkflowDetails(name, version, skip_registration, 63 | cls.swf_options['workflow_registration_options']) 64 | 65 | 66 | def pairwise(iterable): 67 | """ 68 | from the itertools recipes 69 | s -> (s0,s1), (s1,s2), (s2, s3), (s3, None)... 70 | """ 71 | a = next(iterable) 72 | b = None 73 | while True: 74 | b = None 75 | try: 76 | b = next(iterable) 77 | except StopIteration: 78 | break 79 | yield a, b 80 | a = b 81 | yield a, b 82 | 83 | 84 | def extract_workflows_dict(workflow_definitions): 85 | workflows = dict() 86 | for workflow_definition in workflow_definitions: 87 | if not issubclass(workflow_definition, WorkflowDefinition): 88 | raise TypeError("workflow parameter must be an subclass of the " 89 | "WorkflowDefinition") 90 | 91 | # extract activities info from the class 92 | for workflow_type, func_name in six.iteritems(workflow_definition._workflow_types): 93 | 94 | namever = (workflow_type.name, workflow_type.version) 95 | workflows[namever] = (workflow_definition, workflow_type, 96 | func_name) 97 | 98 | return workflows 99 | 100 | 101 | # regex for translating kwarg case 102 | _first_cap_replace = re.compile(r'(.)([A-Z][a-z]+)') 103 | _remainder_cap_replace = re.compile(r'([a-z0-9])([A-Z])') 104 | 105 | 106 | def camel_keys_to_snake_case(dictionary): 107 | """ 108 | Translate a dictionary containing camelCase keys into dictionary with 109 | snake_case keys that match python kwargs well. 110 | """ 111 | output = {} 112 | for original_key in dictionary.keys(): 113 | # insert an underscore before any word beginning with a capital followed by lower case 114 | translated_key = _first_cap_replace.sub(r'\1_\2', original_key) 115 | # insert an underscore before any remaining capitals that follow lower case characters 116 | translated_key = _remainder_cap_replace.sub(r'\1_\2', translated_key).lower() 117 | output[translated_key] = dictionary[original_key] 118 | return output 119 | 120 | 121 | def snake_keys_to_camel_case(dictionary): 122 | """ 123 | Translate a dictionary containing snake_case keys into dictionary with 124 | camelCase keys as required for decision dicts. 125 | """ 126 | output = {} 127 | for original_key in dictionary.keys(): 128 | components = original_key.split('_') 129 | translated_key = components[0] + ''.join([component.title() for component in components[1:]]) 130 | output[translated_key] = dictionary[original_key] 131 | return output 132 | -------------------------------------------------------------------------------- /botoflow/workers/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | from .workflow_worker import WorkflowWorker, GenericWorkflowWorker 15 | from .activity_worker import ActivityWorker 16 | from .threaded_workflow_executor import ThreadedWorkflowExecutor 17 | from .threaded_activity_executor import ThreadedActivityExecutor 18 | from .multiprocessing_workflow_executor import MultiprocessingWorkflowExecutor 19 | from .multiprocessing_activity_executor import MultiprocessingActivityExecutor 20 | -------------------------------------------------------------------------------- /botoflow/workers/activity_task.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | from ..workflow_execution import WorkflowExecution 15 | 16 | 17 | class ActivityTask(object): 18 | """**INTERNAL** 19 | Unit of work sent to an activity worker. 20 | """ 21 | 22 | def __init__(self, task_dict): 23 | """ 24 | Used to construct the object from SWF task dictionary 25 | """ 26 | self.id = task_dict['activityId'] 27 | self.name = task_dict['activityType']['name'] 28 | self.version = task_dict['activityType']['version'] 29 | self.input = task_dict['input'] 30 | self.started_event_id = task_dict['startedEventId'] 31 | self.token = task_dict['taskToken'] 32 | self.workflow_execution = WorkflowExecution( 33 | task_dict['workflowExecution']['workflowId'], 34 | task_dict['workflowExecution']['runId']) 35 | 36 | # TODO implement __repr__ 37 | -------------------------------------------------------------------------------- /botoflow/workers/base_worker.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | import os 15 | import socket 16 | import threading 17 | import logging 18 | 19 | from copy import copy 20 | 21 | from botocore.session import Session 22 | 23 | from ..core import async_traceback 24 | 25 | log = logging.getLogger(__name__) 26 | 27 | 28 | class BaseWorker(object): 29 | """ 30 | Base for the Workflow and Activity workers 31 | """ 32 | 33 | def __init__(self, session, aws_region, domain, task_list): 34 | if not isinstance(session, Session): 35 | raise TypeError("session must be an instance " 36 | "of botocore.session.Session") 37 | 38 | self._identity = None 39 | self._session = session 40 | self._aws_region = aws_region 41 | 42 | self._domain = domain 43 | self._task_list = task_list 44 | 45 | # set user agent to botoflow 46 | # import here to avoid import cycles 47 | from .. import __version__ 48 | session.user_agent_name = 'botoflow' 49 | session.user_agent_version = __version__ 50 | self._fix_endpoint() 51 | 52 | def __repr__(self): 53 | return "<%s at %s domain=%s task_list=%s>" % ( 54 | self.__class__.__name__, hex(id(self)), self.domain, 55 | self.task_list) 56 | 57 | def _fix_endpoint(self): 58 | timeout = self.client._endpoint.timeout 59 | 60 | # newer versions of botocore create a timeout tuple of the form (connect_timeout, read_timeout) 61 | # older versions just use a scalar int 62 | if isinstance(timeout, tuple) and timeout[1] < 65: 63 | self.client._endpoint.timeout = (timeout[0], 65) 64 | elif not isinstance(timeout, tuple) and timeout < 65: 65 | self.client._endpoint.timeout = 65 66 | 67 | def __setstate__(self, dct): 68 | self.__dict__ = dct 69 | self._fix_endpoint() 70 | 71 | def __getstate__(self): 72 | dct = copy(self.__dict__) 73 | try: 74 | del dct['_client'] # for pickling 75 | except KeyError: 76 | pass 77 | return dct 78 | 79 | @property 80 | def client(self): 81 | """Returns the botocore SWF client 82 | :rtype: botocore.client.swf 83 | """ 84 | try: 85 | return self._client 86 | except AttributeError: # create a new client 87 | self._client = self._session.create_client( 88 | service_name='swf', region_name=self._aws_region) 89 | return self._client 90 | 91 | @property 92 | def domain(self): 93 | """Returns the worker's domain""" 94 | return self._domain 95 | 96 | @property 97 | def task_list(self): 98 | """Returns the task list""" 99 | return self._task_list 100 | 101 | @property 102 | def unhandled_exception_handler(self): 103 | """Returns the current unhandled exception handler. 104 | 105 | Handler notified about poll request and other unexpected failures. The 106 | default implementation logs the failures using ERROR level. 107 | """ 108 | return self._unhandled_exception_handler 109 | 110 | @unhandled_exception_handler.setter 111 | def unhandled_exception_handler(self, func): 112 | self._unhandled_exception_handler = func 113 | 114 | @property 115 | def identity(self): 116 | """Returns the worker's worker's identity 117 | 118 | This value ends up stored in the identity field of the corresponding 119 | Start history event. Default is "pid":"host". 120 | """ 121 | if self._identity is None: 122 | self._identity = "%d:%s" % (os.getpid(), socket.gethostname()) 123 | return self._identity 124 | 125 | @identity.setter 126 | def identity(self, value): 127 | self._identity = value 128 | 129 | def run(self): 130 | """Should be implemented by the worker 131 | 132 | :raises: NotImplementedError 133 | """ 134 | raise NotImplementedError() 135 | 136 | def run_once(self): 137 | """Should be implemented by the worker 138 | 139 | :raises: NotImplementedError 140 | """ 141 | raise NotImplementedError() 142 | 143 | @staticmethod 144 | def _unhandled_exception_handler(exc, tb_list): 145 | """Handler notified about poll request and other unexpected failures. 146 | 147 | This default implementation logs the failures using ERROR level. 148 | """ 149 | thread_name = threading.current_thread().name 150 | tb_str = async_traceback.format_exc(None, exc, tb_list) 151 | log.error("Unhandled exception raised in thread %s:\n%s", 152 | thread_name, "\n".join(tb_str)) 153 | -------------------------------------------------------------------------------- /botoflow/workers/multiprocessing_activity_executor.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | import sys 15 | import traceback 16 | import multiprocessing 17 | import signal 18 | import logging 19 | 20 | import dill 21 | 22 | from .multiprocessing_executor import MultiprocessingExecutor 23 | 24 | log = logging.getLogger(__name__) 25 | 26 | 27 | class MultiprocessingActivityExecutor(MultiprocessingExecutor): 28 | """This is an executor for :py:class:`~.ActivityWorker` that uses multiple processes to 29 | parallelize the activity work. 30 | 31 | """ 32 | 33 | def start(self, pollers=1, workers=1): 34 | """Start the worker. This method does not block. 35 | 36 | :param int pollers: Count of poller processes to use. Must be equal or 37 | less than the `workers` attribute. 38 | :param int workers: Count of worker processes to use. 39 | """ 40 | if pollers < 1: 41 | raise ValueError("pollers count must be greater than 0") 42 | if workers < 1: 43 | raise ValueError("workers count must be greater than 0") 44 | if pollers > workers: 45 | raise ValueError("pollers must be less or equal to " 46 | "workers") 47 | 48 | super(MultiprocessingActivityExecutor, self).start() 49 | 50 | # we use this semaphore to ensure we have at most poller_tasks running 51 | poller_semaphore = self._process_manager().Semaphore(pollers) 52 | 53 | def run_poller_worker_with_exc(executor_pickle): 54 | executor = dill.loads(executor_pickle) 55 | try: 56 | executor._process_queue.get() 57 | # ignore any SIGINT, so it looks closer to threading 58 | signal.signal(signal.SIGINT, signal.SIG_IGN) 59 | run_poller_worker(executor) 60 | except Exception as err: 61 | _, _, tb = sys.exc_info() 62 | tb_list = traceback.extract_tb(tb) 63 | handler = executor._worker.unhandled_exception_handler 64 | handler(err, tb_list) 65 | finally: 66 | process = multiprocessing.current_process() 67 | log.debug("Poller/executor %s terminating", process.name) 68 | executor._process_queue.task_done() 69 | 70 | def run_poller_worker(executor): 71 | process = multiprocessing.current_process() 72 | log.debug("Poller/executor %s started", process.name) 73 | initializer = self.initializer 74 | initializer(executor) 75 | while executor._worker_shutdown.empty(): 76 | work_callable = None 77 | with poller_semaphore: 78 | 79 | while work_callable is None: 80 | # make sure that after we wake up we're still relevant 81 | if not executor._worker_shutdown.empty(): 82 | return 83 | try: 84 | work_callable = executor._worker.poll_for_activities() 85 | except Exception as err: 86 | _, _, tb = sys.exc_info() 87 | tb_list = traceback.extract_tb(tb) 88 | handler = executor._worker.unhandled_exception_handler 89 | handler(err, tb_list) 90 | 91 | try: 92 | work_callable() 93 | except Exception as err: 94 | _, _, tb = sys.exc_info() 95 | tb_list = traceback.extract_tb(tb) 96 | handler = executor._worker.unhandled_exception_handler 97 | handler(err, tb_list) 98 | 99 | for i in range(workers): 100 | process = multiprocessing.Process( 101 | target=run_poller_worker_with_exc, args=(dill.dumps(self),)) 102 | self._process_queue.put(i) 103 | process.daemon = True 104 | process.name = "%r Process-%d" % (self, i) 105 | process.start() 106 | -------------------------------------------------------------------------------- /botoflow/workers/multiprocessing_executor.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | import sys 15 | import multiprocessing 16 | import multiprocessing.managers 17 | import signal 18 | import logging 19 | 20 | import six 21 | 22 | 23 | log = logging.getLogger(__name__) 24 | 25 | 26 | def _manager_initializer(): 27 | signal.signal(signal.SIGINT, signal.SIG_IGN) 28 | 29 | 30 | class _ProcessManager(object): 31 | def __init__(self): 32 | self._process_manager = None 33 | 34 | def process_manager(self): 35 | if self._process_manager is None: 36 | self._process_manager = multiprocessing.managers.SyncManager() 37 | self._process_manager.start(_manager_initializer) 38 | return self._process_manager 39 | 40 | process_manager = _ProcessManager().process_manager 41 | 42 | 43 | class MultiprocessingExecutor(object): 44 | """A base for all multiprocessing executors""" 45 | 46 | def __init__(self, worker): 47 | self._worker = worker 48 | 49 | def start(self): 50 | """Start the worker. This method does not block.""" 51 | log.debug("Starting worker %s", self) 52 | 53 | # use this simple queue (and .empty) to check if we should shut down 54 | self._worker_shutdown = self._process_manager().Queue() 55 | # to track the active processes 56 | self._process_queue = self._process_manager().JoinableQueue() 57 | self._running = True 58 | 59 | def stop(self): 60 | """Stops the worker processes. 61 | To wait for all the processes to terminate, call: 62 | 63 | .. code-block:: python 64 | 65 | worker.start() 66 | time.sleep(360) 67 | worker.stop() 68 | worker.join() # will block 69 | 70 | """ 71 | log.debug("Stopping worker %s", self) 72 | if not self.is_running: 73 | return False 74 | 75 | self._worker_shutdown.put(1) 76 | 77 | def join(self): 78 | """Will wait till all the processes are terminated 79 | """ 80 | try: 81 | self._process_queue.join() 82 | except KeyboardInterrupt: 83 | six.print_("\nTerminating, please wait...", file=sys.stderr) 84 | self._process_queue.join() 85 | self._running = False 86 | 87 | @property 88 | def initializer(self): 89 | """If set, the initializer function will be called after the subprocess 90 | is started with the worker object as the first argument. 91 | 92 | You can use this to, for example, set the process name suffix, to 93 | distinguish between activity and workflow workers (when starting them 94 | from the same process): 95 | 96 | .. code-block:: python 97 | 98 | from setproctitle import getproctitle, setproctitle 99 | 100 | def set_worker_title(worker): 101 | name = getproctitle() 102 | if isinstance(worker, WorkflowWorker): 103 | setproctitle(name + ' (WorkflowWorker)') 104 | elif isinstance(worker, ActivityWorker): 105 | setproctitle(name + ' (ActivityWorker)') 106 | 107 | worker.initializer = set_worker_title 108 | """ 109 | try: 110 | return self.__initializer 111 | except AttributeError: 112 | return lambda obj: None 113 | 114 | @initializer.setter 115 | def initializer(self, func): 116 | self.__initializer = func 117 | 118 | @property 119 | def is_running(self): 120 | """Returns True if the worker is running""" 121 | if hasattr(self, '_running'): 122 | return self._running 123 | return False 124 | 125 | @classmethod 126 | def _process_manager(cls): 127 | return process_manager() 128 | -------------------------------------------------------------------------------- /botoflow/workers/multiprocessing_workflow_executor.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | import multiprocessing 15 | import multiprocessing.queues 16 | import signal 17 | import logging 18 | 19 | import dill 20 | 21 | from ..core import async_traceback 22 | 23 | from .multiprocessing_executor import MultiprocessingExecutor 24 | 25 | log = logging.getLogger(__name__) 26 | 27 | 28 | class MultiprocessingWorkflowExecutor(MultiprocessingExecutor): 29 | """This is a multiprocessing workflow executor, suitable for handling lots of 30 | workflow decisions in parallel on CPython. 31 | """ 32 | 33 | def start(self, pollers=1): 34 | """Start the worker. 35 | 36 | :param int pollers: Poller/worker count to start. Because the expected 37 | lifetime of the decider is short (should be seconds at most), we 38 | don't need a separate worker queue. 39 | 40 | Example of starting and terminating the worker: 41 | 42 | .. code-block:: python 43 | 44 | worker.start(pollers=10) 45 | time.sleep(360) 46 | worker.stop() 47 | worker.join() # will block 48 | 49 | """ 50 | if pollers < 1: 51 | raise ValueError("pollers count must be greater than 0") 52 | 53 | super(MultiprocessingWorkflowExecutor, self).start() 54 | 55 | start_condition = self._process_manager().Condition() 56 | 57 | def run_decider(executor): 58 | executor._process_queue.get() 59 | # ignore any SIGINT, so it looks closer to threading 60 | signal.signal(signal.SIGINT, signal.SIG_IGN) 61 | 62 | process = multiprocessing.current_process() 63 | log.debug("Poller/decider %s started", process.name) 64 | 65 | while executor._worker_shutdown.empty(): 66 | with start_condition: 67 | start_condition.notify_all() 68 | try: 69 | executor._worker.run_once() 70 | 71 | except Exception as err: 72 | tb_list = async_traceback.extract_tb() 73 | handler = executor._worker.unhandled_exception_handler 74 | handler(err, tb_list) 75 | 76 | def run_decider_with_exc(executor_pickle): 77 | executor = dill.loads(executor_pickle) 78 | initializer = self.initializer 79 | initializer(executor) 80 | try: 81 | run_decider(executor) 82 | except Exception as err: 83 | tb_list = async_traceback.extract_tb() 84 | handler = executor._worker.unhandled_exception_handler 85 | handler(err, tb_list) 86 | finally: 87 | process = multiprocessing.current_process() 88 | log.debug("Poller/decider %s terminating", process.name) 89 | executor._process_queue.task_done() 90 | 91 | for i in range(pollers): 92 | with start_condition: 93 | self._process_queue.put(i) 94 | process = multiprocessing.Process(target=run_decider_with_exc, 95 | args=(dill.dumps(self),)) 96 | process.daemon = True 97 | process.name = "%r Process-%d" % (self, i) 98 | process.start() 99 | # wait for the process to "ready" before starting next one 100 | # or returning 101 | start_condition.wait() 102 | -------------------------------------------------------------------------------- /botoflow/workers/swf_op_callable.py: -------------------------------------------------------------------------------- 1 | from .. import swf_exceptions 2 | 3 | 4 | class SWFOp(object): 5 | """Callable wrapper for SWF Operations that inspects the replies and raises appropriate 6 | :py:mod:`botoflow.swf_exceptions`. 7 | """ 8 | 9 | def __init__(self, endpoint, op): 10 | self.endpoint = endpoint 11 | self.op = op 12 | 13 | def __call__(self, **kwargs): 14 | response, response_data = self.op.call(self.endpoint, **kwargs) 15 | if response.ok: 16 | return response_data 17 | 18 | exception = swf_exceptions.SWFResponseError 19 | 20 | if 'Errors' in response_data: 21 | _type = response_data['Errors'][0]['Type'] 22 | if _type in swf_exceptions._swf_fault_exception: 23 | exception = swf_exceptions._swf_fault_exception[_type] 24 | 25 | if exception == swf_exceptions.SWFResponseError: 26 | error = exception(response_data.get('message'), 27 | 'No error provided by SWF: {0}' 28 | .format(response_data)) 29 | else: 30 | error = exception(response_data.get('message'), 31 | response_data) 32 | raise error # exception from SWF Service 33 | -------------------------------------------------------------------------------- /botoflow/workers/threaded_activity_executor.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | import sys 15 | import threading 16 | import traceback 17 | import logging 18 | 19 | from .threaded_executor import ThreadedExecutor 20 | 21 | log = logging.getLogger(__name__) 22 | 23 | 24 | class ThreadedActivityExecutor (ThreadedExecutor): 25 | """This is an executor for :py:class:`~.ActivityWorker` that uses threads to parallelize 26 | the activity work. 27 | 28 | Because of the GIL in CPython, it is recomended to use this worker only on 29 | Jython or IronPython. 30 | """ 31 | 32 | def start(self, pollers=1, workers=1): 33 | """Start the worker. This method does not block. 34 | 35 | :param int pollers: Count of poller threads to use. Must be equal or 36 | less than the `workers` attribute. 37 | :param int workers: Count of worker threads to use. 38 | """ 39 | if pollers < 1: 40 | raise ValueError("poller_threads count must be greater than 0") 41 | if workers < 1: 42 | raise ValueError("worker_threads count must be greater than 0") 43 | if pollers > workers: 44 | raise ValueError("poller_thread must be less or equal to " 45 | "worker_threads") 46 | 47 | super(ThreadedActivityExecutor, self).start() 48 | 49 | # we use this semaphore to ensure we have at most poller_tasks running 50 | poller_semaphore = threading.Semaphore(pollers) 51 | 52 | # noinspection PyShadowingNames 53 | def run_poller_worker(self): 54 | self._thread_queue.get() 55 | # noinspection PyShadowingNames 56 | thread = threading.current_thread() 57 | log.debug("Poller/worker %s started", thread.name) 58 | try: 59 | while not self._worker_shutdown: 60 | work_callable = None 61 | with poller_semaphore: 62 | 63 | while work_callable is None: 64 | # make sure that after we wake up we're still 65 | # relevant 66 | if self._worker_shutdown: 67 | return 68 | work_callable = self._worker.poll_for_activities() 69 | work_callable() 70 | 71 | except Exception as err: 72 | _, _, tb = sys.exc_info() 73 | tb_list = traceback.extract_tb(tb) 74 | handler = self._worker.unhandled_exception_handler 75 | handler(err, tb_list) 76 | finally: 77 | log.debug("Poller/worker %s terminating", thread.name) 78 | self._thread_queue.task_done() 79 | 80 | for i in range(workers): 81 | self._thread_queue.put(i) 82 | thread = threading.Thread(target=run_poller_worker, args=(self,)) 83 | thread.daemon = True 84 | thread.name = "%r Thread-%d" % (self, i) 85 | thread.start() 86 | -------------------------------------------------------------------------------- /botoflow/workers/threaded_executor.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | from six.moves import queue 15 | import logging 16 | 17 | log = logging.getLogger(__name__) 18 | 19 | 20 | class ThreadedExecutor(object): 21 | """This will execute a worker using multiple threads.""" 22 | 23 | def __init__(self, worker): 24 | self._worker = worker 25 | 26 | def start(self): 27 | """Start the worker. This method does not block.""" 28 | log.debug("Starting worker %s", self) 29 | 30 | self._worker_shutdown = False # when True will indicate shutdown 31 | self._thread_queue = queue.Queue() # to track the active threads 32 | self._running = True 33 | 34 | def stop(self): 35 | """Stops the worker threads. 36 | To wait for all the threads to terminate, call: 37 | 38 | .. code-block:: python 39 | 40 | worker.start() 41 | time.sleep(360) 42 | worker.stop() 43 | worker.join() # will block 44 | 45 | """ 46 | log.debug("Stopping worker %s", self) 47 | if not self.is_running: 48 | return False 49 | 50 | self._worker_shutdown = True 51 | 52 | def join(self): 53 | """Will wait till all the threads are terminated 54 | """ 55 | self._thread_queue.join() 56 | self._running = False 57 | 58 | @property 59 | def initializer(self): 60 | """If set, the initializer function will be called after the thread 61 | is started with the worker object as the first argument. 62 | """ 63 | try: 64 | return self.__initializer 65 | except AttributeError: 66 | return lambda obj: None 67 | 68 | @initializer.setter 69 | def initializer(self, func): 70 | self.__initializer = func 71 | 72 | @property 73 | def is_running(self): 74 | """Returns True if the worker is running""" 75 | if hasattr(self, '_running'): 76 | return self._running 77 | return False 78 | -------------------------------------------------------------------------------- /botoflow/workers/threaded_workflow_executor.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | import threading 15 | import logging 16 | 17 | from ..core import async_traceback 18 | 19 | from .threaded_executor import ThreadedExecutor 20 | 21 | log = logging.getLogger(__name__) 22 | 23 | 24 | class ThreadedWorkflowExecutor (ThreadedExecutor): 25 | """This is a threaded workflow executor. 26 | 27 | It will execute a :py:class:`~WorkflowWorker` in multiple threads. 28 | 29 | As in the case with the :py:class:`~.ThreadedActivityWorker` it is not 30 | recomended to use it on CPython because of the GIL unless the poller/worker 31 | count is low (less than 5). 32 | """ 33 | 34 | def start(self, pollers=1): 35 | """Start the worker. 36 | 37 | :param int pollers: Poller/worker count to start. Because the expected 38 | lifetime of the decider is short (should be seconds at most), we 39 | don't need a separate worker queue. 40 | 41 | Example of starting and terminating the worker: 42 | 43 | .. code-block:: python 44 | 45 | worker.start(pollers=2) 46 | time.sleep(360) 47 | worker.stop() 48 | worker.join() # will block 49 | 50 | """ 51 | if pollers < 1: 52 | raise ValueError("poller_threads count must be greater than 0") 53 | 54 | super(ThreadedWorkflowExecutor, self).start() 55 | 56 | start_condition = threading.Condition() 57 | 58 | # noinspection PyShadowingNames 59 | def run_decider(self): 60 | self._thread_queue.get() 61 | thread = threading.current_thread() 62 | log.debug("Poller/decider %s started", thread.name) 63 | initializer = self.initializer 64 | initializer(self) 65 | 66 | try: 67 | while not self._worker_shutdown: 68 | with start_condition: 69 | start_condition.notifyAll() 70 | self._worker.run_once() 71 | except Exception as err: 72 | tb_list = async_traceback.extract_tb() 73 | handler = self._worker.unhandled_exception_handler 74 | handler(err, tb_list) 75 | finally: 76 | log.debug("Poller/decider %s terminating", thread.name) 77 | self._thread_queue.task_done() 78 | 79 | for i in range(pollers): 80 | with start_condition: 81 | self._thread_queue.put(i) 82 | thread = threading.Thread(target=run_decider, args=(self,)) 83 | thread.daemon = True 84 | thread.name = "%r Thread-%d" % (self, i) 85 | thread.start() 86 | # wait for the thread to "ready" before starting next one 87 | # or returning 88 | start_condition.wait() 89 | -------------------------------------------------------------------------------- /botoflow/workflow_execution.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | 14 | from collections import namedtuple 15 | 16 | 17 | class WorkflowExecution(namedtuple('WorkflowExecution', 'workflow_id run_id')): 18 | """Contains workflow execution information provided by SWF. 19 | 20 | .. py:attribute:: workflow_id 21 | 22 | Either provided or randomly generated Workflow ID. There can only be one workflow running with the same 23 | Workflow ID. 24 | 25 | :rtype: str 26 | 27 | .. py:attribute:: run_id 28 | 29 | SWF generated and provided Run ID associated with a particular workflow execution 30 | 31 | :rtype: str 32 | 33 | """ 34 | 35 | 36 | def workflow_execution_from_swf_event(event): 37 | attributes = event.attributes 38 | if 'workflowExecution' in attributes: 39 | attributes = attributes['workflowExecution'] 40 | return WorkflowExecution(attributes['workflowId'], attributes['runId']) 41 | -------------------------------------------------------------------------------- /botoflow/workflow_time.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). 4 | # You may not use this file except in compliance with the License. 5 | # A copy of the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0 8 | # 9 | # or in the "license" file accompanying this file. This file is distributed 10 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | # express or implied. See the License for the specific language governing 12 | # permissions and limitations under the License. 13 | """ 14 | It is important that :py:mod:`~botofolow.workflow_time` functions be used isntead of the regular ``time`` within the decider. 15 | The decider requires deterministic ``time`` and ``sleep`` related to the workflow execution. 16 | """ 17 | 18 | from time import mktime 19 | 20 | from .context import get_context, DecisionContext 21 | 22 | __all__ = ('time', 'sleep', 'is_replaying') 23 | 24 | 25 | def time(): 26 | """ 27 | time() -> integer 28 | 29 | Return the current time in seconds since the Epoch. 30 | Fractions of a second will not be presented as in the :py:func:`time.time`. 31 | 32 | .. code-block:: python 33 | 34 | 35 | from botoflow import coroutine 36 | from botoflow.workflow_time import time 37 | 38 | ... 39 | 40 | @coroutine 41 | def run_after(self, when): 42 | if time() > when: 43 | yield Activities.some_activity() 44 | 45 | :raises TypeError: If the function is called not in DecisionContext 46 | :returns: Returns the workflow's time in seconds since epoch. 47 | :rtype: int 48 | """ 49 | try: 50 | context = get_context() 51 | if isinstance(context, DecisionContext): 52 | return int(mktime(context._workflow_time.timetuple())) 53 | except AttributeError: 54 | pass 55 | raise TypeError("workflow_time.time() should be run inside of a workflow") 56 | 57 | 58 | def sleep(seconds): 59 | """ 60 | Value that becomes ready after the specified delay. 61 | It acts like time.sleep() if used together with a yield. 62 | 63 | .. code-block:: python 64 | 65 | from botoflow import coroutine 66 | from botoflow.workflow_time import sleep 67 | 68 | ... 69 | 70 | @coroutine 71 | def sleeping(self, time_to_sleep): 72 | yield sleep(time_to_sleep) 73 | 74 | @coroutine 75 | def manual_timeout(self, time_to_sleep): 76 | # *for illustration purposes*, you should prefer using activity start_to_close timeout instead 77 | activity_future = Activities.long_activity() 78 | yield sleep(time_to_sleep) 79 | return activity_future.done() 80 | 81 | 82 | :raises TypeError: If the function is called not in DecisionContext 83 | :raises botoflow.core.exceptions.CancelledError: If the timer/sleep was cancelled 84 | :returns: Future representing the timer 85 | :rtype: botoflow.core.future.Future 86 | """ 87 | try: 88 | context = get_context() 89 | if not isinstance(context, DecisionContext): 90 | raise AttributeError() 91 | except AttributeError: 92 | raise TypeError("flow_time.Timer() should be run inside of a " 93 | "workflow") 94 | 95 | decider = context.decider 96 | return decider.handle_execute_timer(seconds) 97 | 98 | 99 | def is_replaying(): 100 | """Indicates if the workflow is currently replaying (True) or generating 101 | (False) new decisions. 102 | 103 | This could be useful for filtering out logs for transitions that have 104 | already completed. See: :py:class:`~botoflow.logging_filters.BotoflowFilter`. 105 | 106 | :returns: True if the current state in the workflow being replayed. 107 | :rtype: bool 108 | :raises TypeError: If the method is called not in the DecisionContext. 109 | """ 110 | try: 111 | context = get_context() 112 | if isinstance(context, DecisionContext): 113 | return context._replaying 114 | except AttributeError: 115 | pass 116 | raise TypeError("workflow_time.time() should be run inside of a workflow") 117 | -------------------------------------------------------------------------------- /circle.yml: -------------------------------------------------------------------------------- 1 | dependencies: 2 | override: 3 | - pip install tox tox-pyenv 4 | - pyenv local 2.7.10 3.4.3 3.5.0 5 | 6 | test: 7 | override: 8 | - tox 9 | 10 | post: 11 | - mkdir -p $CIRCLE_TEST_REPORTS/pytest; cp -av build/xunit/* $CIRCLE_TEST_REPORTS/ 12 | - bash <(curl -s https://codecov.io/bash) -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | codecov: 2 | notify: 3 | require_ci_to_pass: true 4 | comment: 5 | behavior: default 6 | layout: header, diff 7 | require_changes: false 8 | coverage: 9 | precision: 2 10 | range: 11 | - 70.0 12 | - 100.0 13 | round: down 14 | status: 15 | changes: false 16 | patch: true 17 | project: true 18 | parsers: 19 | gcov: 20 | branch_detection: 21 | conditional: true 22 | loop: true 23 | macro: false 24 | method: false 25 | javascript: 26 | enable_partials: false 27 | -------------------------------------------------------------------------------- /docs/source/_static/botoflow_favico.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/boto/botoflow/49d8ed3bc9c57294504be82e933a051e1901b76e/docs/source/_static/botoflow_favico.png -------------------------------------------------------------------------------- /docs/source/_static/li_bullet.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/boto/botoflow/49d8ed3bc9c57294504be82e933a051e1901b76e/docs/source/_static/li_bullet.gif -------------------------------------------------------------------------------- /docs/source/_static/style.css: -------------------------------------------------------------------------------- 1 | h1 { 2 | color: white; 3 | padding: 5px 5px 5px 45px; 4 | margin: 20px 0 35px -60px; 5 | font-size: 28px; 6 | } 7 | 8 | #right-column { 9 | padding: 20px 50px; 10 | } 11 | 12 | .sidebar-toc ul li a:hover { 13 | background-color: #EBB5D3; 14 | color: #fff; 15 | } 16 | 17 | .botoflow-logo { 18 | margin: 15px; 19 | } 20 | 21 | li { 22 | list-style-image: url('li_bullet.gif'); 23 | } 24 | 25 | a:hover { 26 | color: #EBB5D3; 27 | } 28 | 29 | div.body { 30 | padding: 10px 30px 0 30px; 31 | } 32 | 33 | div.sphinxsidebar h3 { 34 | font-family: 'Arial', 'Georgia', serif; 35 | } 36 | 37 | div.sphinxsidebar input { 38 | border: 2px solid #38ec86; 39 | } 40 | 41 | div.sphinxsidebar textarea:focus, div.sphinxsidebar input:focus, div.sphinxsidebar input[type]:focus, div.sphinxsidebar .uneditable-input:focus { 42 | border-color: rgba(56, 236, 134, 0.8); 43 | box-shadow: 0 1px 1px rgba(56, 236, 134, 0.075) inset, 0 0 8px rgba(56, 236, 134, 0.6); 44 | outline: 0 none; 45 | } 46 | 47 | div h1 { 48 | font-family: 'Arial', 'Georgia', serif; 49 | background-color: #666A7A; 50 | } 51 | 52 | div.body h2 { 53 | font-family: 'Arial', 'Georgia', serif; 54 | } 55 | 56 | div.body h3 { 57 | font-family: 'Arial', 'Georgia', serif; 58 | } 59 | 60 | div.body h4 { 61 | font-family: 'Arial', 'Georgia', serif; 62 | } 63 | 64 | div.body h5 { 65 | font-family: 'Arial', 'Georgia', serif; 66 | } 67 | 68 | dt { 69 | font-weight: normal; 70 | font-size: 16px; 71 | margin: 15px 0px 10px 0px; 72 | } 73 | 74 | dt code.descname { 75 | font-weight: bold; 76 | } 77 | 78 | body { 79 | font-size: 15px; 80 | } -------------------------------------------------------------------------------- /docs/source/_static/ul_bullet.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/boto/botoflow/49d8ed3bc9c57294504be82e933a051e1901b76e/docs/source/_static/ul_bullet.png -------------------------------------------------------------------------------- /docs/source/_templates/layout.html: -------------------------------------------------------------------------------- 1 | {% extends "!layout.html" %} 2 | {% block extraheadbboo %} 3 | 4 | 5 | {{ super() }} 6 | {% endblock %} 7 | {% set css_files = css_files + ['_static/style.css'] %} -------------------------------------------------------------------------------- /docs/source/_templates/logo-text.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 |
6 | -------------------------------------------------------------------------------- /docs/source/changelog.rst: -------------------------------------------------------------------------------- 1 | *Generated from:* ``_ 2 | 3 | .. include:: ../../CHANGELOG.rst 4 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. botoflow documentation master file, created by 2 | sphinx-quickstart on Mon Feb 25 16:12:26 2013. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | 7 | About Botoflow 8 | -------------- 9 | 10 | **Botoflow is a programming framework that works together with 11 | Amazon Simple Workflow Service** (Amazon SWF) to help developers build 12 | asynchronous and distributed applications that process work asynchronously and 13 | distribute the processing across components that execute remotely. 14 | 15 | Using the framework, such work can be organized into **discrete units 16 | called tasks**. You can create tasks that are independent of each other 17 | and may execute concurrently. You can also create tasks that depend on 18 | the outcome of other tasks and need to be sequenced. In fact, the 19 | framework allows you to implement complex graphs of tasks by 20 | interconnecting them through control flow. Best of all, the flow of 21 | tasks can be expressed naturally through the control flow of the 22 | application itself, using features of Python that you are already 23 | familiar with. This makes development easy by allowing you to focus on 24 | your application's business logic while the framework takes care of 25 | the mechanics of creating and coordinating tasks. 26 | 27 | Under the hood, the framework is powered by the scheduling, routing, 28 | and state management features of Amazon SWF. The use of Amazon SWF 29 | also makes your applications **scalable, reliable, and 30 | auditable**. Applications written using the framework are highly 31 | concurrent and can be readily distributed across processes and 32 | machines. The framework is ideal for a broad set of use cases such as 33 | business process workflows, media encoding, long running tasks, and 34 | background processing. 35 | 36 | 37 | .. warning:: 38 | 39 | This documentation as well as the underlying APIs are 'work in progress' 40 | (!) and may/will change before the first release. 41 | 42 | 43 | 44 | Overview 45 | -------- 46 | 47 | .. toctree:: 48 | :maxdepth: 2 49 | 50 | overview 51 | 52 | Concepts 53 | -------- 54 | 55 | .. toctree:: 56 | :maxdepth: 2 57 | 58 | concepts 59 | 60 | Setting up Development Environment 61 | ---------------------------------- 62 | 63 | .. toctree:: 64 | :maxdepth: 2 65 | 66 | setting_up_dev_env 67 | 68 | Feature Details 69 | --------------- 70 | 71 | .. toctree:: 72 | :maxdepth: 2 73 | 74 | feature_details 75 | 76 | API Reference 77 | ------------- 78 | 79 | .. toctree:: 80 | :maxdepth: 3 81 | 82 | reference/index 83 | 84 | Internal Reference 85 | ------------------ 86 | 87 | .. toctree:: 88 | :maxdepth: 3 89 | 90 | internal_reference/index 91 | 92 | Change Log 93 | ---------- 94 | 95 | .. toctree:: 96 | :maxdepth: 2 97 | 98 | changelog 99 | 100 | ================== 101 | Indices and tables 102 | ================== 103 | 104 | * :ref:`genindex` 105 | * :ref:`modindex` 106 | * :ref:`search` 107 | 108 | -------------------------------------------------------------------------------- /docs/source/internal_reference/activity_retrying.rst: -------------------------------------------------------------------------------- 1 | Activity Retrying 2 | ================= 3 | 4 | .. automodule:: botoflow.activity_retrying 5 | :members: 6 | :undoc-members: 7 | -------------------------------------------------------------------------------- /docs/source/internal_reference/core.rst: -------------------------------------------------------------------------------- 1 | Asynchronous Framework Core 2 | =========================== 3 | 4 | At the core of botoflow is asynchronous framework utilizing :py:class:`~botoflow.core.future.Future`, similar to the Future described in :pep:`3156`. Since this code was developed before arrival of :pep:`3156` and :py:mod:`asyncio`. 5 | 6 | .. todo:: 7 | 8 | Eventually we will deprecate this approach in favor of using just :py:mod:`asyncio` as the core. This will take some time as we want to have *Python 2.7* support, for as long as the language itself is supported. 9 | 10 | 11 | Core event loop 12 | --------------- 13 | 14 | .. automodule:: botoflow.core.async_event_loop 15 | :members: 16 | 17 | Async Traceback support and manipulation 18 | ---------------------------------------- 19 | 20 | .. automodule:: botoflow.core.async_traceback 21 | :members: 22 | 23 | Decorators 24 | ---------- 25 | 26 | .. automodule:: botoflow.core.decorators 27 | :members: 28 | 29 | Exceptions 30 | ---------- 31 | 32 | .. automodule:: botoflow.core.exceptions 33 | :members: 34 | 35 | Future and the like 36 | ------------------- 37 | 38 | .. automodule:: botoflow.core.future 39 | :members: 40 | -------------------------------------------------------------------------------- /docs/source/internal_reference/decider.rst: -------------------------------------------------------------------------------- 1 | Decider 2 | ======= 3 | 4 | 5 | botoflow.decider.decider 6 | ------------------------ 7 | 8 | .. automodule:: botoflow.decider.decider 9 | :members: 10 | :undoc-members: 11 | 12 | botoflow.decider.decision_task_poller 13 | ------------------------------------- 14 | 15 | .. automodule:: botoflow.decider.decision_task_poller 16 | :members: 17 | :undoc-members: 18 | 19 | botoflow.decider.workflow_replayer 20 | ---------------------------------- 21 | 22 | .. automodule:: botoflow.decider.workflow_replayer 23 | :members: 24 | :undoc-members: 25 | 26 | 27 | -------------------------------------------------------------------------------- /docs/source/internal_reference/decisions.rst: -------------------------------------------------------------------------------- 1 | ========= 2 | Decisions 3 | ========= 4 | 5 | ``decisions`` are classes/encapsulations of decisions we send to SWF. 6 | 7 | Decision Bases 8 | -------------- 9 | 10 | .. automodule:: botoflow.decisions.decision_bases 11 | :members: 12 | :undoc-members: 13 | 14 | Decision List 15 | ------------- 16 | 17 | .. automodule:: botoflow.decisions.decision_list 18 | :members: 19 | :undoc-members: 20 | 21 | Decisions 22 | --------- 23 | 24 | .. automodule:: botoflow.decisions.decisions 25 | :members: 26 | :undoc-members: 27 | 28 | 29 | -------------------------------------------------------------------------------- /docs/source/internal_reference/decorator_descriptors.rst: -------------------------------------------------------------------------------- 1 | Decorator Descriptors 2 | ===================== 3 | 4 | botoflow.decorator_descriptors 5 | ------------------------------ 6 | 7 | .. automodule:: botoflow.decorator_descriptors 8 | :members: 9 | :undoc-members: 10 | -------------------------------------------------------------------------------- /docs/source/internal_reference/flow_types.rst: -------------------------------------------------------------------------------- 1 | Flow Types 2 | ========== 3 | 4 | .. automodule:: botoflow.flow_types 5 | :members: 6 | :undoc-members: 7 | -------------------------------------------------------------------------------- /docs/source/internal_reference/history_events.rst: -------------------------------------------------------------------------------- 1 | History Events 2 | ============== 3 | 4 | 5 | Event Bases 6 | ----------- 7 | 8 | .. automodule:: botoflow.history_events.event_bases 9 | :members: 10 | :undoc-members: 11 | 12 | Events 13 | ------ 14 | 15 | .. automodule:: botoflow.history_events.events 16 | :members: 17 | :undoc-members: 18 | 19 | 20 | -------------------------------------------------------------------------------- /docs/source/internal_reference/index.rst: -------------------------------------------------------------------------------- 1 | Internal Reference 2 | ================== 3 | 4 | .. note:: 5 | 6 | Internal reference documentation for botoflow developers. 7 | 8 | .. toctree:: 9 | :maxdepth: 4 10 | 11 | activity_retrying 12 | core 13 | decider 14 | decisions 15 | decorator_descriptors 16 | flow_types 17 | history_events 18 | utils 19 | workers_internal 20 | -------------------------------------------------------------------------------- /docs/source/internal_reference/utils.rst: -------------------------------------------------------------------------------- 1 | ===== 2 | Utils 3 | ===== 4 | 5 | .. automodule:: botoflow.utils 6 | :members: 7 | :undoc-members: 8 | -------------------------------------------------------------------------------- /docs/source/internal_reference/workers_internal.rst: -------------------------------------------------------------------------------- 1 | Workers and Executors 2 | ===================== 3 | 4 | SWF Operation Callable 5 | ---------------------- 6 | .. automodule:: botoflow.workers.swf_op_callable 7 | :members: 8 | :undoc-members: 9 | 10 | Workers Activity Task 11 | --------------------- 12 | 13 | .. automodule:: botoflow.workers.activity_task 14 | :show-inheritance: 15 | :members: 16 | :undoc-members: 17 | 18 | Base Worker 19 | ----------- 20 | 21 | .. automodule:: botoflow.workers.base_worker 22 | :show-inheritance: 23 | :members: 24 | :undoc-members: 25 | 26 | 27 | Multiprocessing Executor 28 | ------------------------ 29 | 30 | .. automodule:: botoflow.workers.multiprocessing_executor 31 | :show-inheritance: 32 | :members: 33 | :undoc-members: 34 | 35 | 36 | Threaded Executor 37 | ----------------- 38 | 39 | .. automodule:: botoflow.workers.threaded_executor 40 | :show-inheritance: 41 | :members: 42 | :undoc-members: 43 | -------------------------------------------------------------------------------- /docs/source/reference/constants.rst: -------------------------------------------------------------------------------- 1 | ========= 2 | Constants 3 | ========= 4 | 5 | botoflow.constants 6 | ------------------ 7 | 8 | .. automodule:: botoflow.constants 9 | :members: 10 | :undoc-members: 11 | 12 | -------------------------------------------------------------------------------- /docs/source/reference/context.rst: -------------------------------------------------------------------------------- 1 | Decision, Workflow and Activity Context 2 | ======================================= 3 | 4 | Activity Context 5 | ---------------- 6 | 7 | .. automodule:: botoflow.context.activity_context 8 | :members: 9 | 10 | Decision Context 11 | ---------------- 12 | 13 | .. automodule:: botoflow.context.decision_context 14 | :members: 15 | 16 | Start Workflow Context 17 | ---------------------- 18 | 19 | .. automodule:: botoflow.context.start_workflow_context 20 | :members: 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /docs/source/reference/data_converter.rst: -------------------------------------------------------------------------------- 1 | ============== 2 | Data Converter 3 | ============== 4 | 5 | Data converters are used in botoflow to serialize/deserialize Python objects and exceptions between botoflow and Amazon SWF. 6 | 7 | .. note:: 8 | 9 | The naming ``data_converters`` and not ``serde`` or ``encoder/decoder`` was chosen to match closer with the naming in official Flow Java library, but behaves much like :py:func:`json.dumps` and :py:func:`json.loads` in one. 10 | 11 | 12 | 13 | Abstract Data Converter 14 | ----------------------- 15 | 16 | .. automodule:: botoflow.data_converter.abstract_data_converter 17 | :members: 18 | 19 | JSON Data Converter 20 | ------------------- 21 | 22 | .. automodule:: botoflow.data_converter.json_data_converter 23 | :members: 24 | 25 | Pickle Data Converter 26 | --------------------- 27 | 28 | .. automodule:: botoflow.data_converter.pickle_data_converter 29 | :members: 30 | -------------------------------------------------------------------------------- /docs/source/reference/decorators.rst: -------------------------------------------------------------------------------- 1 | Decorators 2 | ========== 3 | 4 | botoflow.decorators 5 | ------------------- 6 | 7 | .. automodule:: botoflow.decorators 8 | :members: 9 | :undoc-members: 10 | -------------------------------------------------------------------------------- /docs/source/reference/exceptions.rst: -------------------------------------------------------------------------------- 1 | ========== 2 | Exceptions 3 | ========== 4 | 5 | Botoflow Exceptions 6 | ------------------- 7 | 8 | .. automodule:: botoflow.exceptions 9 | :members: 10 | :undoc-members: 11 | 12 | 13 | SWF Response Errors/Exceptions 14 | ------------------------------ 15 | 16 | .. automodule:: botoflow.swf_exceptions 17 | :members: 18 | :undoc-members: 19 | -------------------------------------------------------------------------------- /docs/source/reference/index.rst: -------------------------------------------------------------------------------- 1 | ============= 2 | API Reference 3 | ============= 4 | 5 | .. toctree:: 6 | :maxdepth: 4 7 | 8 | constants 9 | context 10 | data_converter 11 | decorators 12 | exceptions 13 | logging_filters 14 | options 15 | workers 16 | workflow_definition 17 | workflow_execution 18 | workflow_starter 19 | workflow_time 20 | -------------------------------------------------------------------------------- /docs/source/reference/logging_filters.rst: -------------------------------------------------------------------------------- 1 | =============== 2 | Logging Filters 3 | =============== 4 | 5 | .. automodule:: botoflow.logging_filters 6 | :members: 7 | :undoc-members: 8 | -------------------------------------------------------------------------------- /docs/source/reference/options.rst: -------------------------------------------------------------------------------- 1 | Options Contexts 2 | ================ 3 | 4 | .. automodule:: botoflow.options 5 | :members: 6 | :undoc-members: 7 | -------------------------------------------------------------------------------- /docs/source/reference/workers.rst: -------------------------------------------------------------------------------- 1 | Workers and Executors 2 | ===================== 3 | 4 | 5 | Activity Worker 6 | --------------- 7 | 8 | .. automodule:: botoflow.workers.activity_worker 9 | :show-inheritance: 10 | :members: 11 | 12 | Workflow Worker 13 | --------------- 14 | 15 | .. automodule:: botoflow.workers.workflow_worker 16 | :show-inheritance: 17 | :members: 18 | 19 | Multiprocessing Activity Executor 20 | --------------------------------- 21 | 22 | .. automodule:: botoflow.workers.multiprocessing_activity_executor 23 | :show-inheritance: 24 | :members: 25 | 26 | Multiprocessing Workflow Executor 27 | --------------------------------- 28 | 29 | .. automodule:: botoflow.workers.multiprocessing_workflow_executor 30 | :show-inheritance: 31 | :members: 32 | 33 | Threaded Activity Executor 34 | -------------------------- 35 | 36 | .. automodule:: botoflow.workers.threaded_activity_executor 37 | :show-inheritance: 38 | :members: 39 | 40 | Threaded Workflow Executor 41 | -------------------------- 42 | 43 | .. automodule:: botoflow.workers.threaded_workflow_executor 44 | :show-inheritance: 45 | :members: 46 | -------------------------------------------------------------------------------- /docs/source/reference/workflow_definition.rst: -------------------------------------------------------------------------------- 1 | Workflow Definition 2 | =================== 3 | 4 | .. automodule:: botoflow.workflow_definition 5 | :show-inheritance: 6 | :members: 7 | :undoc-members: 8 | 9 | -------------------------------------------------------------------------------- /docs/source/reference/workflow_execution.rst: -------------------------------------------------------------------------------- 1 | Workflow Execution 2 | ================== 3 | 4 | .. automodule:: botoflow.workflow_execution 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/source/reference/workflow_starter.rst: -------------------------------------------------------------------------------- 1 | ================ 2 | Workflow Starter 3 | ================ 4 | 5 | .. automodule:: botoflow.workflow_starting 6 | :members: 7 | :undoc-members: 8 | -------------------------------------------------------------------------------- /docs/source/reference/workflow_time.rst: -------------------------------------------------------------------------------- 1 | ============= 2 | Workflow Time 3 | ============= 4 | 5 | .. automodule:: botoflow.workflow_time 6 | :members: 7 | :undoc-members: 8 | -------------------------------------------------------------------------------- /docs/source/setting_up_dev_env.rst: -------------------------------------------------------------------------------- 1 | ====================================== 2 | Setting up the Development Environment 3 | ====================================== 4 | 5 | The following sections provide instructions for setting up your development 6 | environment for the botoflow. 7 | 8 | 9 | Prerequisites 10 | ------------- 11 | 12 | To develop applications that use the botoflow, you will need: 13 | 14 | * A working Python 2.7 or 3.4+ environment. 15 | * `BotoCore `_ (and it's dependencies). 16 | * `Dill `_ (required for multiprocessing executor). 17 | * Botoflow. 18 | * An active AWS account signed up for `Simple Workflow Service `_. 19 | * An IAM user with privileges in SWF is needed. Please refer to `Amazon IAM SWF`_ 20 | for more details, but the inline policy for an existing user could be: 21 | 22 | .. code-block:: sh 23 | 24 | { 25 | "Version": "2012-10-17", 26 | "Statement": [ 27 | { 28 | "Action": [ 29 | "swf:*" 30 | ], 31 | "Effect": "Allow", 32 | "Resource": "*" 33 | } 34 | ] 35 | } 36 | 37 | * Add a SWF ``domain`` in your AWS account. This is possible with `awscli`_: 38 | 39 | .. code-block:: sh 40 | 41 | $ aws swf register-domain \ 42 | --name helloworld \ 43 | --description "Helloworld domain" \ 44 | --workflow-execution-retention-period-in-days 7 \ 45 | --region us-east-1 46 | 47 | **Note:** ``Workflows Types`` and ``Activities Types`` will be automatically registered 48 | when initializing a :py:mod:`~botoflow.workers.workflow_worker` and a :py:mod:`~botoflow.workers.activity_worker` 49 | so there is no need to pre-register these resources. 50 | 51 | 52 | Developing a Workflow 53 | --------------------- 54 | 55 | After you have set up the development environment and configured the AWS account 56 | you can start developing workflows with the botoflow. The typical steps involved in developing 57 | a workflow are as follows: 58 | 59 | #. Define activity and workflow contracts. First, analyze your application 60 | requirements and identify the workflow and activities that are needed to 61 | fulfill them. For example, in a media processing use case, you may need to 62 | download a file, process it, and upload the processed file to an Amazon 63 | Simple Storage Service (S3) bucket. For this application, you may define a 64 | file processing workflow and activities to download the file, perform 65 | processing on it, upload the processed file, and delete files from the local 66 | disk. 67 | #. Implement activities and workflows. The workflow implementation provides the 68 | business logic, while each activity implements a single logical processing 69 | step in the application. The workflow implementation calls the activities. 70 | #. Implement host programs for activity and workflow implementations. After you 71 | have implemented your workflow and activities, you need to create host 72 | programs. A host program is responsible for getting tasks from Amazon SWF 73 | and dispatching them to the appropriate implementation method. AWS Flow 74 | Framework provides worker classes that make implementing these host programs 75 | trivial. 76 | #. Test your workflow. TODO: botoflow does not yet provide nice 77 | testing facilities 78 | #. Deploy the workers. You can now deploy your workers as desired - for 79 | example, you can deploy them to instances in the cloud or in your own data 80 | centers. Once deployed, the workers start polling Amazon SWF for tasks. 81 | #. Start executions. You can start an execution of your workflow from any 82 | program using the workflow definition. You can also use the Amazon SWF 83 | console to start and view workflow executions in your Amazon SWF account. 84 | 85 | 86 | Examples 87 | -------- 88 | 89 | * Helloworld: `examples/helloworld `_ 90 | 91 | .. _awscli: https://aws.amazon.com/cli/ 92 | .. _Amazon IAM SWF: http://docs.aws.amazon.com/amazonswf/latest/developerguide/swf-dev-iam.html 93 | -------------------------------------------------------------------------------- /examples/helloworld/README.rst: -------------------------------------------------------------------------------- 1 | ================== 2 | Helloworld Example 3 | ================== 4 | 5 | **Requires: python3** 6 | 7 | This example creates two activities and one decider for those activities. The first activity will 8 | ask for a name and then the second activity will print that name. 9 | 10 | How To 11 | ------ 12 | 13 | Make sure you have an environment configured: `Setup development environment `_ 14 | 15 | 16 | Configuration 17 | ~~~~~~~~~~~~~ 18 | 19 | Configure the ``config.ini``: 20 | 21 | .. code-block:: sh 22 | 23 | $ cat << EOF > config.ini 24 | [default] 25 | domain = helloworld 26 | tasklist = tasklist1 27 | EOF 28 | 29 | **NOTE:** Take a look at the ``sample_config.ini`` for all options 30 | 31 | 32 | Execute 33 | ~~~~~~~ 34 | 35 | Activate the ``virtualenv`` that has ``botoflow`` installed: 36 | 37 | .. code-block:: sh 38 | 39 | $ . venv/bin/activate 40 | 41 | Now run the example: 42 | 43 | .. code-block:: sh 44 | 45 | $ python helloworld.py 46 | -------------------------------------------------------------------------------- /examples/helloworld/helloworld.py: -------------------------------------------------------------------------------- 1 | import botocore 2 | import configparser 3 | import os.path 4 | import sys 5 | from botoflow import activity, activities, execute, workflow_starter, WorkflowDefinition 6 | from botoflow.workers import workflow_worker, activity_worker 7 | from botoflow.constants import MINUTES 8 | 9 | 10 | @activities(schedule_to_start_timeout=1*MINUTES, 11 | start_to_close_timeout=1*MINUTES) 12 | class HelloWorldActivities(object): 13 | 14 | @activity('1.0') 15 | def get_name(self): 16 | return input('Whats your name: ') 17 | 18 | @activity('1.0') 19 | def print_greeting(self, name): 20 | print("Hello {}!".format(name)) 21 | 22 | 23 | class HelloWorldWorkflow(WorkflowDefinition): 24 | 25 | @execute(version='1.0', execution_start_to_close_timeout=1*MINUTES) 26 | def hello_world(self): 27 | name = yield HelloWorldActivities.get_name() 28 | yield HelloWorldActivities.print_greeting(name) 29 | 30 | 31 | def main(config_filename='config.ini'): 32 | config = configparser.ConfigParser() 33 | if os.path.isfile(config_filename): 34 | config.read(config_filename) 35 | else: 36 | print("Cannot file config file: {}".format(config_filename)) 37 | sys.exit(1) 38 | 39 | PROFILE = config.get('default', 'profile', fallback=None) 40 | REGION = config.get('default', 'region', fallback='us-east-1') 41 | DOMAIN = config.get('default', 'domain', fallback=None) 42 | TASKLIST = config.get('default', 'tasklist', fallback=None) 43 | 44 | if not DOMAIN or not TASKLIST: 45 | print("You must define a domain and tasklist in config.ini") 46 | sys.exit(1) 47 | 48 | if PROFILE: 49 | session = botocore.session.Session(profile=PROFILE) 50 | else: 51 | session = botocore.session.get_session() 52 | 53 | # Create decider 54 | # Registers workflowtype 55 | decider = workflow_worker.WorkflowWorker(session, 56 | REGION, 57 | DOMAIN, 58 | TASKLIST, 59 | HelloWorldWorkflow) 60 | 61 | # Create worker 62 | # Registers activities 63 | worker = activity_worker.ActivityWorker(session, 64 | REGION, 65 | DOMAIN, 66 | TASKLIST, 67 | HelloWorldActivities()) 68 | 69 | # Now that all workflow and activies are registered initialize the 70 | # workflow 71 | with workflow_starter(session, 72 | REGION, 73 | DOMAIN, 74 | TASKLIST): 75 | print("Workflow started") 76 | HelloWorldWorkflow.hello_world() # starts the workflow 77 | 78 | print("Fire decider") 79 | decider.run_once() 80 | 81 | print("Fire worker") 82 | worker.run_once() 83 | 84 | print("Fire decider again") 85 | decider.run_once() 86 | 87 | print("Fire worker again") 88 | worker.run_once() 89 | 90 | print("Fire decider to complete workflow") 91 | decider.run_once() 92 | 93 | 94 | if __name__ == '__main__': 95 | main() 96 | -------------------------------------------------------------------------------- /examples/helloworld/sample.config.ini: -------------------------------------------------------------------------------- 1 | [default] 2 | profile = swfprofile 3 | domain = helloworld 4 | region = us-east-1 5 | tasklist = tasklist1 6 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | addopts = -v --cov-report=xml --cov-report=term-missing --cov=botoflow --ignore=test/integration -------------------------------------------------------------------------------- /requirements-docs.txt: -------------------------------------------------------------------------------- 1 | Sphinx>=1.3 2 | guzzle_sphinx_theme>=0.7.11,<0.8 3 | sphinxcontrib-seqdiag>=0.8,<1.0 4 | sphinxcontrib-blockdiag>=1.5,<2.0 5 | -rrequirements.txt 6 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | tox>=2.3.1,<3.0.0 2 | dill>=0.2 3 | retrying>=1.3.3 4 | nose==1.3.3 5 | mock==1.3.0 6 | pytest==2.9.2 7 | pytest-catchlog>=1.1 8 | pytest-cov>=2.2 9 | coverage>=4 10 | wheel==0.24.0 11 | -e git://github.com/boto/botocore.git@develop#egg=botocore 12 | -------------------------------------------------------------------------------- /scripts/ci/fix-coverage-path: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | import sys 5 | import argparse 6 | import xml.etree.ElementTree as ET 7 | 8 | 9 | def parse_args(argv): 10 | parser = argparse.ArgumentParser() 11 | parser.add_argument('--strip', type=str, required=True, help="path to strip out") 12 | parser.add_argument('xmlpath', nargs=1, help="path to junit.xml") 13 | return parser.parse_args(argv) 14 | 15 | 16 | def main(argv): 17 | args = parse_args(argv) 18 | strip_path = args.strip.split(os.path.sep) 19 | 20 | et = ET.parse(args.xmlpath[0]) 21 | for elem in et.findall('.//*/class'): 22 | file_path = os.path.abspath(elem.get('filename')).split(os.path.sep) 23 | if strip_path == file_path[:len(strip_path)]: 24 | file_path = file_path[len(strip_path):] 25 | 26 | elem.set('filename', os.path.join(*file_path)) 27 | 28 | et.write(args.xmlpath[0]) 29 | return 0 30 | 31 | 32 | if __name__ == '__main__': 33 | sys.exit(main(sys.argv[1:])) 34 | -------------------------------------------------------------------------------- /scripts/ci/install: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | from subprocess import check_call 5 | import shutil 6 | 7 | _dname = os.path.dirname 8 | 9 | REPO_ROOT = _dname(_dname(_dname(os.path.abspath(__file__)))) 10 | os.chdir(REPO_ROOT) 11 | 12 | 13 | def run(command): 14 | return check_call(command, shell=True) 15 | 16 | 17 | try: 18 | # Has the form "major.minor" 19 | python_version = os.environ['PYTHON_VERSION'] 20 | except KeyError: 21 | python_version = '.'.join([str(i) for i in sys.version_info[:2]]) 22 | 23 | 24 | run('pip install -r requirements.txt') 25 | if os.path.isdir('dist') and os.listdir('dist'): 26 | shutil.rmtree('dist') 27 | run('python setup.py bdist_wheel') 28 | wheel_dist = os.listdir('dist')[0] 29 | run('pip install %s' % (os.path.join('dist', wheel_dist))) 30 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [wheel] 2 | universal = 1 3 | 4 | [flake8] 5 | ignore = F401,W293 6 | max-line-length = 120 7 | exclude = test/*,build/*,dist/*,docs/* 8 | max-complexity = 10 -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | import botoflow 4 | 5 | requires = ['botocore>=1.1.10', 6 | 'six>=1.2.0', 7 | 'dill>=0.2', 8 | 'retrying>=1.3.3'] 9 | 10 | args = dict( 11 | name='botoflow', 12 | version=botoflow.__version__, 13 | author='Amazon.com, Darjus Loktevic', 14 | author_email='darjus@gmail.com', 15 | description="Botoflow is an asynchronous framework for Amazon SWF that helps you " 16 | "build SWF applications using Python", 17 | long_description=open('README.rst').read(), 18 | url='https://github.com/boto/botoflow', 19 | packages=find_packages(exclude=['test*']), 20 | scripts=[], 21 | cmdclass={}, 22 | install_requires=requires, 23 | license="Apache License 2.0", 24 | classifiers=( 25 | 'Development Status :: 4 - Beta', 26 | 'Intended Audience :: Developers', 27 | 'Intended Audience :: System Administrators', 28 | 'Natural Language :: English', 29 | 'License :: OSI Approved :: Apache Software License', 30 | 'Programming Language :: Python', 31 | 'Programming Language :: Python :: 2.7', 32 | 'Programming Language :: Python :: 3', 33 | 'Programming Language :: Python :: 3.4', 34 | 'Programming Language :: Python :: 3.5', 35 | ), 36 | ) 37 | 38 | setup(**args) 39 | 40 | -------------------------------------------------------------------------------- /test/integration/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | def pytest_addoption(parser): 4 | parser.addoption('--domain', action='store', default='mydomain2', 5 | help='Domain to use for integration tests') 6 | parser.addoption('--task-list', action='store', default='tasklist', 7 | help='Task list name for integration tests') 8 | parser.addoption('--region', action='store', default='us-east-1', 9 | help='Region for integration tests') 10 | 11 | @pytest.fixture(scope='session') 12 | def integration_test_args(request): 13 | return dict( 14 | domain=request.config.getoption('domain'), 15 | tasklist=request.config.getoption('task_list'), 16 | region=request.config.getoption('region'), 17 | ) 18 | 19 | 20 | -------------------------------------------------------------------------------- /test/integration/multiprocessing_workflows.py: -------------------------------------------------------------------------------- 1 | from botoflow import * 2 | 3 | from various_activities import BunchOfActivities 4 | 5 | 6 | class NoActivitiesWorkflow(WorkflowDefinition): 7 | @execute(version='1.2', execution_start_to_close_timeout=60) 8 | def execute(self, arg1): 9 | return_(arg1) 10 | 11 | class OneActivityWorkflow(WorkflowDefinition): 12 | 13 | @execute(version='1.1', execution_start_to_close_timeout=60) 14 | def execute(self, arg1, arg2): 15 | arg_sum = yield BunchOfActivities.sum(arg1, arg2) 16 | return_(arg_sum) 17 | 18 | 19 | class NoActivitiesFailureWorkflow(WorkflowDefinition): 20 | 21 | @execute(version='1.1', execution_start_to_close_timeout=60) 22 | def execute(self, arg1): 23 | raise RuntimeError("ExecutionFailed") 24 | 25 | 26 | class OneActivityWorkflow(WorkflowDefinition): 27 | 28 | @execute(version='1.1', execution_start_to_close_timeout=60) 29 | def execute(self, arg1, arg2): 30 | arg_sum = yield BunchOfActivities.sum(arg1, arg2) 31 | return_(arg_sum) 32 | 33 | 34 | class OneMultiWorkflow(WorkflowDefinition): 35 | @execute(version='1.2', execution_start_to_close_timeout=60) 36 | def execute(self, arg1, arg2): 37 | arg_sum = yield BunchOfActivities.sum(arg1, arg2) 38 | return_(arg_sum) 39 | 40 | 41 | class TwoMultiWorkflow(WorkflowDefinition): 42 | @execute(version='1.2', execution_start_to_close_timeout=60) 43 | def execute(self, arg1, arg2): 44 | arg_sum = yield BunchOfActivities.sum(arg1, arg2) 45 | return_(arg_sum) 46 | 47 | -------------------------------------------------------------------------------- /test/integration/test_generic_workflows.py: -------------------------------------------------------------------------------- 1 | # -*- mode:python ; fill-column:120 -*- 2 | import time 3 | import unittest 4 | 5 | from botoflow import (WorkflowDefinition, execute, return_, ThreadedActivityExecutor, GenericWorkflowWorker, ActivityWorker, 6 | workflow_starter) 7 | 8 | from botoflow.utils import extract_workflows_dict 9 | from utils import SWFMixIn 10 | from various_activities import BunchOfActivities 11 | 12 | 13 | class WorkflowFinder (object): 14 | def __init__(self, *definitions): 15 | self._workflow_defintions = definitions 16 | 17 | @property 18 | def workflows(self): 19 | return extract_workflows_dict(self._workflow_defintions) 20 | 21 | def __call__(self, name, version): 22 | return self.workflows[(name, version)] 23 | 24 | class TestGenericWorkflows(SWFMixIn, unittest.TestCase): 25 | 26 | def _register_workflows(self, generic_worker): 27 | for _, workflow_type, _ in generic_worker._get_workflow_finder().workflows.values(): 28 | generic_worker._register_workflow_type(workflow_type) 29 | 30 | def test_one_activity(self): 31 | class OneActivityWorkflow(WorkflowDefinition): 32 | def __init__(self, workflow_execution): 33 | super(OneActivityWorkflow, self).__init__(workflow_execution) 34 | self.activities_client = BunchOfActivities() 35 | 36 | @execute(version='1.1', execution_start_to_close_timeout=60) 37 | def execute(self, arg1, arg2): 38 | arg_sum = yield self.activities_client.sum(arg1, arg2) 39 | return_(arg_sum) 40 | 41 | wf_worker = GenericWorkflowWorker( 42 | self.session, self.region, self.domain, self.task_list, WorkflowFinder(OneActivityWorkflow)) 43 | 44 | self._register_workflows(wf_worker) 45 | 46 | act_worker = ThreadedActivityExecutor(ActivityWorker( 47 | self.session, self.region, self.domain, self.task_list, BunchOfActivities())) 48 | 49 | with workflow_starter(self.session, self.region, self.domain, self.task_list): 50 | instance = OneActivityWorkflow.execute(arg1=1, arg2=2) 51 | self.workflow_execution = instance.workflow_execution 52 | 53 | wf_worker.run_once() 54 | act_worker.start(1, 4) 55 | act_worker.stop() 56 | wf_worker.run_once() 57 | act_worker.join() 58 | time.sleep(1) 59 | 60 | hist = self.get_workflow_execution_history() 61 | self.assertEqual(len(hist), 11) 62 | self.assertEqual(hist[-1]['eventType'], 'WorkflowExecutionCompleted') 63 | self.assertEqual(self.serializer.loads( 64 | hist[-1]['workflowExecutionCompletedEventAttributes']['result']), 3) 65 | 66 | 67 | 68 | if __name__ == '__main__': 69 | unittest.main() 70 | -------------------------------------------------------------------------------- /test/integration/test_manual_activities.py: -------------------------------------------------------------------------------- 1 | # -*- mode:python ; fill-column:120 -*- 2 | import time 3 | import unittest 4 | import os 5 | from threading import Thread 6 | 7 | from botoflow import (WorkflowDefinition, execute, return_, 8 | ThreadedActivityExecutor, WorkflowWorker, ActivityWorker, 9 | workflow_starter) 10 | 11 | from botoflow.manual_activity_completion_client import ManualActivityCompletionClient 12 | from utils import SWFMixIn 13 | from various_activities import BunchOfActivities, ManualActivities 14 | 15 | 16 | class TestManualActivities(SWFMixIn, unittest.TestCase): 17 | 18 | def test_one_manual_activity(self): 19 | swf_client = self.client 20 | class OneManualActivityWorkflow(WorkflowDefinition): 21 | def __init__(self, workflow_execution): 22 | super(OneManualActivityWorkflow, self).__init__(workflow_execution) 23 | 24 | @execute(version='1.1', execution_start_to_close_timeout=60) 25 | def execute(self, template): 26 | result = yield ManualActivities.perform_task(template=template) 27 | return_(result) 28 | 29 | wf_worker = WorkflowWorker( 30 | self.session, self.region, self.domain, self.task_list, OneManualActivityWorkflow) 31 | 32 | act_executor = ThreadedActivityExecutor(ActivityWorker( 33 | self.session, self.region, self.domain, self.task_list, ManualActivities())) 34 | 35 | with workflow_starter(self.session, self.region, self.domain, self.task_list): 36 | instance = OneManualActivityWorkflow.execute(template='instructions.tmpl') 37 | self.workflow_execution = instance.workflow_execution 38 | 39 | def complete_this_activity(): 40 | activities_client = ManualActivityCompletionClient(swf_client) 41 | with open('task_token.txt', 'r') as shared_file: 42 | task_token = shared_file.read() 43 | os.remove('task_token.txt') 44 | activities_client.complete('Manual Activity Done', task_token) 45 | 46 | 47 | wf_worker.run_once() 48 | act_executor.start(1, 4) 49 | time.sleep(5) 50 | 51 | activity_finisher = Thread(target=complete_this_activity) 52 | activity_finisher.start() 53 | activity_finisher.join() 54 | 55 | act_executor.stop() 56 | wf_worker.run_once() 57 | act_executor.join() 58 | wf_worker.run_once() 59 | time.sleep(1) 60 | 61 | hist = self.get_workflow_execution_history() 62 | self.assertEqual(len(hist), 11) 63 | self.assertEqual(hist[-1]['eventType'], 'WorkflowExecutionCompleted') 64 | self.assertEqual(self.serializer.loads( 65 | hist[-1]['workflowExecutionCompletedEventAttributes']['result']), 'Manual Activity Done') 66 | 67 | 68 | def test_one_manual_one_automatic_activity(self): 69 | swf_client = self.client 70 | class OneManualOneAutomaticActivityWorkflow(WorkflowDefinition): 71 | def __init__(self, workflow_execution): 72 | super(OneManualOneAutomaticActivityWorkflow, self).__init__(workflow_execution) 73 | 74 | @execute(version='1.1', execution_start_to_close_timeout=60) 75 | def execute(self, template): 76 | (x, y) = yield ManualActivities.perform_task(template=template) 77 | arg_sum = yield BunchOfActivities.sum(x, y) 78 | return_(arg_sum) 79 | 80 | wf_worker = WorkflowWorker( 81 | self.session, self.region, self.domain, self.task_list, OneManualOneAutomaticActivityWorkflow) 82 | 83 | act_worker = ActivityWorker( 84 | self.session, self.region, self.domain, self.task_list, 85 | BunchOfActivities(), ManualActivities()) 86 | 87 | with workflow_starter(self.session, self.region, self.domain, self.task_list): 88 | instance = OneManualOneAutomaticActivityWorkflow.execute(template='instructions.tmpl') 89 | self.workflow_execution = instance.workflow_execution 90 | 91 | def complete_this_activity(): 92 | activities_client = ManualActivityCompletionClient(swf_client) 93 | with open('task_token.txt', 'r') as shared_file: 94 | task_token = shared_file.read() 95 | os.remove('task_token.txt') 96 | activities_client.complete((3,4), task_token) 97 | 98 | wf_worker.run_once() 99 | act_worker.run_once() 100 | 101 | time.sleep(5) 102 | activity_finisher = Thread(target=complete_this_activity) 103 | activity_finisher.start() 104 | activity_finisher.join() 105 | 106 | wf_worker.run_once() 107 | act_worker.run_once() 108 | wf_worker.run_once() 109 | time.sleep(1) 110 | 111 | hist = self.get_workflow_execution_history() 112 | self.assertEqual(len(hist), 17) 113 | self.assertEqual(hist[-1]['eventType'], 'WorkflowExecutionCompleted') 114 | self.assertEqual(self.serializer.loads( 115 | hist[-1]['workflowExecutionCompletedEventAttributes']['result']), 7) 116 | 117 | if __name__ == '__main__': 118 | unittest.main() 119 | -------------------------------------------------------------------------------- /test/integration/test_multi_workflow.py: -------------------------------------------------------------------------------- 1 | import time 2 | import unittest 3 | 4 | from botoflow import WorkflowWorker, ActivityWorker, workflow_starter 5 | from multiprocessing_workflows import OneMultiWorkflow, TwoMultiWorkflow 6 | from various_activities import BunchOfActivities 7 | from utils import SWFMixIn 8 | 9 | 10 | class TestMultiWorkflows(SWFMixIn, unittest.TestCase): 11 | 12 | def test_two_workflows(self): 13 | wf_worker = WorkflowWorker( 14 | self.session, self.region, self.domain, self.task_list, 15 | OneMultiWorkflow, TwoMultiWorkflow) 16 | act_worker = ActivityWorker( 17 | self.session, self.region, self.domain, self.task_list, BunchOfActivities()) 18 | with workflow_starter(self.session, self.region, self.domain, self.task_list): 19 | instance = OneMultiWorkflow.execute(arg1=1, arg2=2) 20 | self.workflow_executions.append(instance.workflow_execution) 21 | instance = TwoMultiWorkflow.execute(arg1=1, arg2=2) 22 | self.workflow_executions.append(instance.workflow_execution) 23 | 24 | for i in range(2): 25 | wf_worker.run_once() 26 | act_worker.run_once() 27 | 28 | wf_worker.run_once() 29 | wf_worker.run_once() 30 | time.sleep(1) 31 | 32 | if __name__ == '__main__': 33 | unittest.main() 34 | -------------------------------------------------------------------------------- /test/integration/test_multiprocessing_workers.py: -------------------------------------------------------------------------------- 1 | import time 2 | import unittest 3 | 4 | 5 | from botoflow import ( 6 | MultiprocessingActivityExecutor, MultiprocessingWorkflowExecutor, workflow_starter, 7 | WorkflowWorker, ActivityWorker) 8 | from multiprocessing_workflows import ( 9 | NoActivitiesWorkflow, NoActivitiesFailureWorkflow, OneActivityWorkflow) 10 | from various_activities import BunchOfActivities 11 | from utils import SWFMixIn 12 | 13 | 14 | class TestMultiprocessingWorkers(SWFMixIn, unittest.TestCase): 15 | 16 | def test_no_activities(self): 17 | 18 | worker = MultiprocessingWorkflowExecutor(WorkflowWorker( 19 | self.session, self.region, self.domain, self.task_list, NoActivitiesWorkflow)) 20 | with workflow_starter(self.session, self.region, self.domain, self.task_list): 21 | instance = NoActivitiesWorkflow.execute(arg1="TestExecution") 22 | self.workflow_execution = instance.workflow_execution 23 | 24 | # start + stop should run the worker's Decider once 25 | worker.start() 26 | worker.stop() 27 | worker.join() 28 | time.sleep(2) 29 | 30 | hist = self.get_workflow_execution_history() 31 | self.assertEqual(len(hist), 5) 32 | self.assertEqual(hist[-1]['eventType'], 'WorkflowExecutionCompleted') 33 | self.assertEqual(self.serializer.loads( 34 | hist[-1]['workflowExecutionCompletedEventAttributes']['result']), 'TestExecution') 35 | 36 | def test_no_activities_failure(self): 37 | 38 | worker = MultiprocessingWorkflowExecutor(WorkflowWorker( 39 | self.session, self.region, self.domain, self.task_list, NoActivitiesFailureWorkflow)) 40 | with workflow_starter(self.session, self.region, self.domain, self.task_list): 41 | instance = NoActivitiesFailureWorkflow.execute(arg1="TestExecution") 42 | self.workflow_execution = instance.workflow_execution 43 | 44 | worker.start() 45 | worker.stop() 46 | worker.join() 47 | time.sleep(1) 48 | 49 | hist = self.get_workflow_execution_history() 50 | self.assertEqual(len(hist), 5) 51 | self.assertEqual(hist[-1]['eventType'], 'WorkflowExecutionFailed') 52 | self.assertEqual(str(self.serializer.loads( 53 | hist[-1]['workflowExecutionFailedEventAttributes']['details'])[0]), 54 | "ExecutionFailed") 55 | 56 | def test_one_activity(self): 57 | wf_worker = MultiprocessingWorkflowExecutor(WorkflowWorker( 58 | self.session, self.region, self.domain, self.task_list, OneActivityWorkflow)) 59 | 60 | act_worker = MultiprocessingActivityExecutor(ActivityWorker( 61 | self.session, self.region, self.domain, self.task_list, BunchOfActivities())) 62 | 63 | with workflow_starter(self.session, self.region, self.domain, self.task_list): 64 | instance = OneActivityWorkflow.execute(arg1=1, arg2=2) 65 | self.workflow_execution = instance.workflow_execution 66 | 67 | wf_worker.start() 68 | act_worker.start() 69 | time.sleep(20) 70 | act_worker.stop() 71 | wf_worker.stop() 72 | act_worker.join() 73 | wf_worker.join() 74 | time.sleep(1) 75 | 76 | hist = self.get_workflow_execution_history() 77 | self.assertEqual(len(hist), 11) 78 | self.assertEqual(hist[-1]['eventType'], 'WorkflowExecutionCompleted') 79 | self.assertEqual(self.serializer.loads( 80 | hist[-1]['workflowExecutionCompletedEventAttributes']['result']), 3) 81 | 82 | if __name__ == '__main__': 83 | unittest.main() 84 | -------------------------------------------------------------------------------- /test/integration/test_retrying_workflows.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import time 3 | import unittest 4 | 5 | from calendar import timegm 6 | 7 | from botoflow import (WorkflowDefinition, execute, return_, coroutine, activity, ThreadedWorkflowExecutor, 8 | ThreadedActivityExecutor, WorkflowWorker, ActivityWorker, activity_options, workflow_time, 9 | flow_types, logging_filters, workflow_starter, workflow, activities, retry_activity, 10 | retry_on_exception, swf_exceptions) 11 | 12 | from botoflow.exceptions import ActivityTaskTimedOutError, ActivityTaskFailedError 13 | from utils import SWFMixIn 14 | 15 | 16 | @activities(schedule_to_start_timeout=60, 17 | start_to_close_timeout=60) 18 | class RetryingActivities(object): 19 | 20 | @retry_activity(stop_max_attempt_number=2, wait_fixed=0.5) 21 | @activity(version='1.0', start_to_close_timeout=1) 22 | def activity_timing_out(self, sleep_time): 23 | return time.sleep(sleep_time) 24 | 25 | @retry_activity(stop_max_attempt_number=2, retry_on_exception=retry_on_exception(RuntimeError)) 26 | @activity(version='1.0', start_to_close_timeout=2) 27 | def activity_raises_errors(self, exception): 28 | raise exception 29 | 30 | 31 | logging.basicConfig(level=logging.DEBUG, 32 | format='%(filename)s:%(lineno)d (%(funcName)s) - %(message)s') 33 | logging.getLogger().addFilter(logging_filters.BotoflowFilter()) 34 | 35 | 36 | class TestRetryingActivitiesWorkflows(SWFMixIn, unittest.TestCase): 37 | 38 | def test_activity_timeouts(self): 39 | class ActivityRetryOnTimeoutWorkflow(WorkflowDefinition): 40 | @execute(version='1.2', execution_start_to_close_timeout=60) 41 | def execute(self, sleep): 42 | try: 43 | yield RetryingActivities.activity_timing_out(sleep) 44 | except ActivityTaskTimedOutError: 45 | pass 46 | 47 | wf_worker = WorkflowWorker( 48 | self.session, self.region, self.domain, self.task_list, ActivityRetryOnTimeoutWorkflow) 49 | 50 | act_worker = ActivityWorker( 51 | self.session, self.region, self.domain, self.task_list, RetryingActivities()) 52 | 53 | with workflow_starter(self.session, self.region, self.domain, self.task_list): 54 | instance = ActivityRetryOnTimeoutWorkflow.execute(sleep=2) 55 | self.workflow_execution = instance.workflow_execution 56 | 57 | for i in range(2): 58 | wf_worker.run_once() 59 | try: 60 | act_worker.run_once() 61 | except swf_exceptions.UnknownResourceError: 62 | # we expect the activity to have timed out already 63 | pass 64 | # for the timer 65 | wf_worker.run_once() 66 | 67 | time.sleep(1) 68 | 69 | hist = self.get_workflow_execution_history() 70 | self.assertEqual(len(hist), 22) 71 | self.assertEqual(hist[-1]['eventType'], 'WorkflowExecutionCompleted') 72 | self.assertEqual(hist[10]['eventType'], 'TimerStarted') 73 | self.assertEqual(hist[10]['timerStartedEventAttributes']['startToFireTimeout'], "1") 74 | 75 | def test_activity_exception_retries(self): 76 | class ActivityRetryOnExceptionWorkflow(WorkflowDefinition): 77 | @execute(version='1.2', execution_start_to_close_timeout=60) 78 | def execute(self): 79 | try: 80 | yield RetryingActivities.activity_raises_errors(RuntimeError) 81 | except ActivityTaskFailedError as err: 82 | assert isinstance(err.cause, RuntimeError) 83 | 84 | try: 85 | yield RetryingActivities.activity_raises_errors(AttributeError) 86 | except ActivityTaskFailedError as err: 87 | assert isinstance(err.cause, AttributeError) 88 | pass 89 | 90 | wf_worker = WorkflowWorker( 91 | self.session, self.region, self.domain, self.task_list, ActivityRetryOnExceptionWorkflow) 92 | 93 | act_worker = ActivityWorker( 94 | self.session, self.region, self.domain, self.task_list, RetryingActivities()) 95 | 96 | with workflow_starter(self.session, self.region, self.domain, self.task_list): 97 | instance = ActivityRetryOnExceptionWorkflow.execute() 98 | self.workflow_execution = instance.workflow_execution 99 | 100 | for i in range(2): 101 | wf_worker.run_once() 102 | act_worker.run_once() 103 | # for the timer 104 | wf_worker.run_once() 105 | 106 | act_worker.run_once() 107 | wf_worker.run_once() 108 | 109 | # wf_worker.run_once() 110 | # print 'wfrun' 111 | 112 | time.sleep(1) 113 | 114 | # check that we have a timer started and that the workflow length is the same to validate 115 | # that retries happened only on one of the exceptions 116 | hist = self.get_workflow_execution_history() 117 | self.assertEqual(len(hist), 28) 118 | self.assertEqual(hist[-1]['eventType'], 'WorkflowExecutionCompleted') 119 | self.assertEqual(hist[10]['eventType'], 'TimerStarted') 120 | self.assertEqual(hist[10]['timerStartedEventAttributes']['startToFireTimeout'], "0") 121 | 122 | 123 | if __name__ == '__main__': 124 | unittest.main() 125 | -------------------------------------------------------------------------------- /test/integration/test_signaling_workflows.py: -------------------------------------------------------------------------------- 1 | # -*- mode:python ; fill-column:120 -*- 2 | import time 3 | import unittest 4 | 5 | from botoflow import (workflow_time, WorkflowDefinition, WorkflowWorker, 6 | signal, execute, return_, workflow_starter, 7 | Future) 8 | from utils import SWFMixIn 9 | 10 | 11 | class SignalledWorkflow(WorkflowDefinition): 12 | 13 | def __init__(self, workflow_execution): 14 | super(SignalledWorkflow, self).__init__(workflow_execution) 15 | self.msg = "Not signalled" 16 | 17 | @execute(version='1.0', execution_start_to_close_timeout=60) 18 | def execute(self): 19 | yield workflow_time.sleep(4) 20 | return_(self.msg) 21 | 22 | @signal() 23 | def signal(self, msg): 24 | self.msg = msg 25 | 26 | 27 | class SignalledManyInputWorkflow(WorkflowDefinition): 28 | 29 | @execute(version='1.0', execution_start_to_close_timeout=60) 30 | def execute(self): 31 | self._wait_for_signal = Future() 32 | result = [] 33 | while True: 34 | signal_result = yield self._wait_for_signal 35 | if not signal_result: 36 | break 37 | result.append(signal_result) 38 | # reset the future 39 | self._wait_for_signal = Future() 40 | 41 | return_(result) 42 | 43 | @signal() 44 | def add_data(self, input): 45 | self._wait_for_signal.set_result(input) 46 | 47 | 48 | class TestSignalledWorkflows(SWFMixIn, unittest.TestCase): 49 | 50 | def test_signalled_workflow(self): 51 | wf_worker = WorkflowWorker( 52 | self.session, self.region, self.domain, self.task_list, 53 | SignalledWorkflow) 54 | 55 | with workflow_starter(self.session, self.region, self.domain, self.task_list): 56 | instance = SignalledWorkflow.execute() 57 | self.workflow_execution = instance.workflow_execution 58 | 59 | # wait and signal the workflow 60 | time.sleep(1) 61 | instance.signal("Signaled") 62 | 63 | for i in range(2): 64 | wf_worker.run_once() 65 | 66 | time.sleep(1) 67 | 68 | hist = self.get_workflow_execution_history() 69 | self.assertEqual(len(hist), 11) 70 | self.assertEqual(hist[-1]['eventType'], 'WorkflowExecutionCompleted') 71 | self.assertEqual(self.serializer.loads( 72 | hist[-1]['workflowExecutionCompletedEventAttributes']['result']), 'Signaled') 73 | 74 | def test_signalled_many_input_workflow(self): 75 | wf_worker = WorkflowWorker( 76 | self.session, self.region, self.domain, self.task_list, 77 | SignalledManyInputWorkflow) 78 | 79 | with workflow_starter(self.session, self.region, self.domain, self.task_list): 80 | instance = SignalledManyInputWorkflow.execute() 81 | self.workflow_execution = instance.workflow_execution 82 | 83 | # wait and signal the workflow 84 | for i in range(1, 5): 85 | instance.add_data(i) 86 | instance.add_data(None) # stop looping 87 | 88 | wf_worker.run_once() 89 | 90 | time.sleep(1) 91 | 92 | hist = self.get_workflow_execution_history() 93 | self.assertEqual(len(hist), 10) 94 | self.assertEqual(hist[-1]['eventType'], 'WorkflowExecutionCompleted') 95 | self.assertEqual(self.serializer.loads( 96 | hist[-1]['workflowExecutionCompletedEventAttributes']['result']), 97 | [1,2,3,4]) 98 | 99 | 100 | if __name__ == '__main__': 101 | unittest.main() 102 | 103 | -------------------------------------------------------------------------------- /test/integration/various_activities.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from botoflow import activities, activity, manual_activity, get_context 4 | from botoflow.core.exceptions import CancellationError, CancelledError 5 | 6 | 7 | class MySpecialCancelledError(CancelledError): 8 | 9 | def __init__(self, extra_data): 10 | self.extra_data = extra_data 11 | 12 | 13 | @activities(schedule_to_start_timeout=60, 14 | start_to_close_timeout=60) 15 | class BunchOfActivities(object): 16 | @activity(version='1.1', task_priority=100) 17 | def priority_sum(self, x, y): 18 | return x + y 19 | 20 | @activity(version='1.1') 21 | def sum(self, x, y): 22 | return x + y 23 | 24 | @activity(version='1.4', 25 | schedule_to_close_timeout=60 * 2) 26 | def mul(self, x, y): 27 | return x * y 28 | 29 | @activity('1.1') 30 | def throw(self): 31 | raise ValueError("Hello-Error") 32 | 33 | @activity('1.0') 34 | def heartbeating_activity(self, repeat_num): 35 | for i in range(repeat_num): 36 | time.sleep(0.2) 37 | get_context().heartbeat(str(i)) 38 | 39 | @activity('1.0') 40 | def sleep_activity(self, sleep_secs): 41 | time.sleep(sleep_secs) 42 | 43 | @activity('1.1') 44 | def cleanup_state_activity(self): 45 | return 'clean' 46 | 47 | @activity('1.0', task_list='FAKE') 48 | def wrong_tasklist_activity(self): 49 | return 50 | 51 | @activity('1.0') 52 | def heartbeating_custom_error_activity(self, repeat_num): 53 | for i in range(repeat_num): 54 | time.sleep(0.2) 55 | try: 56 | get_context().heartbeat(str(i)) 57 | except CancellationError: 58 | raise MySpecialCancelledError("spam") 59 | 60 | 61 | @activities(schedule_to_start_timeout=60, 62 | start_to_close_timeout=60) 63 | class ManualActivities(object): 64 | @manual_activity(version='1.0') 65 | def perform_task(self, **kwargs): 66 | activity_context = get_context() 67 | task_token = activity_context.task.token 68 | 69 | with open('task_token.txt', 'w') as shared_file: 70 | shared_file.write(task_token) 71 | -------------------------------------------------------------------------------- /test/unit/core/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | @pytest.yield_fixture(scope='session') 5 | def core_debug(): 6 | from botoflow.core import ( 7 | async_context, async_event_loop, async_task, async_task_context, 8 | base_future, decorators, future) 9 | 10 | async_context.DEBUG = True 11 | async_event_loop.DEBUG = True 12 | async_task.DEBUG = True 13 | async_task_context.DEBUG = True 14 | base_future.DEBUG = True 15 | decorators.DEBUG = True 16 | future.DEBUG = True 17 | 18 | yield None 19 | 20 | async_context.DEBUG = False 21 | async_event_loop.DEBUG = False 22 | async_task.DEBUG = False 23 | async_task_context.DEBUG = False 24 | base_future.DEBUG = False 25 | decorators.DEBUG = False 26 | future.DEBUG = False 27 | -------------------------------------------------------------------------------- /test/unit/core/test_async_event_loop.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from botoflow.core import async_event_loop 3 | 4 | pytestmark = pytest.mark.usefixtures('core_debug') 5 | 6 | 7 | def test_smoke(): 8 | ev = async_event_loop.AsyncEventLoop() 9 | assert None == ev.execute_all_tasks() 10 | 11 | -------------------------------------------------------------------------------- /test/unit/core/test_async_traceback.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import unittest 3 | import logging 4 | import pytest 5 | 6 | import sys 7 | import six 8 | 9 | from botoflow.core.async_event_loop import AsyncEventLoop 10 | from botoflow.core.decorators import coroutine, task 11 | from botoflow.core.async_traceback import format_exc, print_exc 12 | from botoflow.logging_filters import BotoflowFilter 13 | 14 | logging.basicConfig(level=logging.DEBUG, 15 | format='%(filename)s:%(lineno)d (%(funcName)s) - %(message)s') 16 | logging.getLogger('botoflow').addFilter(BotoflowFilter()) 17 | 18 | pytestmark = pytest.mark.usefixtures('core_debug') 19 | 20 | 21 | class TestTraceback(unittest.TestCase): 22 | 23 | def setUp(self): 24 | self.tb_str = None 25 | 26 | @pytest.mark.xfail(sys.version_info >= (3,5,0), 27 | reason="Some kind of brokennes on 3.5.0 specifically") 28 | def test_format(self): 29 | @task 30 | def task_func(): 31 | raise RuntimeError("Test") 32 | 33 | @task_func.do_except 34 | def except_func(err): 35 | self.tb_str = "".join(format_exc()) 36 | 37 | ev = AsyncEventLoop() 38 | with ev: 39 | task_func() 40 | ev.execute_all_tasks() 41 | 42 | self.assertTrue(self.tb_str) 43 | self.assertEqual(1, self.tb_str.count('---continuation---')) 44 | 45 | @pytest.mark.xfail(sys.version_info >= (3,5,0), 46 | reason="Some kind of brokennes on 3.5.0 specifically") 47 | def test_print(self): 48 | @task 49 | def task_func(): 50 | raise RuntimeError("Test") 51 | 52 | @task_func.do_except 53 | def except_func(err): 54 | strfile = six.StringIO() 55 | print_exc(file=strfile) 56 | self.tb_str = strfile.getvalue() 57 | 58 | ev = AsyncEventLoop() 59 | with ev: 60 | task_func() 61 | ev.execute_all_tasks() 62 | 63 | self.assertTrue(self.tb_str) 64 | self.assertEqual(1, self.tb_str.count('---continuation---')) 65 | 66 | def test_recursive(self): 67 | @task 68 | def task_raises_recursive(count=3): 69 | if not count: 70 | raise RuntimeError("Test") 71 | count -= 1 72 | task_raises_recursive(count) 73 | 74 | @task 75 | def task_func(): 76 | task_raises_recursive() 77 | 78 | @task_func.do_except 79 | def except_func(err): 80 | self.tb_str = format_exc() 81 | 82 | ev = AsyncEventLoop() 83 | with ev: 84 | task_func() 85 | ev.execute_all_tasks() 86 | 87 | self.assertTrue(self.tb_str) 88 | 89 | @pytest.mark.xfail(sys.version_info >= (3,5,0), 90 | reason="Some kind of brokennes on 3.5.0 specifically") 91 | def test_async(self): 92 | @coroutine 93 | def raises(): 94 | raise RuntimeError("TestErr") 95 | 96 | @coroutine 97 | def main(): 98 | try: 99 | yield raises() 100 | except RuntimeError: 101 | self.tb_str = "".join(format_exc()) 102 | 103 | ev = AsyncEventLoop() 104 | with ev: 105 | main() 106 | ev.execute_all_tasks() 107 | 108 | self.assertTrue(self.tb_str) 109 | self.assertEqual(2, self.tb_str.count('---continuation---')) 110 | 111 | 112 | if __name__ == '__main__': 113 | unittest.main() 114 | -------------------------------------------------------------------------------- /test/unit/decider/test_activity_future.py: -------------------------------------------------------------------------------- 1 | try: 2 | from unittest.mock import MagicMock 3 | except ImportError: 4 | from mock import MagicMock 5 | 6 | 7 | from botoflow.core import AsyncEventLoop, BaseFuture, Future, coroutine, return_ 8 | from botoflow.decider.activity_future import ActivityFuture 9 | 10 | 11 | def test_activity_future(): 12 | 13 | ev = AsyncEventLoop() 14 | with ev: 15 | 16 | future = Future() 17 | 18 | m_activity_task_handler = MagicMock() 19 | activity_future = ActivityFuture(future, m_activity_task_handler, 1) 20 | 21 | @coroutine 22 | def main(): 23 | result = yield activity_future 24 | return_(result) 25 | 26 | main_future = main() 27 | future.set_result(3) 28 | 29 | ev.execute_all_tasks() 30 | assert main_future.result() == 3 31 | 32 | 33 | def test_activity_future_cancel(): 34 | 35 | ev = AsyncEventLoop() 36 | with ev: 37 | 38 | future = Future() 39 | cancel_future = BaseFuture() 40 | 41 | m_activity_task_handler = MagicMock() 42 | m_activity_task_handler.request_cancel_activity_task.return_value = cancel_future 43 | activity_future = ActivityFuture(future, m_activity_task_handler, 1) 44 | 45 | @coroutine 46 | def main(): 47 | cancel_future.cancel() 48 | result = yield activity_future.cancel() 49 | assert result == cancel_future 50 | 51 | main() 52 | future.set_result(3) 53 | 54 | ev.execute_all_tasks() 55 | assert cancel_future.cancelled() 56 | 57 | 58 | def test_activity_future_cancel_failed(): 59 | 60 | ev = AsyncEventLoop() 61 | with ev: 62 | 63 | future = Future() 64 | cancel_future = BaseFuture() 65 | 66 | m_activity_task_handler = MagicMock() 67 | m_activity_task_handler.request_cancel_activity_task.return_value = cancel_future 68 | activity_future = ActivityFuture(future, m_activity_task_handler, 1) 69 | 70 | @coroutine 71 | def main(): 72 | cancel_future.set_exception(RuntimeError()) 73 | yield activity_future.cancel() 74 | 75 | main_future = main() 76 | future.set_result(3) 77 | 78 | ev.execute_all_tasks() 79 | assert isinstance(main_future.exception(), RuntimeError) 80 | -------------------------------------------------------------------------------- /test/unit/decider/test_activity_task_handler.py: -------------------------------------------------------------------------------- 1 | from mock import MagicMock, call, patch 2 | 3 | from botoflow.constants import USE_WORKER_TASK_LIST 4 | from botoflow.core import AllFuture, Future, AsyncEventLoop 5 | from botoflow.decider import activity_task_handler 6 | 7 | 8 | def test_init(): 9 | ath = activity_task_handler.ActivityTaskHandler('decider', 'task_list') 10 | 11 | assert ath._decider == 'decider' 12 | assert ath._task_list == 'task_list' 13 | assert ath._open_activities == {} 14 | assert ath._schedule_event_to_activity_id == {} 15 | assert ath._open_cancels == {} 16 | 17 | 18 | def test_del(): 19 | ath = activity_task_handler.ActivityTaskHandler('decider', 'task_list') 20 | m_future = MagicMock() 21 | ath._open_activities = {'activity': {'handler': m_future}} 22 | 23 | del ath 24 | assert m_future.close.mock_calls == [call()] 25 | 26 | 27 | @patch.object(activity_task_handler, 'ActivityFuture') 28 | def test_handle_execute_activity(m_ActivityFuture): 29 | m_decider = MagicMock() 30 | m_data_converter = MagicMock() 31 | m_activity_type = MagicMock(data_converter=m_data_converter) 32 | ath = activity_task_handler.ActivityTaskHandler(m_decider, 'task_list') 33 | 34 | decision_dict = {'task_list': {'name': USE_WORKER_TASK_LIST}, 35 | 'activity_type_name': 'name', 'activity_type_version': 'version'} 36 | activity_future = ath.handle_execute_activity(m_activity_type, decision_dict, [], {}) 37 | assert activity_future == m_ActivityFuture() 38 | assert m_data_converter.dumps.mock_calls == [call([[], {}])] 39 | 40 | 41 | def test_request_cancel_activity_task_all(): 42 | m_decider = MagicMock() 43 | ath = activity_task_handler.ActivityTaskHandler(m_decider, 'task_list') 44 | ath.request_cancel_activity_task = MagicMock() 45 | my_future = Future() 46 | ath._open_activities = {'1': {'future': my_future}} 47 | 48 | ev = AsyncEventLoop() 49 | with ev: 50 | assert isinstance(ath.request_cancel_activity_task_all(), AllFuture) 51 | 52 | ev.execute_all_tasks() 53 | assert call(my_future, '1') in ath.request_cancel_activity_task.mock_calls 54 | 55 | 56 | def test_request_cancel_activity_task_duplicate(): 57 | m_decider = MagicMock() 58 | ath = activity_task_handler.ActivityTaskHandler(m_decider, 'task_list') 59 | ath._open_cancels = {'1': {'future': 'future'}} 60 | 61 | assert ath.request_cancel_activity_task(None, '1') == 'future' 62 | -------------------------------------------------------------------------------- /test/unit/decider/test_decider.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | from mock import patch, MagicMock, call 4 | 5 | from botoflow.decider import decider 6 | from botoflow.workers import GenericWorkflowWorker 7 | from botoflow.decider.decision_task_poller import DecisionTaskPoller, DecisionTask 8 | from botoflow.history_events.events import ( 9 | WorkflowExecutionStarted, WorkflowExecutionCompleted, DecisionTaskScheduled, DecisionTaskStarted, 10 | DecisionTaskCompleted, StartChildWorkflowExecutionInitiated, ChildWorkflowExecutionStarted, 11 | ChildWorkflowExecutionCompleted, ChildWorkflowExecutionFailed) 12 | 13 | 14 | @patch.object(decider, 'get_context') 15 | @patch.object(decider, 'set_context') 16 | def test_decide_reordered(m_old_context, m_context): 17 | date = datetime(1975, 5, 25) 18 | id_range = iter(range(1, 100)) 19 | 20 | events = [WorkflowExecutionStarted(next(id_range), date, {}), 21 | DecisionTaskScheduled(next(id_range), date, {}), 22 | DecisionTaskStarted(next(id_range), date, {}), 23 | DecisionTaskCompleted(next(id_range), date, {}), 24 | StartChildWorkflowExecutionInitiated(next(id_range), date, {}), 25 | ChildWorkflowExecutionStarted(next(id_range), date, {}), 26 | DecisionTaskScheduled(next(id_range), date, {}), 27 | DecisionTaskCompleted(next(id_range), date, {}), 28 | StartChildWorkflowExecutionInitiated(next(id_range), date, {}), 29 | DecisionTaskStarted(next(id_range), date, {}), 30 | ChildWorkflowExecutionStarted(next(id_range), date, {}), 31 | ChildWorkflowExecutionCompleted(next(id_range), date, {}), 32 | DecisionTaskScheduled(next(id_range), date, {}), 33 | DecisionTaskCompleted(next(id_range), date, {}), 34 | ChildWorkflowExecutionFailed(next(id_range), date, {}), 35 | DecisionTaskStarted(next(id_range), date, {}), 36 | ChildWorkflowExecutionCompleted(next(id_range), date, {},), # must go after StartChildWorkflow... 37 | ChildWorkflowExecutionCompleted(next(id_range), date, {}), # this one too 38 | DecisionTaskCompleted(next(id_range), date, {}), 39 | StartChildWorkflowExecutionInitiated(next(id_range), date, {}), 40 | DecisionTaskStarted(next(id_range), date, {}), 41 | DecisionTaskCompleted(next(id_range), date, {}), 42 | WorkflowExecutionCompleted(next(id_range), date, {})] 43 | 44 | m_worker = MagicMock(spec=GenericWorkflowWorker) 45 | m_poller = MagicMock(spec=DecisionTaskPoller) 46 | m_decision_task = MagicMock(spec=DecisionTask, workflow_id='unit-wfid', run_id='unit-runid', task_token='unit-tt', 47 | previous_started_event_id=1, events=iter(events)) 48 | m_poller().poll.return_value = m_decision_task 49 | 50 | m_handle_history_event = MagicMock(spec=decider.Decider._handle_history_event) 51 | 52 | decider_inst = decider.Decider(m_worker, 'unit-domain', 'unit-tlist', MagicMock(), 53 | 'unit-id', _Poller=m_poller) 54 | decider_inst._process_decisions = MagicMock(spec=decider.Decider._process_decisions) 55 | decider_inst._handle_history_event = m_handle_history_event 56 | 57 | decider_inst.decide() 58 | 59 | assert m_handle_history_event.mock_calls == [call(events[0]), call(events[4]), call(events[5]), call(events[8]), 60 | call(events[10]), call(events[11]), call(events[14]), 61 | call(events[19]), call(events[16]), call(events[17])] 62 | -------------------------------------------------------------------------------- /test/unit/decider/test_workflow_execution_handler.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from mock import Mock, patch 4 | import pytest 5 | 6 | from botoflow.decider.workflow_execution_handler import WorkflowExecutionHandler 7 | 8 | @pytest.fixture 9 | def handler(): 10 | return WorkflowExecutionHandler(Mock(name='decider'), 'task-list') 11 | 12 | kwarg_dict = {'keyword': 'argument'} 13 | arg_list = ['arg', 'list'] 14 | arg_tuple = tuple(arg_list) 15 | 16 | valid_kwargs = ( 17 | [[], kwarg_dict], 18 | ([], kwarg_dict), 19 | [(), kwarg_dict], 20 | ((), kwarg_dict), 21 | kwarg_dict, 22 | ) 23 | @pytest.mark.parametrize('input', valid_kwargs) 24 | def test_load_kwarg_input(handler, input): 25 | event = Mock() 26 | event.attributes = {'input': json.dumps(input)} 27 | print(event.attributes) 28 | assert handler._load_input(event)[0] == [] 29 | assert handler._load_input(event)[1] == kwarg_dict 30 | 31 | 32 | @pytest.mark.parametrize('args, kwargs', ( 33 | [{'__tuple': ()}, kwarg_dict], 34 | [{'__tuple': ('a',)}, kwarg_dict], 35 | )) 36 | def test_arg_return_values(handler, args, kwargs): 37 | event = Mock() 38 | event.attributes = {'input': json.dumps([args, kwargs])} 39 | assert handler._load_input(event)[0] == args['__tuple'] 40 | assert handler._load_input(event)[1] == kwargs 41 | 42 | def test_load_null_input(handler): 43 | event = Mock() 44 | event.attributes = {} 45 | assert handler._load_input(event) == ([], {}) 46 | -------------------------------------------------------------------------------- /test/unit/decisions/test_decision_list.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from botoflow.decisions import decision_list, decisions 4 | class TestDecisionList(unittest.TestCase): 5 | 6 | def test_delete_decision(self): 7 | dlist = decision_list.DecisionList() 8 | dlist.append(decisions.CancelTimer(123)) 9 | 10 | self.assertTrue(dlist) 11 | dlist.delete_decision(decisions.CancelTimer, 999) 12 | self.assertTrue(dlist) 13 | dlist.delete_decision(decisions.CancelTimer, 123) 14 | self.assertFalse(dlist) 15 | 16 | def test_to_swf(self): 17 | dlist = decision_list.DecisionList() 18 | dlist.append(decisions.CancelTimer(123)) 19 | 20 | swf_list = dlist.to_swf() 21 | self.assertTrue(swf_list) 22 | self.assertEqual(swf_list, [{'cancelTimerDecisionAttributes': 23 | {'timerId': 123}, 24 | 'decisionType': 'CancelTimer'}]) 25 | 26 | if __name__ == '__main__': 27 | unittest.main() 28 | -------------------------------------------------------------------------------- /test/unit/test/test_simple_workflow_testing.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from mock import patch 3 | 4 | from botoflow import WorkflowDefinition, execute, activities, activity, return_, Future, coroutine 5 | from botoflow.test.workflow_testing_context import WorkflowTestingContext 6 | 7 | 8 | @activities(task_priority=100, 9 | schedule_to_start_timeout=60, 10 | start_to_close_timeout=60) 11 | class BunchOfActivities(object): 12 | @activity(version='1.1') 13 | def sum(self, x, y): 14 | return x + y 15 | 16 | @activity(version='1.4', 17 | schedule_to_close_timeout=60*2) 18 | def mul(self, x, y): 19 | return x * y 20 | 21 | @activity('1.1') 22 | def throw(self): 23 | raise ValueError("Hello-Error") 24 | 25 | 26 | class SimpleWorkflow(WorkflowDefinition): 27 | 28 | @execute('1.0', 5) 29 | def run(self, x, y): 30 | retval = None 31 | sum_result = yield BunchOfActivities.sum(x, y) 32 | if sum_result > 2: 33 | retval = yield BunchOfActivities.mul(retval, 2) 34 | 35 | return_(retval) 36 | 37 | @coroutine 38 | def sync_method(self, x, y): 39 | return_(x+y) 40 | 41 | @coroutine 42 | def async_method(self, x, y): 43 | result = yield BunchOfActivities.sum(x, y) 44 | return_(result) 45 | 46 | 47 | def test_simple_workflow_testing(): 48 | with patch.object(BunchOfActivities, 'sum', return_value=Future.with_result(3)), \ 49 | patch.object(BunchOfActivities, 'mul', return_value=Future.with_result(6)): 50 | 51 | with WorkflowTestingContext(): 52 | result = SimpleWorkflow.run(1,2) 53 | assert 6 == result.result() 54 | assert BunchOfActivities.__dict__['sum'].__dict__['func'].swf_options['activity_type'].schedule_to_start_timeout == 60 55 | assert BunchOfActivities.__dict__['mul'].__dict__['func'].swf_options['activity_type'].schedule_to_start_timeout == 60 56 | assert BunchOfActivities.__dict__['mul'].__dict__['func'].swf_options['activity_type'].schedule_to_close_timeout == 120 57 | 58 | 59 | def test_activity_not_stubbed_exception(): 60 | with WorkflowTestingContext(): 61 | result = SimpleWorkflow.run(1,2) 62 | 63 | assert NotImplementedError == type(result.exception()) 64 | assert "Activity BunchOfActivities.sum must be stubbed" in repr(result.exception()) 65 | 66 | 67 | def test_sync_method(): 68 | with WorkflowTestingContext(): 69 | result = SimpleWorkflow(None).sync_method(1, 2) 70 | assert 3 == result.result() 71 | 72 | 73 | @patch.object(BunchOfActivities, 'sum', return_value=Future.with_result(3)) 74 | def test_async_method(m_sum): 75 | with WorkflowTestingContext(): 76 | result = SimpleWorkflow(None).async_method(1, 2) 77 | assert 3 == result.result() 78 | -------------------------------------------------------------------------------- /test/unit/test_decorators.py: -------------------------------------------------------------------------------- 1 | from botoflow import activities, activity 2 | from botoflow.data_converter.json_data_converter import JSONDataConverter 3 | 4 | 5 | def test_activities_decorator_adds_data_converter(): 6 | class MyDataConverter(object): 7 | pass 8 | 9 | @activities(data_converter=MyDataConverter()) 10 | class ActivitiesWithCustomDataConverter(object): 11 | @activity(None) 12 | def foobar(self): 13 | pass 14 | 15 | assert isinstance(ActivitiesWithCustomDataConverter().foobar.swf_options['activity_type'].data_converter, MyDataConverter) 16 | 17 | 18 | def test_activities_decorator_uses_default_data_converter(): 19 | @activities() 20 | class ActivitiesWithDefaultDataConverter(object): 21 | @activity(None) 22 | def foobar(self): 23 | pass 24 | 25 | assert isinstance(ActivitiesWithDefaultDataConverter().foobar.swf_options['activity_type'].data_converter, JSONDataConverter) 26 | -------------------------------------------------------------------------------- /test/unit/test_options.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from botoflow.context import set_context, DecisionContext 3 | 4 | from botoflow.options import activity_options, workflow_options 5 | 6 | class TestOptions(unittest.TestCase): 7 | 8 | def tearDown(self): 9 | set_context(None) 10 | 11 | def test_activity_overrides(self): 12 | context = DecisionContext(None) 13 | set_context(context) 14 | 15 | self.assertFalse(context._activity_options_overrides) 16 | with activity_options(task_list='Test'): 17 | self.assertEqual(context._activity_options_overrides['task_list'], 18 | {'name':'Test'}) 19 | self.assertFalse(context._activity_options_overrides) 20 | 21 | def test_workflow_overrides(self): 22 | context = DecisionContext(None) 23 | set_context(context) 24 | 25 | self.assertFalse(context._workflow_options_overrides) 26 | with workflow_options(child_policy='TERMINATE'): 27 | self.assertEqual( 28 | context._workflow_options_overrides['child_policy'], 29 | 'TERMINATE') 30 | self.assertFalse(context._workflow_options_overrides) 31 | 32 | if __name__ == '__main__': 33 | unittest.main() 34 | -------------------------------------------------------------------------------- /test/unit/test_swf_exceptions.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from botocore.exceptions import ClientError 4 | 5 | from botoflow import swf_exceptions 6 | 7 | 8 | def test_swf_exception_wrapper_with_no_exception(): 9 | with swf_exceptions.swf_exception_wrapper(): 10 | pass # nothing went wrong 11 | 12 | 13 | def test_swf_exception_wrapper_with_unknown_exception(): 14 | with pytest.raises(swf_exceptions.SWFResponseError) as exc: 15 | with swf_exceptions.swf_exception_wrapper(): 16 | raise ClientError({'Error': {'Code': 'SomethingBrokeError', 'Message': 'Oops'}}, 'foobar') 17 | 18 | assert str(exc.value) == "Oops" 19 | 20 | 21 | @pytest.mark.parametrize('code_key,expected_exception', [ 22 | ('DomainDeprecatedFault', swf_exceptions.DomainDeprecatedError), 23 | ('DomainAlreadyExistsFault', swf_exceptions.DomainAlreadyExistsError), 24 | ('DefaultUndefinedFault', swf_exceptions.DefaultUndefinedError), 25 | ('LimitExceededFault', swf_exceptions.LimitExceededError), 26 | ('WorkflowExecutionAlreadyStartedFault', swf_exceptions.WorkflowExecutionAlreadyStartedError), 27 | ('TypeDeprecatedFault', swf_exceptions.TypeDeprecatedError), 28 | ('TypeAlreadyExistsFault', swf_exceptions.TypeAlreadyExistsError), 29 | ('OperationNotPermittedFault', swf_exceptions.OperationNotPermittedError), 30 | ('UnknownResourceFault', swf_exceptions.UnknownResourceError), 31 | ('SWFResponseError', swf_exceptions.SWFResponseError), 32 | ('ThrottlingException', swf_exceptions.ThrottlingException), 33 | ('ValidationException', swf_exceptions.ValidationException), 34 | ('UnrecognizedClientException', swf_exceptions.UnrecognizedClientException), 35 | ('InternalFailure', swf_exceptions.InternalFailureError), 36 | ]) 37 | def test_swf_exception_wrapper_with_known_exception(code_key, expected_exception): 38 | with pytest.raises(expected_exception) as exc: 39 | with swf_exceptions.swf_exception_wrapper(): 40 | raise ClientError({'Error': {'Code': code_key, 'Message': 'Oops'}}, 'foobar') 41 | 42 | assert str(exc.value) == "Oops" 43 | 44 | 45 | def test_swf_exception_wrapper_with_no_error_response_details(): 46 | with pytest.raises(swf_exceptions.SWFResponseError) as exc: 47 | with swf_exceptions.swf_exception_wrapper(): 48 | raise ClientError({'Error': {'Code': None, 'Message': None}}, 'foobar') 49 | 50 | 51 | def test_swf_exception_wrapper_with_malformed_code_key(): 52 | with pytest.raises(swf_exceptions.SWFResponseError) as exc: 53 | with swf_exceptions.swf_exception_wrapper(): 54 | raise ClientError({'Error': {'Code': (123, "this is not a key"), 'Message': None}}, 'foobar') 55 | 56 | 57 | def test_swf_exception_wrapper_with_non_client_error(): 58 | exception = RuntimeError("Not handled by swf_exception_wrapper") 59 | with pytest.raises(RuntimeError) as exc: 60 | with swf_exceptions.swf_exception_wrapper(): 61 | raise exception 62 | 63 | assert exc.value == exception 64 | -------------------------------------------------------------------------------- /test/unit/test_utils.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from botoflow.utils import camel_keys_to_snake_case, snake_keys_to_camel_case 4 | 5 | 6 | class TestUtils(unittest.TestCase): 7 | 8 | def test_camel_keys_to_snake_case(self): 9 | d = { 10 | 'workflowType': 'A', 11 | 'taskList': 'B', 12 | 'childPolicy': 'C', 13 | 'executionStartToCloseTimeout': 'D', 14 | 'taskStartToCloseTimeout': 'E', 15 | 'input': 'F', 16 | 'workflowId': 'G', 17 | 'domain': 'H' 18 | } 19 | 20 | self.assertDictEqual(camel_keys_to_snake_case(d), { 21 | 'workflow_type': 'A', 22 | 'task_list': 'B', 23 | 'child_policy': 'C', 24 | 'execution_start_to_close_timeout': 'D', 25 | 'task_start_to_close_timeout': 'E', 26 | 'input': 'F', 27 | 'workflow_id': 'G', 28 | 'domain': 'H' 29 | }) 30 | 31 | 32 | def test_snake_keys_to_camel_case(self): 33 | d = { 34 | 'workflow_type': 'A', 35 | 'task_list': 'B', 36 | 'child_policy': 'C', 37 | 'execution_start_to_close_timeout': 'D', 38 | 'task_start_to_close_timeout': 'E', 39 | 'input': 'F', 40 | 'workflow_id': 'G', 41 | 'domain': 'H' 42 | } 43 | 44 | self.assertDictEqual(snake_keys_to_camel_case(d), { 45 | 'workflowType': 'A', 46 | 'taskList': 'B', 47 | 'childPolicy': 'C', 48 | 'executionStartToCloseTimeout': 'D', 49 | 'taskStartToCloseTimeout': 'E', 50 | 'input': 'F', 51 | 'workflowId': 'G', 52 | 'domain': 'H' 53 | }) 54 | 55 | 56 | if __name__ == '__main__': 57 | unittest.main() 58 | -------------------------------------------------------------------------------- /test/unit/test_workflow_definition.py: -------------------------------------------------------------------------------- 1 | 2 | from botoflow import execute, signal 3 | from botoflow import workflow_definition as wodef 4 | 5 | 6 | class SpamWorkflow(wodef.WorkflowDefinition): 7 | 8 | @execute('1.0', 1) 9 | def execute0(self): 10 | pass 11 | 12 | @execute('1.0', 1) 13 | def execute1(self): 14 | pass 15 | 16 | @signal() 17 | def signal0(self): 18 | pass 19 | 20 | @signal() 21 | def signal1(self): 22 | pass 23 | 24 | 25 | class SubSpamWorkflow(SpamWorkflow): 26 | 27 | @execute('1.1', 2) 28 | def execute0(self): 29 | pass 30 | 31 | @signal() 32 | def signal0(self): 33 | pass 34 | 35 | 36 | def test_meta_subclass(): 37 | assert set(SubSpamWorkflow._workflow_types.values()) == {'execute0', 'execute1'} 38 | assert SubSpamWorkflow._workflow_signals['signal0'][1] == SubSpamWorkflow.signal0 39 | assert SubSpamWorkflow._workflow_signals['signal1'][1] == SpamWorkflow.signal1 40 | -------------------------------------------------------------------------------- /test/unit/test_workflow_time.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | try: 4 | from unittest.mock import MagicMock, patch, call 5 | except ImportError: 6 | from mock import MagicMock, patch, call 7 | 8 | from datetime import datetime 9 | 10 | from botoflow import workflow_time 11 | from botoflow.context import DecisionContext 12 | from botoflow.decider import Decider 13 | 14 | 15 | @patch.object(workflow_time, 'get_context') 16 | def test_time(m_get_context): 17 | dt = datetime.fromtimestamp(10.0) 18 | m_get_context.return_value = MagicMock(spec=DecisionContext, _workflow_time=dt) 19 | assert 10 == workflow_time.time() 20 | 21 | 22 | def test_time_wrong_context(): 23 | with pytest.raises(TypeError): 24 | workflow_time.time() 25 | 26 | @patch.object(workflow_time, 'get_context') 27 | def test_time_no_context(m_get_context): 28 | m_get_context.side_effect = AttributeError() 29 | 30 | with pytest.raises(TypeError): 31 | workflow_time.time() 32 | 33 | 34 | @patch.object(workflow_time, 'get_context') 35 | def test_sleep(m_get_context): 36 | m_decider = MagicMock(spec=Decider) 37 | m_decider.handle_execute_timer.return_value = 'Works' 38 | m_get_context.return_value = MagicMock(spec=DecisionContext, decider=m_decider) 39 | 40 | assert 'Works' == workflow_time.sleep(10) 41 | assert m_decider.handle_execute_timer.mock_calls == [call(10)] 42 | 43 | 44 | def test_sleep_wrong_context(): 45 | with pytest.raises(TypeError): 46 | workflow_time.sleep(10) 47 | 48 | @patch.object(workflow_time, 'get_context') 49 | def test_sleep_no_context(m_get_context): 50 | m_get_context.side_effect = AttributeError() 51 | 52 | with pytest.raises(TypeError): 53 | workflow_time.sleep() 54 | 55 | 56 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27,py34,py35 3 | skipsdist = True 4 | 5 | [testenv] 6 | commands = 7 | {toxinidir}/scripts/ci/install 8 | {envbindir}/python setup.py develop 9 | py.test --cov {envsitepackagesdir}/botoflow --junitxml={toxinidir}/build/xunit/{envname}/junit.xml 10 | {toxinidir}/scripts/ci/fix-coverage-path --strip="{envsitepackagesdir}" {toxinidir}/coverage.xml 11 | --------------------------------------------------------------------------------