├── tests ├── __init__.py ├── fixtures │ ├── __init__.py │ ├── flows │ │ ├── __init__.py │ │ └── example.py │ ├── log.py │ └── decider.py ├── conftest.py ├── test_context.py ├── test_log.py ├── test_utils.py ├── test_param.py ├── test_runner.py ├── test_task.py ├── test_decider.py └── test_activity.py ├── requirements.txt ├── requirements-doc.txt ├── garcon ├── __init__.py ├── utils.py ├── log.py ├── param.py ├── context.py ├── event.py ├── runner.py ├── task.py ├── decider.py └── activity.py ├── docs ├── releases.rst ├── guide.rst ├── releases │ └── v0.0.4.rst ├── api.rst ├── guide │ ├── intro.rst │ ├── sample.rst │ └── generators.rst ├── index.rst ├── Makefile ├── make.bat └── conf.py ├── requirements-tests.txt ├── tox.ini ├── example ├── simple │ ├── run.py │ └── workflow.py └── custom_decider │ ├── run.py │ └── workflow.py ├── .github └── workflows │ ├── publish.yml │ ├── testing.yml │ └── codeql-analysis.yml ├── setup.py ├── LICENSE ├── .gitignore └── README.rst /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/fixtures/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/fixtures/flows/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | backoff==2.2.1 2 | boto3==1.35.0 3 | -------------------------------------------------------------------------------- /requirements-doc.txt: -------------------------------------------------------------------------------- 1 | -r requirements.txt 2 | 3 | futures 4 | sphinxcontrib-napoleon 5 | -------------------------------------------------------------------------------- /garcon/__init__.py: -------------------------------------------------------------------------------- 1 | from pkgutil import extend_path 2 | __path__ = extend_path(__path__, __name__) 3 | -------------------------------------------------------------------------------- /docs/releases.rst: -------------------------------------------------------------------------------- 1 | Release notes 2 | ============= 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | 7 | releases/v0.0.4 8 | -------------------------------------------------------------------------------- /requirements-tests.txt: -------------------------------------------------------------------------------- 1 | flake8 2 | wheel 3 | pytest-cov==5.0.0 4 | pytest==8.3.2 5 | tox==4.18.1 6 | coveralls==4.0.0 7 | -------------------------------------------------------------------------------- /docs/guide.rst: -------------------------------------------------------------------------------- 1 | User's guide 2 | ============ 3 | 4 | .. toctree:: 5 | 6 | guide/intro 7 | guide/sample 8 | guide/generators 9 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py38,py39,py310,py311,py312 3 | 4 | [testenv] 5 | deps = -rrequirements-tests.txt 6 | commands = py.test 7 | 8 | [testenv:py3] 9 | deps = -rrequirements.txt 10 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | try: 2 | from unittest.mock import MagicMock 3 | except: 4 | from mock import MagicMock 5 | 6 | import pytest 7 | import boto3 8 | 9 | 10 | @pytest.fixture 11 | def boto_client(monkeypatch): 12 | """Create a fake boto client.""" 13 | return MagicMock(spec=boto3.client('swf', region_name='us-east-1')) 14 | -------------------------------------------------------------------------------- /docs/releases/v0.0.4.rst: -------------------------------------------------------------------------------- 1 | What's new in Garcon 0.0.4 2 | ========================== 3 | 4 | May 12th, 2015 5 | -------------- 6 | 7 | External Activities 8 | ~~~~~~~~~~~~~~~~~~~ 9 | 10 | SWF allows any activities to be written in any language (allowing the service 11 | to be fully polyglot). This change makes it easier for Garcon to handle and 12 | work with external activities. 13 | 14 | To define an external activity: simply set the flag `external` on the activity 15 | creation, and define (it's mandatory) the `timeout` and, if needed, the 16 | `heartbeat` timeout. 17 | 18 | Change: `#52 `` 19 | -------------------------------------------------------------------------------- /example/simple/run.py: -------------------------------------------------------------------------------- 1 | from garcon import activity 2 | from garcon import decider 3 | from threading import Thread 4 | import boto3 5 | import time 6 | 7 | import workflow 8 | 9 | client = boto3.client('swf', region_name='us-east-1') 10 | deciderworker = decider.DeciderWorker(client, workflow) 11 | 12 | client.start_workflow_execution( 13 | domain=workflow.domain, 14 | workflowId='unique-workflow-identifier', 15 | workflowType=dict( 16 | name=workflow.name, 17 | version='1.0'), 18 | taskList=dict(name=workflow.name)) 19 | 20 | Thread(target=activity.ActivityWorker(client, workflow).run).start() 21 | while(True): 22 | deciderworker.run() 23 | time.sleep(1) 24 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Upload Python Package 2 | 3 | on: 4 | release: 5 | types: [created] 6 | 7 | jobs: 8 | deploy: 9 | runs-on: ubuntu-latest 10 | permissions: 11 | id-token: write 12 | steps: 13 | - uses: actions/checkout@v2 14 | - name: Set up Python 15 | uses: actions/setup-python@v1 16 | with: 17 | python-version: '3.x' 18 | - name: Install dependencies 19 | run: | 20 | python -m pip install --upgrade pip 21 | pip install setuptools wheel twine 22 | - name: Build 23 | run: | 24 | python setup.py sdist bdist_wheel 25 | - name: Publish 26 | uses: pypa/gh-action-pypi-publish@release/v1 27 | -------------------------------------------------------------------------------- /docs/api.rst: -------------------------------------------------------------------------------- 1 | Api 2 | === 3 | 4 | .. automodule:: garcon.activity 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | 9 | .. automodule:: garcon.decider 10 | :members: 11 | :undoc-members: 12 | :show-inheritance: 13 | 14 | .. automodule:: garcon.event 15 | :members: 16 | :undoc-members: 17 | :show-inheritance: 18 | 19 | .. automodule:: garcon.log 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | 24 | .. automodule:: garcon.runner 25 | :members: 26 | :undoc-members: 27 | :show-inheritance: 28 | 29 | .. automodule:: garcon.task 30 | :members: 31 | :undoc-members: 32 | :show-inheritance: 33 | 34 | .. automodule:: garcon.utils 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | -------------------------------------------------------------------------------- /docs/guide/intro.rst: -------------------------------------------------------------------------------- 1 | Introduction 2 | ============ 3 | 4 | Garcon is a Python library for Amazon SWF, originally built at 5 | `The Orchard `_. 6 | 7 | The goal of this library is to allow the creation of workflows using SWF 8 | without the need to worry about the orchestration of the different activities, 9 | and build out the complex different workers. 10 | 11 | Main Features: 12 | 13 | * Simple: when you write a flow, the deciders and the activity workers are 14 | automatically generated. No extra work is required. 15 | * Retry mechanisms: if an activity has failed, you can set a maximum of retries. 16 | It ends up very useful when you work with external APIs. 17 | * Scalable timeouts: all the timeout are calculated and consider other running 18 | workflows. 19 | * :doc:`Activity Generators `: some workflows requires more 20 | than one instance of a specific activity. 21 | -------------------------------------------------------------------------------- /example/custom_decider/run.py: -------------------------------------------------------------------------------- 1 | from garcon import activity 2 | from garcon import decider 3 | from threading import Thread 4 | import time 5 | 6 | import boto3 7 | import workflow 8 | 9 | # Initiate the workflow on the dev domain and custom_decider name. 10 | client = boto3.client('swf', region_name='us-east-1') 11 | workflow = workflow.Workflow(client, 'dev', 'custom_decider') 12 | deciderworker = decider.DeciderWorker(workflow) 13 | 14 | client.start_workflow_execution( 15 | domain=workflow.domain, 16 | workflowId='unique-workflow-identifier', 17 | workflowType=dict( 18 | name=workflow.name, 19 | version='1.0'), 20 | executionStartToCloseTimeout='3600', 21 | taskStartToCloseTimeout='3600', 22 | childPolicy='TERMINATE', 23 | taskList=dict(name=workflow.name)) 24 | 25 | Thread(target=activity.ActivityWorker(workflow).run).start() 26 | while(True): 27 | deciderworker.run() 28 | time.sleep(1) 29 | -------------------------------------------------------------------------------- /tests/fixtures/log.py: -------------------------------------------------------------------------------- 1 | from garcon import log 2 | 3 | 4 | class MockLogClient(log.GarconLogger): 5 | """Mock of an object for which we want to add a Garcon logger 6 | """ 7 | domain = 'test_domain' 8 | 9 | 10 | # Valid execution context 11 | execution_context = { 12 | 'execution.domain': 'dev', 13 | 'execution.run_id': '123abc=', 14 | 'execution.workflow_id': 'test-workflow-id'} 15 | 16 | # Invalid execution context. Keys are incorrect 17 | invalid_execution_context = { 18 | 'abcd.domain': 'dev', 19 | '123.run_id': '123abc=', 20 | 'XYZ.workflow_id': 'test-workflow-id'} 21 | 22 | 23 | def log_enabled_object(): 24 | """Creates a mock object with log enabled 25 | """ 26 | 27 | mock = MockLogClient() 28 | mock.set_log_context(execution_context) 29 | 30 | return mock 31 | 32 | 33 | def log_disabled_object(): 34 | """Creates a mock object with no log 35 | """ 36 | 37 | mock = MockLogClient() 38 | mock.set_log_context(invalid_execution_context) 39 | 40 | return mock 41 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages 2 | from setuptools import setup 3 | from codecs import open 4 | from os import path 5 | 6 | here = path.abspath(path.dirname(__file__)) 7 | # Get the long description from the README file 8 | with open(path.join(here, 'README.rst'), encoding='utf-8') as f: 9 | long_description = f.read() 10 | 11 | 12 | setup( 13 | name='Garcon', 14 | version='1.1.1', 15 | url='https://github.com/xethorn/garcon/', 16 | author='Michael Ortali', 17 | author_email='hello@xethorn.net', 18 | description=('Lightweight library for AWS SWF.'), 19 | long_description=long_description, 20 | license='MIT', 21 | packages=find_packages(), 22 | include_package_data=True, 23 | install_requires=['boto3', 'backoff'], 24 | zip_safe=False, 25 | classifiers=[ 26 | 'Programming Language :: Python :: 3.8', 27 | 'Programming Language :: Python :: 3.9', 28 | 'Programming Language :: Python :: 3.10', 29 | 'Programming Language :: Python :: 3.11', 30 | 'Programming Language :: Python :: 3.12', 31 | ],) 32 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Michael Ortali 4 | Copyright (c) 2014 The Orchard 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | 24 | -------------------------------------------------------------------------------- /.github/workflows/testing.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a single version of Python 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: Build 5 | 6 | on: [push, pull_request] 7 | 8 | jobs: 9 | build: 10 | 11 | runs-on: ubuntu-latest 12 | strategy: 13 | matrix: 14 | python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] 15 | 16 | steps: 17 | - uses: actions/checkout@v2 18 | - name: Set up Python ${{ matrix.python-version }} 19 | uses: actions/setup-python@v1 20 | with: 21 | python-version: ${{ matrix.python-version }} 22 | - name: Install dependencies 23 | run: | 24 | python -m pip install --upgrade pip 25 | pip install -r requirements.txt 26 | pip install -r requirements-tests.txt 27 | - name: Lint with flake8 28 | run: | 29 | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics 30 | flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 31 | - name: Test with pytest 32 | env: 33 | COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} 34 | run: | 35 | pytest 36 | -------------------------------------------------------------------------------- /tests/fixtures/flows/example.py: -------------------------------------------------------------------------------- 1 | from garcon import activity 2 | from garcon import runner 3 | 4 | import boto3 5 | 6 | client = boto3.client('swf', region_name='us-east-1') 7 | domain = 'dev' 8 | name = 'workflow_name' 9 | create = activity.create(client, domain, name) 10 | 11 | activity_1 = create( 12 | name='activity_1', 13 | tasks=runner.Sync( 14 | lambda activity, context: 15 | print('activity_1'))) 16 | 17 | activity_2 = create( 18 | name='activity_2', 19 | requires=[activity_1], 20 | tasks=runner.Async( 21 | lambda activity, context: 22 | print('activity_2_task_1'), 23 | lambda activity, context: 24 | print('activity_2_task_2'))) 25 | 26 | activity_3 = create( 27 | name='activity_3', 28 | requires=[activity_1], 29 | tasks=runner.Sync( 30 | lambda activity, context: 31 | print('activity_3'))) 32 | 33 | activity_4 = create( 34 | name='activity_4', 35 | requires=[activity_3, activity_2], 36 | tasks=runner.Sync( 37 | lambda activity, context: 38 | print('activity_4'))) 39 | 40 | 41 | def on_exception(actor, exception): 42 | """Handler for exceptions. 43 | 44 | Useful if you use sentry or other similar systems. 45 | """ 46 | 47 | print(exception) 48 | -------------------------------------------------------------------------------- /tests/test_context.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import MagicMock 2 | 3 | from garcon import context 4 | from tests.fixtures import decider as decider_events 5 | 6 | 7 | def test_context_creation_without_events(monkeypatch): 8 | """Check the basic context creation. 9 | """ 10 | 11 | current_context = context.ExecutionContext() 12 | assert not current_context.current 13 | assert not current_context.workflow_input 14 | 15 | 16 | def test_context_creation_with_events(monkeypatch): 17 | """Test context creation with events. 18 | """ 19 | 20 | from tests.fixtures import decider as poll 21 | 22 | current_context = context.ExecutionContext(poll.history.get('events')) 23 | assert current_context.current == {'k': 'v'} 24 | 25 | def test_get_workflow_execution_info(monkeypatch): 26 | """Check that the workflow execution info are properly extracted 27 | """ 28 | 29 | from tests.fixtures import decider as poll 30 | 31 | current_context = context.ExecutionContext() 32 | current_context.set_workflow_execution_info(poll.history, 'dev') 33 | 34 | # Test extracting workflow execution info 35 | assert current_context.current == { 36 | 'execution.domain': 'dev', 37 | 'execution.run_id': '123abc=', 38 | 'execution.workflow_id': 'test-workflow-id'} 39 | -------------------------------------------------------------------------------- /example/simple/workflow.py: -------------------------------------------------------------------------------- 1 | from garcon import activity 2 | from garcon import runner 3 | import boto3 4 | import logging 5 | import random 6 | 7 | 8 | logger = logging.getLogger(__name__) 9 | client = boto3.client('swf', region_name='us-east-1') 10 | domain = 'dev' 11 | name = 'workflow_sample' 12 | create = activity.create(client, domain, name) 13 | 14 | 15 | def activity_failure(context, activity): 16 | num = int(random.random() * 4) 17 | if num != 3: 18 | logger.warn('activity_3: fails') 19 | raise Exception('fails') 20 | logger.debug('activity_3: end') 21 | 22 | 23 | test_activity_1 = create( 24 | name='activity_1', 25 | run=runner.Sync( 26 | lambda context, activity: logger.debug('activity_1'))) 27 | 28 | test_activity_2 = create( 29 | name='activity_2', 30 | requires=[test_activity_1], 31 | run=runner.Async( 32 | lambda context, activity: logger.debug('activity_2_task_1'), 33 | lambda context, activity: logger.debug('activity_2_task_2'))) 34 | 35 | test_activity_3 = create( 36 | name='activity_3', 37 | retry=10, 38 | requires=[test_activity_1], 39 | run=runner.Sync(activity_failure)) 40 | 41 | test_activity_4 = create( 42 | name='activity_4', 43 | requires=[test_activity_3, test_activity_2], 44 | run=runner.Sync( 45 | lambda context, activity: logger.debug('activity_4'))) 46 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *.cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | local_settings.py 55 | 56 | # Flask stuff: 57 | instance/ 58 | .webassets-cache 59 | 60 | # Scrapy stuff: 61 | .scrapy 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | # Jupyter Notebook 70 | .ipynb_checkpoints 71 | 72 | # pyenv 73 | .python-version 74 | 75 | # celery beat schedule file 76 | celerybeat-schedule 77 | 78 | # SageMath parsed files 79 | *.sage.py 80 | 81 | # Environments 82 | .env 83 | .venv 84 | env/ 85 | venv/ 86 | ENV/ 87 | 88 | # Spyder project settings 89 | .spyderproject 90 | .spyproject 91 | 92 | # Rope project settings 93 | .ropeproject 94 | 95 | # mkdocs documentation 96 | /site 97 | 98 | # mypy 99 | .mypy_cache/ -------------------------------------------------------------------------------- /docs/guide/sample.rst: -------------------------------------------------------------------------------- 1 | Code Sample 2 | =========== 3 | 4 | Before going onto the details, let’s take a quick look at the Garcon’s 5 | implementation of `Serial Activity Execution `_: 6 | 7 | .. code-block:: python 8 | 9 | from garcon import activity 10 | from garcon import runners 11 | 12 | 13 | domain = 'dev' 14 | name = 'boto_tutorial' 15 | create = activity.create(domain, name) 16 | 17 | a_tasks = create( 18 | name='a_tasks', 19 | run=runner.Sync( 20 | lambda context, activity: dict(result='Now don’t be givin him sambuca!')) 21 | 22 | b_tasks = create( 23 | name='b_tasks', 24 | requires=[a_tasks], 25 | run=runner.Sync( 26 | lambda context, activity: print(context))) 27 | 28 | c_tasks = create( 29 | name='c_tasks', 30 | requires=[b_tasks], 31 | run=runner.Sync( 32 | lambda context, activity: print(context))) 33 | 34 | By way of comparison, check out the `implementation `_ 35 | using directly boto. 36 | 37 | Note: 38 | Notes: Executing this code shows that the activity “a_tasks” returns a 39 | dictionary which hydrates the execution context. When the activity “b_tasks” 40 | is executed, the context passed for its execution contains the key/value 41 | previously passed as an output. Same observation can be done in “c_tasks”. 42 | 43 | All activities are running in series. `More examples `_ 44 | (including runners) are available online. 45 | -------------------------------------------------------------------------------- /tests/test_log.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from garcon import log 4 | from tests.fixtures import log as fixture 5 | 6 | 7 | def test_get_logger_namespace(): 8 | """Test that the logger nsmaespace is generated properly for a given 9 | execution context. 10 | """ 11 | 12 | assert log.get_logger_namespace(fixture.execution_context) == '.'.join([ 13 | log.LOGGER_PREFIX, 14 | fixture.execution_context.get('execution.domain'), 15 | fixture.execution_context.get('execution.workflow_id'), 16 | fixture.execution_context.get('execution.run_id')]) 17 | 18 | 19 | def test_set_log_context(): 20 | """Test that the logger_name property has set properly or has not been set 21 | if an invalid execution context is passed in. 22 | """ 23 | 24 | valid_mock = fixture.log_enabled_object() 25 | invalid_mock = fixture.log_disabled_object() 26 | 27 | assert valid_mock.logger_name == '.'.join([ 28 | log.LOGGER_PREFIX, 29 | fixture.execution_context.get('execution.domain'), 30 | fixture.execution_context.get('execution.workflow_id'), 31 | fixture.execution_context.get('execution.run_id')]) 32 | assert valid_mock.logger is logging.getLogger(valid_mock.logger_name) 33 | 34 | assert getattr(invalid_mock, 'logger_name', None) is None 35 | assert invalid_mock.logger is logging.getLogger(log.LOGGER_PREFIX) 36 | 37 | 38 | def test_unset_log_context(): 39 | """Test that the logger_name property has been unset. 40 | """ 41 | 42 | valid_mock = fixture.log_enabled_object() 43 | valid_mock.unset_log_context() 44 | 45 | assert getattr(valid_mock, 'logger_name', None) is None 46 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Garcon 2 | ====== 3 | 4 | Lightweight library for AWS SWF. 5 | 6 | Requirements 7 | ------------ 8 | 9 | * Python 3.5, 3.6, 3.7, 3.8 (tested) 10 | * Boto 2.34.0 (tested) 11 | 12 | 13 | Goal 14 | ---- 15 | 16 | The goal of this library is to allow the creation of Amazon Simple Workflow without the need to worry about the orchestration of the different activities and building out the different workers. This framework aims to help simple workflows. If you have a more complex case, you might want to use directly boto. 17 | 18 | Code sample 19 | ----------- 20 | 21 | The code sample shows a workflow that has 4 activities. It starts with activity_1, which after being completed schedule activity_2 and activity_3 to be ran in parallel. The workflow ends after the completion of activity_4 which requires activity_2 and activity_3 to be completed:: 22 | 23 | from garcon import activity 24 | from garcon import runner 25 | 26 | 27 | domain = 'dev' 28 | create = activity.create(domain) 29 | 30 | test_activity_1 = create( 31 | name='activity_1', 32 | tasks=runner.Sync( 33 | lambda activity, context: print('activity_1'))) 34 | 35 | test_activity_2 = create( 36 | name='activity_2', 37 | requires=[test_activity_1], 38 | run=runner.Async( 39 | lambda activity, context: print('activity_2_task_1'), 40 | lambda activity, context: print('activity_2_task_2'))) 41 | 42 | test_activity_3 = create( 43 | name='activity_3', 44 | requires=[test_activity_1], 45 | run=runner.Sync( 46 | lambda activity, context: print('activity_3'))) 47 | 48 | test_activity_4 = create( 49 | name='activity_4', 50 | requires=[test_activity_3, test_activity_2], 51 | run=runner.Sync( 52 | lambda activity, context: print('activity_4'))) 53 | 54 | Documentation 55 | ------------- 56 | 57 | .. toctree:: 58 | :titlesonly: 59 | 60 | guide 61 | api 62 | releases 63 | 64 | 65 | Licence 66 | ------- 67 | 68 | This web site and all documentation is licensed under `Creative 69 | Commons 3.0 `_. 70 | -------------------------------------------------------------------------------- /garcon/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Utils 3 | ===== 4 | 5 | """ 6 | 7 | import hashlib 8 | 9 | 10 | def create_dictionary_key(dictionary): 11 | """Create a key that represents the content of the dictionary. 12 | 13 | Args: 14 | dictionary (dict): the dictionary to use. 15 | Return: 16 | str: the key that represents the content of the dictionary. 17 | """ 18 | 19 | if not isinstance(dictionary, dict): 20 | raise TypeError('The value passed should be a dictionary.') 21 | 22 | if not dictionary: 23 | raise ValueError('The dictionary cannot be empty.') 24 | 25 | sorted_dict = sorted(dictionary.items()) 26 | 27 | key_parts = ''.join([ 28 | "'{key}':'{val}';".format(key=key, val=val) 29 | for (key, val) in sorted_dict]) 30 | 31 | return hashlib.sha1(key_parts.encode('utf-8')).hexdigest() 32 | 33 | def non_throttle_error(exception): 34 | """Activity Runner. 35 | 36 | Determine whether SWF Exception was a throttle or a different error. 37 | 38 | Args: 39 | exception: botocore.exceptions.Client instance. 40 | Return: 41 | bool: True if exception was a throttle, False otherwise. 42 | """ 43 | 44 | return exception.response.get('Error').get('Code') != 'ThrottlingException' 45 | 46 | def throttle_backoff_handler(details): 47 | """Callback to be used when a throttle backoff is invoked. 48 | 49 | For more details see: https://github.com/litl/backoff/#event-handlers 50 | 51 | Args: 52 | dictionary (dict): Details of the backoff invocation. Valid keys 53 | include: 54 | target: reference to the function or method being invoked. 55 | args: positional arguments to func. 56 | kwargs: keyword arguments to func. 57 | tries: number of invocation tries so far. 58 | wait: seconds to wait (on_backoff handler only). 59 | value: value triggering backoff (on_predicate decorator only). 60 | """ 61 | 62 | activity = details['args'][0] 63 | activity.logger.info( 64 | 'Throttle Exception occurred on try {}. ' 65 | 'Sleeping for {} seconds'.format( 66 | details['tries'], details['wait'])) 67 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | try: 2 | from unittest.mock import MagicMock 3 | except: 4 | from mock import MagicMock 5 | from botocore import exceptions 6 | import datetime 7 | import pytest 8 | import json 9 | 10 | from garcon import utils 11 | 12 | 13 | def test_create_dictionary_key_with_string(): 14 | """Try creating a key using a string instead of a dict. 15 | """ 16 | 17 | with pytest.raises(TypeError): 18 | utils.create_dictionary_key('something') 19 | 20 | 21 | def test_create_dictionary_key_with_empty_dict(): 22 | """Try creating a key using an empty dictionary. 23 | """ 24 | 25 | with pytest.raises(ValueError): 26 | utils.create_dictionary_key(dict()) 27 | 28 | 29 | def test_create_dictionary_key(): 30 | """Try creating a unique key from a dict. 31 | """ 32 | 33 | values = [ 34 | dict(foo=10), 35 | dict(foo2=datetime.datetime.now())] 36 | 37 | for value in values: 38 | assert len(utils.create_dictionary_key(value)) == 40 39 | 40 | def test_create_dictionary_key(): 41 | """Try creating a unique key from a dict. 42 | """ 43 | 44 | values = [ 45 | dict(foo=10), 46 | dict(foo2=datetime.datetime.now())] 47 | 48 | for value in values: 49 | assert len(utils.create_dictionary_key(value)) == 40 50 | 51 | 52 | def test_non_throttle_error(): 53 | """Assert SWF error is evaluated as non-throttle error properly. 54 | """ 55 | 56 | exception = exceptions.ClientError( 57 | {'Error': {'Code': 'ThrottlingException'}}, 58 | 'operationName') 59 | result = utils.non_throttle_error(exception) 60 | assert not utils.non_throttle_error(exception) 61 | 62 | exception = exceptions.ClientError( 63 | {'Error': {'Code': 'RateExceeded'}}, 64 | 'operationName') 65 | assert utils.non_throttle_error(exception) 66 | 67 | def test_throttle_backoff_handler(): 68 | """Assert backoff is logged correctly. 69 | """ 70 | 71 | mock_activity = MagicMock() 72 | details = dict( 73 | args=(mock_activity,), 74 | tries=5, 75 | wait=10) 76 | utils.throttle_backoff_handler(details) 77 | mock_activity.logger.info.assert_called_with( 78 | 'Throttle Exception occurred on try {}. ' 79 | 'Sleeping for {} seconds'.format( 80 | details['tries'], details['wait'])) 81 | -------------------------------------------------------------------------------- /tests/test_param.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from garcon import param 4 | 5 | 6 | def test_base_param_class(): 7 | """Test the base class: cannot get data and return no requirements. 8 | """ 9 | 10 | current_param = param.BaseParam() 11 | with pytest.raises(NotImplementedError): 12 | current_param.get_data({}) 13 | 14 | assert not list(current_param.requirements) 15 | 16 | 17 | def test_static_param(): 18 | """Test the behavior of the static param class 19 | """ 20 | 21 | message = 'Hello World' 22 | current_param = param.StaticParam(message) 23 | assert current_param.get_data({}) is message 24 | assert not list(current_param.requirements) 25 | 26 | 27 | def test_default_param(): 28 | """Test the behavior of the default param class. 29 | """ 30 | 31 | key = 'context.key' 32 | message = 'Hello World' 33 | current_param = param.Param(key) 34 | requirements = list(current_param.requirements) 35 | assert current_param.get_data({key: message}) is message 36 | assert requirements[0] is key 37 | 38 | 39 | def test_all_requirements(): 40 | """Test getting all the requirements. 41 | """ 42 | 43 | keys = ['context.key1', 'context.key2', 'context.key3'] 44 | manual_keys = ['context.manual_key1', 'context.manual_key2'] 45 | params = [param.Param(key) for key in keys] 46 | params += manual_keys 47 | params += [param.StaticParam('Value')] 48 | params = [param.parametrize(current_param) for current_param in params] 49 | 50 | resp = param.get_all_requirements(params) 51 | for key in keys: 52 | assert key in resp 53 | 54 | for manual_key in manual_keys: 55 | assert manual_key in resp 56 | 57 | assert 'Value' not in resp 58 | 59 | 60 | def test_parametrize(): 61 | """Test parametrize. 62 | 63 | Parametrize only allows objects that inherits BaseParam or string. 64 | """ 65 | 66 | keys = ['context.key1', 'context.key2', 'context.key3'] 67 | manual_keys = ['context.manual_key1', 'context.manual_key2'] 68 | params = [param.Param(key) for key in keys] 69 | params += manual_keys 70 | params += [param.StaticParam('Value')] 71 | params = [param.parametrize(current_param) for current_param in params] 72 | 73 | for current_param in params: 74 | assert isinstance(current_param, param.BaseParam) 75 | 76 | with pytest.raises(param.UnknownParamException): 77 | param.parametrize(list('Unknown')) 78 | -------------------------------------------------------------------------------- /garcon/log.py: -------------------------------------------------------------------------------- 1 | """Garcon logger module 2 | """ 3 | 4 | import logging 5 | 6 | 7 | LOGGER_PREFIX = 'garcon' 8 | 9 | 10 | class GarconLogger: 11 | """This class is meant to be extended to get the Garcon logger feature 12 | The logger injects the execution context into the logger name. 13 | 14 | This is used by the Activity class in Garcon and allows you to log from 15 | the activity object. Typically, you can log from a Garcon task and it will 16 | prefix your log messages with execution context information (domain, 17 | workflow_id, run_id). 18 | 19 | Requirements: 20 | Your loggers need to be set up so there is at least one of them with a name 21 | prefixed with LOGGER_PREFIX. The Garcon logger will inherit the handlers 22 | from that logger. 23 | 24 | The formatter for your handler(s) must use the logger name. 25 | Formatter Example:: 26 | 27 | %(asctime)s - %(name)s - %(levelname)s - %(message)s 28 | 29 | This formatter will generate a log message as follow: 30 | '2015-01-15 - garcon.[domain].[workflow_id].[run_id] - [level] - [message]' 31 | """ 32 | 33 | @property 34 | def logger(self): 35 | """Return the appropriate logger. Default to LOGGER_PREFIX if 36 | no logger name was set. 37 | 38 | Return: 39 | logging.Logger: a logger object 40 | """ 41 | 42 | return logging.getLogger( 43 | getattr(self, 'logger_name', None) or LOGGER_PREFIX) 44 | 45 | def set_log_context(self, execution_context): 46 | """Set a logger name with execution context passed in. 47 | 48 | Args: 49 | execution_context (dict): execution context information 50 | """ 51 | 52 | if ('execution.domain' in execution_context and 53 | 'execution.workflow_id' in execution_context and 54 | 'execution.run_id' in execution_context): 55 | 56 | self.logger_name = get_logger_namespace(execution_context) 57 | 58 | def unset_log_context(self): 59 | """Unset the logger name. 60 | """ 61 | 62 | self.logger_name = None 63 | 64 | 65 | def get_logger_namespace(execution_context): 66 | """Return the logger namespace for a given execution context. 67 | 68 | Args: 69 | execution_context (dict): execution context information 70 | """ 71 | 72 | return '.'.join([ 73 | LOGGER_PREFIX, 74 | execution_context.get('execution.domain'), 75 | execution_context.get('execution.workflow_id'), 76 | execution_context.get('execution.run_id')]) 77 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ master ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ master ] 20 | schedule: 21 | - cron: '39 1 * * 5' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'python' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] 37 | # Learn more: 38 | # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed 39 | 40 | steps: 41 | - name: Checkout repository 42 | uses: actions/checkout@v2 43 | 44 | # Initializes the CodeQL tools for scanning. 45 | - name: Initialize CodeQL 46 | uses: github/codeql-action/init@v1 47 | with: 48 | languages: ${{ matrix.language }} 49 | # If you wish to specify custom queries, you can do so here or in a config file. 50 | # By default, queries listed here will override any specified in a config file. 51 | # Prefix the list here with "+" to use these queries and those in the config file. 52 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 53 | 54 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 55 | # If this step fails, then you should remove it and run the build manually (see below) 56 | - name: Autobuild 57 | uses: github/codeql-action/autobuild@v1 58 | 59 | # ℹ️ Command-line programs to run using the OS shell. 60 | # 📚 https://git.io/JvXDl 61 | 62 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 63 | # and modify them (or add more) to build your code if your project 64 | # uses a compiled language 65 | 66 | #- run: | 67 | # make bootstrap 68 | # make release 69 | 70 | - name: Perform CodeQL Analysis 71 | uses: github/codeql-action/analyze@v1 72 | -------------------------------------------------------------------------------- /docs/guide/generators.rst: -------------------------------------------------------------------------------- 1 | Generators 2 | ========== 3 | 4 | Generators spawn one or more instances of an activity based on values provided 5 | in the context. 6 | 7 | One of our use case includes a job that calls an API each day to get metrics 8 | for all the countries in the world. If the API fails for one country, the entire 9 | activity fails — retrying it means we will have to restart the entire list of 10 | countries. 11 | 12 | Instead of having one activity to do all calls, it’s a lot more robust to have 13 | one activity per country and have a retry mechanism applied to it. Failures 14 | will only be contained for one country that has failed instead of all. 15 | 16 | Note: 17 | Be aware that SWF has a limit on the number of events the history can hold, 18 | always make sure the number of activities spawned by the generator will 19 | allow enough room. 20 | 21 | Example: 22 | 23 | .. code-block:: python 24 | 25 | from garcon import activity 26 | from garcon import runner 27 | from garcon import task 28 | import random 29 | 30 | 31 | domain = 'dev' 32 | name = 'country_flow' 33 | create = activity.create(domain, name) 34 | 35 | 36 | def country_generator(context): 37 | # We limit this so you can more easily see the failures / retries. 38 | total_countries = 6 39 | for country_id in range(1, total_countries): 40 | yield {'generator.country_id': country_id} 41 | 42 | 43 | @task.decorate() 44 | def unstable_country_task(activity, country_id): 45 | num = int(random.random() * 4) 46 | base = 'activity_2_country_id_{country_id}'.format( 47 | country_id=country_id) 48 | 49 | if num == 3: 50 | # Make the task randomly fail. 51 | print(base, 'has failed') 52 | raise Exception('fails') 53 | 54 | print(base, 'has succeeded') 55 | 56 | 57 | test_activity_1 = create( 58 | name='activity_1', 59 | tasks=runner.Sync( 60 | lambda context, activity: print('activity_1'))) 61 | 62 | test_activity_2 = create( 63 | name='activity_2', 64 | requires=[test_activity_1], 65 | generators=[ 66 | country_generator], 67 | retry=3, 68 | tasks=runner.Sync( 69 | unstable_country_task.fill(country_id='generator.country_id'))) 70 | 71 | test_activity_3 = create( 72 | name='activity_3', 73 | requires=[test_activity_2], 74 | tasks=runner.Sync( 75 | lambda context, activity: print('end of flow'))) 76 | 77 | Note: 78 | Generators attribute takes a list of generators. If you have a flow that 79 | has a date range, list of countries, you can create activities that 80 | corresponds to one day and one specific countries. If you have 10 days in 81 | your range and 20 countries, you will run 200 activities. 82 | -------------------------------------------------------------------------------- /garcon/param.py: -------------------------------------------------------------------------------- 1 | """ 2 | Param 3 | ===== 4 | 5 | Params are values that are passed to the activities. They are either provided 6 | by the execution context (see Param) or are statically provided at the runtime 7 | of the activity (see StaticParam). Custom params should extend the Param class. 8 | 9 | Note: 10 | Make sure the custom param class lists out all the dependencies to the 11 | execution context in `requirements`. 12 | """ 13 | 14 | 15 | class BaseParam: 16 | """Base Param Class. 17 | 18 | Provides the structure and required methods of any param class. 19 | """ 20 | 21 | @property 22 | def requirements(self): 23 | """Return the requirements for this param. 24 | """ 25 | 26 | return 27 | yield 28 | 29 | def get_data(self, context): 30 | """Get the data. 31 | 32 | Args: 33 | context (dict): the context (optional) in which the data might be 34 | found. For Static Param this won't be necessary. 35 | """ 36 | 37 | raise NotImplementedError() 38 | 39 | 40 | class Param(BaseParam): 41 | 42 | def __init__(self, context_key): 43 | """Create a default param. 44 | 45 | Args: 46 | context_key (str): the context key. 47 | """ 48 | 49 | self.context_key = context_key 50 | 51 | @property 52 | def requirements(self): 53 | """Return the requirements for this param. 54 | """ 55 | 56 | yield self.context_key 57 | 58 | def get_data(self, context): 59 | """Get value from the context. 60 | 61 | Args: 62 | context (dict): the context in which the data might be found based 63 | on the key provided. 64 | 65 | Return: 66 | obj: an object from the context that corresponds to the context 67 | key. 68 | """ 69 | 70 | return context.get(self.context_key, None) 71 | 72 | 73 | class StaticParam(BaseParam): 74 | 75 | def __init__(self, value): 76 | """Create a static param. 77 | 78 | Args: 79 | value (str): the value of the param. 80 | """ 81 | 82 | self.value = value 83 | 84 | def get_data(self, context): 85 | """Get value from the context. 86 | 87 | Args: 88 | context (dict): execution context (not used.) 89 | """ 90 | 91 | return self.value 92 | 93 | 94 | def get_all_requirements(params): 95 | """Get all the requirements from a list of params. 96 | 97 | Args: 98 | params (list): The list of params. 99 | """ 100 | 101 | requirements = [] 102 | for param in params: 103 | requirements += list(param.requirements) 104 | return requirements 105 | 106 | 107 | def parametrize(requirement): 108 | """Parametrize a requirement. 109 | 110 | Args: 111 | requirement (*): the requirement to parametrize. 112 | """ 113 | 114 | if isinstance(requirement, str): 115 | return Param(requirement) 116 | elif isinstance(requirement, BaseParam): 117 | return requirement 118 | raise UnknownParamException() 119 | 120 | 121 | class UnknownParamException(Exception): 122 | pass 123 | -------------------------------------------------------------------------------- /garcon/context.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Context 4 | ======= 5 | 6 | Context carries information that have been retrieved from the different SWF 7 | events of an execution. 8 | """ 9 | 10 | import json 11 | 12 | 13 | class ExecutionContext: 14 | 15 | def __init__(self, events=None): 16 | """Create the execution context. 17 | 18 | An execution context gathers the execution input and the result of all 19 | the activities that have successfully ran. It also adds the execution 20 | input into the mix (for logger purposes). 21 | 22 | Args: 23 | events (list): optional list of all the events. 24 | """ 25 | 26 | self.current = {} 27 | self.workflow_input = {} 28 | 29 | if events: 30 | for event in events: 31 | self.add(event) 32 | 33 | def add(self, event): 34 | """Add an event into the execution context. 35 | 36 | The events are the ones coming from SWF directly (so the fields are the 37 | ones we expect). 38 | 39 | Args: 40 | event (dict): the event to add to the context. 41 | """ 42 | 43 | event_type = event.get('eventType') 44 | if event_type == 'ActivityTaskCompleted': 45 | self.add_activity_result(event) 46 | elif event_type == 'WorkflowExecutionStarted': 47 | self.set_execution_input(event) 48 | 49 | def set_workflow_execution_info(self, execution_info, domain): 50 | """Add the workflow execution info. 51 | 52 | Workflow execution info contains the domain, workflow id and run id. 53 | This allows the logger to properly namespace the messages and 54 | facilitate debugging. 55 | 56 | Args: 57 | execution_info (dict): the execution information. 58 | domain (str): the current domain 59 | """ 60 | 61 | if ('workflowExecution' in execution_info and 62 | 'workflowId' in execution_info['workflowExecution'] and 63 | 'runId' in execution_info['workflowExecution']): 64 | 65 | workflow_execution = execution_info['workflowExecution'] 66 | self.current.update({ 67 | 'execution.domain': domain, 68 | 'execution.workflow_id': workflow_execution['workflowId'], 69 | 'execution.run_id': workflow_execution['runId'] 70 | }) 71 | 72 | def set_execution_input(self, execution_event): 73 | """Add the workflow execution input. 74 | 75 | Please note the input within the execution event should always be a 76 | json string. 77 | 78 | Args: 79 | execution_event (str): the execution event information. 80 | """ 81 | 82 | attributes = execution_event['workflowExecutionStartedEventAttributes'] 83 | result = attributes.get('input') 84 | if result: 85 | result = json.loads(result) 86 | self.workflow_input = result 87 | self.current.update(result) 88 | 89 | def add_activity_result(self, activity_event): 90 | """Add an activity result. 91 | 92 | Please note: the result of an activity event should always be a json 93 | string. 94 | 95 | Args: 96 | activity_event (str): json object that represents the activity 97 | information. 98 | """ 99 | 100 | attributes = activity_event['activityTaskCompletedEventAttributes'] 101 | result = attributes.get('result') 102 | 103 | if result: 104 | self.current.update(json.loads(result)) 105 | -------------------------------------------------------------------------------- /garcon/event.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from garcon import activity 3 | from garcon import context 4 | import json 5 | 6 | 7 | def activity_states_from_events(events): 8 | """Get activity states from a list of events. 9 | 10 | The workflow events contains the different states of our activities. This 11 | method consumes the logs, and regenerates a dictionnary with the list of 12 | all the activities and their states. 13 | 14 | Note: 15 | Please note: from the list of events, only activities that have been 16 | registered are accessible. For all the others that have not yet 17 | started, they won't be part of this list. 18 | 19 | Args: 20 | events (dict): list of all the events. 21 | Return: 22 | `dict`: the activities and their state. 23 | """ 24 | 25 | events = sorted(events, key=lambda item: item.get('eventId')) 26 | event_id_info = dict() 27 | activity_events = dict() 28 | 29 | for event in events: 30 | event_id = event.get('eventId') 31 | event_type = event.get('eventType') 32 | 33 | if event_type == 'ActivityTaskScheduled': 34 | activity_info = event.get('activityTaskScheduledEventAttributes') 35 | activity_id = activity_info.get('activityId') 36 | activity_name = activity_info.get('activityType').get('name') 37 | event_id_info.update({ 38 | event_id: { 39 | 'activity_name': activity_name, 40 | 'activity_id': activity_id} 41 | }) 42 | 43 | activity_events.setdefault( 44 | activity_name, {}).setdefault( 45 | activity_id, 46 | activity.ActivityState(activity_id)).add_state( 47 | activity.ACTIVITY_SCHEDULED) 48 | 49 | elif event_type == 'ActivityTaskFailed': 50 | activity_info = event.get('activityTaskFailedEventAttributes') 51 | activity_event = event_id_info.get( 52 | activity_info.get('scheduledEventId')) 53 | activity_id = activity_event.get('activity_id') 54 | 55 | activity_events.setdefault( 56 | activity_event.get('activity_name'), {}).setdefault( 57 | activity_id, 58 | activity.ActivityState(activity_id)).add_state( 59 | activity.ACTIVITY_FAILED) 60 | 61 | elif event_type == 'ActivityTaskCompleted': 62 | activity_info = event.get('activityTaskCompletedEventAttributes') 63 | activity_event = event_id_info.get( 64 | activity_info.get('scheduledEventId')) 65 | activity_id = activity_event.get('activity_id') 66 | 67 | activity_events.setdefault( 68 | activity_event.get('activity_name'), {}).setdefault( 69 | activity_id, 70 | activity.ActivityState(activity_id)).add_state( 71 | activity.ACTIVITY_COMPLETED) 72 | 73 | result = json.loads(activity_info.get('result') or '{}') 74 | activity_events.get( 75 | activity_event.get('activity_name')).get( 76 | activity_id).set_result(result) 77 | 78 | return activity_events 79 | 80 | 81 | def get_current_context(events): 82 | """Get the current context from the list of events. 83 | 84 | Each activity returns bits of information that needs to be provided to the 85 | next activities. 86 | 87 | Args: 88 | events (list): List of events. 89 | Return: 90 | dict: The current context. 91 | """ 92 | 93 | events = sorted(events, key=lambda item: item.get('eventId')) 94 | execution_context = context.ExecutionContext(events) 95 | return execution_context 96 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | |BuildStatus| |Downloads| |CoverageStatus| 2 | 3 | Lightweight library for AWS SWF. 4 | 5 | Garcon deals with easy going clients and kitchens. It takes orders 6 | from clients (deciders), and send them to the kitchen (activities). 7 | Difficult clients and kitchens can be handled directly by the 8 | restaurant manager. 9 | 10 | Requirements 11 | ~~~~~~~~~~~~ 12 | 13 | - Python 3.8, 3.9, 3.10, 3.11, 3.12 (tested) 14 | - Boto3 (tested) 15 | 16 | Goal 17 | ~~~~ 18 | 19 | The goal of this library is to allow the creation of Amazon Simple 20 | Workflow without the need to worry about the orchestration of the 21 | different activities and building out the different workers. This 22 | framework aims to help simple workflows. If you have a more complex 23 | case, you might want to use directly boto3. 24 | 25 | Code sample 26 | ~~~~~~~~~~~ 27 | 28 | The code sample shows a workflow where a user enters a coffee shop, orders 29 | a coffee and a chocolate chip cookie. All ordered items are prepared and 30 | completed, the user pays the order, receives the ordered items, then leave 31 | the shop. 32 | 33 | The code below represents the workflow decider. For the full code sample, 34 | see the `example`_. 35 | 36 | .. code:: python 37 | 38 | enter = schedule('enter', self.create_enter_coffee_activity) 39 | enter.wait() 40 | 41 | total = 0 42 | for item in ['coffee', 'chocolate_chip_cookie']: 43 | activity_name = 'order_{item}'.format(item=item) 44 | activity = schedule(activity_name, 45 | self.create_order_activity, 46 | input={'item': item}) 47 | total += activity.result.get('price') 48 | 49 | pay_activity = schedule( 50 | 'pay', self.create_payment_activity, 51 | input={'total': total}) 52 | 53 | get_order = schedule('get_order', self.create_get_order_activity) 54 | 55 | # Waiting for paying and getting the order to complete before 56 | # we let the user leave the coffee shop. 57 | pay_activity.wait(), get_order.wait() 58 | schedule('leave_coffee_shop', self.create_leave_coffee_shop) 59 | 60 | 61 | Application architecture 62 | ~~~~~~~~~~~~~~~~~~~~~~~~ 63 | 64 | :: 65 | 66 | . 67 | ├── cli.py # Instantiate the workers 68 | ├── flows # All your application flows. 69 | │ ├── __init__.py 70 | │ └── example.py # Should contain a structure similar to the code sample. 71 | ├── tasks # All your tasks 72 | │ ├── __init__.py 73 | │ └── s3.py # Task that focuses on s3 files. 74 | └── task_example.py # Your different tasks. 75 | 76 | Trusted by 77 | ~~~~~~~~~~ 78 | 79 | |The Orchard| |Sony Music| |DataArt| 80 | 81 | Contributors 82 | ~~~~~~~~~~~~ 83 | 84 | - Michael Ortali (Author) 85 | - Adam Griffiths 86 | - Raphael Antonmattei 87 | - John Penner 88 | 89 | .. _xethorn: github.com/xethorn 90 | .. _rantonmattei: github.com/rantonmattei 91 | .. _someboredkiddo: github.com/someboredkiddo 92 | .. _example: https://github.com/xethorn/garcon/tree/master/example/custom_decider 93 | 94 | .. |BuildStatus| image:: https://github.com/xethorn/garcon/workflows/Build/badge.svg 95 | :target: https://github.com/xethorn/garcon/actions?query=workflow%3ABuild+branch%3Amaster 96 | 97 | .. |Downloads| image:: https://img.shields.io/pypi/dm/garcon.svg 98 | :target: https://coveralls.io/r/xethorn/garcon?branch=master 99 | 100 | .. |CoverageStatus| image:: https://coveralls.io/repos/xethorn/garcon/badge.svg?branch=master 101 | :target: https://coveralls.io/r/xethorn/garcon?branch=master 102 | 103 | .. |The Orchard| image:: https://media-exp1.licdn.com/dms/image/C4E0BAQGi7o5g9l4JWg/company-logo_200_200/0/1519855981606?e=2159024400&v=beta&t=WBe-gOK2b30vUTGKbA025i9NFVDyOrS4Fotx9fMEZWo 104 | :target: https://theorchard.com 105 | 106 | .. |Sony Music| image:: https://media-exp1.licdn.com/dms/image/C4D0BAQE9rvU-3ig-jg/company-logo_200_200/0/1604099587507?e=2159024400&v=beta&t=eAAubphf_fI-5GEb0ak1QnmtRHmc8466Qj4sGrCsWYc 107 | :target: https://www.sonymusic.com/ 108 | 109 | .. |DataArt| image:: https://media-exp1.licdn.com/dms/image/C4E0BAQGRi6OIlNQG8Q/company-logo_200_200/0/1519856519357?e=2159024400&v=beta&t=oi6HQpzoeTKA082s-8Ft75vGTvAkEp4VHRyMLeOHXoo 110 | :target: https://www.dataart.com/ 111 | -------------------------------------------------------------------------------- /garcon/runner.py: -------------------------------------------------------------------------------- 1 | """ 2 | Task runners 3 | ============ 4 | 5 | The task runners are responsible for running all the tasks (either in series 6 | or in parallel). There's only one task runner per activity. 7 | """ 8 | 9 | from concurrent import futures 10 | from concurrent.futures import ThreadPoolExecutor 11 | 12 | from garcon.task import flatten 13 | 14 | 15 | DEFAULT_TASK_TIMEOUT = 600 # 10 minutes. 16 | DEFAULT_TASK_HEARTBEAT = 600 # 10 minutes 17 | 18 | 19 | class NoRunnerRequirementsFound(Exception): 20 | pass 21 | 22 | 23 | class RunnerMissing(Exception): 24 | pass 25 | 26 | 27 | class BaseRunner(): 28 | 29 | def __init__(self, *args): 30 | self.tasks = args 31 | 32 | def timeout(self, context): 33 | """Calculate and return the timeout for an activity. 34 | 35 | The calculation of the timeout is pessimistic: it takes the worse case 36 | scenario (even for asynchronous task lists, it supposes there is only 37 | one thread completed at a time.) 38 | 39 | Return: 40 | str: The timeout (boto requires the timeout to be a string and not 41 | a regular number.) 42 | """ 43 | 44 | timeout = 0 45 | 46 | for task in flatten(self.tasks, context): 47 | task_timeout = DEFAULT_TASK_TIMEOUT 48 | task_details = getattr(task, '__garcon__', None) 49 | 50 | if task_details: 51 | task_timeout = task_details.get( 52 | 'timeout', DEFAULT_TASK_TIMEOUT) 53 | 54 | timeout = timeout + task_timeout 55 | 56 | return timeout 57 | 58 | def heartbeat(self, context): 59 | """Calculate and return the heartbeat for an activity. 60 | 61 | The heartbeat represents when an actvitity should be sending a signal 62 | to SWF that it has not completed yet. The heartbeat is sent everytime 63 | a new task is going to be launched. 64 | 65 | Similar to the `BaseRunner.timeout`, the heartbeat is pessimistic, it 66 | looks at the largest heartbeat and set it up. 67 | 68 | Return: 69 | str: The heartbeat timeout (boto requires the timeout to be a 70 | string not a regular number.) 71 | """ 72 | 73 | heartbeat = 0 74 | 75 | for task in flatten(self.tasks, context): 76 | task_details = getattr(task, '__garcon__', None) 77 | task_heartbeat = DEFAULT_TASK_HEARTBEAT 78 | 79 | if task_details: 80 | task_heartbeat = task_details.get( 81 | 'heartbeat', DEFAULT_TASK_HEARTBEAT) 82 | 83 | if task_heartbeat > heartbeat: 84 | heartbeat = task_heartbeat 85 | 86 | return heartbeat 87 | 88 | def requirements(self, context): 89 | """Find all the requirements from the list of tasks and return it. 90 | 91 | If a task does not use the `task.decorate`, no assumptions can be made 92 | on which values from the context will be used, and it will raise an 93 | exception. 94 | 95 | Raise: 96 | NoRequirementFound: The exception when no requirements have been 97 | mentioned in at least one or more tasks. 98 | 99 | Return: 100 | set: the list of the required values from the context. 101 | """ 102 | 103 | requirements = [] 104 | 105 | # Get all the tasks and the lists (so the .fill on lists are also 106 | # considered.) 107 | all_tasks = list(self.tasks) + list(flatten(self.tasks, context)) 108 | for task in all_tasks: 109 | task_details = getattr(task, '__garcon__', None) 110 | if task_details: 111 | requirements += task_details.get('requirements', []) 112 | else: 113 | raise NoRunnerRequirementsFound() 114 | return set(requirements) 115 | 116 | def execute(self, activity, context): 117 | """Execution of the tasks. 118 | """ 119 | 120 | raise NotImplementedError 121 | 122 | 123 | class Sync(BaseRunner): 124 | 125 | def execute(self, activity, context): 126 | result = dict() 127 | for task in flatten(self.tasks, context): 128 | activity.heartbeat() 129 | task_context = dict(list(result.items()) + list(context.items())) 130 | resp = task(task_context, activity=activity) 131 | result.update(resp or dict()) 132 | return result 133 | 134 | 135 | class Async(BaseRunner): 136 | 137 | def __init__(self, *args, **kwargs): 138 | self.tasks = args 139 | self.max_workers = kwargs.get('max_workers', 3) 140 | 141 | def execute(self, activity, context): 142 | result = dict() 143 | with ThreadPoolExecutor(max_workers=self.max_workers) as executor: 144 | tasks = [] 145 | for task in flatten(self.tasks, context): 146 | tasks.append(executor.submit(task, context, activity=activity)) 147 | 148 | for future in futures.as_completed(tasks): 149 | activity.heartbeat() 150 | data = future.result() 151 | result.update(data or {}) 152 | return result 153 | 154 | 155 | class External(BaseRunner): 156 | 157 | def __init__(self, timeout=None, heartbeat=None): 158 | """Create the External Runner. 159 | 160 | Args: 161 | timeout (int): activity timeout in seconds (mandatory) 162 | heartbeat (int): heartbeat timeout in seconds, if not defined, it 163 | will be equal to the timeout. 164 | """ 165 | 166 | assert timeout, 'External runner requires a timeout.' 167 | 168 | self.timeout = lambda ctx=None: timeout 169 | self.heartbeat = lambda ctx=None: (heartbeat or timeout) 170 | -------------------------------------------------------------------------------- /example/custom_decider/workflow.py: -------------------------------------------------------------------------------- 1 | """ 2 | Workflow: Ordering a coffee in a shop. 3 | -------------------------------------- 4 | 5 | This is a workflow when someone is entering in a shop, ordering a coffee and 6 | a chocolate chip cookie, pays the bill and tips, get the order and leave the 7 | coffee shop. 8 | 9 | Workflow: 10 | 1. Enter in the coffee shop. 11 | 2. Order. 12 | 2.1 Coffee. 13 | 2.2 Chocolate chip cookie. 14 | 3. Finalize the order (any of the activites below can be done in any order) 15 | 3.1 Pays. 16 | 3.2 Get the order. 17 | 4. Leave the coffee shop. 18 | 19 | Result: 20 | entering coffee shop 21 | ordering: coffee 22 | ordering: chocolate_chip_cookie 23 | pay $6.98 24 | get order 25 | leaving the coffee shop 26 | """ 27 | 28 | from garcon import activity 29 | from garcon import runner 30 | from garcon import task 31 | 32 | 33 | class Workflow: 34 | 35 | def __init__(self, client, domain, name): 36 | """Create the workflow. 37 | 38 | Args: 39 | domain (str): the domain to attach this workflow to. 40 | name (str): the name of the workflow. 41 | """ 42 | self.name = name 43 | self.domain = domain 44 | self.client = client 45 | self.create_activity = activity.create(client, domain, name) 46 | 47 | def decider(self, schedule, context=None): 48 | """Custom deciders. 49 | 50 | The custom decider is a method that allows you to write more complex 51 | workflows based on some input and output. In our case, we have a 52 | workflow that triggers n steps based on a value passed in the context 53 | and triggers a specific behavior if another value is present. 54 | 55 | Args: 56 | schedule (callable): method that is used to schedule a specific 57 | activity. 58 | context (dict): the current context. 59 | """ 60 | 61 | # The schedule method takes an "activity id" (it's not the name of 62 | # the activity, it's a unique id that defines the activity you 63 | # are trying to schedule, it's your responsibility to make it unique). 64 | # The second argument is the reference to the ActivityWorker. 65 | enter = schedule('enter', self.create_enter_coffee_activity) 66 | enter.wait() 67 | 68 | total = 0 69 | for item in ['coffee', 'chocolate_chip_cookie']: 70 | activity_name = 'order_{item}'.format(item=item) 71 | activity = schedule(activity_name, 72 | self.create_order_activity, 73 | input={'item': item}) 74 | 75 | # Getting the result of the previous activity to calculate the 76 | # total the user will be charged. 77 | total += activity.result.get('price') 78 | 79 | # The `input` param is the data that will be sent to the activity. 80 | pay_activity = schedule( 81 | 'pay', self.create_payment_activity, 82 | input={'total': total}) 83 | 84 | get_order = schedule('get_order', self.create_get_order_activity) 85 | pay_activity.wait(), get_order.wait() 86 | 87 | # Leaving the coffee shop will not happen before the payment has 88 | # been processed and the order has been taken. 89 | schedule('leave_coffee_shop', self.create_leave_coffee_shop) 90 | 91 | @property 92 | def create_enter_coffee_activity(self): 93 | """Create the activity when user enters coffee shop.""" 94 | return self.create_activity( 95 | name='enter', 96 | run=runner.Sync( 97 | lambda context, activity: 98 | print('entering coffee shop'))) 99 | 100 | @property 101 | def create_order_activity(self): 102 | """Create an order for an item. 103 | 104 | Returns: 105 | ActivityWorker: the activity that create an order item. 106 | """ 107 | @task.decorate() 108 | def order(activity, item=None): 109 | print('ordering: {}'.format(item)) 110 | price = 0.00 111 | if item == 'coffee': 112 | price = 3.99 113 | if item == 'chocolate_chip_cookie': 114 | price = 2.99 115 | return {'price': price} 116 | 117 | return self.create_activity( 118 | name='order', 119 | run=runner.Sync(order.fill(item='item'))) 120 | 121 | @property 122 | def create_payment_activity(self): 123 | """Pay the bill. 124 | 125 | Returns: 126 | ActivityWorker: the activity that pays the bills. 127 | """ 128 | @task.decorate() 129 | def payment(activity, total=None): 130 | print('pay ${}'.format(total)) 131 | 132 | return self.create_activity( 133 | name='payment', 134 | run=runner.Sync(payment.fill(total='total'))) 135 | 136 | @property 137 | def create_get_order_activity(self): 138 | """Get order. 139 | 140 | Returns: 141 | ActivityWorker: the activity that gets the order. 142 | """ 143 | @task.decorate() 144 | def payment(activity): 145 | print('get order') 146 | 147 | return self.create_activity( 148 | name='get_order', 149 | run=runner.Sync(payment.fill())) 150 | 151 | @property 152 | def create_leave_coffee_shop(self): 153 | """Leave the coffee shop. 154 | 155 | Returns: 156 | ActivityWorker: the activity that leaves the coffee shop. 157 | """ 158 | @task.decorate() 159 | def leave(activity): 160 | print('Leaving the coffee shop') 161 | 162 | return self.create_activity( 163 | name='leave', 164 | run=runner.Sync(leave.fill())) 165 | -------------------------------------------------------------------------------- /tests/test_runner.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import MagicMock 2 | 3 | import pytest 4 | 5 | from garcon import activity 6 | from garcon import runner 7 | from garcon import task 8 | 9 | 10 | EMPTY_CONTEXT = dict() 11 | 12 | 13 | def test_execute_default_task_runner(): 14 | """Should throw an exception. 15 | """ 16 | 17 | current_runner = runner.BaseRunner() 18 | with pytest.raises(NotImplementedError): 19 | current_runner.execute(None, None) 20 | 21 | 22 | def test_synchronous_tasks(monkeypatch, boto_client): 23 | """Test synchronous tasks. 24 | """ 25 | 26 | monkeypatch.setattr(activity.ActivityExecution, 'heartbeat', 27 | lambda self: None) 28 | 29 | resp = dict(foo='bar') 30 | current_runner = runner.Sync( 31 | MagicMock(), MagicMock(return_value=resp)) 32 | current_activity = activity.ActivityExecution( 33 | boto_client, 'activityId', 'taskToken', '{"context": "value"}') 34 | 35 | result = current_runner.execute(current_activity, EMPTY_CONTEXT) 36 | 37 | assert len(current_runner.tasks) == 2 38 | 39 | for current_task in task.flatten(current_runner.tasks, EMPTY_CONTEXT): 40 | assert current_task.called 41 | 42 | assert resp == result 43 | 44 | 45 | def test_aynchronous_tasks(monkeypatch, boto_client): 46 | """Test asynchronous tasks. 47 | """ 48 | 49 | monkeypatch.setattr(activity.ActivityExecution, 'heartbeat', 50 | lambda self: None) 51 | 52 | tasks = [MagicMock() for i in range(5)] 53 | tasks[2].return_value = dict(oi='mondo') 54 | tasks[4].return_value = dict(bonjour='monde') 55 | expected_response = dict( 56 | list(tasks[2].return_value.items()) + 57 | list(tasks[4].return_value.items())) 58 | 59 | workers = 2 60 | current_runner = runner.Async(*tasks, max_workers=workers) 61 | 62 | assert current_runner.max_workers == workers 63 | assert len(current_runner.tasks) == len(tasks) 64 | 65 | current_activity = activity.ActivityExecution(boto_client, 66 | 'activityId', 'taskToken', '{"hello": "world"}') 67 | 68 | resp = current_runner.execute(current_activity, current_activity.context) 69 | 70 | for current_task in tasks: 71 | assert current_task.called 72 | 73 | assert resp == expected_response 74 | 75 | 76 | def test_calculate_timeout_with_no_tasks(): 77 | """Task list without task has no timeout. 78 | """ 79 | 80 | task_list = runner.BaseRunner() 81 | assert task_list.timeout(EMPTY_CONTEXT) == 0 82 | 83 | 84 | def test_calculate_heartbeat_with_no_tasks(): 85 | """Task list without tasks has no heartbeat. 86 | """ 87 | 88 | task_list = runner.BaseRunner() 89 | assert task_list.heartbeat(EMPTY_CONTEXT) == 0 90 | 91 | 92 | def test_calculate_default_timeout(): 93 | """Tasks that do not have a set timeout get the default timeout. 94 | """ 95 | 96 | task_list = runner.BaseRunner(lambda x: x) 97 | assert task_list.timeout(EMPTY_CONTEXT) == runner.DEFAULT_TASK_TIMEOUT 98 | 99 | 100 | def test_calculate_default_heartbeat(): 101 | """Tasks that do not have a set timeout get the default timeout. 102 | """ 103 | 104 | task_list = runner.BaseRunner(lambda x: x) 105 | assert task_list.heartbeat(EMPTY_CONTEXT) == runner.DEFAULT_TASK_HEARTBEAT 106 | 107 | 108 | def test_calculate_timeout(): 109 | """Check methods that have set timeout. 110 | """ 111 | 112 | timeout = 10 113 | 114 | @task.timeout(timeout) 115 | def task_a(): 116 | pass 117 | 118 | current_runner = runner.BaseRunner(task_a) 119 | assert current_runner.timeout(EMPTY_CONTEXT) == timeout 120 | 121 | @task.decorate(timeout=timeout) 122 | def task_b(): 123 | pass 124 | 125 | current_runner = runner.BaseRunner(task_b) 126 | assert current_runner.timeout(EMPTY_CONTEXT) == timeout 127 | 128 | def task_c(): 129 | pass 130 | 131 | current_runner = runner.BaseRunner(task_a, task_c) 132 | current_timeout = current_runner.timeout(EMPTY_CONTEXT) 133 | expected_timeout = timeout + runner.DEFAULT_TASK_TIMEOUT 134 | assert current_timeout == expected_timeout 135 | 136 | 137 | def test_calculate_heartbeat(): 138 | """Check methods that have set timeout. 139 | """ 140 | 141 | @task.decorate(heartbeat=5) 142 | def task_a(): 143 | pass 144 | 145 | current_runner = runner.BaseRunner(task_a) 146 | assert current_runner.heartbeat(EMPTY_CONTEXT) == 5 147 | 148 | @task.decorate(heartbeat=3) 149 | def task_b(): 150 | pass 151 | 152 | current_runner = runner.BaseRunner(task_b) 153 | assert current_runner.heartbeat(EMPTY_CONTEXT) == 3 154 | 155 | @task.decorate(heartbeat=4498) 156 | def task_c(): 157 | pass 158 | 159 | def task_d(): 160 | pass 161 | 162 | current_runner = runner.BaseRunner( 163 | task_a, task_b, task_c, task_d) 164 | assert current_runner.heartbeat(EMPTY_CONTEXT) == 4498 165 | 166 | 167 | def test_runner_requirements(): 168 | """Test the requirements for the runner 169 | """ 170 | 171 | @task.decorate() 172 | def task_a(): 173 | pass 174 | 175 | @task.decorate(timeout=20) 176 | def task_b(): 177 | pass 178 | 179 | value_1 = 'context.value' 180 | value_2 = 'context.value_1' 181 | current_runner = runner.BaseRunner( 182 | task_a.fill(foo=value_1), 183 | task_b.fill(foobar=value_2)) 184 | 185 | requirements = current_runner.requirements(EMPTY_CONTEXT) 186 | assert len(requirements) == 2 187 | assert value_1 in requirements 188 | assert value_2 in requirements 189 | 190 | 191 | def test_runner_requirements_without_decoration(): 192 | """Should just throw an exception. 193 | """ 194 | 195 | def task_a(): 196 | pass 197 | 198 | current_runner = runner.BaseRunner(task_a) 199 | 200 | with pytest.raises(runner.NoRunnerRequirementsFound): 201 | current_runner.requirements(EMPTY_CONTEXT) 202 | -------------------------------------------------------------------------------- /tests/fixtures/decider.py: -------------------------------------------------------------------------------- 1 | history = {'events': [ 2 | {'eventId': 1, 3 | 'eventTimestamp': 1419350960.696, 4 | 'eventType': 'WorkflowExecutionStarted', 5 | 'workflowExecutionStartedEventAttributes': { 6 | 'childPolicy': 'TERMINATE', 7 | 'executionStartToCloseTimeout': '3600', 8 | 'parentInitiatedEventId': 0, 9 | 'taskList': {'name': 'garcon_decider'}, 10 | 'taskStartToCloseTimeout': '300', 11 | 'workflowType': {'name': 'garcon_decider', 'version': '1.0'}}}, 12 | {'decisionTaskScheduledEventAttributes': { 13 | 'startToCloseTimeout': '300', 14 | 'taskList': {'name': 'garcon_decider'}}, 15 | 'eventId': 2, 16 | 'eventTimestamp': 1419350960.696, 17 | 'eventType': 'DecisionTaskScheduled'}, 18 | {'decisionTaskStartedEventAttributes': { 19 | 'scheduledEventId': 2}, 20 | 'eventId': 3, 21 | 'eventTimestamp': 1419350960.767, 22 | 'eventType': 'DecisionTaskStarted'}, 23 | {'decisionTaskCompletedEventAttributes': { 24 | 'scheduledEventId': 2, 'startedEventId': 3}, 25 | 'eventId': 4, 26 | 'eventTimestamp': 1419350960.945, 27 | 'eventType': 'DecisionTaskCompleted'}, 28 | {'activityTaskScheduledEventAttributes': { 29 | 'activityId': 'workflow_name_activity_1-1', 30 | 'activityType': {'name': 'workflow_name_activity_1', 'version': '1.0'}, 31 | 'decisionTaskCompletedEventId': 4, 32 | 'heartbeatTimeout': '600', 33 | 'input': 'json object', 34 | 'scheduleToCloseTimeout': '3900', 35 | 'scheduleToStartTimeout': '300', 36 | 'startToCloseTimeout': '3600', 37 | 'taskList': {'name': 'garcon_workflow_name_activity_1'}}, 38 | 'eventId': 5, 39 | 'eventTimestamp': 1419350960.945, 40 | 'eventType': 'ActivityTaskScheduled'}, 41 | {'activityTaskStartedEventAttributes': { 42 | 'scheduledEventId': 5}, 43 | 'eventId': 6, 44 | 'eventTimestamp': 1419351025.473, 45 | 'eventType': 'ActivityTaskStarted'}, 46 | {'activityTaskCompletedEventAttributes': { 47 | 'scheduledEventId': 5, 'startedEventId': 6}, 48 | 'eventId': 7, 49 | 'eventTimestamp': 1419351025.534, 50 | 'eventType': 'ActivityTaskCompleted'}, 51 | {'decisionTaskScheduledEventAttributes': { 52 | 'startToCloseTimeout': '300', 53 | 'taskList': {'name': 'garcon_decider'}}, 54 | 'eventId': 8, 55 | 'eventTimestamp': 1419351025.534, 56 | 'eventType': 'DecisionTaskScheduled'}, 57 | {'decisionTaskStartedEventAttributes': {'scheduledEventId': 8}, 58 | 'eventId': 9, 59 | 'eventTimestamp': 1419351026.066, 60 | 'eventType': 'DecisionTaskStarted'}, 61 | {'decisionTaskCompletedEventAttributes': { 62 | 'scheduledEventId': 8, 'startedEventId': 9}, 63 | 'eventId': 10, 64 | 'eventTimestamp': 1419351026.232, 65 | 'eventType': 'DecisionTaskCompleted'}, 66 | {'activityTaskScheduledEventAttributes': { 67 | 'activityId': 'workflow_name_activity_2-1', 68 | 'activityType': {'name': 'workflow_name_activity_2', 'version': '1.0'}, 69 | 'decisionTaskCompletedEventId': 10, 70 | 'heartbeatTimeout': '600', 71 | 'input': 'json object', 72 | 'scheduleToCloseTimeout': '3900', 73 | 'scheduleToStartTimeout': '300', 74 | 'startToCloseTimeout': '3600', 75 | 'taskList': {'name': 'garcon_workflow_name_activity_2'}}, 76 | 'eventId': 11, 77 | 'eventTimestamp': 1419351026.232, 78 | 'eventType': 'ActivityTaskScheduled'}, 79 | {'activityTaskScheduledEventAttributes': { 80 | 'activityId': 'workflow_name_activity_3-1', 81 | 'activityType': {'name': 'workflow_name_activity_3', 'version': '1.0'}, 82 | 'decisionTaskCompletedEventId': 10, 83 | 'heartbeatTimeout': '600', 84 | 'input': 'json object', 85 | 'scheduleToCloseTimeout': '3900', 86 | 'scheduleToStartTimeout': '300', 87 | 'startToCloseTimeout': '3600', 88 | 'taskList': {'name': 'garcon_workflow_name_activity_3'}}, 89 | 'eventId': 12, 90 | 'eventTimestamp': 1419351026.232, 91 | 'eventType': 'ActivityTaskScheduled'}, 92 | {'activityTaskStartedEventAttributes': { 93 | 'scheduledEventId': 12}, 94 | 'eventId': 13, 95 | 'eventTimestamp': 1419351026.357, 96 | 'eventType': 'ActivityTaskStarted'}, 97 | {'activityTaskCompletedEventAttributes': { 98 | 'scheduledEventId': 12, 'startedEventId': 13}, 99 | 'eventId': 14, 100 | 'eventTimestamp': 1419351026.437, 101 | 'eventType': 'ActivityTaskCompleted'}, 102 | {'decisionTaskScheduledEventAttributes': { 103 | 'startToCloseTimeout': '300', 104 | 'taskList': {'name': 'garcon_decider'}}, 105 | 'eventId': 15, 106 | 'eventTimestamp': 1419351026.437, 107 | 'eventType': 'DecisionTaskScheduled'}, 108 | {'activityTaskStartedEventAttributes': { 109 | 'scheduledEventId': 11}, 110 | 'eventId': 16, 111 | 'eventTimestamp': 1419351026.542, 112 | 'eventType': 'ActivityTaskStarted'}, 113 | {'decisionTaskStartedEventAttributes': { 114 | 'scheduledEventId': 15}, 115 | 'eventId': 17, 116 | 'eventTimestamp': 1419351026.622, 117 | 'eventType': 'DecisionTaskStarted'}, 118 | {'decisionTaskCompletedEventAttributes': { 119 | 'scheduledEventId': 15, 'startedEventId': 17}, 120 | 'eventId': 18, 121 | 'eventTimestamp': 1419351026.803, 122 | 'eventType': 'DecisionTaskCompleted'}, 123 | {'activityTaskCompletedEventAttributes': { 124 | 'scheduledEventId': 11, 'startedEventId': 16, 'result': '{"k": "v"}'}, 125 | 'eventId': 19, 126 | 'eventTimestamp': 1419351026.932, 127 | 'eventType': 'ActivityTaskCompleted'}, 128 | {'decisionTaskScheduledEventAttributes': { 129 | 'startToCloseTimeout': '300', 130 | 'taskList': {'name': 'garcon_decider'}}, 131 | 'eventId': 20, 132 | 'eventTimestamp': 1419351026.932, 133 | 'eventType': 'DecisionTaskScheduled'}, 134 | {'decisionTaskStartedEventAttributes': {'scheduledEventId': 20}, 135 | 'eventId': 21, 136 | 'eventTimestamp': 1419351027.276, 137 | 'eventType': 'DecisionTaskStarted'}, 138 | {'decisionTaskCompletedEventAttributes': { 139 | 'scheduledEventId': 20, 'startedEventId': 21}, 140 | 'eventId': 22, 141 | 'eventTimestamp': 1419351027.381, 142 | 'eventType': 'DecisionTaskCompleted'}, 143 | {'activityTaskScheduledEventAttributes': { 144 | 'activityId': 'workflow_name_activity_4-1', 145 | 'activityType': {'name': 'workflow_name_activity_4', 'version': '1.0'}, 146 | 'decisionTaskCompletedEventId': 22, 147 | 'heartbeatTimeout': '600', 148 | 'input': 'json object', 149 | 'scheduleToCloseTimeout': '3900', 150 | 'scheduleToStartTimeout': '300', 151 | 'startToCloseTimeout': '3600', 152 | 'taskList': {'name': 'garcon_workflow_name_activity_4'}}, 153 | 'eventId': 23, 154 | 'eventTimestamp': 1419351027.381, 155 | 'eventType': 'ActivityTaskScheduled'}, 156 | {'activityTaskStartedEventAttributes': {'scheduledEventId': 23}, 157 | 'eventId': 24, 158 | 'eventTimestamp': 1419351027.43, 159 | 'eventType': 'ActivityTaskStarted'}, 160 | {'activityTaskCompletedEventAttributes': { 161 | 'scheduledEventId': 23, 'startedEventId': 24}, 162 | 'eventId': 25, 163 | 'eventTimestamp': 1419351027.492, 164 | 'eventType': 'ActivityTaskCompleted'}, 165 | {'decisionTaskScheduledEventAttributes': { 166 | 'startToCloseTimeout': '300', 167 | 'taskList': {'name': 'garcon_decider'}}, 168 | 'eventId': 26, 169 | 'eventTimestamp': 1419351027.492, 170 | 'eventType': 'DecisionTaskScheduled'}, 171 | {'decisionTaskStartedEventAttributes': {'scheduledEventId': 26}, 172 | 'eventId': 27, 173 | 'eventTimestamp': 1419351027.555, 174 | 'eventType': 'DecisionTaskStarted'}], 175 | 'workflowExecution': { 176 | 'runId': '123abc=', 177 | 'workflowId': 'test-workflow-id'} 178 | } 179 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " applehelp to make an Apple Help Book" 34 | @echo " devhelp to make HTML files and a Devhelp project" 35 | @echo " epub to make an epub" 36 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 37 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 38 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 39 | @echo " text to make text files" 40 | @echo " man to make manual pages" 41 | @echo " texinfo to make Texinfo files" 42 | @echo " info to make Texinfo files and run them through makeinfo" 43 | @echo " gettext to make PO message catalogs" 44 | @echo " changes to make an overview of all changed/added/deprecated items" 45 | @echo " xml to make Docutils-native XML files" 46 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 47 | @echo " linkcheck to check all external links for integrity" 48 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 49 | @echo " coverage to run coverage check of the documentation (if enabled)" 50 | 51 | clean: 52 | rm -rf $(BUILDDIR)/* 53 | 54 | html: 55 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 56 | @echo 57 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 58 | 59 | dirhtml: 60 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 61 | @echo 62 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 63 | 64 | singlehtml: 65 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 66 | @echo 67 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 68 | 69 | pickle: 70 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 71 | @echo 72 | @echo "Build finished; now you can process the pickle files." 73 | 74 | json: 75 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 76 | @echo 77 | @echo "Build finished; now you can process the JSON files." 78 | 79 | htmlhelp: 80 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 81 | @echo 82 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 83 | ".hhp project file in $(BUILDDIR)/htmlhelp." 84 | 85 | qthelp: 86 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 87 | @echo 88 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 89 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 90 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/GarconAWSSWF.qhcp" 91 | @echo "To view the help file:" 92 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/GarconAWSSWF.qhc" 93 | 94 | applehelp: 95 | $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp 96 | @echo 97 | @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." 98 | @echo "N.B. You won't be able to view it unless you put it in" \ 99 | "~/Library/Documentation/Help or install it in your application" \ 100 | "bundle." 101 | 102 | devhelp: 103 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 104 | @echo 105 | @echo "Build finished." 106 | @echo "To view the help file:" 107 | @echo "# mkdir -p $$HOME/.local/share/devhelp/GarconAWSSWF" 108 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/GarconAWSSWF" 109 | @echo "# devhelp" 110 | 111 | epub: 112 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 113 | @echo 114 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 115 | 116 | latex: 117 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 118 | @echo 119 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 120 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 121 | "(use \`make latexpdf' here to do that automatically)." 122 | 123 | latexpdf: 124 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 125 | @echo "Running LaTeX files through pdflatex..." 126 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 127 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 128 | 129 | latexpdfja: 130 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 131 | @echo "Running LaTeX files through platex and dvipdfmx..." 132 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 133 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 134 | 135 | text: 136 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 137 | @echo 138 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 139 | 140 | man: 141 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 142 | @echo 143 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 144 | 145 | texinfo: 146 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 147 | @echo 148 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 149 | @echo "Run \`make' in that directory to run these through makeinfo" \ 150 | "(use \`make info' here to do that automatically)." 151 | 152 | info: 153 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 154 | @echo "Running Texinfo files through makeinfo..." 155 | make -C $(BUILDDIR)/texinfo info 156 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 157 | 158 | gettext: 159 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 160 | @echo 161 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 162 | 163 | changes: 164 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 165 | @echo 166 | @echo "The overview file is in $(BUILDDIR)/changes." 167 | 168 | linkcheck: 169 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 170 | @echo 171 | @echo "Link check complete; look for any errors in the above output " \ 172 | "or in $(BUILDDIR)/linkcheck/output.txt." 173 | 174 | doctest: 175 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 176 | @echo "Testing of doctests in the sources finished, look at the " \ 177 | "results in $(BUILDDIR)/doctest/output.txt." 178 | 179 | coverage: 180 | $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage 181 | @echo "Testing of coverage in the sources finished, look at the " \ 182 | "results in $(BUILDDIR)/coverage/python.txt." 183 | 184 | xml: 185 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 186 | @echo 187 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 188 | 189 | pseudoxml: 190 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 191 | @echo 192 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 193 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. xml to make Docutils-native XML files 37 | echo. pseudoxml to make pseudoxml-XML files for display purposes 38 | echo. linkcheck to check all external links for integrity 39 | echo. doctest to run all doctests embedded in the documentation if enabled 40 | echo. coverage to run coverage check of the documentation if enabled 41 | goto end 42 | ) 43 | 44 | if "%1" == "clean" ( 45 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 46 | del /q /s %BUILDDIR%\* 47 | goto end 48 | ) 49 | 50 | 51 | REM Check if sphinx-build is available and fallback to Python version if any 52 | %SPHINXBUILD% 2> nul 53 | if errorlevel 9009 goto sphinx_python 54 | goto sphinx_ok 55 | 56 | :sphinx_python 57 | 58 | set SPHINXBUILD=python -m sphinx.__init__ 59 | %SPHINXBUILD% 2> nul 60 | if errorlevel 9009 ( 61 | echo. 62 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 63 | echo.installed, then set the SPHINXBUILD environment variable to point 64 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 65 | echo.may add the Sphinx directory to PATH. 66 | echo. 67 | echo.If you don't have Sphinx installed, grab it from 68 | echo.http://sphinx-doc.org/ 69 | exit /b 1 70 | ) 71 | 72 | :sphinx_ok 73 | 74 | 75 | if "%1" == "html" ( 76 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 77 | if errorlevel 1 exit /b 1 78 | echo. 79 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 80 | goto end 81 | ) 82 | 83 | if "%1" == "dirhtml" ( 84 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 85 | if errorlevel 1 exit /b 1 86 | echo. 87 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 88 | goto end 89 | ) 90 | 91 | if "%1" == "singlehtml" ( 92 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 93 | if errorlevel 1 exit /b 1 94 | echo. 95 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 96 | goto end 97 | ) 98 | 99 | if "%1" == "pickle" ( 100 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 101 | if errorlevel 1 exit /b 1 102 | echo. 103 | echo.Build finished; now you can process the pickle files. 104 | goto end 105 | ) 106 | 107 | if "%1" == "json" ( 108 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 109 | if errorlevel 1 exit /b 1 110 | echo. 111 | echo.Build finished; now you can process the JSON files. 112 | goto end 113 | ) 114 | 115 | if "%1" == "htmlhelp" ( 116 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 117 | if errorlevel 1 exit /b 1 118 | echo. 119 | echo.Build finished; now you can run HTML Help Workshop with the ^ 120 | .hhp project file in %BUILDDIR%/htmlhelp. 121 | goto end 122 | ) 123 | 124 | if "%1" == "qthelp" ( 125 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 129 | .qhcp project file in %BUILDDIR%/qthelp, like this: 130 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\GarconAWSSWF.qhcp 131 | echo.To view the help file: 132 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\GarconAWSSWF.ghc 133 | goto end 134 | ) 135 | 136 | if "%1" == "devhelp" ( 137 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 138 | if errorlevel 1 exit /b 1 139 | echo. 140 | echo.Build finished. 141 | goto end 142 | ) 143 | 144 | if "%1" == "epub" ( 145 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 146 | if errorlevel 1 exit /b 1 147 | echo. 148 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 149 | goto end 150 | ) 151 | 152 | if "%1" == "latex" ( 153 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 154 | if errorlevel 1 exit /b 1 155 | echo. 156 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 157 | goto end 158 | ) 159 | 160 | if "%1" == "latexpdf" ( 161 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 162 | cd %BUILDDIR%/latex 163 | make all-pdf 164 | cd %~dp0 165 | echo. 166 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 167 | goto end 168 | ) 169 | 170 | if "%1" == "latexpdfja" ( 171 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 172 | cd %BUILDDIR%/latex 173 | make all-pdf-ja 174 | cd %~dp0 175 | echo. 176 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 177 | goto end 178 | ) 179 | 180 | if "%1" == "text" ( 181 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 182 | if errorlevel 1 exit /b 1 183 | echo. 184 | echo.Build finished. The text files are in %BUILDDIR%/text. 185 | goto end 186 | ) 187 | 188 | if "%1" == "man" ( 189 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 190 | if errorlevel 1 exit /b 1 191 | echo. 192 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 193 | goto end 194 | ) 195 | 196 | if "%1" == "texinfo" ( 197 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 198 | if errorlevel 1 exit /b 1 199 | echo. 200 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 201 | goto end 202 | ) 203 | 204 | if "%1" == "gettext" ( 205 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 206 | if errorlevel 1 exit /b 1 207 | echo. 208 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 209 | goto end 210 | ) 211 | 212 | if "%1" == "changes" ( 213 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 214 | if errorlevel 1 exit /b 1 215 | echo. 216 | echo.The overview file is in %BUILDDIR%/changes. 217 | goto end 218 | ) 219 | 220 | if "%1" == "linkcheck" ( 221 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 222 | if errorlevel 1 exit /b 1 223 | echo. 224 | echo.Link check complete; look for any errors in the above output ^ 225 | or in %BUILDDIR%/linkcheck/output.txt. 226 | goto end 227 | ) 228 | 229 | if "%1" == "doctest" ( 230 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 231 | if errorlevel 1 exit /b 1 232 | echo. 233 | echo.Testing of doctests in the sources finished, look at the ^ 234 | results in %BUILDDIR%/doctest/output.txt. 235 | goto end 236 | ) 237 | 238 | if "%1" == "coverage" ( 239 | %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage 240 | if errorlevel 1 exit /b 1 241 | echo. 242 | echo.Testing of coverage in the sources finished, look at the ^ 243 | results in %BUILDDIR%/coverage/python.txt. 244 | goto end 245 | ) 246 | 247 | if "%1" == "xml" ( 248 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 249 | if errorlevel 1 exit /b 1 250 | echo. 251 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 252 | goto end 253 | ) 254 | 255 | if "%1" == "pseudoxml" ( 256 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 257 | if errorlevel 1 exit /b 1 258 | echo. 259 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 260 | goto end 261 | ) 262 | 263 | :end 264 | -------------------------------------------------------------------------------- /tests/test_task.py: -------------------------------------------------------------------------------- 1 | import functools 2 | from unittest.mock import MagicMock 3 | 4 | import pytest 5 | 6 | from garcon import task 7 | from garcon import param 8 | 9 | 10 | def test_timeout_decorator(): 11 | """Test the timeout decorator. 12 | """ 13 | 14 | timeout = 10 15 | 16 | @task.timeout(timeout) 17 | def test(): 18 | pass 19 | 20 | assert test.__garcon__.get('timeout') == timeout 21 | 22 | 23 | def test_timeout_decorator_with_heartbeat(): 24 | """Test the timeout decorator with heartbeat. 25 | """ 26 | 27 | timeout = 20 28 | heartbeat = 30 29 | 30 | @task.timeout(timeout, heartbeat=heartbeat) 31 | def test(): 32 | pass 33 | 34 | assert test.__garcon__.get('heartbeat') == heartbeat 35 | assert test.__garcon__.get('timeout') == timeout 36 | 37 | @task.timeout(timeout) 38 | @task.decorate(timeout=heartbeat) 39 | def test2(): 40 | pass 41 | 42 | assert test2.__garcon__.get('heartbeat') == heartbeat 43 | assert test2.__garcon__.get('timeout') == timeout 44 | 45 | 46 | def test_decorator(): 47 | """Test the Decorator. 48 | 49 | It should add __garcon__ to the method and if a key / value is 50 | passed, it should add it. 51 | """ 52 | 53 | def test(): 54 | pass 55 | 56 | task._decorate(test) 57 | assert hasattr(test, '__garcon__') 58 | 59 | task._decorate(test, 'foo') 60 | assert test.__garcon__.get('foo') is None 61 | 62 | task._decorate(test, 'foo', 'bar') 63 | assert test.__garcon__.get('foo') == 'bar' 64 | 65 | 66 | def test_generator_decorator(): 67 | """Test the geneartor decorator. 68 | """ 69 | 70 | @task.list 71 | def test(): 72 | pass 73 | 74 | assert test.__garcon__.get('list') 75 | assert task.is_task_list(test) 76 | 77 | 78 | def test_link_decorator(): 79 | """Test linking the decorator between two methods. 80 | """ 81 | 82 | def testA(): 83 | pass 84 | 85 | def testB(): 86 | pass 87 | 88 | task._decorate(testA, 'foo', 'value') 89 | task._link_decorator(testA, testB) 90 | assert testA.__garcon__ == testB.__garcon__ 91 | assert testA.__garcon__.get('foo') == 'value' 92 | assert testB.__garcon__.get('foo') == 'value' 93 | 94 | 95 | def test_link_decorator_with_empty_source(): 96 | """Test linking decorators when garcon is not set on the source. 97 | """ 98 | 99 | def testA(): 100 | pass 101 | 102 | def testB(): 103 | pass 104 | 105 | task._link_decorator(testA, testB) 106 | assert not getattr(testA, '__garcon__', None) 107 | assert len(testB.__garcon__) == 0 108 | 109 | task._decorate(testB, 'foo', 'value') 110 | assert testB.__garcon__.get('foo') == 'value' 111 | 112 | 113 | def test_task_decorator(): 114 | """Test the task decorator. 115 | """ 116 | 117 | timeout = 40 118 | userinfo = 'something' 119 | 120 | @task.decorate(timeout=timeout) 121 | def test(user): 122 | assert user == userinfo 123 | 124 | assert test.__garcon__.get('timeout') == timeout 125 | assert test.__garcon__.get('heartbeat') == timeout 126 | assert callable(test.fill) 127 | 128 | call = test.fill(user='user') 129 | call(dict(user='something')) 130 | 131 | 132 | def test_task_decorator_attributes(): 133 | """Test that the decorated task has attributes similar to the original 134 | function. 135 | """ 136 | 137 | decorator = task.decorate() 138 | 139 | def test(): 140 | pass 141 | 142 | test_task = decorator(test) 143 | for attr_name in functools.WRAPPER_ASSIGNMENTS: 144 | assert getattr(test_task, attr_name) == getattr(test, attr_name) 145 | 146 | 147 | def test_task_decorator_with_heartbeat(): 148 | """Test the task decorator with heartbeat. 149 | """ 150 | 151 | heartbeat = 50 152 | userinfo = None 153 | 154 | @task.decorate(heartbeat=heartbeat) 155 | def test(user): 156 | assert user is userinfo 157 | 158 | assert test.__garcon__.get('heartbeat') == heartbeat 159 | 160 | 161 | def test_task_decorator_with_activity(): 162 | """Test the task decorator with an activity. 163 | """ 164 | 165 | current_activity = MagicMock() 166 | 167 | @task.decorate() 168 | def test(activity): 169 | activity() 170 | assert activity is current_activity 171 | 172 | call = test.fill() 173 | call(dict(), activity=current_activity) 174 | 175 | assert current_activity.called 176 | 177 | 178 | def test_task_decorator_with_context(): 179 | """Test the task decorator with an activity. 180 | """ 181 | 182 | current_context = {} 183 | spy = MagicMock() 184 | 185 | @task.decorate() 186 | def test(context): 187 | spy() 188 | 189 | call = test.fill() 190 | 191 | with pytest.raises(Exception): 192 | call(current_context) 193 | 194 | assert not spy.called 195 | 196 | 197 | def test_contextify_added_fill(): 198 | """Verify that calling contextify on a method added .fill on the method. 199 | """ 200 | 201 | @task.contextify 202 | def test(activity, context, something): 203 | pass 204 | 205 | assert test 206 | assert test.fill 207 | 208 | 209 | def test_contextify_default_method_call(): 210 | """Test that the contextify decorator is not altering the method itself. 211 | The contextify decorator should preserve the format of the method, it 212 | allows us to use it in more than one context. 213 | """ 214 | 215 | response = dict(somekey='somevalue') 216 | spy = MagicMock() 217 | 218 | @task.contextify 219 | def method(activity, key_to_replace, key_to_not_replace=None): 220 | assert key_to_replace == 'value' 221 | assert key_to_not_replace is None 222 | spy() 223 | return response 224 | 225 | assert method(None, 'value') is response 226 | assert spy.called 227 | 228 | 229 | def test_contextify(): 230 | """Try using contextify on a method. 231 | """ 232 | 233 | value = 'random' 234 | kwargs_value = 'more-random' 235 | return_value = 'a value' 236 | spy = MagicMock() 237 | 238 | @task.contextify 239 | def method( 240 | activity, key_to_replace, key_not_to_replace=None, 241 | kwargs_to_replace=None): 242 | 243 | assert not key_not_to_replace 244 | assert key_to_replace == value 245 | assert kwargs_to_replace == kwargs_value 246 | spy() 247 | return dict(return_value=return_value) 248 | 249 | fn = method.fill( 250 | key_to_replace='context.key', 251 | kwargs_to_replace='context.kwarg_key') 252 | 253 | resp = fn({'context.key': value, 'context.kwarg_key': kwargs_value}) 254 | 255 | assert isinstance(resp, dict) 256 | assert resp.get('return_value') is return_value 257 | 258 | 259 | def test_contextify_with_mapped_response(): 260 | """Test the contextify method with mapped response. 261 | """ 262 | 263 | return_value = 'a value' 264 | 265 | @task.contextify 266 | def method( 267 | activity, key_to_replace, key_not_to_replace=None, 268 | kwargs_to_replace=None): 269 | 270 | assert activity == 'activity' 271 | return dict(return_value=return_value) 272 | 273 | fn = method.fill( 274 | key_to_replace='context.key', 275 | kwargs_to_replace='context.kwarg_key', 276 | namespace='somethingrandom') 277 | 278 | resp = fn( 279 | {'context.key': 'test', 'context.kwarg_key': 'a'}, 280 | activity='activity') 281 | 282 | assert isinstance(resp, dict) 283 | assert len(resp) == 1 284 | assert resp.get('somethingrandom.return_value') is return_value 285 | 286 | 287 | def test_flatten(): 288 | """Test the flatten function. 289 | """ 290 | 291 | spy = MagicMock 292 | 293 | @task.decorate(timeout=10) 294 | def task_a(name): 295 | pass 296 | 297 | @task.decorate(timeout=10) 298 | def task_b(name): 299 | pass 300 | 301 | @task.list 302 | def task_generator(context): 303 | yield task_b 304 | if context.get('value'): 305 | yield task_a 306 | 307 | value = list(task.flatten( 308 | (task_a, task_b, task_generator, task_a), 309 | dict(value='something'))) 310 | assert value == [task_a, task_b, task_b, task_a, task_a] 311 | 312 | 313 | def test_fill_function_call(): 314 | """Test filling the function call. 315 | """ 316 | 317 | def test_function(activity, arg_one, key, kwarg_one=None, kwarg_two=None): 318 | pass 319 | 320 | requirements = dict( 321 | arg_one=param.Param('context.arg'), 322 | kwarg_one=param.Param('context.kwarg')) 323 | activity = None 324 | context = { 325 | 'context.arg': 'arg.value', 326 | 'context.kwarg': 'kwarg.value'} 327 | 328 | data = task.fill_function_call( 329 | test_function, requirements, activity, context) 330 | 331 | assert not data.get('activity') 332 | assert not data.get('context') 333 | assert not data.get('key') 334 | assert not data.get('kwarg_two') 335 | assert data.get('arg_one') == 'arg.value' 336 | assert data.get('kwarg_one') == 'kwarg.value' 337 | test_function(**data) 338 | 339 | 340 | def test_namespace_result(): 341 | """Test namespacing the results. 342 | """ 343 | 344 | value = 'foo' 345 | 346 | resp = task.namespace_result(dict(test=value), '') 347 | assert resp.get('test') == value 348 | 349 | resp = task.namespace_result(dict(test=value), 'namespace') 350 | assert resp.get('namespace.test') == value 351 | -------------------------------------------------------------------------------- /garcon/task.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Task 4 | ==== 5 | 6 | Tasks are small discrete applications that are meant to perform a defined 7 | action within an activity. An activity can have more than one task, they can 8 | run in series or in parallel. 9 | 10 | Tasks can add values to the context by returning a dictionary that contains 11 | the informations to add (useful if you need to pass information from one 12 | task – in an activity, to another activity's task.) 13 | 14 | Note: 15 | If you need a task runner that is not covered by the two scenarios below, 16 | you may need to just have a main task, and have this task split the work 17 | the way you want. 18 | """ 19 | 20 | import copy 21 | from functools import update_wrapper 22 | 23 | from garcon import param 24 | 25 | 26 | def decorate(timeout=None, heartbeat=None, enable_contextify=True): 27 | """Generic task decorator for tasks. 28 | 29 | Args: 30 | timeout (int): The timeout of the task (see timeout). 31 | heartbeat (int): The heartbeat timeout. 32 | contextify (boolean): If the task can be contextified (see contextify). 33 | Return: 34 | callable: The wrapper. 35 | """ 36 | 37 | def wrapper(fn): 38 | if timeout: 39 | _decorate(fn, 'timeout', timeout) 40 | 41 | # If the task does not have a heartbeat, but instead the task has 42 | # a timeout, the heartbeat should be adjusted to the timeout. In 43 | # most case, most people will probably opt for this option. 44 | if heartbeat or timeout: 45 | _decorate(fn, 'heartbeat', heartbeat or timeout) 46 | 47 | if enable_contextify: 48 | contextify(fn) 49 | return fn 50 | 51 | return wrapper 52 | 53 | 54 | def timeout(time, heartbeat=None): 55 | """Wrapper for a task to define its timeout. 56 | 57 | Args: 58 | time (int): the timeout in seconds 59 | heartbeat (int): the heartbeat timeout (in seconds too.) 60 | """ 61 | 62 | def wrapper(fn): 63 | _decorate(fn, 'timeout', time) 64 | if heartbeat: 65 | _decorate(fn, 'heartbeat', heartbeat) 66 | return fn 67 | 68 | return wrapper 69 | 70 | 71 | def list(fn): 72 | """Wrapper for a callable to define a task generator. 73 | 74 | Generators are used to check values in the context and schedule different 75 | tasks based on it. Note: depending on the tasks returned by the generator, 76 | the timout values will be calculated differently. 77 | 78 | For instance:: 79 | 80 | @task.list 81 | def create_client(context): 82 | yield create_user.fill( 83 | username='context.username', 84 | email='context.email') 85 | if context.get('context.credit_card'): 86 | yield create_credit_card.fill( 87 | username='context.username', 88 | credit_card='context.credit_card') 89 | yield send_email.fill(email='context.email') 90 | """ 91 | 92 | _decorate(fn, key='list', value=True) 93 | contextify(fn) 94 | return fn 95 | 96 | 97 | def is_task_list(fn): 98 | """Check if a function is a task list. 99 | 100 | Return: 101 | boolean: if a function is a task list. 102 | """ 103 | 104 | return getattr(fn, '__garcon__', {}).get('list') 105 | 106 | 107 | def _decorate(fn, key=None, value=None): 108 | """Add the garcon property to the function. 109 | 110 | Args: 111 | fn (callable): The function to alter. 112 | key (string): The key to set (optional.) 113 | value (any): The value to set (optional.) 114 | """ 115 | 116 | if not hasattr(fn, '__garcon__'): 117 | setattr(fn, '__garcon__', dict()) 118 | 119 | if key: 120 | fn.__garcon__.update({ 121 | key: value 122 | }) 123 | 124 | 125 | def _link_decorator(source_fn, dest_fn): 126 | """Link the garcon decorator values between two methods. 127 | 128 | If the destination method already have a value on `__garcon__`, we get it 129 | and merge it with the other one (so no values are lost.) 130 | 131 | Args: 132 | source_fn (callable): The method that contains `__garcon__`. 133 | dest_fn (callable): The method that receives the decorator. 134 | """ 135 | 136 | source_values = copy.deepcopy(getattr(source_fn, '__garcon__', dict())) 137 | 138 | if hasattr(dest_fn, '__garcon__'): 139 | source_values.update(dest_fn.__garcon__) 140 | 141 | setattr(dest_fn, '__garcon__', source_values) 142 | 143 | 144 | def contextify(fn): 145 | """Decorator to take values from the context and apply them to fn. 146 | 147 | The goal of this decorator is to allow methods to be called with different 148 | values from the same context. For instance: if you need to increase the 149 | throughtput of two different dynamodb tables, you will need to pass a 150 | table name, table index, and the new throughtput. 151 | 152 | If you have more than one table, it gets difficult to manage. With this 153 | decorator, it's a little easier:: 154 | 155 | @contextify 156 | def increase_dynamodb_throughtput( 157 | activity, context, table_name=None, table_index=None, 158 | table_throughtput=None): 159 | print(table_name) 160 | activity_task = increase_dynamodb_throughtput.fill( 161 | table_name='dynamodb.table_name1', 162 | table_index='dynamodb.table_index1', 163 | table_throughtput='dynamodb.table_throughtput1') 164 | context = dict( 165 | 'dynamodb.table_name1': 'table_name', 166 | 'dynamodb.table_index1': 'index', 167 | 'dynamodb.table_throughtput1': 'throughtput1') 168 | activity_task(..., context) # shows table_name 169 | """ 170 | 171 | def fill(namespace=None, **requirements): 172 | 173 | requirements = { 174 | key: param.parametrize(current_param) 175 | for key, current_param in requirements.items()} 176 | 177 | def wrapper(context, **kwargs): 178 | kwargs.update( 179 | fill_function_call( 180 | fn, requirements, kwargs.get('activity'), context)) 181 | 182 | response = fn(**kwargs) 183 | if not response or not namespace: 184 | return response 185 | 186 | return namespace_result(response, namespace) 187 | 188 | update_wrapper(wrapper, fn) 189 | 190 | # Keep a record of the requirements value. This allows us to trim the 191 | # size of the context sent to the activity as an input. 192 | _link_decorator(fn, wrapper) 193 | _decorate( 194 | wrapper, 195 | 'requirements', 196 | param.get_all_requirements(requirements.values())) 197 | return wrapper 198 | 199 | fn.fill = fill 200 | return fn 201 | 202 | 203 | def flatten(callables, context=None): 204 | """Flatten the tasks. 205 | 206 | The task list is a mix of tasks and generators. The task generators are 207 | consuming the context and spawning new tasks. This method flattens 208 | everything into one list. 209 | 210 | Args: 211 | callables (list): list of callables (including tasks and generators.) 212 | 213 | Yield: 214 | callable: one of the task. 215 | """ 216 | 217 | for task in callables: 218 | if is_task_list(task): 219 | for subtask in task(context): 220 | yield subtask 221 | continue 222 | yield task 223 | 224 | 225 | def fill_function_call(fn, requirements, activity, context): 226 | """Fill a function calls from values from the context to the variable. 227 | 228 | Args: 229 | fn (callable): the function to call. 230 | requirements (dict): the requirements. The key represent the variable 231 | name and the value represents where the value is in the context. 232 | activity (ActivityWorker): the current activity worker. 233 | context (dict): the current context. 234 | 235 | Return: 236 | dict: The arguments to call the method with. 237 | """ 238 | 239 | function_arguments = fn.__code__.co_varnames[:fn.__code__.co_argcount] 240 | kwargs = dict() 241 | 242 | for argument in function_arguments: 243 | param = requirements.get(argument, None) 244 | value = None 245 | 246 | if argument == 'context': 247 | raise Exception( 248 | 'Data used from the context should be explicit. A task should' 249 | ' not randomly access information from the context.') 250 | 251 | elif argument == 'activity': 252 | value = activity 253 | 254 | elif param: 255 | value = param.get_data(context) 256 | 257 | kwargs.update({ 258 | argument: value 259 | }) 260 | 261 | return kwargs 262 | 263 | 264 | def namespace_result(dictionary, namespace): 265 | """Namespace the response 266 | 267 | This method takes the keys in the map and add a prefix to all the keys 268 | (the namespace):: 269 | 270 | resp = dict(key='value', index='storage') 271 | namespace_response(resp, 'namespace') 272 | # Returns: {'namespace.index': 'storage', 'namespace.key': 'value'} 273 | 274 | Args: 275 | dictionary (dict): The dictionary to update. 276 | map (dict): The keys to update. 277 | 278 | Return: 279 | Dict: the updated dictionary 280 | """ 281 | 282 | if not namespace: 283 | return dictionary 284 | 285 | return { 286 | (namespace + '.' + key): value for key, value in dictionary.items()} 287 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Garcon, AWS SWF documentation build configuration file, created by 5 | # sphinx-quickstart on Tue May 12 10:01:26 2015. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | import sys 17 | import os 18 | import shlex 19 | 20 | # If extensions (or modules to document with autodoc) are in another directory, 21 | # add these directories to sys.path here. If the directory is relative to the 22 | # documentation root, use os.path.abspath to make it absolute, like shown here. 23 | sys.path.insert(0, os.path.abspath('../')) 24 | 25 | # -- General configuration ------------------------------------------------ 26 | 27 | # If your documentation needs a minimal Sphinx version, state it here. 28 | #needs_sphinx = '1.0' 29 | 30 | # Add any Sphinx extension module names here, as strings. They can be 31 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 32 | # ones. 33 | extensions = [ 34 | 'sphinx.ext.autodoc', 35 | 'sphinx.ext.intersphinx', 36 | 'sphinxcontrib.napoleon', 37 | ] 38 | 39 | # Add any paths that contain templates here, relative to this directory. 40 | templates_path = ['_templates'] 41 | 42 | # The suffix(es) of source filenames. 43 | # You can specify multiple suffix as a list of string: 44 | # source_suffix = ['.rst', '.md'] 45 | source_suffix = '.rst' 46 | 47 | # The encoding of source files. 48 | #source_encoding = 'utf-8-sig' 49 | 50 | # The master toctree document. 51 | master_doc = 'index' 52 | 53 | # General information about the project. 54 | project = 'Garcon' 55 | copyright = '2015, Michael Ortali' 56 | author = 'Michael Ortali' 57 | 58 | # The version info for the project you're documenting, acts as replacement for 59 | # |version| and |release|, also used in various other places throughout the 60 | # built documents. 61 | # 62 | # The short X.Y version. 63 | version = '0.0.4' 64 | # The full version, including alpha/beta/rc tags. 65 | release = '0.0.4' 66 | 67 | # The language for content autogenerated by Sphinx. Refer to documentation 68 | # for a list of supported languages. 69 | # 70 | # This is also used if you do content translation via gettext catalogs. 71 | # Usually you set "language" from the command line for these cases. 72 | language = None 73 | 74 | # There are two options for replacing |today|: either, you set today to some 75 | # non-false value, then it is used: 76 | #today = '' 77 | # Else, today_fmt is used as the format for a strftime call. 78 | #today_fmt = '%B %d, %Y' 79 | 80 | # List of patterns, relative to source directory, that match files and 81 | # directories to ignore when looking for source files. 82 | exclude_patterns = ['_build'] 83 | 84 | # The reST default role (used for this markup: `text`) to use for all 85 | # documents. 86 | #default_role = None 87 | 88 | # If true, '()' will be appended to :func: etc. cross-reference text. 89 | #add_function_parentheses = True 90 | 91 | # If true, the current module name will be prepended to all description 92 | # unit titles (such as .. function::). 93 | #add_module_names = True 94 | 95 | # If true, sectionauthor and moduleauthor directives will be shown in the 96 | # output. They are ignored by default. 97 | #show_authors = False 98 | 99 | # The name of the Pygments (syntax highlighting) style to use. 100 | pygments_style = 'sphinx' 101 | 102 | # A list of ignored prefixes for module index sorting. 103 | #modindex_common_prefix = [] 104 | 105 | # If true, keep warnings as "system message" paragraphs in the built documents. 106 | #keep_warnings = False 107 | 108 | # If true, `todo` and `todoList` produce output, else they produce nothing. 109 | todo_include_todos = False 110 | 111 | 112 | # -- Options for HTML output ---------------------------------------------- 113 | 114 | # The theme to use for HTML and HTML Help pages. See the documentation for 115 | # a list of builtin themes. 116 | # html_theme = 'alabaster' 117 | 118 | # Theme options are theme-specific and customize the look and feel of a theme 119 | # further. For a list of options available for each theme, see the 120 | # documentation. 121 | #html_theme_options = {} 122 | 123 | # Add any paths that contain custom themes here, relative to this directory. 124 | #html_theme_path = [] 125 | 126 | # The name for this set of Sphinx documents. If None, it defaults to 127 | # " v documentation". 128 | #html_title = None 129 | 130 | # A shorter title for the navigation bar. Default is the same as html_title. 131 | #html_short_title = None 132 | 133 | # The name of an image file (relative to this directory) to place at the top 134 | # of the sidebar. 135 | #html_logo = None 136 | 137 | # The name of an image file (within the static path) to use as favicon of the 138 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 139 | # pixels large. 140 | #html_favicon = None 141 | 142 | # Add any paths that contain custom static files (such as style sheets) here, 143 | # relative to this directory. They are copied after the builtin static files, 144 | # so a file named "default.css" will overwrite the builtin "default.css". 145 | html_static_path = ['_static'] 146 | 147 | # Add any extra paths that contain custom files (such as robots.txt or 148 | # .htaccess) here, relative to this directory. These files are copied 149 | # directly to the root of the documentation. 150 | #html_extra_path = [] 151 | 152 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 153 | # using the given strftime format. 154 | #html_last_updated_fmt = '%b %d, %Y' 155 | 156 | # If true, SmartyPants will be used to convert quotes and dashes to 157 | # typographically correct entities. 158 | #html_use_smartypants = True 159 | 160 | # Custom sidebar templates, maps document names to template names. 161 | #html_sidebars = {} 162 | 163 | # Additional templates that should be rendered to pages, maps page names to 164 | # template names. 165 | #html_additional_pages = {} 166 | 167 | # If false, no module index is generated. 168 | #html_domain_indices = True 169 | 170 | # If false, no index is generated. 171 | #html_use_index = True 172 | 173 | # If true, the index is split into individual pages for each letter. 174 | #html_split_index = False 175 | 176 | # If true, links to the reST sources are added to the pages. 177 | #html_show_sourcelink = True 178 | 179 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 180 | #html_show_sphinx = True 181 | 182 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 183 | #html_show_copyright = True 184 | 185 | # If true, an OpenSearch description file will be output, and all pages will 186 | # contain a tag referring to it. The value of this option must be the 187 | # base URL from which the finished HTML is served. 188 | #html_use_opensearch = '' 189 | 190 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 191 | #html_file_suffix = None 192 | 193 | # Language to be used for generating the HTML full-text search index. 194 | # Sphinx supports the following languages: 195 | # 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja' 196 | # 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr' 197 | #html_search_language = 'en' 198 | 199 | # A dictionary with options for the search language support, empty by default. 200 | # Now only 'ja' uses this config value 201 | #html_search_options = {'type': 'default'} 202 | 203 | # The name of a javascript file (relative to the configuration directory) that 204 | # implements a search results scorer. If empty, the default will be used. 205 | #html_search_scorer = 'scorer.js' 206 | 207 | # Output file base name for HTML help builder. 208 | htmlhelp_basename = 'GarconAWSSWFdoc' 209 | 210 | # -- Options for LaTeX output --------------------------------------------- 211 | 212 | latex_elements = { 213 | # The paper size ('letterpaper' or 'a4paper'). 214 | #'papersize': 'letterpaper', 215 | 216 | # The font size ('10pt', '11pt' or '12pt'). 217 | #'pointsize': '10pt', 218 | 219 | # Additional stuff for the LaTeX preamble. 220 | #'preamble': '', 221 | 222 | # Latex figure (float) alignment 223 | #'figure_align': 'htbp', 224 | } 225 | 226 | # Grouping the document tree into LaTeX files. List of tuples 227 | # (source start file, target name, title, 228 | # author, documentclass [howto, manual, or own class]). 229 | latex_documents = [ 230 | (master_doc, 'GarconAWSSWF.tex', 'Garcon, Documentation', 231 | 'Michael Ortali', 'manual'), 232 | ] 233 | 234 | # The name of an image file (relative to this directory) to place at the top of 235 | # the title page. 236 | #latex_logo = None 237 | 238 | # For "manual" documents, if this is true, then toplevel headings are parts, 239 | # not chapters. 240 | #latex_use_parts = False 241 | 242 | # If true, show page references after internal links. 243 | #latex_show_pagerefs = False 244 | 245 | # If true, show URL addresses after external links. 246 | #latex_show_urls = False 247 | 248 | # Documents to append as an appendix to all manuals. 249 | #latex_appendices = [] 250 | 251 | # If false, no module index is generated. 252 | #latex_domain_indices = True 253 | 254 | 255 | # -- Options for manual page output --------------------------------------- 256 | 257 | # One entry per manual page. List of tuples 258 | # (source start file, name, description, authors, manual section). 259 | man_pages = [ 260 | (master_doc, 'garconawsswf', 'Garcon, Documentation', 261 | [author], 1) 262 | ] 263 | 264 | # If true, show URL addresses after external links. 265 | #man_show_urls = False 266 | 267 | 268 | # -- Options for Texinfo output ------------------------------------------- 269 | 270 | # Grouping the document tree into Texinfo files. List of tuples 271 | # (source start file, target name, title, author, 272 | # dir menu entry, description, category) 273 | texinfo_documents = [ 274 | (master_doc, 'GarconAWSSWF', 'Garcon, Documentation', 275 | author, 'GarconAWSSWF', 'Lightweight library for AWS SWF.', 276 | 'Miscellaneous'), 277 | ] 278 | 279 | # Documents to append as an appendix to all manuals. 280 | #texinfo_appendices = [] 281 | 282 | # If false, no module index is generated. 283 | #texinfo_domain_indices = True 284 | 285 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 286 | #texinfo_show_urls = 'footnote' 287 | 288 | # If true, do not generate a @detailmenu in the "Top" node's menu. 289 | #texinfo_no_detailmenu = False 290 | 291 | 292 | # -- Options for Epub output ---------------------------------------------- 293 | 294 | # Bibliographic Dublin Core info. 295 | epub_title = project 296 | epub_author = author 297 | epub_publisher = author 298 | epub_copyright = copyright 299 | 300 | # The basename for the epub file. It defaults to the project name. 301 | #epub_basename = project 302 | 303 | # The HTML theme for the epub output. Since the default themes are not optimized 304 | # for small screen space, using the same theme for HTML and epub output is 305 | # usually not wise. This defaults to 'epub', a theme designed to save visual 306 | # space. 307 | #epub_theme = 'epub' 308 | 309 | # The language of the text. It defaults to the language option 310 | # or 'en' if the language is not set. 311 | #epub_language = '' 312 | 313 | # The scheme of the identifier. Typical schemes are ISBN or URL. 314 | #epub_scheme = '' 315 | 316 | # The unique identifier of the text. This can be a ISBN number 317 | # or the project homepage. 318 | #epub_identifier = '' 319 | 320 | # A unique identification for the text. 321 | #epub_uid = '' 322 | 323 | # A tuple containing the cover image and cover page html template filenames. 324 | #epub_cover = () 325 | 326 | # A sequence of (type, uri, title) tuples for the guide element of content.opf. 327 | #epub_guide = () 328 | 329 | # HTML files that should be inserted before the pages created by sphinx. 330 | # The format is a list of tuples containing the path and title. 331 | #epub_pre_files = [] 332 | 333 | # HTML files shat should be inserted after the pages created by sphinx. 334 | # The format is a list of tuples containing the path and title. 335 | #epub_post_files = [] 336 | 337 | # A list of files that should not be packed into the epub file. 338 | epub_exclude_files = ['search.html'] 339 | 340 | # The depth of the table of contents in toc.ncx. 341 | #epub_tocdepth = 3 342 | 343 | # Allow duplicate toc entries. 344 | #epub_tocdup = True 345 | 346 | # Choose between 'default' and 'includehidden'. 347 | #epub_tocscope = 'default' 348 | 349 | # Fix unsupported image types using the Pillow. 350 | #epub_fix_images = False 351 | 352 | # Scale large images. 353 | #epub_max_image_width = 0 354 | 355 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 356 | #epub_show_urls = 'inline' 357 | 358 | # If false, no index is generated. 359 | #epub_use_index = True 360 | 361 | 362 | # Example configuration for intersphinx: refer to the Python standard library. 363 | intersphinx_mapping = {'https://docs.python.org/': None} 364 | -------------------------------------------------------------------------------- /tests/test_decider.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import MagicMock 2 | import json 3 | 4 | import pytest 5 | 6 | from garcon import decider 7 | from garcon import activity 8 | from tests.fixtures import decider as decider_events 9 | 10 | 11 | def test_create_decider(monkeypatch): 12 | """Create a decider and check the behavior of the registration. 13 | """ 14 | 15 | from tests.fixtures.flows import example 16 | 17 | d = decider.DeciderWorker(example) 18 | assert d.client 19 | assert len(d.activities) == 4 20 | assert d.flow 21 | assert d.domain 22 | assert d.on_exception 23 | 24 | monkeypatch.setattr(decider.DeciderWorker, 'register', MagicMock()) 25 | d = decider.DeciderWorker(example) 26 | assert d.register.called 27 | 28 | monkeypatch.setattr(decider.DeciderWorker, 'register', MagicMock()) 29 | dec = decider.DeciderWorker(example, register=False) 30 | assert not dec.register.called 31 | 32 | 33 | def test_get_history(monkeypatch): 34 | """Test the decider history 35 | """ 36 | 37 | from tests.fixtures.flows import example 38 | 39 | events = decider_events.history.get('events') 40 | half = int(len(events) / 2) 41 | events = events[:half * 2] 42 | pool_1 = events[half:] 43 | pool_2 = events[:half] 44 | identity = 'identity' 45 | 46 | d = decider.DeciderWorker(example) 47 | d.client.poll_for_decision_task = MagicMock(return_value={'events': pool_2}) 48 | 49 | resp = d.get_history( 50 | identity, {'events': pool_1, 'nextPageToken': 'nextPage'}) 51 | 52 | d.client.poll_for_decision_task.assert_called_with( 53 | domain=example.domain, 54 | nextPageToken='nextPage', 55 | identity=identity, 56 | taskList=dict(name=d.task_list)) 57 | assert len(resp) == len([ 58 | evt for evt in events if evt['eventType'].startswith('Decision')]) 59 | 60 | 61 | def test_get_activity_states(monkeypatch): 62 | """Test get activity states from history. 63 | """ 64 | 65 | from tests.fixtures.flows import example 66 | 67 | identity= 'identity' 68 | events = decider_events.history.get('events') 69 | d = decider.DeciderWorker(example) 70 | history = d.get_history(identity, {'events': events}) 71 | states = d.get_activity_states(history) 72 | 73 | for activity_name, activity_instances in states.items(): 74 | for activity_instance, activity_state in activity_instances.items(): 75 | assert isinstance(activity_state, activity.ActivityState) 76 | 77 | 78 | def test_running_workflow(monkeypatch): 79 | """Test running a workflow. 80 | """ 81 | 82 | from tests.fixtures.flows import example 83 | 84 | d = decider.DeciderWorker(example, register=False) 85 | d.client.poll_for_decision_task = MagicMock( 86 | return_value=decider_events.history) 87 | d.client.respond_decision_task_completed = MagicMock() 88 | d.complete = MagicMock() 89 | d.create_decisions_from_flow = MagicMock() 90 | 91 | # Via flow. 92 | d.run() 93 | assert d.create_decisions_from_flow.called 94 | 95 | # Via custom decider 96 | spy = MagicMock() 97 | 98 | def custom_decider(schedule): 99 | spy() 100 | 101 | example.decider = custom_decider 102 | d.run() 103 | assert spy.called 104 | 105 | 106 | def test_running_workflow_identity(monkeypatch): 107 | """Test running a workflow with and without identity. 108 | """ 109 | 110 | from tests.fixtures.flows import example 111 | 112 | d = decider.DeciderWorker(example, register=False) 113 | d.client.poll_for_decision_task = MagicMock() 114 | d.complete = MagicMock() 115 | d.create_decisions_from_flow = MagicMock() 116 | 117 | # assert running decider without identity 118 | d.run() 119 | d.client.poll_for_decision_task.assert_called_with( 120 | domain=d.domain, 121 | taskList=dict(name=d.task_list), 122 | identity='') 123 | 124 | # assert running decider with identity 125 | d.run('foo') 126 | d.client.poll_for_decision_task.assert_called_with( 127 | domain=d.domain, 128 | taskList=dict(name=d.task_list), 129 | identity='foo') 130 | 131 | 132 | def test_running_workflow_exception(monkeypatch): 133 | """Run a decider with an exception raised during poll. 134 | """ 135 | 136 | from tests.fixtures.flows import example 137 | 138 | d = decider.DeciderWorker(example, register=False) 139 | d.client.poll_for_decision_task = MagicMock( 140 | return_value=decider_events.history) 141 | d.complete = MagicMock() 142 | d.create_decisions_from_flow = MagicMock() 143 | exception = Exception('test') 144 | d.client.poll_for_decision_task.side_effect = exception 145 | d.on_exception = MagicMock() 146 | d.logger.error = MagicMock() 147 | d.run() 148 | assert d.on_exception.called 149 | d.logger.error.assert_called_with(exception, exc_info=True) 150 | assert not d.complete.called 151 | 152 | 153 | def test_create_decisions_from_flow_exception(monkeypatch): 154 | """Test exception is raised and workflow fails when exception raised. 155 | """ 156 | 157 | from tests.fixtures.flows import example 158 | 159 | decider_worker = decider.DeciderWorker(example, register=False) 160 | decider_worker.logger.error = MagicMock() 161 | decider_worker.on_exception = MagicMock() 162 | 163 | exception = Exception('test') 164 | monkeypatch.setattr(decider.activity, 165 | 'find_available_activities', MagicMock(side_effect = exception)) 166 | 167 | decisions = [] 168 | mock_activity_states = MagicMock() 169 | mock_context = MagicMock() 170 | decider_worker.create_decisions_from_flow( 171 | decisions, mock_activity_states, mock_context) 172 | 173 | failed_execution = dict( 174 | decisionType='FailWorkflowExecution', 175 | failWorkflowExecutionDecisionAttributes=dict(reason=str(exception))) 176 | 177 | assert failed_execution in decisions 178 | assert decider_worker.on_exception.called 179 | decider_worker.logger.error.assert_called_with(exception, exc_info=True) 180 | 181 | 182 | def test_running_workflow_without_events(monkeypatch): 183 | """Test running a workflow without having any events. 184 | """ 185 | 186 | from tests.fixtures.flows import example 187 | 188 | d = decider.DeciderWorker(example, register=False) 189 | d.client.poll_for_decision_task = MagicMock(return_value={}) 190 | d.get_history = MagicMock() 191 | d.run() 192 | 193 | assert not d.get_history.called 194 | 195 | 196 | def test_schedule_context(): 197 | """Test the schedule context. 198 | """ 199 | 200 | context = decider.ScheduleContext() 201 | assert context.completed 202 | 203 | context.mark_uncompleted() 204 | assert not context.completed 205 | 206 | 207 | def test_schedule_with_unscheduled_activity(monkeypatch): 208 | """Test the scheduling of an activity. 209 | """ 210 | 211 | from tests.fixtures.flows import example 212 | 213 | monkeypatch.setattr(decider, 'schedule_activity_task', MagicMock()) 214 | 215 | decisions = MagicMock() 216 | schedule_context = decider.ScheduleContext() 217 | history = {} 218 | current_activity = example.activity_1 219 | 220 | decider.schedule( 221 | decisions, schedule_context, history, {}, 'schedule_id', 222 | current_activity) 223 | 224 | assert decider.schedule_activity_task.called 225 | assert not schedule_context.completed 226 | 227 | 228 | def test_schedule_with_scheduled_activity(monkeypatch): 229 | """Test the scheduling of an activity. 230 | """ 231 | 232 | from tests.fixtures.flows import example 233 | 234 | monkeypatch.setattr(decider, 'schedule_activity_task', MagicMock()) 235 | 236 | decisions = MagicMock() 237 | schedule_context = decider.ScheduleContext() 238 | instance_state = activity.ActivityState('activity_1') 239 | instance_state.add_state(activity.ACTIVITY_SCHEDULED) 240 | current_activity = example.activity_1 241 | history = { 242 | current_activity.name: { 243 | 'workflow_name_activity_1-1-schedule_id': instance_state 244 | } 245 | } 246 | 247 | resp = decider.schedule( 248 | decisions, schedule_context, history, {}, 'schedule_id', 249 | current_activity) 250 | 251 | assert not decider.schedule_activity_task.called 252 | assert not schedule_context.completed 253 | assert resp.get_last_state() == activity.ACTIVITY_SCHEDULED 254 | 255 | with pytest.raises(activity.ActivityInstanceNotReadyException): 256 | resp.result.get('foo') 257 | 258 | 259 | def test_schedule_with_completed_activity(monkeypatch): 260 | """Test the scheduling of an activity. 261 | """ 262 | 263 | from tests.fixtures.flows import example 264 | 265 | monkeypatch.setattr(decider, 'schedule_activity_task', MagicMock()) 266 | 267 | decisions = MagicMock() 268 | schedule_context = decider.ScheduleContext() 269 | instance_state = activity.ActivityState('activity_1') 270 | instance_state.add_state(activity.ACTIVITY_COMPLETED) 271 | current_activity = example.activity_1 272 | history = { 273 | current_activity.name: { 274 | 'workflow_name_activity_1-1-schedule_id': instance_state 275 | } 276 | } 277 | 278 | resp = decider.schedule( 279 | decisions, schedule_context, history, {}, 'schedule_id', 280 | current_activity) 281 | 282 | assert not decider.schedule_activity_task.called 283 | assert resp.get_last_state() == activity.ACTIVITY_COMPLETED 284 | assert schedule_context.completed 285 | resp.result.get('foo') 286 | 287 | 288 | def test_schedule_requires_with_incomplete_activities(): 289 | """Test the scheduler. 290 | """ 291 | 292 | activity_state = activity.ActivityState('activity_name') 293 | with pytest.raises(activity.ActivityInstanceNotReadyException): 294 | decider.ensure_requirements([activity_state]) 295 | 296 | with pytest.raises(activity.ActivityInstanceNotReadyException): 297 | decider.ensure_requirements([None]) 298 | 299 | activity_state.add_state(activity.ACTIVITY_COMPLETED) 300 | decider.ensure_requirements(requires=[activity_state]) 301 | 302 | 303 | def test_schedule_activity_task(monkeypatch): 304 | """Test scheduling an activity task. 305 | """ 306 | 307 | from tests.fixtures.flows import example 308 | 309 | instance = list(example.activity_1.instances({}))[0] 310 | decisions = [] 311 | decider.schedule_activity_task(decisions, instance) 312 | expects = dict( 313 | decisionType='ScheduleActivityTask', 314 | scheduleActivityTaskDecisionAttributes=dict( 315 | activityId=instance.id, 316 | activityType=dict( 317 | name=instance.activity_name, 318 | version='1.0'), 319 | taskList=dict(name=instance.activity_worker.task_list), 320 | input=json.dumps(instance.create_execution_input()), 321 | heartbeatTimeout=str(instance.heartbeat_timeout), 322 | startToCloseTimeout=str(instance.timeout), 323 | scheduleToStartTimeout=str(instance.schedule_to_start), 324 | scheduleToCloseTimeout=str(instance.schedule_to_close))) 325 | assert expects in decisions 326 | 327 | 328 | def test_schedule_activity_task_with_version(monkeypatch): 329 | """Test scheduling an activity task with a version. 330 | """ 331 | 332 | from tests.fixtures.flows import example 333 | 334 | instance = list(example.activity_1.instances({}))[0] 335 | decisions = [] 336 | version = '2.0' 337 | decider.schedule_activity_task(decisions, instance, version=version) 338 | expects = dict( 339 | decisionType='ScheduleActivityTask', 340 | scheduleActivityTaskDecisionAttributes=dict( 341 | activityId=instance.id, 342 | activityType=dict( 343 | name=instance.activity_name, 344 | version=version), 345 | taskList=dict(name=instance.activity_worker.task_list), 346 | input=json.dumps(instance.create_execution_input()), 347 | heartbeatTimeout=str(instance.heartbeat_timeout), 348 | startToCloseTimeout=str(instance.timeout), 349 | scheduleToStartTimeout=str(instance.schedule_to_start), 350 | scheduleToCloseTimeout=str(instance.schedule_to_close))) 351 | assert expects in decisions 352 | 353 | 354 | def test_schedule_activity_task_with_custom_id(monkeypatch): 355 | """Test scheduling an activity task with a custom id. 356 | """ 357 | 358 | from tests.fixtures.flows import example 359 | 360 | instance = list(example.activity_1.instances({}))[0] 361 | decisions = [] 362 | custom_id = 'special_id' 363 | decider.schedule_activity_task(decisions, instance, id=custom_id) 364 | expects = dict( 365 | decisionType='ScheduleActivityTask', 366 | scheduleActivityTaskDecisionAttributes=dict( 367 | activityId=custom_id, 368 | activityType=dict( 369 | name=instance.activity_name, 370 | version='1.0'), 371 | taskList=dict(name=instance.activity_worker.task_list), 372 | input=json.dumps(instance.create_execution_input()), 373 | heartbeatTimeout=str(instance.heartbeat_timeout), 374 | startToCloseTimeout=str(instance.timeout), 375 | scheduleToStartTimeout=str(instance.schedule_to_start), 376 | scheduleToCloseTimeout=str(instance.schedule_to_close))) 377 | assert expects in decisions 378 | -------------------------------------------------------------------------------- /garcon/decider.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Decider Worker 4 | =============== 5 | 6 | The decider worker is focused on orchestrating which activity needs to be 7 | executed and when based on the flow procided. 8 | """ 9 | 10 | import functools 11 | import json 12 | import uuid 13 | 14 | from garcon import activity 15 | from garcon import event 16 | from garcon import log 17 | 18 | class DeciderWorker(log.GarconLogger): 19 | 20 | def __init__(self, flow, register=True): 21 | """Initialize the Decider Worker. 22 | 23 | Args: 24 | flow (module): Flow module. 25 | register (boolean): If this flow needs to be register on AWS. 26 | """ 27 | 28 | self.client = flow.client 29 | self.flow = flow 30 | self.domain = flow.domain 31 | self.version = getattr(flow, 'version', '1.0') 32 | self.activities = activity.find_workflow_activities(flow) 33 | self.task_list = flow.name 34 | self.on_exception = getattr(flow, 'on_exception', None) 35 | 36 | if register: 37 | self.register() 38 | 39 | def get_history(self, identity, poll): 40 | """Get all the history. 41 | 42 | The full history needs to be recovered from SWF to make sure that all 43 | the activities have been properly scheduled. With boto, only the last 44 | 100 events are provided, this methods retrieves all events. 45 | 46 | Args: 47 | identity (string): The identity of the decider that pulls the information. 48 | poll (object): The poll object (see AWS SWF for details.) 49 | Return: 50 | list: All the events. 51 | """ 52 | 53 | events = poll['events'] 54 | while 'nextPageToken' in poll: 55 | poll = self.client.poll_for_decision_task( 56 | domain=self.domain, 57 | identity=identity, 58 | taskList=dict(name=self.task_list), 59 | nextPageToken=poll['nextPageToken']) 60 | 61 | if 'events' in poll: 62 | events += poll['events'] 63 | 64 | # Remove all the events that are related to decisions and only. 65 | return [e for e in events if not e['eventType'].startswith('Decision')] 66 | 67 | def get_activity_states(self, history): 68 | """Get the activity states from the history. 69 | 70 | From the full history extract the different activity states. Those 71 | states contain 72 | 73 | Args: 74 | history (list): the full history. 75 | Return: 76 | dict: list of all the activities and their state. It only contains 77 | activities that have been scheduled with AWS. 78 | """ 79 | 80 | return event.activity_states_from_events(history) 81 | 82 | def register(self): 83 | """Register the Workflow on SWF. 84 | 85 | To work, SWF needs to have pre-registered the domain, the workflow, 86 | and the different activities, this method takes care of this part. 87 | """ 88 | 89 | registerables = [] 90 | registerables.append(( 91 | self.client.register_domain, 92 | dict(name=self.domain, 93 | workflowExecutionRetentionPeriodInDays='90'))) 94 | registerables.append(( 95 | self.client.register_workflow_type, 96 | dict( 97 | domain=self.domain, 98 | name=self.task_list, 99 | version=self.version))) 100 | 101 | for current_activity in self.activities: 102 | registerables.append(( 103 | self.client.register_activity_type, 104 | dict( 105 | domain=self.domain, 106 | name=current_activity.name, 107 | version=self.version))) 108 | 109 | for callable, data in registerables: 110 | try: 111 | callable(**data) 112 | except Exception as e: 113 | print(e) 114 | print(data.get('name'), 'already exists') 115 | 116 | def create_decisions_from_flow(self, decisions, activity_states, context): 117 | """Create the decisions from the flow. 118 | 119 | Simple flows don't need a custom decider, since all the requirements 120 | can be provided at the activity level. Discovery of the next activity 121 | to schedule is thus very straightforward. 122 | 123 | Args: 124 | decisions (list): the layer decision for swf. 125 | activity_states (dict): all the state activities. 126 | context (dict): the context of the activities. 127 | """ 128 | 129 | try: 130 | for current in activity.find_available_activities( 131 | self.flow, activity_states, context.current): 132 | schedule_activity_task( 133 | decisions, current, version=self.version) 134 | else: 135 | activities = list( 136 | activity.find_uncomplete_activities( 137 | self.flow, activity_states, context.current)) 138 | if not activities: 139 | decisions.append(dict( 140 | decisionType='CompleteWorkflowExecution')) 141 | except Exception as e: 142 | decisions.append(dict( 143 | decisionType='FailWorkflowExecution', 144 | failWorkflowExecutionDecisionAttributes=dict( 145 | reason=str(e)))) 146 | if self.on_exception: 147 | self.on_exception(self, e) 148 | self.logger.error(e, exc_info=True) 149 | 150 | def delegate_decisions(self, decisions, decider, history, context): 151 | """Delegate the decisions. 152 | 153 | For more complex flows (the ones that have, for instance, optional 154 | activities), you can write your own decider. The decider receives a 155 | method `schedule` which schedule the activity if not scheduled yet, 156 | and if scheduled, returns its result. 157 | 158 | Args: 159 | decisions (list): the layer decision for swf. 160 | decider (callable): the decider (it needs to have schedule) 161 | history (dict): all the state activities and its history. 162 | context (dict): the context of the activities. 163 | """ 164 | 165 | schedule_context = ScheduleContext() 166 | decider_schedule = functools.partial( 167 | schedule, decisions, schedule_context, history, context.current, 168 | version=self.version) 169 | 170 | try: 171 | kwargs = dict(schedule=decider_schedule) 172 | 173 | # retro-compatibility. 174 | if 'context' in decider.__code__.co_varnames: 175 | kwargs.update(context=context.workflow_input) 176 | 177 | decider(**kwargs) 178 | 179 | # When no exceptions are raised and the method decider has returned 180 | # it means that there i nothing left to do in the current decider. 181 | if schedule_context.completed: 182 | decisions.append(dict( 183 | decisionType='CompleteWorkflowExecution')) 184 | except activity.ActivityInstanceNotReadyException: 185 | pass 186 | except Exception as e: 187 | decisions.append(dict( 188 | decisionType='FailWorkflowExecution', 189 | failWorkflowExecutionDecisionAttributes=dict( 190 | reason=str(e)))) 191 | if self.on_exception: 192 | self.on_exception(self, e) 193 | self.logger.error(e, exc_info=True) 194 | 195 | def run(self, identity=None): 196 | """Run the decider. 197 | 198 | The decider defines which task needs to be launched and when based on 199 | the list of events provided. It looks at the list of all the available 200 | activities, and launch the ones that: 201 | 202 | * are not been scheduled yet. 203 | * have all the dependencies resolved. 204 | 205 | If the decider is not able to find an uncompleted activity, the 206 | workflow can safely mark its execution as complete. 207 | 208 | Args: 209 | identity (str): Identity of the worker making the request, which 210 | is recorded in the DecisionTaskStarted event in the AWS 211 | console. This enables diagnostic tracing when problems arise. 212 | 213 | Return: 214 | boolean: Always return true, so any loop on run can act as a long 215 | running process. 216 | """ 217 | 218 | try: 219 | poll = self.client.poll_for_decision_task( 220 | domain=self.domain, 221 | taskList=dict(name=self.task_list), 222 | identity=identity or '') 223 | except Exception as error: 224 | # Catch exceptions raised during poll() to avoid a Decider thread 225 | # dying & the daemon unable to process subsequent workflows. 226 | # AWS api limits on SWF calls are a common source of such 227 | # exceptions. 228 | 229 | # on_exception() can be overriden by the flow to send an alert 230 | # when such an exception occurs. 231 | if self.on_exception: 232 | self.on_exception(self, error) 233 | self.logger.error(error, exc_info=True) 234 | return True 235 | 236 | custom_decider = getattr(self.flow, 'decider', None) 237 | 238 | if 'events' not in poll: 239 | return True 240 | 241 | history = self.get_history(identity or '', poll) 242 | activity_states = self.get_activity_states(history) 243 | current_context = event.get_current_context(history) 244 | current_context.set_workflow_execution_info(poll, self.domain) 245 | 246 | decisions = [] 247 | if not custom_decider: 248 | self.create_decisions_from_flow( 249 | decisions, activity_states, current_context) 250 | else: 251 | self.delegate_decisions( 252 | decisions, custom_decider, activity_states, current_context) 253 | self.client.respond_decision_task_completed( 254 | taskToken=poll.get('taskToken'), 255 | decisions=decisions) 256 | return True 257 | 258 | 259 | class ScheduleContext: 260 | """ 261 | Schedule Context 262 | ================ 263 | 264 | The schedule context keeps track of all the current scheduling progress – 265 | which allows to easy determinate if there are more decisions to be taken 266 | or if the execution can be closed. 267 | """ 268 | 269 | def __init__(self): 270 | """Create a schedule context. 271 | """ 272 | 273 | self.completed = True 274 | 275 | def mark_uncompleted(self): 276 | """Mark the scheduling as completed. 277 | 278 | When a scheduling is completed, it means all the activities have been 279 | properly scheduled and they have all completed. 280 | """ 281 | 282 | self.completed = False 283 | 284 | 285 | def schedule_activity_task( 286 | decisions, instance, version='1.0', id=None): 287 | """Schedule an activity task. 288 | 289 | Args: 290 | decisions (Layer1Decisions): the layer decision for swf. 291 | instance (ActivityInstance): the activity instance to schedule. 292 | version (str): the version of the activity instance. 293 | id (str): optional id of the activity instance. 294 | """ 295 | 296 | decisions.append(dict( 297 | decisionType='ScheduleActivityTask', 298 | scheduleActivityTaskDecisionAttributes=dict( 299 | activityId=id or instance.id, 300 | activityType=dict( 301 | name=instance.activity_name, 302 | version=version), 303 | taskList=dict(name=instance.activity_worker.task_list), 304 | input=json.dumps(instance.create_execution_input()), 305 | heartbeatTimeout=str(instance.heartbeat_timeout), 306 | startToCloseTimeout=str(instance.timeout), 307 | scheduleToStartTimeout=str(instance.schedule_to_start), 308 | scheduleToCloseTimeout=str(instance.schedule_to_close)))) 309 | 310 | 311 | def schedule( 312 | decisions, schedule_context, history, context, schedule_id, 313 | current_activity, requires=None, input=None, version='1.0'): 314 | """Schedule an activity. 315 | 316 | Scheduling an activity requires all the requirements to be completed (all 317 | activities should be marked as completed). The scheduler also mixes the 318 | input with the full execution context to send the data to the activity. 319 | 320 | Args: 321 | decisions (list): the layer decision for swf. 322 | schedule_context (dict): information about the schedule. 323 | history (dict): history of the execution. 324 | context (dict): context of the execution. 325 | schedule_id (str): the id of the activity to schedule. 326 | current_activity (Activity): the activity to run. 327 | requires (list): list of all requirements. 328 | input (dict): additional input for the context. 329 | 330 | Throws: 331 | ActivityInstanceNotReadyException: if one of the activity in the 332 | requirements is not ready. 333 | 334 | Return: 335 | State: the state of the schedule (contains the response). 336 | """ 337 | 338 | ensure_requirements(requires) 339 | activity_completed = set() 340 | result = dict() 341 | 342 | instance_context = dict() 343 | instance_context.update(context or {}) 344 | instance_context.update(input or {}) 345 | 346 | for current in current_activity.instances(instance_context): 347 | current_id = '{}-{}'.format(current.id, schedule_id) 348 | states = history.get(current.activity_name, {}).get(current_id) 349 | 350 | if states: 351 | if states.get_last_state() == activity.ACTIVITY_COMPLETED: 352 | result.update(states.result or dict()) 353 | activity_completed.add(True) 354 | continue 355 | 356 | activity_completed.add(False) 357 | schedule_context.mark_uncompleted() 358 | 359 | if states.get_last_state() != activity.ACTIVITY_FAILED: 360 | continue 361 | elif (not current.retry or 362 | current.retry < activity.count_activity_failures(states)): 363 | raise Exception( 364 | 'The activity failures has exceeded its retry limit.') 365 | 366 | activity_completed.add(False) 367 | schedule_context.mark_uncompleted() 368 | schedule_activity_task( 369 | decisions, current, id=current_id, version=version) 370 | 371 | state = activity.ActivityState(current_activity.name) 372 | state.add_state(activity.ACTIVITY_SCHEDULED) 373 | 374 | if len(activity_completed) == 1 and True in activity_completed: 375 | state.add_state(activity.ACTIVITY_COMPLETED) 376 | state.set_result(result) 377 | return state 378 | 379 | 380 | def ensure_requirements(requires): 381 | """Ensure scheduling meets requirements. 382 | 383 | Verify the state of the requirements to make sure the activity can be 384 | scheduled. 385 | 386 | Args: 387 | requires (list): list of all requirements. 388 | 389 | Throws: 390 | ActivityInstanceNotReadyException: if one of the activity in the 391 | requirements is not ready. 392 | """ 393 | 394 | requires = requires or [] 395 | for require in requires: 396 | if (not require or 397 | require.get_last_state() != activity.ACTIVITY_COMPLETED): 398 | raise activity.ActivityInstanceNotReadyException() 399 | -------------------------------------------------------------------------------- /tests/test_activity.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import MagicMock 2 | from unittest.mock import ANY 3 | import json 4 | import sys 5 | 6 | from botocore import exceptions 7 | import pytest 8 | 9 | from garcon import activity 10 | from garcon import event 11 | from garcon import runner 12 | from garcon import task 13 | from garcon import utils 14 | from tests.fixtures import decider 15 | 16 | 17 | def activity_run( 18 | monkeypatch, boto_client, poll=None, complete=None, fail=None, 19 | execute=None): 20 | """Create an activity. 21 | """ 22 | 23 | current_activity = activity.Activity(boto_client) 24 | poll = poll or dict() 25 | 26 | monkeypatch.setattr( 27 | current_activity, 'execute_activity', 28 | execute or MagicMock(return_value=dict())) 29 | monkeypatch.setattr( 30 | boto_client, 'poll_for_activity_task', MagicMock(return_value=poll)) 31 | monkeypatch.setattr( 32 | boto_client, 'respond_activity_task_completed', complete or MagicMock()) 33 | monkeypatch.setattr( 34 | boto_client, 'respond_activity_task_failed', fail or MagicMock()) 35 | 36 | return current_activity 37 | 38 | 39 | @pytest.fixture(params=[0, 1, 2]) 40 | def generators(request): 41 | generators = [] 42 | 43 | if request.param >= 1: 44 | def igenerator(context): 45 | for i in range(10): 46 | yield {'i': i} 47 | 48 | generators.append(igenerator) 49 | 50 | if request.param == 2: 51 | def dgenerator(context): 52 | for i in range(10): 53 | yield {'d': i * 2} 54 | generators.append(dgenerator) 55 | 56 | return generators 57 | 58 | 59 | @pytest.fixture 60 | def poll(): 61 | return dict( 62 | activityId='something', 63 | taskToken='taskToken') 64 | 65 | 66 | def test_poll_for_activity(monkeypatch, poll, boto_client): 67 | """Test that poll_for_activity successfully polls. 68 | """ 69 | 70 | activity_task = poll 71 | current_activity = activity_run(monkeypatch, boto_client, poll) 72 | boto_client.poll_for_activity_task.return_value = activity_task 73 | 74 | activity_execution = current_activity.poll_for_activity() 75 | assert boto_client.poll_for_activity_task.called 76 | assert activity_execution.task_token is poll.get('taskToken') 77 | 78 | 79 | def test_poll_for_activity_throttle_retry(monkeypatch, poll, boto_client): 80 | """Test that SWF throttles are retried during polling. 81 | """ 82 | 83 | current_activity = activity_run(monkeypatch, boto_client, poll) 84 | boto_client.poll_for_activity_task.side_effect = exceptions.ClientError( 85 | {'Error': {'Code': 'ThrottlingException'}}, 86 | 'operation name') 87 | 88 | with pytest.raises(exceptions.ClientError): 89 | current_activity.poll_for_activity() 90 | assert boto_client.poll_for_activity_task.call_count == 5 91 | 92 | 93 | def test_poll_for_activity_error(monkeypatch, poll, boto_client): 94 | """Test that non-throttle errors during poll are thrown. 95 | """ 96 | 97 | current_activity = activity_run(monkeypatch, boto_client, poll) 98 | 99 | exception = Exception() 100 | boto_client.poll_for_activity_task.side_effect = exception 101 | 102 | with pytest.raises(Exception): 103 | current_activity.poll_for_activity() 104 | 105 | 106 | def test_poll_for_activity_identity(monkeypatch, poll, boto_client): 107 | """Test that identity is passed to poll_for_activity. 108 | """ 109 | 110 | current_activity = activity_run(monkeypatch, boto_client, poll) 111 | 112 | current_activity.poll_for_activity(identity='foo') 113 | boto_client.poll_for_activity_task.assert_called_with( 114 | domain=ANY, taskList=ANY, identity='foo') 115 | 116 | 117 | def test_poll_for_activity_no_identity(monkeypatch, poll, boto_client): 118 | """Test poll_for_activity works without identity passed as param. 119 | """ 120 | 121 | current_activity = activity_run(monkeypatch, boto_client, poll) 122 | 123 | current_activity.poll_for_activity() 124 | boto_client.poll_for_activity_task.assert_called_with( 125 | domain=ANY, taskList=ANY) 126 | 127 | 128 | def test_run_activity(monkeypatch, poll, boto_client): 129 | """Run an activity. 130 | """ 131 | 132 | current_activity = activity_run(monkeypatch, boto_client, poll=poll) 133 | current_activity.run() 134 | 135 | boto_client.poll_for_activity_task.assert_called_with( 136 | domain=ANY, taskList=ANY) 137 | assert current_activity.execute_activity.called 138 | assert boto_client.respond_activity_task_completed.called 139 | 140 | 141 | def test_run_activity_identity(monkeypatch, poll, boto_client): 142 | """Run an activity with identity as param. 143 | """ 144 | 145 | current_activity = activity_run(monkeypatch, boto_client, poll=poll) 146 | current_activity.run(identity='foo') 147 | 148 | boto_client.poll_for_activity_task.assert_called_with( 149 | domain=ANY, taskList=ANY, identity='foo') 150 | assert current_activity.execute_activity.called 151 | assert boto_client.respond_activity_task_completed.called 152 | 153 | 154 | def test_run_capture_exception(monkeypatch, poll, boto_client): 155 | """Run an activity with an exception raised during activity execution. 156 | """ 157 | 158 | current_activity = activity_run(monkeypatch, boto_client, poll) 159 | current_activity.on_exception = MagicMock() 160 | current_activity.execute_activity = MagicMock() 161 | error_msg_long = "Error" * 100 162 | actual_error_msg = error_msg_long[:255] 163 | current_activity.execute_activity.side_effect = Exception(error_msg_long) 164 | current_activity.run() 165 | 166 | assert boto_client.poll_for_activity_task.called 167 | assert current_activity.execute_activity.called 168 | assert current_activity.on_exception.called 169 | boto_client.respond_activity_task_failed.assert_called_with( 170 | taskToken=poll.get('taskToken'), 171 | reason=actual_error_msg) 172 | assert not boto_client.respond_activity_task_completed.called 173 | 174 | 175 | def test_run_capture_fail_exception(monkeypatch, poll, boto_client): 176 | """Run an activity with an exception raised during failing execution. 177 | """ 178 | 179 | current_activity = activity_run(monkeypatch, boto_client, poll) 180 | current_activity.on_exception = MagicMock() 181 | current_activity.execute_activity = MagicMock() 182 | current_activity.complete = MagicMock() 183 | current_activity.fail = MagicMock() 184 | error_msg_long = "Error" * 100 185 | current_activity.complete.side_effect = Exception(error_msg_long) 186 | current_activity.fail.side_effect = Exception(error_msg_long) 187 | current_activity.run() 188 | 189 | assert boto_client.poll_for_activity_task.called 190 | assert current_activity.execute_activity.called 191 | assert not current_activity.complete.called 192 | assert not current_activity.fail.called 193 | assert current_activity.on_exception.called 194 | 195 | 196 | def test_run_capture_poll_exception(monkeypatch, boto_client, poll): 197 | """Run an activity with an exception raised during poll. 198 | """ 199 | 200 | current_activity = activity_run(monkeypatch, boto_client, poll=poll) 201 | 202 | current_activity.on_exception = MagicMock() 203 | current_activity.execute_activity = MagicMock() 204 | 205 | exception = Exception('poll exception') 206 | boto_client.poll_for_activity_task.side_effect = exception 207 | current_activity.run() 208 | 209 | assert boto_client.poll_for_activity_task.called 210 | assert current_activity.on_exception.called 211 | assert not current_activity.execute_activity.called 212 | assert not boto_client.respond_activity_task_completed.called 213 | 214 | current_activity.on_exception = None 215 | current_activity.logger.error = MagicMock() 216 | current_activity.run() 217 | current_activity.logger.error.assert_called_with(exception, exc_info=True) 218 | 219 | 220 | def test_run_activity_without_id(monkeypatch, boto_client): 221 | """Run an activity without an activity id. 222 | """ 223 | 224 | current_activity = activity_run(monkeypatch, boto_client, poll=dict()) 225 | current_activity.run() 226 | 227 | assert boto_client.poll_for_activity_task.called 228 | assert not current_activity.execute_activity.called 229 | assert not boto_client.respond_activity_task_completed.called 230 | 231 | 232 | def test_run_activity_with_context(monkeypatch, boto_client, poll): 233 | """Run an activity with a context. 234 | """ 235 | 236 | context = dict(foo='bar') 237 | poll.update(input=json.dumps(context)) 238 | 239 | current_activity = activity_run(monkeypatch, boto_client, poll=poll) 240 | current_activity.run() 241 | 242 | activity_execution = current_activity.execute_activity.call_args[0][0] 243 | assert activity_execution.context == context 244 | 245 | 246 | def test_run_activity_with_result(monkeypatch, boto_client, poll): 247 | """Run an activity with a result. 248 | """ 249 | 250 | result = dict(foo='bar') 251 | mock = MagicMock(return_value=result) 252 | current_activity = activity_run(monkeypatch, boto_client, poll=poll, 253 | execute=mock) 254 | current_activity.run() 255 | boto_client.respond_activity_task_completed.assert_called_with( 256 | result=json.dumps(result), taskToken=poll.get('taskToken')) 257 | 258 | 259 | def test_task_failure(monkeypatch, boto_client, poll): 260 | """Run an activity that has a bad task. 261 | """ 262 | 263 | resp = dict(foo='bar') 264 | mock = MagicMock(return_value=resp) 265 | reason = 'fail' 266 | current_activity = activity_run(monkeypatch, boto_client, poll=poll, 267 | execute=mock) 268 | current_activity.on_exception = MagicMock() 269 | current_activity.execute_activity.side_effect = Exception(reason) 270 | current_activity.run() 271 | 272 | boto_client.respond_activity_task_failed.assert_called_with( 273 | taskToken=poll.get('taskToken'), 274 | reason=reason) 275 | 276 | 277 | def test_task_failure_on_close_activity(monkeypatch, boto_client, poll): 278 | """Run an activity failure when the task is already closed. 279 | """ 280 | 281 | resp = dict(foo='bar') 282 | mock = MagicMock(return_value=resp) 283 | current_activity = activity_run(monkeypatch, boto_client, poll=poll, 284 | execute=mock) 285 | current_activity.on_exception = MagicMock() 286 | current_activity.execute_activity.side_effect = Exception('fail') 287 | boto_client.respond_activity_task_failed.side_effect = Exception('fail') 288 | current_activity.unset_log_context = MagicMock() 289 | current_activity.run() 290 | 291 | assert current_activity.unset_log_context.called 292 | 293 | 294 | def test_execute_activity(monkeypatch, boto_client): 295 | """Test the execution of an activity. 296 | """ 297 | 298 | monkeypatch.setattr(activity.ActivityExecution, 'heartbeat', 299 | lambda self: None) 300 | 301 | resp = dict(task_resp='something') 302 | custom_task = MagicMock(return_value=resp) 303 | 304 | current_activity = activity.Activity(boto_client) 305 | current_activity.runner = runner.Sync(custom_task) 306 | 307 | val = current_activity.execute_activity(activity.ActivityExecution( 308 | boto_client, 'activityId', 'taskToken', '{"context": "value"}')) 309 | 310 | assert custom_task.called 311 | assert val == resp 312 | 313 | 314 | def test_hydrate_activity(monkeypatch, boto_client): 315 | """Test the hydratation of an activity. 316 | """ 317 | 318 | current_activity = activity.Activity(boto_client) 319 | current_activity.hydrate(dict( 320 | name='activity', 321 | domain='domain', 322 | requires=[], 323 | on_exception=lambda actor, exception: print(exception), 324 | tasks=[lambda: dict('val')])) 325 | 326 | 327 | def test_create_activity(monkeypatch, boto_client): 328 | """Test the creation of an activity via `create`. 329 | """ 330 | 331 | create = activity.create(boto_client, 'domain_name', 'flow_name') 332 | 333 | current_activity = create(name='activity_name') 334 | assert isinstance(current_activity, activity.Activity) 335 | assert current_activity.name == 'flow_name_activity_name' 336 | assert current_activity.task_list == 'flow_name_activity_name' 337 | assert current_activity.domain == 'domain_name' 338 | assert current_activity.client == boto_client 339 | 340 | 341 | def test_create_external_activity(monkeypatch, boto_client): 342 | """Test the creation of an external activity via `create`. 343 | """ 344 | 345 | create = activity.create(boto_client, 'domain_name', 'flow_name') 346 | 347 | current_activity = create( 348 | name='activity_name', 349 | timeout=60, 350 | heartbeat=40, 351 | external=True) 352 | 353 | assert isinstance(current_activity, activity.ExternalActivity) 354 | assert current_activity.name == 'flow_name_activity_name' 355 | assert current_activity.task_list == 'flow_name_activity_name' 356 | assert current_activity.domain == 'domain_name' 357 | 358 | assert isinstance(current_activity.runner, runner.External) 359 | assert current_activity.runner.heartbeat() == 40 360 | assert current_activity.runner.timeout() == 60 361 | 362 | 363 | def test_create_activity_worker(monkeypatch): 364 | """Test the creation of an activity worker. 365 | """ 366 | 367 | from tests.fixtures.flows import example 368 | 369 | worker = activity.ActivityWorker(example) 370 | assert len(worker.activities) == 4 371 | 372 | assert worker.flow is example 373 | assert not worker.worker_activities 374 | 375 | 376 | def test_instances_creation(monkeypatch, boto_client, generators): 377 | """Test the creation of an activity instance id with the use of a local 378 | context. 379 | """ 380 | 381 | local_activity = activity.Activity(boto_client) 382 | external_activity = activity.ExternalActivity(timeout=60) 383 | 384 | for current_activity in [local_activity, external_activity]: 385 | current_activity.generators = generators 386 | 387 | if len(current_activity.generators): 388 | instances = list(current_activity.instances(dict())) 389 | assert len(instances) == pow(10, len(generators)) 390 | for instance in instances: 391 | assert isinstance(instance.local_context.get('i'), int) 392 | 393 | if len(generators) == 2: 394 | assert isinstance(instance.local_context.get('d'), int) 395 | else: 396 | instances = list(current_activity.instances(dict())) 397 | assert len(instances) == 1 398 | assert isinstance(instances[0].local_context, dict) 399 | # Context is empty since no generator was used. 400 | assert not instances[0].local_context 401 | 402 | 403 | def test_activity_timeouts(monkeypatch, boto_client, generators): 404 | """Test the creation of an activity timeouts. 405 | 406 | More details: the timeout of a task is 120s, the schedule to start is 1000, 407 | 100 activities are going to be scheduled when the generator is set. The 408 | schedule_to_start for all activities instance is: 10000 * 100 = 100k. The 409 | schedule to close is 100k + duration of an activity (which is 120s * 2). 410 | """ 411 | 412 | timeout = 120 413 | start_timeout = 1000 414 | 415 | @task.decorate(timeout=timeout) 416 | def local_task(): 417 | return 418 | 419 | current_activity = activity.Activity(boto_client) 420 | current_activity.hydrate(dict(schedule_to_start=start_timeout)) 421 | current_activity.generators = generators 422 | current_activity.runner = runner.Sync( 423 | local_task.fill(), 424 | local_task.fill()) 425 | 426 | total_generators = pow(10, len(current_activity.generators)) 427 | schedule_to_start = start_timeout * total_generators 428 | for instance in current_activity.instances({}): 429 | assert current_activity.pool_size == total_generators 430 | assert instance.schedule_to_start == schedule_to_start 431 | assert instance.timeout == timeout * 2 432 | assert instance.schedule_to_close == ( 433 | schedule_to_start + instance.timeout) 434 | 435 | 436 | def test_external_activity_timeouts(monkeypatch, boto_client, generators): 437 | """Test the creation of an external activity timeouts. 438 | """ 439 | 440 | timeout = 120 441 | start_timeout = 1000 442 | 443 | current_activity = activity.ExternalActivity(timeout=timeout) 444 | current_activity.hydrate(dict(schedule_to_start=start_timeout)) 445 | current_activity.generators = generators 446 | 447 | total_generators = pow(10, len(current_activity.generators)) 448 | schedule_to_start = start_timeout * total_generators 449 | for instance in current_activity.instances({}): 450 | assert current_activity.pool_size == total_generators 451 | assert instance.schedule_to_start == schedule_to_start 452 | assert instance.timeout == timeout 453 | assert instance.schedule_to_close == ( 454 | schedule_to_start + instance.timeout) 455 | 456 | 457 | @pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3") 458 | def test_worker_run(monkeypatch, boto_client): 459 | """Test running the worker. 460 | """ 461 | 462 | from tests.fixtures.flows import example 463 | 464 | worker = activity.ActivityWorker(example) 465 | assert len(worker.activities) == 4 466 | for current_activity in worker.activities: 467 | monkeypatch.setattr( 468 | current_activity, 'run', MagicMock(return_value=False)) 469 | 470 | worker.run() 471 | 472 | assert len(worker.activities) == 4 473 | for current_activity in worker.activities: 474 | assert current_activity.run.called 475 | 476 | 477 | def test_worker_run_with_skipped_activities(monkeypatch): 478 | """Test running the worker with defined activities. 479 | """ 480 | 481 | monkeypatch.setattr(activity.Activity, 'run', MagicMock(return_value=False)) 482 | 483 | from tests.fixtures.flows import example 484 | 485 | worker = activity.ActivityWorker(example, activities=['activity_1']) 486 | assert len(worker.worker_activities) == 1 487 | for current_activity in worker.activities: 488 | monkeypatch.setattr( 489 | current_activity, 'run', MagicMock(return_value=False)) 490 | 491 | worker.run() 492 | 493 | for current_activity in worker.activities: 494 | if current_activity.name == 'activity_1': 495 | assert current_activity.run.called 496 | else: 497 | assert not current_activity.run.called 498 | 499 | 500 | def test_worker_infinite_loop(): 501 | """Test the worker runner. 502 | """ 503 | 504 | spy = MagicMock() 505 | 506 | class Activity: 507 | def __init__(self): 508 | self.count = 0 509 | 510 | def run(self, identity=None): 511 | spy() 512 | self.count = self.count + 1 513 | if self.count < 5: 514 | return True 515 | return False 516 | 517 | activity_worker = Activity() 518 | activity_worker.name = 'activity_name' 519 | activity_worker.logger = MagicMock() 520 | activity.worker_runner(activity_worker) 521 | assert spy.called 522 | assert spy.call_count == 5 523 | 524 | 525 | def test_worker_infinite_loop_on_external(monkeypatch): 526 | """There is no worker for external activities. 527 | """ 528 | 529 | external_activity = activity.ExternalActivity(timeout=10) 530 | current_run = external_activity.run 531 | spy = MagicMock() 532 | 533 | def run(): 534 | spy() 535 | return current_run() 536 | 537 | monkeypatch.setattr(external_activity, 'run', run) 538 | activity.worker_runner(external_activity) 539 | 540 | # This test might not fail, but it will hang the test suite since it is 541 | # going to trigger an infinite loop. 542 | assert spy.call_count == 1 543 | 544 | 545 | def test_activity_launch_sequence(): 546 | """Test available activities. 547 | """ 548 | 549 | from tests.fixtures.flows import example 550 | 551 | # First available activity is the activity_1. 552 | context = dict() 553 | history = event.activity_states_from_events(decider.history['events'][:1]) 554 | activities = list( 555 | activity.find_available_activities(example, history, context)) 556 | uncomplete = list( 557 | activity.find_uncomplete_activities(example, history, context)) 558 | assert len(activities) == 1 559 | assert len(uncomplete) == 4 560 | assert activities[0].activity_worker == example.activity_1 561 | 562 | # In between activities should not launch activities. 563 | history = event.activity_states_from_events(decider.history['events'][:5]) 564 | activities = list( 565 | activity.find_available_activities(example, history, context)) 566 | uncomplete = list( 567 | activity.find_uncomplete_activities(example, history, context)) 568 | assert len(activities) == 0 569 | assert len(uncomplete) == 4 570 | 571 | # Two activities are launched in parallel: 2 and 3. 572 | history = event.activity_states_from_events(decider.history['events'][:7]) 573 | activities = list( 574 | activity.find_available_activities(example, history, context)) 575 | uncomplete = list( 576 | activity.find_uncomplete_activities(example, history, context)) 577 | assert len(activities) == 2 578 | assert example.activity_1 not in uncomplete 579 | 580 | # Activity 3 completes before activity 2. Activity 4 depends on 2 and 3 to 581 | # complete. 582 | history = event.activity_states_from_events(decider.history['events'][:14]) 583 | activities = list( 584 | activity.find_available_activities(example, history, context)) 585 | uncomplete = list( 586 | activity.find_uncomplete_activities(example, history, context)) 587 | assert len(activities) == 0 588 | assert example.activity_3 not in uncomplete 589 | 590 | # Activity 2 - 3 completed. 591 | history = event.activity_states_from_events(decider.history['events'][:22]) 592 | activities = list( 593 | activity.find_available_activities(example, history, context)) 594 | uncomplete = list( 595 | activity.find_uncomplete_activities(example, history, context)) 596 | assert len(activities) == 1 597 | assert activities[0].activity_worker == example.activity_4 598 | assert example.activity_1 not in uncomplete 599 | assert example.activity_2 not in uncomplete 600 | assert example.activity_3 not in uncomplete 601 | 602 | # Close 603 | history = event.activity_states_from_events(decider.history['events'][:25]) 604 | activities = list( 605 | activity.find_available_activities(example, history, context)) 606 | uncomplete = list( 607 | activity.find_uncomplete_activities(example, history, context)) 608 | assert not activities 609 | assert not uncomplete 610 | 611 | 612 | def test_create_activity_instance(): 613 | """Test the creation of an activity instance. 614 | """ 615 | 616 | activity_mock = MagicMock() 617 | activity_mock.name = 'foobar' 618 | activity_mock.retry = 20 619 | 620 | instance = activity.ActivityInstance(activity_mock) 621 | 622 | assert activity_mock.name == instance.activity_name 623 | assert activity_mock.retry == instance.retry 624 | 625 | 626 | def test_create_activity_instance_id(monkeypatch): 627 | """Test the creation of an activity instance id. 628 | """ 629 | 630 | monkeypatch.setattr(utils, 'create_dictionary_key', MagicMock()) 631 | 632 | activity_mock = MagicMock() 633 | activity_mock.name = 'activity' 634 | instance = activity.ActivityInstance(activity_mock) 635 | 636 | # No context was passed, so create_dictionary key didn't need to be 637 | # called. 638 | assert instance.id == activity_mock.name + '-1' 639 | assert not utils.create_dictionary_key.called 640 | 641 | 642 | def test_create_activity_instance_id_with_local_context(monkeypatch): 643 | """Test the creation of an activity instance id with the use of a local 644 | context. 645 | """ 646 | 647 | monkeypatch.setattr(utils, 'create_dictionary_key', MagicMock()) 648 | 649 | activity_mock = MagicMock() 650 | activity_mock.name = 'activity' 651 | instance = activity.ActivityInstance(activity_mock, dict(foobar='yes')) 652 | 653 | assert instance.id.startswith(activity_mock.name) 654 | assert utils.create_dictionary_key.called 655 | 656 | 657 | def test_create_activity_instance_input_without_runner(monkeypatch): 658 | """Test the creation of a context for an activity instance input without 659 | specifying a runner. 660 | """ 661 | 662 | activity_mock = MagicMock() 663 | activity_mock.name = 'activity' 664 | activity_mock.runner = None 665 | context = dict(context='yes') 666 | instance = activity.ActivityInstance(activity_mock, context) 667 | 668 | with pytest.raises(runner.RunnerMissing): 669 | instance.create_execution_input() 670 | 671 | 672 | def test_create_activity_instance_input(monkeypatch): 673 | """Test the creation of a context for an activity instance input. 674 | """ 675 | 676 | @task.decorate() 677 | def task_a(value): 678 | pass 679 | 680 | activity_mock = MagicMock() 681 | activity_mock.name = 'activity' 682 | activity_mock.runner = runner.BaseRunner(task_a.fill(value='context')) 683 | instance = activity.ActivityInstance( 684 | activity_mock, local_context=dict(context='yes', unused='no'), 685 | execution_context=dict(somemore='values')) 686 | resp = instance.create_execution_input() 687 | 688 | assert len(resp) == 4 689 | assert resp.get('context') == 'yes' 690 | assert 'somemore' not in resp 691 | assert 'unused' not in resp 692 | assert 'execution.domain' in resp 693 | assert 'execution.run_id' in resp 694 | assert 'execution.workflow_id' in resp 695 | 696 | 697 | def test_create_activity_instance_input_without_decorate(monkeypatch): 698 | """Test the creation of a context input without the use of a decorator. 699 | """ 700 | 701 | def task_a(value): 702 | pass 703 | 704 | activity_mock = MagicMock() 705 | activity_mock.name = 'activity' 706 | context = dict(foo='bar') 707 | local_context = dict(context='yes') 708 | 709 | activity_mock.runner = runner.BaseRunner(task_a) 710 | instance = activity.ActivityInstance( 711 | activity_mock, local_context=local_context, 712 | execution_context=context) 713 | 714 | resp = instance.create_execution_input() 715 | assert resp.get('foo') == 'bar' 716 | assert resp.get('context') == 'yes' 717 | 718 | 719 | def test_create_activity_instance_input_with_zero_or_empty_values( 720 | monkeypatch): 721 | """Test the creation of a context for an activity instance input. 722 | """ 723 | 724 | @task.decorate() 725 | def task_a(value1, value2, value3, value4): 726 | pass 727 | 728 | activity_mock = MagicMock() 729 | activity_mock.name = 'activity' 730 | activity_mock.runner = runner.BaseRunner( 731 | task_a.fill( 732 | value1='zero', 733 | value2='empty_list', 734 | value3='empty_dict', 735 | value4='none')) 736 | instance = activity.ActivityInstance( 737 | activity_mock, 738 | local_context=dict( 739 | zero=0, empty_list=[], empty_dict={}, none=None)) 740 | 741 | resp = instance.create_execution_input() 742 | 743 | assert len(resp) == 6 744 | assert resp.get('zero') == 0 745 | assert resp.get('empty_list') == [] 746 | assert resp.get('empty_dict') == {} 747 | assert 'none' not in resp 748 | 749 | 750 | def test_activity_state(): 751 | """Test the creation of the activity state. 752 | """ 753 | 754 | activity_id = 'id' 755 | state = activity.ActivityState(activity_id) 756 | assert state.activity_id is activity_id 757 | assert not state.get_last_state() 758 | 759 | state.add_state(activity.ACTIVITY_FAILED) 760 | state.add_state(activity.ACTIVITY_COMPLETED) 761 | assert len(state.states) == 2 762 | assert state.get_last_state() is activity.ACTIVITY_COMPLETED 763 | 764 | result = 'foobar' 765 | state.set_result(result) 766 | assert state.result == result 767 | 768 | with pytest.raises(Exception): 769 | state.set_result('shouldnt reset') 770 | 771 | assert state.result == result 772 | -------------------------------------------------------------------------------- /garcon/activity.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Activity 4 | ======== 5 | 6 | Activities are self generated classes to which you can pass an identifier, 7 | and a list of tasks to perform. The activities are in between the decider and 8 | the tasks. 9 | 10 | For ease, two types of task runners are available: Sync and Async. If 11 | you need something more specific, you should either create your own runner, or 12 | you should create a main task that will then split the work. 13 | 14 | Create an activity:: 15 | 16 | import boto3 17 | from garcon import activity 18 | 19 | # First step is to create the workflow on a specific domain. 20 | client = boto3.client('swf') 21 | create = activity.create(client, 'domain', 'workflow-name') 22 | 23 | initial_activity = create( 24 | # Name of your activity 25 | name='activity_name', 26 | 27 | # List of tasks to run (here we use the Sync runner) 28 | run=runner.Sync(task1), 29 | 30 | # No requires since it's the first one. Later in your flow, if you have 31 | # a dependency, just use the variable that contains the activity. 32 | requires=[], 33 | 34 | # If the activity fails, number of times you want to retry. 35 | retry=0, 36 | 37 | # If you want to run the activity `n` times, you can use a generator. 38 | generator=[generator_name]) 39 | 40 | """ 41 | 42 | from botocore import exceptions 43 | import itertools 44 | import json 45 | import threading 46 | import backoff 47 | 48 | from garcon import log 49 | from garcon import utils 50 | from garcon import runner 51 | 52 | ACTIVITY_STANDBY = 0 53 | ACTIVITY_SCHEDULED = 1 54 | ACTIVITY_COMPLETED = 2 55 | ACTIVITY_FAILED = 3 56 | 57 | DEFAULT_ACTIVITY_SCHEDULE_TO_START = 600 # 10 minutes 58 | 59 | 60 | class ActivityInstanceNotReadyException(Exception): 61 | """Exception when an activity instance is not ready. 62 | 63 | Activity instances that are considered not ready are instances that have 64 | not completed. 65 | """ 66 | 67 | pass 68 | 69 | 70 | class ActivityInstance: 71 | 72 | def __init__( 73 | self, activity_worker, local_context=None, execution_context=None): 74 | """Activity Instance. 75 | 76 | In SWF, Activity is a worker: it will get information from the context, 77 | and will launch activity instances (only one, unless you have a 78 | generator.) The activity instance generates its key (visible in the SWF 79 | console) from the local context. Activity instances are owned by an 80 | execution. 81 | 82 | Args: 83 | activity_worker (ActivityWorker): The activity worker that owns 84 | this specific Activity Instance. 85 | local_context (dict): the local context of the activity (it does 86 | not include the execution context.) Most times the context will 87 | be empty since it is only filled with data that comes from the 88 | generators. 89 | execution_context (dict): the execution context of when an activity 90 | will be scheduled with. 91 | """ 92 | 93 | self.activity_worker = activity_worker 94 | self.execution_context = execution_context or dict() 95 | self.local_context = local_context or dict() 96 | self.global_context = dict( 97 | list(self.execution_context.items()) + 98 | list(self.local_context.items())) 99 | 100 | @property 101 | def activity_name(self): 102 | """Return the activity name of the worker. 103 | """ 104 | 105 | return self.activity_worker.name 106 | 107 | @property 108 | def retry(self): 109 | """Return the number of retries allowed (matches the worker.) 110 | """ 111 | 112 | return self.activity_worker.retry 113 | 114 | @property 115 | def id(self): 116 | """Generate the id of the activity. 117 | 118 | The id is crutial (not just important): it allows to indentify the 119 | state the activity instance in the event history (if it has failed, 120 | been executed, or marked as completed.) 121 | 122 | Return: 123 | str: composed of the activity name (task list), and the activity 124 | id. 125 | """ 126 | 127 | if not self.local_context: 128 | activity_id = 1 129 | else: 130 | activity_id = utils.create_dictionary_key(self.local_context) 131 | 132 | return '{name}-{id}'.format( 133 | name=self.activity_name, 134 | id=activity_id) 135 | 136 | @property 137 | def schedule_to_start(self): 138 | """Return the schedule to start timeout. 139 | 140 | The schedule to start timeout assumes that only one activity worker is 141 | available (since swf does not provide a count of available workers). So 142 | if the default value is 5 minutes, and you have 10 instances: the 143 | schedule to start will be 50 minutes for all instances. 144 | 145 | Return: 146 | int: Schedule to start timeout. 147 | """ 148 | 149 | return ( 150 | self.activity_worker.pool_size * 151 | self.activity_worker.schedule_to_start_timeout) 152 | 153 | @property 154 | def schedule_to_close(self): 155 | """Return the schedule to close timeout. 156 | 157 | The schedule to close timeout is a simple calculation that defines when 158 | an activity (from the moment it has been scheduled) should end. It is 159 | a calculation between the schedule to start timeout and the activity 160 | timeout. 161 | 162 | Return: 163 | int: Schedule to close timeout. 164 | """ 165 | 166 | return self.schedule_to_start + self.timeout 167 | 168 | @property 169 | def timeout(self): 170 | """Return the timeout in seconds. 171 | 172 | This timeout corresponds on when the activity has started and when we 173 | assume the activity has ended (which corresponds in boto to 174 | start_to_close_timeout.) 175 | 176 | Return: 177 | int: Task list timeout. 178 | """ 179 | 180 | return self.runner.timeout(self.global_context) 181 | 182 | @property 183 | def heartbeat_timeout(self): 184 | """Return the heartbeat in seconds. 185 | 186 | This heartbeat corresponds on when an activity needs to send a signal 187 | to swf that it is still running. This will set the value when the 188 | activity is scheduled. 189 | 190 | Return: 191 | int: Task list timeout. 192 | """ 193 | 194 | return self.runner.heartbeat(self.global_context) 195 | 196 | @property 197 | def runner(self): 198 | """Shortcut to get access to the runner. 199 | 200 | Raises: 201 | runner.RunnerMissing: an activity should always have a runner, 202 | if the runner is missing an exception is raised (we will not 203 | be able to calculate values such as timeouts without a runner.) 204 | 205 | Return: 206 | Runner: the activity runner. 207 | """ 208 | 209 | activity_runner = getattr(self.activity_worker, 'runner', None) 210 | if not activity_runner: 211 | raise runner.RunnerMissing() 212 | return activity_runner 213 | 214 | def create_execution_input(self): 215 | """Create the input of the activity from the context. 216 | 217 | AWS has a limit on the number of characters that can be used (32k). If 218 | you use the `task.decorate`, the data sent to the activity is optimized 219 | to match the values of the context as well as the execution context. 220 | 221 | Return: 222 | dict: the input to send to the activity. 223 | """ 224 | 225 | activity_input = dict() 226 | 227 | try: 228 | for requirement in self.runner.requirements(self.global_context): 229 | value = self.global_context.get(requirement) 230 | if value is not None: 231 | activity_input.update({requirement: value}) 232 | 233 | activity_input.update({ 234 | 'execution.domain': self.global_context.get('execution.domain'), 235 | 'execution.run_id': self.global_context.get('execution.run_id'), 236 | 'execution.workflow_id': self.global_context.get( 237 | 'execution.workflow_id') 238 | }) 239 | 240 | except runner.NoRunnerRequirementsFound: 241 | return self.global_context 242 | 243 | return activity_input 244 | 245 | 246 | class Activity(log.GarconLogger): 247 | version = '1.0' 248 | task_list = None 249 | 250 | def __init__(self, client): 251 | """Instantiates an activity. 252 | 253 | Args: 254 | client: the boto client used for this activity. 255 | """ 256 | 257 | self.client = client 258 | self.name = None 259 | self.domain = None 260 | self.task_list = None 261 | 262 | @backoff.on_exception( 263 | backoff.expo, 264 | exceptions.ClientError, 265 | max_tries=5, 266 | giveup=utils.non_throttle_error, 267 | on_backoff=utils.throttle_backoff_handler, 268 | jitter=backoff.full_jitter) 269 | def poll_for_activity(self, identity=None): 270 | """Runs Activity Poll. 271 | 272 | If a SWF throttling exception is raised during a poll, the poll will 273 | be retried up to 5 times using exponential backoff algorithm. 274 | 275 | Upgrading to boto3 would make this retry logic redundant. 276 | 277 | Args: 278 | identity (str): Identity of the worker making the request, which 279 | is recorded in the ActivityTaskStarted event in the AWS 280 | console. This enables diagnostic tracing when problems arise. 281 | Return: 282 | ActivityExecution: activity execution. 283 | """ 284 | 285 | additional_params = {} 286 | if identity: 287 | additional_params.update(identity=identity) 288 | 289 | execution_definition = self.client.poll_for_activity_task( 290 | domain=self.domain, taskList=dict(name=self.task_list), 291 | **additional_params) 292 | 293 | return ActivityExecution( 294 | self.client, execution_definition.get('activityId'), 295 | execution_definition.get('taskToken'), 296 | execution_definition.get('input')) 297 | 298 | def run(self, identity=None): 299 | """Activity Runner. 300 | 301 | Information is being pulled down from SWF and it checks if the Activity 302 | can be ran. As part of the information provided, the input of the 303 | previous activity is consumed (context). 304 | 305 | Args: 306 | identity (str): Identity of the worker making the request, which 307 | is recorded in the ActivityTaskStarted event in the AWS 308 | console. This enables diagnostic tracing when problems arise. 309 | """ 310 | try: 311 | if identity: 312 | self.logger.debug('Polling with {}'.format(identity)) 313 | execution = self.poll_for_activity(identity) 314 | except Exception as error: 315 | # Catch exceptions raised during poll() to avoid an Activity thread 316 | # dying & worker daemon unable to process the affected Activity. 317 | # AWS api limits on SWF calls are a common source of such 318 | # exceptions (see https://github.com/xethorn/garcon/pull/75) 319 | 320 | # on_exception() can be overriden by the flow to send an alert 321 | # when an exception occurs. 322 | if self.on_exception: 323 | self.on_exception(self, error) 324 | self.logger.error(error, exc_info=True) 325 | return True 326 | 327 | self.set_log_context(execution.context) 328 | if execution.activity_id: 329 | try: 330 | context = self.execute_activity(execution) 331 | execution.complete(context) 332 | except Exception as error: 333 | # If the workflow has been stopped, it is not possible for the 334 | # activity to be updated – it throws an exception which stops 335 | # the worker immediately. 336 | try: 337 | execution.fail(str(error)[:255]) 338 | if self.on_exception: 339 | self.on_exception(self, error) 340 | except Exception as error2: # noqa: E722 341 | if self.on_exception: 342 | self.on_exception(self, error2) 343 | 344 | self.unset_log_context() 345 | return True 346 | 347 | def execute_activity(self, activity): 348 | """Execute the runner. 349 | 350 | Args: 351 | execution (ActivityExecution): the activity execution. 352 | 353 | Return: 354 | dict: The result of the operation. 355 | """ 356 | 357 | return self.runner.execute(activity, activity.context) 358 | 359 | def hydrate(self, data): 360 | """Hydrate the task with information provided. 361 | 362 | Args: 363 | data (dict): the data to use (if defined.) 364 | """ 365 | 366 | self.pool_size = 0 367 | self.version = data.get('version') or self.version 368 | self.name = self.name or data.get('name') 369 | self.domain = getattr(self, 'domain', '') or data.get('domain') 370 | self.requires = getattr(self, 'requires', []) or data.get('requires') 371 | self.retry = getattr(self, 'retry', None) or data.get('retry', 0) 372 | self.task_list = self.task_list or data.get('task_list') 373 | self.on_exception = ( 374 | getattr(self, 'on_exception', None) or data.get('on_exception')) 375 | 376 | # The start timeout is how long it will take between the scheduling 377 | # of the activity and the start of the activity. 378 | self.schedule_to_start_timeout = ( 379 | getattr(self, 'schedule_to_start_timeout', None) or 380 | data.get('schedule_to_start') or 381 | DEFAULT_ACTIVITY_SCHEDULE_TO_START) 382 | 383 | # The previous way to create an activity was to fill a `tasks` param, 384 | # which is not `run`. 385 | self.runner = ( 386 | getattr(self, 'runner', None) or 387 | data.get('run') or data.get('tasks')) 388 | 389 | self.generators = getattr( 390 | self, 'generators', None) or data.get('generators') 391 | 392 | def instances(self, context): 393 | """Get all instances for one activity based on the current context. 394 | 395 | There are two scenarios: when the activity worker has a generator and 396 | when it does not. When it doesn't (the most simple case), there will 397 | always be one instance returned. 398 | 399 | Generators will however consume the context to calculate how many 400 | instances of the activity are needed – and it will generate them 401 | (regardless of their state.) 402 | 403 | Args: 404 | context (dict): the current context. 405 | Return: 406 | list: all the instances of the activity (for a current workflow 407 | execution.) 408 | """ 409 | 410 | if not self.generators: 411 | self.pool_size = 1 412 | yield ActivityInstance(self, execution_context=context) 413 | return 414 | 415 | generator_values = [] 416 | for generator in self.generators: 417 | generator_values.append(generator(context)) 418 | 419 | contexts = list(itertools.product(*generator_values)) 420 | self.pool_size = len(contexts) 421 | for generator_contexts in contexts: 422 | # Each generator returns a context, merge all the contexts 423 | # to only be one - which can be used to 1/ create the id of the 424 | # activity and 2/ be passed as a local context. 425 | instance_context = dict() 426 | for current_generator_context in generator_contexts: 427 | instance_context.update(current_generator_context.items()) 428 | 429 | yield ActivityInstance( 430 | self, execution_context=context, 431 | local_context=instance_context) 432 | 433 | 434 | class ExternalActivity(Activity): 435 | """External activity 436 | 437 | One of the main advantages of SWF is the ability to write a workflow that 438 | has activities written in any languages. The external activity class allows 439 | to write the workflow in Garcon and benefit from some features (timeout 440 | calculation among other things, sending context data.) 441 | """ 442 | 443 | def __init__(self, timeout=None, heartbeat=None): 444 | """Create the External Activity. 445 | 446 | Args: 447 | timeout (int): activity timeout in seconds (mandatory) 448 | heartbeat (int): heartbeat timeout in seconds, if not defined, it 449 | will be equal to the timeout. 450 | """ 451 | 452 | Activity.__init__(self, client=None) 453 | self.runner = runner.External(timeout=timeout, heartbeat=heartbeat) 454 | 455 | def run(self): 456 | """Run the external activity. 457 | 458 | This activity is handled outside, so the run method should remain 459 | unimplemented and return False (so the run loop stops.) 460 | """ 461 | 462 | return False 463 | 464 | 465 | class ActivityExecution(log.GarconLogger): 466 | 467 | def __init__(self, client, activity_id, task_token, context): 468 | """Create an an activity execution. 469 | 470 | Args: 471 | client (boto3.client): the boto client (for easy access if needed). 472 | activity_id (str): the activity id. 473 | task_token (str): the task token. 474 | context (str): data for the execution. 475 | """ 476 | 477 | self.client = client 478 | self.activity_id = activity_id 479 | self.task_token = task_token 480 | self.context = context and json.loads(context) or dict() 481 | 482 | def heartbeat(self, details=None): 483 | """Create a task heartbeat. 484 | 485 | Args: 486 | details (str): details to add to the heartbeat. 487 | """ 488 | 489 | self.client.record_activity_task_heartbeat(taskToken=self.task_token, 490 | details=details or '') 491 | 492 | def fail(self, reason=None): 493 | """Mark the activity execution as failed. 494 | 495 | Args: 496 | reason (str): optional reason for the failure. 497 | """ 498 | 499 | self.client.respond_activity_task_failed( 500 | taskToken=self.task_token, 501 | reason=reason or '') 502 | 503 | def complete(self, context=None): 504 | """Mark the activity execution as completed. 505 | 506 | Args: 507 | context (str or dict): the context result of the operation. 508 | """ 509 | 510 | self.client.respond_activity_task_completed( 511 | taskToken=self.task_token, 512 | result=json.dumps(context)) 513 | 514 | 515 | class ActivityWorker(): 516 | 517 | def __init__(self, flow, activities=None): 518 | """Initiate an activity worker. 519 | 520 | The activity worker take in consideration all the activities from a 521 | flow, or specific activities. Some activities (tasks) might require 522 | more power than others, and be then launched on different machines. 523 | 524 | If a list of activities is passed, the worker will be focused on 525 | completing those and will ignore all the others. 526 | 527 | Args: 528 | flow (module): the flow module. 529 | activities (list): the list of activities that this worker should 530 | handle. 531 | """ 532 | 533 | self.flow = flow 534 | self.activities = find_workflow_activities(self.flow) 535 | self.worker_activities = activities 536 | 537 | def run(self): 538 | """Run the activities. 539 | """ 540 | threads = [] 541 | for activity in self.activities: 542 | if (self.worker_activities and 543 | activity.name not in self.worker_activities): 544 | continue 545 | thread = threading.Thread( 546 | target=worker_runner, 547 | args=(activity,)) 548 | thread.start() 549 | threads.append(thread) 550 | 551 | for thread in threads: 552 | thread.join() 553 | 554 | 555 | class ActivityState: 556 | """ 557 | Activity State 558 | ============== 559 | 560 | Provides information about a specific activity instance state (if the 561 | instance is already scheduled, has failed, or has been completed.) Along 562 | with the default values, this class also provides additional metadata such 563 | as the result of an activity instance. 564 | """ 565 | 566 | def __init__(self, activity_id): 567 | """Create a State. 568 | 569 | Args: 570 | activity_id (str): the activity id. 571 | """ 572 | 573 | self.activity_id = activity_id 574 | self._result = None 575 | self.states = [] 576 | 577 | @property 578 | def result(self): 579 | """Get the result. 580 | """ 581 | 582 | if not self.ready: 583 | raise ActivityInstanceNotReadyException() 584 | return self._result 585 | 586 | @property 587 | def ready(self): 588 | """Check if an activity is ready. 589 | """ 590 | 591 | return self.get_last_state() == ACTIVITY_COMPLETED 592 | 593 | def get_last_state(self): 594 | """Get the last state of the activity execution. 595 | 596 | Return: 597 | int: the state of the activity (see: activity.py) 598 | """ 599 | 600 | if len(self.states): 601 | return self.states[-1] 602 | return None 603 | 604 | def add_state(self, state): 605 | """Add a state in the activity execution. 606 | 607 | Args: 608 | state (int): the state of the activity to add (see activity.py) 609 | """ 610 | 611 | self.states.append(state) 612 | 613 | def set_result(self, result): 614 | """Set the result of the activity. 615 | 616 | This method sometimes throws an exception: an activity id can only have 617 | one result. 618 | 619 | Args: 620 | result (dict): Result of the activity. 621 | """ 622 | 623 | if self._result: 624 | raise Exception('Result is ummutable – it should not be changed.') 625 | self._result = result 626 | 627 | def wait(self): 628 | """Wait until ready. 629 | """ 630 | 631 | if not self.ready: 632 | raise ActivityInstanceNotReadyException() 633 | 634 | 635 | def worker_runner(worker): 636 | """Run indefinitely the worker. 637 | 638 | Args: 639 | worker (object): the Activity worker. 640 | """ 641 | 642 | while (worker.run()): 643 | continue 644 | 645 | 646 | def create(client, domain, workflow_name, version='1.0', on_exception=None): 647 | """Helper method to create Activities. 648 | 649 | The helper method simplifies the creation of an activity by setting the 650 | domain, the task list, and the activity dependencies (what other 651 | activities) need to be completed before this one can run. 652 | 653 | Note: 654 | The task list is generated based on the domain and the name of the 655 | activity. Always make sure your activity name is unique. 656 | 657 | Args: 658 | client (boto3.client): the boto3 client. 659 | domain (str): the domain name. 660 | workflow_name (str): workflow name. 661 | version (str): activity version. 662 | on_exception (callable): the error handler. 663 | 664 | Return: 665 | callable: activity generator. 666 | """ 667 | 668 | def wrapper(**options): 669 | activity = Activity(client) 670 | 671 | if options.get('external'): 672 | activity = ExternalActivity( 673 | timeout=options.get('timeout'), 674 | heartbeat=options.get('heartbeat')) 675 | 676 | activity_name = '{name}_{activity}'.format( 677 | name=workflow_name, 678 | activity=options.get('name')) 679 | 680 | activity.hydrate(dict( 681 | domain=domain, 682 | version=version, 683 | name=activity_name, 684 | generators=options.get('generators', []), 685 | requires=options.get('requires', []), 686 | retry=options.get('retry'), 687 | task_list=activity_name, 688 | tasks=options.get('tasks'), 689 | run=options.get('run'), 690 | schedule_to_start=options.get('schedule_to_start'), 691 | on_exception=options.get('on_exception') or on_exception)) 692 | return activity 693 | 694 | return wrapper 695 | 696 | 697 | def find_available_activities(flow, history, context): 698 | """Find all available activity instances of a flow. 699 | 700 | The history contains all the information of our activities (their state). 701 | This method focuses on finding all the activities that need to run. 702 | 703 | Args: 704 | flow (module): the flow module. 705 | history (dict): the history information. 706 | context (dict): from the context find the available activities. 707 | """ 708 | 709 | for instance in find_activities(flow, context): 710 | # If an event is already available for the activity, it means it is 711 | # not in standby anymore, it's either processing or has been completed. 712 | # The activity is thus not available anymore. 713 | states = history.get(instance.activity_name, {}).get(instance.id) 714 | 715 | if states: 716 | if states.get_last_state() != ACTIVITY_FAILED: 717 | continue 718 | elif (not instance.retry or 719 | instance.retry < count_activity_failures(states)): 720 | raise Exception( 721 | 'The activity failures has exceeded its retry limit.') 722 | 723 | can_yield = True 724 | for requirement in instance.activity_worker.requires: 725 | require_history = history.get(requirement.name) 726 | 727 | if not require_history: 728 | can_yield = False 729 | break 730 | 731 | for requirement_states in require_history.values(): 732 | if ACTIVITY_COMPLETED not in requirement_states.states: 733 | can_yield = False 734 | break 735 | 736 | if can_yield: 737 | yield instance 738 | 739 | 740 | def find_uncomplete_activities(flow, history, context): 741 | """Find uncomplete activity instances. 742 | 743 | Uncomplete activities are all the activities that are not marked as 744 | completed. 745 | 746 | Args: 747 | flow (module): the flow module. 748 | history (dict): the history information. 749 | context (dict): from the context find the available activities. 750 | Yield: 751 | activity: The available activity. 752 | """ 753 | 754 | for instance in find_activities(flow, context): 755 | states = history.get(instance.activity_name, {}).get(instance.id) 756 | if not states or ACTIVITY_COMPLETED not in states.states: 757 | yield instance 758 | 759 | 760 | def find_workflow_activities(flow): 761 | """Retrieves all the activities from a flow 762 | 763 | Args: 764 | flow (module): the flow module. 765 | Return: 766 | list: all the activities. 767 | """ 768 | 769 | activities = [] 770 | for module_attribute in dir(flow): 771 | current_activity = getattr(flow, module_attribute) 772 | if isinstance(current_activity, Activity): 773 | activities.append(current_activity) 774 | return activities 775 | 776 | 777 | def find_activities(flow, context): 778 | """Retrieves all the activities from a flow. 779 | 780 | Args: 781 | flow (module): the flow module. 782 | Return: 783 | list: All the activity instances for the flow. 784 | """ 785 | 786 | activities = [] 787 | for module_attribute in dir(flow): 788 | current_activity = getattr(flow, module_attribute) 789 | 790 | if isinstance(current_activity, Activity): 791 | for activity_instance in current_activity.instances(context): 792 | activities.append(activity_instance) 793 | 794 | return activities 795 | 796 | 797 | def count_activity_failures(states): 798 | """Count the number of times an activity has failed. 799 | 800 | Args: 801 | states (dict): list of activity states. 802 | Return: 803 | int: The number of times an activity has failed. 804 | """ 805 | 806 | return len([evt for evt in states.states if evt == ACTIVITY_FAILED]) 807 | --------------------------------------------------------------------------------